├── .circleci └── config.yml ├── .dockerignore ├── .gitattributes ├── .gitignore ├── .gitmodules ├── CHANGELOG.md ├── LICENSE ├── README.md ├── config ├── README.md └── generate-kitchensink.sh ├── core ├── Event.go ├── EventDispatchEngine.go ├── EventEmitter.go ├── EventListener.go ├── Kraken.go ├── Logger.go ├── ModuleAPIClient.go ├── ModuleAPIServer.go ├── Node.go ├── Query.go ├── QueryEngine.go ├── Registry.go ├── ServiceInstance.go ├── ServiceManager.go ├── State.go ├── StateDifferenceEngine.go ├── StateMutation.go ├── StateMutationEngine.go ├── StateSpec.go ├── StateSyncEngine.go ├── proto │ ├── ModuleAPI.pb.go │ ├── Node.go │ ├── Node.pb.go │ ├── ServiceInstance.pb.go │ ├── StateSyncMessage.pb.go │ ├── customtypes │ │ └── NodeID_type.go │ └── src │ │ ├── ModuleAPI.proto │ │ ├── Node.proto │ │ ├── ServiceInstance.proto │ │ ├── StateSyncMessage.proto │ │ ├── github.com │ │ └── gogo │ │ │ └── protobuf │ │ │ └── gogoproto │ │ │ └── gogo.proto │ │ └── google │ │ └── protobuf │ │ ├── any.proto │ │ ├── api.proto │ │ ├── compiler │ │ └── plugin.proto │ │ ├── descriptor.proto │ │ ├── duration.proto │ │ ├── empty.proto │ │ ├── field_mask.proto │ │ ├── source_context.proto │ │ ├── struct.proto │ │ ├── timestamp.proto │ │ ├── type.proto │ │ └── wrappers.proto └── tests │ ├── api_client │ └── api_client.go │ ├── core_test.go │ ├── events_test.go │ ├── sme_test.go │ ├── sse_test.go │ └── sync_client │ └── sync_client.go ├── extensions └── ipv4 │ ├── customtypes │ ├── IP_type.go │ └── MAC_type.go │ ├── ipv4.go │ ├── ipv4.pb.go │ └── ipv4.proto ├── generators ├── app.go ├── extension.go ├── global.go ├── module.go └── templates │ ├── app │ ├── includes.go.tpl │ ├── main.go.tpl │ └── pprof.go.tpl │ ├── extension │ ├── customtype.type.go.tpl │ ├── template.ext.go.tpl │ ├── template.go.tpl │ └── template.proto.tpl │ ├── import.go │ ├── module │ ├── README.md.tpl │ ├── template.config.proto.tpl │ ├── template.go.tpl │ └── template.mod.go.tpl │ └── templates.go ├── go.mod ├── go.sum ├── kraken.go ├── lib ├── build │ └── buildutil.go ├── copy │ ├── cmp │ │ └── cmp.go │ ├── copy.go │ └── copy_test.go ├── json │ └── json.go ├── types │ └── types.go └── util │ ├── test.pb.go │ ├── test.proto │ ├── util.go │ └── util_test.go ├── modules ├── README.md ├── restapi │ ├── restapi.go │ ├── restapi.pb.go │ ├── restapi.proto │ └── swagger.yaml └── websocket │ ├── README.md │ ├── websocket.go │ ├── websocket.pb.go │ └── websocket.proto └── triad-notice.txt /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 # use CircleCI 2.0 2 | jobs: # basic units of work in a run 3 | build: # make sure the "kitcken sink" builds 4 | docker: 5 | - image: circleci/golang:1.15 6 | steps: 7 | - restore_cache: # restores saved cache if no changes are detected since last run 8 | keys: 9 | - v1-pkg-cache 10 | 11 | - checkout 12 | - run: 13 | name: Build kraken code generator 14 | command: go build . 15 | - run: 16 | name: Create kitchen sink config 17 | command: bash config/generate-kitchensink.sh . 18 | - run: 19 | name: Generate code for kitchen sink 20 | command: ./kraken -f -l debug app generate -o kitchen-sink -c config/kitchensink.yaml 21 | - run: 22 | name: Build kitchen-sink 23 | command: go build . 24 | working_directory: ./kitchen-sink 25 | - run: 26 | name: Print kitchen-sink build version 27 | command: ./kitchen-sink -version 28 | working_directory: ./kitchen-sink 29 | - save_cache: # store cache in the /go/pkg directory 30 | key: v1-pkg-cache 31 | paths: 32 | - "/go/pkg" 33 | 34 | test: 35 | docker: # run the steps with Docker 36 | # CircleCI Go images available at: https://hub.docker.com/r/circleci/golang/ 37 | - image: circleci/golang:1.15 38 | 39 | environment: # environment variables for the build itself 40 | TEST_RESULTS: /tmp/test-results # path to where test results will be saved 41 | 42 | steps: # steps that comprise the `build` job 43 | - checkout # check out source code to working directory 44 | - run: mkdir -p $TEST_RESULTS # create the test results directory 45 | 46 | - restore_cache: # restores saved cache if no changes are detected since last run 47 | keys: 48 | - v1-pkg-cache 49 | 50 | # Normally, this step would be in a custom primary image; 51 | # we've added it here for the sake of explanation. 52 | - run: go get github.com/jstemmer/go-junit-report 53 | 54 | - run: 55 | name: Run core unit tests 56 | # store the results of our tests in the $TEST_RESULTS directory 57 | command: | 58 | trap "go-junit-report <${TEST_RESULTS}/go-test.out > ${TEST_RESULTS}/go-test-report.xml" EXIT 59 | cd core/tests 60 | go test -v | tee ${TEST_RESULTS}/go-test.out 61 | 62 | - save_cache: # store cache in the /go/pkg directory 63 | key: v1-pkg-cache 64 | paths: 65 | - "/go/pkg" 66 | 67 | - store_artifacts: # upload test summary for display in Artifacts 68 | path: /tmp/test-results 69 | destination: raw-test-output 70 | 71 | - store_test_results: # upload test results for display in Test Summary 72 | path: /tmp/test-results 73 | 74 | workflows: 75 | version: 2 76 | test_and_build: 77 | jobs: 78 | - test 79 | - build -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .circleci 2 | .git* 3 | config.yaml 4 | state.json 5 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | vendor/** linguist-vendored 2 | *.tpl linguist-language=golang 3 | *.pb.go linguist-generated=true 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/.vscode 2 | /build 3 | config.yaml 4 | config/kitchensink.yaml 5 | tmp 6 | munge.key 7 | ssh_host_*key* 8 | state.json 9 | **/.DS_Store 10 | /kraken-build 11 | **/*.swp 12 | /kraken -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "utils/kraken-dashboard"] 2 | path = utils/kraken-dashboard 3 | url = ../kraken-dashboard.git 4 | [submodule "utils/krakenctl"] 5 | path = utils/krakenctl 6 | url = ../krakenctl.git 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ## [0.2.0] - 2021-05-17 10 | ### Added 11 | - Added the new `kraken` command to perform project code generation 12 | - Added support for module code generation via the `kraken` command 13 | - Added support for extension code generation via the `kraken` command 14 | - Added support for app entry-point code generation via the `kraken` command 15 | ### Changed 16 | - Updated README to document code generation 17 | - Support multiple extension opbjects in one extension package 18 | - Relocated some common tasks for module init into core 19 | ### Deprecated 20 | ### Removed 21 | - Removed the old, no-longer functional `kraken-builder` command 22 | ### Fixed 23 | - Fixed an issue where state lookups for enum v alues would fail with improper URLShift usage 24 | 25 | ### Security 26 | 27 | ## [0.1.1] - 2021-04-15 28 | ### Added 29 | - Added this changelog (`CHANGELOG.md`) 30 | - Added a submodule for `krakenctl` at [utils/krakenctl](utils/krakenctl) 31 | ### Fixed 32 | - Submodules now use relative paths so we don't accidentally 33 | 34 | ## [0.1.0] - 2021-04-13 35 | ### Added 36 | - Semantic versioning started. Note: this project has been in dev for some time, but never previously versioned. 37 | ### Changed 38 | - Migrate from github.com/hpc/kraken to github.com/kraken-hpc/kraken 39 | - Split-out modules & extensions that are not "core" to the kraken framework 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2020, Triad National Security, LLC 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | 1. Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | 2. Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | 3. Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /config/README.md: -------------------------------------------------------------------------------- 1 | # Config 2 | 3 | This folder contains some simple examples of `kraken` code generation configurations. 4 | -------------------------------------------------------------------------------- /config/generate-kitchensink.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This script creates a config that includes all available modules and extensions 4 | # It will create build targets as configured in this script 5 | # This is mostly intended for running tests 6 | 7 | # This takes the root of the kraken source directory as the first (and only) argument 8 | 9 | if [ $# -ne 1 ]; then 10 | echo "Usage: $0 " 11 | exit 1 12 | fi 13 | 14 | KRAKEN_SRC="$1" 15 | 16 | PKG="github.com/kraken-hpc/kraken" 17 | CONFIG_FILE="$KRAKEN_SRC/config/kitchensink.yaml" 18 | EXTENSIONS_DIR="$KRAKEN_SRC/extensions" 19 | MODULES_DIR="$KRAKEN_SRC/modules" 20 | 21 | echo "Adding build targets" 22 | META=" 23 | name: kitchen-sink 24 | version: v0.0.0 25 | " 26 | 27 | # start with our build targets 28 | echo "$META" > "$CONFIG_FILE" 29 | 30 | # include extensions 31 | echo "Adding extensions" 32 | echo "extensions: " >> "$CONFIG_FILE" 33 | for e_dir in "$EXTENSIONS_DIR"/*; do 34 | if [ ! -d "$e_dir" ]; then 35 | continue 36 | fi 37 | e=$(basename "$e_dir") 38 | echo "Adding extension: $e" 39 | echo " - $PKG/extensions/$e" >> "$CONFIG_FILE" 40 | done 41 | 42 | # include modules 43 | echo "Adding modules" 44 | echo "modules: " >> "$CONFIG_FILE" 45 | for m_dir in "$MODULES_DIR"/*; do 46 | if [ ! -d "$m_dir" ]; then 47 | continue 48 | fi 49 | m=$(basename "$m_dir") 50 | echo "Adding module: $m" 51 | echo " - $PKG/modules/$m" >> "$CONFIG_FILE" 52 | done 53 | 54 | echo "created kitchen sink config: $CONFIG_FILE" -------------------------------------------------------------------------------- /core/Event.go: -------------------------------------------------------------------------------- 1 | /* Event.go: Event objects get distributed through the EventDispatchEngine 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "github.com/kraken-hpc/kraken/lib/types" 14 | ) 15 | 16 | ////////////////// 17 | // Event Object / 18 | //////////////// 19 | 20 | var _ types.Event = (*Event)(nil) 21 | 22 | // An Event is a generic container for internal events, like state changes 23 | type Event struct { 24 | t types.EventType 25 | url string 26 | data interface{} 27 | } 28 | 29 | // NewEvent creates an initialized, fully specified Event 30 | func NewEvent(t types.EventType, url string, data interface{}) types.Event { 31 | ev := &Event{ 32 | t: t, 33 | url: url, 34 | data: data, 35 | } 36 | return ev 37 | } 38 | 39 | func (v *Event) Type() types.EventType { return v.t } 40 | func (v *Event) URL() string { return v.url } 41 | func (v *Event) Data() interface{} { return v.data } 42 | -------------------------------------------------------------------------------- /core/EventDispatchEngine.go: -------------------------------------------------------------------------------- 1 | /* EventDispatchEngine.go: the EventDispatchEngine is a communications hub between pieces 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "fmt" 14 | "sync" 15 | 16 | "github.com/kraken-hpc/kraken/lib/types" 17 | ) 18 | 19 | ////////////////////////// 20 | // EventDispatchEngine Object / 21 | //////////////////////// 22 | 23 | var _ types.EventDispatchEngine = (*EventDispatchEngine)(nil) 24 | 25 | // EventDispatchEngine redistributes event to (possibly filtered) listeners. 26 | type EventDispatchEngine struct { 27 | lists map[string]types.EventListener // place we send Events to 28 | schan chan types.EventListener // where we get subscriptions 29 | // should we have a unique echan for each source? 30 | echan chan []types.Event // where we get events 31 | log types.Logger 32 | lock *sync.RWMutex // Make sure we don't concurrently add a listener and iterate through the list 33 | } 34 | 35 | // NewEventDispatchEngine creates an initialized EventDispatchEngine 36 | func NewEventDispatchEngine(ctx Context) (v *EventDispatchEngine) { 37 | v = &EventDispatchEngine{ 38 | lists: make(map[string]types.EventListener), 39 | schan: make(chan types.EventListener), 40 | echan: make(chan []types.Event), 41 | log: &ctx.Logger, 42 | lock: &sync.RWMutex{}, 43 | } 44 | v.log.SetModule("EventDispatchEngine") 45 | return 46 | } 47 | 48 | // AddListener gets called to add a Listener; Listeners are filtered subscribers 49 | func (v *EventDispatchEngine) AddListener(el types.EventListener) (e error) { 50 | v.lock.Lock() 51 | defer v.lock.Unlock() 52 | k := el.Name() 53 | switch el.State() { 54 | case types.EventListener_UNSUBSCRIBE: 55 | if _, ok := v.lists[k]; ok { 56 | delete(v.lists, k) 57 | } else { 58 | e = fmt.Errorf("cannot unsubscribe unknown listener: %s", k) 59 | } 60 | return 61 | case types.EventListener_STOP: 62 | fallthrough 63 | case types.EventListener_RUN: 64 | // should we check validity? 65 | v.lists[k] = el 66 | return 67 | default: 68 | e = fmt.Errorf("unknown EventListener state: %d", el.State()) 69 | } 70 | return 71 | } 72 | 73 | // SubscriptionChan gets the channel we can subscribe new EventListeners with 74 | func (v *EventDispatchEngine) SubscriptionChan() chan<- types.EventListener { return v.schan } 75 | 76 | // EventChan is the channel emitters should send new events on 77 | func (v *EventDispatchEngine) EventChan() chan<- []types.Event { return v.echan } 78 | 79 | // Run is a goroutine than handles event dispatch and subscriptions 80 | // There's currently no way to stop this once it's started. 81 | func (v *EventDispatchEngine) Run(ready chan<- interface{}) { 82 | v.Log(INFO, "starting EventDispatchEngine") 83 | ready <- nil 84 | for { 85 | select { 86 | case el := <-v.schan: 87 | if e := v.AddListener(el); e != nil { 88 | v.Logf(ERROR, "failed to add new listener: %s, %v", el.Name(), e) 89 | } else { 90 | v.Logf(DEBUG, "successfully added new listener: %s", el.Name()) 91 | } 92 | break 93 | case e := <-v.echan: 94 | if len(e) == 0 { 95 | v.Log(ERROR, "got empty event list") 96 | break 97 | } else { 98 | v.Logf(DEBUG, "dispatching event: %s %s %v\n", types.EventTypeString[e[0].Type()], e[0].URL(), e[0].Data()) 99 | } 100 | go v.sendEvents(e) 101 | break 102 | } 103 | } 104 | } 105 | 106 | //////////////////////// 107 | // Unexported methods / 108 | ////////////////////// 109 | 110 | // goroutine 111 | func (v *EventDispatchEngine) sendEvents(evs []types.Event) { 112 | v.lock.RLock() 113 | defer v.lock.RUnlock() 114 | for _, ev := range evs { 115 | for _, el := range v.lists { 116 | if el.Type() == types.Event_ALL { 117 | el.Send(ev) 118 | } else if ev.Type() == el.Type() { 119 | el.Send(ev) 120 | } 121 | } 122 | } 123 | } 124 | 125 | //////////////////////////// 126 | // Passthrough Interfaces / 127 | ////////////////////////// 128 | 129 | /* 130 | * Consume Logger 131 | */ 132 | var _ types.Logger = (*EventDispatchEngine)(nil) 133 | 134 | func (v *EventDispatchEngine) Log(level types.LoggerLevel, m string) { v.log.Log(level, m) } 135 | func (v *EventDispatchEngine) Logf(level types.LoggerLevel, fmt string, va ...interface{}) { 136 | v.log.Logf(level, fmt, va...) 137 | } 138 | func (v *EventDispatchEngine) SetModule(name string) { v.log.SetModule(name) } 139 | func (v *EventDispatchEngine) GetModule() string { return v.log.GetModule() } 140 | func (v *EventDispatchEngine) SetLoggerLevel(level types.LoggerLevel) { v.log.SetLoggerLevel(level) } 141 | func (v *EventDispatchEngine) GetLoggerLevel() types.LoggerLevel { return v.log.GetLoggerLevel() } 142 | func (v *EventDispatchEngine) IsEnabledFor(level types.LoggerLevel) bool { 143 | return v.log.IsEnabledFor(level) 144 | } 145 | -------------------------------------------------------------------------------- /core/EventEmitter.go: -------------------------------------------------------------------------------- 1 | /* EventEmitter.go: event emitters can send events to EventDispatchEngine for distribution 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "fmt" 14 | 15 | "github.com/kraken-hpc/kraken/lib/types" 16 | ) 17 | 18 | ///////////////////////// 19 | // EventEmitter Object / 20 | /////////////////////// 21 | 22 | var _ types.EventEmitter = (*EventEmitter)(nil) 23 | 24 | // EventEmitter is really just a helper object, and not a core type. 25 | // It simplifies making an engine that Emits events to event dispatch. 26 | type EventEmitter struct { 27 | subs map[string]chan<- []types.Event 28 | t types.EventType 29 | } 30 | 31 | // NewEventEmitter creates a new initialized EventEmitter. 32 | // It must be Subscribed to do anything interesting. 33 | func NewEventEmitter(t types.EventType) *EventEmitter { 34 | ne := &EventEmitter{ 35 | subs: make(map[string]chan<- []types.Event), 36 | t: t, 37 | } 38 | return ne 39 | } 40 | 41 | // Subscribe links the Emitter to an Event chan, allowing it to actually send events. 42 | func (m *EventEmitter) Subscribe(id string, c chan<- []types.Event) (e error) { 43 | if _, ok := m.subs[id]; ok { 44 | e = fmt.Errorf("subscription id already in use: %s", id) 45 | return 46 | } 47 | m.subs[id] = c 48 | return 49 | } 50 | 51 | // Unsubscribe removes an event chan from the subscriber list 52 | func (m *EventEmitter) Unsubscribe(id string) (e error) { 53 | if _, ok := m.subs[id]; !ok { 54 | e = fmt.Errorf("cannot unsubscribe, no such subscription: %s", id) 55 | return 56 | } 57 | delete(m.subs, id) 58 | return 59 | } 60 | 61 | // EventType returns the event type that this Emitter sends 62 | func (m *EventEmitter) EventType() types.EventType { return m.t } 63 | 64 | // Emit emits (non-blocking) a slice of Events 65 | // NOT a goroutine; handles that internally 66 | func (m *EventEmitter) Emit(v []types.Event) { 67 | go m.emit(v) 68 | } 69 | 70 | // EmitOne is a helper for when we have a single event 71 | func (m *EventEmitter) EmitOne(v types.Event) { 72 | m.Emit([]types.Event{v}) 73 | } 74 | 75 | //////////////////////// 76 | // Unexported methods / 77 | ////////////////////// 78 | 79 | func (m *EventEmitter) emit(v []types.Event) { 80 | for _, s := range m.subs { 81 | s <- v // should we introduce timeouts? 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /core/EventListener.go: -------------------------------------------------------------------------------- 1 | /* EventListener.go: event listeners listen for events in dispatch. They include filtering. 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "regexp" 14 | 15 | "github.com/kraken-hpc/kraken/lib/types" 16 | ) 17 | 18 | /////////////////////// 19 | // Auxiliary Objects / 20 | ///////////////////// 21 | 22 | /* 23 | * Filter generators - for URL based filtering 24 | */ 25 | 26 | // FilterSimple is mostly for an example; it's not very useful 27 | func FilterSimple(ev types.Event, match []string) (r bool) { 28 | for _, v := range match { 29 | if ev.URL() == v { 30 | r = true 31 | } 32 | } 33 | return 34 | } 35 | 36 | // FilterRegexpStr matches URL to a regexp (string) 37 | // FilterRegexp is probably more efficient for repeated filtering 38 | func FilterRegexpStr(ev types.Event, re string) (r bool) { 39 | r, e := regexp.Match(re, []byte(ev.URL())) 40 | if e != nil { 41 | r = false 42 | } 43 | return 44 | } 45 | 46 | // FilterRegexp matches URL to a compiled Regexp 47 | func FilterRegexp(ev types.Event, re *regexp.Regexp) (r bool) { 48 | return re.Match([]byte(ev.URL())) 49 | } 50 | 51 | /* 52 | * Sender generators 53 | */ 54 | 55 | // ChanSender is for the simple case were we just retransmit on a chan 56 | func ChanSender(ev types.Event, c chan<- types.Event) error { 57 | c <- ev 58 | return nil 59 | } 60 | 61 | ////////////////////////// 62 | // EventListener Object / 63 | //////////////////////// 64 | 65 | var _ types.EventListener = (*EventListener)(nil) 66 | 67 | // An EventListener implementation that leaves filter/send as arbitrary function pointers. 68 | type EventListener struct { 69 | name string 70 | s types.EventListenerState 71 | filter func(types.Event) bool 72 | send func(types.Event) error 73 | t types.EventType 74 | } 75 | 76 | // NewEventListener creates a new initialized, full specified EventListener 77 | func NewEventListener(name string, t types.EventType, filter func(types.Event) bool, send func(types.Event) error) *EventListener { 78 | el := &EventListener{} 79 | el.name = name 80 | el.s = types.EventListener_RUN 81 | el.filter = filter 82 | el.send = send 83 | el.t = t 84 | return el 85 | } 86 | 87 | // Name returns the name of this listener; names are used to make unique keys, must be unique 88 | func (v *EventListener) Name() string { return v.name } 89 | 90 | // State is the current state of the listener; listeners can be temporarily muted, for instance 91 | func (v *EventListener) State() types.EventListenerState { return v.s } 92 | 93 | // SetState sets the listener runstate 94 | func (v *EventListener) SetState(s types.EventListenerState) { v.s = s } 95 | 96 | // Send processes the callback to send the event to the object listening. 97 | func (v *EventListener) Send(ev types.Event) (e error) { 98 | if v.filter(ev) { 99 | return v.send(ev) 100 | } 101 | return 102 | } 103 | 104 | // Filter processes a filter callback, returns whether this event would be filtered. 105 | // Send uses this automatically. 106 | func (v *EventListener) Filter(ev types.Event) (r bool) { 107 | return v.filter(ev) 108 | } 109 | 110 | // Type returns the type of event we're listening for. This is another kind of filter. 111 | func (v *EventListener) Type() types.EventType { return v.t } 112 | -------------------------------------------------------------------------------- /core/Logger.go: -------------------------------------------------------------------------------- 1 | /* Logger.go: Implementation of the WriterLogger & ServiceLogger 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "fmt" 14 | "io" 15 | "strings" 16 | "time" 17 | 18 | "github.com/kraken-hpc/kraken/lib/types" 19 | ) 20 | 21 | /////////////////////// 22 | // Auxiliary Objects / 23 | ///////////////////// 24 | 25 | // create shortcut aliases for log levels 26 | const ( 27 | DDDEBUG = types.LLDDDEBUG 28 | DDEBUG = types.LLDDEBUG 29 | DEBUG = types.LLDEBUG 30 | INFO = types.LLINFO 31 | NOTICE = types.LLNOTICE 32 | WARNING = types.LLWARNING 33 | ERROR = types.LLERROR 34 | CRITICAL = types.LLCRITICAL 35 | FATAL = types.LLFATAL 36 | PANIC = types.LLPANIC 37 | ) 38 | 39 | // LoggerEvent is used by ServiceLogger to send log events over channels 40 | type LoggerEvent struct { 41 | Level types.LoggerLevel 42 | Module string 43 | Message string 44 | } 45 | 46 | // ServiceLoggerListener receives log events through a channel and passes them to a secondary logger interface 47 | // This can be run as a goroutine 48 | func ServiceLoggerListener(l types.Logger, c <-chan LoggerEvent) { 49 | for { 50 | le := <-c 51 | l.Log(le.Level, le.Module+":"+le.Message) 52 | } 53 | } 54 | 55 | ///////////////////////// 56 | // WriterLogger Object / 57 | /////////////////////// 58 | 59 | var _ types.Logger = (*WriterLogger)(nil) 60 | 61 | // A WriterLogger writes to any io.Writer interface, e.g. stdout, stderr, or an open file 62 | // NOTE: Does not close the interface 63 | type WriterLogger struct { 64 | w io.Writer 65 | m string 66 | lv types.LoggerLevel 67 | DisablePrefix bool 68 | } 69 | 70 | // Log submits a Log message with a LoggerLevel 71 | func (l *WriterLogger) Log(lv types.LoggerLevel, m string) { 72 | if l.IsEnabledFor(lv) { 73 | plv := string(lv) 74 | if int(lv) <= len(types.LoggerLevels)+1 { 75 | plv = types.LoggerLevels[lv] 76 | } 77 | var s []string 78 | if !l.DisablePrefix { 79 | s = []string{ 80 | time.Now().Format("15:04:05.000"), 81 | l.m, 82 | plv, 83 | strings.TrimSpace(m) + "\n", 84 | } 85 | } else { 86 | s = []string{ 87 | l.m, 88 | plv, 89 | strings.TrimSpace(m) + "\n", 90 | } 91 | } 92 | l.w.Write([]byte(strings.Join(s, ":"))) 93 | } 94 | } 95 | 96 | // Logf is the same as Log but with sprintf formatting 97 | func (l *WriterLogger) Logf(lv types.LoggerLevel, f string, v ...interface{}) { 98 | if l.IsEnabledFor(lv) { 99 | l.Log(lv, fmt.Sprintf(f, v...)) 100 | } 101 | } 102 | 103 | // SetModule sets an identifier string for the component that will use this Logger 104 | func (l *WriterLogger) SetModule(m string) { l.m = m } 105 | 106 | // GetModule gets the current module string 107 | func (l *WriterLogger) GetModule() string { return l.m } 108 | 109 | // SetLoggerLevel sets the log filtering level 110 | func (l *WriterLogger) SetLoggerLevel(lv types.LoggerLevel) { l.lv = lv } 111 | 112 | // GetLoggerLevel gets the log filtering level 113 | func (l *WriterLogger) GetLoggerLevel() types.LoggerLevel { return l.lv } 114 | 115 | // IsEnabledFor determines if this Logger would send a message at a particular level 116 | func (l *WriterLogger) IsEnabledFor(lv types.LoggerLevel) (r bool) { 117 | if lv <= l.lv { 118 | return true 119 | } 120 | return 121 | } 122 | 123 | // RegisterWriter sets the writer interface this logger will use 124 | func (l *WriterLogger) RegisterWriter(w io.Writer) { 125 | l.w = w 126 | } 127 | 128 | ////////////////////////// 129 | // ServiceLogger Object / 130 | //////////////////////// 131 | 132 | var _ types.Logger = (*ServiceLogger)(nil) 133 | 134 | // A ServiceLogger is a channel interface for aggregating logs from services running as separate goroutines 135 | type ServiceLogger struct { 136 | c chan<- LoggerEvent 137 | m string 138 | lv types.LoggerLevel 139 | } 140 | 141 | func (l *ServiceLogger) Log(lv types.LoggerLevel, m string) { 142 | if l.IsEnabledFor(lv) { 143 | l.c <- LoggerEvent{ 144 | Level: lv, 145 | Module: l.m, 146 | Message: m, 147 | } 148 | } 149 | } 150 | func (l *ServiceLogger) Logf(lv types.LoggerLevel, f string, v ...interface{}) { 151 | if l.IsEnabledFor(lv) { 152 | l.Log(lv, fmt.Sprintf(f, v...)) 153 | } 154 | } 155 | func (l *ServiceLogger) SetModule(m string) { l.m = m } 156 | func (l *ServiceLogger) GetModule() string { return l.m } 157 | func (l *ServiceLogger) SetLoggerLevel(lv types.LoggerLevel) { l.lv = lv } 158 | func (l *ServiceLogger) GetLoggerLevel() types.LoggerLevel { return l.lv } 159 | func (l *ServiceLogger) IsEnabledFor(lv types.LoggerLevel) (r bool) { 160 | if lv <= l.lv { 161 | return true 162 | } 163 | return 164 | } 165 | 166 | // RegisterChannel sets the chan that the ServiceLogger will send events over 167 | func (l *ServiceLogger) RegisterChannel(c chan<- LoggerEvent) { l.c = c } 168 | -------------------------------------------------------------------------------- /core/Query.go: -------------------------------------------------------------------------------- 1 | /* Query.go: defines the Query object used by the QueryEngine for querying state 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "reflect" 14 | 15 | "github.com/kraken-hpc/kraken/lib/types" 16 | ) 17 | 18 | ////////////////// 19 | // Query Object / 20 | //////////////// 21 | 22 | var _ types.Query = (*Query)(nil) 23 | 24 | // Query objects describe a state query 25 | type Query struct { 26 | t types.QueryType 27 | s types.QueryState 28 | u string 29 | v []reflect.Value 30 | c chan types.QueryResponse 31 | } 32 | 33 | // NewQuery creates an initialized query; this is how all Queries should be created 34 | func NewQuery(t types.QueryType, s types.QueryState, url string, v []reflect.Value) (*Query, chan types.QueryResponse) { 35 | q := &Query{} 36 | q.t = t 37 | q.s = s 38 | q.u = url 39 | q.v = v 40 | q.c = make(chan types.QueryResponse) 41 | return q, q.c 42 | } 43 | 44 | // Type returns the type of the query (e.g., Create, Update...) 45 | func (q *Query) Type() types.QueryType { return q.t } 46 | 47 | // State returns the state (Dsc, Cfg, or Both) we are querying 48 | func (q *Query) State() types.QueryState { return q.s } 49 | 50 | // URL returns a string representing the object being queried 51 | func (q *Query) URL() string { return q.u } 52 | 53 | // Value returns an array of associated refelct.Value's with this query 54 | func (q *Query) Value() []reflect.Value { return q.v } 55 | 56 | // ResponseChan returns the channel that a QueryResponse should be sent on 57 | func (q *Query) ResponseChan() chan<- types.QueryResponse { return q.c } 58 | 59 | ////////////////////////// 60 | // QueryResponse Object / 61 | //////////////////////// 62 | 63 | var _ types.QueryResponse = (*QueryResponse)(nil) 64 | 65 | // A QueryResponse is sent by the Engine to the requester with results and possible errors 66 | type QueryResponse struct { 67 | e error 68 | v []reflect.Value 69 | } 70 | 71 | // NewQueryResponse creates an initialized and fully specified QueryResponse 72 | func NewQueryResponse(v []reflect.Value, e error) *QueryResponse { 73 | qr := &QueryResponse{ 74 | e: e, 75 | v: v, 76 | } 77 | return qr 78 | } 79 | 80 | // Error returns the error value of the QueryResponse 81 | func (q *QueryResponse) Error() error { return q.e } 82 | 83 | // Value returns an array of []reflect.Value's that may have resulted from the query 84 | func (q *QueryResponse) Value() []reflect.Value { return q.v } 85 | -------------------------------------------------------------------------------- /core/Registry.go: -------------------------------------------------------------------------------- 1 | /* Registry.go: the registry manages static module and extension properties on initialization 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "fmt" 14 | "reflect" 15 | 16 | "github.com/gogo/protobuf/jsonpb" 17 | "github.com/gogo/protobuf/proto" 18 | "github.com/kraken-hpc/kraken/lib/json" 19 | "github.com/kraken-hpc/kraken/lib/types" 20 | ) 21 | 22 | ////////////////////// 23 | // Global Variables / 24 | //////////////////// 25 | 26 | // Registry is where we register modules & extensions 27 | // It provides various internal functions 28 | // We need this prior to init(), so it must be a global 29 | var Registry = NewKrakenRegistry() 30 | 31 | // Set our registry to be our json resolver 32 | func init() { 33 | json.Marshaler.AnyResolver = Registry 34 | json.Unmarshaler.AnyResolver = Registry 35 | } 36 | 37 | /////////////////////////// 38 | // KrakenRegistry Object / 39 | ///////////////////////// 40 | 41 | var _ jsonpb.AnyResolver = (*KrakenRegistry)(nil) 42 | 43 | type KrakenRegistry struct { 44 | Modules map[string]types.Module 45 | Extensions map[string]types.Extension 46 | Discoverables map[string]map[string]map[string]reflect.Value // d["instance_id"]["property_url"]["value_id"] 47 | Mutations map[string]map[string]types.StateMutation // m["instance_id"]["mutation_id"] 48 | ServiceInstances map[string]map[string]types.ServiceInstance // s["module"]["instance_id"] 49 | } 50 | 51 | func NewKrakenRegistry() *KrakenRegistry { 52 | r := &KrakenRegistry{ 53 | Modules: make(map[string]types.Module), 54 | Extensions: make(map[string]types.Extension), 55 | Discoverables: make(map[string]map[string]map[string]reflect.Value), 56 | Mutations: make(map[string]map[string]types.StateMutation), 57 | ServiceInstances: make(map[string]map[string]types.ServiceInstance), 58 | } 59 | return r 60 | } 61 | 62 | // RegisterModule adds an module to the map if it hasn't been already 63 | // It's probably a good idea for this to be done in init() 64 | func (r *KrakenRegistry) RegisterModule(m types.Module) { 65 | if _, ok := r.Modules[m.Name()]; !ok { 66 | r.Modules[m.Name()] = m 67 | } 68 | } 69 | 70 | // RegisterExtension adds an extension to the map if it hasn't been already 71 | // It's probably a good idea for this to be done in init() 72 | func (r *KrakenRegistry) RegisterExtension(e types.Extension) { 73 | if _, ok := r.Extensions[e.Name()]; !ok { 74 | r.Extensions[e.Name()] = e 75 | } 76 | } 77 | 78 | // RegisterDiscoverable adds a map of discoverables the module can emit 79 | func (r *KrakenRegistry) RegisterDiscoverable(si types.ServiceInstance, d map[string]map[string]reflect.Value) { 80 | r.Discoverables[si.ID()] = d 81 | } 82 | 83 | // RegisterMutations declares mutations a module can perform 84 | func (r *KrakenRegistry) RegisterMutations(si types.ServiceInstance, d map[string]types.StateMutation) { 85 | r.Mutations[si.ID()] = d 86 | } 87 | 88 | // RegisterServiceInstance creates a service instance with a particular module.RegisterServiceInstance 89 | // Note: This can be done after the fact, but serviceinstances that are added after runtime cannot 90 | // (currently) be used as part of mutation chains. 91 | func (r *KrakenRegistry) RegisterServiceInstance(m types.Module, d map[string]types.ServiceInstance) { 92 | r.ServiceInstances[m.Name()] = d 93 | } 94 | 95 | func (r *KrakenRegistry) Resolve(url string) (proto.Message, error) { 96 | if e, ok := r.Extensions[url]; ok { 97 | return e.New(), nil 98 | } 99 | for _, m := range r.Modules { 100 | if m, ok := m.(types.ModuleWithConfig); ok { 101 | if m.ConfigURL() == url { 102 | return m.NewConfig(), nil 103 | } 104 | } 105 | } 106 | return nil, fmt.Errorf("proto not found") 107 | } 108 | -------------------------------------------------------------------------------- /core/ServiceInstance.go: -------------------------------------------------------------------------------- 1 | /* ServiceInstance.go: provides the interface for controlling service instance processes 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2020, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | //go:generate protoc -I proto/src -I proto --gogo_out=Mgoogle/protobuf/any.proto=github.com/gogo/protobuf/types,plugins=grpc:proto proto/src/ServiceInstance.proto 11 | 12 | package core 13 | 14 | import ( 15 | "fmt" 16 | "os" 17 | "sync" 18 | 19 | "github.com/gogo/protobuf/proto" 20 | ptypes "github.com/gogo/protobuf/types" 21 | 22 | fork "github.com/kraken-hpc/go-fork" 23 | "github.com/kraken-hpc/kraken/lib/types" 24 | ) 25 | 26 | //////////////////////////// 27 | // ServiceInstance Object / 28 | ////////////////////////// 29 | 30 | var _ types.ServiceInstance = (*ServiceInstance)(nil) 31 | 32 | // A ServiceInstance describes a service that will be built-in to the binary and exec'ed by forking 33 | // note: state information is stored in the node proto object, this object manages a running context 34 | type ServiceInstance struct { 35 | id string // ID must be unique 36 | module string // name doesn't need to be unique; we can run multiple configs of the same service 37 | exe string // gets set automatically 38 | entry func() // needs to run as a goroutine 39 | sock string 40 | fork *fork.Function 41 | ctl chan<- types.ServiceControl 42 | wchan chan<- types.ServiceInstanceUpdate 43 | state types.ServiceState // note: these states mean a slightly different: RUN means process is running, INIT means nothing 44 | m types.ModuleSelfService 45 | mutex *sync.Mutex 46 | } 47 | 48 | // NewServiceInstance provides a new, initialized ServiceInstance object 49 | func NewServiceInstance(id, module string, entry func()) *ServiceInstance { 50 | si := &ServiceInstance{ 51 | id: id, 52 | module: module, 53 | entry: entry, 54 | fork: nil, 55 | mutex: &sync.Mutex{}, 56 | } 57 | si.setState((types.Service_STOP)) // we're obviously stopped right now 58 | si.exe, _ = os.Executable() 59 | return si 60 | } 61 | 62 | // ID gets the ID string for the service 63 | func (si *ServiceInstance) ID() string { return si.id } 64 | 65 | // Module returns the name of the module this is an instance of 66 | func (si *ServiceInstance) Module() string { return si.module } 67 | 68 | // GetState returns the current run state of the service 69 | func (si *ServiceInstance) GetState() types.ServiceState { 70 | si.mutex.Lock() 71 | defer si.mutex.Unlock() 72 | return si.state 73 | } 74 | 75 | // UpdateConfig will send a signal to the running si to check for a config update 76 | func (si *ServiceInstance) UpdateConfig() { 77 | if si.ctl != nil { 78 | si.ctl <- types.ServiceControl{Command: types.ServiceControl_UPDATE} 79 | } 80 | } 81 | 82 | // Start will execute the process 83 | func (si *ServiceInstance) Start() { 84 | e := si.start() 85 | if e != nil { 86 | si.setState(types.Service_ERROR) 87 | return 88 | } 89 | si.setState(types.Service_RUN) 90 | go si.watcher() 91 | } 92 | 93 | // Stop sends a signal to the running si to stop 94 | func (si *ServiceInstance) Stop() { 95 | if si.ctl != nil { 96 | si.ctl <- types.ServiceControl{Command: types.ServiceControl_STOP} 97 | } 98 | } 99 | 100 | // Watch provides a channel where process state changes will be reported 101 | func (si *ServiceInstance) Watch(wchan chan<- types.ServiceInstanceUpdate) { 102 | si.wchan = wchan 103 | } 104 | 105 | // SetCtl sets the channel to send control message to (to pass through the API) 106 | func (si *ServiceInstance) SetCtl(ctl chan<- types.ServiceControl) { 107 | si.ctl = ctl 108 | } 109 | 110 | // SetSock sets the path to the API socket 111 | func (si *ServiceInstance) SetSock(sock string) { 112 | si.sock = sock 113 | } 114 | 115 | // setState sets the state, but should only be done internally. This makes sure we notify any watcher 116 | func (si *ServiceInstance) setState(state types.ServiceState) { 117 | si.mutex.Lock() 118 | defer si.mutex.Unlock() 119 | si.state = state 120 | if si.wchan != nil { 121 | si.wchan <- types.ServiceInstanceUpdate{ 122 | ID: si.id, 123 | State: si.state, 124 | } 125 | } 126 | } 127 | 128 | func (si *ServiceInstance) watcher() { 129 | e := si.fork.Wait() 130 | if e != nil { 131 | si.setState(types.Service_ERROR) 132 | return 133 | } 134 | si.setState(types.Service_STOP) 135 | } 136 | 137 | func (si *ServiceInstance) start() (e error) { 138 | si.mutex.Lock() 139 | defer si.mutex.Unlock() 140 | if si.state == types.Service_RUN { 141 | return fmt.Errorf("cannot start service instance not in stop state") 142 | } 143 | if _, e = os.Stat(si.exe); os.IsNotExist(e) { 144 | return e 145 | } 146 | si.fork = fork.NewFork("ModuleExecute", ModuleExecute, os.Args[0], "["+si.id+"]") 147 | si.fork.Stdin = os.Stdin 148 | si.fork.Stdout = os.Stdout 149 | si.fork.Stderr = os.Stderr 150 | return si.fork.Fork(si.ID(), si.module, si.sock) 151 | } 152 | 153 | // moduleExecute does all of the necessary steps to start the service instance 154 | // this is the actual entry point for a new module process 155 | func ModuleExecute(id, module, sock string) { 156 | m, ok := Registry.Modules[module] 157 | if !ok { 158 | fmt.Printf("trying to launch non-existent module: %s", module) 159 | return 160 | } 161 | mss, ok := m.(types.ModuleSelfService) 162 | if !ok { 163 | fmt.Printf("module is not executable: %s", module) 164 | return 165 | } 166 | config := false 167 | mc, ok := m.(types.ModuleWithConfig) 168 | if ok { 169 | config = true 170 | } 171 | 172 | fmt.Printf("I am: %s\n", id) 173 | 174 | api := NewModuleAPIClient(sock) 175 | // call in, and get a control chan 176 | cc, e := api.ServiceInit(id, module) 177 | if e != nil { 178 | fmt.Printf("sock: %v\nid: %v\nmodule: %v\nerror: %v\n", os.Getenv("KRAKEN_SOCK"), os.Getenv("KRAKEN_ID"), os.Getenv("KRAKEN_MODULE"), e) 179 | return 180 | } 181 | 182 | // Setup logger stream 183 | if e = api.LoggerInit(id); e != nil { 184 | fmt.Printf("failed to create logger stream: %v\n", e) 185 | return 186 | } 187 | 188 | // Setup mutation stream if we need it 189 | mm, ok := m.(types.ModuleWithMutations) 190 | if ok { 191 | cc, e := api.MutationInit(id, module) 192 | if e != nil { 193 | api.Logf(ERROR, "failed to create mutation stream: %v\n", e) 194 | return 195 | } 196 | mm.SetMutationChan(cc) 197 | } 198 | 199 | // Setup event stream if we need it 200 | me, ok := m.(types.ModuleWithAllEvents) 201 | if ok { 202 | cc, e := api.EventInit(id, module) 203 | if e != nil { 204 | api.Logf(ERROR, "failed to create event stream: %v\n", e) 205 | return 206 | } 207 | me.SetEventsChan(cc) 208 | } 209 | 210 | // Setup discovery stream if we need it 211 | md, ok := m.(types.ModuleWithDiscovery) 212 | if ok { 213 | cc, e := api.DiscoveryInit(id) 214 | if e != nil { 215 | api.Logf(ERROR, "failed to create discovery stream: %v\n", e) 216 | return 217 | } 218 | md.SetDiscoveryChan(cc) 219 | } 220 | 221 | updateConfig := func() { 222 | if !config { 223 | api.Logf(ERROR, "tried to update config on module with no config") 224 | return 225 | } 226 | // Get a copy of the config 227 | n, _ := api.QueryRead(api.Self().String()) 228 | srv := n.GetService(id) 229 | p, e := Registry.Resolve(srv.GetConfig().GetTypeUrl()) 230 | if e != nil { 231 | api.Logf(ERROR, "resolve config error (%s): %v\n", srv.GetConfig().GetTypeUrl(), e) 232 | return 233 | } 234 | e = ptypes.UnmarshalAny(srv.GetConfig(), p) 235 | if e != nil { 236 | api.Logf(ERROR, "unmarshal config failure: %v\n", e) 237 | return 238 | } 239 | defaults := mc.NewConfig() 240 | proto.Merge(defaults, p) 241 | mc.UpdateConfig(defaults) 242 | } 243 | 244 | mss.Init(api) 245 | if config { 246 | updateConfig() 247 | } 248 | go mss.Entry() 249 | 250 | for { 251 | select { 252 | case cmd := <-cc: 253 | switch cmd.Command { 254 | case types.ServiceControl_STOP: 255 | api.Logf(NOTICE, "stopping") 256 | mss.Stop() 257 | break 258 | case types.ServiceControl_UPDATE: 259 | updateConfig() 260 | break 261 | default: 262 | } 263 | } 264 | } 265 | } 266 | 267 | func init() { 268 | fork.RegisterFunc("ModuleExecute", ModuleExecute) 269 | } 270 | -------------------------------------------------------------------------------- /core/ServiceManager.go: -------------------------------------------------------------------------------- 1 | /* ServiceManagement.go: provides management of external service modules 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "reflect" 14 | "regexp" 15 | "sync" 16 | 17 | pb "github.com/kraken-hpc/kraken/core/proto" 18 | ct "github.com/kraken-hpc/kraken/core/proto/customtypes" 19 | "github.com/kraken-hpc/kraken/lib/types" 20 | "github.com/kraken-hpc/kraken/lib/util" 21 | ) 22 | 23 | /////////////////////////// 24 | // ServiceManager Object / 25 | ///////////////////////// 26 | 27 | var _ types.ServiceManager = (*ServiceManager)(nil) 28 | 29 | type ServiceManager struct { 30 | srv map[string]types.ServiceInstance // map of si IDs to ServiceInstances 31 | mutex *sync.Mutex 32 | sock string // socket that we use for API comms 33 | sclist types.EventListener 34 | echan chan types.Event 35 | wchan chan types.ServiceInstanceUpdate 36 | ctx Context 37 | query *QueryEngine 38 | log types.Logger 39 | } 40 | 41 | func NewServiceManager(ctx Context, sock string) *ServiceManager { 42 | sm := &ServiceManager{ 43 | srv: make(map[string]types.ServiceInstance), 44 | mutex: &sync.Mutex{}, 45 | sock: sock, 46 | echan: make(chan types.Event), 47 | wchan: make(chan types.ServiceInstanceUpdate), 48 | ctx: ctx, 49 | log: &ctx.Logger, 50 | query: &ctx.Query, 51 | } 52 | sm.log.SetModule("ServiceManager") 53 | return sm 54 | } 55 | 56 | func (sm *ServiceManager) Run(ready chan<- interface{}) { 57 | // subscribe to STATE_CHANGE events for "/Services" 58 | smurl := regexp.MustCompile(`^\/?Services\/`) 59 | sm.sclist = NewEventListener( 60 | "ServiceManager", 61 | types.Event_STATE_CHANGE, 62 | func(v types.Event) bool { 63 | node, url := util.NodeURLSplit(v.URL()) 64 | if !ct.NewNodeID(node).EqualTo(sm.ctx.Self) { 65 | return false 66 | } 67 | if smurl.MatchString(url) { 68 | return true 69 | } 70 | return false 71 | }, 72 | func(v types.Event) error { return ChanSender(v, sm.echan) }, 73 | ) 74 | sm.ctx.SubChan <- sm.sclist 75 | 76 | // initialize service instances 77 | for m := range Registry.ServiceInstances { 78 | for _, si := range Registry.ServiceInstances[m] { 79 | sm.log.Logf(types.LLINFO, "adding service: %s", si.ID()) 80 | sm.AddService(si) 81 | } 82 | } 83 | 84 | go func() { 85 | sm.log.Logf(types.LLDEBUG, "starting initial service sync") 86 | for _, si := range sm.srv { 87 | sm.log.Logf(types.LLDDEBUG, "starting initial service sync: %s", si.ID()) 88 | sm.syncService(si.ID()) 89 | sm.setServiceStateDsc(si.ID(), pb.ServiceInstance_STOP) 90 | } 91 | }() 92 | 93 | // ready to go 94 | ready <- nil 95 | 96 | // main listening loop 97 | for { 98 | select { 99 | case v := <-sm.echan: 100 | // state change for services 101 | sm.log.Logf(types.LLDDEBUG, "processing state change event: %s", v.URL()) 102 | sm.processStateChange(v.Data().(*StateChangeEvent)) 103 | case su := <-sm.wchan: 104 | // si changed process state 105 | sm.log.Logf(types.LLDDEBUG, "processing SI state update: %s -> %s", su.ID, su.State) 106 | go sm.processUpdate(su) 107 | } 108 | } 109 | } 110 | 111 | func (sm *ServiceManager) AddService(si types.ServiceInstance) { 112 | sm.mutex.Lock() 113 | defer sm.mutex.Unlock() 114 | if _, ok := sm.srv[si.ID()]; ok { 115 | sm.log.Logf(types.LLERROR, "tried to add service that already exists: %s", si.ID()) 116 | } 117 | sm.srv[si.ID()] = si 118 | si.Watch(sm.wchan) 119 | si.SetSock(sm.sock) 120 | } 121 | 122 | func (sm *ServiceManager) DelService(si string) { 123 | sm.mutex.Lock() 124 | defer sm.mutex.Unlock() 125 | if si, ok := sm.srv[si]; ok { 126 | si.Watch(nil) // we don't want to watch this anymore 127 | si.SetSock("") 128 | delete(sm.srv, si.ID()) 129 | } 130 | } 131 | 132 | func (sm *ServiceManager) GetService(si string) types.ServiceInstance { 133 | sm.mutex.Lock() 134 | defer sm.mutex.Unlock() 135 | if si, ok := sm.srv[si]; ok { 136 | return si 137 | } 138 | return nil 139 | } 140 | 141 | func (sm *ServiceManager) processStateChange(v *StateChangeEvent) { 142 | // extract SI 143 | _, url := util.NodeURLSplit(v.URL) 144 | us := util.URLToSlice(url) 145 | si := "" 146 | // this makes sure we don't get tripped up by leading slashes 147 | for i := range us { 148 | if us[i] == "Services" { 149 | si = us[i+1] 150 | } 151 | } 152 | if si == "" { 153 | sm.log.Logf(types.LLDEBUG, "failed to parse URL for /Services state change: %s", v.URL) 154 | return 155 | } 156 | sm.syncService(si) 157 | } 158 | 159 | func (sm *ServiceManager) processUpdate(su types.ServiceInstanceUpdate) { 160 | // set the state in the SDE 161 | switch su.State { 162 | case types.Service_STOP: 163 | sm.setServiceStateDsc(su.ID, pb.ServiceInstance_STOP) 164 | case types.Service_RUN: 165 | // this is actually pb state INIT; it's up to 166 | sm.setServiceStateDsc(su.ID, pb.ServiceInstance_INIT) 167 | case types.Service_ERROR: 168 | sm.setServiceStateDsc(su.ID, pb.ServiceInstance_ERROR) 169 | } 170 | } 171 | 172 | // syncService is what actually does most of the work. It compares cfg to dsc and decides what to do 173 | func (sm *ServiceManager) syncService(si string) { 174 | sm.log.Logf(types.LLDDEBUG, "syncing service: %s", si) 175 | srv := sm.GetService(si) 176 | if srv == nil { 177 | sm.log.Logf(types.LLERROR, "tried to sync non-existent service: %s", si) 178 | return 179 | } 180 | c := sm.getServiceStateCfg(si) 181 | d := sm.getServiceStateDsc(si) 182 | 183 | if c == d { // nothing to do 184 | sm.log.Logf(types.LLDDDEBUG, "service already synchronized: %s (%+v == %+v)", si, c, d) 185 | return 186 | } 187 | if d == pb.ServiceInstance_ERROR { // don't clear errors 188 | return 189 | } 190 | switch c { 191 | case pb.ServiceInstance_RUN: // we're supposed to be running 192 | if d != pb.ServiceInstance_INIT { // did we already try to start? 193 | sm.setServiceStateDsc(si, pb.ServiceInstance_INIT) 194 | sm.log.Logf(types.LLDDEBUG, "starting service: %s", si) 195 | go srv.Start() // startup 196 | } 197 | case pb.ServiceInstance_STOP: // we're supposed to be stopped 198 | sm.log.Logf(types.LLDDEBUG, "stopping service: %s", si) 199 | srv.Stop() // stop 200 | } 201 | } 202 | 203 | // Some helper functions... 204 | 205 | func (sm *ServiceManager) getServiceStateCfg(si string) pb.ServiceInstance_ServiceState { 206 | n, _ := sm.query.Read(sm.ctx.Self) 207 | v, e := n.GetValue(sm.stateURL(si)) 208 | if e != nil { 209 | sm.log.Logf(types.LLERROR, "failed to get cfg state value (%s): %s", sm.stateURL(si), e.Error()) 210 | return pb.ServiceInstance_UNKNOWN 211 | } 212 | return pb.ServiceInstance_ServiceState(v.Int()) 213 | } 214 | 215 | func (sm *ServiceManager) getServiceStateDsc(si string) pb.ServiceInstance_ServiceState { 216 | n, _ := sm.query.ReadDsc(sm.ctx.Self) 217 | v, e := n.GetValue(sm.stateURL(si)) 218 | if e != nil { 219 | sm.log.Logf(types.LLERROR, "failed to get dsc state value (%s): %s", sm.stateURL(si), e.Error()) 220 | return pb.ServiceInstance_UNKNOWN 221 | } 222 | return pb.ServiceInstance_ServiceState(v.Int()) 223 | } 224 | 225 | func (sm *ServiceManager) setServiceStateDsc(si string, state pb.ServiceInstance_ServiceState) { 226 | _, e := sm.query.SetValueDsc(util.NodeURLJoin(sm.ctx.Self.String(), sm.stateURL(si)), reflect.ValueOf(state)) 227 | if e != nil { 228 | sm.log.Logf(types.LLERROR, "failed to set dsc state value (%s): %s", sm.stateURL(si), e.Error()) 229 | return 230 | } 231 | } 232 | 233 | func (sm *ServiceManager) stateURL(si string) string { 234 | return util.URLPush(util.URLPush("/Services", si), "State") 235 | } 236 | -------------------------------------------------------------------------------- /core/State.go: -------------------------------------------------------------------------------- 1 | /* State.go: provides an intermediate between Node and Engine 2 | * States store collections of Nodes, and implement node:/node/value URL resolution. 3 | * 4 | * Author: J. Lowell Wofford 5 | * 6 | * This software is open source software available under the BSD-3 license. 7 | * Copyright (c) 2018, Triad National Security, LLC 8 | * See LICENSE file for details. 9 | */ 10 | 11 | package core 12 | 13 | import ( 14 | "fmt" 15 | "reflect" 16 | "sync" 17 | 18 | ct "github.com/kraken-hpc/kraken/core/proto/customtypes" 19 | "github.com/kraken-hpc/kraken/lib/types" 20 | "github.com/kraken-hpc/kraken/lib/util" 21 | ) 22 | 23 | ////////////////// 24 | // State Object / 25 | //////////////// 26 | 27 | var _ types.CRUD = (*State)(nil) 28 | var _ types.BulkCRUD = (*State)(nil) 29 | var _ types.Resolver = (*State)(nil) 30 | var _ types.State = (*State)(nil) 31 | 32 | // A State stores and manipulates a collection of Nodes 33 | type State struct { 34 | nodesMutex *sync.RWMutex 35 | nodes map[string]*Node 36 | } 37 | 38 | // NewState creates an initialized state 39 | func NewState() *State { 40 | s := &State{} 41 | s.nodes = make(map[string]*Node) 42 | s.nodesMutex = &sync.RWMutex{} 43 | return s 44 | } 45 | 46 | /* 47 | * CRUD funcs 48 | */ 49 | 50 | // Create creates a node in the state 51 | func (s *State) Create(n types.Node) (r types.Node, e error) { 52 | s.nodesMutex.Lock() 53 | defer s.nodesMutex.Unlock() 54 | 55 | idstr := n.ID().String() 56 | if _, ok := s.nodes[idstr]; ok { 57 | e = fmt.Errorf("not creating node with duplicate ID: %s", idstr) 58 | return 59 | } 60 | s.nodes[idstr] = n.(*Node) 61 | r = s.nodes[idstr] 62 | return 63 | } 64 | 65 | // Read returns a node from the state 66 | func (s *State) Read(nid types.NodeID) (r types.Node, e error) { 67 | s.nodesMutex.RLock() 68 | defer s.nodesMutex.RUnlock() 69 | 70 | idstr := nid.String() 71 | if v, ok := s.nodes[idstr]; ok { 72 | r = v 73 | return 74 | } 75 | e = fmt.Errorf("no node found by id: %s", idstr) 76 | return 77 | } 78 | 79 | // Update updates a node in the state 80 | func (s *State) Update(n types.Node) (r types.Node, e error) { 81 | s.nodesMutex.Lock() 82 | defer s.nodesMutex.Unlock() 83 | 84 | idstr := n.ID().String() 85 | if _, ok := s.nodes[idstr]; ok { 86 | s.nodes[idstr] = n.(*Node) 87 | r = s.nodes[idstr] 88 | return 89 | } 90 | e = fmt.Errorf("could not update node, id does not exist: %s", idstr) 91 | return 92 | } 93 | 94 | // Delete removes a node from the state 95 | func (s *State) Delete(n types.Node) (r types.Node, e error) { 96 | return s.DeleteByID(n.ID()) 97 | } 98 | 99 | // DeleteByID deletes a node from the state keyed by NodeID 100 | func (s *State) DeleteByID(nid types.NodeID) (r types.Node, e error) { 101 | s.nodesMutex.Lock() 102 | defer s.nodesMutex.Unlock() 103 | 104 | idstr := nid.String() 105 | if v, ok := s.nodes[idstr]; ok { 106 | delete(s.nodes, idstr) 107 | r = v 108 | return 109 | } 110 | e = fmt.Errorf("could not delete non-existent node: %s", idstr) 111 | return 112 | } 113 | 114 | /* 115 | * BulkCRUD funcs 116 | */ 117 | 118 | // BulkCreate creates multiple nodes 119 | func (s *State) BulkCreate(ns []types.Node) (r []types.Node, e error) { 120 | return bulkCRUD(ns, s.Create) 121 | } 122 | 123 | // BulkRead reads multiple nodes 124 | func (s *State) BulkRead(nids []types.NodeID) (r []types.Node, e error) { 125 | return bulkCRUDByID(nids, s.Read) 126 | } 127 | 128 | // BulkUpdate updates multiple nodes 129 | func (s *State) BulkUpdate(ns []types.Node) (r []types.Node, e error) { 130 | return bulkCRUD(ns, s.Update) 131 | } 132 | 133 | // BulkDelete removes multiple nodes 134 | func (s *State) BulkDelete(ns []types.Node) (r []types.Node, e error) { 135 | return bulkCRUD(ns, s.Delete) 136 | } 137 | 138 | // BulkDeleteByID removes multiple nodes keyed by NodeID 139 | func (s *State) BulkDeleteByID(nids []types.NodeID) (r []types.Node, e error) { 140 | return bulkCRUDByID(nids, s.DeleteByID) 141 | } 142 | 143 | // ReadAll returns a slice of all nodes from the state 144 | func (s *State) ReadAll() (r []types.Node, e error) { 145 | s.nodesMutex.RLock() 146 | defer s.nodesMutex.RUnlock() 147 | 148 | for _, v := range s.nodes { 149 | r = append(r, v) 150 | } 151 | return 152 | } 153 | 154 | // DeleteAll will remove all nodes from the state 155 | func (s *State) DeleteAll() (r []types.Node, e error) { 156 | s.nodesMutex.Lock() 157 | defer s.nodesMutex.Unlock() 158 | 159 | r, e = s.ReadAll() 160 | s.nodes = make(map[string]*Node) 161 | return 162 | } 163 | 164 | /* 165 | * Resolver funcs 166 | */ 167 | 168 | // GetValue will query a property with URL, where node is mapped by ID 169 | func (s *State) GetValue(url string) (r reflect.Value, e error) { 170 | n, _, sub, e := s.resolveNode(url) 171 | if e != nil { 172 | return 173 | } 174 | r, e = n.(*Node).GetValue(sub) 175 | return 176 | } 177 | 178 | // SetValue will set a property with URL, where node is mapped by ID 179 | func (s *State) SetValue(url string, v reflect.Value) (r reflect.Value, e error) { 180 | n, _, sub, e := s.resolveNode(url) 181 | if e != nil { 182 | return 183 | } 184 | r, e = n.(*Node).SetValue(sub, v) 185 | return 186 | } 187 | 188 | /* 189 | * TODO: Index funcs 190 | func (s *State) CreateIndex(key string) (e error) { return } 191 | func (s *State) DeleteIndex(key string) (e error) { return } 192 | func (s *State) RebuildIndex(key string) (e error) { return } 193 | func (s *State) QueryIndex(key string, value string) (ns []types.IndexableNode, e error) { return } 194 | */ 195 | 196 | /* 197 | * TODO: Queryable funcs 198 | func (s *State) Search(key string, value reflect.Value) (r []string) { return } 199 | func (s *State) QuerySelect(query string) (r []types.Node, e error) { return } 200 | func (s *State) QueryUpdate(query string, value reflect.Value) (r []reflect.Value, e error) { return } 201 | func (s *State) QueryDelete(query string) (r []types.Node, e error) { return } 202 | */ 203 | 204 | //////////////////////// 205 | // Unexported methods / 206 | ////////////////////// 207 | 208 | /* 209 | * resolveNode is a way to separate URL -> Node resolver 210 | * n - the node resolved (if any) 211 | * root - the root that matched the node 212 | * sub - the sub-URL remaining with root removed (has leading /) 213 | * e - error should be nil on success. Should always be set if no node found. 214 | */ 215 | func (s *State) resolveNode(url string) (n types.Node, root, sub string, e error) { 216 | root, sub = util.NodeURLSplit(url) 217 | n, e = s.Read(ct.NewNodeIDFromURL(root)) 218 | if n == nil { 219 | e = fmt.Errorf("no such node: %s", root) 220 | } 221 | return 222 | } 223 | 224 | func bulkCRUD(ns []types.Node, f func(types.Node) (types.Node, error)) (r []types.Node, e error) { 225 | for _, n := range ns { 226 | var ret types.Node 227 | ret, e = f(n) 228 | if e != nil { 229 | return 230 | } 231 | r = append(r, ret) 232 | } 233 | return 234 | } 235 | 236 | func bulkCRUDByID(ns []types.NodeID, f func(types.NodeID) (types.Node, error)) (r []types.Node, e error) { 237 | for _, n := range ns { 238 | var ret types.Node 239 | ret, e = f(n) 240 | if e != nil { 241 | return 242 | } 243 | r = append(r, ret) 244 | } 245 | return 246 | } 247 | -------------------------------------------------------------------------------- /core/StateMutation.go: -------------------------------------------------------------------------------- 1 | /* StateMutation.go: a state mutation describes a mutation of state 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package core 11 | 12 | import ( 13 | "reflect" 14 | "time" 15 | 16 | "github.com/kraken-hpc/kraken/lib/types" 17 | ) 18 | 19 | ////////////////////////// 20 | // StateMutation Object / 21 | //////////////////////// 22 | 23 | var _ types.StateMutation = (*StateMutation)(nil) 24 | 25 | // A StateMutation describes a possible mutation of state 26 | // These are declared by modules 27 | // These are used to construct the state evolution graph 28 | type StateMutation struct { 29 | mut map[string][2]reflect.Value 30 | context types.StateMutationContext 31 | base *StateSpec // this is the spec, less the mutation value 32 | timeout time.Duration 33 | failto [3]string 34 | } 35 | 36 | // NewStateMutation creates an initialized, specified StateMutation object 37 | func NewStateMutation(mut map[string][2]reflect.Value, req map[string]reflect.Value, exc map[string]reflect.Value, context types.StateMutationContext, timeout time.Duration, failto [3]string) *StateMutation { 38 | for u := range mut { 39 | if _, ok := req[u]; ok { 40 | // FIXME: this should probably error out, but we just fix the problem 41 | delete(req, u) 42 | } 43 | if _, ok := exc[u]; ok { 44 | // FIXME: this should probably error out, but we just fix the problem 45 | delete(exc, u) 46 | } 47 | } 48 | r := &StateMutation{} 49 | r.mut = mut 50 | r.base = NewStateSpec(req, exc) 51 | r.context = context 52 | r.timeout = timeout 53 | r.failto = failto 54 | return r 55 | } 56 | 57 | // Mutates returns the map of URLs/values (before & after) that mutate in this mutation 58 | func (s *StateMutation) Mutates() map[string][2]reflect.Value { return s.mut } 59 | 60 | // Requires returns the map of URLs/values that are required for this mutation 61 | func (s *StateMutation) Requires() map[string]reflect.Value { return s.base.Requires() } 62 | 63 | // Excludes returns the map of URLs/values that are mutally exclusive with this mutation 64 | func (s *StateMutation) Excludes() map[string]reflect.Value { return s.base.Excludes() } 65 | 66 | // Context specifies in which context (Self/Child/All) this mutation applies to 67 | // Note: this doesn't affect the graph; just who does the work. 68 | func (s *StateMutation) Context() types.StateMutationContext { return s.context } 69 | 70 | // Before returns a StatSpec representing the state of of a matching Node before the mutation 71 | func (s *StateMutation) Before() types.StateSpec { 72 | r := make(map[string]reflect.Value) 73 | for u, v := range s.mut { 74 | r[u] = v[0] 75 | } 76 | before, _ := s.base.SpecMerge(NewStateSpec(r, make(map[string]reflect.Value))) 77 | return before 78 | } 79 | 80 | // After returns a StatSpec representing the state of of a matching Node after the mutation 81 | func (s *StateMutation) After() types.StateSpec { 82 | r := make(map[string]reflect.Value) 83 | for u, v := range s.mut { 84 | r[u] = v[1] 85 | } 86 | after, _ := s.base.SpecMerge(NewStateSpec(r, make(map[string]reflect.Value))) 87 | return after 88 | } 89 | 90 | // SpecCompatIn decides if this mutation can form an in arrow in the graph 91 | // This is used for graph building. 92 | func (s *StateMutation) SpecCompatIn(sp types.StateSpec, muts map[string]uint32) bool { 93 | // Philosphy: assume true, fail fast 94 | 95 | // 1. If the specs aren't compatible, we're done 96 | // Provides basic requires/excludes matching 97 | if !s.After().SpecCompat(sp) { 98 | return false 99 | } 100 | 101 | // 1.5 We have stronger requirements for mutation excludes in the case of zero value exclude 102 | for url, val := range s.Excludes() { 103 | if val.IsZero() { 104 | // It's not OK to be unset, and SpecCompat won't catch this (for good reason) 105 | // SpecCompat catches the case of set, but zero 106 | if _, ok := sp.Requires()[url]; !ok { 107 | return false 108 | } 109 | } 110 | } 111 | 112 | // 2. Do the requirements of the spec match the end of what this mutation mutates? 113 | for u, v := range s.Mutates() { 114 | spv, ok := sp.Requires()[u] 115 | if !ok { 116 | // make a zero value of the type we're comparing 117 | spv = reflect.Indirect(reflect.New(v[0].Type())) 118 | } 119 | if spv.Interface() != v[1].Interface() { 120 | return false 121 | } 122 | } 123 | 124 | // 3. If our mutation requires x & and x is a mutator (globally) -> they must match 125 | for r := range s.Requires() { 126 | if _, ok := muts[r]; ok { // our requires is also a mutator 127 | spv, ok := sp.Requires()[r] 128 | if !ok { 129 | return false 130 | } 131 | if spv.Interface() != s.Requires()[r].Interface() { 132 | return false 133 | } 134 | } 135 | } 136 | 137 | // Ok, we're copatible 138 | return true 139 | } 140 | 141 | // SpecCompatOut decides if this mutaiton can form an out arrow in the graph 142 | // This is used for graph building. 143 | func (s *StateMutation) SpecCompatOut(sp types.StateSpec, muts map[string]uint32) bool { 144 | // Philosphy: assume true, fail fast 145 | 146 | // 1. If the specs aren't compatible, we're done 147 | // Provides basic requires/excludes matching 148 | if !s.Before().SpecCompat(sp) { 149 | return false 150 | } 151 | 152 | // 1.5 We have stronger requirements for mutation excludes in the case of zero value exclude 153 | for url, val := range s.Excludes() { 154 | if val.IsZero() { 155 | // It's not OK to be unset, and SpecCompat won't catch this (for good reason) 156 | // SpecCompat catches the case of set, but zero 157 | if _, ok := sp.Requires()[url]; !ok { 158 | return false 159 | } 160 | } 161 | } 162 | 163 | // 2. Do the requirements of the spec match the begining of what this mutation mutates? 164 | for u, v := range s.Mutates() { 165 | spv, ok := sp.Requires()[u] 166 | if !ok { 167 | // make a zero value of the type we're comparing 168 | spv = reflect.Indirect(reflect.New(v[0].Type())) 169 | } 170 | if spv.Interface() != v[0].Interface() { 171 | return false 172 | } 173 | } 174 | 175 | // 3. If our mutation requires x & and x is a mutator (globally) -> they must match 176 | for r := range s.Requires() { 177 | if _, ok := muts[r]; ok { // our requires is also a mutator 178 | spv, ok := sp.Requires()[r] 179 | if !ok { 180 | return false 181 | } 182 | if spv.Interface() != s.Requires()[r].Interface() { 183 | return false 184 | } 185 | } 186 | } 187 | 188 | // Ok, we're copatible 189 | return true 190 | } 191 | 192 | func (s *StateMutation) Timeout() time.Duration { return s.timeout } 193 | 194 | func (s *StateMutation) SetTimeout(t time.Duration) { s.timeout = t } 195 | 196 | func (s *StateMutation) FailTo() [3]string { return s.failto } 197 | -------------------------------------------------------------------------------- /core/proto/Node.go: -------------------------------------------------------------------------------- 1 | /* Node.go: Provides the necessary additional functionality to make the Node a lib.Message 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package proto 11 | 12 | import ( 13 | "github.com/kraken-hpc/kraken/core/proto/customtypes" 14 | "github.com/kraken-hpc/kraken/lib/json" 15 | ) 16 | 17 | // MarshalJSON creats a JSON version of Node 18 | func (n *Node) MarshalJSON() ([]byte, error) { 19 | return json.Marshal(n) 20 | } 21 | 22 | // UnmarshalJSON populates a node from JSON 23 | func (n *Node) UnmarshalJSON(j []byte) error { 24 | return json.Unmarshal(j, n) 25 | } 26 | 27 | func (n *Node) GetId() *customtypes.NodeID { 28 | return n.Id 29 | } 30 | -------------------------------------------------------------------------------- /core/proto/customtypes/NodeID_type.go: -------------------------------------------------------------------------------- 1 | /* UUID_type.go: define methods to make a CustomType that stores UUIDs 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package customtypes 11 | 12 | import ( 13 | "strconv" 14 | "strings" 15 | 16 | uuid "github.com/satori/go.uuid" 17 | ) 18 | 19 | // UUID implements a CustomType as well as a NodeID 20 | // We can't test for that here because importing util/types would cause a loop 21 | 22 | type NodeID struct { 23 | uuid.UUID 24 | } 25 | 26 | func (u NodeID) Marshal() ([]byte, error) { 27 | return u.MarshalBinary() 28 | } 29 | 30 | func (u NodeID) MarshalJSON() ([]byte, error) { 31 | return []byte(strconv.Quote(u.String())), nil 32 | } 33 | 34 | func (u *NodeID) MarshalTo(data []byte) (int, error) { 35 | copy(data, u.Bytes()) 36 | return len(u.Bytes()), nil 37 | } 38 | 39 | func (u *NodeID) Unmarshal(data []byte) error { 40 | return u.UnmarshalBinary(data) 41 | } 42 | 43 | func (u *NodeID) Size() int { return len(u.Bytes()) } 44 | 45 | func (u *NodeID) UnmarshalJSON(data []byte) error { 46 | s := strings.Trim(string(data), "\"'") 47 | return u.UnmarshalText([]byte(s)) 48 | } 49 | 50 | func (u *NodeID) EqualTo(i interface{}) bool { 51 | if u2, ok := i.(*NodeID); ok && u2 != nil { 52 | return uuid.Equal(u.UUID, u2.UUID) 53 | } 54 | return false 55 | } 56 | 57 | func (u NodeID) Equal(u2 NodeID) bool { 58 | return uuid.Equal(u.UUID, u2.UUID) 59 | } 60 | 61 | func (u NodeID) Compare(u2 NodeID) int { 62 | return strings.Compare(u.String(), u2.String()) 63 | } 64 | 65 | func (u *NodeID) Nil() bool { 66 | return uuid.Equal(u.UUID, uuid.Nil) 67 | } 68 | 69 | func NewNodeID(id string) *NodeID { 70 | u := uuid.FromStringOrNil(id) 71 | return &NodeID{u} 72 | } 73 | 74 | func NewNodeIDFromBinary(b []byte) *NodeID { 75 | u := uuid.FromBytesOrNil(b) 76 | return &NodeID{u} 77 | } 78 | 79 | func NewNodeIDFromURL(url string) *NodeID { 80 | id, _ := NodeURLSplit(url) 81 | return NewNodeID(id) 82 | } 83 | 84 | // we can't import this from util 85 | func NodeURLSplit(s string) (node string, url string) { 86 | ret := strings.SplitN(s, ":", 2) 87 | switch len(ret) { 88 | case 0: // empty string 89 | node = "" 90 | url = "" 91 | case 1: // we have just a node? 92 | node = ret[0] 93 | url = "" 94 | case 2: 95 | node = ret[0] 96 | url = ret[1] 97 | break 98 | } 99 | return 100 | } 101 | -------------------------------------------------------------------------------- /core/proto/src/ModuleAPI.proto: -------------------------------------------------------------------------------- 1 | /* API.proto: describes the RPC API protocol 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | syntax = "proto3"; 11 | package proto; 12 | option go_package = ".;proto"; 13 | 14 | import "Node.proto"; 15 | import "google/protobuf/any.proto"; 16 | import "google/protobuf/empty.proto"; 17 | 18 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 19 | option (gogoproto.marshaler_all) = true; 20 | option (gogoproto.unmarshaler_all) = true; 21 | option (gogoproto.sizer_all) = true; 22 | option (gogoproto.goproto_registration) = true; 23 | option (gogoproto.messagename_all) = true; 24 | 25 | message Query { 26 | string URL = 1; 27 | oneof payload { 28 | Node node = 2; 29 | string text = 3; 30 | bool bool = 4; 31 | MutationNodeList mutationNodeList = 5; 32 | MutationEdgeList mutationEdgeList = 6; 33 | MutationPath mutationPath = 7; 34 | } 35 | repeated string filter = 8; // this is used to e.g. select updates only specific URLs 36 | } 37 | 38 | message QueryMulti { 39 | repeated Query queries = 1; 40 | } 41 | 42 | message ServiceInitRequest { 43 | string id = 1; 44 | string module = 2; 45 | } 46 | 47 | message ServiceControl { 48 | enum Command { 49 | STOP = 0; 50 | UPDATE = 1; 51 | INIT = 2; // special control to send our node info one time 52 | } 53 | Command command = 1; 54 | google.protobuf.Any config = 2; 55 | } 56 | 57 | message MutationControl { 58 | enum Type { 59 | MUTATE = 0; 60 | INTERRUPT = 1; 61 | } 62 | string module = 1; 63 | string id = 2; 64 | Type type = 3; 65 | Node cfg = 4; 66 | Node dsc = 5; 67 | } 68 | 69 | message StateChangeControl { 70 | enum Type { 71 | CREATE = 0; 72 | READ = 1; //unused 73 | UPDATE = 2; 74 | DELETE = 3; 75 | CFG_READ = 4; //unused 76 | CFG_UPDATE = 5; 77 | } 78 | Type type = 1; 79 | string url = 2; 80 | string value = 3; 81 | } 82 | 83 | message EventControl { 84 | enum Type { 85 | StateChange = 0; 86 | Mutation = 1; 87 | Discovery = 2; 88 | } 89 | Type type = 1; 90 | oneof event{ 91 | StateChangeControl stateChangeControl =2; 92 | MutationControl mutationControl = 3; 93 | DiscoveryEvent discoveryEvent =4; 94 | } 95 | } 96 | 97 | message DiscoveryEvent { 98 | string id = 1; 99 | string url = 2; 100 | string value_id = 3; 101 | } 102 | 103 | message MutationNodeList { 104 | repeated MutationNode MutationNodeList = 1; 105 | } 106 | 107 | message MutationEdgeList { 108 | repeated MutationEdge MutationEdgeList = 1; 109 | } 110 | 111 | message MutationPath { 112 | int64 cur = 1; 113 | bool cmplt = 2; 114 | repeated MutationEdge chain = 3; 115 | } 116 | 117 | message MutationNode { 118 | string label = 1; 119 | string id = 2; 120 | NodeColor color = 3; 121 | } 122 | 123 | message MutationEdge { 124 | string from = 1; 125 | string to = 2; 126 | string id = 3; 127 | EdgeColor color = 4; 128 | } 129 | 130 | // This is only nessessary for the json mutation edge color to output in the correct format for the dashboard 131 | message EdgeColor { 132 | string color = 1; 133 | string highlight = 2; 134 | bool inherit = 3; 135 | } 136 | 137 | // This is only nessessary for the json mutation node color to output in the correct format for the dashboard 138 | message NodeColor { 139 | string border = 1; 140 | string background = 2; 141 | } 142 | 143 | message LogMessage { 144 | string origin = 1; 145 | uint32 level = 2; 146 | string msg = 3; 147 | } 148 | 149 | service ModuleAPI { 150 | // Query language 151 | // TODO: create API for bulk CRUD operations 152 | rpc QueryCreate(Query) returns (Query) {} 153 | rpc QueryRead(Query) returns (Query) {} 154 | rpc QueryReadDsc(Query) returns (Query) {} 155 | rpc QueryUpdate(Query) returns (Query) {} 156 | rpc QueryUpdateDsc(Query) returns (Query) {} 157 | rpc QueryDelete(Query) returns (Query) {} 158 | rpc QueryReadAll(google.protobuf.Empty) returns (QueryMulti) {} 159 | rpc QueryReadAllDsc(google.protobuf.Empty) returns (QueryMulti) {} 160 | rpc QueryMutationNodes(google.protobuf.Empty) returns (Query) {} 161 | rpc QueryMutationEdges(google.protobuf.Empty) returns (Query) {} 162 | rpc QueryNodeMutationNodes(Query) returns (Query) {} 163 | rpc QueryNodeMutationEdges(Query) returns (Query) {} 164 | rpc QueryNodeMutationPath(Query) returns (Query) {} 165 | rpc QueryDeleteAll(google.protobuf.Empty) returns (QueryMulti) {} 166 | rpc QueryFreeze(google.protobuf.Empty)returns (Query) {} 167 | rpc QueryThaw(google.protobuf.Empty)returns (Query) {} 168 | rpc QueryFrozen(google.protobuf.Empty)returns (Query) {} 169 | 170 | // Service management 171 | rpc ServiceInit(ServiceInitRequest) returns (stream ServiceControl) {} 172 | 173 | // Mutation/Discover management 174 | rpc MutationInit(ServiceInitRequest) returns (stream MutationControl) {} 175 | 176 | // Event management 177 | rpc EventInit(ServiceInitRequest) returns (stream EventControl) {} 178 | 179 | // Discovery management 180 | rpc DiscoveryInit(stream DiscoveryEvent) returns (google.protobuf.Empty) {} 181 | 182 | // Logging 183 | rpc LoggerInit(stream LogMessage) returns (google.protobuf.Empty) {} 184 | } 185 | -------------------------------------------------------------------------------- /core/proto/src/Node.proto: -------------------------------------------------------------------------------- 1 | /* Node.proto: describes the base Node object 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | syntax = "proto3"; 11 | package proto; 12 | option go_package = ".;proto"; 13 | 14 | import "google/protobuf/any.proto"; 15 | import "ServiceInstance.proto"; 16 | 17 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 18 | option (gogoproto.marshaler_all) = true; 19 | option (gogoproto.unmarshaler_all) = true; 20 | option (gogoproto.sizer_all) = true; 21 | option (gogoproto.goproto_registration) = true; 22 | option (gogoproto.messagename_all) = true; 23 | 24 | message NodeList { 25 | repeated Node nodes = 1; 26 | } 27 | 28 | message Node { 29 | bytes id = 1 [(gogoproto.customtype) = "github.com/kraken-hpc/kraken/core/proto/customtypes.NodeID"]; 30 | string nodename = 2; /* may or may not be the hostname */ 31 | enum RunState { 32 | UNKNOWN = 0; 33 | INIT = 1; 34 | SYNC = 2; 35 | ERROR = 3; 36 | } 37 | RunState run_state = 3; 38 | enum PhysState { 39 | PHYS_UNKNOWN = 0; 40 | POWER_OFF = 1; 41 | POWER_ON = 2; 42 | POWER_CYCLE = 3; // probably won't be used 43 | PHYS_HANG = 4; // possibly recoverable (by reboot) physical hange 44 | PHYS_ERROR = 5; // in a permanent, unrecoverable state 45 | } 46 | PhysState phys_state = 4; 47 | string arch = 5; 48 | string platform = 6; 49 | bytes parent_id = 7 [(gogoproto.customtype) = "github.com/kraken-hpc/kraken/core/proto/customtypes.NodeID"]; 50 | enum FreeBusy { 51 | FREE = 0; // I.e. we currently control states on this node 52 | BUSY = 1; // Control of this node has been handed off to something else; we should leave it alone. 53 | } 54 | FreeBusy busy = 8; 55 | reserved 9 to 13; 56 | repeated ServiceInstance services = 14; 57 | repeated google.protobuf.Any extensions = 15; 58 | } -------------------------------------------------------------------------------- /core/proto/src/ServiceInstance.proto: -------------------------------------------------------------------------------- 1 | /* ServiceInstance.proto: describes a Service Instance of a module 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | syntax = "proto3"; 11 | package proto; 12 | option go_package = ".;proto"; 13 | 14 | import "google/protobuf/any.proto"; 15 | 16 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 17 | option (gogoproto.marshaler_all) = true; 18 | option (gogoproto.unmarshaler_all) = true; 19 | option (gogoproto.sizer_all) = true; 20 | option (gogoproto.goproto_registration) = true; 21 | option (gogoproto.messagename_all) = true; 22 | 23 | message ServiceInstance { 24 | string id = 1; // this needs to be unique 25 | string module = 2; // this is potentially non-unique 26 | enum ServiceState { 27 | UNKNOWN = 0; 28 | INIT = 1; 29 | STOP = 2; 30 | RUN = 3; 31 | ERROR = 4; 32 | } 33 | ServiceState state = 3; 34 | google.protobuf.Any config = 4; 35 | string error_msg = 5; 36 | } -------------------------------------------------------------------------------- /core/proto/src/StateSyncMessage.proto: -------------------------------------------------------------------------------- 1 | /* StateSyncMessage.proto: describes the state sync protocol packets & PhoneHome 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | syntax = "proto3"; 11 | package proto; 12 | option go_package = ".;proto"; 13 | 14 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 15 | option (gogoproto.marshaler_all) = true; 16 | option (gogoproto.unmarshaler_all) = true; 17 | option (gogoproto.sizer_all) = true; 18 | option (gogoproto.goproto_registration) = true; 19 | option (gogoproto.messagename_all) = true; 20 | 21 | /* StateSyncMessage are messages sent by StateSync */ 22 | message StateSyncMessage { 23 | bytes id = 1; 24 | bytes hmac = 2; 25 | bytes message = 3; 26 | } 27 | 28 | message PhoneHomeRequest { 29 | bytes id = 1; 30 | } 31 | 32 | message PhoneHomeReply { 33 | bytes pid = 1; 34 | bytes key = 2; 35 | StateSyncMessage cfg = 3; 36 | StateSyncMessage dsc = 4; 37 | } 38 | 39 | service StateSync { 40 | rpc RPCPhoneHome (PhoneHomeRequest) returns (PhoneHomeReply) {} 41 | } -------------------------------------------------------------------------------- /core/proto/src/github.com/gogo/protobuf/gogoproto/gogo.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers for Go with Gadgets 2 | // 3 | // Copyright (c) 2013, The GoGo Authors. All rights reserved. 4 | // http://github.com/gogo/protobuf 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | syntax = "proto2"; 30 | package gogoproto; 31 | 32 | import "google/protobuf/descriptor.proto"; 33 | 34 | option java_package = "com.google.protobuf"; 35 | option java_outer_classname = "GoGoProtos"; 36 | option go_package = "github.com/gogo/protobuf/gogoproto"; 37 | 38 | extend google.protobuf.EnumOptions { 39 | optional bool goproto_enum_prefix = 62001; 40 | optional bool goproto_enum_stringer = 62021; 41 | optional bool enum_stringer = 62022; 42 | optional string enum_customname = 62023; 43 | optional bool enumdecl = 62024; 44 | } 45 | 46 | extend google.protobuf.EnumValueOptions { 47 | optional string enumvalue_customname = 66001; 48 | } 49 | 50 | extend google.protobuf.FileOptions { 51 | optional bool goproto_getters_all = 63001; 52 | optional bool goproto_enum_prefix_all = 63002; 53 | optional bool goproto_stringer_all = 63003; 54 | optional bool verbose_equal_all = 63004; 55 | optional bool face_all = 63005; 56 | optional bool gostring_all = 63006; 57 | optional bool populate_all = 63007; 58 | optional bool stringer_all = 63008; 59 | optional bool onlyone_all = 63009; 60 | 61 | optional bool equal_all = 63013; 62 | optional bool description_all = 63014; 63 | optional bool testgen_all = 63015; 64 | optional bool benchgen_all = 63016; 65 | optional bool marshaler_all = 63017; 66 | optional bool unmarshaler_all = 63018; 67 | optional bool stable_marshaler_all = 63019; 68 | 69 | optional bool sizer_all = 63020; 70 | 71 | optional bool goproto_enum_stringer_all = 63021; 72 | optional bool enum_stringer_all = 63022; 73 | 74 | optional bool unsafe_marshaler_all = 63023; 75 | optional bool unsafe_unmarshaler_all = 63024; 76 | 77 | optional bool goproto_extensions_map_all = 63025; 78 | optional bool goproto_unrecognized_all = 63026; 79 | optional bool gogoproto_import = 63027; 80 | optional bool protosizer_all = 63028; 81 | optional bool compare_all = 63029; 82 | optional bool typedecl_all = 63030; 83 | optional bool enumdecl_all = 63031; 84 | 85 | optional bool goproto_registration = 63032; 86 | optional bool messagename_all = 63033; 87 | 88 | optional bool goproto_sizecache_all = 63034; 89 | optional bool goproto_unkeyed_all = 63035; 90 | } 91 | 92 | extend google.protobuf.MessageOptions { 93 | optional bool goproto_getters = 64001; 94 | optional bool goproto_stringer = 64003; 95 | optional bool verbose_equal = 64004; 96 | optional bool face = 64005; 97 | optional bool gostring = 64006; 98 | optional bool populate = 64007; 99 | optional bool stringer = 67008; 100 | optional bool onlyone = 64009; 101 | 102 | optional bool equal = 64013; 103 | optional bool description = 64014; 104 | optional bool testgen = 64015; 105 | optional bool benchgen = 64016; 106 | optional bool marshaler = 64017; 107 | optional bool unmarshaler = 64018; 108 | optional bool stable_marshaler = 64019; 109 | 110 | optional bool sizer = 64020; 111 | 112 | optional bool unsafe_marshaler = 64023; 113 | optional bool unsafe_unmarshaler = 64024; 114 | 115 | optional bool goproto_extensions_map = 64025; 116 | optional bool goproto_unrecognized = 64026; 117 | 118 | optional bool protosizer = 64028; 119 | optional bool compare = 64029; 120 | 121 | optional bool typedecl = 64030; 122 | 123 | optional bool messagename = 64033; 124 | 125 | optional bool goproto_sizecache = 64034; 126 | optional bool goproto_unkeyed = 64035; 127 | } 128 | 129 | extend google.protobuf.FieldOptions { 130 | optional bool nullable = 65001; 131 | optional bool embed = 65002; 132 | optional string customtype = 65003; 133 | optional string customname = 65004; 134 | optional string jsontag = 65005; 135 | optional string moretags = 65006; 136 | optional string casttype = 65007; 137 | optional string castkey = 65008; 138 | optional string castvalue = 65009; 139 | 140 | optional bool stdtime = 65010; 141 | optional bool stdduration = 65011; 142 | optional bool wktpointer = 65012; 143 | 144 | } 145 | -------------------------------------------------------------------------------- /core/proto/src/google/protobuf/any.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option go_package = "github.com/golang/protobuf/ptypes/any"; 37 | option java_package = "com.google.protobuf"; 38 | option java_outer_classname = "AnyProto"; 39 | option java_multiple_files = true; 40 | option objc_class_prefix = "GPB"; 41 | 42 | // `Any` contains an arbitrary serialized protocol buffer message along with a 43 | // URL that describes the type of the serialized message. 44 | // 45 | // Protobuf library provides support to pack/unpack Any values in the form 46 | // of utility functions or additional generated methods of the Any type. 47 | // 48 | // Example 1: Pack and unpack a message in C++. 49 | // 50 | // Foo foo = ...; 51 | // Any any; 52 | // any.PackFrom(foo); 53 | // ... 54 | // if (any.UnpackTo(&foo)) { 55 | // ... 56 | // } 57 | // 58 | // Example 2: Pack and unpack a message in Java. 59 | // 60 | // Foo foo = ...; 61 | // Any any = Any.pack(foo); 62 | // ... 63 | // if (any.is(Foo.class)) { 64 | // foo = any.unpack(Foo.class); 65 | // } 66 | // 67 | // Example 3: Pack and unpack a message in Python. 68 | // 69 | // foo = Foo(...) 70 | // any = Any() 71 | // any.Pack(foo) 72 | // ... 73 | // if any.Is(Foo.DESCRIPTOR): 74 | // any.Unpack(foo) 75 | // ... 76 | // 77 | // Example 4: Pack and unpack a message in Go 78 | // 79 | // foo := &pb.Foo{...} 80 | // any, err := ptypes.MarshalAny(foo) 81 | // ... 82 | // foo := &pb.Foo{} 83 | // if err := ptypes.UnmarshalAny(any, foo); err != nil { 84 | // ... 85 | // } 86 | // 87 | // The pack methods provided by protobuf library will by default use 88 | // 'type.googleapis.com/full.type.name' as the type URL and the unpack 89 | // methods only use the fully qualified type name after the last '/' 90 | // in the type URL, for example "foo.bar.com/x/y.z" will yield type 91 | // name "y.z". 92 | // 93 | // 94 | // JSON 95 | // ==== 96 | // The JSON representation of an `Any` value uses the regular 97 | // representation of the deserialized, embedded message, with an 98 | // additional field `@type` which contains the type URL. Example: 99 | // 100 | // package google.profile; 101 | // message Person { 102 | // string first_name = 1; 103 | // string last_name = 2; 104 | // } 105 | // 106 | // { 107 | // "@type": "type.googleapis.com/google.profile.Person", 108 | // "firstName": , 109 | // "lastName": 110 | // } 111 | // 112 | // If the embedded message type is well-known and has a custom JSON 113 | // representation, that representation will be embedded adding a field 114 | // `value` which holds the custom JSON in addition to the `@type` 115 | // field. Example (for message [google.protobuf.Duration][]): 116 | // 117 | // { 118 | // "@type": "type.googleapis.com/google.protobuf.Duration", 119 | // "value": "1.212s" 120 | // } 121 | // 122 | message Any { 123 | // A URL/resource name whose content describes the type of the 124 | // serialized protocol buffer message. 125 | // 126 | // For URLs which use the scheme `http`, `https`, or no scheme, the 127 | // following restrictions and interpretations apply: 128 | // 129 | // * If no scheme is provided, `https` is assumed. 130 | // * The last segment of the URL's path must represent the fully 131 | // qualified name of the type (as in `path/google.protobuf.Duration`). 132 | // The name should be in a canonical form (e.g., leading "." is 133 | // not accepted). 134 | // * An HTTP GET on the URL must yield a [google.protobuf.Type][] 135 | // value in binary format, or produce an error. 136 | // * Applications are allowed to cache lookup results based on the 137 | // URL, or have them precompiled into a binary to avoid any 138 | // lookup. Therefore, binary compatibility needs to be preserved 139 | // on changes to types. (Use versioned type names to manage 140 | // breaking changes.) 141 | // 142 | // Schemes other than `http`, `https` (or the empty scheme) might be 143 | // used with implementation specific semantics. 144 | // 145 | string type_url = 1; 146 | 147 | // Must be a valid serialized protocol buffer of the above specified type. 148 | bytes value = 2; 149 | } 150 | -------------------------------------------------------------------------------- /core/proto/src/google/protobuf/duration.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option cc_enable_arenas = true; 37 | option go_package = "github.com/golang/protobuf/ptypes/duration"; 38 | option java_package = "com.google.protobuf"; 39 | option java_outer_classname = "DurationProto"; 40 | option java_multiple_files = true; 41 | option objc_class_prefix = "GPB"; 42 | 43 | // A Duration represents a signed, fixed-length span of time represented 44 | // as a count of seconds and fractions of seconds at nanosecond 45 | // resolution. It is independent of any calendar and concepts like "day" 46 | // or "month". It is related to Timestamp in that the difference between 47 | // two Timestamp values is a Duration and it can be added or subtracted 48 | // from a Timestamp. Range is approximately +-10,000 years. 49 | // 50 | // # Examples 51 | // 52 | // Example 1: Compute Duration from two Timestamps in pseudo code. 53 | // 54 | // Timestamp start = ...; 55 | // Timestamp end = ...; 56 | // Duration duration = ...; 57 | // 58 | // duration.seconds = end.seconds - start.seconds; 59 | // duration.nanos = end.nanos - start.nanos; 60 | // 61 | // if (duration.seconds < 0 && duration.nanos > 0) { 62 | // duration.seconds += 1; 63 | // duration.nanos -= 1000000000; 64 | // } else if (durations.seconds > 0 && duration.nanos < 0) { 65 | // duration.seconds -= 1; 66 | // duration.nanos += 1000000000; 67 | // } 68 | // 69 | // Example 2: Compute Timestamp from Timestamp + Duration in pseudo code. 70 | // 71 | // Timestamp start = ...; 72 | // Duration duration = ...; 73 | // Timestamp end = ...; 74 | // 75 | // end.seconds = start.seconds + duration.seconds; 76 | // end.nanos = start.nanos + duration.nanos; 77 | // 78 | // if (end.nanos < 0) { 79 | // end.seconds -= 1; 80 | // end.nanos += 1000000000; 81 | // } else if (end.nanos >= 1000000000) { 82 | // end.seconds += 1; 83 | // end.nanos -= 1000000000; 84 | // } 85 | // 86 | // Example 3: Compute Duration from datetime.timedelta in Python. 87 | // 88 | // td = datetime.timedelta(days=3, minutes=10) 89 | // duration = Duration() 90 | // duration.FromTimedelta(td) 91 | // 92 | // # JSON Mapping 93 | // 94 | // In JSON format, the Duration type is encoded as a string rather than an 95 | // object, where the string ends in the suffix "s" (indicating seconds) and 96 | // is preceded by the number of seconds, with nanoseconds expressed as 97 | // fractional seconds. For example, 3 seconds with 0 nanoseconds should be 98 | // encoded in JSON format as "3s", while 3 seconds and 1 nanosecond should 99 | // be expressed in JSON format as "3.000000001s", and 3 seconds and 1 100 | // microsecond should be expressed in JSON format as "3.000001s". 101 | // 102 | // 103 | message Duration { 104 | 105 | // Signed seconds of the span of time. Must be from -315,576,000,000 106 | // to +315,576,000,000 inclusive. Note: these bounds are computed from: 107 | // 60 sec/min * 60 min/hr * 24 hr/day * 365.25 days/year * 10000 years 108 | int64 seconds = 1; 109 | 110 | // Signed fractions of a second at nanosecond resolution of the span 111 | // of time. Durations less than one second are represented with a 0 112 | // `seconds` field and a positive or negative `nanos` field. For durations 113 | // of one second or more, a non-zero value for the `nanos` field must be 114 | // of the same sign as the `seconds` field. Must be from -999,999,999 115 | // to +999,999,999 inclusive. 116 | int32 nanos = 2; 117 | } 118 | -------------------------------------------------------------------------------- /core/proto/src/google/protobuf/empty.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option go_package = "github.com/golang/protobuf/ptypes/empty"; 37 | option java_package = "com.google.protobuf"; 38 | option java_outer_classname = "EmptyProto"; 39 | option java_multiple_files = true; 40 | option objc_class_prefix = "GPB"; 41 | option cc_enable_arenas = true; 42 | 43 | // A generic empty message that you can re-use to avoid defining duplicated 44 | // empty messages in your APIs. A typical example is to use it as the request 45 | // or the response type of an API method. For instance: 46 | // 47 | // service Foo { 48 | // rpc Bar(google.protobuf.Empty) returns (google.protobuf.Empty); 49 | // } 50 | // 51 | // The JSON representation for `Empty` is empty JSON object `{}`. 52 | message Empty {} 53 | -------------------------------------------------------------------------------- /core/proto/src/google/protobuf/source_context.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option java_package = "com.google.protobuf"; 37 | option java_outer_classname = "SourceContextProto"; 38 | option java_multiple_files = true; 39 | option objc_class_prefix = "GPB"; 40 | option go_package = "google.golang.org/genproto/protobuf/source_context;source_context"; 41 | 42 | // `SourceContext` represents information about the source of a 43 | // protobuf element, like the file in which it is defined. 44 | message SourceContext { 45 | // The path-qualified name of the .proto file that contained the associated 46 | // protobuf element. For example: `"google/protobuf/source_context.proto"`. 47 | string file_name = 1; 48 | } 49 | -------------------------------------------------------------------------------- /core/proto/src/google/protobuf/struct.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option cc_enable_arenas = true; 37 | option go_package = "github.com/golang/protobuf/ptypes/struct;structpb"; 38 | option java_package = "com.google.protobuf"; 39 | option java_outer_classname = "StructProto"; 40 | option java_multiple_files = true; 41 | option objc_class_prefix = "GPB"; 42 | 43 | 44 | // `Struct` represents a structured data value, consisting of fields 45 | // which map to dynamically typed values. In some languages, `Struct` 46 | // might be supported by a native representation. For example, in 47 | // scripting languages like JS a struct is represented as an 48 | // object. The details of that representation are described together 49 | // with the proto support for the language. 50 | // 51 | // The JSON representation for `Struct` is JSON object. 52 | message Struct { 53 | // Unordered map of dynamically typed values. 54 | map fields = 1; 55 | } 56 | 57 | // `Value` represents a dynamically typed value which can be either 58 | // null, a number, a string, a boolean, a recursive struct value, or a 59 | // list of values. A producer of value is expected to set one of that 60 | // variants, absence of any variant indicates an error. 61 | // 62 | // The JSON representation for `Value` is JSON value. 63 | message Value { 64 | // The kind of value. 65 | oneof kind { 66 | // Represents a null value. 67 | NullValue null_value = 1; 68 | // Represents a double value. 69 | double number_value = 2; 70 | // Represents a string value. 71 | string string_value = 3; 72 | // Represents a boolean value. 73 | bool bool_value = 4; 74 | // Represents a structured value. 75 | Struct struct_value = 5; 76 | // Represents a repeated `Value`. 77 | ListValue list_value = 6; 78 | } 79 | } 80 | 81 | // `NullValue` is a singleton enumeration to represent the null value for the 82 | // `Value` type union. 83 | // 84 | // The JSON representation for `NullValue` is JSON `null`. 85 | enum NullValue { 86 | // Null value. 87 | NULL_VALUE = 0; 88 | } 89 | 90 | // `ListValue` is a wrapper around a repeated field of values. 91 | // 92 | // The JSON representation for `ListValue` is JSON array. 93 | message ListValue { 94 | // Repeated field of dynamically typed values. 95 | repeated Value values = 1; 96 | } 97 | -------------------------------------------------------------------------------- /core/proto/src/google/protobuf/timestamp.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 36 | option cc_enable_arenas = true; 37 | option go_package = "github.com/golang/protobuf/ptypes/timestamp"; 38 | option java_package = "com.google.protobuf"; 39 | option java_outer_classname = "TimestampProto"; 40 | option java_multiple_files = true; 41 | option objc_class_prefix = "GPB"; 42 | 43 | // A Timestamp represents a point in time independent of any time zone 44 | // or calendar, represented as seconds and fractions of seconds at 45 | // nanosecond resolution in UTC Epoch time. It is encoded using the 46 | // Proleptic Gregorian Calendar which extends the Gregorian calendar 47 | // backwards to year one. It is encoded assuming all minutes are 60 48 | // seconds long, i.e. leap seconds are "smeared" so that no leap second 49 | // table is needed for interpretation. Range is from 50 | // 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. 51 | // By restricting to that range, we ensure that we can convert to 52 | // and from RFC 3339 date strings. 53 | // See [https://www.ietf.org/rfc/rfc3339.txt](https://www.ietf.org/rfc/rfc3339.txt). 54 | // 55 | // # Examples 56 | // 57 | // Example 1: Compute Timestamp from POSIX `time()`. 58 | // 59 | // Timestamp timestamp; 60 | // timestamp.set_seconds(time(NULL)); 61 | // timestamp.set_nanos(0); 62 | // 63 | // Example 2: Compute Timestamp from POSIX `gettimeofday()`. 64 | // 65 | // struct timeval tv; 66 | // gettimeofday(&tv, NULL); 67 | // 68 | // Timestamp timestamp; 69 | // timestamp.set_seconds(tv.tv_sec); 70 | // timestamp.set_nanos(tv.tv_usec * 1000); 71 | // 72 | // Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. 73 | // 74 | // FILETIME ft; 75 | // GetSystemTimeAsFileTime(&ft); 76 | // UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; 77 | // 78 | // // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z 79 | // // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. 80 | // Timestamp timestamp; 81 | // timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); 82 | // timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); 83 | // 84 | // Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. 85 | // 86 | // long millis = System.currentTimeMillis(); 87 | // 88 | // Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) 89 | // .setNanos((int) ((millis % 1000) * 1000000)).build(); 90 | // 91 | // 92 | // Example 5: Compute Timestamp from current time in Python. 93 | // 94 | // timestamp = Timestamp() 95 | // timestamp.GetCurrentTime() 96 | // 97 | // # JSON Mapping 98 | // 99 | // In JSON format, the Timestamp type is encoded as a string in the 100 | // [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the 101 | // format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" 102 | // where {year} is always expressed using four digits while {month}, {day}, 103 | // {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional 104 | // seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), 105 | // are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone 106 | // is required, though only UTC (as indicated by "Z") is presently supported. 107 | // 108 | // For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past 109 | // 01:30 UTC on January 15, 2017. 110 | // 111 | // In JavaScript, one can convert a Date object to this format using the 112 | // standard [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString] 113 | // method. In Python, a standard `datetime.datetime` object can be converted 114 | // to this format using [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) 115 | // with the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one 116 | // can use the Joda Time's [`ISODateTimeFormat.dateTime()`]( 117 | // http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime--) 118 | // to obtain a formatter capable of generating timestamps in this format. 119 | // 120 | // 121 | message Timestamp { 122 | 123 | // Represents seconds of UTC time since Unix epoch 124 | // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to 125 | // 9999-12-31T23:59:59Z inclusive. 126 | int64 seconds = 1; 127 | 128 | // Non-negative fractions of a second at nanosecond resolution. Negative 129 | // second values with fractions must still have non-negative nanos values 130 | // that count forward in time. Must be from 0 to 999,999,999 131 | // inclusive. 132 | int32 nanos = 2; 133 | } 134 | -------------------------------------------------------------------------------- /core/proto/src/google/protobuf/type.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | syntax = "proto3"; 32 | 33 | package google.protobuf; 34 | 35 | import "google/protobuf/any.proto"; 36 | import "google/protobuf/source_context.proto"; 37 | 38 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 39 | option cc_enable_arenas = true; 40 | option java_package = "com.google.protobuf"; 41 | option java_outer_classname = "TypeProto"; 42 | option java_multiple_files = true; 43 | option objc_class_prefix = "GPB"; 44 | option go_package = "google.golang.org/genproto/protobuf/ptype;ptype"; 45 | 46 | // A protocol buffer message type. 47 | message Type { 48 | // The fully qualified message name. 49 | string name = 1; 50 | // The list of fields. 51 | repeated Field fields = 2; 52 | // The list of types appearing in `oneof` definitions in this type. 53 | repeated string oneofs = 3; 54 | // The protocol buffer options. 55 | repeated Option options = 4; 56 | // The source context. 57 | SourceContext source_context = 5; 58 | // The source syntax. 59 | Syntax syntax = 6; 60 | } 61 | 62 | // A single field of a message type. 63 | message Field { 64 | // Basic field types. 65 | enum Kind { 66 | // Field type unknown. 67 | TYPE_UNKNOWN = 0; 68 | // Field type double. 69 | TYPE_DOUBLE = 1; 70 | // Field type float. 71 | TYPE_FLOAT = 2; 72 | // Field type int64. 73 | TYPE_INT64 = 3; 74 | // Field type uint64. 75 | TYPE_UINT64 = 4; 76 | // Field type int32. 77 | TYPE_INT32 = 5; 78 | // Field type fixed64. 79 | TYPE_FIXED64 = 6; 80 | // Field type fixed32. 81 | TYPE_FIXED32 = 7; 82 | // Field type bool. 83 | TYPE_BOOL = 8; 84 | // Field type string. 85 | TYPE_STRING = 9; 86 | // Field type group. Proto2 syntax only, and deprecated. 87 | TYPE_GROUP = 10; 88 | // Field type message. 89 | TYPE_MESSAGE = 11; 90 | // Field type bytes. 91 | TYPE_BYTES = 12; 92 | // Field type uint32. 93 | TYPE_UINT32 = 13; 94 | // Field type enum. 95 | TYPE_ENUM = 14; 96 | // Field type sfixed32. 97 | TYPE_SFIXED32 = 15; 98 | // Field type sfixed64. 99 | TYPE_SFIXED64 = 16; 100 | // Field type sint32. 101 | TYPE_SINT32 = 17; 102 | // Field type sint64. 103 | TYPE_SINT64 = 18; 104 | }; 105 | 106 | // Whether a field is optional, required, or repeated. 107 | enum Cardinality { 108 | // For fields with unknown cardinality. 109 | CARDINALITY_UNKNOWN = 0; 110 | // For optional fields. 111 | CARDINALITY_OPTIONAL = 1; 112 | // For required fields. Proto2 syntax only. 113 | CARDINALITY_REQUIRED = 2; 114 | // For repeated fields. 115 | CARDINALITY_REPEATED = 3; 116 | }; 117 | 118 | // The field type. 119 | Kind kind = 1; 120 | // The field cardinality. 121 | Cardinality cardinality = 2; 122 | // The field number. 123 | int32 number = 3; 124 | // The field name. 125 | string name = 4; 126 | // The field type URL, without the scheme, for message or enumeration 127 | // types. Example: `"type.googleapis.com/google.protobuf.Timestamp"`. 128 | string type_url = 6; 129 | // The index of the field type in `Type.oneofs`, for message or enumeration 130 | // types. The first type has index 1; zero means the type is not in the list. 131 | int32 oneof_index = 7; 132 | // Whether to use alternative packed wire representation. 133 | bool packed = 8; 134 | // The protocol buffer options. 135 | repeated Option options = 9; 136 | // The field JSON name. 137 | string json_name = 10; 138 | // The string value of the default value of this field. Proto2 syntax only. 139 | string default_value = 11; 140 | } 141 | 142 | // Enum type definition. 143 | message Enum { 144 | // Enum type name. 145 | string name = 1; 146 | // Enum value definitions. 147 | repeated EnumValue enumvalue = 2; 148 | // Protocol buffer options. 149 | repeated Option options = 3; 150 | // The source context. 151 | SourceContext source_context = 4; 152 | // The source syntax. 153 | Syntax syntax = 5; 154 | } 155 | 156 | // Enum value definition. 157 | message EnumValue { 158 | // Enum value name. 159 | string name = 1; 160 | // Enum value number. 161 | int32 number = 2; 162 | // Protocol buffer options. 163 | repeated Option options = 3; 164 | } 165 | 166 | // A protocol buffer option, which can be attached to a message, field, 167 | // enumeration, etc. 168 | message Option { 169 | // The option's name. For protobuf built-in options (options defined in 170 | // descriptor.proto), this is the short name. For example, `"map_entry"`. 171 | // For custom options, it should be the fully-qualified name. For example, 172 | // `"google.api.http"`. 173 | string name = 1; 174 | // The option's value packed in an Any message. If the value is a primitive, 175 | // the corresponding wrapper type defined in google/protobuf/wrappers.proto 176 | // should be used. If the value is an enum, it should be stored as an int32 177 | // value using the google.protobuf.Int32Value type. 178 | Any value = 2; 179 | } 180 | 181 | // The syntax in which a protocol buffer element is defined. 182 | enum Syntax { 183 | // Syntax `proto2`. 184 | SYNTAX_PROTO2 = 0; 185 | // Syntax `proto3`. 186 | SYNTAX_PROTO3 = 1; 187 | } 188 | -------------------------------------------------------------------------------- /core/proto/src/google/protobuf/wrappers.proto: -------------------------------------------------------------------------------- 1 | // Protocol Buffers - Google's data interchange format 2 | // Copyright 2008 Google Inc. All rights reserved. 3 | // https://developers.google.com/protocol-buffers/ 4 | // 5 | // Redistribution and use in source and binary forms, with or without 6 | // modification, are permitted provided that the following conditions are 7 | // met: 8 | // 9 | // * Redistributions of source code must retain the above copyright 10 | // notice, this list of conditions and the following disclaimer. 11 | // * Redistributions in binary form must reproduce the above 12 | // copyright notice, this list of conditions and the following disclaimer 13 | // in the documentation and/or other materials provided with the 14 | // distribution. 15 | // * Neither the name of Google Inc. nor the names of its 16 | // contributors may be used to endorse or promote products derived from 17 | // this software without specific prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | // Wrappers for primitive (non-message) types. These types are useful 32 | // for embedding primitives in the `google.protobuf.Any` type and for places 33 | // where we need to distinguish between the absence of a primitive 34 | // typed field and its default value. 35 | 36 | syntax = "proto3"; 37 | 38 | package google.protobuf; 39 | 40 | option csharp_namespace = "Google.Protobuf.WellKnownTypes"; 41 | option cc_enable_arenas = true; 42 | option go_package = "github.com/golang/protobuf/ptypes/wrappers"; 43 | option java_package = "com.google.protobuf"; 44 | option java_outer_classname = "WrappersProto"; 45 | option java_multiple_files = true; 46 | option objc_class_prefix = "GPB"; 47 | 48 | // Wrapper message for `double`. 49 | // 50 | // The JSON representation for `DoubleValue` is JSON number. 51 | message DoubleValue { 52 | // The double value. 53 | double value = 1; 54 | } 55 | 56 | // Wrapper message for `float`. 57 | // 58 | // The JSON representation for `FloatValue` is JSON number. 59 | message FloatValue { 60 | // The float value. 61 | float value = 1; 62 | } 63 | 64 | // Wrapper message for `int64`. 65 | // 66 | // The JSON representation for `Int64Value` is JSON string. 67 | message Int64Value { 68 | // The int64 value. 69 | int64 value = 1; 70 | } 71 | 72 | // Wrapper message for `uint64`. 73 | // 74 | // The JSON representation for `UInt64Value` is JSON string. 75 | message UInt64Value { 76 | // The uint64 value. 77 | uint64 value = 1; 78 | } 79 | 80 | // Wrapper message for `int32`. 81 | // 82 | // The JSON representation for `Int32Value` is JSON number. 83 | message Int32Value { 84 | // The int32 value. 85 | int32 value = 1; 86 | } 87 | 88 | // Wrapper message for `uint32`. 89 | // 90 | // The JSON representation for `UInt32Value` is JSON number. 91 | message UInt32Value { 92 | // The uint32 value. 93 | uint32 value = 1; 94 | } 95 | 96 | // Wrapper message for `bool`. 97 | // 98 | // The JSON representation for `BoolValue` is JSON `true` and `false`. 99 | message BoolValue { 100 | // The bool value. 101 | bool value = 1; 102 | } 103 | 104 | // Wrapper message for `string`. 105 | // 106 | // The JSON representation for `StringValue` is JSON string. 107 | message StringValue { 108 | // The string value. 109 | string value = 1; 110 | } 111 | 112 | // Wrapper message for `bytes`. 113 | // 114 | // The JSON representation for `BytesValue` is JSON string. 115 | message BytesValue { 116 | // The bytes value. 117 | bytes value = 1; 118 | } 119 | -------------------------------------------------------------------------------- /core/tests/api_client/api_client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "reflect" 7 | "time" 8 | 9 | ptypes "github.com/gogo/protobuf/types" 10 | pbd "github.com/kraken-hpc/kraken-legacy/modules/dummy" 11 | "github.com/kraken-hpc/kraken/core" 12 | cpb "github.com/kraken-hpc/kraken/core/proto" 13 | ) 14 | 15 | func main() { 16 | 17 | api := core.NewModuleAPIClient(os.Args[1]) 18 | 19 | ns, e := api.QueryReadAll() 20 | if e != nil { 21 | fmt.Println(e) 22 | } 23 | for _, n := range ns { 24 | fmt.Printf("%v\n", string(n.JSON())) 25 | } 26 | 27 | fmt.Println("changing dummy0 config") 28 | dconf := &pbd.Config{ 29 | Say: []string{"one", "more", "test"}, 30 | Wait: "1s", 31 | } 32 | any, _ := ptypes.MarshalAny(dconf) 33 | 34 | ns[0].SetValue("/Services/dummy0/Config", reflect.ValueOf(any)) 35 | api.QueryUpdate(ns[0]) 36 | 37 | fmt.Println("waiting 10s") 38 | time.Sleep(10 * time.Second) 39 | 40 | self := ns[0].ID().String() 41 | 42 | fmt.Println("stopping restapi") 43 | me, _ := api.QueryRead(self) 44 | me.SetValue("/Services/restapi/State", reflect.ValueOf(cpb.ServiceInstance_STOP)) 45 | api.QueryUpdate(me) 46 | 47 | fmt.Println("waiting 10s") 48 | time.Sleep(10 * time.Second) 49 | 50 | fmt.Println("starting restapi") 51 | me.SetValue("/Services/restapi/State", reflect.ValueOf(cpb.ServiceInstance_RUN)) 52 | api.QueryUpdate(me) 53 | } 54 | -------------------------------------------------------------------------------- /core/tests/events_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | 6 | . "github.com/kraken-hpc/kraken/core" 7 | "github.com/kraken-hpc/kraken/lib/types" 8 | ) 9 | 10 | // TestEventDispatchEngine tests that NewEventDispatchEngine initializes correctly 11 | /* FIXME: broken test 12 | func TestEventDispatchEngine(t *testing.T) { 13 | ed := NewEventDispatchEngine() 14 | if ed == nil { 15 | t.Fatal("failed to create EventDispatchEngine") 16 | } 17 | t.Run("valid echan", func(t *testing.T) { 18 | echan := ed.EventChan() 19 | if echan == nil { 20 | t.Fatal("echan was not succesfully created") 21 | } 22 | }) 23 | t.Run("valid schan", func(t *testing.T) { 24 | schan := ed.SubscriptionChan() 25 | if schan == nil { 26 | t.Fatal("schan was not succesfully created") 27 | } 28 | }) 29 | 30 | } 31 | */ 32 | 33 | // ExampleFilterSimple shows how to use a filter generator 34 | func ExampleFilterSimple() { 35 | list := []string{ 36 | "/this/is", 37 | "/a/test", 38 | } 39 | el := NewEventListener("example", 40 | types.Event_STATE_CHANGE, 41 | func(ev types.Event) bool { 42 | return FilterSimple(ev, list) 43 | }, 44 | nil) 45 | 46 | ev1 := NewEvent( 47 | types.Event_STATE_CHANGE, 48 | "/this/is", 49 | nil) 50 | ev2 := NewEvent( 51 | types.Event_STATE_CHANGE, 52 | "/blatz", 53 | nil) 54 | fmt.Printf("%s -> %v\n", ev1.URL(), el.Filter(ev1)) 55 | fmt.Printf("%s -> %v\n", ev2.URL(), el.Filter(ev2)) 56 | // Output: 57 | // /this/is -> true 58 | // /blatz -> false 59 | } 60 | -------------------------------------------------------------------------------- /core/tests/sse_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | _ "github.com/kraken-hpc/kraken/core" 5 | ) 6 | 7 | /* FIXME: broken test 8 | func TestGenerateKey(t *testing.T) { 9 | ctx := Context{} 10 | sse := NewStateSyncEngine(ctx) 11 | key1 := sse.generateKey() 12 | key2 := sse.generateKey() 13 | fmt.Println(hex.Dump(key1)) 14 | fmt.Println(hex.Dump(key2)) 15 | if bytes.Equal(key1, key2) { 16 | t.Errorf("generated identical keys!") 17 | } 18 | } 19 | */ 20 | -------------------------------------------------------------------------------- /core/tests/sync_client/sync_client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "time" 7 | 8 | "github.com/kraken-hpc/kraken/core" 9 | pb "github.com/kraken-hpc/kraken/core/proto" 10 | "github.com/kraken-hpc/kraken/lib/types" 11 | 12 | "google.golang.org/grpc" 13 | ) 14 | 15 | // should work 16 | const simpleNode1 string = ` 17 | "id": "Ej5FZ+ibEtOkVkJmVUQAAA==", 18 | "nodename": "", 19 | "runState": "INIT", 20 | "physState": "PHYS_UNKNOWN", 21 | "extensions": [ 22 | { 23 | "@type": "type.googleapis.com/IPv4.IPv4OverEthernet", 24 | "ifaces": [ 25 | { 26 | "ip": { 27 | "ip": "CgkIBw==" 28 | } 29 | } 30 | ] 31 | } 32 | ] 33 | } 34 | ` 35 | 36 | // should fail on lack of ip extension 37 | const simpleNode2 string = ` 38 | "id": "Ej5FZ+ibEtOkVkJmVUQAAB==", 39 | "nodename": "", 40 | "runState": "INIT", 41 | "physState": "PHYS_UNKNOWN", 42 | "extensions": [ 43 | ] 44 | } 45 | ] 46 | } 47 | ` 48 | 49 | // should fail because not in INIT 50 | const simpleNode3 string = ` 51 | "id": "Ej5FZ+ibEtOkVkJmVUQAAA==", 52 | "nodename": "", 53 | "runState": "SYNC", 54 | "physState": "PHYS_UNKNOWN", 55 | "extensions": [ 56 | { 57 | "@type": "type.googleapis.com/IPv4.IPv4OverEthernet", 58 | "ifaces": [ 59 | { 60 | "ip": { 61 | "ip": "CgkIBw==" 62 | } 63 | } 64 | ] 65 | } 66 | ] 67 | } 68 | ` 69 | 70 | func send(n types.Node) { 71 | conn, e := grpc.Dial("127.0.0.1:31415", grpc.WithInsecure()) 72 | if e != nil { 73 | log.Printf("did not connect: %v", e) 74 | } 75 | defer conn.Close() 76 | c := pb.NewStateSyncClient(conn) 77 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 78 | defer cancel() 79 | r, e := c.RPCPhoneHome(ctx, &pb.PhoneHomeRequest{Id: n.ID().Bytes()}) 80 | if e != nil { 81 | log.Printf("could not phone home: %v", e) 82 | } 83 | log.Printf("%v\n", r) 84 | } 85 | 86 | func main() { 87 | n1 := core.NewNodeFromJSON([]byte(simpleNode1)) 88 | n2 := core.NewNodeFromJSON([]byte(simpleNode2)) 89 | n3 := core.NewNodeFromJSON([]byte(simpleNode3)) 90 | 91 | send(n1) 92 | time.Sleep(time.Second * 3) 93 | send(n2) 94 | time.Sleep(time.Second * 3) 95 | send(n3) 96 | } 97 | -------------------------------------------------------------------------------- /extensions/ipv4/customtypes/IP_type.go: -------------------------------------------------------------------------------- 1 | /* IP_type.go: define methods to make a CustomType that stores IP addresses 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package customtypes 11 | 12 | import ( 13 | fmt "fmt" 14 | "net" 15 | "strconv" 16 | "strings" 17 | 18 | "github.com/kraken-hpc/kraken/lib/types" 19 | ) 20 | 21 | var _ (types.ExtensionCustomType) = IP{} 22 | var _ (types.ExtensionCustomTypePtr) = (*IP)(nil) 23 | 24 | type IP struct { 25 | net.IP 26 | } 27 | 28 | func (i IP) Marshal() ([]byte, error) { 29 | return []byte(i.To4()), nil 30 | } 31 | 32 | func (i *IP) MarshalTo(data []byte) (int, error) { 33 | copy(data, []byte(i.To4())) 34 | return 4, nil 35 | } 36 | 37 | func (i *IP) Unmarshal(data []byte) error { 38 | if len(data) != 4 { 39 | return fmt.Errorf("incorrect IP address lenght: %d != 4", len(data)) 40 | } 41 | i.IP = net.IPv4(data[0], data[1], data[2], data[3]) 42 | return nil 43 | } 44 | 45 | func (i *IP) Size() int { 46 | return 4 47 | } 48 | 49 | func (i IP) MarshalJSON() (j []byte, e error) { 50 | return []byte(strconv.Quote(i.String())), nil 51 | } 52 | 53 | func (i *IP) UnmarshalJSON(data []byte) error { 54 | s := strings.Trim(string(data), "\"'") 55 | return i.UnmarshalText([]byte(s)) 56 | } 57 | -------------------------------------------------------------------------------- /extensions/ipv4/customtypes/MAC_type.go: -------------------------------------------------------------------------------- 1 | /* MAC_type.go: define methods to make a CustomType that stores Hardware addresses 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package customtypes 11 | 12 | import ( 13 | fmt "fmt" 14 | "net" 15 | "strconv" 16 | "strings" 17 | 18 | "github.com/kraken-hpc/kraken/lib/types" 19 | ) 20 | 21 | var _ (types.ExtensionCustomType) = IP{} 22 | var _ (types.ExtensionCustomTypePtr) = (*IP)(nil) 23 | 24 | type MAC struct { 25 | net.HardwareAddr 26 | } 27 | 28 | func (m MAC) Marshal() ([]byte, error) { 29 | return []byte(m.HardwareAddr), nil 30 | } 31 | 32 | func (m *MAC) MarshalTo(data []byte) (int, error) { 33 | copy(data, []byte(m.HardwareAddr)) 34 | return 4, nil 35 | } 36 | 37 | func (m *MAC) Unmarshal(data []byte) error { 38 | if len(data) > 20 { // IPoIB is longest we allow 39 | return fmt.Errorf("incorrect hardware address lenght: %d > 20", len(data)) 40 | } 41 | m.HardwareAddr = net.HardwareAddr(data) 42 | return nil 43 | } 44 | 45 | func (m *MAC) Size() int { return len(m.HardwareAddr) } 46 | 47 | func (m MAC) MarshalJSON() ([]byte, error) { 48 | return []byte(strconv.Quote(m.String())), nil 49 | } 50 | 51 | func (m *MAC) UnmarshalJSON(data []byte) (e error) { 52 | s := strings.Trim(string(data), "\"'") 53 | m.HardwareAddr, e = net.ParseMAC(s) 54 | return 55 | } 56 | -------------------------------------------------------------------------------- /extensions/ipv4/ipv4.go: -------------------------------------------------------------------------------- 1 | /* IPv4.go: this extension adds standard IPv4 and Ethernet properties to the Node state 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | //go:generate protoc -I ../../core/proto/src -I . --gogo_out=plugin=grpc:. ipv4.proto 11 | 12 | package ipv4 13 | 14 | import ( 15 | "github.com/kraken-hpc/kraken/core" 16 | "github.com/kraken-hpc/kraken/lib/types" 17 | ) 18 | 19 | const Name = "type.googleapis.com/IPv4.IPv4OverEthernet" 20 | 21 | ///////////////////////////// 22 | // IPv4OverEthernet Object / 23 | /////////////////////////// 24 | 25 | var _ types.Extension = (*IPv4OverEthernet)(nil) 26 | 27 | func (i *IPv4OverEthernet) New() types.Message { 28 | return &IPv4OverEthernet{ 29 | /* 30 | Ifaces: []*pb.IPv4OverEthernet_ConfiguredInterface{ 31 | &pb.IPv4OverEthernet_ConfiguredInterface{ 32 | Eth: &pb.Ethernet{}, 33 | Ip: &pb.IPv4{}, 34 | }, 35 | &pb.IPv4OverEthernet_ConfiguredInterface{ 36 | Eth: &pb.Ethernet{}, 37 | Ip: &pb.IPv4{}, 38 | }, 39 | &pb.IPv4OverEthernet_ConfiguredInterface{ 40 | Eth: &pb.Ethernet{}, 41 | Ip: &pb.IPv4{}, 42 | }, 43 | &pb.IPv4OverEthernet_ConfiguredInterface{ 44 | Eth: &pb.Ethernet{}, 45 | Ip: &pb.IPv4{}, 46 | }, 47 | }, 48 | Routes: []*pb.IPv4{ 49 | &pb.IPv4{}, 50 | &pb.IPv4{}, 51 | &pb.IPv4{}, 52 | &pb.IPv4{}, 53 | }, 54 | DnsNameservers: []*pb.IPv4{ 55 | &pb.IPv4{}, 56 | &pb.IPv4{}, 57 | &pb.IPv4{}, 58 | &pb.IPv4{}, 59 | }, 60 | Hostname: &pb.DNSA{Ip: &pb.IPv4{}}, 61 | */ 62 | } 63 | } 64 | 65 | func (*IPv4OverEthernet) Name() string { 66 | return Name 67 | } 68 | 69 | func init() { 70 | core.Registry.RegisterExtension(&IPv4OverEthernet{}) 71 | } 72 | -------------------------------------------------------------------------------- /extensions/ipv4/ipv4.proto: -------------------------------------------------------------------------------- 1 | /* IPv4.proto: describes IPv4 specific state structures 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | syntax = "proto3"; 11 | package IPv4; 12 | option go_package = ".;ipv4"; 13 | 14 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 15 | option (gogoproto.marshaler_all) = true; 16 | option (gogoproto.unmarshaler_all) = true; 17 | option (gogoproto.sizer_all) = true; 18 | option (gogoproto.goproto_registration) = true; 19 | option (gogoproto.messagename_all) = true; 20 | 21 | 22 | message IPv4 { 23 | bytes ip = 2 [(gogoproto.customtype) = "github.com/kraken-hpc/kraken/extensions/ipv4/customtypes.IP"]; 24 | bytes subnet = 3 [(gogoproto.customtype) = "github.com/kraken-hpc/kraken/extensions/ipv4/customtypes.IP"]; 25 | } 26 | 27 | message Ethernet { 28 | string iface = 1; 29 | bytes mac = 2 [(gogoproto.customtype) = "github.com/kraken-hpc/kraken/extensions/ipv4/customtypes.MAC"]; /* mac address of the interface */ 30 | uint32 mtu = 3; 31 | bool control = 4; /* should we assume control (if we can)? */ 32 | } 33 | 34 | message IPv4OverEthernet { /* Extension */ 35 | message ConfiguredInterface { 36 | Ethernet eth = 1; 37 | IPv4 ip = 2; 38 | } 39 | map ifaces = 1; 40 | repeated IPv4 routes = 2; 41 | repeated IPv4 dns_nameservers = 3; 42 | repeated IPv4 dns_domains = 4; 43 | DNSA hostname = 5; 44 | } 45 | 46 | message DNSA { /* implies a ptr record */ 47 | string hostname = 1; 48 | string domainname = 2; 49 | IPv4 ip = 3; 50 | } 51 | 52 | message DNSCNAME { 53 | string cname = 1; 54 | string hostname = 2; 55 | } -------------------------------------------------------------------------------- /generators/app.go: -------------------------------------------------------------------------------- 1 | /* app.go: generators for making kraken entry points 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package generators 11 | 12 | import ( 13 | "flag" 14 | "fmt" 15 | "io/ioutil" 16 | "os" 17 | "path/filepath" 18 | "strings" 19 | "text/template" 20 | 21 | "github.com/kraken-hpc/kraken/generators/templates" 22 | "gopkg.in/yaml.v2" 23 | ) 24 | 25 | // App generation 26 | type AppConfig struct { 27 | Global *GlobalConfigType 28 | Name string // Application name 29 | Version string // Application version (not Kraken version) 30 | Pprof bool // Build with pprof (default: false)? 31 | Extensions []string // List of extensions to include (url paths) 32 | Modules []string // List of modules to include (url paths) 33 | } 34 | 35 | func appCompileTemplate(tplFile, tmpDir string, cfg *AppConfig) (target string, e error) { 36 | var tpl *template.Template 37 | var out *os.File 38 | parts := strings.Split(filepath.Base(tplFile), ".") 39 | target = strings.Join(parts[:len(parts)-1], ".") 40 | dest := filepath.Join(tmpDir, target) 41 | if _, err := os.Stat(dest); err == nil { 42 | if !Global.Force { 43 | return "", fmt.Errorf("refusing to overwrite file: %s (force not specified)", dest) 44 | } 45 | } 46 | tpl = template.New(tplFile) 47 | if tpl, e = tpl.Parse(string(templates.MustAsset(tplFile))); e != nil { 48 | return 49 | } 50 | if out, e = os.Create(dest); e != nil { 51 | return 52 | } 53 | defer out.Close() 54 | e = tpl.Execute(out, cfg) 55 | return 56 | } 57 | 58 | func AppGenerate(global *GlobalConfigType, args []string) { 59 | Global = global 60 | Log = global.Log 61 | var configFile string 62 | var outDir string 63 | var help bool 64 | fs := flag.NewFlagSet("app generate", flag.ExitOnError) 65 | fs.StringVar(&configFile, "c", "kraken.yaml", "name of app config file to use") 66 | fs.StringVar(&outDir, "o", ".", "output directory for app") 67 | fs.BoolVar(&help, "h", false, "print this usage") 68 | fs.Usage = func() { 69 | fmt.Println("[app]lication generate will generate a kraken entry point based on an app config.") 70 | fmt.Println("Usage: kraken app generate [-h] [-c ] [-o ]") 71 | fs.PrintDefaults() 72 | } 73 | fs.Parse(args) 74 | if help { 75 | fs.Usage() 76 | os.Exit(0) 77 | } 78 | if len(fs.Args()) != 0 { 79 | Log.Fatalf("unknown option: %s", fs.Args()[0]) 80 | } 81 | 82 | ts, err := templates.AssetDir("app") 83 | if err != nil || len(ts) < 2 { 84 | Log.Fatalf("no assets found for app generation: %v", err) 85 | } 86 | 87 | stat, err := os.Stat(outDir) 88 | if err == nil && !stat.IsDir() { 89 | Log.Fatalf("output directory %s exists, but is not a directory", outDir) 90 | } 91 | if err != nil { 92 | // create the dir 93 | if err = os.MkdirAll(outDir, 0777); err != nil { 94 | Log.Fatalf("failed to create output directory %s: %v", outDir, err) 95 | } 96 | } 97 | cfgData, err := ioutil.ReadFile(configFile) 98 | if err != nil { 99 | Log.Fatalf("could not read config file %s: %v", configFile, err) 100 | } 101 | cfg := &AppConfig{} 102 | if err = yaml.Unmarshal(cfgData, cfg); err != nil { 103 | Log.Fatalf("failed to parse config file %s: %v", configFile, err) 104 | } 105 | cfg.Global = Global 106 | if cfg.Name == "" { 107 | Log.Fatal("an application name must be specified in the app config") 108 | } 109 | if cfg.Version == "" { 110 | Log.Info("app version was not specified, will default to v0.0.0") 111 | } 112 | for _, t := range ts { 113 | written, err := appCompileTemplate("app/"+t, outDir, cfg) 114 | if err != nil { 115 | Log.Fatalf("failed to write template: %v", err) 116 | } 117 | Log.Debugf("wrote file: %s", written) 118 | } 119 | Log.Infof("app \"%s\" generated at %s", cfg.Name, outDir) 120 | } 121 | -------------------------------------------------------------------------------- /generators/extension.go: -------------------------------------------------------------------------------- 1 | /* extension.go: generators for making kraken extensions 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package generators 11 | 12 | import ( 13 | "flag" 14 | "fmt" 15 | "io/ioutil" 16 | "os" 17 | "path/filepath" 18 | "strings" 19 | "text/template" 20 | 21 | "github.com/kraken-hpc/kraken/generators/templates" 22 | "github.com/kraken-hpc/kraken/lib/util" 23 | "gopkg.in/yaml.v2" 24 | ) 25 | 26 | type ExtensionConfig struct { 27 | // Global should not be specified on config; it will get overwritten regardless. 28 | Global *GlobalConfigType `yaml:""` 29 | // URL of package, e.g. "github.com/kraken-hpc/kraken/extensions/ipv4" (required) 30 | PackageUrl string `yaml:"package_url"` 31 | // Go package name (default: last element of PackageUrl) 32 | PackageName string `yaml:"package_name"` 33 | // Proto package name (default: camel case of PackageName) 34 | ProtoPackage string `yaml:"proto_name"` 35 | // Extension object name (required) 36 | // More than one extension object can be declared in the same package 37 | // Objects are referenced as ProtoName.Name, e.g. IPv4.IPv4OverEthernet 38 | Name string `yaml:"name"` 39 | // CustomTypes are an advanced feature and most people won't use them 40 | // Declaring a custom type will create a stub to develop a gogo customtype on 41 | // It will also include a commented example of linking a customtype in the proto 42 | CustomTypes []string `yaml:"custom_types"` 43 | // LowerName is intended for internal use only 44 | LowerName string `yaml:""` 45 | } 46 | 47 | type CustomTypeConfig struct { 48 | Global *GlobalConfigType `yaml:"__global,omitempty"` 49 | Name string 50 | } 51 | 52 | func extensionReadConfig(file string) (cfg *ExtensionConfig, err error) { 53 | cfgData, err := ioutil.ReadFile(file) 54 | if err != nil { 55 | return nil, fmt.Errorf("could not read config file %s: %v", file, err) 56 | } 57 | cfg = &ExtensionConfig{} 58 | if err = yaml.Unmarshal(cfgData, cfg); err != nil { 59 | return nil, fmt.Errorf("failed to parse config file %s: %v", file, err) 60 | } 61 | // now, apply sanity checks & defaults 62 | if cfg.PackageUrl == "" { 63 | return nil, fmt.Errorf("package_url must be specified") 64 | } 65 | if cfg.Name == "" { 66 | return nil, fmt.Errorf("name must be specified") 67 | } 68 | urlParts := util.URLToSlice(cfg.PackageUrl) 69 | name := urlParts[len(urlParts)-1] 70 | if cfg.PackageName == "" { 71 | cfg.PackageName = name 72 | } 73 | if cfg.ProtoPackage == "" { 74 | parts := strings.Split(cfg.PackageName, "_") // in the off chance _ is used 75 | cname := "" 76 | for _, s := range parts { 77 | cname += strings.Title(s) 78 | } 79 | cfg.ProtoPackage = cname 80 | } 81 | cfg.LowerName = strings.ToLower(cfg.Name) 82 | return 83 | } 84 | 85 | func extensionCompileTemplate(tplFile, outDir string, cfg *ExtensionConfig) (target string, err error) { 86 | var tpl *template.Template 87 | var out *os.File 88 | parts := strings.Split(filepath.Base(tplFile), ".") 89 | if parts[0] == "template" { 90 | parts = append([]string{cfg.LowerName}, parts[1:len(parts)-1]...) 91 | } else { 92 | parts = parts[:len(parts)-1] 93 | } 94 | target = strings.Join(parts, ".") 95 | dest := filepath.Join(outDir, target) 96 | if _, err := os.Stat(dest); err == nil { 97 | if !Global.Force { 98 | return "", fmt.Errorf("refusing to overwrite file: %s (force not specified)", dest) 99 | } 100 | } 101 | tpl = template.New(tplFile) 102 | data, err := templates.Asset(tplFile) 103 | if err != nil { 104 | return 105 | } 106 | if tpl, err = tpl.Parse(string(data)); err != nil { 107 | return 108 | } 109 | if out, err = os.Create(dest); err != nil { 110 | return 111 | } 112 | defer out.Close() 113 | err = tpl.Execute(out, cfg) 114 | return 115 | } 116 | 117 | func extensionCompileCustomtype(outDir string, cfg *CustomTypeConfig) (target string, err error) { 118 | var tpl *template.Template 119 | var out *os.File 120 | target = cfg.Name + ".type.go" 121 | dest := filepath.Join(outDir, target) 122 | if _, err := os.Stat(dest); err == nil { 123 | if !Global.Force { 124 | return "", fmt.Errorf("refusing to overwrite file: %s (force not specified)", dest) 125 | } 126 | } 127 | tpl = template.New("extension/customtype.type.go.tpl") 128 | data, err := templates.Asset("extension/customtype.type.go.tpl") 129 | if err != nil { 130 | return 131 | } 132 | if tpl, err = tpl.Parse(string(data)); err != nil { 133 | return 134 | } 135 | if out, err = os.Create(dest); err != nil { 136 | return 137 | } 138 | defer out.Close() 139 | err = tpl.Execute(out, cfg) 140 | return 141 | } 142 | 143 | func ExtensionGenerate(global *GlobalConfigType, args []string) { 144 | Global = global 145 | Log = global.Log 146 | var configFile string 147 | var outDir string 148 | var help bool 149 | fs := flag.NewFlagSet("extension generate", flag.ExitOnError) 150 | fs.StringVar(&configFile, "c", "extension.yaml", "name of extension config file to use") 151 | fs.StringVar(&outDir, "o", ".", "output directory for extension") 152 | fs.BoolVar(&help, "h", false, "print this usage") 153 | fs.Usage = func() { 154 | fmt.Println("extension [gen]erate will generate a kraken extension based on an extension config.") 155 | fmt.Println("Usage: kraken extension generate [-h] [-c ] [-o ]") 156 | fs.PrintDefaults() 157 | } 158 | fs.Parse(args) 159 | if help { 160 | fs.Usage() 161 | os.Exit(0) 162 | } 163 | if len(fs.Args()) != 0 { 164 | Log.Fatalf("unknown option: %s", fs.Args()[0]) 165 | } 166 | stat, err := os.Stat(outDir) 167 | if err == nil && !stat.IsDir() { 168 | Log.Fatalf("output directory %s exists, but is not a directory", outDir) 169 | } 170 | if err != nil { 171 | // create the dir 172 | if err = os.MkdirAll(outDir, 0777); err != nil { 173 | Log.Fatalf("failed to create output directory %s: %v", outDir, err) 174 | } 175 | } 176 | cfg, err := extensionReadConfig(configFile) 177 | if err != nil { 178 | Log.Fatalf("failed to read config file: %v", err) 179 | } 180 | Log.Debugf("generating %s.%s with %d custom types", 181 | cfg.ProtoPackage, 182 | cfg.Name, 183 | len(cfg.CustomTypes)) 184 | cfg.Global = global 185 | // Ok, that's all the prep, now fill/write the templates 186 | common := []string{ 187 | "extension/template.proto.tpl", 188 | "extension/template.ext.go.tpl", 189 | "extension/template.go.tpl", 190 | } 191 | for _, f := range common { 192 | written, err := extensionCompileTemplate(f, outDir, cfg) 193 | if err != nil { 194 | Log.Fatalf("failed to write template: %v", err) 195 | } 196 | Log.Debugf("wrote file: %s", written) 197 | } 198 | if len(cfg.CustomTypes) > 0 { 199 | // generate customtypes 200 | ctypeDir := filepath.Join(outDir, "customtypes") 201 | stat, err := os.Stat(ctypeDir) 202 | if err == nil && !stat.IsDir() { 203 | Log.Fatalf("customtypes directory %s exists, but is not a directory", outDir) 204 | } 205 | if err != nil { 206 | // create the dir 207 | if err = os.MkdirAll(ctypeDir, 0777); err != nil { 208 | Log.Fatalf("failed to create customtypes directory %s: %v", outDir, err) 209 | } 210 | } 211 | for _, name := range cfg.CustomTypes { 212 | ctCfg := &CustomTypeConfig{ 213 | Global: global, 214 | Name: name, 215 | } 216 | written, err := extensionCompileCustomtype(ctypeDir, ctCfg) 217 | if err != nil { 218 | Log.Fatalf("failed to write template: %v", err) 219 | } 220 | Log.Debugf("wrote file: %s", written) 221 | } 222 | } 223 | Log.Infof("extension \"%s.%s\" generated at %s", cfg.ProtoPackage, cfg.Name, outDir) 224 | } 225 | -------------------------------------------------------------------------------- /generators/global.go: -------------------------------------------------------------------------------- 1 | /* global.go: global config for generators 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | //go:generate go-bindata -o templates/templates.go -fs -pkg templates -prefix templates templates/app templates/module templates/extension 11 | 12 | package generators 13 | 14 | import ( 15 | log "github.com/sirupsen/logrus" 16 | ) 17 | 18 | type GlobalConfigType struct { 19 | Version string 20 | Log *log.Logger 21 | Force bool 22 | } 23 | 24 | var Global *GlobalConfigType 25 | var Log *log.Logger 26 | -------------------------------------------------------------------------------- /generators/templates/app/includes.go.tpl: -------------------------------------------------------------------------------- 1 | // Code generated by kraken {{ .Global.Version }}. DO NOT EDIT 2 | package main 3 | 4 | // Extensions 5 | {{ range .Extensions }} 6 | import _ "{{ . }}" 7 | {{ end }} 8 | 9 | // Modules 10 | {{ range .Modules }} 11 | import _ "{{ . }}" 12 | {{ end }} 13 | -------------------------------------------------------------------------------- /generators/templates/app/pprof.go.tpl: -------------------------------------------------------------------------------- 1 | // Code generated by kraken {{ .Global.Version }}. DO NOT EDIT 2 | 3 | package main 4 | 5 | //{{if .Pprof}} 6 | import ( 7 | "log" 8 | "net" 9 | "net/http" 10 | _ "net/http/pprof" 11 | ) 12 | 13 | func init() { 14 | go func() { 15 | listener, e := net.Listen("tcp", ":0") 16 | if e != nil { 17 | log.Printf("failed to start pprof: %v", e) 18 | return 19 | } 20 | log.Printf("pprof listening on port: %s:%d", listener.Addr().(*net.TCPAddr).IP.String(), listener.Addr().(*net.TCPAddr).Port) 21 | log.Println(http.Serve(listener, nil)) 22 | }() 23 | } 24 | 25 | //{{end}} 26 | -------------------------------------------------------------------------------- /generators/templates/extension/customtype.type.go.tpl: -------------------------------------------------------------------------------- 1 | // One-time generated by kraken {{ .Global.Version }}. 2 | // You probably want to edit this file to add values you need. 3 | // `kraken extension update` *will not* replace this file. 4 | // `kraken -f extension generate` *will* replace this file. 5 | 6 | // The generated template is supposed to act more as an example than something useable out-of-the-box 7 | // You'll probably need to update most of these methods to make something functional. 8 | 9 | package customtypes 10 | 11 | import ( 12 | "strconv" 13 | "strings" 14 | 15 | "github.com/kraken-hpc/kraken/lib/types" 16 | ) 17 | 18 | var _ (types.ExtensionCustomType) = {{ .Name }}{} 19 | var _ (types.ExtensionCustomTypePtr) = (*{{ .Name }})(nil) 20 | 21 | type {{ .Name }} struct { 22 | // TODO: you might want this type to be based on some other kind of data 23 | // if you change this, all of the methods will need to change appropriately 24 | s string 25 | } 26 | 27 | // Marshal controls how the type is converted to binary 28 | func (t {{ .Name }}) Marshal() ([]byte, error) { 29 | return []byte(t.s), nil 30 | } 31 | 32 | // MarshalTo is like Marshal, but writes the result to `data` 33 | func (t *{{ .Name }}) MarshalTo(data []byte) (int, error) { 34 | copy(data, []byte(t.s)) 35 | return len(t.s), nil 36 | } 37 | 38 | // Unmarshal converts an encoded []byte to populate the type 39 | func (t *{{ .Name }}) Unmarshal(data []byte) error { 40 | t.s = string(data) 41 | return nil 42 | } 43 | 44 | // Size returns the size (encoded) of the type 45 | func (t *{{ .Name }}) Size() int { 46 | return len(t.s) 47 | } 48 | 49 | // MarshalJSON writes the type to JSON format in the form of a byte string 50 | func (t {{ .Name }}) MarshalJSON() (j []byte, e error) { 51 | return []byte(strconv.Quote(t.s)), nil 52 | } 53 | 54 | // UnmarshalJSON takes byte string JSON to populate the type 55 | func (t *{{ .Name }}) UnmarshalJSON(data []byte) error { 56 | t.s = strings.Trim(string(data), "\"'") 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /generators/templates/extension/template.ext.go.tpl: -------------------------------------------------------------------------------- 1 | // Generated by kraken {{- .Global.Version }}. DO NOT EDIT 2 | // To update, run `kraken extension update` 3 | package {{ .PackageName }} 4 | 5 | import ( 6 | "github.com/kraken-hpc/kraken/core" 7 | ) 8 | 9 | ////////////////////////// 10 | // Boilerplate Methods // 11 | //////////////////////// 12 | 13 | 14 | func (*{{ .Name }}) Name() string { 15 | return "type.googleapis.com/{{ .ProtoPackage }}.{{ .Name }}" 16 | } 17 | 18 | func init() { 19 | core.Registry.RegisterExtension(&{{ .Name }}{}) 20 | } 21 | -------------------------------------------------------------------------------- /generators/templates/extension/template.go.tpl: -------------------------------------------------------------------------------- 1 | // One-time generated by kraken {{ .Global.Version }}. 2 | // You probably want to edit this file to add values you need. 3 | // `kraken extension update` *will not* replace this file. 4 | // `kraken -f extension generate` *will* replace this file. 5 | // To generate your config protobuf run `go generate` (requires `protoc` and `gogo-protobuf`) 6 | 7 | //go:generate protoc -I $GOPATH/src -I . --gogo_out=plugins=grpc:. {{ .LowerName }}.proto 8 | 9 | package {{ .PackageName }} 10 | 11 | import ( 12 | "github.com/kraken-hpc/kraken/lib/types" 13 | ) 14 | 15 | ///////////////////////// 16 | // {{ .Name }} Object // 17 | /////////////////////// 18 | 19 | // This null declaration ensures that we adhere to the interface 20 | var _ types.Extension = (*{{ .Name }})(nil) 21 | 22 | // New creates a new initialized instance of an extension 23 | func (i *{{ .Name }}) New() (m types.Message) { 24 | m = &{{ .Name }}{} 25 | // TODO: you can add extra actions that should happen when a new instance is created here 26 | // for instance, you could set default values. 27 | return 28 | } 29 | -------------------------------------------------------------------------------- /generators/templates/extension/template.proto.tpl: -------------------------------------------------------------------------------- 1 | // One-time generated by kraken {{ .Global.Version }}. 2 | // `kraken module update` *will not* replace this file. 3 | // `kraken -f module generate` *will* replace this file. 4 | // You probably want to edit this file to add values you need. 5 | 6 | syntax = "proto3"; 7 | package {{ .ProtoPackage }}; 8 | option go_package = ".;{{ .PackageName }}"; 9 | 10 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 11 | option (gogoproto.marshaler_all) = true; 12 | option (gogoproto.unmarshaler_all) = true; 13 | option (gogoproto.sizer_all) = true; 14 | option (gogoproto.goproto_registration) = true; 15 | option (gogoproto.messagename_all) = true; 16 | 17 | // TODO: (optional) Define sub-messages here 18 | 19 | message {{ .Name }} { /* Extension */ 20 | // TODO: Add variables here 21 | {{- range .CustomTypes }} 22 | // To define a variable for custom type {{ . }}: 23 | // bytes my{{ . }} = 2 [(gogoproto.customtype) = "{{ $.PackageUrl }}/customtypes.{{ . }}; 24 | {{- end }} 25 | } 26 | -------------------------------------------------------------------------------- /generators/templates/import.go: -------------------------------------------------------------------------------- 1 | // this file is a placeholder for imports to make sure imports needed only in templates get added to go.mod 2 | package templates 3 | 4 | import ( 5 | _ "github.com/coreos/go-systemd/daemon" 6 | ) 7 | -------------------------------------------------------------------------------- /generators/templates/module/README.md.tpl: -------------------------------------------------------------------------------- 1 | # This module was generated by kraken version {{ .Global.Version }} 2 | 3 | ## What was generated? 4 | 5 | We generated these files: 6 | 7 | - `{{ .Name }}.mod.go` - This file shouldn't be directly modified. Use `kraken module update` instead. 8 | - `{{ .Name }}.go` - This file is basic starting-point file to build on. If you run `kraken module update` it *won't* be modified, but `kraken -f module generate` will overwrite it. 9 | - `{{ .Name }}.config.proto` - This is your basis protobuf config. You'll need to generate code from this when it's got the values you need (wiht `go generate`). If you run `kraken module update` it *won't* be modified, but `kraken -f module generate` will overwrite it. 10 | 11 | It might be a good idea to use `{{ .Name }}.go` only to link larger functions in some other source file. This will make it easier for major upgrades when you may need to run `kraken module generate` again. 12 | 13 | ## What to do now 14 | 15 | 1. You need to generate your protobuf code (if you selected `with_config: true` (default)). This will require `protoc` and `gogo-protobuf`. You can then run `go generate` in this directory to produce `{{ .Name }}.config.pb.go`. 16 | 2. At this point the module is runnable if you build it into an app, and it will generate graphs, though it won't *do* anything. This is a good point to make sure your graphs make sense. 17 | 3. In `{{ .Name }}.go` insude of `func Init()` you need to add real functions for mutation handlers. The default handler is just one that prints a "not implemented" message. 18 | 4. You can also add custom code to places like `func Entry()` to change startup behavior, or to `func Poll()` to use your polling loop (if you chose to create one). Look for comments starting with `// TODO:` for suggestions of where you might want to edit this file. 19 | -------------------------------------------------------------------------------- /generators/templates/module/template.config.proto.tpl: -------------------------------------------------------------------------------- 1 | // One-time generated by kraken {{ .Global.Version }}. 2 | // `kraken module update` *will not* replace this file. 3 | // `kraken -f module generate` *will* replace this file. 4 | // You probably want to edit this file to add values you need. 5 | syntax = "proto3"; 6 | package {{ .Name }}; 7 | option go_package = ".;{{ .PackageName }}"; 8 | 9 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 10 | option (gogoproto.marshaler_all) = true; 11 | option (gogoproto.unmarshaler_all) = true; 12 | option (gogoproto.sizer_all) = true; 13 | option (gogoproto.goproto_registration) = true; 14 | option (gogoproto.messagename_all) = true; 15 | 16 | message Config { 17 | {{ if .WithPolling }} 18 | string polling_interval = 1; // Polling interval for the polling timer 19 | {{ end }} 20 | // add variable entries here 21 | } 22 | -------------------------------------------------------------------------------- /generators/templates/module/template.go.tpl: -------------------------------------------------------------------------------- 1 | // One-time generated by kraken {{ .Global.Version }}. 2 | // You probably want to edit this file to add values you need. 3 | // `kraken module update` *will not* replace this file. 4 | // `kraken -f module generate` *will* replace this file. 5 | // To generate your config protobuf run `go generate` (requires `protoc` and `gogo-protobuf`) 6 | 7 | {{ if .WithConfig }} 8 | //go:generate protoc -I $GOPATH/src -I . --gogo_out=plugins=grpc:. {{ .PackageName }}.config.proto 9 | {{ end }} 10 | 11 | package {{ .PackageName }} 12 | 13 | import ( 14 | {{- if .WithConfig }} 15 | "fmt" 16 | {{ end }} 17 | "os" 18 | {{- if .WithPolling }} 19 | "time" 20 | {{ end }} 21 | 22 | {{- if .WithConfig }} 23 | proto "github.com/gogo/protobuf/proto" 24 | {{ end }} 25 | "github.com/kraken-hpc/kraken/lib/types" 26 | ) 27 | 28 | ///////////////////////// 29 | // {{ .Name }} Object // 30 | /////////////////////// 31 | 32 | // These dummy declarations exist to ensure that we're adhering to all of our intended interfaces 33 | var _ types.Module = (*{{ .Name }})(nil) 34 | {{- if .WithConfig }} 35 | var _ types.ModuleWithConfig = (*{{- .Name }})(nil) 36 | {{- end }} 37 | var _ types.ModuleWithMutations = (*{{ .Name }})(nil) 38 | var _ types.ModuleWithDiscovery = (*{{ .Name }})(nil) 39 | var _ types.ModuleSelfService = (*{{ .Name }})(nil) 40 | 41 | // {{ .Name }} is the primary object interface of the module 42 | type {{ .Name }} struct { 43 | {{- if .WithConfig }} 44 | cfg *Config 45 | {{ end }} 46 | {{- if .WithPolling }} 47 | pollTicker *time.Ticker 48 | {{ end }} 49 | // TODO: You can add more internal variables here 50 | } 51 | 52 | /////////////////////// 53 | // Module Execution // 54 | ///////////////////// 55 | 56 | // Init is used to intialize an executable module prior to entrypoint 57 | func (mod *{{ .Name }}) Init(a types.ModuleAPIClient) { 58 | api = a 59 | self = api.Self() 60 | {{- if .WithConfig }} 61 | mod.cfg = mod.NewConfig().(*Config) 62 | {{ end }} 63 | // TODO: You should set these mutation handlers to real functions 64 | {{- range $name, $mutation := .Mutations }} 65 | mutations["{{- $name }}"].Handler = hNotImplemented 66 | {{- end }} 67 | // TODO: You may need to initialize more things here 68 | Log(DEBUG, "initialized") 69 | } 70 | 71 | // Entry is the module's executable entrypoint 72 | func (mod *{{ .Name }}) Entry() { 73 | Log(INFO, "starting") 74 | 75 | {{- if .WithPolling }} 76 | // setup a ticker for polling discovery 77 | dur, _ := time.ParseDuration(mod.cfg.GetPollingInterval()) 78 | mod.pollTicker = time.NewTicker(dur) 79 | 80 | {{ end }} 81 | // TODO: You may need to start more things here. 82 | 83 | // set our own state to be running 84 | mod.State(StateRun) 85 | 86 | // enter our event listener 87 | mod.Listen() 88 | } 89 | 90 | // Stop should perform a graceful exit 91 | func (mod *{{ .Name }}) Stop() { 92 | Log(INFO, "stopping") 93 | // TODO: You may need to safely stop more things. 94 | os.Exit(0) 95 | } 96 | 97 | {{- if .WithPolling }} 98 | ////////////// 99 | // Polling // 100 | //////////// 101 | 102 | // Poll is called repeated only a timer. 103 | func (mod *{{ .Name }}) Poll() { 104 | Log(DDEBUG, "polling loop called") 105 | 106 | // TODO: Add some stuff for Poll to do. Discovering this is a common option. 107 | } 108 | 109 | {{ end }} 110 | {{- if .WithConfig }} 111 | ////////////////////// 112 | // Config Handling // 113 | //////////////////// 114 | 115 | // NewConfig returns a fully initialized default config 116 | func (*{{ .Name }}) NewConfig() proto.Message { 117 | // TODO: fill out this structure with default values for your config 118 | r := &Config{ 119 | {{- if .WithPolling }} 120 | PollingInterval: "10s", 121 | {{ end }} 122 | } 123 | return r 124 | } 125 | 126 | // UpdateConfig updates the running config 127 | func (mod *{{ .Name }}) UpdateConfig(cfg proto.Message) (e error) { 128 | if cfgConfig, ok := cfg.(*Config); ok { 129 | {{- if .WithPolling }} 130 | // Update the polling ticker 131 | if mod.pollTicker != nil { 132 | mod.pollTicker.Stop() 133 | dur, _ := time.ParseDuration(mod.cfg.GetPollingInterval()) 134 | mod.pollTicker = time.NewTicker(dur) 135 | } 136 | {{ end }} 137 | // TODO: You made need to update other things here when your config is changed. 138 | mod.cfg = cfgConfig 139 | return 140 | } 141 | return fmt.Errorf("invalid config type") 142 | } 143 | {{ end }} 144 | -------------------------------------------------------------------------------- /generators/templates/module/template.mod.go.tpl: -------------------------------------------------------------------------------- 1 | // Generated by kraken {{- .Global.Version }}. DO NOT EDIT 2 | // To update, run `kraken module update` 3 | 4 | package {{ .PackageName }} 5 | 6 | import ( 7 | "reflect" 8 | 9 | {{- if .WithConfig }} 10 | ptypes "github.com/gogo/protobuf/types" 11 | {{ end }} 12 | "github.com/kraken-hpc/kraken/core" 13 | cpb "github.com/kraken-hpc/kraken/core/proto" 14 | "github.com/kraken-hpc/kraken/lib/types" 15 | "github.com/kraken-hpc/kraken/lib/util" 16 | ) 17 | 18 | /////////////////////// 19 | // Global Variables // 20 | ///////////////////// 21 | 22 | var api types.ModuleAPIClient // the ModuleAPI client 23 | var self types.NodeID // our own node ID 24 | var si types.ServiceInstance // our own service instance 25 | var mchan <-chan types.Event // mutation event channel (recieve) 26 | var dchan chan<- types.Event // discovery event channel (send) 27 | var interruptHandler = hInterruptUnhandled // set this in order to process mutation interrupts 28 | 29 | /////////////////////////// 30 | // Mutation Definitions // 31 | ///////////////////////// 32 | 33 | var mutations = map[string]*core.Mutation{ 34 | {{- range $name, $mutation := .Mutations }} 35 | "{{- $name }}": { 36 | Mutates: map[string][2]string{ 37 | {{- range $url, $values := $mutation.Mutates }} 38 | "{{- $url }}": {"{{- $values.From }}", "{{- $values.To }}"}, 39 | {{- end }} 40 | }, 41 | Requires: map[string]string{ 42 | {{- range $url, $value := $mutation.Requires }} 43 | "{{- $url }}": "{{- $value }}", 44 | {{- end }} 45 | }, 46 | Excludes: map[string]string{ 47 | {{- range $url, $value := $mutation.Excludes }} 48 | "{{- $url }}": "{{- $value }}", 49 | {{- end }} 50 | }, 51 | Context: "{{- $mutation.Context }}", 52 | Timeout: "{{- $mutation.Timeout }}", 53 | FailTo: [2]string{"{{- $mutation.FailTo.Url }}", "{{- $mutation.FailTo.Value }}"}, 54 | }, 55 | {{- end }} 56 | } 57 | 58 | //////////////////////////// 59 | // Discovery Definitions // 60 | ////////////////////////// 61 | 62 | // Note: Discoveries that are implicit by mutations are automatically included 63 | // Discoveries for the service state are automatically included. 64 | var discoveries = []string{ 65 | {{- range .Discoveries }} 66 | "{{- . }}", 67 | {{- end }} 68 | } 69 | 70 | ///////////////////////////////// 71 | // Boilerplate Object Methods // 72 | /////////////////////////////// 73 | 74 | // Name returns the FQDN of the module 75 | func (*{{- .Name }}) Name() string { return "{{- .PackageUrl }}" } 76 | 77 | {{- if .WithConfig }} 78 | // ConfigURL gives the any resolver URL for the config 79 | func (*{{- .Name }}) ConfigURL() string { 80 | cfg := &Config{} 81 | any, _ := ptypes.MarshalAny(cfg) 82 | return any.GetTypeUrl() 83 | } 84 | {{ end }} 85 | 86 | //////////////////////// 87 | // Wrapper functions // 88 | ////////////////////// 89 | 90 | // Discover submits a discovery event 91 | func Discover(node string, url string, value string) { 92 | nurl := util.NodeURLJoin(node, url) 93 | dchan <- core.NewEvent( 94 | types.Event_DISCOVERY, 95 | nurl, 96 | &core.DiscoveryEvent{ 97 | URL: nurl, 98 | ValueID: value, 99 | }, 100 | ) 101 | } 102 | 103 | // We intentionally exclude Update wrappers as these methods should generally be avoided. 104 | // Use SetValue(s) instead. 105 | 106 | // SetState is a helper function that discovers our own ServiceInstance State 107 | type SelfState string 108 | 109 | const ( 110 | StateRun SelfState = "RUN" 111 | StateStop SelfState = "STOP" 112 | StateError SelfState = "ERROR" 113 | ) 114 | 115 | // State sets our own (ServiceInstance) running state 116 | func (mod *{{- .Name }}) State(s SelfState) { 117 | url := util.URLPush(util.URLPush("/Services", si.ID()), "State") 118 | Discover(self.String(), url, string(s)) 119 | } 120 | 121 | // Listen for events. This is used as our primary entrypoint function. 122 | func (mod *{{- .Name }}) Listen() { 123 | // main loop 124 | for { 125 | select { 126 | {{- if .WithPolling }} 127 | case <-mod.pollTicker.C: // our ticker ticked 128 | go mod.Poll() 129 | break 130 | {{- end }} 131 | case m := <-mchan: // mutation request 132 | go mod.handleMutation(m) 133 | break 134 | } 135 | } 136 | } 137 | 138 | // SetMutationChan sets the current mutation channel 139 | // this is generally done by the API 140 | func (mod *{{- .Name }}) SetMutationChan(c <-chan types.Event) { mchan = c } 141 | 142 | // SetDiscoveryChan sets the current discovery channel 143 | // this is generally done by the API 144 | func (mod *{{- .Name }}) SetDiscoveryChan(c chan<- types.Event) { dchan = c } 145 | 146 | // Make some logging shortcuts 147 | const ( 148 | PANIC = types.LLPANIC 149 | FATAL = types.LLFATAL 150 | CRITICAL = types.LLCRITICAL 151 | ERROR = types.LLERROR 152 | WARNING = types.LLWARNING 153 | NOTICE = types.LLNOTICE 154 | INFO = types.LLINFO 155 | DEBUG = types.LLDEBUG 156 | DDEBUG = types.LLDDEBUG 157 | DDDEBUG = types.LLDDDEBUG 158 | ) 159 | 160 | // Log is a convenience function wrapping api.Log 161 | func Log(level types.LoggerLevel, m string) { 162 | api.Log(level, m) 163 | } 164 | 165 | // Logf is a convenience function wrapping the api.Logf 166 | func Logf(level types.LoggerLevel, fmt string, v ...interface{}) { 167 | api.Logf(level, fmt, v...) 168 | } 169 | 170 | //////////////////////// 171 | // Unexported methods / 172 | ////////////////////// 173 | 174 | func hNotImplemented(mid string, cfg, dsc types.Node) { 175 | Logf(ERROR, "mutation request for node=%s, mutation=%s, but mutation is not implemented", cfg.ID().String(), mid) 176 | } 177 | 178 | func hInterruptUnhandled(*core.MutationEvent) { 179 | Logf(DDEBUG, "received a mutation interrupt, but interrupts are not currently handled") 180 | } 181 | 182 | // handle mutation events 183 | func (mod *{{- .Name }}) handleMutation(m types.Event) { 184 | if m.Type() != types.Event_STATE_MUTATION { 185 | api.Log(types.LLINFO, "got an unexpected event type on mutation channel") 186 | } 187 | me := m.Data().(*core.MutationEvent) 188 | // mutation switch 189 | switch me.Type { 190 | case core.MutationEvent_MUTATE: 191 | mid := me.Mutation[1] 192 | if mdef, ok := mutations[mid]; ok { 193 | if mdef.Handler != nil { 194 | mdef.Handler(mid, me.NodeCfg, me.NodeDsc) 195 | } else { 196 | Logf(ERROR, "hanlder for mutation %s is nil", mid) 197 | } 198 | } else { 199 | Logf(types.LLDEBUG, "mutation request for undefined mutation: %s", me.Mutation[1]) 200 | } 201 | case core.MutationEvent_INTERRUPT: 202 | // nothing to do 203 | interruptHandler(me) 204 | } 205 | } 206 | 207 | // initialization. We rely on this to declare ourselves with Kraken. 208 | func init() { 209 | module := &{{- .Name }}{} 210 | core.Registry.RegisterModule(module) 211 | 212 | si = core.NewServiceInstance("{{- .PackageName }}", module.Name(), module.Entry) 213 | core.Registry.RegisterServiceInstance(module, map[string]types.ServiceInstance{si.ID(): si}) 214 | 215 | // Build list of mutations 216 | muts := make(map[string]types.StateMutation) 217 | for mid, mdef := range mutations { 218 | mut, err := core.CreateMutation(si, mdef) 219 | if err != nil { 220 | panic(err) 221 | } 222 | muts[mid] = mut 223 | // add mutators to discoverables 224 | for url := range mdef.Mutates { 225 | discoveries = append(discoveries, url) 226 | } 227 | } 228 | core.Registry.RegisterMutations(si, muts) 229 | 230 | // Build list of discoveries 231 | discs := make(map[string]map[string]reflect.Value) 232 | for _, url := range discoveries { 233 | d, err := core.CreateDiscovers(url) 234 | if err != nil { 235 | panic(err) 236 | } 237 | discs[url] = d 238 | } 239 | // Add our own service discoveries 240 | discs["/Services/{{- .PackageName }}/State"] = map[string]reflect.Value{ 241 | "UNKNOWN": reflect.ValueOf(cpb.ServiceInstance_UNKNOWN), 242 | "INIT": reflect.ValueOf(cpb.ServiceInstance_INIT), 243 | "STOP": reflect.ValueOf(cpb.ServiceInstance_STOP), 244 | "RUN": reflect.ValueOf(cpb.ServiceInstance_RUN), 245 | "ERROR": reflect.ValueOf(cpb.ServiceInstance_ERROR), 246 | } 247 | core.Registry.RegisterDiscoverable(si, discs) 248 | } 249 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kraken-hpc/kraken 2 | 3 | require ( 4 | github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf 5 | github.com/gogo/protobuf v1.3.2 6 | github.com/golang/protobuf v1.4.3 7 | github.com/google/go-cmp v0.5.2 // indirect 8 | github.com/gorilla/handlers v1.5.1 9 | github.com/gorilla/mux v1.8.0 10 | github.com/gorilla/websocket v1.4.2 11 | github.com/kr/text v0.2.0 // indirect 12 | github.com/kraken-hpc/go-fork v0.1.1 13 | github.com/kraken-hpc/kraken-legacy v0.0.0-20210324182500-03891ea96b57 14 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect 15 | github.com/satori/go.uuid v1.2.0 16 | github.com/sirupsen/logrus v1.8.1 17 | golang.org/x/net v0.0.0-20210226101413-39120d07d75e // indirect 18 | golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 // indirect 19 | golang.org/x/text v0.3.5 // indirect 20 | google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect 21 | google.golang.org/grpc v1.33.2 22 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect 23 | gopkg.in/yaml.v2 v2.4.0 24 | ) 25 | 26 | go 1.15 27 | -------------------------------------------------------------------------------- /kraken.go: -------------------------------------------------------------------------------- 1 | /* kraken.go: the kraken executable can be used to generate kraken application, module, and extension stubs 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package main 11 | 12 | import ( 13 | "flag" 14 | "fmt" 15 | "go/build" 16 | "os" 17 | 18 | "github.com/kraken-hpc/kraken/generators" 19 | log "github.com/sirupsen/logrus" 20 | ) 21 | 22 | var Global = &generators.GlobalConfigType{ 23 | Version: "v0.2.0", 24 | } 25 | 26 | var strToLL = map[string]log.Level{ 27 | "panic": log.PanicLevel, 28 | "fatal": log.FatalLevel, 29 | "error": log.ErrorLevel, 30 | "warn": log.WarnLevel, 31 | "info": log.InfoLevel, 32 | "debug": log.DebugLevel, 33 | "trace": log.TraceLevel, 34 | } 35 | 36 | func cmdApp(args []string) { 37 | var help bool 38 | fs := flag.NewFlagSet("app", flag.ExitOnError) 39 | fs.BoolVar(&help, "h", false, "print this usage") 40 | fs.Usage = func() { 41 | fmt.Println("Usage: kraken [app]lication [-h] [command] [opts]") 42 | fmt.Println("Commands:") 43 | fmt.Println("\tgenerate : generate an application entry point from a config") 44 | fs.PrintDefaults() 45 | } 46 | fs.Parse(args) 47 | if help { 48 | fs.Usage() 49 | os.Exit(0) 50 | } 51 | args = fs.Args() 52 | if len(args) < 1 { 53 | Log.Fatal("no app sub-command") 54 | } 55 | 56 | cmd := args[0] 57 | args = args[1:] 58 | switch cmd { 59 | case "gen", "generate": 60 | generators.AppGenerate(Global, args) 61 | default: 62 | Log.Errorf("unknown app sub-command: %s", cmd) 63 | fs.Usage() 64 | os.Exit(1) 65 | } 66 | } 67 | 68 | func cmdModule(args []string) { 69 | var help bool 70 | fs := flag.NewFlagSet("module", flag.ExitOnError) 71 | fs.BoolVar(&help, "h", false, "print this message") 72 | fs.Usage = func() { 73 | fmt.Println("Usage: kraken [mod]ule [-h] [command] [opts]") 74 | fmt.Println("Commands:") 75 | fmt.Println("\tgenerate : generate a module from a config") 76 | fmt.Println("\tupdate : update the .mod.go file only") 77 | fs.PrintDefaults() 78 | } 79 | fs.Parse(args) 80 | if help { 81 | fs.Usage() 82 | os.Exit(0) 83 | } 84 | args = fs.Args() 85 | if len(args) < 1 { 86 | Log.Fatal("no module sub-command") 87 | } 88 | cmd := args[0] 89 | args = args[1:] 90 | switch cmd { 91 | case "gen", "generate": 92 | generators.ModuleGenerate(Global, args) 93 | case "up", "update": 94 | generators.ModuleUpdate(Global, args) 95 | default: 96 | Log.Errorf("unknown module sub-command: %s", cmd) 97 | fs.Usage() 98 | os.Exit(1) 99 | } 100 | } 101 | 102 | func cmdExtension(args []string) { 103 | var help bool 104 | fs := flag.NewFlagSet("extension", flag.ExitOnError) 105 | fs.BoolVar(&help, "h", false, "print this message") 106 | fs.Usage = func() { 107 | fmt.Println("Usage: kraken [ext]ension [-h] [command] [opts]") 108 | fmt.Println("Commands:") 109 | fmt.Println("\tgenerate : generate an extension from a config") 110 | fs.PrintDefaults() 111 | } 112 | fs.Parse(args) 113 | if help { 114 | fs.Usage() 115 | os.Exit(0) 116 | } 117 | args = fs.Args() 118 | if len(args) < 1 { 119 | Log.Fatal("no extension sub-command") 120 | } 121 | cmd := args[0] 122 | args = args[1:] 123 | switch cmd { 124 | case "gen", "generate": 125 | generators.ExtensionGenerate(Global, args) 126 | default: 127 | Log.Errorf("unknown extension sub-command: %s", cmd) 128 | fs.Usage() 129 | os.Exit(1) 130 | } 131 | } 132 | 133 | // Entry point 134 | var Log *log.Logger 135 | 136 | func main() { 137 | Log = log.New() 138 | Global.Log = Log 139 | var help = false 140 | fs := flag.NewFlagSet("kraken", flag.ContinueOnError) 141 | usage := func() { 142 | fmt.Println("kraken is a code-generator for kraken-based applications, modules, and extensions") 143 | fmt.Println() 144 | fmt.Println("Usage: kraken [-fv] [-l ] [options]") 145 | fmt.Println("Commands:") 146 | fmt.Println("\t[app]lication") 147 | fmt.Println("\t[mod]ule") 148 | fmt.Println("\t[ext]ension") 149 | fmt.Println("For command help: kraken -h") 150 | fs.PrintDefaults() 151 | } 152 | gopath := os.Getenv("GOPATH") 153 | if gopath == "" { 154 | gopath = build.Default.GOPATH 155 | } 156 | var logLevel string 157 | fs.BoolVar(&Global.Force, "f", false, "overwrite files if they exist") 158 | fs.StringVar(&logLevel, "l", "info", "Log level. Valid values: panic, fatal, error, warn, info, debug, trace.") 159 | fs.BoolVar(&help, "h", false, "print usage and exit") 160 | if err := fs.Parse(os.Args[1:]); err != nil { 161 | Log.Fatalf("failed to parse arguments: %v", err) 162 | } 163 | if help { 164 | usage() 165 | os.Exit(0) 166 | } 167 | if ll, ok := strToLL[logLevel]; ok { 168 | Log.SetLevel(ll) 169 | } else { 170 | Log.Fatalf("unknown log level: %s", logLevel) 171 | } 172 | args := fs.Args() 173 | if len(args) < 1 { 174 | log.Error("no command specified") 175 | usage() 176 | os.Exit(1) 177 | } 178 | cmd := args[0] 179 | args = args[1:] 180 | switch cmd { 181 | case "app", "application": 182 | cmdApp(args) 183 | case "mod", "module": 184 | cmdModule(args) 185 | case "ext", "extension": 186 | cmdExtension(args) 187 | default: 188 | Log.Fatalf("unknown command: %s", cmd) 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /lib/build/buildutil.go: -------------------------------------------------------------------------------- 1 | /* buildutil.go: provides useful methods used in building Kraken 2 | * 3 | * Author: Devon Bautista 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package build 11 | 12 | import ( 13 | "fmt" 14 | "io/ioutil" 15 | "log" 16 | "os" 17 | "path/filepath" 18 | "regexp" 19 | "strings" 20 | ) 21 | 22 | // simpleSearchAndReplace searches all lines in a file and replaces all 23 | // instances of srchStr with replStr. 24 | // This is a helper function for SearchAndReplace only works with regular 25 | // files. 26 | func simpleSearchAndReplace(filename, srchStr, replStr string) (e error) { 27 | // Try to open file 28 | input, e := ioutil.ReadFile(filename) 29 | if e != nil { 30 | log.Fatalf("error opening file %s for search and replace", filename) 31 | } 32 | 33 | // Put lines of file into array 34 | lines := strings.Split(string(input), "\n") 35 | 36 | // Iterate through lines to search and replace 37 | for i, line := range lines { 38 | if strings.Contains(line, srchStr) { 39 | // Replace srchStr with replStr in line[i] 40 | lines[i] = strings.ReplaceAll(string(lines[i]), srchStr, replStr) 41 | } 42 | } 43 | 44 | // Merge array into one string 45 | output := strings.Join(lines, "\n") 46 | 47 | // Attempt to write new contents to file 48 | e = ioutil.WriteFile(filename, []byte(output), 0644) 49 | if e != nil { 50 | log.Fatal("error writing file during search and replace") 51 | } 52 | return 53 | } 54 | 55 | // simpleRegexReplace applies a regular expression src to filename and replaces 56 | // matches with repl, which supports formatting specified in the regexp package. 57 | // This is a helper function for RegexReplace and only works on regular files. 58 | func simpleRegexReplace(filename, src, repl string) (e error) { 59 | // Get file perms 60 | var fInfo os.FileInfo 61 | if fInfo, e = os.Stat(filename); e != nil { 62 | e = fmt.Errorf("could not stat %s: %v", filename, e) 63 | return 64 | } 65 | 66 | // Open file 67 | var fileContents []byte 68 | if fileContents, e = ioutil.ReadFile(filename); e != nil { 69 | e = fmt.Errorf("could not read %s: %v", filename, e) 70 | return 71 | } 72 | 73 | // Replace src with repl 74 | var newContents []byte 75 | re := regexp.MustCompile(src) 76 | newContents = re.ReplaceAll(fileContents, []byte(repl)) 77 | 78 | // Replace file 79 | if e = ioutil.WriteFile(filename, newContents, fInfo.Mode()); e != nil { 80 | e = fmt.Errorf("could not write %s: %v", filename, e) 81 | } 82 | 83 | return 84 | } 85 | 86 | // SearchAndReplace replaces all instances of srchStr in filename with replStr. 87 | // If filename is a directory, it does this recursively for all files and 88 | // directories contained in it. 89 | func SearchAndReplace(filename, srchStr, replStr string) (e error) { 90 | // Get info of filename 91 | var info os.FileInfo 92 | info, e = os.Lstat(filename) 93 | if e != nil { 94 | return 95 | } 96 | 97 | // Is this a directory? 98 | if info.IsDir() { 99 | // If so, read each child and recurse until regular file found. 100 | var contents []os.FileInfo 101 | contents, e = ioutil.ReadDir(filename) 102 | for _, content := range contents { 103 | e = SearchAndReplace(filepath.Join(filename, content.Name()), srchStr, replStr) 104 | if e != nil { 105 | return 106 | } 107 | } 108 | } else { 109 | // If not, perform a simple search and replace on the file 110 | e = simpleSearchAndReplace(filename, srchStr, replStr) 111 | } 112 | 113 | return 114 | } 115 | 116 | // RegexReplace applies the regular expression src to filename and replaces 117 | // matches with repl. If filename is a directory, it does this recursively 118 | // for all files and directories contained in it. The replace string repl 119 | // supports formatting according to the regexp package. 120 | func RegexReplace(filename, src, repl string) (e error) { 121 | // What type of file is this? 122 | var info os.FileInfo 123 | if info, e = os.Lstat(filename); e != nil { 124 | e = fmt.Errorf("could not stat %s: %v", filename, e) 125 | return 126 | } 127 | 128 | // If file is a directory, recurse until regular file is found 129 | if info.IsDir() { 130 | var contents []os.FileInfo 131 | if contents, e = ioutil.ReadDir(filename); e != nil { 132 | e = fmt.Errorf("could not read %s: %v", filename, e) 133 | return 134 | } 135 | for _, file := range contents { 136 | filePath := filepath.Join(filename, file.Name()) 137 | if e = RegexReplace(filePath, src, repl); e != nil { 138 | e = fmt.Errorf("could not deep regex replacement in %s: %v", filePath, e) 139 | return 140 | } 141 | } 142 | } else { 143 | if e = simpleRegexReplace(filename, src, repl); e != nil { 144 | e = fmt.Errorf("could not perform simple regex replacement on %s: %v", filename, e) 145 | } 146 | } 147 | 148 | return 149 | } 150 | -------------------------------------------------------------------------------- /lib/copy/cmp/cmp.go: -------------------------------------------------------------------------------- 1 | // Package cmp provides file- and tree-comparison routines. 2 | // 3 | // It is a sub-package to copy's test suite. 4 | package cmp 5 | 6 | import ( 7 | "bytes" 8 | "errors" 9 | "fmt" 10 | "io" 11 | "io/ioutil" 12 | "os" 13 | "path/filepath" 14 | "reflect" 15 | 16 | "github.com/kraken-hpc/kraken/lib/copy" 17 | ) 18 | 19 | const ( 20 | // Compare this many bytes of a file at a time 21 | chunkSize = 64000 // 64KB 22 | ) 23 | 24 | // IsSameFile returns true if file1 and file2 are the 25 | // same file as determined by os.SameFile. 26 | // This will work with directories as well. 27 | // What to do with symlinks is configured by opt. 28 | func IsSameFile(opt copy.Options, file1, file2 string) (bool, error) { 29 | var fileInfo1, fileInfo2 os.FileInfo 30 | var err error 31 | 32 | fileInfo1, err = opt.Stat(file1) 33 | if err != nil { 34 | return false, err 35 | } 36 | fileInfo2, err = opt.Stat(file2) 37 | if err != nil { 38 | return false, err 39 | } 40 | 41 | return os.SameFile(fileInfo1, fileInfo2), err 42 | } 43 | 44 | // readDirNames returns an array of file names of the 45 | // contents of the directory at path. 46 | func readDirNames(path string) ([]string, error) { 47 | items, err := ioutil.ReadDir(path) 48 | if err != nil { 49 | return nil, err 50 | } 51 | var basenames []string 52 | for _, item := range items { 53 | basenames = append(basenames, item.Name()) 54 | } 55 | return basenames, nil 56 | } 57 | 58 | // IsEqualSymlink checks if two symlinks have the same name and target. 59 | func IsEqualSymlink(link1, link2 string, lInfo1, lInfo2 os.FileInfo) error { 60 | // Check targets 61 | targ1, err := os.Readlink(link1) 62 | if err != nil { 63 | return err 64 | } 65 | targ2, err := os.Readlink(link2) 66 | if err != nil { 67 | return err 68 | } 69 | if targ1 != targ2 { 70 | return fmt.Errorf("symlinks point to different locations:\n%q -> %q\n%q -> %q", link1, targ1, link2, targ2) 71 | } 72 | return nil 73 | } 74 | 75 | // areFilesOrEqualSymlinks returns nil if (1) the files are 76 | // both symlinks AND they point to the same location or (2) 77 | // neither file is a symlink. Otherwise, it returns an error. 78 | func areFilesOrEqualSymlinks(opt copy.Options, fPath1, fPath2 string, fInfo1, fInfo2 os.FileInfo) (file1, file2 string, err error) { 79 | fIsSymLink1 := fInfo1.Mode()&os.ModeSymlink == os.ModeSymlink 80 | fIsSymLink2 := fInfo2.Mode()&os.ModeSymlink == os.ModeSymlink 81 | file1 = fPath1 82 | file2 = fPath2 83 | 84 | // Check if either or both files are symlinks 85 | if fIsSymLink1 || fIsSymLink2 { 86 | if opt.FollowSymlinks { 87 | // If we're following symlinks, find their targets in order 88 | // to compare the end results 89 | file1, err = filepath.EvalSymlinks(file1) 90 | if err != nil { 91 | return 92 | } 93 | file2, err = filepath.EvalSymlinks(file2) 94 | if err != nil { 95 | return 96 | } 97 | } else { 98 | if fIsSymLink1 && fIsSymLink2 { 99 | // If BOTH are symlinks and we're not following them, 100 | // compare their targets 101 | err = IsEqualSymlink(file1, file2, fInfo1, fInfo2) 102 | } else { 103 | // Only one of the files is a symlink and we're not 104 | // following them, they are not considered the same 105 | err = fmt.Errorf("one file is symlink, one is regular file: %q, %q", file1, file2) 106 | } 107 | } 108 | } 109 | return 110 | } 111 | 112 | // IsSameFile compares two files by byte contents. 113 | // What to do with symlinks is configured by opt. 114 | func IsEqualFile(opt copy.Options, file1, file2 string) error { 115 | // What kind of files are these? 116 | fInfo1, err := opt.Stat(file1) 117 | if err != nil { 118 | return err 119 | } 120 | fInfo2, err := opt.Stat(file2) 121 | if err != nil { 122 | return err 123 | } 124 | if file1, file2, err = areFilesOrEqualSymlinks(opt, file1, file2, fInfo1, fInfo2); err != nil { 125 | return err 126 | } 127 | 128 | // Open files 129 | f1, err := os.Open(file1) 130 | if err != nil { 131 | return err 132 | } 133 | defer f1.Close() 134 | f2, err := os.Open(file2) 135 | if err != nil { 136 | return err 137 | } 138 | defer f2.Close() 139 | 140 | // Compare chunks of files until different or EOF 141 | // in one or both 142 | for { 143 | // Create buffers in loop since Go is garbage-collected 144 | buf1 := make([]byte, chunkSize) 145 | _, err1 := f1.Read(buf1) 146 | 147 | buf2 := make([]byte, chunkSize) 148 | _, err2 := f2.Read(buf2) 149 | 150 | // Check for EOF 151 | if err1 != nil || err2 != nil { 152 | if errors.Is(err1, io.EOF) && errors.Is(err2, io.EOF) { 153 | // EOF for both files reached; files equal 154 | return nil 155 | } else if errors.Is(err1, io.EOF) || errors.Is(err2, io.EOF) { 156 | // Files different lengths; not equal 157 | return fmt.Errorf("files different lengths: %s, %s", file1, file2) 158 | } else { 159 | return fmt.Errorf("non-EOF errors thrown: %s, %s", err1, err2) 160 | } 161 | } 162 | 163 | // Compare buffers 164 | if !bytes.Equal(buf1, buf2) { 165 | return fmt.Errorf("%q and %q do not have equal content", file1, file2) 166 | } 167 | } 168 | } 169 | 170 | // IsEqualDir compares the contents of two directories. 171 | // What to do with symlinks is configured by opt. 172 | func IsEqualDir(opt copy.Options, src, dst string) error { 173 | // Get info for each dir 174 | srcInfo, err := opt.Stat(src) 175 | if err != nil { 176 | return err 177 | } 178 | dstInfo, err := opt.Stat(dst) 179 | if err != nil { 180 | return err 181 | } 182 | 183 | // Do they have the same mode? 184 | if srcMode, dstMode := srcInfo.Mode()&os.ModeType, dstInfo.Mode()&os.ModeType; srcMode != dstMode { 185 | return fmt.Errorf("mismatched mode: %q (%s) and %q (%s)", src, srcMode, dst, dstMode) 186 | } 187 | 188 | // Decide what to do based on file type (for recursion) 189 | if srcInfo.Mode().IsDir() { 190 | // Directory 191 | if opt.Recursive { 192 | // Compare directory contents' names 193 | srcNames, err := readDirNames(src) 194 | if err != nil { 195 | return err 196 | } 197 | dstNames, err := readDirNames(dst) 198 | if err != nil { 199 | return err 200 | } 201 | 202 | if !reflect.DeepEqual(srcNames, dstNames) { 203 | return fmt.Errorf("directory contents mismatch:\n%q: %v\n%q: %v", src, srcNames, dst, dstNames) 204 | } 205 | 206 | // Recursively compare directory contents 207 | for _, item := range srcNames { 208 | if err := IsEqualDir(opt, filepath.Join(src, item), filepath.Join(dst, item)); err != nil { 209 | return err 210 | } 211 | } 212 | return nil 213 | } else { 214 | if srcInfo.Mode() == dstInfo.Mode() { 215 | return nil 216 | } 217 | return fmt.Errorf("directory mode mismatch: %q (%v), %q (%v)", src, srcInfo.Mode(), dst, dstInfo.Mode()) 218 | } 219 | } else if srcInfo.Mode().IsRegular() || srcInfo.Mode()&os.ModeSymlink == os.ModeSymlink { 220 | // Regular file or symlink 221 | return IsEqualFile(opt, src, dst) 222 | } else { 223 | return fmt.Errorf("unsupported file mode: %s", srcInfo.Mode()) 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /lib/copy/copy.go: -------------------------------------------------------------------------------- 1 | package copy 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "path" 10 | ) 11 | 12 | // ErrorSkip can be returned by PreHook to skip a file 13 | // when copying. 14 | type ErrorSkip struct { 15 | // File is what is being skipped. 16 | File string 17 | } 18 | 19 | func (e *ErrorSkip) Error() string { return "skipped: " + e.File } 20 | 21 | func NewErrorSkip(file string) *ErrorSkip { return &ErrorSkip{File: file} } 22 | 23 | // Options provides configuration for copying. 24 | type Options struct { 25 | // Recurse into subdirectories, copying their 26 | // contents. 27 | Recursive bool 28 | 29 | // Clobber (overwrite) file or directory if it 30 | // already exists. 31 | Clobber bool 32 | 33 | // If FollowSymlinks is set, copy the file pointed to by 34 | // the symlink instead of the symlink itself. 35 | FollowSymlinks bool 36 | 37 | // Be verbose, showing each file being copied. 38 | Verbose bool 39 | 40 | // PreHook is called, if specified, on each file before 41 | // it is copied. 42 | // 43 | // If PreHook returns ErrorSkip, the current file 44 | // is skipped and Copy returns nil. 45 | // 46 | // If PreHook returns another non-nil error, the 47 | // current file is skipped and Copy returns the error. 48 | PreHook func(src, dst string, srcInfo os.FileInfo) error 49 | 50 | // PostHook is called, if specified, on the current 51 | // file after it is copied. 52 | PostHook func(src, dst string) 53 | } 54 | 55 | // Copy copies a regular file, symlink, or directory 56 | // located at src to the destination dst, using the 57 | // options configured. 58 | func (opt Options) Copy(src, dst string) error { 59 | var srcInfo os.FileInfo 60 | var err error 61 | 62 | // Make sure source file exists 63 | srcInfo, err = opt.Stat(src) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | // Call pre-copy hook if specified 69 | if opt.PreHook != nil { 70 | err := opt.PreHook(src, dst, srcInfo) 71 | if _, ok := err.(*ErrorSkip); ok { 72 | return nil 73 | } else if err != nil { 74 | return err 75 | } 76 | } 77 | 78 | // Only overwrite dst if Clobber is true 79 | if err := opt.copyFile(src, dst, srcInfo); err != nil { 80 | return err 81 | } 82 | 83 | // Call post-copy hook if specified 84 | if opt.PostHook != nil { 85 | opt.PostHook(src, dst) 86 | } 87 | 88 | return nil 89 | } 90 | 91 | // copyFile determines which copy helper function to call 92 | // based on the type of file src is. 93 | func (opt Options) copyFile(src, dst string, srcInfo os.FileInfo) error { 94 | if srcInfo == nil { 95 | return fmt.Errorf("srcInfo (os.FileInfo) cannot be nil") 96 | } 97 | srcMode := srcInfo.Mode() 98 | 99 | switch { 100 | // Directory 101 | case srcMode.IsDir(): 102 | return opt.copyTree(src, dst, srcInfo) 103 | 104 | // Regular file 105 | case srcMode.IsRegular(): 106 | return opt.copyRegularFile(src, dst, srcInfo) 107 | 108 | // Symlink 109 | case srcMode&os.ModeSymlink == os.ModeSymlink: 110 | return opt.handleSymlink(src, dst) 111 | 112 | // Anything else 113 | default: 114 | return &os.PathError{ 115 | Op: "copy", 116 | Path: src, 117 | Err: fmt.Errorf("unsupported file mode %s", srcMode), 118 | } 119 | } 120 | } 121 | 122 | // copyRegularFile copies a regular file located at src to the 123 | // destination dst. 124 | func (opt Options) copyRegularFile(src, dst string, srcInfo os.FileInfo) error { 125 | if opt.Verbose { 126 | log.Printf("[F] %q -> %q\n", src, dst) 127 | } 128 | 129 | // Open source file for reading 130 | srcFile, err := os.Open(src) 131 | if err != nil { 132 | return err 133 | } 134 | defer srcFile.Close() 135 | 136 | // Open destination file for creation/writing 137 | dstFile, err := os.OpenFile(dst, os.O_RDWR|os.O_CREATE|os.O_TRUNC, srcInfo.Mode().Perm()) 138 | if err != nil { 139 | return err 140 | } 141 | defer dstFile.Close() 142 | 143 | // Copy file contents 144 | _, err = io.Copy(dstFile, srcFile) 145 | return err 146 | } 147 | 148 | // handleSymlink copies a symlink src to dst by reading it and 149 | // creating a new symlink. 150 | func (opt Options) handleSymlink(src, dst string) error { 151 | // Get symlink's destination 152 | target, err := os.Readlink(src) 153 | if err != nil { 154 | return err 155 | } 156 | 157 | if opt.Verbose { 158 | log.Printf("[L] %q (%s) -> %q\n", src, target, dst) 159 | } 160 | 161 | if opt.FollowSymlinks { 162 | // Recursively follow symlink 163 | if _, err := os.Lstat(target); err != nil { 164 | return err 165 | } 166 | return opt.Copy(target, dst) 167 | } else { 168 | // "Copy" symlink itself 169 | if err := os.Symlink(target, dst); err != nil { 170 | if opt.Clobber { 171 | if err := os.RemoveAll(dst); err != nil { 172 | return err 173 | } 174 | return os.Symlink(target, dst) 175 | } else { 176 | return err 177 | } 178 | } 179 | return nil 180 | } 181 | } 182 | 183 | // copyTree copies a directory at src and its contents, recursively, 184 | // to the destination dst. 185 | func (opt Options) copyTree(src, dst string, info os.FileInfo) error { 186 | if opt.Verbose { 187 | log.Printf("[D] %q -> %q\n", src, dst) 188 | } 189 | 190 | // TODO: Add preservation of mode 191 | if info == nil { 192 | return fmt.Errorf("info (os.FileInfo) cannot be nil") 193 | } else if !info.Mode().IsDir() { 194 | return fmt.Errorf("%q is not a directory", src) 195 | } 196 | 197 | // Make destination dir 198 | err := os.Mkdir(dst, info.Mode()) 199 | if err != nil { 200 | if opt.Clobber { 201 | if err = os.RemoveAll(dst); err != nil { 202 | return err 203 | } 204 | if err = os.Mkdir(dst, info.Mode()); err != nil { 205 | return err 206 | } 207 | } else { 208 | return err 209 | } 210 | } 211 | 212 | if opt.Recursive { 213 | contents, err := ioutil.ReadDir(src) 214 | if err != nil { 215 | return err 216 | } 217 | for _, item := range contents { 218 | srcItem, dstItem := path.Join(src, item.Name()), path.Join(dst, item.Name()) 219 | if _, err := opt.Stat(srcItem); err != nil { 220 | return err 221 | } 222 | if err := opt.Copy(srcItem, dstItem); err != nil { 223 | return err 224 | } 225 | } 226 | } 227 | return nil 228 | } 229 | 230 | // Stat returns a file's information. If path is a symlink, 231 | // then Stat does or does not follow it based on the value 232 | // of FollowSymLinks. 233 | func (opt Options) Stat(path string) (os.FileInfo, error) { 234 | if opt.FollowSymlinks { 235 | return os.Stat(path) 236 | } else { 237 | return os.Lstat(path) 238 | } 239 | } 240 | 241 | // GetDefaultOptions returns an Options struct populated 242 | // with the default options. 243 | func GetDefaultOptions() Options { 244 | return Options{ 245 | Recursive: false, // Don't copy dir contents 246 | Clobber: true, // cp overwrites by default 247 | FollowSymlinks: true, // cp by default copies targets 248 | Verbose: false, // Don't print filenames 249 | PreHook: nil, 250 | PostHook: nil, 251 | } 252 | } 253 | 254 | // Copy uses the default options to copy 255 | // src to dst. 256 | func Copy(src, dst string) error { 257 | var defaultOpts = GetDefaultOptions() 258 | return defaultOpts.Copy(src, dst) 259 | } 260 | -------------------------------------------------------------------------------- /lib/json/json.go: -------------------------------------------------------------------------------- 1 | /* json.go: json handlers for messages 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018-2021, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | package json 11 | 12 | import ( 13 | "bytes" 14 | 15 | "github.com/gogo/protobuf/jsonpb" 16 | "github.com/gogo/protobuf/proto" 17 | ) 18 | 19 | // Marshaler is a global marshaler that sets our default options 20 | var Marshaler = jsonpb.Marshaler{} 21 | 22 | // Unmarshaler is a global unmarshaler that sets our default options 23 | var Unmarshaler = jsonpb.Unmarshaler{} 24 | 25 | // Marshal turns a gogo proto.Message into json with the default marshaler 26 | func Marshal(m proto.Message) ([]byte, error) { 27 | buf := bytes.NewBuffer([]byte{}) 28 | e := Marshaler.Marshal(buf, m) 29 | return buf.Bytes(), e 30 | } 31 | 32 | // Unmarshal turns a json message into a proto.Message with the default marshaler 33 | func Unmarshal(in []byte, m proto.Message) error { 34 | buf := bytes.NewBuffer(in) 35 | return Unmarshaler.Unmarshal(buf, m) 36 | } 37 | -------------------------------------------------------------------------------- /lib/util/test.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-gogo. DO NOT EDIT. 2 | // source: test.proto 3 | 4 | package util 5 | 6 | import ( 7 | fmt "fmt" 8 | proto "github.com/gogo/protobuf/proto" 9 | math "math" 10 | ) 11 | 12 | // Reference imports to suppress errors if they are not otherwise used. 13 | var _ = proto.Marshal 14 | var _ = fmt.Errorf 15 | var _ = math.Inf 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the proto package it is being compiled against. 19 | // A compilation error at this line likely means your copy of the 20 | // proto package needs to be updated. 21 | const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package 22 | 23 | type SubFixture struct { 24 | A string `protobuf:"bytes,1,opt,name=A,proto3" json:"A,omitempty"` 25 | B string `protobuf:"bytes,2,opt,name=B,proto3" json:"B,omitempty"` 26 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 27 | XXX_unrecognized []byte `json:"-"` 28 | XXX_sizecache int32 `json:"-"` 29 | } 30 | 31 | func (m *SubFixture) Reset() { *m = SubFixture{} } 32 | func (m *SubFixture) String() string { return proto.CompactTextString(m) } 33 | func (*SubFixture) ProtoMessage() {} 34 | func (*SubFixture) Descriptor() ([]byte, []int) { 35 | return fileDescriptor_c161fcfdc0c3ff1e, []int{0} 36 | } 37 | func (m *SubFixture) XXX_Unmarshal(b []byte) error { 38 | return xxx_messageInfo_SubFixture.Unmarshal(m, b) 39 | } 40 | func (m *SubFixture) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 41 | return xxx_messageInfo_SubFixture.Marshal(b, m, deterministic) 42 | } 43 | func (m *SubFixture) XXX_Merge(src proto.Message) { 44 | xxx_messageInfo_SubFixture.Merge(m, src) 45 | } 46 | func (m *SubFixture) XXX_Size() int { 47 | return xxx_messageInfo_SubFixture.Size(m) 48 | } 49 | func (m *SubFixture) XXX_DiscardUnknown() { 50 | xxx_messageInfo_SubFixture.DiscardUnknown(m) 51 | } 52 | 53 | var xxx_messageInfo_SubFixture proto.InternalMessageInfo 54 | 55 | func (m *SubFixture) GetA() string { 56 | if m != nil { 57 | return m.A 58 | } 59 | return "" 60 | } 61 | 62 | func (m *SubFixture) GetB() string { 63 | if m != nil { 64 | return m.B 65 | } 66 | return "" 67 | } 68 | 69 | type Fixture struct { 70 | Boolean bool `protobuf:"varint,1,opt,name=Boolean,proto3" json:"Boolean,omitempty"` 71 | UInt uint32 `protobuf:"varint,2,opt,name=UInt,proto3" json:"UInt,omitempty"` 72 | Slice []string `protobuf:"bytes,3,rep,name=Slice,proto3" json:"Slice,omitempty"` 73 | Map map[string]uint32 `protobuf:"bytes,4,rep,name=Map,proto3" json:"Map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` 74 | Sub *SubFixture `protobuf:"bytes,5,opt,name=Sub,proto3" json:"Sub,omitempty"` 75 | SliceSub []*SubFixture `protobuf:"bytes,6,rep,name=SliceSub,proto3" json:"SliceSub,omitempty"` 76 | MapSub map[uint32]*SubFixture `protobuf:"bytes,7,rep,name=MapSub,proto3" json:"MapSub,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` 77 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 78 | XXX_unrecognized []byte `json:"-"` 79 | XXX_sizecache int32 `json:"-"` 80 | } 81 | 82 | func (m *Fixture) Reset() { *m = Fixture{} } 83 | func (m *Fixture) String() string { return proto.CompactTextString(m) } 84 | func (*Fixture) ProtoMessage() {} 85 | func (*Fixture) Descriptor() ([]byte, []int) { 86 | return fileDescriptor_c161fcfdc0c3ff1e, []int{1} 87 | } 88 | func (m *Fixture) XXX_Unmarshal(b []byte) error { 89 | return xxx_messageInfo_Fixture.Unmarshal(m, b) 90 | } 91 | func (m *Fixture) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 92 | return xxx_messageInfo_Fixture.Marshal(b, m, deterministic) 93 | } 94 | func (m *Fixture) XXX_Merge(src proto.Message) { 95 | xxx_messageInfo_Fixture.Merge(m, src) 96 | } 97 | func (m *Fixture) XXX_Size() int { 98 | return xxx_messageInfo_Fixture.Size(m) 99 | } 100 | func (m *Fixture) XXX_DiscardUnknown() { 101 | xxx_messageInfo_Fixture.DiscardUnknown(m) 102 | } 103 | 104 | var xxx_messageInfo_Fixture proto.InternalMessageInfo 105 | 106 | func (m *Fixture) GetBoolean() bool { 107 | if m != nil { 108 | return m.Boolean 109 | } 110 | return false 111 | } 112 | 113 | func (m *Fixture) GetUInt() uint32 { 114 | if m != nil { 115 | return m.UInt 116 | } 117 | return 0 118 | } 119 | 120 | func (m *Fixture) GetSlice() []string { 121 | if m != nil { 122 | return m.Slice 123 | } 124 | return nil 125 | } 126 | 127 | func (m *Fixture) GetMap() map[string]uint32 { 128 | if m != nil { 129 | return m.Map 130 | } 131 | return nil 132 | } 133 | 134 | func (m *Fixture) GetSub() *SubFixture { 135 | if m != nil { 136 | return m.Sub 137 | } 138 | return nil 139 | } 140 | 141 | func (m *Fixture) GetSliceSub() []*SubFixture { 142 | if m != nil { 143 | return m.SliceSub 144 | } 145 | return nil 146 | } 147 | 148 | func (m *Fixture) GetMapSub() map[uint32]*SubFixture { 149 | if m != nil { 150 | return m.MapSub 151 | } 152 | return nil 153 | } 154 | 155 | func init() { 156 | proto.RegisterType((*SubFixture)(nil), "Test.SubFixture") 157 | proto.RegisterType((*Fixture)(nil), "Test.Fixture") 158 | proto.RegisterMapType((map[string]uint32)(nil), "Test.Fixture.MapEntry") 159 | proto.RegisterMapType((map[uint32]*SubFixture)(nil), "Test.Fixture.MapSubEntry") 160 | } 161 | 162 | func init() { proto.RegisterFile("test.proto", fileDescriptor_c161fcfdc0c3ff1e) } 163 | 164 | var fileDescriptor_c161fcfdc0c3ff1e = []byte{ 165 | // 282 bytes of a gzipped FileDescriptorProto 166 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xc1, 0x4b, 0xc3, 0x30, 167 | 0x18, 0xc5, 0xc9, 0xd2, 0xb5, 0xdd, 0x57, 0x07, 0x23, 0x88, 0xc4, 0x9d, 0x4a, 0x0f, 0x92, 0x83, 168 | 0x14, 0x9c, 0x20, 0xa2, 0xa7, 0x15, 0x14, 0x44, 0x7a, 0x49, 0xf5, 0xe2, 0x2d, 0x95, 0x1c, 0x8a, 169 | 0xa5, 0x2d, 0x5d, 0x22, 0xee, 0xe8, 0x7f, 0x2e, 0x49, 0x5a, 0x57, 0xd8, 0x6e, 0xdf, 0x23, 0xbf, 170 | 0xf7, 0xf2, 0xbe, 0x04, 0x40, 0xc9, 0x9d, 0x4a, 0xbb, 0xbe, 0x55, 0x2d, 0xf1, 0xde, 0xe4, 0x4e, 171 | 0x25, 0x0c, 0xa0, 0xd0, 0xe5, 0x73, 0xf5, 0xa3, 0x74, 0x2f, 0xc9, 0x19, 0xa0, 0x2d, 0x45, 0x31, 172 | 0x62, 0x0b, 0x8e, 0xb6, 0x46, 0x65, 0x74, 0xe6, 0x54, 0x96, 0xfc, 0x62, 0x08, 0x46, 0x8e, 0x42, 173 | 0x90, 0xb5, 0x6d, 0x2d, 0x45, 0x63, 0xe9, 0x90, 0x8f, 0x92, 0x10, 0xf0, 0xde, 0x5f, 0x1a, 0x65, 174 | 0x6d, 0x4b, 0x6e, 0x67, 0x72, 0x0e, 0xf3, 0xa2, 0xae, 0x3e, 0x25, 0xc5, 0x31, 0x66, 0x0b, 0xee, 175 | 0x04, 0x61, 0x80, 0x73, 0xd1, 0x51, 0x2f, 0xc6, 0x2c, 0xda, 0x5c, 0xa4, 0xa6, 0x4d, 0x3a, 0xe4, 176 | 0xa7, 0xb9, 0xe8, 0x9e, 0x1a, 0xd5, 0xef, 0xb9, 0x41, 0x48, 0x02, 0xb8, 0xd0, 0x25, 0x9d, 0xc7, 177 | 0x88, 0x45, 0x9b, 0x95, 0x23, 0x0f, 0xa5, 0xb9, 0x39, 0x24, 0xd7, 0x10, 0xda, 0x58, 0x03, 0xfa, 178 | 0x36, 0xf2, 0x18, 0xfc, 0x27, 0xc8, 0x0d, 0xf8, 0xb9, 0xe8, 0x0c, 0x1b, 0x58, 0xf6, 0xf2, 0xe8, 179 | 0xfa, 0x42, 0x97, 0xae, 0xc1, 0x00, 0xae, 0xef, 0x20, 0x1c, 0x5b, 0x91, 0x15, 0xe0, 0x2f, 0xb9, 180 | 0x1f, 0x1e, 0xca, 0x8c, 0x66, 0xc5, 0x6f, 0x51, 0x6b, 0x39, 0xec, 0xed, 0xc4, 0xc3, 0xec, 0x1e, 181 | 0xad, 0x5f, 0x21, 0x9a, 0xc4, 0x4d, 0xad, 0x4b, 0x67, 0xbd, 0x9a, 0x5a, 0x4f, 0xd5, 0x3e, 0x84, 182 | 0x65, 0xe1, 0x87, 0x9f, 0x3e, 0x6a, 0x55, 0xd5, 0xa5, 0x6f, 0x3f, 0xf1, 0xf6, 0x2f, 0x00, 0x00, 183 | 0xff, 0xff, 0x48, 0x84, 0x85, 0x75, 0xd2, 0x01, 0x00, 0x00, 184 | } 185 | -------------------------------------------------------------------------------- /lib/util/test.proto: -------------------------------------------------------------------------------- 1 | /* test.proto: this creates a protobuf fixture to run tests against 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | syntax = "proto3"; 11 | package Test; 12 | option go_package = ".;util"; 13 | 14 | message SubFixture { 15 | string A = 1; 16 | string B = 2; 17 | } 18 | 19 | message Fixture { 20 | bool Boolean = 1; 21 | uint32 UInt = 2; 22 | repeated string Slice = 3; 23 | map Map = 4; 24 | SubFixture Sub = 5; 25 | repeated SubFixture SliceSub = 6; 26 | map MapSub = 7; 27 | } -------------------------------------------------------------------------------- /lib/util/util_test.go: -------------------------------------------------------------------------------- 1 | //go:generate protoc -I ../../core/proto/src -I . --gogo_out=grpc:. test.proto 2 | 3 | package util 4 | 5 | import ( 6 | "reflect" 7 | "testing" 8 | ) 9 | 10 | var n1 = &Fixture{ 11 | Boolean: true, 12 | UInt: 42, 13 | Slice: []string{"this", "is", "a", "test"}, 14 | Map: map[string]uint32{"one": 1, "two": 2, "three": 3}, 15 | Sub: &SubFixture{A: "A", B: "B"}, 16 | SliceSub: []*SubFixture{{A: "A", B: "B"}, {A: "C", B: "D"}}, 17 | MapSub: map[uint32]*SubFixture{ 18 | 7: { 19 | A: "lucky", 20 | B: "prime", 21 | }, 22 | 13: { 23 | A: "unlucky", 24 | B: "also prime", 25 | }, 26 | 42: { 27 | A: "meaning", 28 | B: "life", 29 | }, 30 | }, 31 | } 32 | var n2 = &Fixture{ 33 | Boolean: true, 34 | UInt: 43, 35 | Slice: []string{"this", "is", "the", "test"}, 36 | Map: map[string]uint32{"one": 1, "two": 3, "four": 4}, 37 | Sub: &SubFixture{A: "A", B: "C"}, 38 | SliceSub: []*SubFixture{{A: "A", B: "D"}, {A: "C", B: "D"}}, 39 | MapSub: map[uint32]*SubFixture{ 40 | 7: { 41 | A: "lucky", 42 | B: "prime", 43 | }, 44 | 13: { 45 | A: "very unlucky", 46 | B: "also prime", 47 | }, 48 | 11: { 49 | A: "also lucky", 50 | B: "also prime", 51 | }, 52 | }, 53 | } 54 | 55 | func TestDiff(t *testing.T) { 56 | 57 | d, e := MessageDiff(n1, n2, "") 58 | if e != nil { 59 | t.Error(e) 60 | } 61 | t.Logf("diff: %v", d) 62 | s := []string{ 63 | "/UInt", 64 | "/Slice/2", 65 | "/Map/two", 66 | "/Map/three", 67 | "/Map/four", 68 | "/Sub/B", 69 | "/SliceSub/0/B", 70 | "/MapSub/13/A", 71 | "/MapSub/42", 72 | "/MapSub/11", 73 | } 74 | if !reflect.DeepEqual(d, s) { 75 | t.Error("Incorrect diff") 76 | } 77 | } 78 | 79 | func TestResolveURL(t *testing.T) { 80 | // do some lookups on n1 81 | tests := [][2]string{ 82 | {"/UInt", "42"}, 83 | {"/Slice/2", "a"}, 84 | {"/Map/three", "3"}, 85 | {"/Sub/B", "B"}, 86 | {"/SliceSub/0/A", "A"}, 87 | {"/MapSub/13/A", "unlucky"}, 88 | } 89 | for _, test := range tests { 90 | test := test 91 | t.Run(test[0], func(t *testing.T) { 92 | v, e := ResolveURL(test[0], reflect.ValueOf(n1)) 93 | if e != nil { 94 | t.Errorf("Unexected error: %v", e) 95 | } 96 | if ValueToString(v) != test[1] { 97 | t.Errorf("Result mismatch: %s != %s", ValueToString(v), test[1]) 98 | } 99 | }) 100 | } 101 | } 102 | 103 | func TestResolveOrMakeURL(t *testing.T) { 104 | t.Run("map lookup", func(t *testing.T) { 105 | nv := reflect.ValueOf(n1) 106 | v, e := ResolveOrMakeURL("/MapSub/99/A", nv) 107 | if e != nil { 108 | t.Errorf("unexpected ResolvOrMakeURL failure: %v", e) 109 | return 110 | } 111 | v.SetString("testing") 112 | v2, e := ResolveURL("/MapSub/99/A", nv) 113 | if e != nil { 114 | t.Errorf("unexpected ResolveURL failure: %v", e) 115 | return 116 | } 117 | if v2.String() != v.String() { 118 | t.Errorf("second lookup failed: %s != %s", v2.String(), v.String()) 119 | } 120 | }) 121 | t.Run("nil map lookup", func(t *testing.T) { 122 | n := &Fixture{} 123 | nv := reflect.ValueOf(n) 124 | v, e := ResolveOrMakeURL("/MapSub/99/A", nv) 125 | if e != nil { 126 | t.Errorf("unexpected ResolvOrMakeURL failure: %v", e) 127 | return 128 | } 129 | v.SetString("testing") 130 | v2, e := ResolveURL("/MapSub/99/A", nv) 131 | if e != nil { 132 | t.Errorf("unexpected ResolveURL failure: %v", e) 133 | return 134 | } 135 | if v2.String() != v.String() { 136 | t.Errorf("second lookup failed: %s != %s", v2.String(), v.String()) 137 | } 138 | }) 139 | t.Run("new map entry", func(t *testing.T) { 140 | n := &Fixture{} 141 | nv := reflect.ValueOf(n) 142 | v, e := ResolveOrMakeURL("/MapSub/99", nv) 143 | if e != nil { 144 | t.Errorf("unexpected ResolvOrMakeURL failure: %v", e) 145 | return 146 | } 147 | sf := SubFixture{A: "eh", B: "bee"} 148 | sfv := reflect.ValueOf(sf) 149 | v.Set(sfv) 150 | v2, e := ResolveURL("/MapSub/99", nv) 151 | if e != nil { 152 | t.Errorf("unexpected ResolveURL failure: %v", e) 153 | return 154 | } 155 | if !reflect.DeepEqual(sf, v2.Interface()) { 156 | t.Errorf("second lookup failed: %s != %s", v2.String(), sfv.String()) 157 | } 158 | }) 159 | t.Run("nil slice lookup", func(t *testing.T) { 160 | n := &Fixture{} 161 | nv := reflect.ValueOf(n) 162 | v, e := ResolveOrMakeURL("/SliceSub/1/A", nv) 163 | if e != nil { 164 | t.Errorf("unexpected ResolvOrMakeURL failure: %v", e) 165 | return 166 | } 167 | v.SetString("testing") 168 | v2, e := ResolveURL("/SliceSub/1/A", nv) 169 | if e != nil { 170 | t.Errorf("unexpected ResolveURL failure: %v", e) 171 | return 172 | } 173 | if v2.String() != v.String() { 174 | t.Errorf("second lookup failed: %s != %s", v2.String(), v.String()) 175 | } 176 | }) 177 | } 178 | 179 | func TestURLShift(t *testing.T) { 180 | type test struct { 181 | url string 182 | root string 183 | sub string 184 | } 185 | tests := []test{ 186 | { 187 | url: "This/is/a/test", 188 | root: "This", 189 | sub: "is/a/test", 190 | }, 191 | { 192 | url: "/This/is/a/test", 193 | root: "This", 194 | sub: "is/a/test", 195 | }, 196 | { 197 | url: "", 198 | root: "", 199 | sub: "", 200 | }, 201 | { 202 | url: "test", 203 | root: "test", 204 | sub: "", 205 | }, 206 | { 207 | url: "test/", 208 | root: "test", 209 | sub: "", 210 | }, 211 | } 212 | for _, v := range tests { 213 | t.Run(v.url, 214 | func(t *testing.T) { 215 | root, sub := URLShift(v.url) 216 | t.Logf("url: %s, root: %s, sub: %s", v.url, root, sub) 217 | if root != v.root { 218 | t.Errorf("url: %s, root: %s != %s", v.url, root, v.root) 219 | } 220 | if sub != v.sub { 221 | t.Errorf("url: %s, sub: %s != %s", v.url, sub, v.sub) 222 | } 223 | }) 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /modules/README.md: -------------------------------------------------------------------------------- 1 | # What is a Kraken Module? 2 | - A Kraken module is an add-on to the core Kraken program. 3 | - Modules speak to Kraken through discoveries and mutations. 4 | - A discovery occurs when a module determines that a change has occurred. The module passes that change along to Kraken so that it can keep track of what's going on. 5 | - A mutation is a desired change which corresponds with an edge in the state mutation graph. Mutations are sent by Kraken to modules. The module is responsible for performing a task based on the received mutation. 6 | 7 | # Module File Tree 8 | An example file tree is shown below: 9 | - kraken 10 | - modules 11 | - module_name 12 | - module_name.go 13 | - proto 14 | - module_name.proto 15 | - module_name.pb.go (generated) 16 | 17 | The module_name.go and module_name.proto are both written by the programmer of the module. The module_name.pb.go file is generated based on the module_name.proto file. 18 | 19 | # Writing a Kraken Module 20 | For ease of documentation, it will be assumed that the module being worked on is `Ipmipower`. When starting work on a module, we should start by creating an object for that module, in this case called `Ipmipower`. That object will implement some number of the interfaces listed below: 21 | 22 | # Interfaces 23 | ## Module 24 | - Must be implemented 25 | ### Required Methods 26 | - `func (p *Ipmipower) Name() string` 27 | - returns the name of the module 28 | 29 | ## ModuleSelfService 30 | - Must be implemented 31 | ### Required Methods 32 | - `func (p *Ipmipower) Entry()` 33 | - `Entry()` is the start point of the module 34 | - `Entry()` is where the main portion of the module code will reside 35 | - `func (p *Ipmipower) Init(api lib.ModuleAPIClient)` 36 | - `Init()` runs before `Entry()` is called 37 | - Initializes the member variables of the module struct 38 | - `func (p *Ipmipower) Stop()` 39 | - `Stop()` should ensure clean exit of the module 40 | 41 | ## ModuleWithConfig 42 | - Should be implemented if the module is to allow any customizability 43 | ### Required Methods 44 | - `func (*Ipmipower) NewConfig() proto.Message` 45 | - Returns the default configuration 46 | - `func (p *Ipmipower) UpdateConfig(cfg proto.Message) (e error)` 47 | - Writes a new module configuration 48 | - Ensures config updates take effect in the running module 49 | - `func (p *Ipmipower) ConfigURL() string` 50 | - Returns the configuration URL (a unique protobuf url string) 51 | ### Required Variables 52 | - `var cfg *proto.IpmipowerConfig` 53 | - `cfg` stores the module configuration 54 | - `*proto.IpmipowerConfig` is defined in the ipmipower.proto file 55 | 56 | ## ModuleWithDiscovery 57 | - Should be implemented if the module is to communicate discoveries to Kraken 58 | ### Required Methods 59 | - `func (p *Ipmipower) SetDiscoveryChan(c chan<- lib.Event)` 60 | - Sets the channel on which discoveries will be sent 61 | ### Required Variables 62 | - `var dchan chan<- lib.Event` 63 | - The channel through which discoveries are sent by the module 64 | 65 | ## ModuleWithMutations 66 | - Should be implemented if the module is to receive notification of mutations 67 | ### Required Methods 68 | - `func (p *Ipmipower) SetMutationChan(c <-chan lib.Event)` 69 | - Sets the channel on which mutations will be received 70 | ### Required Variables 71 | - `var mchan <-chan lib.Event` 72 | - The channel through which mutations are received by the module 73 | 74 | # Module Registration 75 | - There are certain things that must be registered with Kraken in order for proper communication to occur 76 | - `core.Registry.RegisterModule(&Ipmipower)` 77 | - Required for every module 78 | - `core.Registry.RegisterServiceInstance(&Ipmipower, d map[string]lib.ServiceInstance)` 79 | - Required for every module 80 | - `core.Registry.RegisterDiscoverable(&Ipmipower, discoverables)` 81 | - Required if the module implements the `ModuleWithDiscovery` interface 82 | - `discoverables` is of type `map[string]map[string]reflect.Value` 83 | - `core.Registry.RegisterMutations(&Ipmipower, mutations)` 84 | - Required if the module implements the `ModuleWithMutations` interface 85 | - `mutations` is of type `map[string]lib.StateMutation` 86 | 87 | # The `MutationEvent` Object 88 | - There are two `Node` member variables in the `MutationEvent` struct: `NodeCfg` and `NodeDsc` 89 | - `NodeCfg` contains complete information about the desired configuration of the node 90 | - `NodeDsc` contains incomplete information about the current configuration of the node 91 | 92 | # Logging 93 | - Logging is the preferred method to indicate failure 94 | - May be performed using `&Ipmitool.api.Logf(loggingLevel, errorMessage)` 95 | - `loggingLevel` is of type `level LoggerLevel` 96 | - `errorMessage` is of type `fmt string` 97 | 98 | # How do I add my module to the run list? 99 | - Kraken must be rebuilt each time you would like to modify the module run list 100 | - Kraken modules are added to the run list through a config file. A Kraken binary is then built using this config file. 101 | - The following command builds Kraken using `config_file` for its configuration: `go run kraken-build.go -config config/config_file`. If the target you are building to already exists you may use the `-force` flag to force a rebuild. 102 | - dynamic module loading will likely be added as a feature in the future 103 | -------------------------------------------------------------------------------- /modules/restapi/restapi.proto: -------------------------------------------------------------------------------- 1 | /* restapi.proto: describes the RestAPIConfig object 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | syntax = "proto3"; 11 | package restapi; 12 | option go_package = ".;restapi"; 13 | 14 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 15 | option (gogoproto.marshaler_all) = true; 16 | option (gogoproto.unmarshaler_all) = true; 17 | option (gogoproto.sizer_all) = true; 18 | option (gogoproto.goproto_registration) = true; 19 | option (gogoproto.messagename_all) = true; 20 | 21 | message Config { 22 | string addr = 1; 23 | int32 port = 2; 24 | } -------------------------------------------------------------------------------- /modules/websocket/README.md: -------------------------------------------------------------------------------- 1 | # Websocket Module 2 | The websocket module allows for a client to subscribe to a stream of kraken events. Currently the Module supports subscribing to these events: 3 | 4 | - STATE_CHANGE 5 | - STATE_MUTATION 6 | - DISCOVERY 7 | 8 | ## How to use websocket with Kraken 9 | These steps are assuming that Kraken is running and is built with the websocket module. It also assumes that both the restapi module and websocket module are in RUN state. 10 | 11 | 1. Retrieve the websocket information from the restapi by making a GET request to `http://{RESTAPI_IP}:{RESTAPI_PORT}/ws` 12 | - This will return an object of this schema: 13 | ```JSON 14 | { 15 | "websocket": { 16 | "host": "{WEBSOCKET_IP}", 17 | "port": "{WEBSOCKET_PORT}", 18 | "url": "{WEBSOCKET_URL}" 19 | } 20 | } 21 | ``` 22 | 23 | 2. You can now open a websocket with the information provided. Here's an example in javascript with fetch and WebSocket: 24 | ```javascript 25 | fetch("http://{RESTAPI_IP}:{RESTAPI_PORT}/ws") 26 | .then(resp => resp.json()) 27 | .then(json => { 28 | const wsurl = `ws://${json.websocket.host}:${json.websocket.port}${json.websocket.url}` 29 | websocket = new WebSocket(wsurl) 30 | ``` 31 | 32 | 3. Once a websocket connection is established you can send a subscription request to subscribe to an event stream. Here's another javascript example: 33 | ```javascript 34 | this.websocket.send("{ \"command\": \"SUBSCRIBE\", \"type\": \"STATE_CHANGE\" }") 35 | this.websocket.send("{ \"command\": \"SUBSCRIBE\", \"type\": \"STATE_MUTATION\" }") 36 | this.websocket.send("{ \"command\": \"SUBSCRIBE\", \"type\": \"DISCOVERY\" }") 37 | ``` -------------------------------------------------------------------------------- /modules/websocket/websocket.proto: -------------------------------------------------------------------------------- 1 | /* restapi.proto: describes the RestAPIConfig object 2 | * 3 | * Author: J. Lowell Wofford 4 | * 5 | * This software is open source software available under the BSD-3 license. 6 | * Copyright (c) 2018, Triad National Security, LLC 7 | * See LICENSE file for details. 8 | */ 9 | 10 | syntax = "proto3"; 11 | package WebSocket; 12 | option go_package = ".;websocket"; 13 | 14 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 15 | option (gogoproto.marshaler_all) = true; 16 | option (gogoproto.unmarshaler_all) = true; 17 | option (gogoproto.sizer_all) = true; 18 | option (gogoproto.goproto_registration) = true; 19 | option (gogoproto.messagename_all) = true; 20 | 21 | message Config { 22 | int32 port = 2; 23 | string writeWait = 3; // Time allowed to write a message to the peer. 24 | string pongWait = 4; // Time allowed to read the next pong message from the peer. 25 | string pingPeriod = 5; // Send pings to peer with this period. Must be less than pongWait. 26 | int64 maxMessageSize = 6; // Maximum message size allowed from peer. 27 | } -------------------------------------------------------------------------------- /triad-notice.txt: -------------------------------------------------------------------------------- 1 | This software is open source software available under the BSD-3 license. 2 | 3 | Copyright (c) 2018, Triad National Security, LLC 4 | 5 | All rights reserved. 6 | 7 | This program was produced under U.S. Government contract 89233218CNA000001 for 8 | Los Alamos National Laboratory (LANL), which is operated by Triad National Security, 9 | LLC for the U.S. Department of Energy/National Nuclear Security Administration. 10 | 11 | All rights in the program are reserved by Triad National Security, LLC, and 12 | the U.S. Department of Energy/National Nuclear Security Administration. 13 | The Government is granted for itself and others acting on its behalf a nonexclusive, 14 | paid-up, irrevocable worldwide license in this material to reproduce, prepare 15 | derivative works, distribute copies to the public, perform publicly and display 16 | publicly, and to permit others to do so. 17 | 18 | This is open source software; you can redistribute it and/or modify it under the 19 | terms of the License. If software is modified to produce derivative works, such 20 | modified software should be clearly marked, so as not to confuse it with the 21 | version available from LANL. Full text of the License can be found in the LICENSE 22 | file in the main development branch of the repository. 23 | 24 | Additionally, redistribution and use in source and binary forms, with or without 25 | modification, are permitted provided that the following conditions are met: 26 | 1. Redistributions of source code must retain the above copyright notice, this 27 | list of conditions and the following disclaimer. 28 | 2. Redistributions in binary form must reproduce the above copyright notice, 29 | this list of conditions and the following disclaimer in the documentation and/or 30 | other materials provided with the distribution. 31 | 3. Neither the name of Triad National Security, LLC, Los Alamos National 32 | Laboratory, LANL, the U.S. Government, nor the names of its contributors may be 33 | used to endorse or promote products derived from this software without specific 34 | prior written permission. 35 | 36 | THIS SOFTWARE IS PROVIDED BY Triad National Security, LLC AND CONTRIBUTORS 37 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 38 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 39 | ARE DISCLAIMED. IN NO EVENT SHALL Triad National Security, LLC OR 40 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, 41 | OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 42 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 43 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 44 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 45 | IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 46 | OF SUCH DAMAGE. 47 | --------------------------------------------------------------------------------