├── .gitignore ├── LICENSE.txt ├── Makefile ├── README.md ├── cmd └── memex │ ├── atexit.go │ ├── extensions.go │ ├── fswatch.go │ ├── fswatch_test.go │ ├── indexdb.go │ ├── main.go │ ├── os.go │ ├── os_android.go │ ├── os_posix.go │ ├── os_windows.go │ ├── service-config.go │ ├── service-supervisor.go │ ├── service.go │ └── util.go ├── example-memexdir ├── foo │ ├── foo.sh │ └── memexservice.yml └── twitter │ ├── memexservice.yml │ └── twitter-credentials.ini.in ├── extension ├── extension.go ├── fs.go ├── test │ └── test.go └── time.go ├── go.mod ├── go.sum ├── service-api ├── ipc.go └── msg.go ├── services └── twitter │ ├── .gitignore │ ├── ioutil.go │ ├── main.go │ ├── tweeter │ ├── conversions.go │ ├── like.go │ ├── media.go │ ├── response.go │ ├── tweet.go │ ├── tweeter.go │ └── user.go │ └── twitter-credentials.ini.in └── version.txt /.gitignore: -------------------------------------------------------------------------------- 1 | .memex-index 2 | .DS_Store 3 | *credentials.ini 4 | 5 | /bin 6 | /_* 7 | /example-memexdir/twitter/tweets/ 8 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | ISC License 2 | 3 | Copyright (c) 2021, Rasmus Andersson 4 | 5 | Permission to use, copy, modify, and/or distribute this software for any 6 | purpose with or without fee is hereby granted, provided that the above 7 | copyright notice and this permission notice appear in all copies. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | VERSION := $(shell cat version.txt) 2 | BUILDTAG := \ 3 | $(shell [ -d .git ] && git rev-parse --short=10 HEAD 2>/dev/null || date '+src@%Y%m%d%H%M%S') 4 | 5 | BIN_MEMEX_SOURCES := \ 6 | cmd/memex/*.go \ 7 | service-api/*.go \ 8 | extension/*.go \ 9 | extension/*/*.go 10 | 11 | SERVICES_TWITTER_SOURCES := \ 12 | services/twitter/*.go \ 13 | services/twitter/tweeter/*.go 14 | 15 | ALL_SOURCES := $(BIN_MEMEX_SOURCES) $(SERVICES_TWITTER_SOURCES) 16 | 17 | TEST_SRC_FILES := \ 18 | $(shell find . -type f -name '*_test.go' -not -path '*/_*' -not -path './vendor/*') 19 | TEST_DIRS := $(sort $(dir $(TEST_SRC_FILES))) 20 | 21 | # ------------------------------------------------------------------------------------------ 22 | # products 23 | 24 | all: bin/memex services/twitter/twitter 25 | 26 | bin/memex: $(BIN_MEMEX_SOURCES) 27 | @echo "go build $@" 28 | @go build \ 29 | -ldflags="-X 'main.MEMEX_VERSION=$(VERSION)' -X 'main.MEMEX_BUILDTAG=$(BUILDTAG)'" \ 30 | -o "$@" ./cmd/memex/ 31 | 32 | services/twitter/twitter: $(SERVICES_TWITTER_SOURCES) 33 | @echo "go build $@" 34 | @go build -o "$@" ./services/twitter/ 35 | 36 | clean: 37 | rm -f bin/* 38 | 39 | .PHONY: all clean 40 | 41 | # ------------------------------------------------------------------------------------------ 42 | # development & maintenance 43 | 44 | fmt: 45 | @gofmt -l -w -s $(ALL_SOURCES) 46 | 47 | tidy: fmt 48 | go mod tidy 49 | 50 | test: 51 | @go test $(TEST_DIRS) 52 | 53 | 54 | # files that affect building but not the source. Used by dev-* targets 55 | WATCH_MISC_FILES := $(firstword $(MAKEFILE_LIST)) go.sum 56 | 57 | run-service: fmt bin/memex 58 | ./bin/memex -debug -D example-memexdir 59 | 60 | dev-service: 61 | @autorun -no-clear $(WATCH_MISC_FILES) $(DEV_CONFIG) $(SERVICE_SOURCES) -- \ 62 | "$(MAKE) run-service" 63 | 64 | .PHONY: fmt tidy test 65 | .PHONY: run-service dev-service 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rsms's memex 2 | 3 | Software for managing my digital information, like tweets. 4 | 5 | 6 | ## Usage 7 | 8 | First check out the source and build. You'll need Make and Go installed. 9 | 10 | ```sh 11 | git clone https://github.com/rsms/memex.git 12 | cd memex 13 | make 14 | ``` 15 | 16 | Then make your memex data directory by starting from a copy of the example directory. 17 | This directory can be anywhere you like and will hold your data and service configs. 18 | 19 | ```sh 20 | cp -a path-to-source-of/memex/example-memexdir ~/memex 21 | ``` 22 | 23 | You can also start with an empty directory, e.g. `mkdir ~/memex`. 24 | 25 | Next, export the path of your memex directory in your environment: 26 | 27 | ```sh 28 | export MEMEX_DIR=$HOME/memex 29 | # optional: add to you shell's init script: 30 | echo "export MEMEX_DIR=$MEMEX_DIR" \ 31 | >> ~/`[[ $SHELL == *"/zsh" ]] && echo .zshrc || echo .bashrc` 32 | ``` 33 | 34 | Finally, run the memex service: 35 | 36 | ```sh 37 | path-to-source-of/memex/bin/memex 38 | ``` 39 | 40 | The `memex` program is a service manager, much like an operating system service manager like 41 | systemd it manages services (processes). It takes care of restarting processes when they exit, 42 | start, restart or stop processes when their configuration files changes and collect all logs 43 | and outputs in one place. This makes it easy to "run a lot of processes" with a single command 44 | (`memex`), and similarly to stop them all (^C.) 45 | 46 | The `memex` program looks in your `MEMEX_DIR` for files called `memexservice.yml` which it treats 47 | as "service configurations". These YAML files describes what command to run as the service process, 48 | what arguments to pass it, its environment, it it's been temporarily disabled or not, and so on. 49 | 50 | Changes to these files are automatically detected and you don't need to restart memex when making 51 | changes to these "service config" files. 52 | 53 | Service processes are started with a working directory of the configuration file. 54 | I.e. `~/memex/foo/bar/memexservice.yml` with `start:./bar` will start a process `./bar` in 55 | the directory `~/memex/foo/bar`. 56 | 57 | See `example-memexdir/foo/memexservice.yml` for a full example and more details. 58 | -------------------------------------------------------------------------------- /cmd/memex/atexit.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "os/signal" 7 | "runtime/debug" 8 | "sync" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/rsms/go-log" 13 | ) 14 | 15 | type ExitHandler = func(context.Context) error 16 | 17 | var ( 18 | ExitCh chan struct{} // closes when all exit handlers have completed 19 | 20 | sigch chan os.Signal 21 | exitExitCode = 0 22 | exitHandlersMu sync.Mutex // protects exitHandlers 23 | exitHandlers []ExitHandler 24 | exitTimeouts = map[os.Signal]time.Duration{} 25 | exitSignals = []os.Signal{syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM} 26 | ) 27 | 28 | const defaultExitTimeout = 5 * time.Second 29 | 30 | func init() { 31 | ExitCh = make(chan struct{}) 32 | sigch = make(chan os.Signal, 1) 33 | 34 | for _, sig := range exitSignals { 35 | exitTimeouts[sig] = defaultExitTimeout 36 | } 37 | 38 | signal.Notify(sigch, exitSignals...) 39 | go func() { 40 | sig := <-sigch 41 | 42 | // reset signal handler so that a second signal has the default effect 43 | signal.Reset(exitSignals...) 44 | 45 | // log that we are shutting down 46 | log.Info("shutting down...") 47 | 48 | // create context for shutdown 49 | timeout, ok := exitTimeouts[sig] 50 | if !ok { 51 | timeout = defaultExitTimeout 52 | } 53 | ctx, cancel := context.WithTimeout(context.Background(), timeout) 54 | defer cancel() // just in case 55 | defer log.RootLogger.Sync() // sync logger after returning 56 | 57 | // Note: copy and don't lock to avoid deadlock in case a handler calls RegisterExitHandler 58 | handlers := exitHandlers[:] 59 | fnch := make(chan struct{}) // "done" signals 60 | exitCode := exitExitCode 61 | 62 | // invoke all shutdown handlers in goroutines 63 | for _, fn := range handlers { 64 | go func(fn ExitHandler) { 65 | defer func() { 66 | if r := recover(); r != nil { 67 | log.Error("panic in RegisterExitHandler function: %v\n", r) 68 | if log.RootLogger.Level <= log.LevelDebug { 69 | debug.PrintStack() 70 | } 71 | // cancel the shutdown context 72 | cancel() 73 | } 74 | }() 75 | 76 | // invoke handler and log error 77 | if err := fn(ctx); err != nil { 78 | if err != context.DeadlineExceeded && err != context.Canceled { 79 | log.Error("RegisterExitHandler function: %v", err) 80 | } 81 | // cancel the shutdown context 82 | cancel() 83 | } else { 84 | // signal to outer function that this handler has completed 85 | fnch <- struct{}{} 86 | } 87 | }(fn) 88 | } 89 | 90 | // wait for all shutdown handler goroutines to finish 91 | wait_loop: 92 | for range handlers { 93 | select { 94 | case <-fnch: // ok 95 | case <-ctx.Done(): 96 | // Context canceled 97 | if ctx.Err() == context.DeadlineExceeded { 98 | log.Warn("shutdown timeout (%s)", timeout) 99 | } 100 | if exitCode == 0 { 101 | exitCode = 1 102 | } 103 | break wait_loop 104 | } 105 | } 106 | 107 | // finished 108 | log.RootLogger.Sync() 109 | os.Exit(exitCode) 110 | }() 111 | } 112 | 113 | // Shutdown is like os.Exit but invokes shutdown handlers before exiting 114 | func Shutdown(exitCode int) { 115 | exitExitCode = exitCode 116 | close(sigch) 117 | <-ExitCh // never returns 118 | } 119 | 120 | func SetExitTimeout(timeout time.Duration, onlySignals ...os.Signal) { 121 | if onlySignals == nil { 122 | onlySignals = exitSignals 123 | } 124 | for _, sig := range onlySignals { 125 | if _, ok := exitTimeouts[sig]; ok { 126 | exitTimeouts[sig] = timeout 127 | } 128 | } 129 | } 130 | 131 | func GetExitTimeout(signal os.Signal) time.Duration { 132 | if signal != nil { 133 | if timeout, ok := exitTimeouts[signal]; ok { 134 | return timeout 135 | } 136 | } 137 | return defaultExitTimeout 138 | } 139 | 140 | // RegisterExitHandler adds a function to be called during program shutdown. 141 | // The following signatures are accepted: 142 | // func(context.Context)error 143 | // func(context.Context) 144 | // func()error 145 | // func() 146 | // 147 | // A panics inside a handler will cause the context to be cancelled and the 148 | // panic reported. I.e. a panic inside an exit handler is isolated to that handler 149 | // but "speeds up" shutdown. 150 | // 151 | func RegisterExitHandler(handlerFunc interface{}) { 152 | var fn ExitHandler 153 | if f, ok := handlerFunc.(ExitHandler); ok { 154 | fn = f 155 | } else if f, ok := handlerFunc.(func(context.Context)); ok { 156 | fn = func(ctx context.Context) error { 157 | // log.Info("atexit synthetic handler %p for %p invoked", fn, handlerFunc) 158 | f(ctx) 159 | return nil 160 | } 161 | } else if f, ok := handlerFunc.(func() error); ok { 162 | fn = func(_ context.Context) error { 163 | // log.Info("atexit synthetic handler %p for %p invoked", fn, handlerFunc) 164 | return f() 165 | } 166 | } else if f, ok := handlerFunc.(func()); ok { 167 | fn = func(_ context.Context) error { 168 | // log.Info("atexit synthetic handler %p for %p invoked", fn, handlerFunc) 169 | f() 170 | return nil 171 | } 172 | } else { 173 | panic("invalid handler signature (see RegisterExitHandler documentation)") 174 | } 175 | exitHandlersMu.Lock() 176 | defer exitHandlersMu.Unlock() 177 | exitHandlers = append(exitHandlers, fn) 178 | } 179 | -------------------------------------------------------------------------------- /cmd/memex/extensions.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "runtime/debug" 7 | "sync" 8 | "time" 9 | 10 | "github.com/rsms/go-log" 11 | "github.com/rsms/memex/extension" 12 | ) 13 | 14 | // ————————————————————————————————————————————————————————————————————————————————————— 15 | 16 | // implementation of extension.Extension 17 | type Extension struct { 18 | name string 19 | version string 20 | ctx context.Context 21 | 22 | // fields used only by supervisor; never exposed to the extension itself 23 | cancel context.CancelFunc // cancel function for ctx 24 | donech chan struct{} // closed when a extension's run function returns 25 | 26 | lazymu sync.Mutex // protects the following fields (lazy-initialized data) 27 | logger *log.Logger 28 | } 29 | 30 | // check conformance to extension.Extension 31 | var _ extension.Extension = &Extension{} 32 | 33 | func (si *Extension) Name() string { return si.name } 34 | func (si *Extension) Version() string { return si.version } 35 | func (si *Extension) Done() <-chan struct{} { return si.ctx.Done() } 36 | func (si *Extension) Context() context.Context { return si.ctx } 37 | 38 | func (si *Extension) Logger() *log.Logger { 39 | si.lazymu.Lock() 40 | defer si.lazymu.Unlock() 41 | if si.logger == nil { 42 | si.logger = log.SubLogger("[" + si.String() + "]") 43 | } 44 | return si.logger 45 | } 46 | 47 | func (si *Extension) WatchFiles(paths ...string) extension.FSWatcher { 48 | // TODO maybe share a watcher (Each FSWatcher uses considerable resources) 49 | w1, err := NewFSWatcher(si.ctx) 50 | if err != nil { 51 | // fails only when the underlying OS extension fails; we are generally in trouble. 52 | panic(err) 53 | } 54 | w := &ServiceFSWatcher{FSWatcher: *w1} 55 | for _, path := range paths { 56 | w.Add(path) 57 | } 58 | return w 59 | } 60 | 61 | func (si *Extension) String() string { 62 | return fmt.Sprintf("%s@%s", si.name, si.version) 63 | } 64 | 65 | // ————————————————————————————————————————————————————————————————————————————————————— 66 | 67 | type ServiceFSWatcher struct { 68 | FSWatcher 69 | } 70 | 71 | func (w *ServiceFSWatcher) Events() <-chan []FSEvent { 72 | return w.FSWatcher.Events 73 | } 74 | func (w *ServiceFSWatcher) Latency() time.Duration { 75 | return w.FSWatcher.Latency 76 | } 77 | func (w *ServiceFSWatcher) SetLatency(latency time.Duration) { 78 | w.FSWatcher.Latency = latency 79 | } 80 | 81 | // ————————————————————————————————————————————————————————————————————————————————————— 82 | 83 | type ExtSupervisor struct { 84 | runmu sync.RWMutex 85 | runmap map[string]*Extension 86 | log *log.Logger 87 | } 88 | 89 | func NewExtSupervisor(l *log.Logger) *ExtSupervisor { 90 | return &ExtSupervisor{ 91 | runmap: make(map[string]*Extension), 92 | log: l, 93 | } 94 | } 95 | 96 | // Start starts all extensions described in registry. 97 | func (s *ExtSupervisor) Start(ctx context.Context, registry map[string]extension.RunFunc) error { 98 | s.runmu.Lock() 99 | defer s.runmu.Unlock() 100 | for name, runf := range registry { 101 | s.startService(ctx, name, runf) 102 | } 103 | return nil 104 | } 105 | 106 | // Shutdown stops all extensions 107 | func (s *ExtSupervisor) Shutdown(shutdownCtx context.Context) error { 108 | // hold a lock during entire "stop all" process to prevent stopService or startService 109 | // calls from interfering. 110 | s.runmu.Lock() 111 | defer s.runmu.Unlock() 112 | return s.stopAllServices(shutdownCtx) 113 | } 114 | 115 | // startService creates a new extension Extension and runs it via runService in a new goroutine. 116 | // NOT THREAD SAFE: Caller must hold lock on s.runmu 117 | func (s *ExtSupervisor) startService(ctx context.Context, name string, runf extension.RunFunc) { 118 | extensionCtx, cancel := context.WithCancel(ctx) 119 | si := &Extension{ 120 | name: name, 121 | version: "1", // TODO 122 | ctx: extensionCtx, 123 | cancel: cancel, 124 | donech: make(chan struct{}), 125 | } 126 | s.log.Info("starting extension %s", si) 127 | s.runmap[name] = si 128 | 129 | // start the extension goroutine, which calls runf and blocks until it either: 130 | // - returns, 131 | // - panics (logs panic), or 132 | // - times out (si.ctx) 133 | go func() { 134 | func() { 135 | defer func() { 136 | if r := recover(); r != nil { 137 | s.log.Error("panic in extension %q: %v\n%s", si.name, r, debug.Stack()) 138 | } 139 | }() 140 | runf(si) 141 | select { 142 | case <-si.ctx.Done(): // ok 143 | s.log.Debug("extension %s exited", si) 144 | default: 145 | s.log.Warn("extension %s exited prematurely (did not wait for Done())", si) 146 | si.cancel() 147 | } 148 | }() 149 | 150 | // signal to supervisor that the extension has completed shutdown 151 | close(si.donech) 152 | }() 153 | } 154 | 155 | // stopService stops a specific extension. 156 | // Thread-safe. 157 | func (s *ExtSupervisor) stopService(ctx context.Context, name string) error { 158 | // retrieve and remove extension instance from runmap 159 | s.runmu.Lock() 160 | si := s.runmap[name] 161 | if si != nil { 162 | delete(s.runmap, name) 163 | } 164 | s.runmu.Unlock() 165 | if si == nil { 166 | return errorf("can not find extension %q", name) 167 | } 168 | 169 | // cancel the extension and await its shutdown 170 | si.cancel() 171 | select { 172 | case <-si.donech: 173 | // ok; extension shut down ok 174 | case <-ctx.Done(): 175 | // Context cancelled or timed out 176 | // NOTE: There is a possibility that an ill-behaving extension just keeps on truckin' here. 177 | return ctx.Err() 178 | } 179 | return nil 180 | } 181 | 182 | // stopService stops all extensions. 183 | // NOT THREAD SAFE: Caller must hold lock on s.runmu 184 | func (s *ExtSupervisor) stopAllServices(ctx context.Context) error { 185 | // cancel all extensions 186 | for _, si := range s.runmap { 187 | si.cancel() 188 | } 189 | 190 | // wait for extensions to stop 191 | var err error 192 | wait_loop: 193 | for _, si := range s.runmap { 194 | select { 195 | case <-si.donech: // ok; extension shut down in time 196 | case <-ctx.Done(): 197 | s.log.Warn("timeout while waiting for extension %s to shut down", si) 198 | err = ctx.Err() 199 | break wait_loop 200 | } 201 | } 202 | 203 | // clear runmap 204 | s.runmap = nil 205 | return err 206 | } 207 | -------------------------------------------------------------------------------- /cmd/memex/fswatch.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "sync/atomic" 6 | "time" 7 | 8 | "github.com/fsnotify/fsnotify" 9 | "github.com/rsms/memex/extension" 10 | ) 11 | 12 | // event op (bitmask) 13 | const ( 14 | FSEventCreate = extension.FSEventCreate 15 | FSEventWrite = extension.FSEventWrite 16 | FSEventRemove = extension.FSEventRemove 17 | FSEventRename = extension.FSEventRename 18 | FSEventChmod = extension.FSEventChmod 19 | ) 20 | 21 | type FSEvent = extension.FSEvent 22 | type FSEventFlags = extension.FSEventFlags 23 | 24 | // FSWatcher observes file system for changes. 25 | // A new FSWatcher starts in Resumed mode. 26 | type FSWatcher struct { 27 | Latency time.Duration // Time window for bundling changes together. Default: 100ms 28 | Events chan []FSEvent // Channel on which event batches are delivered 29 | Error error // After Events channel is closed, this indicates the reason 30 | 31 | w *fsnotify.Watcher 32 | start uint32 33 | ctx context.Context 34 | } 35 | 36 | func NewFSWatcher(ctx context.Context) (*FSWatcher, error) { 37 | w2, err := fsnotify.NewWatcher() 38 | if err != nil { 39 | // fails only when the underlying OS extension fails. 40 | // E.g. golang.org/x/sys/unix.{Kqueue,InotifyInit1} or syscall.CreateIoCompletionPort 41 | return nil, err 42 | } 43 | w := &FSWatcher{ 44 | Latency: 100 * time.Millisecond, 45 | Events: make(chan []FSEvent), 46 | w: w2, 47 | ctx: ctx, 48 | } 49 | return w, nil 50 | } 51 | 52 | // Add file or directory (non-recursively) to be watched. 53 | // If path is already watched nil is returned (duplicates ignored.) 54 | func (w *FSWatcher) Add(path string) error { 55 | err := w.w.Add(path) 56 | if err == nil && atomic.CompareAndSwapUint32(&w.start, 0, 1) { 57 | go w.runLoop() 58 | } 59 | return err 60 | } 61 | 62 | // Remove unregisters a file or directory that was previously registered with Add. 63 | // If path is not being watched an error is returned. 64 | func (w *FSWatcher) Remove(path string) error { 65 | return w.w.Remove(path) 66 | } 67 | 68 | // Stop begins the shutdown process of the watcher. Causes Run() to return. 69 | func (w *FSWatcher) Close() error { 70 | if atomic.CompareAndSwapUint32(&w.start, 0, 1) { 71 | // never started so w.Events would not close from runLoop exiting 72 | close(w.Events) 73 | } 74 | err := w.w.Close() 75 | if err == nil { 76 | err = w.Error 77 | } 78 | return err 79 | } 80 | 81 | func (w *FSWatcher) runLoop() { 82 | changeq := make(map[string]FSEventFlags) 83 | foreverDuration := time.Duration(0x7fffffffffffffff) 84 | flushTimer := time.NewTimer(foreverDuration) 85 | flushTimerActive := false 86 | var errCounter int 87 | 88 | defer close(w.Events) 89 | 90 | for { 91 | select { 92 | 93 | case <-w.ctx.Done(): 94 | w.Error = w.ctx.Err() 95 | return 96 | 97 | case <-flushTimer.C: 98 | if len(changeq) > 0 { 99 | // logd("[fswatcher] flush %+v", changeq) 100 | events := make([]FSEvent, 0, len(changeq)) 101 | for name, flags := range changeq { 102 | events = append(events, FSEvent{ 103 | Flags: flags, 104 | Name: name, 105 | }) 106 | } 107 | for _, ev := range events { 108 | delete(changeq, ev.Name) 109 | } 110 | w.Events <- events 111 | } 112 | flushTimer.Reset(foreverDuration) 113 | flushTimerActive = false 114 | 115 | case event, more := <-w.w.Events: 116 | if !more { 117 | // closed 118 | return 119 | } 120 | 121 | // logd("[fswatcher] event: %v %q", event.Op, event.Name) 122 | 123 | // reset error counter 124 | errCounter = 0 125 | 126 | // update event mask 127 | prev := changeq[event.Name] 128 | next := FSEventFlags(event.Op) 129 | if prev&FSEventCreate != 0 && next&FSEventRemove != 0 && next&FSEventCreate == 0 { 130 | // was created and is now removed 131 | next = ((prev | next) &^ FSEventCreate) &^ FSEventWrite 132 | } else if prev&FSEventRemove != 0 && next&FSEventCreate != 0 && next&FSEventRemove == 0 { 133 | // was removed and is now created -> modified 134 | next = (prev | FSEventWrite) &^ FSEventRemove 135 | } else { 136 | next = prev | next 137 | } 138 | changeq[event.Name] = next 139 | 140 | if !flushTimerActive { 141 | flushTimerActive = true 142 | flushTimer.Reset(w.Latency) 143 | } 144 | 145 | case err := <-w.w.Errors: 146 | w.Error = err 147 | errCounter++ 148 | if errCounter > 10 { 149 | // There were many errors without any file events. 150 | // Close the watcher and return as there may be unrecoverable 151 | w.Close() 152 | return 153 | } 154 | 155 | } // select 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /cmd/memex/fswatch_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/fsnotify/fsnotify" 7 | "github.com/rsms/go-testutil" 8 | ) 9 | 10 | func TestFSEvent(t *testing.T) { 11 | assert := testutil.NewAssert(t) 12 | 13 | assert.Eq("FSEventCreate == fsnotify.Create", int(FSEventCreate), int(fsnotify.Create)) 14 | assert.Eq("FSEventWrite == fsnotify.Write", int(FSEventWrite), int(fsnotify.Write)) 15 | assert.Eq("FSEventRemove == fsnotify.Remove", int(FSEventRemove), int(fsnotify.Remove)) 16 | assert.Eq("FSEventRename == fsnotify.Rename", int(FSEventRename), int(fsnotify.Rename)) 17 | assert.Eq("FSEventChmod == fsnotify.Chmod", int(FSEventChmod), int(fsnotify.Chmod)) 18 | } 19 | -------------------------------------------------------------------------------- /cmd/memex/indexdb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | // "bytes" 5 | // "encoding/gob" 6 | // "time" 7 | 8 | "github.com/rsms/go-log" 9 | "github.com/syndtr/goleveldb/leveldb" 10 | ) 11 | 12 | type IndexDB struct { 13 | db *leveldb.DB 14 | logger *log.Logger 15 | } 16 | 17 | type DocType struct { 18 | Id string `json:"id"` 19 | ParentId string `json:"parent_id"` 20 | Name string `json:"name"` 21 | } 22 | 23 | type Doc struct { 24 | Id string `json:"id"` 25 | Version string `json:"version"` 26 | Type string `json:"type"` 27 | Tags []string `json:"tags"` 28 | Body interface{} `json:"body"` 29 | } 30 | 31 | func IndexDBOpen(path string, logger *log.Logger) (*IndexDB, error) { 32 | db, err := leveldb.OpenFile(path, nil) 33 | return &IndexDB{ 34 | db: db, 35 | logger: logger, 36 | }, err 37 | } 38 | 39 | func (db *IndexDB) Close() error { 40 | if db.db == nil { 41 | return nil 42 | } 43 | err := db.db.Close() 44 | db.db = nil 45 | return err 46 | } 47 | 48 | func (db *IndexDB) PutDoc(doc ...*Doc) error { 49 | // TODO 50 | // consider returning conflicts 51 | return nil 52 | } 53 | 54 | func (db *IndexDB) DefineTypes(types ...DocType) error { 55 | batch := new(leveldb.Batch) 56 | for _, dt := range types { 57 | key := []byte("dtype:" + dt.Id) 58 | val := []byte(dt.Name + "\n" + dt.ParentId) 59 | batch.Put(key, val) 60 | } 61 | return db.db.Write(batch, nil) 62 | } 63 | -------------------------------------------------------------------------------- /cmd/memex/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | 10 | "github.com/rsms/go-log" 11 | "github.com/rsms/memex/extension" 12 | // _ "github.com/rsms/memex/extension/webbrowser" 13 | // _ "github.com/rsms/memex/extension/test" 14 | ) 15 | 16 | var MEMEX_VERSION string = "0.0.0" // set at compile time 17 | var MEMEX_BUILDTAG string = "src" // set at compile time 18 | var MEMEX_ROOT string // root file directory for the memex program & its files 19 | var MEMEX_DIR string // root file directory for the memex repo 20 | 21 | // program-wide index database 22 | var indexdb *IndexDB 23 | 24 | func main() { 25 | defer log.Sync() 26 | var err error 27 | 28 | // parse CLI flags 29 | optMemexDir := flag.String("D", "", "Set MEMEX_DIR (overrides environment var)") 30 | optVersion := flag.Bool("version", false, "Print version and exit") 31 | optDebug := flag.Bool("debug", false, "Enable debug mode") 32 | flag.Parse() 33 | 34 | // want version? 35 | if *optVersion { 36 | fmt.Printf("memex version %s build %s\n", MEMEX_VERSION, MEMEX_BUILDTAG) 37 | os.Exit(0) 38 | } 39 | 40 | // update log level based on CLI options 41 | if *optDebug { 42 | log.RootLogger.Level = log.LevelDebug 43 | log.RootLogger.EnableFeatures(log.FSync) 44 | } else { 45 | log.RootLogger.EnableFeatures(log.FSyncError) 46 | } 47 | 48 | // log version 49 | log.Info("memex v%s+%s\n", MEMEX_VERSION, MEMEX_BUILDTAG) 50 | 51 | // MEMEX_ROOT = dirname(dirname(argv[0])) (since {MEMEX_ROOT}/bin/memex) 52 | MEMEX_ROOT = filepath.Dir(filepath.Dir(mustRealPath(os.Args[0]))) 53 | 54 | // MEMEX_DIR 55 | MEMEX_DIR = *optMemexDir 56 | crashOnErr(init_MEMEX_DIR()) 57 | log.Debug("MEMEX_ROOT=%q", MEMEX_ROOT) 58 | log.Debug("MEMEX_DIR=%q", MEMEX_DIR) 59 | 60 | // update env 61 | os.Setenv("MEMEX_ROOT", MEMEX_ROOT) 62 | os.Setenv("MEMEX_DIR", MEMEX_DIR) 63 | os.Setenv("MEMEX_VERSION", MEMEX_VERSION) 64 | 65 | // open index database 66 | indexdb, err = IndexDBOpen( 67 | filepath.Join(MEMEX_DIR, ".memex-index"), 68 | log.SubLogger("[indexdb]")) 69 | crashOnErr(err) 70 | RegisterExitHandler(indexdb.Close) 71 | 72 | // change working directory to MEMEX_DIR 73 | crashOnErr(os.Chdir(MEMEX_DIR)) 74 | 75 | // start extension supervisor 76 | if len(extension.Registry) > 0 { 77 | extSupervisor := NewExtSupervisor(log.SubLogger("[extsv]")) 78 | RegisterExitHandler(extSupervisor.Shutdown) 79 | if err := extSupervisor.Start(context.Background(), extension.Registry); err != nil { 80 | log.Error("extension supervisor: %v", err) 81 | } 82 | } 83 | 84 | // start service supervisor 85 | srvSupervisor := NewServiceSupervisor(log.RootLogger, MEMEX_DIR) 86 | RegisterExitHandler(srvSupervisor.Shutdown) 87 | if err := srvSupervisor.Start(context.Background()); err != nil { 88 | log.Error("service supervisor: %v", err) 89 | } 90 | 91 | // channel closes when all exit handlers have completed 92 | <-ExitCh 93 | } 94 | 95 | // init_MEMEX_DIR initializes MEMEX_DIR 96 | func init_MEMEX_DIR() error { 97 | var err error 98 | if MEMEX_DIR == "" { 99 | MEMEX_DIR = os.Getenv("MEMEX_DIR") 100 | if MEMEX_DIR == "" { 101 | return errorf("MEMEX_DIR not set in environment nor passed as a CLI option") 102 | } 103 | } 104 | // realpath 105 | MEMEX_DIR, err = filepath.Abs(MEMEX_DIR) 106 | if err != nil { 107 | return err 108 | } 109 | // make sure it's a directory 110 | if err = isdir(MEMEX_DIR); err != nil { 111 | return errorf("MEMEX_DIR: %v", err) 112 | } 113 | // SERVICE_DIR = filepath.Join(MEMEX_DIR, "services") 114 | // if err = os.MkdirAll(SERVICE_DIR, 0700); err != nil { 115 | // return err 116 | // } 117 | return nil 118 | } 119 | 120 | func mustRealPath(path string) string { 121 | abspath, err := filepath.Abs(path) 122 | if err == nil { 123 | abspath, err = filepath.EvalSymlinks(abspath) 124 | } 125 | if err != nil { 126 | panic(err) 127 | } 128 | return abspath 129 | } 130 | 131 | func crashOnErr(err error) { 132 | if err != nil { 133 | fatalf(err) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /cmd/memex/os.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // func OSShellArgs(command string) []string 4 | -------------------------------------------------------------------------------- /cmd/memex/os_android.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func OSShellArgs(command string) []string { 4 | return []string{"/system/bin/sh", "-c", command} 5 | } 6 | -------------------------------------------------------------------------------- /cmd/memex/os_posix.go: -------------------------------------------------------------------------------- 1 | // +build !android 2 | 3 | package main 4 | 5 | import ( 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func OSShellArgs(command string) []string { 11 | SHELL := strings.TrimSpace(os.Getenv("SHELL")) 12 | if SHELL == "" || SHELL[0] != '/' { 13 | SHELL = "/bin/sh" 14 | } 15 | return []string{SHELL, "-c", command} 16 | } 17 | -------------------------------------------------------------------------------- /cmd/memex/os_windows.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "strings" 6 | ) 7 | 8 | func OSShellArgs(command string) []string { 9 | args := []string{os.Getenv("comspec")} 10 | if len(args[0]) == 0 { 11 | args[0] = "cmd.exe" 12 | } 13 | if strings.Contains(args[0], "cmd.exe") { 14 | command = strings.ReplaceAll(command, "\"", "\\\"") 15 | args = append(args, "/d", "/s", "/c", "\""+command+"\"") 16 | } else { 17 | args = append(args, "-c", command) 18 | } 19 | return args 20 | } 21 | -------------------------------------------------------------------------------- /cmd/memex/service-config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "regexp" 8 | "strings" 9 | "syscall" 10 | "time" 11 | 12 | "gopkg.in/yaml.v2" 13 | ) 14 | 15 | type ServiceConfig struct { 16 | Start interface{} `yaml:",flow"` // string or []interface{} 17 | RestartLimitInterval time.Duration `yaml:"restart-limit-interval"` // e.g. 10s, 4.5s, 123ms 18 | Chroot string 19 | Disabled bool 20 | Env map[string]interface{} 21 | 22 | _start []string // parsed Start 23 | _env map[string]string // preprocessed Env with expanded $(NAME)s 24 | _filename string // non-empty when loaded from file 25 | } 26 | 27 | func ServiceConfigLoad(filename string) (*ServiceConfig, error) { 28 | s := &ServiceConfig{} 29 | return s, s.LoadFile(filename) 30 | } 31 | 32 | func (c *ServiceConfig) GetStartArgs() []string { return c._start } 33 | func (c *ServiceConfig) GetSourceFilename() string { return c._filename } 34 | func (c *ServiceConfig) GetEnvEncodedList() []string { 35 | if len(c._env) == 0 { 36 | return nil 37 | } 38 | pairs := make([]string, len(c.Env)) 39 | i := 0 40 | for k, v := range c._env { 41 | pairs[i] = k + "=" + v 42 | i++ 43 | } 44 | return pairs 45 | } 46 | 47 | // IsEqual compares the recevier with other and returns true if they are equivalent 48 | func (c *ServiceConfig) IsEqual(other *ServiceConfig) bool { 49 | if c.RestartLimitInterval != other.RestartLimitInterval || 50 | c.Chroot != other.Chroot || 51 | c.Disabled != other.Disabled || 52 | len(c.GetStartArgs()) != len(other.GetStartArgs()) || 53 | len(c.Env) != len(other.Env) { 54 | return false 55 | } 56 | for i := 0; i < len(c.GetStartArgs()); i++ { 57 | if c.GetStartArgs()[i] != other.GetStartArgs()[i] { 58 | return false 59 | } 60 | } 61 | for k, v := range c.Env { 62 | if other.Env[k] != v { 63 | return false 64 | } 65 | } 66 | for k, v := range other.Env { 67 | if c.Env[k] != v { 68 | return false 69 | } 70 | } 71 | return true 72 | } 73 | 74 | func (c *ServiceConfig) LoadFile(filename string) error { 75 | f, err := os.Open(filename) 76 | if err != nil { 77 | return err 78 | } 79 | defer f.Close() 80 | return c.Load(f, filename) 81 | } 82 | 83 | func (c *ServiceConfig) YAMLString() string { 84 | data, _ := yaml.Marshal(c) 85 | return string(data) 86 | } 87 | 88 | func (c *ServiceConfig) Load(r io.Reader, filename string) error { 89 | c._filename = filename 90 | 91 | c.RestartLimitInterval = -1 92 | err := yaml.NewDecoder(r).Decode(c) 93 | if err != nil { 94 | return err 95 | } 96 | 97 | switch cmd := c.Start.(type) { 98 | 99 | case string: 100 | c._start = OSShellArgs(cmd) 101 | 102 | case []interface{}: 103 | if len(cmd) == 0 { 104 | return errorf("start is an empty list; missing program") 105 | } 106 | args := make([]string, len(cmd)) 107 | for i, any := range cmd { 108 | switch v := any.(type) { 109 | case string: 110 | args[i] = v 111 | default: 112 | args[i] = fmt.Sprintf("%v", any) 113 | } 114 | } 115 | c._start = args 116 | 117 | default: 118 | return errorf("start must be string or lsit of strings (got %T)", cmd) 119 | 120 | } // switch 121 | 122 | // preprocess env 123 | c._env = make(map[string]string, len(c.Env)) 124 | for k, v := range c.Env { 125 | c._env[k] = fmt.Sprint(v) 126 | } 127 | 128 | // replace $(NAME) or $(NAME:default) with env vars 129 | re := regexp.MustCompile(`\$\([A-Za-z0-9_][^):]*(?::[^)]+|)\)`) 130 | var envNotFound []string 131 | subEnvVars := func(str string) string { 132 | return re.ReplaceAllStringFunc(str, func(s string) string { 133 | name := s[2 : len(s)-1] 134 | // see if there'a a default value 135 | defaultValue := "" 136 | defaultIndex := strings.IndexByte(name, ':') 137 | if defaultIndex >= 0 { 138 | defaultValue = name[defaultIndex+1:] 139 | name = name[:defaultIndex] 140 | } 141 | name = strings.TrimSpace(name) 142 | value, found := c._env[name] 143 | if found { 144 | return value 145 | } 146 | value, found = syscall.Getenv(name) 147 | if !found { 148 | if defaultIndex >= 0 { 149 | value = defaultValue 150 | } else { 151 | envNotFound = append(envNotFound, name) 152 | } 153 | } 154 | return value 155 | }) 156 | } 157 | 158 | // subEnvVars of env 159 | for k, v := range c._env { 160 | c._env[k] = subEnvVars(v) 161 | } 162 | 163 | // subEnvVars of args 164 | for i := 0; i < len(c._start); i++ { 165 | c._start[i] = subEnvVars(c._start[i]) 166 | } 167 | 168 | if len(envNotFound) > 0 { 169 | if len(envNotFound) == 1 { 170 | err = errorf("variable not found in environment: %q", envNotFound[0]) 171 | } else { 172 | err = errorf("variables not found in environment: %q", envNotFound) 173 | } 174 | } 175 | 176 | return err 177 | } 178 | -------------------------------------------------------------------------------- /cmd/memex/service-supervisor.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "path/filepath" 7 | "strings" 8 | "sync" 9 | "sync/atomic" 10 | "time" 11 | 12 | "github.com/rsms/go-log" 13 | "github.com/rsms/memex/extension" 14 | ) 15 | 16 | var configFilenames = map[string]struct{}{ 17 | "memexservice.yml": {}, 18 | "memexservice.yaml": {}, 19 | } 20 | 21 | func isConfigFile(filename string) bool { 22 | _, ok := configFilenames[filename] 23 | return ok 24 | } 25 | 26 | type ServiceSupervisor struct { 27 | servicedir string 28 | log *log.Logger 29 | fsw *FSWatcher 30 | shutdown uint32 31 | 32 | mu sync.RWMutex // protects the following fields 33 | servicem map[string]*Service // all known services keyed by serviceId 34 | } 35 | 36 | func NewServiceSupervisor(logger *log.Logger, serviceDir string) *ServiceSupervisor { 37 | return &ServiceSupervisor{ 38 | servicedir: serviceDir, 39 | log: logger, 40 | } 41 | } 42 | 43 | func (s *ServiceSupervisor) Start(ctx context.Context) error { 44 | var err error 45 | s.servicem = make(map[string]*Service) 46 | 47 | // fs watcher 48 | if s.fsw, err = NewFSWatcher(ctx); err != nil { 49 | return err 50 | } 51 | s.fsw.Latency = 500 * time.Millisecond 52 | s.fsw.Add(s.servicedir) 53 | 54 | // initial scan of service directory 55 | if err := s.initialServiceDirScan(); err != nil { 56 | return err 57 | } 58 | 59 | // spawn runloop goroutine 60 | go func() { 61 | err := s.runLoop(ctx) 62 | s.log.Debug("runLoop exited") 63 | if err != nil { 64 | s.log.Error("%v", err) // FIXME 65 | } 66 | }() 67 | return nil 68 | } 69 | 70 | func (s *ServiceSupervisor) Shutdown() error { 71 | if !atomic.CompareAndSwapUint32(&s.shutdown, 0, 1) { 72 | return nil // race lost or already shut down 73 | } 74 | 75 | // stop fs watcher 76 | if s.fsw != nil { 77 | s.fsw.Close() 78 | } 79 | 80 | // stop services 81 | s.mu.Lock() 82 | defer s.mu.Unlock() 83 | n := len(s.servicem) 84 | // make a copy of services as they are removed from servicem via stopAndEndService 85 | services := make([]*Service, len(s.servicem)) 86 | i := 0 87 | for _, s := range s.servicem { 88 | services[i] = s 89 | i++ 90 | } 91 | 92 | // now actually shut down all services 93 | if n > 0 { 94 | s.log.Info("stopping %d %s...", n, plural(n, "service", "services")) 95 | for _, service := range services { 96 | s.stopAndEndService(service) 97 | } 98 | // wait for all services to stop before we return, which may cause the main process to exit 99 | for _, service := range services { 100 | service.Wait() 101 | s.log.Info("service %s stopped", service.id) 102 | } 103 | } 104 | 105 | return nil 106 | } 107 | 108 | // --- 109 | 110 | // stopAndEndService; must hold full lock s.mu 111 | func (s *ServiceSupervisor) stopAndEndService(service *Service) { 112 | delete(s.servicem, service.id) 113 | service.StopAndEnd() 114 | } 115 | 116 | func (s *ServiceSupervisor) serviceIdFromFile(configfile string) string { 117 | // e.g. "/memexdir/service/foo/bar/memexservice.yml" => "foo/bar" 118 | return filepath.Dir(relPath(s.servicedir, configfile)) 119 | } 120 | 121 | func (s *ServiceSupervisor) getService(sid string) *Service { 122 | s.mu.RLock() 123 | service := s.servicem[sid] 124 | s.mu.RUnlock() 125 | return service 126 | } 127 | 128 | func (s *ServiceSupervisor) getOrAddService(sid, configfile string) *Service { 129 | service := s.getService(sid) 130 | if service != nil { 131 | return service 132 | } 133 | s.mu.Lock() 134 | defer s.mu.Unlock() 135 | service = s.servicem[sid] 136 | if service != nil { // data race 137 | return service 138 | } 139 | service, err := LoadServiceFromConfig(sid, configfile, s.log) 140 | if err != nil { 141 | s.log.Error("failed to load service %q config (%q): %v", sid, configfile, err) 142 | } 143 | if s.log.Level <= log.LevelDebug { 144 | s.log.Debug("config for service %q:\n%s", sid, service.config.YAMLString()) 145 | } 146 | s.servicem[sid] = service 147 | s.log.Debug("new service registered %q", sid) 148 | return service 149 | } 150 | 151 | func (s *ServiceSupervisor) onServiceFileAppeared(configfile string) { 152 | sid := s.serviceIdFromFile(configfile) 153 | s.log.Debug("service %q file appeared %q", sid, relPath(s.servicedir, configfile)) 154 | service := s.getOrAddService(sid, configfile) 155 | if service.config.Disabled { 156 | if service.IsRunning() { 157 | s.log.Debug("stopping disabled service %q", sid) 158 | service.Stop() 159 | } 160 | } else { 161 | if !service.IsRunning() { 162 | service.Restart() 163 | } 164 | } 165 | s.fsw.Add(configfile) 166 | } 167 | 168 | func (s *ServiceSupervisor) onServiceFileDisappeared(configfile string) { 169 | sid := s.serviceIdFromFile(configfile) 170 | s.log.Debug("service %q file disappeared %q", sid, relPath(s.servicedir, configfile)) 171 | s.mu.Lock() 172 | defer s.mu.Unlock() 173 | if service := s.servicem[sid]; service != nil { 174 | s.stopAndEndService(service) 175 | } 176 | } 177 | 178 | func (s *ServiceSupervisor) onServiceFileChanged(configfile string) { 179 | sid := s.serviceIdFromFile(configfile) 180 | s.log.Debug("service %q file modified %q", sid, relPath(s.servicedir, configfile)) 181 | 182 | s.mu.RLock() 183 | defer s.mu.RUnlock() 184 | service := s.servicem[sid] 185 | if service == nil { 186 | s.log.Debug("unknown service %q", sid) 187 | return 188 | } 189 | 190 | config, err := ServiceConfigLoad(configfile) 191 | if err != nil { 192 | s.log.Error("failed to load config %q: %v", configfile, err) 193 | // TODO: should we stop and end the service in this case? 194 | } else { 195 | if service.config.IsEqual(config) && !service.CheckExeChangedSinceSpawn() { 196 | s.log.Info("%q did not change", sid) 197 | } else { 198 | service.config = config 199 | if s.log.Level <= log.LevelDebug { 200 | s.log.Debug("updated config for service %q:\n%s", sid, service.config.YAMLString()) 201 | } 202 | if service.config.Disabled { 203 | if service.IsRunning() { 204 | s.log.Info("stopping disabled service %q", sid) 205 | service.Stop() 206 | } 207 | } else { 208 | s.log.Info("restaring service %q", sid) 209 | service.Restart() 210 | } 211 | } 212 | } 213 | } 214 | 215 | func (s *ServiceSupervisor) onServiceFileChange(ev FSEvent) { 216 | // s.log.Info("fs event: %s %q", ev.Flags.String(), relPath(s.servicedir, ev.Name)) 217 | configfile := ev.Name 218 | if ev.IsRemove() { 219 | s.onServiceFileDisappeared(configfile) 220 | } else { 221 | s.onServiceFileChanged(configfile) 222 | } 223 | } 224 | 225 | // initial scan of servicedir 226 | func (s *ServiceSupervisor) initialServiceDirScan() error { 227 | if len(s.servicem) != 0 { 228 | panic("servicem is not empty") 229 | } 230 | s.log.Debug("start initial scan") 231 | err := filepath.Walk(s.servicedir, func(path string, info os.FileInfo, err error) error { 232 | // handle error 233 | if err != nil { 234 | s.log.Warn("can not read %q: %v", path, err) 235 | if info.IsDir() { 236 | return filepath.SkipDir // skip directory 237 | } 238 | return nil 239 | } 240 | // // debug log 241 | // if s.log.Level <= log.LevelDebug { 242 | // s.log.Debug("%v %s", info.Mode(), relPath(s.servicedir, path)) 243 | // } 244 | // consider service file 245 | if info.Mode().IsRegular() { 246 | name := filepath.Base(path) 247 | if isConfigFile(name) { 248 | dir := filepath.Dir(path) 249 | reldir := relPath(s.servicedir, dir) 250 | s.log.Info("found service config %q", filepath.Join(reldir, name)) 251 | // s.log.Debug("fs watch dir %q", reldir) 252 | // s.fsw.Add(dir) 253 | s.onServiceFileAppeared(path) 254 | } 255 | } 256 | return nil 257 | }) 258 | return err 259 | } 260 | 261 | func (s *ServiceSupervisor) runLoop(ctx context.Context) error { 262 | s.log.Info("watching filesystem for changes") 263 | for { 264 | changes, more := <-s.fsw.Events 265 | if !more { 266 | break 267 | } 268 | s.log.Debug("fs changes: %v", changes) 269 | for _, ev := range changes { 270 | if ev.Flags != extension.FSEventChmod { 271 | s.onServiceFileChange(ev) 272 | } 273 | } 274 | } 275 | return nil 276 | } 277 | 278 | // relPath returns a relative name of path rooted in dir. 279 | // If path is outside dir path is returned verbatim. 280 | // path is assumed to be absolute. 281 | func relPath(dir string, path string) string { 282 | if len(path) > len(dir) && path[len(dir)] == os.PathSeparator && strings.HasPrefix(path, dir) { 283 | return path[len(dir)+1:] 284 | } else if path == dir { 285 | return "." 286 | } 287 | return path 288 | } 289 | -------------------------------------------------------------------------------- /cmd/memex/service.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | "sync/atomic" 10 | "syscall" 11 | "time" 12 | 13 | "github.com/rsms/go-log" 14 | "github.com/rsms/memex/service-api" 15 | ) 16 | 17 | const ( 18 | ServiceStateInit = uint32(iota) 19 | ServiceStateStart // special state for when the runloop is entered 20 | ServiceStateRun // should be running 21 | ServiceStateStop // should be stopped 22 | ServiceStateEnd // end 23 | ) 24 | 25 | // defaultRestartLimitInterval is the default value for services without restart-limit-interval 26 | // in their configs 27 | const defaultRestartLimitInterval = 5 * time.Second 28 | 29 | // Service is a subprocess that provides some SAM service like processing email. 30 | type Service struct { 31 | id string 32 | cwd string 33 | config *ServiceConfig 34 | log *log.Logger 35 | proc *exec.Cmd 36 | ipc *service.IPC 37 | runch chan string 38 | runstate uint32 // desired "target" run state 39 | exepath string // absolute path of executable, from args[0], set by spawn() 40 | runexeStat os.FileInfo // stat at the time of last spawn() 41 | 42 | // procstate is only touched by runloop goroutine 43 | procstate uint32 44 | spawntime time.Time // time of last spawn() attempt (successful or not) 45 | restartTimer *time.Timer 46 | restartTimerCancelCh chan struct{} 47 | } 48 | 49 | func NewService(id, cwd string, supervisorLog *log.Logger) *Service { 50 | return &Service{ 51 | id: id, 52 | cwd: cwd, 53 | log: supervisorLog.SubLogger("[" + id + "]"), 54 | runch: make(chan string, 64), 55 | } 56 | } 57 | 58 | func LoadServiceFromConfig(id, configfile string, supervisorLog *log.Logger) (*Service, error) { 59 | s := NewService(id, filepath.Dir(configfile), supervisorLog) 60 | config, err := ServiceConfigLoad(configfile) 61 | s.config = config 62 | return s, err 63 | } 64 | 65 | func (s *Service) String() string { 66 | return s.id 67 | } 68 | 69 | // IsRunning returns true if the logical state of the service is "running" 70 | func (s *Service) IsRunning() bool { 71 | runstate := atomic.LoadUint32(&s.runstate) 72 | return runstate == ServiceStateStart || runstate == ServiceStateRun 73 | // return s.proc != nil && (s.proc.ProcessState == nil || s.proc.ProcessState.Exited()) 74 | } 75 | 76 | // Restart causes the service to restart (or simply start if it's not running) 77 | func (s *Service) Restart() { 78 | if atomic.CompareAndSwapUint32(&s.runstate, ServiceStateInit, ServiceStateStart) { 79 | go s.runloop() 80 | } 81 | s.runch <- "restart" 82 | } 83 | 84 | // Stop causes the service to stop (has no effect if it's not running) 85 | func (s *Service) Stop() { 86 | s.runch <- "stop" 87 | } 88 | 89 | // StopAndEnd stops the process if its running and then ends the service's lifetime. 90 | // The service object is invalid after this call. 91 | // Trying to call methods that change the run state like Restart, Stop etc after calling this 92 | // function will lead to a panic. 93 | func (s *Service) StopAndEnd() { 94 | s.runch <- "end" 95 | } 96 | 97 | // Wait blocks until the service's process has terminated. 98 | // Note: this may block forever if a call to Stop has not been made. 99 | func (s *Service) Wait() int { 100 | if s.proc == nil { 101 | return 0 102 | } 103 | s.proc.Wait() // ignore error; instead read ProcessState 104 | // runtime.Gosched() 105 | return s.proc.ProcessState.ExitCode() 106 | } 107 | 108 | func (s *Service) Pid() int { 109 | if s.proc == nil || s.proc.Process == nil { 110 | return 0 111 | } 112 | return s.proc.Process.Pid 113 | } 114 | 115 | func (s *Service) CheckExeChangedSinceSpawn() bool { 116 | if s.runexeStat == nil || s.exepath == "" { 117 | return false 118 | } 119 | finfo, err := os.Stat(s.exepath) 120 | if err != nil { 121 | return true 122 | } 123 | // log.Debug("s.exepath: %q", s.exepath) 124 | // log.Debug("size: %v <> %v", finfo.Size(), s.runexeStat.Size()) 125 | // log.Debug("mode: %v <> %v", finfo.Mode(), s.runexeStat.Mode()) 126 | // log.Debug("mtime: %v <> %v", finfo.ModTime(), s.runexeStat.ModTime()) 127 | return finfo.Size() != s.runexeStat.Size() || 128 | finfo.Mode() != s.runexeStat.Mode() || 129 | finfo.ModTime() != s.runexeStat.ModTime() 130 | } 131 | 132 | // ---------------------------------------------------------------------------------- 133 | // command & notification handlers 134 | 135 | type commandHandler func(s *Service, args [][]byte) ([][]byte, error) 136 | type notificationHandler func(s *Service, args [][]byte) 137 | 138 | var ( 139 | notificationHandlers map[string]notificationHandler 140 | commandHandlers map[string]commandHandler 141 | ) 142 | 143 | func init() { 144 | notificationHandlers = make(map[string]notificationHandler) 145 | commandHandlers = make(map[string]commandHandler) 146 | 147 | commandHandlers["ping"] = func(_ *Service, args [][]byte) ([][]byte, error) { 148 | return args, nil 149 | } 150 | 151 | // commandHandlers["type.register"] = func(s *Service, args [][]byte) ([][]byte, error) { 152 | // var typedefs []service.TypeDef 153 | // if err := json.Unmarshal(args[0], &typedefs); err != nil { 154 | // return nil, err 155 | // } 156 | // doctypes := make([]DocType, len(typedefs)) 157 | // for i, t := range typedefs { 158 | // dt := &doctypes[i] 159 | // dt.Id = t.Id 160 | // dt.ParentId = t.ParentId 161 | // dt.Name = t.Name 162 | // } 163 | // err := indexdb.DefineTypes(doctypes...) 164 | // return nil, err 165 | // } 166 | } 167 | 168 | func (s *Service) handleIncomingNotification(name string, args [][]byte) { 169 | // s.log.Debug("ipc notification: %q, args=%v", name, args) 170 | if f, ok := notificationHandlers[name]; ok { 171 | f(s, args) 172 | } else { 173 | s.log.Info("unknown notification %q (ignoring)", name) 174 | } 175 | } 176 | 177 | func (s *Service) handleIncomingCommand(cmd string, args [][]byte) ([][]byte, error) { 178 | // s.log.Debug("ipc command: %q, args=%v", cmd, args) 179 | if f, ok := commandHandlers[cmd]; ok { 180 | return f(s, args) 181 | } 182 | s.log.Info("received unknown command %q (ignoring)", cmd) 183 | return nil, errorf("invalid command %q", cmd) 184 | } 185 | 186 | // ------------------------------------------------------------ 187 | // runloop 188 | 189 | var timeInDistantPast = time.Unix(0, 0) 190 | 191 | func serviceStateName(state uint32) string { 192 | switch state { 193 | case ServiceStateInit: 194 | return "init" 195 | case ServiceStateStart: 196 | return "start" 197 | case ServiceStateRun: 198 | return "run" 199 | case ServiceStateStop: 200 | return "stop" 201 | case ServiceStateEnd: 202 | return "end" 203 | default: 204 | return "?" 205 | } 206 | } 207 | 208 | func (s *Service) runchClosed() bool { 209 | return atomic.LoadUint32(&s.runstate) == ServiceStateEnd 210 | } 211 | 212 | func (s *Service) transitionRunState(runstate uint32) bool { 213 | if s.runstate == runstate { 214 | return false 215 | } 216 | s.log.Debug("runstate transition %s -> %s", 217 | serviceStateName(s.runstate), serviceStateName(runstate)) 218 | atomic.StoreUint32(&s.runstate, runstate) 219 | return true 220 | } 221 | 222 | func (s *Service) cancelRestartTimer() { 223 | if s.restartTimer != nil { 224 | s.log.Debug("cancelling restartTimer") 225 | s.restartTimerCancelCh <- struct{}{} 226 | s.restartTimer.Stop() 227 | select { 228 | case <-s.restartTimer.C: 229 | default: 230 | } 231 | s.restartTimer = nil 232 | } 233 | } 234 | 235 | func (s *Service) startRestartTimer(delay time.Duration) { 236 | s.cancelRestartTimer() 237 | s.restartTimer = time.NewTimer(delay) 238 | s.restartTimerCancelCh = make(chan struct{}, 1) 239 | go func() { 240 | select { 241 | case _, more := <-s.restartTimer.C: 242 | s.log.Debug("restartTimer => more: %v", more) 243 | s.restartTimer = nil 244 | s.runch <- "continue-spawn" 245 | case <-s.restartTimerCancelCh: 246 | s.log.Debug("restartTimer cancelled") 247 | } 248 | }() 249 | return 250 | } 251 | 252 | func (s *Service) getRestartLimitInterval() time.Duration { 253 | if s.config.RestartLimitInterval < 0 { 254 | return defaultRestartLimitInterval 255 | } 256 | return s.config.RestartLimitInterval 257 | } 258 | 259 | func (s *Service) dospawn() { 260 | for { 261 | restartLimitInterval := s.getRestartLimitInterval() 262 | timeSinceLastSpawn := time.Since(s.spawntime) 263 | if timeSinceLastSpawn < restartLimitInterval { 264 | waittime := restartLimitInterval - timeSinceLastSpawn 265 | s.log.Info( 266 | "restarting too quickly; delaying restart by %s imposed by restart-limit-interval=%s", 267 | waittime.String(), restartLimitInterval) 268 | s.startRestartTimer(waittime) 269 | return 270 | } 271 | // actually spawn 272 | if err := s.spawn(); err != nil { 273 | s.log.Error("spawn failed: %v", err) 274 | s.transitionProcState(ServiceStateStop) 275 | // continue for loop which will branch to startRestartTimer 276 | } else { 277 | s.transitionProcState(ServiceStateRun) 278 | return 279 | } 280 | } 281 | } 282 | 283 | func (s *Service) transitionProcState(nextState uint32) bool { 284 | if s.procstate == nextState { 285 | return false 286 | } 287 | s.log.Debug("procstate transition %s -> %s", 288 | serviceStateName(s.procstate), serviceStateName(nextState)) 289 | s.procstate = nextState 290 | runstate := atomic.LoadUint32(&s.runstate) 291 | switch s.procstate { 292 | case ServiceStateRun: 293 | if runstate == ServiceStateStop { 294 | s.log.Info("stopping process") 295 | s.sendSignal(syscall.SIGINT) 296 | } 297 | case ServiceStateStop: 298 | if runstate == ServiceStateRun { 299 | s.log.Info("spawning new process") 300 | s.dospawn() 301 | } 302 | } 303 | return true 304 | } 305 | 306 | func (s *Service) handleRunloopMsg(msg string) bool { 307 | s.log.Debug("runloop got message: %v", msg) 308 | if msg == "restart" { 309 | // request to start or restart the service 310 | s.cancelRestartTimer() 311 | s.transitionRunState(ServiceStateRun) 312 | // reset spawntime so that a process that terminated early doesn't delay 313 | // this explicit user-requested restart. 314 | s.spawntime = timeInDistantPast 315 | if s.procstate == ServiceStateRun { 316 | // The process is running; stop it before we can restart it. 317 | // Zero spawntime so that we can start it again immediately. 318 | s.killProcess() 319 | } else { 320 | // the process is not running; spawn a new process 321 | s.dospawn() 322 | } 323 | } else if msg == "continue-spawn" { 324 | // dospawn() timer expired 325 | s.dospawn() 326 | } else if msg == "stop" { 327 | // request to stop the service 328 | s.cancelRestartTimer() 329 | if s.transitionRunState(ServiceStateStop) { 330 | s.killProcess() 331 | } 332 | } else if msg == "end" { 333 | // request to stop the service and & end its life, existing the control runloop 334 | s.cancelRestartTimer() 335 | if s.transitionRunState(ServiceStateStop) { 336 | s.killProcess() 337 | } 338 | if s.transitionRunState(ServiceStateEnd) { 339 | close(s.runch) 340 | return false 341 | } 342 | } else if msg == "spawned" { 343 | // process just started 344 | s.transitionProcState(ServiceStateRun) 345 | } else if msg == "exited" { 346 | // process just exited 347 | s.transitionProcState(ServiceStateStop) 348 | } else { 349 | s.log.Error("runloop got unexpected message %q", msg) 350 | } 351 | return true 352 | } 353 | 354 | func (s *Service) runloop() { 355 | s.log.Debug("runloop start") 356 | if atomic.LoadUint32(&s.runstate) != ServiceStateStart { 357 | panic("s.state != ServiceStateStart") 358 | } 359 | 360 | loop: 361 | for { 362 | select { 363 | case msg, ok := <-s.runch: 364 | if !ok || !s.handleRunloopMsg(msg) { // runch closed 365 | break loop 366 | } 367 | } 368 | } 369 | s.log.Debug("runloop end") 370 | } 371 | 372 | func (s *Service) sendSignal(sig syscall.Signal) bool { 373 | if s.procstate == ServiceStateRun { 374 | pid := s.proc.Process.Pid 375 | s.log.Debug("sending signal %v to process %d", sig, pid) 376 | // negative pid == process group (not supported on Windows) 377 | if err := syscall.Kill(-pid, sig); err != nil { 378 | s.log.Debug("failed to signal process group %d; trying just the process...", -pid) 379 | if err := syscall.Kill(pid, sig); err != nil { 380 | s.log.Warn("failed to send signal to process %d: %v", pid, err) 381 | return false 382 | } 383 | } 384 | } 385 | return true 386 | } 387 | 388 | func (s *Service) killProcess() { 389 | s.sendSignal(syscall.SIGINT) 390 | if s.restartTimer != nil { 391 | s.log.Debug("stopping restartTimer") 392 | s.restartTimer.Stop() 393 | } 394 | } 395 | 396 | func (s *Service) resolveUserFilePath(path string) (string, error) { 397 | if filepath.IsAbs(path) { 398 | return path, nil 399 | } 400 | var reldir string 401 | configfile := s.config.GetSourceFilename() 402 | if configfile != "" { 403 | reldir = filepath.Dir(configfile) 404 | } else { 405 | reldir = MEMEX_DIR 406 | } 407 | var err error 408 | if path, err = filepath.Abs(filepath.Join(reldir, path)); err == nil { 409 | path, err = filepath.EvalSymlinks(path) 410 | } 411 | if err != nil { 412 | s.log.Warn("failed to resolve path to file %q in dir %q (%v)", path, reldir, err) 413 | } 414 | return path, err 415 | } 416 | 417 | func (s *Service) resolveExePath(args []string) error { 418 | // resolve executable path 419 | s.exepath = args[0] 420 | if filepath.Base(s.exepath) == s.exepath { 421 | if lp, err := exec.LookPath(s.exepath); err != nil { 422 | return err 423 | } else { 424 | s.exepath = lp 425 | } 426 | } else if !filepath.IsAbs(s.exepath) { 427 | if fn, err := s.resolveUserFilePath(s.exepath); err == nil { 428 | s.exepath = fn 429 | } 430 | } 431 | 432 | // stat 433 | finfo, err := os.Stat(s.exepath) 434 | if err != nil { 435 | return err 436 | } 437 | if finfo.IsDir() { 438 | return errorf("%s is a directory (expected an executable file)", args[0]) 439 | } 440 | s.runexeStat = finfo 441 | return nil 442 | } 443 | 444 | func (s *Service) spawn() error { 445 | if s.ipc != nil { 446 | panic("spawn with active ipc driver") 447 | } 448 | 449 | s.spawntime = time.Now() // must update before any return statement 450 | 451 | // get command args from config 452 | args := s.config.GetStartArgs() 453 | if len(args) == 0 { 454 | return errorf("empty args in config or missing \"start\"") 455 | } 456 | 457 | // resolve executable path; sets s.exepath and s.runexeStat 458 | err := s.resolveExePath(args) 459 | if err != nil { 460 | return err 461 | } 462 | 463 | // chroot 464 | chroot := s.config.Chroot 465 | if chroot != "" { 466 | if os.Getuid() != 0 { 467 | s.log.Warn("ignoring chroot in config (memex is not running as root)") 468 | chroot = "" 469 | } else { 470 | chroot, err = s.resolveUserFilePath(chroot) 471 | if err != nil { 472 | return err 473 | } 474 | s.log.Info("chroot=%q", chroot) 475 | } 476 | } 477 | 478 | // environment variables 479 | env := append(append(os.Environ(), s.config.GetEnvEncodedList()...), 480 | "MEMEX_IPC_RECV_FD=3", 481 | "MEMEX_IPC_SEND_FD=4", 482 | ) 483 | 484 | // build process/command 485 | s.proc = &exec.Cmd{ 486 | Path: s.exepath, 487 | Args: args, 488 | Dir: s.cwd, 489 | Env: env, 490 | SysProcAttr: &syscall.SysProcAttr{ 491 | Chroot: chroot, 492 | Setsid: true, // start process with its own session id, to support -PID signalling 493 | }, 494 | } 495 | 496 | // IPC files 497 | ipcout, ipcRemoteWrite, err := os.Pipe() 498 | if err != nil { 499 | return err 500 | } 501 | ipcRemoteRead, ipcin, err := os.Pipe() 502 | if err != nil { 503 | return err 504 | } 505 | // the order of ExtraFiles must be synced with SAM_IPC_RECV_FD & SAM_IPC_SEND_FD above 506 | // Note: Cmd.ExtraFiles are not available on Windows. 507 | s.proc.ExtraFiles = []*os.File{ipcRemoteRead, ipcRemoteWrite} 508 | defer func() { 509 | // close remote end of IPC pipes 510 | ipcRemoteRead.Close() 511 | ipcRemoteWrite.Close() 512 | }() 513 | 514 | // create stdio pipes for forwarding output 515 | stdout, err := s.proc.StdoutPipe() // io.ReadCloser 516 | if err != nil { 517 | return err 518 | } 519 | stderr, err := s.proc.StderrPipe() // io.ReadCloser 520 | if err != nil { 521 | return err 522 | } 523 | 524 | // start process 525 | s.log.Info("starting process with %q %q", s.exepath, args[1:]) 526 | if err := s.proc.Start(); err != nil { 527 | return err 528 | } 529 | s.spawntime = time.Now() // update again 530 | s.log.Info("process %d started", s.proc.Process.Pid) 531 | s.runch <- "spawned" 532 | 533 | s.ipc = service.NewIPC() 534 | s.ipc.OnCommand = s.handleIncomingCommand 535 | s.ipc.OnNotification = s.handleIncomingNotification 536 | s.ipc.Start(ipcout, ipcin) 537 | 538 | // goroutine which reads stdout and stderr, prints line by line with prefix 539 | go s.readStdioLoop(stdout, "stdout") 540 | go s.readStdioLoop(stderr, "stderr") 541 | 542 | go func() { 543 | s.proc.Wait() 544 | s.ipc.Stop() 545 | s.ipc = nil 546 | ps := s.proc.ProcessState 547 | s.log.Info("process %d exited with status %v", ps.Pid(), ps.ExitCode()) 548 | if !s.runchClosed() { 549 | // note: s.runch is closed in ServiceStateEnd 550 | s.runch <- "exited" 551 | } 552 | }() 553 | 554 | return nil 555 | } 556 | 557 | func (s *Service) readStdioLoop(r io.ReadCloser, ioname string) { 558 | buffer := make([]byte, 4096) 559 | tail := []byte{} 560 | logger := s.log.SubLogger(" [" + ioname + "]") 561 | for { 562 | n, err := r.Read(buffer) 563 | if n == 0 || err == io.EOF { 564 | s.log.Debug("closed %s (EOF)", ioname) 565 | break 566 | } 567 | if err != nil { 568 | s.log.Info("error while reading %s of service %s: %s", ioname, s, err) 569 | break 570 | } 571 | // TODO: find line endings and print line by line? 572 | buf := buffer[:n] 573 | for { 574 | i := bytes.IndexByte(buf, '\n') 575 | if i == -1 { 576 | tail = append([]byte{}, buf...) 577 | break 578 | } 579 | line := buf[:i] 580 | buf = buf[i+1:] 581 | if len(tail) > 0 { 582 | tail = append(tail, line...) 583 | logger.Info("%s", string(tail)) 584 | tail = tail[:0] 585 | } else { 586 | logger.Info("%s", string(line)) 587 | } 588 | } 589 | } 590 | } 591 | -------------------------------------------------------------------------------- /cmd/memex/util.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | // create error 9 | func errorf(format string, v ...interface{}) error { 10 | return fmt.Errorf(format, v...) 11 | } 12 | 13 | // log error and exit 14 | func fatalf(msg interface{}, arg ...interface{}) { 15 | var format string 16 | if s, ok := msg.(string); ok { 17 | format = s 18 | } else if s, ok := msg.(fmt.Stringer); ok { 19 | format = s.String() 20 | } else { 21 | format = fmt.Sprintf("%v", msg) 22 | } 23 | fmt.Fprintf(os.Stderr, format+"\n", arg...) 24 | os.Exit(1) 25 | } 26 | 27 | // isdir returns nil if path is a directory, or an error describing the issue 28 | func isdir(path string) error { 29 | finfo, err := os.Stat(path) 30 | if err != nil { 31 | return err 32 | } 33 | if !finfo.IsDir() { 34 | return errorf("%q is not a directory", path) 35 | } 36 | return nil 37 | } 38 | 39 | func plural(n int, one, other string) string { 40 | if n == 1 { 41 | return one 42 | } 43 | return other 44 | } 45 | -------------------------------------------------------------------------------- /example-memexdir/foo/foo.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | echo "hello from foo" 3 | trap exit SIGINT 4 | 5 | echo "args: $@" 6 | echo "env FOO=$FOO" 7 | echo "env BAR=$BAR" 8 | 9 | # env | sort 10 | # exec sleep 30 11 | 12 | for i in `seq 3 1`; do 13 | echo "$i" 14 | sleep 1 15 | done 16 | -------------------------------------------------------------------------------- /example-memexdir/foo/memexservice.yml: -------------------------------------------------------------------------------- 1 | # start is the command for starting a new process. 2 | # It can be an array or string. If it's a string it will be launches as a script in the 3 | # default shell. If it's an array the first argument is the executable file. 4 | # Occurances of $(NAME) is replaced with the value of an environment variable NAME. 5 | # It is an error if a named env variable is not found. For cases when the var may be 6 | # undefined, you can specify a default value with the syntax $(NAME:defaultvalue). 7 | start: [./foo.sh, $(LOL:lolnotfound), $(HOME)] 8 | 9 | # restart-limit-interval: Minimum amount of time between automatic restarts. 10 | # This does not apply to explicit restarts or restarts from changes to the config file. 11 | restart-limit-interval: 10s 12 | 13 | # env is an optional dictionary of environment variables to set for new processes. 14 | # Expansion variables e.g. $(NAME) works here just like they do for command args. 15 | env: 16 | FOO: 123 17 | BAR: bar-$(FOO) 18 | 19 | # chroot causes the process to be "jailed" to the given directory tree. 20 | # This only works when the memex service runs in privileged mode. 21 | # chroot: . 22 | 23 | # disabled causes the service to not be run. 24 | # disabled: true 25 | -------------------------------------------------------------------------------- /example-memexdir/twitter/memexservice.yml: -------------------------------------------------------------------------------- 1 | start: [$(MEMEX_ROOT)/services/twitter/twitter] 2 | restart-limit-interval: 10s 3 | 4 | # remove or change to false to enable this service 5 | disabled: true 6 | -------------------------------------------------------------------------------- /example-memexdir/twitter/twitter-credentials.ini.in: -------------------------------------------------------------------------------- 1 | # Make a Twitter dev app and then generate key & secret at 2 | # https://developer.twitter.com/en/portal/apps/YOURAPPID/keys 3 | ConsumerKey: REPLACEME 4 | ConsumerSecret: REPLACEME 5 | AccessToken: REPLACEME 6 | AccessTokenSecret: REPLACEME 7 | -------------------------------------------------------------------------------- /extension/extension.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/rsms/go-log" 7 | ) 8 | 9 | // Extension represents a specific extension instance and provides an API to memex 10 | type Extension interface { 11 | Name() string // extension name 12 | Version() string // instance version 13 | Context() context.Context // the context of the extension instance 14 | Done() <-chan struct{} // closes when the extension should stop (==Context().Done()) 15 | Logger() *log.Logger // logger for this extension 16 | 17 | // WatchFiles returns a new file watcher which watches the file system for changes 18 | // to the provided paths. The returned watcher operates within the instance's context. 19 | WatchFiles(path ...string) FSWatcher 20 | } 21 | 22 | // RunFunc is the type of function that a extension exposes as it's "main" function. 23 | // The extension should not return from this function until Insance.Done() is closed. 24 | type RunFunc func(Extension) 25 | 26 | // Register a extension. Must be run at init time (i.e. in an init() function) 27 | func Register(extensionName string, run RunFunc) { 28 | if _, ok := Registry[extensionName]; ok { 29 | panic("duplicate extension \"" + extensionName + "\"") 30 | } 31 | Registry[extensionName] = run 32 | } 33 | 34 | // Don't modify from a extension 35 | var Registry = map[string]RunFunc{} 36 | -------------------------------------------------------------------------------- /extension/fs.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type FSWatcher interface { 9 | Add(name string) error 10 | Remove(name string) error 11 | Events() <-chan []FSEvent 12 | Latency() time.Duration 13 | SetLatency(time.Duration) 14 | } 15 | 16 | type FSEvent struct { 17 | Name string // filename 18 | Flags FSEventFlags // one or more FSEvent flag 19 | } 20 | 21 | func (ev FSEvent) IsCreate() bool { return ev.Flags&FSEventCreate != 0 } 22 | func (ev FSEvent) IsWrite() bool { return ev.Flags&FSEventWrite != 0 } 23 | func (ev FSEvent) IsRemove() bool { return ev.Flags&FSEventRemove != 0 } 24 | func (ev FSEvent) IsRename() bool { return ev.Flags&FSEventRename != 0 } 25 | func (ev FSEvent) IsChmod() bool { return ev.Flags&FSEventChmod != 0 } 26 | 27 | type FSEventFlags uint32 28 | 29 | const ( 30 | FSEventCreate FSEventFlags = 1 << iota 31 | FSEventWrite 32 | FSEventRemove 33 | FSEventRename 34 | FSEventChmod 35 | ) 36 | 37 | func (ev FSEvent) String() string { 38 | return fmt.Sprintf("%q %s", ev.Name, ev.Flags.String()) 39 | } 40 | 41 | func (fl FSEventFlags) String() string { 42 | a := [33]byte{} // max: "|CREATE|REMOVE|WRITE|RENAME|CHMOD" 43 | buf := a[:0] 44 | if fl&FSEventCreate != 0 { 45 | buf = append(buf, "|CREATE"...) 46 | } 47 | if fl&FSEventRemove != 0 { 48 | buf = append(buf, "|REMOVE"...) 49 | } 50 | if fl&FSEventWrite != 0 { 51 | buf = append(buf, "|WRITE"...) 52 | } 53 | if fl&FSEventRename != 0 { 54 | buf = append(buf, "|RENAME"...) 55 | } 56 | if fl&FSEventChmod != 0 { 57 | buf = append(buf, "|CHMOD"...) 58 | } 59 | if len(buf) == 0 { 60 | return "0" 61 | } 62 | return string(buf[1:]) // sans "|" 63 | } 64 | -------------------------------------------------------------------------------- /extension/test/test.go: -------------------------------------------------------------------------------- 1 | package supervisor_test 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/rsms/memex/extension" 7 | ) 8 | 9 | func init() { 10 | extension.Register("supervisor-test", run) 11 | } 12 | 13 | func run(ext extension.Extension) { 14 | log := ext.Logger() 15 | log.Info("starting") 16 | 17 | // // panic right away 18 | // panic("meow") 19 | 20 | // // panic during shutdown 21 | // <-ext.Done() 22 | // panic("meow") 23 | 24 | // properly wait 25 | <-ext.Done() 26 | 27 | // // wait forever (times out) 28 | // ch := make(chan bool) 29 | // <-ch 30 | 31 | // simulate doing some slow shutdown work 32 | for i := 0; i < 3; i++ { 33 | log.Info("simulating slow shutdown (%d/3)", i+1) 34 | time.Sleep(200 * time.Millisecond) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /extension/time.go: -------------------------------------------------------------------------------- 1 | package extension 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Timestamp is a millisecond-precision Unix timestamp (in UTC) 8 | type Timestamp int64 9 | 10 | // returns a millisecond-precision Unix timestamp (in UTC) 11 | func Now() Timestamp { 12 | return TimeTimestamp(time.Now()) 13 | } 14 | 15 | func TimeTimestamp(t time.Time) Timestamp { 16 | sec := t.Unix() // doesn't care about time zone 17 | nsec := t.UnixNano() - (sec * 1000000000) 18 | return Timestamp((sec * 1000) + (nsec / 1000000)) 19 | } 20 | 21 | func (t Timestamp) Time() time.Time { 22 | sec := int64(t) / 1000 23 | nsec := (int64(t) - (sec * 1000)) * 1000000 24 | return time.Unix(sec, nsec) 25 | } 26 | 27 | func (t Timestamp) String() string { 28 | return t.Time().String() 29 | } 30 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rsms/memex 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/fsnotify/fsnotify v1.4.9 7 | github.com/kurrik/oauth1a v0.1.1 8 | github.com/rsms/go-log v0.1.2 9 | github.com/rsms/go-testutil v0.1.1 10 | github.com/syndtr/goleveldb v1.0.0 11 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 // indirect 12 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect 13 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect 14 | gopkg.in/yaml.v2 v2.3.0 15 | howett.net/plist v0.0.0-20201203080718-1454fab16a06 16 | ) 17 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 2 | github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= 3 | github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= 4 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 5 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 6 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= 7 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 8 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 9 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 10 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 11 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 12 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 13 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 14 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 15 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 16 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 17 | github.com/kurrik/oauth1a v0.1.1 h1:3myAVza5bCMnyW/0gcVtQUeYaqcMKmniNxOIm0ESjek= 18 | github.com/kurrik/oauth1a v0.1.1/go.mod h1:2lmEMbW1BVM6RfQ6aN+b7kQSegGdXU4XeVfHKm4qxM0= 19 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 20 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 21 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 22 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 23 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 24 | github.com/rsms/go-log v0.1.2 h1:rnA+V1ccv+fsZcDtj5rwZ0B1PPS+iEJVB+F2emQs2s8= 25 | github.com/rsms/go-log v0.1.2/go.mod h1:yhPudnQQV6DCSCmGPQa4agpQB6jtXokgjEHGIgurDms= 26 | github.com/rsms/go-testutil v0.1.0/go.mod h1:Jm6EzhXOLcqNmqWbqOYMXOat3diHHyH1L5MLuP+6PyI= 27 | github.com/rsms/go-testutil v0.1.1 h1:IC5+Iruf368jqSovAvQCC1bQyiIo8+gCeLSCCxExmbY= 28 | github.com/rsms/go-testutil v0.1.1/go.mod h1:Jm6EzhXOLcqNmqWbqOYMXOat3diHHyH1L5MLuP+6PyI= 29 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 30 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 31 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 32 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= 33 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= 34 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 35 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= 36 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 37 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 38 | golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 39 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 40 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 41 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= 42 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 43 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 44 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 45 | golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= 46 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 47 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 48 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 49 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 50 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 51 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 52 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 53 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 54 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 55 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 56 | gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= 57 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 58 | howett.net/plist v0.0.0-20201203080718-1454fab16a06 h1:QDxUo/w2COstK1wIBYpzQlHX/NqaQTcf9jyz347nI58= 59 | howett.net/plist v0.0.0-20201203080718-1454fab16a06/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= 60 | -------------------------------------------------------------------------------- /service-api/ipc.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | // 4 | // IPC implements inter-process communication over io.Reader and io.Writer. 5 | // 6 | // Message wire format: 7 | // Msg = u32(len) MsgType u32(id) u32(cmdlen) string(cmd) u32(argc) Arg* 8 | // Arg = u32(len) []byte(value) 9 | // MsgType = CommandType | ResultType 10 | // CommandType = byte(">") 11 | // ResultType = byte("<") 12 | // 13 | // Command example: 14 | // <26> ">" <1> <4> "ping" <1> <5> "hello" 15 | // 16 | // Result examples: 17 | // <24> "<" <1> <2> "ok" <1> <5> "hello" 18 | // <39> "<" <1> <5> "error" <1> <17> "error description" 19 | // 20 | import ( 21 | "encoding/binary" 22 | "encoding/json" 23 | "errors" 24 | "fmt" 25 | "io" 26 | "os" 27 | "runtime/debug" 28 | "strconv" 29 | "sync" 30 | "sync/atomic" 31 | ) 32 | 33 | type IPCCommandHandler func(cmd string, args [][]byte) ([][]byte, error) 34 | type IPCNotificationHandler func(name string, args [][]byte) 35 | 36 | type IPC struct { 37 | OnCommand IPCCommandHandler 38 | OnNotification IPCNotificationHandler 39 | Running bool // true in-between calls to Start() and Stop() 40 | 41 | stopch chan bool // stop signal 42 | writech chan *ipcMsg // outgoing messages 43 | reqid uint32 // next request id 44 | reqw []ipcPromise // pending outgoing requests' response channels 45 | reqwmu sync.Mutex // lock for reqm 46 | } 47 | 48 | type ipcMsg struct { 49 | typ byte 50 | id uint32 51 | cmd string 52 | args [][]byte 53 | } 54 | 55 | type ipcPromise struct { 56 | id uint32 57 | ch chan *ipcMsg // response channel 58 | } 59 | 60 | const ( 61 | msgTypeCommand = byte('>') 62 | msgTypeResult = byte('<') 63 | msgTypeNotification = byte('N') 64 | ) 65 | 66 | var ( 67 | ErrNoMemex error // returned from Connect when Memex is unavailable 68 | ErrNotConnected error 69 | ) 70 | 71 | // default IPC object, created by Connect and used by package-level functions 72 | var DefaultIPC *IPC 73 | 74 | // true when DefaultIPC is valid; after a successful call to Connect() 75 | var Connected bool 76 | 77 | func init() { 78 | ErrNoMemex = errors.New("not running in Memex (ErrNoMemex)") 79 | ErrNotConnected = errors.New("not connected to Memex (ErrNotConnected)") 80 | } 81 | 82 | // connect DefaultIPC 83 | func Connect(onCommand func(cmd string, args [][]byte) ([][]byte, error)) error { 84 | if DefaultIPC != nil { 85 | return fmt.Errorf("already connected") 86 | } 87 | // check if this process is running as a Memex subprocess 88 | recvfd, err := strconv.Atoi(os.Getenv("MEMEX_IPC_RECV_FD")) 89 | if err != nil { 90 | return ErrNoMemex 91 | } 92 | sendfd, err := strconv.Atoi(os.Getenv("MEMEX_IPC_SEND_FD")) 93 | if err != nil { 94 | return ErrNoMemex 95 | } 96 | DefaultIPC = NewIPC() 97 | DefaultIPC.OnCommand = onCommand 98 | ipcin := os.NewFile(uintptr(recvfd), "ipcrecv") 99 | ipcout := os.NewFile(uintptr(sendfd), "ipcsend") 100 | if err := DefaultIPC.Start(ipcin, ipcout); err != nil { 101 | return err 102 | } 103 | Connected = true 104 | return err 105 | } 106 | 107 | func Command(cmd string, args ...[]byte) (res [][]byte, err error) { 108 | if DefaultIPC == nil { 109 | return nil, ErrNotConnected 110 | } 111 | return DefaultIPC.Command(cmd, args...) 112 | } 113 | 114 | func JsonCommand(cmd string, arg0 interface{}, args ...[]byte) (res [][]byte, err error) { 115 | arg0buf, err := json.Marshal(arg0) 116 | if err != nil { 117 | return nil, err 118 | } 119 | if len(args) > 0 { 120 | args2 := append([][]byte{arg0buf}, args...) 121 | return DefaultIPC.Command(cmd, args2...) 122 | } 123 | return DefaultIPC.Command(cmd, arg0buf) 124 | } 125 | 126 | func Notify(name string, args ...[]byte) error { 127 | if DefaultIPC == nil { 128 | return ErrNotConnected 129 | } 130 | return DefaultIPC.Notify(name, args...) 131 | } 132 | 133 | func NewIPC() *IPC { 134 | return &IPC{ 135 | stopch: make(chan bool, 1), 136 | writech: make(chan *ipcMsg, 8), 137 | } 138 | } 139 | 140 | func (ipc *IPC) Notify(name string, args ...[]byte) error { 141 | if !ipc.Running { 142 | return fmt.Errorf("ipc not running") 143 | } 144 | ipc.writech <- &ipcMsg{typ: msgTypeNotification, cmd: name, args: args} 145 | return nil 146 | } 147 | 148 | func (ipc *IPC) CommandAndForget(cmd string, args ...[]byte) error { 149 | if !ipc.Running { 150 | return fmt.Errorf("ipc not running") 151 | } 152 | id := atomic.AddUint32(&ipc.reqid, 1) 153 | ipc.writech <- &ipcMsg{typ: msgTypeCommand, id: id, cmd: cmd, args: args} 154 | return nil 155 | } 156 | 157 | func (ipc *IPC) Command(cmd string, args ...[]byte) (res [][]byte, err error) { 158 | if !ipc.Running { 159 | return nil, fmt.Errorf("ipc not running") 160 | } 161 | 162 | // generate request ID and create response channel 163 | id := atomic.AddUint32(&ipc.reqid, 1) 164 | ch := make(chan *ipcMsg) 165 | 166 | // add wait object 167 | ipc.reqwmu.Lock() 168 | ipc.reqw = append(ipc.reqw, ipcPromise{id: id, ch: ch}) 169 | ipc.reqwmu.Unlock() 170 | 171 | // send request 172 | ipc.writech <- &ipcMsg{typ: msgTypeCommand, id: id, cmd: cmd, args: args} 173 | 174 | // wait for response 175 | msg := <-ch 176 | 177 | if msg.cmd == "error" { 178 | if len(msg.args) > 0 { 179 | err = fmt.Errorf("ipc error: %s", string(msg.args[0])) 180 | } else { 181 | err = fmt.Errorf("ipc error") 182 | } 183 | } else { 184 | res = msg.args 185 | } 186 | return 187 | } 188 | 189 | func (ipc *IPC) Stop() { 190 | if ipc.Running { 191 | ipc.Running = false 192 | ipc.stopch <- true 193 | } 194 | } 195 | 196 | func (ipc *IPC) Start(r io.Reader, w io.Writer) error { 197 | if ipc.Running { 198 | return fmt.Errorf("already started") 199 | } 200 | ipc.Running = true 201 | go func() { 202 | err := ipc.ioLoop(r, w) 203 | ipc.Running = false 204 | if err != nil { 205 | logf("[memex/ipc] I/O error: %v", err) 206 | } 207 | }() 208 | return nil 209 | } 210 | 211 | func (ipc *IPC) ioLoop(r io.Reader, w io.Writer) error { 212 | // start response writer goroutine 213 | go ipc.writeLoop(w) 214 | 215 | // read buffer and total accumulative input 216 | buf := make([]byte, 4096) 217 | input := []byte{} 218 | 219 | // read loop 220 | for { 221 | // Semantics of Read(): 222 | // - if the underlying read buffer is empty, wait until its filled and then return. 223 | // - else return the underlying buffer immediately 224 | // This means that fflush in the service process causes Read here to return. 225 | n, err := r.Read(buf) 226 | if err != nil { 227 | if err == io.EOF { 228 | break 229 | } 230 | return err 231 | } else if n == 0 || !ipc.Running { 232 | break 233 | } 234 | 235 | input = append(input, buf[:n]...) 236 | 237 | // parse each complete request in input 238 | bytes := input 239 | for { 240 | msgdata, remainder, ok := readLengthPrefixedSlice(bytes) 241 | if !ok { 242 | break 243 | } 244 | bytes = remainder 245 | 246 | // Clone the input and run it on another goroutine 247 | data := append([]byte{}, msgdata...) 248 | go ipc.readMsg(data) 249 | } 250 | 251 | // Move the remaining partial request to the end to avoid reallocating 252 | input = append(input[:0], bytes...) 253 | } // for 254 | // EOF or stopped 255 | return nil 256 | } 257 | 258 | func writeMsg(w io.Writer, buf []byte, msg *ipcMsg) { 259 | length := 1 + 4 + 4 + len(msg.cmd) + 4 // MsgType + u32(id) + u32(cmdlen) + len(cmd) + u32(argc) 260 | for _, v := range msg.args { 261 | length += 4 + len(v) 262 | } 263 | 264 | // write u32(len) byte('>') u32(id) u32(len) 265 | offs := uint32(0) 266 | binary.LittleEndian.PutUint32(buf, uint32(length)) 267 | offs += 4 268 | buf[offs] = msg.typ 269 | offs += 1 270 | binary.LittleEndian.PutUint32(buf[offs:], msg.id) 271 | offs += 4 272 | binary.LittleEndian.PutUint32(buf[offs:], uint32(len(msg.cmd))) 273 | offs += 4 274 | w.Write(buf[:offs]) 275 | offs = 0 // flush 276 | w.Write([]byte(msg.cmd)) 277 | binary.LittleEndian.PutUint32(buf, uint32(len(msg.args))) 278 | w.Write(buf[:4]) 279 | 280 | // write args 281 | for _, v := range msg.args { 282 | binary.LittleEndian.PutUint32(buf, uint32(len(v))) 283 | w.Write(buf[:4]) 284 | w.Write(v) 285 | } 286 | } 287 | 288 | func (ipc *IPC) readMsg(data []byte) { 289 | defer func() { 290 | if r := recover(); r != nil { 291 | logf("[memex/ipc] ignoring invalid data: %v\n%s", r, debug.Stack()) 292 | } 293 | }() 294 | 295 | // read header 296 | // Note: readMsg receives data slice that _excludes_ the length prefix (no u32(len) header) 297 | offs := uint32(0) 298 | typ := data[offs] 299 | offs += 1 300 | id := binary.LittleEndian.Uint32(data[offs : offs+4]) 301 | offs += 4 302 | cmdlen := binary.LittleEndian.Uint32(data[offs : offs+4]) 303 | offs += 4 304 | cmd := data[offs : offs+cmdlen] 305 | offs += cmdlen 306 | argc := binary.LittleEndian.Uint32(data[offs : offs+4]) 307 | offs += 4 308 | 309 | // read args 310 | args := [][]byte{} 311 | for i := uint32(0); i < argc; i++ { 312 | z := binary.LittleEndian.Uint32(data[offs : offs+4]) 313 | offs += 4 314 | args = append(args, data[offs:offs+z]) 315 | offs += z 316 | } 317 | 318 | // dispatch 319 | switch typ { 320 | 321 | case msgTypeNotification: 322 | if ipc.OnNotification == nil { 323 | return 324 | } 325 | func() { 326 | // trap panics in OnCommand and create error response 327 | defer func() { 328 | if r := recover(); r != nil { 329 | logf("[memex/ipc] recovered panic in OnNotification: %v\n%s", r, debug.Stack()) 330 | } 331 | }() 332 | ipc.OnNotification(string(cmd), args) 333 | }() 334 | 335 | case msgTypeCommand: 336 | if ipc.OnCommand == nil { 337 | ipc.sendErrorResult(id, "commands not accepted") 338 | return 339 | } 340 | func() { 341 | // trap panics in OnCommand and create error response 342 | defer func() { 343 | if r := recover(); r != nil { 344 | logf("[memex/ipc] recovered panic in OnCommand: %v\n%s", r, debug.Stack()) 345 | ipc.sendErrorResult(id, r) 346 | } 347 | }() 348 | res, err := ipc.OnCommand(string(cmd), args) // -> [][]byte, error 349 | if err != nil { 350 | ipc.sendErrorResult(id, err) 351 | } else { 352 | //logf("IPC readMsg enqueue response on writech") 353 | ipc.writech <- &ipcMsg{ 354 | typ: msgTypeResult, 355 | id: id, 356 | cmd: "ok", 357 | args: res, 358 | } 359 | } 360 | }() 361 | 362 | case msgTypeResult: 363 | // find corresponding wait object 364 | var reqwait *ipcPromise 365 | ipc.reqwmu.Lock() 366 | for i, v := range ipc.reqw { 367 | if v.id == id { 368 | reqwait = &v 369 | ipc.reqw = append(ipc.reqw[:i], ipc.reqw[i+1:]...) 370 | } 371 | } 372 | ipc.reqwmu.Unlock() 373 | if reqwait != nil { 374 | reqwait.ch <- &ipcMsg{ 375 | typ: typ, 376 | id: id, 377 | cmd: string(cmd), 378 | args: args, 379 | } 380 | } 381 | 382 | default: 383 | logf("[memex/ipc] readMsg ignoring invalid message type 0x%02x", typ) 384 | 385 | } 386 | } 387 | 388 | func (ipc *IPC) sendErrorResult(id uint32, err interface{}) { 389 | ipc.writech <- &ipcMsg{ 390 | typ: msgTypeResult, 391 | id: id, 392 | cmd: "error", 393 | args: [][]byte{[]byte(fmt.Sprint(err))}, 394 | } 395 | } 396 | 397 | func (ipc *IPC) writeLoop(w io.Writer) { 398 | // buf is a small buffer used temporarily to encode data 399 | buf := []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} 400 | for { 401 | // logf("IPC writeLoop awaiting message...") 402 | select { 403 | case msg := <-ipc.writech: 404 | // logf("IPC writeLoop writing message %v", msg) 405 | writeMsg(w, buf, msg) 406 | case <-ipc.stopch: 407 | // IPC shutting down -- exit write loop 408 | break 409 | } 410 | } 411 | } 412 | 413 | func writeU32(w io.Writer, v uint32) { 414 | buf := []byte{0, 0, 0, 0} 415 | binary.LittleEndian.PutUint32(buf, v) 416 | w.Write(buf) 417 | } 418 | 419 | func readLengthPrefixedSlice(input []byte) (data []byte, remainder []byte, ok bool) { 420 | inlen := len(input) 421 | if inlen >= 4 { 422 | z := binary.LittleEndian.Uint32(input) 423 | if uint(inlen-4) >= uint(z) { 424 | return input[4 : 4+z], input[4+z:], true 425 | } 426 | } 427 | return []byte{}, input, false 428 | } 429 | 430 | func logf(format string, v ...interface{}) { 431 | fmt.Fprintf(os.Stderr, format+"\n", v...) 432 | } 433 | -------------------------------------------------------------------------------- /service-api/msg.go: -------------------------------------------------------------------------------- 1 | package service 2 | 3 | type PutCommand struct { 4 | // required 5 | Type string `json:"type"` // e.g. "twitter.status" 6 | File string `json:"file"` // either File or Data must be provided 7 | Data []byte `json:"data"` 8 | 9 | // optional 10 | Id string `json:"id"` // if set: replace if exists 11 | Name string `json:"name"` // 12 | Tags []string `json:"tags"` // text to be tokenized for indexing 13 | } 14 | 15 | type TypeDef struct { 16 | Id string `json:"id"` 17 | ParentId string `json:"parent_id"` 18 | Name string `json:"name"` 19 | } 20 | 21 | type TypeRegCommand struct { 22 | Types []TypeDef 23 | } 24 | -------------------------------------------------------------------------------- /services/twitter/.gitignore: -------------------------------------------------------------------------------- 1 | /twitter 2 | -------------------------------------------------------------------------------- /services/twitter/ioutil.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | ) 8 | 9 | // UNIX file mode permission bits 10 | type UnixPermBits uint32 11 | 12 | const ( 13 | UnixPermSetuid UnixPermBits = 1 << (12 - 1 - iota) 14 | UnixPermSetgid 15 | UnixPermSticky 16 | UnixPermUserRead 17 | UnixPermUserWrite 18 | UnixPermUserExecute 19 | UnixPermGroupRead 20 | UnixPermGroupWrite 21 | UnixPermGroupExecute 22 | UnixPermOtherRead 23 | UnixPermOtherWrite 24 | UnixPermOtherExecute 25 | ) 26 | 27 | func mkdirsForFile(filename string, perm os.FileMode) error { 28 | dir, err := filepath.Abs(filename) 29 | if err != nil { 30 | return err 31 | } 32 | dirperm := matchingDirUnixPerm(perm) 33 | return os.MkdirAll(filepath.Dir(dir), dirperm) 34 | } 35 | 36 | func renameFile(oldpath, newpath string, mode os.FileMode) error { 37 | if err := os.Chmod(oldpath, mode); err != nil { 38 | return err 39 | } 40 | err := os.Rename(oldpath, newpath) 41 | if err != nil && os.IsNotExist(err) { 42 | if err = mkdirsForFile(newpath, mode); err == nil { 43 | err = os.Rename(oldpath, newpath) 44 | } 45 | } 46 | return err 47 | } 48 | 49 | func writeFileWithMkdirs(filename string, data []byte, mode os.FileMode) error { 50 | retried := false 51 | for { 52 | if err := ioutil.WriteFile(filename, data, mode); err != nil { 53 | if retried || !os.IsNotExist(err) { 54 | return err 55 | } 56 | retried = true 57 | if err := mkdirsForFile(filename, mode); err != nil { 58 | return err 59 | } 60 | continue 61 | } 62 | break 63 | } 64 | return nil 65 | } 66 | 67 | func matchingDirUnixPerm(perm os.FileMode) os.FileMode { 68 | if perm&os.FileMode(UnixPermUserRead) != 0 { 69 | perm |= os.FileMode(UnixPermUserExecute) 70 | } 71 | if perm&os.FileMode(UnixPermGroupRead) != 0 { 72 | perm |= os.FileMode(UnixPermGroupExecute) 73 | } 74 | if perm&os.FileMode(UnixPermOtherRead) != 0 { 75 | perm |= os.FileMode(UnixPermOtherExecute) 76 | } 77 | return perm 78 | } 79 | -------------------------------------------------------------------------------- /services/twitter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | // "fmt" 5 | // "os" 6 | // "time" 7 | // "bytes" 8 | // "encoding/json" 9 | // "context" 10 | 11 | "bytes" 12 | "encoding/json" 13 | "flag" 14 | "fmt" 15 | "io" 16 | "io/fs" 17 | "io/ioutil" 18 | "os" 19 | "path/filepath" 20 | "sort" 21 | "strconv" 22 | "strings" 23 | "time" 24 | 25 | "github.com/rsms/go-log" 26 | memex "github.com/rsms/memex/service-api" 27 | "github.com/rsms/memex/services/twitter/tweeter" 28 | ) 29 | 30 | var DATA_DIR string = "./data" 31 | var DEBUG = false 32 | var NEW_FILE_MODE fs.FileMode = 0600 // permissions for new files 33 | var TWEETS_DIR = "tweets" // sibdir of DATA_DIR where tweet json files are stored 34 | 35 | type IdRange struct { 36 | min, max uint64 37 | } 38 | 39 | func onMemexMsg(cmd string, args [][]byte) ([][]byte, error) { 40 | log.Info("onMemexMsg %q %v", cmd, args) 41 | return nil, nil 42 | } 43 | 44 | func main() { 45 | defer log.Sync() 46 | 47 | // DATA_DIR 48 | if os.Getenv("MEMEX_DIR") != "" { 49 | DATA_DIR = filepath.Join(os.Getenv("MEMEX_DIR"), "twitter") 50 | } else { 51 | var err error 52 | DATA_DIR, err = filepath.Abs(DATA_DIR) 53 | if err != nil { 54 | panic(err) 55 | } 56 | } 57 | 58 | // parse CLI flags 59 | optDebug := flag.Bool("debug", false, "Enable debug mode") 60 | flag.Parse() 61 | 62 | // update log level based on CLI options 63 | log.RootLogger.EnableFeatures(log.FMilliseconds) 64 | if *optDebug { 65 | DEBUG = true 66 | log.RootLogger.Level = log.LevelDebug 67 | log.RootLogger.EnableFeatures(log.FSync) 68 | } else { 69 | log.RootLogger.EnableFeatures(log.FSyncError) 70 | } 71 | 72 | // connect to memex 73 | if err := memex.Connect(onMemexMsg); err != nil && err != memex.ErrNoMemex { 74 | log.Warn("failed to commect to memex service: %v", err) 75 | } 76 | 77 | // test memex IPC 78 | if memex.Connected { 79 | // disable log prefix 80 | log.RootLogger.DisableFeatures(log.FTime | log.FMilliseconds | 81 | log.FPrefixDebug | log.FPrefixInfo | log.FPrefixWarn | log.FPrefixError) 82 | 83 | log.Info("connected to memex service; sending \"ping\" message") 84 | res, err := memex.Command("ping", []byte("hello")) 85 | if err != nil { 86 | panic(err) 87 | } 88 | log.Info("\"ping\" result from memex service: %q", res) 89 | } 90 | 91 | log.Info("DATA_DIR=%q", DATA_DIR) 92 | 93 | // configure twitter client 94 | tw, err := createTwitterClient("./twitter-credentials.ini") 95 | if err != nil { 96 | panic(err) 97 | } 98 | 99 | downloadTweets(tw) 100 | } 101 | 102 | func getStoredTweetIdRange() (min_id uint64, max_id uint64) { 103 | min_id = 0 104 | max_id = 0 105 | tweetsDir := datapath(TWEETS_DIR) 106 | f, err := os.Open(tweetsDir) 107 | if err != nil { 108 | if !os.IsNotExist(err) { 109 | log.Error("failed to open directory %q: %v", tweetsDir, err) 110 | } 111 | return 112 | } 113 | defer f.Close() 114 | names, err := f.Readdirnames(-1) 115 | if err != nil { 116 | log.Error("failed to read directory %q: %v", tweetsDir, err) 117 | } 118 | if len(names) > 0 { 119 | sort.Strings(names) 120 | // min_id 121 | for _, name := range names { 122 | if len(name) > 0 && name[0] != '.' && strings.HasSuffix(name, ".json") { 123 | // oldest tweet probably found. parse as uint (sans ".json") 124 | id, err := strconv.ParseUint(name[:len(name)-len(".json")], 10, 64) 125 | if err == nil { 126 | min_id = id 127 | break 128 | } 129 | } 130 | } 131 | // max_id 132 | for i := len(names) - 1; i >= 0; i-- { 133 | name := names[i] 134 | if len(name) > 0 && name[0] != '.' && strings.HasSuffix(name, ".json") { 135 | // most recent tweet probably found. parse as uint (sans ".json") 136 | id, err := strconv.ParseUint(name[:len(name)-len(".json")], 10, 64) 137 | if err == nil { 138 | max_id = id 139 | break 140 | } 141 | } 142 | } 143 | } 144 | return 145 | } 146 | 147 | // downloadTweets fetches old tweets. 148 | // It is limited to ~3000 tweets into the past (as of May 2021.) 149 | // See https://twitter.com/settings/download_your_data for an alterante way to get old tweets. 150 | func downloadTweets(tw *tweeter.Client) { 151 | // find oldest tweet we have already saved 152 | minStoredId, maxStoredId := getStoredTweetIdRange() 153 | log.Info("minStoredId, maxStoredId: %v, %v", minStoredId, maxStoredId) 154 | 155 | var max_id, since_id uint64 156 | isFetchingHistory := false 157 | 158 | // if we have existing tweets, we start by fetching additional old tweets 159 | if minStoredId != 0 { 160 | isFetchingHistory = true 161 | max_id = minStoredId 162 | } 163 | 164 | const sleeptimeMax time.Duration = 10 * time.Second 165 | var sleeptime time.Duration 166 | bumpSleeptime := func(initial time.Duration) { 167 | if sleeptime == 0 { 168 | sleeptime = initial 169 | } else { 170 | sleeptime *= 2 171 | if sleeptime > sleeptimeMax { 172 | sleeptime = sleeptimeMax 173 | } 174 | } 175 | } 176 | 177 | for { 178 | // fetch from API 179 | idrange, err := downloadSomeTweets(tw, max_id, since_id) 180 | if err != nil { 181 | // error 182 | if rl, ok := err.(tweeter.RateLimitError); ok { 183 | // rate-limited 184 | log.Warn("rate limited: %v/%v req remaining, reset at %v", 185 | rl.RateLimitRemaining(), rl.RateLimit(), rl.RateLimitReset()) 186 | if rl.RateLimitRemaining() == 0 { 187 | sleeptime = time.Until(rl.RateLimitReset().Add(100 * time.Millisecond)) 188 | } else { 189 | bumpSleeptime(500 * time.Millisecond) 190 | } 191 | } else { 192 | log.Error("error while downloading tweets: %v", err) 193 | // if e, ok := err.(tweeter.ResponseError); ok { 194 | // // communication error (e.g. network error, server error, etc) 195 | // } 196 | bumpSleeptime(100 * time.Millisecond) 197 | } 198 | if sleeptime > 0 { 199 | // wait a bit before we retry 200 | if sleeptime > sleeptimeMax { 201 | log.Info("retrying at %v...", time.Now().Add(sleeptime)) 202 | } else { 203 | log.Info("retrying in %v...", sleeptime) 204 | } 205 | time.Sleep(sleeptime) 206 | } 207 | } else { 208 | // success 209 | sleeptime = 0 // reset sleeptime 210 | maxStoredId = u64max(maxStoredId, idrange.max) 211 | if idrange.min != 0 { 212 | minStoredId = u64min(minStoredId, idrange.min) 213 | } 214 | 215 | if idrange.min == max_id || idrange.min == 0 { 216 | // empty; no tweets found 217 | if isFetchingHistory { 218 | log.Info("no more tweets returned by the Twitter API for max_id=%v", max_id) 219 | isFetchingHistory = false 220 | } else { 221 | // As we are polling for new tweets; wait a little before retrying. 222 | // API rate limit of 900/15min means our upper frequency limit is 1req/sec 223 | waittime := 5 * time.Second 224 | log.Info("checking again in %v...", waittime) 225 | time.Sleep(waittime) 226 | } 227 | } else { 228 | // did find some tweets 229 | log.Info("downloaded tweets in id range %v-%v", idrange.min, idrange.max) 230 | } 231 | 232 | if isFetchingHistory { 233 | // fetching old tweets. Moves: past <- now 234 | max_id = minStoredId - 1 // API max_id has semantics "LESS OR EQUAL" (<=) 235 | since_id = 0 236 | } else { 237 | // fetching new tweets. Moves: past -> now 238 | max_id = 0 239 | since_id = maxStoredId // API since_id has semantics "GREATER" (>) 240 | } 241 | } 242 | } 243 | } 244 | 245 | func mockRateLimitError(limit uint32, remaining uint32, reset time.Time) tweeter.RateLimitError { 246 | return tweeter.RateLimitError{ 247 | Limit: limit, 248 | Remaining: remaining, 249 | Reset: reset, 250 | } 251 | } 252 | 253 | func mockResponseError(code int, body string) tweeter.ResponseError { 254 | return tweeter.NewResponseError(code, body) 255 | } 256 | 257 | // downloadSomeTweets downloads tweets which are older than max_id 258 | func downloadSomeTweets(tw *tweeter.Client, max_id, since_id uint64) (IdRange, error) { 259 | // Note that max_id=0 makes twFetchUserTimeline omit the id and fetch the most recent tweets. 260 | 261 | if max_id == 0 && since_id == 0 { 262 | log.Info("fetching recent tweets") 263 | } else if max_id == 0 { 264 | log.Info("checking for tweets newer than id %v", since_id) 265 | } else { 266 | log.Info("fetching tweets older than id %v", max_id+1 /* "<=" -> "<" */) 267 | } 268 | 269 | // mock error responses for testing & debugging 270 | // return mockRateLimitError(900, 0, time.Now().Add(3*time.Second)), 0 271 | // return mockRateLimitError(900, 10, time.Now().Add(3*time.Second)), 0 272 | 273 | var idrange IdRange 274 | 275 | // talk with the twitter API (upper limit on "count" is 200 as of May 2021) 276 | res, err := twFetchUserTimeline(tw, 200, max_id, since_id) 277 | if err != nil { 278 | // an error here is a network error. 279 | // res.Parse is where we get API errors. 280 | return idrange, err 281 | } 282 | // json_prettyprint(res.ReadBody()) 283 | 284 | // log rate limits 285 | log.Info("twitter API rate limit: %v/%v req remaining, reset at %v", 286 | res.RateLimitRemaining(), res.RateLimit(), res.RateLimitReset()) 287 | 288 | // parse the API response 289 | var tweets []map[string]interface{} // [{key:value}...] 290 | if err := res.Parse(&tweets); err != nil { 291 | return idrange, err 292 | } 293 | 294 | // parse the tweets json 295 | for _, tweet := range tweets { 296 | // extract id_str property 297 | id_str, ok := tweet["id_str"].(string) 298 | if !ok { 299 | err = fmt.Errorf("unexpected json from API response: id_str is not a string\n%+v", tweet) 300 | return idrange, err 301 | } 302 | 303 | // parse id_str property as uint 304 | var id uint64 305 | if id, err = strconv.ParseUint(id_str, 10, 64); err != nil { 306 | err = fmt.Errorf("failed to parse id_str %q: %v", id_str, err) 307 | return idrange, err 308 | } 309 | 310 | idrange.max = u64max(idrange.max, id) 311 | // idrange.min = u64min(idrange.min, id) 312 | if id < idrange.min || idrange.min == 0 { 313 | idrange.min = id 314 | } 315 | 316 | // // print 317 | // jsondata, _ := json.MarshalIndent(tweet, "", " ") 318 | // log.Info("tweet %#v: %s", id, jsondata) 319 | 320 | // write json file 321 | jsonfile := datapath(TWEETS_DIR, id_str+".json") 322 | var f io.WriteCloser 323 | if f, err = createFileAtomic(jsonfile, NEW_FILE_MODE); err != nil { 324 | return idrange, err 325 | } 326 | jsonenc := json.NewEncoder(f) 327 | jsonenc.SetIndent("", " ") 328 | err = jsonenc.Encode(tweet) 329 | f.Close() 330 | if err != nil { 331 | return idrange, fmt.Errorf("failed to encode json %q: %v", jsonfile, err) 332 | } 333 | log.Info("wrote %s", nicepath(jsonfile)) 334 | } 335 | 336 | // TODO: when we encounter a tweet with... 337 | // - "in_reply_to_status_id": 1396149090375720960 -- fetch that tweet 338 | // - "in_reply_to_user_id": 14199907 -- fetch that user 339 | return idrange, nil 340 | } 341 | 342 | type writeFileTransaction struct { 343 | f *os.File 344 | filename string 345 | mode fs.FileMode 346 | } 347 | 348 | func (t *writeFileTransaction) Write(p []byte) (n int, err error) { 349 | return t.f.Write(p) 350 | } 351 | 352 | func (t *writeFileTransaction) Close() error { 353 | if err := t.f.Close(); err != nil { 354 | return err 355 | } 356 | return renameFile(t.f.Name(), t.filename, t.mode) 357 | } 358 | 359 | func createFileAtomic(filename string, mode fs.FileMode) (io.WriteCloser, error) { 360 | f, err := os.CreateTemp("", "memex") 361 | return &writeFileTransaction{f, filename, mode}, err 362 | } 363 | 364 | func twFetchUserTimeline( 365 | tw *tweeter.Client, count, max_id, since_id uint64) (*tweeter.ApiResponse, error) { 366 | // https://developer.twitter.com/en/docs/twitter-api/v1/tweets/timelines/api-reference/get-statuses-user_timeline 367 | path := "/1.1/statuses/user_timeline.json?trim_user=true&include_rts=false&tweet_mode=extended" 368 | if count > 0 { 369 | path += fmt.Sprintf("&count=%v", count) 370 | } 371 | if max_id > 0 { 372 | path += fmt.Sprintf("&max_id=%v", max_id) 373 | } 374 | if since_id > 0 { 375 | path += fmt.Sprintf("&since_id=%v", since_id) 376 | } 377 | //log.Debug("path %q", path) 378 | res, err := tw.GET(path, nil) 379 | if err != nil { 380 | return nil, err 381 | } 382 | return res, nil 383 | } 384 | 385 | func createTwitterClient(credentialsFile string) (*tweeter.Client, error) { 386 | creds, err := loadINIFile(credentialsFile) 387 | if err != nil { 388 | return nil, err 389 | } 390 | getcred := func(k string) string { 391 | v := creds[k] 392 | if v == "" { 393 | err = fmt.Errorf("missing credentials key %q", k) 394 | } 395 | return v 396 | } 397 | params := &tweeter.ClientParams{ 398 | ConsumerKey: getcred("ConsumerKey"), 399 | ConsumerSecret: getcred("ConsumerSecret"), 400 | AccessToken: getcred("AccessToken"), 401 | AccessTokenSecret: getcred("AccessTokenSecret"), 402 | } 403 | return tweeter.NewClient(params), err 404 | } 405 | 406 | // loadINIFile reads an INI-style file, with lines of the format "key:value" or "key=value". 407 | // Skips empty lines and lines starting with ";" or "#". Does not support sections ("[section]".) 408 | func loadINIFile(filename string) (map[string]string, error) { 409 | buf, err := ioutil.ReadFile(filename) 410 | if err != nil { 411 | return nil, err 412 | } 413 | m := map[string]string{} 414 | for lineno, line := range bytes.Split(buf, []byte("\n")) { 415 | if len(line) == 0 || line[0] == ';' || line[0] == '#' { 416 | continue 417 | } 418 | if line[0] == '[' { 419 | return m, fmt.Errorf("sections are no supported (%s:%d)", filename, lineno+1) 420 | } 421 | i := bytes.IndexByte(line, ':') 422 | if i == -1 { 423 | i = bytes.IndexByte(line, '=') 424 | if i == -1 { 425 | return m, fmt.Errorf("key without value (%s:%d)", filename, lineno+1) 426 | } 427 | } 428 | key := string(line[:i]) 429 | value := string(bytes.TrimLeft(line[i+1:], " \t")) 430 | m[key] = value 431 | } 432 | return m, nil 433 | } 434 | 435 | func json_prettyprint(buf []byte) { 436 | var buf2 bytes.Buffer 437 | json.Indent(&buf2, buf, "", " ") 438 | os.Stdout.Write(buf2.Bytes()) 439 | os.Stdout.Write([]byte("\n")) 440 | } 441 | 442 | func createFile(filename string) (*os.File, error) { 443 | return os.OpenFile(filename, os.O_RDWR|os.O_CREATE|os.O_TRUNC, NEW_FILE_MODE) 444 | } 445 | 446 | func datapath(names ...string) string { 447 | names = append([]string{DATA_DIR}, names...) 448 | return filepath.Join(names...) 449 | } 450 | 451 | func nicepath(path string) string { 452 | if s, err := filepath.Rel(DATA_DIR, path); err == nil { 453 | if !strings.HasPrefix(s, "../") { 454 | return s 455 | } 456 | } 457 | return path 458 | } 459 | 460 | func u64min(a, b uint64) uint64 { 461 | if a < b { 462 | return a 463 | } 464 | return b 465 | } 466 | 467 | func u64max(a, b uint64) uint64 { 468 | if a > b { 469 | return a 470 | } 471 | return b 472 | } 473 | -------------------------------------------------------------------------------- /services/twitter/tweeter/conversions.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Arne Roomann-Kurrik 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package tweeter 16 | 17 | func arrayValue(m map[string]interface{}, key string) []interface{} { 18 | v, exists := m[key] 19 | if exists { 20 | return v.([]interface{}) 21 | } else { 22 | return []interface{}{} 23 | } 24 | } 25 | 26 | func boolValue(m map[string]interface{}, key string) bool { 27 | v, exists := m[key] 28 | if exists { 29 | return v.(bool) 30 | } else { 31 | return false 32 | } 33 | } 34 | 35 | func int32Value(m map[string]interface{}, key string) int32 { 36 | v, exists := m[key] 37 | if exists { 38 | return v.(int32) 39 | } else { 40 | return 0 41 | } 42 | } 43 | 44 | func int64Value(m map[string]interface{}, key string) int64 { 45 | v, exists := m[key] 46 | if exists { 47 | i, ok := v.(int64) 48 | if !ok { 49 | i = int64(v.(float64)) 50 | } 51 | return i 52 | } 53 | return 0 54 | } 55 | 56 | func float64Value(m map[string]interface{}, key string) float64 { 57 | v, exists := m[key] 58 | if exists { 59 | f, ok := v.(float64) 60 | if !ok { 61 | f = float64(v.(int64)) 62 | } 63 | return f 64 | } 65 | return 0.0 66 | } 67 | 68 | func mapValue(m map[string]interface{}, key string) map[string]interface{} { 69 | v, exists := m[key] 70 | if exists { 71 | return v.(map[string]interface{}) 72 | } else { 73 | return map[string]interface{}{} 74 | } 75 | } 76 | 77 | func stringValue(m map[string]interface{}, key string) string { 78 | v, exists := m[key] 79 | if exists { 80 | return v.(string) 81 | } else { 82 | return "" 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /services/twitter/tweeter/like.go: -------------------------------------------------------------------------------- 1 | package tweeter 2 | 3 | // Like is "favorite", recorded when you tap a heart in Twitter 4 | type Like struct { 5 | CreatedAtStr string `json:"created_at"` 6 | Id uint64 `json:"id"` 7 | User User `json:"user"` 8 | QuotedStatusId uint64 `json:"quoted_status_id"` 9 | QuotedStatus *Tweet `json:"quoted_status"` 10 | IsTruncated bool `json:"truncated"` 11 | } 12 | -------------------------------------------------------------------------------- /services/twitter/tweeter/media.go: -------------------------------------------------------------------------------- 1 | package tweeter 2 | 3 | import ( 4 | "io" 5 | "mime/multipart" 6 | "time" 7 | ) 8 | 9 | type MediaImage struct { 10 | ImageType string `json:"image_type"` // e.g. "image/jpeg" 11 | Width int `json:"w"` 12 | Height int `json:"h"` 13 | } 14 | 15 | type MediaResponse struct { 16 | Id uint64 `json:"media_id"` 17 | Size int `json:"size"` 18 | ExpiresAfter time.Duration `json:"expires_after_secs"` 19 | Image *MediaImage 20 | } 21 | 22 | type MediaUpload struct { 23 | Name string 24 | BodyReader io.Reader // either set BodyReader or BodyData 25 | BodyData []byte // either set BodyReader or BodyData 26 | } 27 | 28 | func (r *MediaUpload) Send(c *Client) (mr *MediaResponse, err error) { 29 | endpoint := "https://upload.twitter.com/1.1/media/upload.json" 30 | res, err := c.POSTForm(endpoint, func(m *multipart.Writer) error { 31 | mediaWriter, err := m.CreateFormFile("media", r.Name) 32 | if err != nil { 33 | return err 34 | } 35 | if r.BodyReader != nil { 36 | if _, err = io.Copy(mediaWriter, r.BodyReader); err != nil { 37 | return err 38 | } 39 | } else { 40 | if _, err := mediaWriter.Write(r.BodyData); err != nil { 41 | return err 42 | } 43 | } 44 | return nil 45 | }) 46 | 47 | mr = &MediaResponse{} 48 | if err = res.Parse(mr); err != nil { 49 | mr = nil 50 | } else { 51 | mr.ExpiresAfter = mr.ExpiresAfter * time.Second 52 | } 53 | return 54 | } 55 | -------------------------------------------------------------------------------- /services/twitter/tweeter/response.go: -------------------------------------------------------------------------------- 1 | package tweeter 2 | 3 | import ( 4 | "compress/gzip" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "strconv" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | const ( 16 | H_LIMIT = "X-Rate-Limit-Limit" 17 | H_LIMIT_REMAIN = "X-Rate-Limit-Remaining" 18 | H_LIMIT_RESET = "X-Rate-Limit-Reset" 19 | H_MEDIA_LIMIT = "X-MediaRateLimit-Limit" 20 | H_MEDIA_LIMIT_REMAIN = "X-MediaRateLimit-Remaining" 21 | H_MEDIA_LIMIT_RESET = "X-MediaRateLimit-Reset" 22 | ) 23 | 24 | const ( 25 | STATUS_OK = 200 26 | STATUS_CREATED = 201 27 | STATUS_ACCEPTED = 202 28 | STATUS_NO_CONTENT = 204 29 | STATUS_INVALID = 400 30 | STATUS_UNAUTHORIZED = 401 31 | STATUS_FORBIDDEN = 403 32 | STATUS_NOTFOUND = 404 33 | STATUS_LIMIT = 429 34 | STATUS_GATEWAY = 502 35 | ) 36 | 37 | // Error returned if there was an issue parsing the response body. 38 | type ResponseError struct { 39 | Body string 40 | Code int 41 | } 42 | 43 | func NewResponseError(code int, body string) ResponseError { 44 | return ResponseError{Code: code, Body: body} 45 | } 46 | 47 | func (e ResponseError) Error() string { 48 | return fmt.Sprintf( 49 | "Unable to handle response (status code %d): `%v`", 50 | e.Code, 51 | e.Body) 52 | } 53 | 54 | // ----------------------------------------------------------------------------------------- 55 | 56 | // RateLimitError is returned from SendRequest when a rate limit is encountered. 57 | type RateLimitError struct { 58 | Limit uint32 // the rate limit ceiling for that given endpoint 59 | Remaining uint32 // the number of requests left for {Limit} window 60 | Reset time.Time // the remaining window before the rate limit resets 61 | } 62 | 63 | func (e RateLimitError) Error() string { 64 | msg := "Rate limit: %v, Remaining: %v, Reset: %v" 65 | return fmt.Sprintf(msg, e.Limit, e.Remaining, e.Reset) 66 | } 67 | 68 | func (e RateLimitError) HasRateLimit() bool { 69 | return true 70 | } 71 | 72 | func (e RateLimitError) RateLimit() uint32 { 73 | return e.Limit 74 | } 75 | 76 | func (e RateLimitError) RateLimitRemaining() uint32 { 77 | return e.Remaining 78 | } 79 | 80 | func (e RateLimitError) RateLimitReset() time.Time { 81 | return e.Reset 82 | } 83 | 84 | // ----------------------------------------------------------------------------------------- 85 | 86 | type Error map[string]interface{} 87 | 88 | func (e Error) Code() int64 { 89 | return int64(float64Value(e, "code")) 90 | } 91 | 92 | func (e Error) Message() string { 93 | return stringValue(e, "message") 94 | } 95 | 96 | func (e Error) Error() string { 97 | return fmt.Sprintf("Error %v: %v", e.Code(), e.Message()) 98 | } 99 | 100 | type Errors map[string]interface{} 101 | 102 | func (e Errors) Error() string { 103 | var ( 104 | msg string = "" 105 | err Error 106 | ok bool 107 | errs []interface{} 108 | ) 109 | errs = arrayValue(e, "errors") 110 | if len(errs) == 0 { 111 | return msg 112 | } 113 | for _, val := range errs { 114 | if err, ok = val.(map[string]interface{}); ok { 115 | msg += err.Error() + ". " 116 | } 117 | } 118 | return msg 119 | } 120 | 121 | func (e Errors) String() string { 122 | return e.Error() 123 | } 124 | 125 | func (e Errors) Errors() []Error { 126 | var errs = arrayValue(e, "errors") 127 | var out = make([]Error, len(errs)) 128 | for i, val := range errs { 129 | out[i] = Error(val.(map[string]interface{})) 130 | } 131 | return out 132 | } 133 | 134 | // ----------------------------------------------------------------------------------------- 135 | 136 | // ApiResponse provides methods for retrieving information from the HTTP 137 | // headers in a Twitter API response. 138 | type ApiResponse http.Response 139 | 140 | func (r ApiResponse) HasRateLimit() bool { 141 | return r.Header.Get(H_LIMIT) != "" 142 | } 143 | 144 | func (r ApiResponse) RateLimit() uint32 { 145 | h := r.Header.Get(H_LIMIT) 146 | i, _ := strconv.ParseUint(h, 10, 32) 147 | return uint32(i) 148 | } 149 | 150 | func (r ApiResponse) RateLimitRemaining() uint32 { 151 | h := r.Header.Get(H_LIMIT_REMAIN) 152 | i, _ := strconv.ParseUint(h, 10, 32) 153 | return uint32(i) 154 | } 155 | 156 | func (r ApiResponse) RateLimitReset() time.Time { 157 | h := r.Header.Get(H_LIMIT_RESET) 158 | i, _ := strconv.ParseUint(h, 10, 32) 159 | t := time.Unix(int64(i), 0) 160 | return t 161 | } 162 | 163 | func (r ApiResponse) HasMediaRateLimit() bool { 164 | return r.Header.Get(H_MEDIA_LIMIT) != "" 165 | } 166 | 167 | func (r ApiResponse) MediaRateLimit() uint32 { 168 | h := r.Header.Get(H_MEDIA_LIMIT) 169 | i, _ := strconv.ParseUint(h, 10, 32) 170 | return uint32(i) 171 | } 172 | 173 | func (r ApiResponse) MediaRateLimitRemaining() uint32 { 174 | h := r.Header.Get(H_MEDIA_LIMIT_REMAIN) 175 | i, _ := strconv.ParseUint(h, 10, 32) 176 | return uint32(i) 177 | } 178 | 179 | func (r ApiResponse) MediaRateLimitReset() time.Time { 180 | h := r.Header.Get(H_MEDIA_LIMIT_RESET) 181 | i, _ := strconv.ParseUint(h, 10, 32) 182 | t := time.Unix(int64(i), 0) 183 | return t 184 | } 185 | 186 | func (r ApiResponse) GetBodyReader() (io.ReadCloser, error) { 187 | header := strings.ToLower(r.Header.Get("Content-Encoding")) 188 | if header == "" || strings.Index(header, "gzip") == -1 { 189 | return r.Body, nil 190 | } 191 | return gzip.NewReader(r.Body) 192 | } 193 | 194 | func (r ApiResponse) ReadBody() ([]byte, error) { 195 | reader, err := r.GetBodyReader() 196 | if err != nil { 197 | return nil, err 198 | } 199 | defer reader.Close() 200 | return ioutil.ReadAll(reader) 201 | } 202 | 203 | // Parse unmarshals a JSON encoded HTTP response into the supplied interface, 204 | // with handling for the various kinds of errors the Twitter API can return. 205 | // 206 | // The returned error may be of the type Errors, RateLimitError, 207 | // ResponseError, or an error returned from io.Reader.Read(). 208 | func (r ApiResponse) Parse(out interface{}) (err error) { 209 | var b []byte 210 | switch r.StatusCode { 211 | 212 | case STATUS_UNAUTHORIZED: 213 | fallthrough 214 | 215 | case STATUS_NOTFOUND: 216 | fallthrough 217 | 218 | case STATUS_GATEWAY: 219 | fallthrough 220 | 221 | case STATUS_FORBIDDEN: 222 | fallthrough 223 | 224 | case STATUS_INVALID: 225 | e := &Errors{} 226 | if b, err = r.ReadBody(); err != nil { 227 | return 228 | } 229 | if err = json.Unmarshal(b, e); err != nil { 230 | err = NewResponseError(r.StatusCode, string(b)) 231 | } else { 232 | err = *e 233 | } 234 | return 235 | 236 | case STATUS_LIMIT: 237 | err = RateLimitError{ 238 | Limit: r.RateLimit(), 239 | Remaining: r.RateLimitRemaining(), 240 | Reset: r.RateLimitReset(), 241 | } 242 | // consume the request body even if we don't need it 243 | r.ReadBody() 244 | return 245 | 246 | case STATUS_NO_CONTENT: 247 | return 248 | 249 | case STATUS_CREATED: 250 | fallthrough 251 | 252 | case STATUS_ACCEPTED: 253 | fallthrough 254 | 255 | case STATUS_OK: 256 | reader, err := r.GetBodyReader() 257 | if err != nil { 258 | return err 259 | } 260 | defer reader.Close() 261 | dec := json.NewDecoder(reader) 262 | // dec.UseNumber() 263 | if err = dec.Decode(out); err != nil && err == io.EOF { 264 | err = nil 265 | } 266 | // if b, err = r.ReadBody(); err != nil { 267 | // return 268 | // } 269 | // err = json.Unmarshal(b, out) 270 | // if err == io.EOF { 271 | // err = nil 272 | // } 273 | 274 | default: 275 | if b, err = r.ReadBody(); err != nil { 276 | return 277 | } 278 | err = NewResponseError(r.StatusCode, string(b)) 279 | } 280 | return 281 | } 282 | -------------------------------------------------------------------------------- /services/twitter/tweeter/tweet.go: -------------------------------------------------------------------------------- 1 | package tweeter 2 | 3 | import ( 4 | "mime/multipart" 5 | "strconv" 6 | "time" 7 | ) 8 | 9 | // Tweet represents an exisiting tweet 10 | type Tweet struct { 11 | CreatedAtStr string `json:"created_at"` 12 | Id uint64 `json:"id"` 13 | Text string `json:"text"` 14 | User User `json:"user"` 15 | RetweetCount uint64 `json:"retweet_count"` 16 | FavoriteCount uint64 `json:"favorite_count"` 17 | IsFavorited bool `json:"favorited"` 18 | IsRetweeted bool `json:"retweeted"` 19 | InReplyToStatusId uint64 `json:"in_reply_to_status_id"` 20 | InReplyToUserId uint64 `json:"in_reply_to_user_id"` 21 | InReplyToScreenName string `json:"in_reply_to_screen_name"` 22 | } 23 | 24 | func (t *Tweet) CreatedAt() time.Time { 25 | // e.g. "Sat Aug 17 18:05:53 +0000 2019" 26 | tm, _ := time.Parse(time.RubyDate, t.CreatedAtStr) 27 | return tm 28 | } 29 | 30 | // StatusUpdate is used to create a new Tweet 31 | type StatusUpdate struct { 32 | Status string 33 | MediaIds []uint64 34 | } 35 | 36 | func (r *StatusUpdate) Send(c *Client) (*Tweet, error) { 37 | qs := "?trim_user=1" // skip user object in response 38 | 39 | if len(r.MediaIds) > 0 { 40 | qs += "&media_ids=" 41 | for i, u := range r.MediaIds { 42 | if i > 0 { 43 | qs += "," 44 | } 45 | qs += strconv.FormatUint(u, 10) 46 | } 47 | } 48 | 49 | res, err := c.POSTForm("/1.1/statuses/update.json"+qs, func(m *multipart.Writer) error { 50 | // Note: we can't write trim_user or media_ids here for some reason 51 | m.WriteField("status", r.Status) 52 | return nil 53 | }) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | tw := &Tweet{} 59 | return tw, res.Parse(tw) 60 | } 61 | -------------------------------------------------------------------------------- /services/twitter/tweeter/tweeter.go: -------------------------------------------------------------------------------- 1 | // This code is based on https://github.com/kurrik/twittergo licensed as follows (Apache 2.0) 2 | // Copyright 2019 Arne Roomann-Kurrik 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | // 16 | package tweeter 17 | 18 | import ( 19 | "bytes" 20 | "crypto/tls" 21 | "encoding/base64" 22 | "encoding/json" 23 | "fmt" 24 | "io/ioutil" 25 | "log" 26 | "mime/multipart" 27 | "net/http" 28 | "net/url" 29 | "os" 30 | "strconv" 31 | "strings" 32 | 33 | "github.com/kurrik/oauth1a" 34 | ) 35 | 36 | const ( 37 | MaxTweetLength = 280 38 | UrlTweetLength = 23 // number of characters a URL debit from MaxTweetLength 39 | ) 40 | 41 | type Client struct { 42 | Host string 43 | OAuth *oauth1a.Service 44 | User *oauth1a.UserConfig 45 | HttpClient *http.Client 46 | AppToken string 47 | } 48 | 49 | type ClientParams struct { 50 | ConsumerKey string 51 | ConsumerSecret string 52 | AccessToken string 53 | AccessTokenSecret string 54 | } 55 | 56 | func NewClient(p *ClientParams) *Client { 57 | config := &oauth1a.ClientConfig{ 58 | ConsumerKey: p.ConsumerKey, 59 | ConsumerSecret: p.ConsumerSecret, 60 | } 61 | user := oauth1a.NewAuthorizedConfig(p.AccessToken, p.AccessTokenSecret) 62 | return NewClientWithConfig(config, user) 63 | } 64 | 65 | func NewClientWithConfig(config *oauth1a.ClientConfig, user *oauth1a.UserConfig) *Client { 66 | var ( 67 | host = "api.twitter.com" 68 | base = "https://" + host 69 | req, _ = http.NewRequest("GET", "https://api.twitter.com", nil) 70 | proxy, _ = http.ProxyFromEnvironment(req) 71 | transport *http.Transport 72 | tlsconfig *tls.Config 73 | ) 74 | if proxy != nil { 75 | tlsconfig = &tls.Config{ 76 | InsecureSkipVerify: getEnvEitherCase("TLS_INSECURE") != "", 77 | } 78 | if tlsconfig.InsecureSkipVerify { 79 | log.Println("WARNING: SSL cert verification disabled") 80 | } 81 | transport = &http.Transport{ 82 | Proxy: http.ProxyURL(proxy), 83 | TLSClientConfig: tlsconfig, 84 | } 85 | } else { 86 | transport = &http.Transport{} 87 | } 88 | return &Client{ 89 | Host: host, 90 | HttpClient: &http.Client{ 91 | Transport: transport, 92 | }, 93 | User: user, 94 | OAuth: &oauth1a.Service{ 95 | RequestURL: base + "/oauth/request_token", 96 | AuthorizeURL: base + "/oauth/authorize", 97 | AccessURL: base + "/oauth/access_token", 98 | ClientConfig: config, 99 | Signer: new(oauth1a.HmacSha1Signer), 100 | }, 101 | } 102 | } 103 | 104 | // Sends a HTTP request through this instance's HTTP client. 105 | func (c *Client) HttpRequest(req *http.Request) (resp *ApiResponse, err error) { 106 | if len(req.URL.Scheme) == 0 { 107 | req.URL, err = url.Parse("https://" + c.Host + req.URL.String()) 108 | if err != nil { 109 | return 110 | } 111 | } 112 | if c.User != nil { 113 | c.OAuth.Sign(req, c.User) 114 | } else if err = c.Sign(req); err != nil { 115 | return 116 | } 117 | var r *http.Response 118 | r, err = c.HttpClient.Do(req) 119 | resp = (*ApiResponse)(r) 120 | return 121 | } 122 | 123 | func (c *Client) GET(endpoint string, headers map[string]string) (*ApiResponse, error) { 124 | req, err := http.NewRequest("GET", endpoint, nil) 125 | if err != nil { 126 | return nil, err 127 | } 128 | return c.HttpRequest(req) 129 | } 130 | 131 | func (c *Client) POSTForm(endpoint string, fw func(*multipart.Writer) error) (*ApiResponse, error) { 132 | body := bytes.NewBufferString("") 133 | mp := multipart.NewWriter(body) 134 | fw(mp) 135 | mp.Close() 136 | req, err := http.NewRequest("POST", endpoint, body) 137 | if err != nil { 138 | return nil, err 139 | } 140 | req.Header.Set("Content-Type", "multipart/form-data;boundary="+mp.Boundary()) 141 | req.Header.Set("Content-Length", strconv.Itoa(body.Len())) 142 | return c.HttpRequest(req) 143 | } 144 | 145 | // Changes the user authorization credentials for this client. 146 | func (c *Client) SetUser(user *oauth1a.UserConfig) { 147 | c.User = user 148 | } 149 | 150 | func (c *Client) FetchAppToken() (err error) { 151 | var ( 152 | req *http.Request 153 | resp *http.Response 154 | rb []byte 155 | rj = map[string]interface{}{} 156 | url = fmt.Sprintf("https://%v/oauth2/token", c.Host) 157 | ct = "application/x-www-form-urlencoded;charset=UTF-8" 158 | body = "grant_type=client_credentials" 159 | ek = oauth1a.Rfc3986Escape(c.OAuth.ClientConfig.ConsumerKey) 160 | es = oauth1a.Rfc3986Escape(c.OAuth.ClientConfig.ConsumerSecret) 161 | cred = fmt.Sprintf("%v:%v", ek, es) 162 | ec = base64.StdEncoding.EncodeToString([]byte(cred)) 163 | h = fmt.Sprintf("Basic %v", ec) 164 | ) 165 | req, err = http.NewRequest("POST", url, bytes.NewBufferString(body)) 166 | if err != nil { 167 | return 168 | } 169 | req.Header.Set("Authorization", h) 170 | req.Header.Set("Content-Type", ct) 171 | if resp, err = c.HttpClient.Do(req); err != nil { 172 | return 173 | } 174 | if resp.StatusCode != 200 { 175 | err = fmt.Errorf("Got HTTP %v instead of 200", resp.StatusCode) 176 | return 177 | } 178 | if rb, err = ioutil.ReadAll(resp.Body); err != nil { 179 | return 180 | } 181 | if err = json.Unmarshal(rb, &rj); err != nil { 182 | return 183 | } 184 | var ( 185 | token_type = rj["token_type"].(string) 186 | access_token = rj["access_token"].(string) 187 | ) 188 | if token_type != "bearer" { 189 | err = fmt.Errorf("Got invalid token type: %v", token_type) 190 | } 191 | c.AppToken = access_token 192 | return nil 193 | } 194 | 195 | // Signs the request with app-only auth, fetching a bearer token if needed. 196 | func (c *Client) Sign(req *http.Request) error { 197 | if len(c.AppToken) == 0 { 198 | if err := c.FetchAppToken(); err != nil { 199 | return err 200 | } 201 | } 202 | req.Header.Set("Authorization", "Bearer "+c.AppToken) 203 | return nil 204 | } 205 | 206 | func getEnvEitherCase(k string) string { 207 | if v := os.Getenv(strings.ToUpper(k)); v != "" { 208 | return v 209 | } 210 | return os.Getenv(strings.ToLower(k)) 211 | } 212 | -------------------------------------------------------------------------------- /services/twitter/tweeter/user.go: -------------------------------------------------------------------------------- 1 | package tweeter 2 | 3 | type User struct { 4 | Id uint64 `json:"id"` 5 | Name string `json:"name"` 6 | ScreenName string `json:"screen_name"` 7 | Location string `json:"location"` 8 | Description string `json:"description"` 9 | Url string `json:"url"` 10 | CreatedAtStr string `json:"created_at"` 11 | FollowersCount uint64 `json:"followers_count"` 12 | FollowingCount uint64 `json:"friends_count"` 13 | FavoritesCount uint64 `json:"favourites_count"` 14 | } 15 | -------------------------------------------------------------------------------- /services/twitter/twitter-credentials.ini.in: -------------------------------------------------------------------------------- 1 | # Make a Twitter dev app and then generate key & secret at 2 | # https://developer.twitter.com/en/portal/apps/YOURAPPID/keys 3 | ConsumerKey: REPLACEME 4 | ConsumerSecret: REPLACEME 5 | AccessToken: REPLACEME 6 | AccessTokenSecret: REPLACEME 7 | -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 0.1.0 2 | --------------------------------------------------------------------------------