├── jsonxt ├── staticcheck.conf ├── tags.go ├── readme.md ├── LICENSE ├── indent.go ├── fold.go └── tables.go ├── CODE_OF_CONDUCT.md ├── lib ├── config.go ├── makepb.sh ├── clang │ ├── notehub-udp.pb.c │ ├── notehub-udp.pb.h │ ├── notehub.pb.c │ └── notehub.pb.h ├── main_test.go ├── error.go ├── notehub.options ├── cert.go ├── json.go ├── event_test.go ├── storage.go ├── notehub.proto ├── fileio.go ├── notelib.go ├── discover.go ├── event.go ├── notebox_bug_demo_test.go ├── notebox_race_test.go ├── olc64.go ├── json_fix.go ├── notebox_race_summary_test.go ├── concurrency_test.go ├── smazz_test.go ├── debug.go ├── notebox_aggressive_race_test.go ├── float16.go ├── smazz.go ├── wire_test.go ├── notebox_extended_race_test.go └── file.go ├── .gitignore ├── README.md ├── go.mod ├── hub ├── tcp.go ├── hub.sh ├── discover.go ├── main.go ├── session.go ├── event.go ├── http.go ├── device.go └── tcps.go ├── CONTRIBUTING.md └── go.sum /jsonxt/staticcheck.conf: -------------------------------------------------------------------------------- 1 | # Disabling staticcheck in the jsonxt package because it is forked code, and we 2 | # want to leave it as similar to the original as possible. 3 | checks = [] 4 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of conduct 2 | 3 | By participating in this project, you agree to abide by the 4 | [Blues Inc code of conduct][1]. 5 | 6 | [1]: https://blues.github.io/opensource/code-of-conduct 7 | 8 | -------------------------------------------------------------------------------- /lib/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Package notelib config.go is where package-wide constants and vars are defined 6 | package notelib 7 | 8 | // defaultMaxGetNoteboxChangesBatchSize is the default change size for noteboxes 9 | const defaultMaxGetNoteboxChangesBatchSize = 150 10 | -------------------------------------------------------------------------------- /lib/makepb.sh: -------------------------------------------------------------------------------- 1 | ## To set up: 2 | ## brew install protoc-gen-go 3 | ## protoc executable is placed on the standard path after installing standard PB support 4 | 5 | protoc --go_opt=Mnotehub.proto=./notelib --go_out=. notehub.proto 6 | mv notelib/*.go . 7 | rmdir notelib 8 | 9 | ## Download and unzip nanopb sdk into a folder, and point to it with the env var NANOPBSDK 10 | ## https://koti.kapsi.fi/jpa/nanopb/ 11 | ## Then, 12 | export NANOPBSDK=~/dev/nordic/external/nano-pb 13 | ${NANOPBSDK}/generator-bin/protoc -I./ -I${NANOPBSDK}/generator/proto/ --nanopb_out=./clang notehub.proto 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | **/.idea 2 | 3 | # Binaries for programs and plugins 4 | *.exe 5 | *.exe~ 6 | *.dll 7 | *.so 8 | *.dylib 9 | 10 | # env files 11 | *.env 12 | 13 | # Test binary, built with `go test -c` 14 | *.test 15 | 16 | # Output of the go coverage tool, specifically when used with LiteIDE 17 | *.out 18 | 19 | # Go build outputs 20 | **/build 21 | hub/hub 22 | lib/lib 23 | 24 | # auto- generated files # 25 | ###################### 26 | *~ 27 | \#*\# 28 | .DS_Store 29 | .DS_Store? 30 | ._* 31 | .Spotlight-V100 32 | .Trashes 33 | ehthumbs.db 34 | Thumbs.db 35 | 36 | # VS Code configuration files 37 | .vscode 38 | -------------------------------------------------------------------------------- /lib/clang/notehub-udp.pb.c: -------------------------------------------------------------------------------- 1 | /* Automatically generated nanopb constant definitions */ 2 | /* Generated by nanopb-0.3.5 at Sun Sep 10 15:08:14 2023. */ 3 | 4 | #include "notehub-udp.pb.h" 5 | 6 | /* @@protoc_insertion_point(includes) */ 7 | #if PB_PROTO_HEADER_VERSION != 30 8 | #error Regenerate this file with the current version of nanopb generator. 9 | #endif 10 | 11 | 12 | 13 | const pb_field_t notelib_NotehubUdpPB_fields[2] = { 14 | PB_FIELD( 1, BYTES , OPTIONAL, STATIC , FIRST, notelib_NotehubUdpPB, Payload, Payload, 0), 15 | PB_LAST_FIELD 16 | }; 17 | 18 | 19 | /* @@protoc_insertion_point(eof) */ 20 | -------------------------------------------------------------------------------- /lib/main_test.go: -------------------------------------------------------------------------------- 1 | package notelib 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | // TestMain runs before all tests in this package 11 | func TestMain(m *testing.M) { 12 | // Silent checkpoints 13 | CheckpointSilently = true 14 | 15 | // Set errorLoggerFunc to panic immediately for all tests 16 | // This will cause the test to fail properly with a stack trace 17 | errorLoggerFunc = func(ctx context.Context, msg string, v ...interface{}) { 18 | errMsg := fmt.Sprintf(msg, v...) 19 | fmt.Printf("ERROR: %s\n", errMsg) 20 | panic(errMsg) // Panic immediately - tests will fail with proper reporting 21 | } 22 | 23 | // Run tests 24 | code := m.Run() 25 | 26 | // Exit with test result code 27 | os.Exit(code) 28 | } 29 | 30 | func longRunningTest(t *testing.T) { 31 | t.Skip() 32 | } 33 | -------------------------------------------------------------------------------- /lib/error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Package notelib err.go contains things that assist in error handling 6 | package notelib 7 | 8 | import ( 9 | "fmt" 10 | "strings" 11 | ) 12 | 13 | // ErrorContains tests to see if an error contains an error keyword that we might expect 14 | func ErrorContains(err error, errKeyword string) bool { 15 | if err == nil { 16 | return false 17 | } 18 | return strings.Contains(fmt.Sprintf("%s", err), errKeyword) 19 | } 20 | 21 | // ErrorString safely returns a string from any error, returning "" for nil 22 | func ErrorString(err error) string { 23 | if err == nil { 24 | return "" 25 | } 26 | return fmt.Sprintf("%s", err) 27 | } 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NOTEHUB TM 2 | 3 | The Blues toolset for manipulating notes and the the basics for creating a hub 4 | server. 5 | 6 | 7 | See also: 8 | * [note-go][note-go] for the Go API for communicating with the Notecard + other 9 | go tools 10 | 11 | [notehub]: https://notehub.io 12 | [note-go]: https://github.com/blues/note-go 13 | 14 | ## How to contribute 15 | 16 | Contributions are welcome! 17 | 18 | Please read the document [How to contribute](CONTRIBUTING.md) which will guide 19 | you through how to build the source code, run the tests, and contribute your 20 | changes to the project. 21 | 22 | ## How to install 23 | 24 | Add the package to your `go.mod` file: 25 | 26 | require github.com/blues/note 27 | 28 | Or, clone the repository: 29 | 30 | git clone --branch master https://github.com/blues/note.git $GOPATH/src/github.com/blues/note 31 | 32 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/blues/note 2 | 3 | go 1.25.1 4 | 5 | require ( 6 | github.com/blues/note-go v1.7.4 7 | github.com/golang/snappy v0.0.4 8 | github.com/google/open-location-code/go v0.0.0-20220120191843-cafb35c0d74d 9 | github.com/google/uuid v1.3.0 10 | github.com/stretchr/testify v1.11.1 11 | github.com/valyala/fastjson v1.6.4 12 | golang.org/x/crypto v0.39.0 13 | google.golang.org/protobuf v1.33.0 14 | ) 15 | 16 | require ( 17 | github.com/creack/goselect v0.1.2 // indirect 18 | github.com/davecgh/go-spew v1.1.1 // indirect 19 | github.com/go-ole/go-ole v1.2.6 // indirect 20 | github.com/lufia/plan9stats v0.0.0-20220326011226-f1430873d8db // indirect 21 | github.com/pmezard/go-difflib v1.0.0 // indirect 22 | github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect 23 | github.com/shirou/gopsutil/v3 v3.22.2 // indirect 24 | github.com/tklauser/go-sysconf v0.3.10 // indirect 25 | github.com/tklauser/numcpus v0.4.0 // indirect 26 | github.com/yusufpapurcu/wmi v1.2.2 // indirect 27 | go.bug.st/serial v1.6.1 // indirect 28 | golang.org/x/sys v0.33.0 // indirect 29 | gopkg.in/yaml.v3 v3.0.1 // indirect 30 | periph.io/x/conn/v3 v3.7.0 // indirect 31 | periph.io/x/host/v3 v3.8.0 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /hub/tcp.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Inbound TCP support 6 | package main 7 | 8 | import ( 9 | "fmt" 10 | "net" 11 | ) 12 | 13 | // tcpHandler kicks off TCP request server 14 | func tcpHandler() { 15 | fmt.Printf("Serving requests on tcp:%s%s\n", serverAddress, serverPortTCP) 16 | 17 | serverAddr, err := net.ResolveTCPAddr("tcp", serverPortTCP) 18 | if err != nil { 19 | fmt.Printf("tcp: error resolving TCP port: %v\n", err) 20 | return 21 | } 22 | 23 | connServer, err := net.ListenTCP("tcp", serverAddr) 24 | if err != nil { 25 | fmt.Printf("tcp: error listening on TCP port: %v\n", err) 26 | return 27 | } 28 | defer connServer.Close() 29 | 30 | for { 31 | 32 | // Accept the TCP connection 33 | connSession, err := connServer.AcceptTCP() 34 | if err != nil { 35 | fmt.Printf("tcp: error accepting TCP session: %v\n", err) 36 | continue 37 | } 38 | 39 | // The scope of a TCP connection may be many requests, so dispatch 40 | // this to a goroutine which will deal with the session until it is closed. 41 | go sessionHandler(connSession, false) 42 | 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /jsonxt/tags.go: -------------------------------------------------------------------------------- 1 | // Copyright 2011 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package jsonxt 6 | 7 | import ( 8 | "strings" 9 | ) 10 | 11 | // tagOptions is the string following a comma in a struct field's "json" 12 | // tag, or the empty string. It does not include the leading comma. 13 | type tagOptions string 14 | 15 | // parseTag splits a struct field's json tag into its name and 16 | // comma-separated options. 17 | func parseTag(tag string) (string, tagOptions) { 18 | if idx := strings.Index(tag, ","); idx != -1 { 19 | return tag[:idx], tagOptions(tag[idx+1:]) 20 | } 21 | return tag, tagOptions("") 22 | } 23 | 24 | // Contains reports whether a comma-separated list of options 25 | // contains a particular substr flag. substr must be surrounded by a 26 | // string boundary or commas. 27 | func (o tagOptions) Contains(optionName string) bool { 28 | if len(o) == 0 { 29 | return false 30 | } 31 | s := string(o) 32 | for s != "" { 33 | var next string 34 | i := strings.Index(s, ",") 35 | if i >= 0 { 36 | s, next = s[:i], s[i+1:] 37 | } 38 | if s == optionName { 39 | return true 40 | } 41 | s = next 42 | } 43 | return false 44 | } 45 | -------------------------------------------------------------------------------- /lib/notehub.options: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | notelib.NotehubPB.MessageType max_size:25 6 | notelib.NotehubPB.Error max_size:254 7 | notelib.NotehubPB.DeviceUID max_size:100 8 | notelib.NotehubPB.ProductUID max_size:254 9 | notelib.NotehubPB.DeviceEndpointID max_size:100 10 | notelib.NotehubPB.HubEndpointID max_size:100 11 | notelib.NotehubPB.HubSessionHandler max_size:254 12 | notelib.NotehubPB.HubPacketHandler max_size:254 13 | notelib.NotehubPB.NotefileID max_size:254 14 | notelib.NotehubPB.NotefileIDs max_size:254 15 | notelib.NotehubPB.HubSessionTicket max_size:128 16 | notelib.NotehubPB.DeviceSN max_size:254 17 | notelib.NotehubPB.NoteID max_size:254 18 | notelib.NotehubPB.CellID max_size:254 19 | notelib.NotehubPB.MotionOrientation max_size:100 20 | notelib.NotehubPB.SessionTrigger max_size:254 21 | notelib.NotehubPB.HubSessionFactoryResetID max_size:100 22 | notelib.NotehubPB.DeviceSKU max_size:100 23 | notelib.NotehubPB.DeviceOrderingCode max_size:254 24 | notelib.NotehubPB.DevicePIN max_size:254 25 | notelib.NotehubPB.Where max_size:25 26 | notelib.NotehubPB.SocketAlias max_size:100 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /lib/cert.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Package notelib cert.go contains certificates 6 | package notelib 7 | 8 | // GetNotecardSecureElementRootCertificateAsPEM is the ST Microelectronics root certificate 9 | // used to generate the certificate embedded in every Notecard's STSAFE-A100 secure element 10 | // at point of chip manufacture. 11 | func GetNotecardSecureElementRootCertificateAsPEM() (cert []byte) { 12 | return []byte(` 13 | -----BEGIN CERTIFICATE----- 14 | MIIB3TCCAWOgAwIBAgIBATAKBggqhkjOPQQDAzBPMQswCQYDVQQGEwJOTDEeMBwG 15 | A1UECgwVU1RNaWNyb2VsZWN0cm9uaWNzIG52MSAwHgYDVQQDDBdTVE0gU1RTQUZF 16 | LUEgUFJPRCBDQSAyNTAeFw0xNzExMTYwMDAwMDBaFw00NzExMTYwMDAwMDBaME8x 17 | CzAJBgNVBAYTAk5MMR4wHAYDVQQKDBVTVE1pY3JvZWxlY3Ryb25pY3MgbnYxIDAe 18 | BgNVBAMMF1NUTSBTVFNBRkUtQSBQUk9EIENBIDI1MHYwEAYHKoZIzj0CAQYFK4EE 19 | ACIDYgAEzJ3UwY415esvvbw/pz2J/UXW6M234YkY4jNCVJieAYMxwtMHDhO1eQke 20 | TAS/WNMz1SHuQ/1gXaS7hfLuk89XJ7dYGsCnEY1GKh6AHVzLo1xk/W6xNNvofoTe 21 | fooVH1sNoxMwETAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMBG2 22 | LhehNFswV9otQG+8RM8BElZHHHH5I40XECvbHu8cVS1m1bmKFG5qhXPp1y1PbgIx 23 | AISPZKUEFntIoVgNkSJO9zi6/RuI25XrikU42sHNVI5YqR2i2PFkaBugfwCYPpxb 24 | 4g== 25 | -----END CERTIFICATE----- 26 | `) 27 | } 28 | -------------------------------------------------------------------------------- /hub/hub.sh: -------------------------------------------------------------------------------- 1 | if [ "$1" == "" ]; then 2 | echo "Before running your server with your notehub domain name, you must" 3 | echo "make a TLS key/cert for it. If you run your server with this shell" 4 | echo "script, a self-signed cert will be created for you." 5 | echo "" 6 | echo "hub.sh your.notehub.domain.name.io" 7 | echo "" 8 | exit 0 9 | fi 10 | set -x 11 | 12 | # Create the security directory if it doesn't exist. Note that this script creates keys 13 | # in this subfolder. See keyDirectory() in main.go 14 | mkdir -p ~/note/keys; 15 | 16 | # Make a certificate with your domain name 17 | # Note that the key generated by this command is referenced only in tcps.go 18 | openssl req -new -newkey rsa:2048 -x509 -sha256 -days 10000 -nodes -out ~/note/keys/hub.crt -keyout ~/note/keys/hub.key \ 19 | -subj "/CN=$1" \ 20 | -reqexts SAN \ 21 | -extensions SAN \ 22 | -config <(cat /etc/ssl/openssl.cnf \ 23 | <(printf "[SAN]\nsubjectAltName=DNS:$1")) 24 | 25 | # Use the self-signed certificate also as the root CA, in case the device is using bidirectional auth. 26 | # Note that the key generated by this command is referenced only in discover.go 27 | cp ~/note/keys/hub.crt ~/note/keys/root.crt 28 | 29 | # If you'd like to see the request to enable bidirectional auth on the notecard, see the scripts 30 | # in the tls directory. 31 | 32 | # Run the notehub 33 | ./hub $1 34 | -------------------------------------------------------------------------------- /hub/discover.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "os" 9 | "time" 10 | 11 | "github.com/blues/note-go/note" 12 | notelib "github.com/blues/note/lib" 13 | ) 14 | 15 | // NotehubDiscover is responsible for discovery of information about the services and apps 16 | func NotehubDiscover(deviceUID string, deviceSN string, productUID string, appUID string, needHandlerInfo bool, hostname string, packetHandlerVersion string) (info notelib.DiscoverInfo, err error) { 17 | // Return basic info about the server 18 | info.HubEndpointID = note.DefaultHubEndpointID 19 | info.HubTimeNs = time.Now().UnixNano() 20 | 21 | // Return info about a specific device if requested 22 | if deviceUID != "" { 23 | device, err2 := deviceGetOrProvision(deviceUID, deviceSN, productUID) 24 | if err2 != nil { 25 | err = err2 26 | return 27 | } 28 | info.HubDeviceStorageObject = notelib.FileStorageObject(deviceUID) 29 | info.HubDeviceAppUID = device.AppUID 30 | if needHandlerInfo { 31 | info.HubSessionHandler = device.Handler 32 | info.HubSessionTicket = device.Ticket 33 | info.HubSessionTicketExpiresTimeNs = device.TicketExpiresTimeSec * int64(1000000000) 34 | } 35 | } 36 | 37 | // Return the tcps issuer rootca cert, used for device-side certificate rotation 38 | serviceCertFile, err2 := os.ReadFile(keyDirectory() + "root.crt") 39 | if err2 == nil { 40 | info.HubCert = serviceCertFile 41 | } 42 | 43 | // Done 44 | return 45 | } 46 | -------------------------------------------------------------------------------- /lib/clang/notehub-udp.pb.h: -------------------------------------------------------------------------------- 1 | /* Automatically generated nanopb header */ 2 | /* Generated by nanopb-0.3.5 at Sun Sep 10 15:08:14 2023. */ 3 | 4 | #ifndef PB_NOTEHUB_UDP_PB_H_INCLUDED 5 | #define PB_NOTEHUB_UDP_PB_H_INCLUDED 6 | #include 7 | 8 | /* @@protoc_insertion_point(includes) */ 9 | #if PB_PROTO_HEADER_VERSION != 30 10 | #error Regenerate this file with the current version of nanopb generator. 11 | #endif 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /* Struct definitions */ 18 | typedef PB_BYTES_ARRAY_T(254) notelib_NotehubUdpPB_Payload_t; 19 | typedef struct _notelib_NotehubUdpPB { 20 | bool has_Payload; 21 | notelib_NotehubUdpPB_Payload_t Payload; 22 | /* @@protoc_insertion_point(struct:notelib_NotehubUdpPB) */ 23 | } notelib_NotehubUdpPB; 24 | 25 | /* Default values for struct fields */ 26 | 27 | /* Initializer values for message structs */ 28 | #define notelib_NotehubUdpPB_init_default {false, {0, {0}}} 29 | #define notelib_NotehubUdpPB_init_zero {false, {0, {0}}} 30 | 31 | /* Field tags (for use in manual encoding/decoding) */ 32 | #define notelib_NotehubUdpPB_Payload_tag 1 33 | 34 | /* Struct field encoding specification for nanopb */ 35 | extern const pb_field_t notelib_NotehubUdpPB_fields[2]; 36 | 37 | /* Maximum encoded size of messages (where known) */ 38 | #define notelib_NotehubUdpPB_size 257 39 | 40 | /* Message IDs (where set with "msgid" option) */ 41 | #ifdef PB_MSGID 42 | 43 | #define NOTEHUB_UDP_MESSAGES \ 44 | 45 | 46 | #endif 47 | 48 | #ifdef __cplusplus 49 | } /* extern "C" */ 50 | #endif 51 | /* @@protoc_insertion_point(eof) */ 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /lib/json.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | // Derived from the "FeedSync" sample code which was covered 5 | // by the Microsoft Public License (Ms-Pl) of December 3, 2007. 6 | // FeedSync itself was derived from the algorithms underlying 7 | // Lotus Notes replication, developed by Ray Ozzie et al c.1985. 8 | 9 | // Package notelib notefile.go handles management and sync of collections of individual notes 10 | package notelib 11 | 12 | import ( 13 | "github.com/blues/note-go/note" 14 | ) 15 | 16 | // jsonConvertJSONToNotefile deserializes/unmarshals a JSON buffer to an in-memory Notefile. 17 | func jsonConvertJSONToNotefile(jsonNotefile []byte) (*Notefile, error) { 18 | // Unmarshal the JSON 19 | newNotefile := &Notefile{} 20 | err := note.JSONUnmarshal(jsonNotefile, &newNotefile) 21 | if err != nil { 22 | return newNotefile, err 23 | } 24 | 25 | // If any of the core map data structures are empty, make sure they 26 | // have valid maps so that the caller can blindly do range enums, etc. 27 | if newNotefile.Notes == nil { 28 | newNotefile.Notes = map[string]note.Note{} 29 | } 30 | if newNotefile.Trackers == nil { 31 | newNotefile.Trackers = map[string]Tracker{} 32 | } 33 | 34 | return newNotefile, nil 35 | } 36 | 37 | // Body functions for Notebox body 38 | // Todo: Should these return errors? 39 | func noteboxBodyFromJSON(data []byte) (body noteboxBody) { _ = note.JSONUnmarshal(data, &body); return } 40 | func noteboxBodyToJSON(body noteboxBody) (data []byte) { data, _ = note.JSONMarshal(body); return } 41 | -------------------------------------------------------------------------------- /jsonxt/readme.md: -------------------------------------------------------------------------------- 1 | Fork of the golang "json" package because of "bulk template" parsing requirements (see notelib/bulk.go), 2 | with the base being Go 1.12 src/encoding/json 3 | http://golang.org/dl/go1.12.src.tar.gz 4 | 5 | 1. Because we need to decompose and reconstruct the JSON template in a linear manner, we need to parse it and reconstruct it with its delimiters in-place. Unfortunately, the standard JSON Decoder suppresses commas and colons and quotes. And so jsonxt returns all delimiters, not just []{} 6 | 7 | 2. Because we are serializing the values without serializing the keys, we need to know in the Decoder when the string it is sending us is a key vs a value. And so we changed it (in a hacky way) to return keys as "quoted" strings, and values as standard unquoted strings. 8 | 9 | ``` 10 | $ diff stream.go ~/desktop/go1-12/src/encoding/json 11 | 12 | 1,4d0 13 | < // NOTE that this is a copy of https://golang.org/src/encoding/json/stream.go with the only changes being: 14 | < // 1) Token() always returns the next token, not just []{}. The XT prefix means "extended token". 15 | < // 2) For string literals that are KEYS, it returns it "quoted" so we can know it's a field, not a value 16 | < // 17 | 9c5 18 | < package jsonxt 19 | --- 20 | > package json 21 | 22 | 419,420c415 23 | < // continue 24 | < return Delim(':'), nil 25 | --- 26 | > continue 27 | 28 | 426,427c421 29 | < // continue 30 | < return Delim(','), nil 31 | --- 32 | > continue 33 | 34 | 432,433c426 35 | < // continue 36 | < return Delim(','), nil 37 | --- 38 | > continue 39 | 40 | 448,449c441 41 | < // return x, nil 42 | < return "\"" + x + "\"", nil 43 | --- 44 | > return x, nil 45 | 46 | ``` 47 | -------------------------------------------------------------------------------- /lib/event_test.go: -------------------------------------------------------------------------------- 1 | package notelib 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/blues/note-go/note" 7 | "github.com/google/uuid" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func isValidUUID(u string) bool { 12 | _, err := uuid.Parse(u) 13 | return err == nil 14 | } 15 | 16 | func TestGenerateEventUid(t *testing.T) { 17 | // nil events or events without a When field 18 | // always generate a UUID 19 | e := note.Event{} 20 | a := GenerateEventUid(nil) 21 | b := GenerateEventUid(&e) 22 | c := GenerateEventUid(&e) 23 | require.True(t, isValidUUID(a)) 24 | require.True(t, isValidUUID(b)) 25 | require.True(t, isValidUUID(c)) 26 | require.NotEqual(t, a, b) 27 | require.NotEqual(t, b, c) 28 | 29 | // an event with When field set should always 30 | // generate a deterministic UID which will look 31 | // something like this: 32 | // 33 | // 33cdeccc-cebe-8032-9f1f-dbee7f5874cb 34 | e = note.Event{When: 1} 35 | b = GenerateEventUid(&e) 36 | c = GenerateEventUid(&e) 37 | require.Equal(t, b, c) 38 | require.Equal(t, "33cdeccc-cebe-8032-9f1f-dbee7f5874cb", b) 39 | 40 | // The `When`, `Body`, `Payload` and `Details` fields 41 | // are all used to construct the UID 42 | e = note.Event{ 43 | When: 1, 44 | Body: &map[string]interface{}{ 45 | "key": "value", 46 | "obj": map[string]int{ 47 | "a": 1, 48 | "b": 2, 49 | }, 50 | "list": []string{"a", "b", "c"}, 51 | }, 52 | Payload: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9}, 53 | Details: &map[string]interface{}{ 54 | "key": 1, 55 | "key2": "value2", 56 | }, 57 | } 58 | 59 | b = GenerateEventUid(&e) 60 | c = GenerateEventUid(&e) 61 | require.Equal(t, b, c) 62 | require.Equal(t, "edd13f0c-b9b8-836c-8c1c-631230e21e56", b) 63 | } 64 | -------------------------------------------------------------------------------- /jsonxt/LICENSE: -------------------------------------------------------------------------------- 1 | JSONXT is a derivative of the standard Golang JSON package that has 2 | been enhanced to allow a needed form of incremental parsing. See: 3 | https://golang.org/src/encoding/json 4 | 5 | ### 6 | Copyright (c) 2009 The Go Authors. All rights reserved. 7 | 8 | Redistribution and use in source and binary forms, with or without 9 | modification, are permitted provided that the following conditions are 10 | met: 11 | 12 | * Redistributions of source code must retain the above copyright 13 | notice, this list of conditions and the following disclaimer. 14 | * Redistributions in binary form must reproduce the above 15 | copyright notice, this list of conditions and the following disclaimer 16 | in the documentation and/or other materials provided with the 17 | distribution. 18 | * Neither the name of Google Inc. nor the names of its 19 | contributors may be used to endorse or promote products derived from 20 | this software without specific prior written permission. 21 | 22 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 23 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 24 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 25 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 26 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 27 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 28 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 32 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 | ### 34 | -------------------------------------------------------------------------------- /lib/storage.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Package notelib storage.go is the class definition for storage drivers 6 | package notelib 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | ) 12 | 13 | // StorageCreateFunc Creates a new storage object instance, with the supplied string as a hint. 14 | // The hint makes it possible to have reasonably meaningful names within 15 | // certain storage mechanisms that can support it (i.e. file systems), 16 | // however it may be completely ignored by other storage mechanisms 17 | // that only deal in terms of addresses, object IDs, or handles. 18 | type storageCreateFunc func(ctx context.Context, iStorageObjectHint string, iFileNameHint string) (storageObject string, err error) 19 | 20 | // StorageCreateObjectFunc Creates a new storage object instance, 21 | // with the supplied storage object name being exact. 22 | type storageCreateObjectFunc func(ctx context.Context, storageObject string) (err error) 23 | 24 | // StorageDeleteFunc deletes an existing storage object instance 25 | type storageDeleteFunc func(ctx context.Context, iStorageObjectHint string, iObject string) (err error) 26 | 27 | // StorageWriteNotefileFunc writes a notefile to the specified storage instance 28 | type storageWriteNotefileFunc func(ctx context.Context, iNotefile *Notefile, iStorageObjectHint string, iObject string) (err error) 29 | 30 | // StorageReadNotefileFunc reads a notefile from the specified storage instance 31 | type storageReadNotefileFunc func(ctx context.Context, iStorageObjectHint string, iObject string) (oNotefile *Notefile, err error) 32 | 33 | // storageClass is the access method by which we do physical I/O 34 | type storageClass struct { 35 | class string 36 | create storageCreateFunc 37 | createObject storageCreateObjectFunc 38 | delete storageDeleteFunc 39 | writeNotefile storageWriteNotefileFunc 40 | readNotefile storageReadNotefileFunc 41 | } 42 | 43 | // Storage creates a storage object from the class in an object string 44 | func storageProvider(iObject string) (storage storageClass, err error) { 45 | // Enumerate known storage providers 46 | if isFileStorage(iObject) { 47 | return fileStorage(), nil 48 | } 49 | 50 | // Not found 51 | return storageClass{}, fmt.Errorf("storage provider not found: %s", iObject) 52 | } 53 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to blues/note 2 | 3 | We love pull requests from everyone. By participating in this project, you 4 | agree to abide by the Blues Inc [code of conduct]. 5 | 6 | [code of conduct]: https://blues.github.io/opensource/code-of-conduct 7 | 8 | Here are some ways *you* can contribute: 9 | 10 | * by using alpha, beta, and prerelease versions 11 | * by reporting bugs 12 | * by suggesting new features 13 | * by writing or editing documentation 14 | * by writing specifications 15 | * by writing code ( **no patch is too small** : fix typos, add comments, 16 | clean up inconsistent whitespace ) 17 | * by refactoring code 18 | * by closing [issues][] 19 | * by reviewing patches 20 | 21 | [issues]: https://github.com/blues/note/issues 22 | 23 | ## Submitting an Issue 24 | 25 | * We use the [GitHub issue tracker][issues] to track bugs and features. 26 | * Before submitting a bug report or feature request, check to make sure it 27 | hasn't 28 | already been submitted. 29 | * When submitting a bug report, please include a [Gist][] that includes a stack 30 | trace and any details that may be necessary to reproduce the bug, including 31 | your release version, Go version, and operating system. Ideally, a bug report 32 | should include a pull request with failing specs. 33 | 34 | [gist]: https://gist.github.com/ 35 | 36 | ## Cleaning up issues 37 | 38 | * Issues that have no response from the submitter will be closed after 30 days. 39 | * Issues will be closed once they're assumed to be fixed or answered. If the 40 | maintainer is wrong, it can be opened again. 41 | * If your issue is closed by mistake, please understand and explain the issue. 42 | We will happily reopen the issue. 43 | 44 | ## Submitting a Pull Request 45 | 1. [Fork][fork] the [official repository][repo]. 46 | 2. [Create a topic branch.][branch] 47 | 3. Implement your feature or bug fix. 48 | 4. Add, commit, and push your changes. 49 | 5. [Submit a pull request.][pr] 50 | 51 | ## Notes 52 | * Please add tests if you changed code. Contributions without tests won't be 53 | * accepted. If you don't know how to add tests, please put in a PR and leave a 54 | * comment asking for help. We love helping! 55 | 56 | [repo]: https://github.com/blues/note/tree/master 57 | [fork]: https://help.github.com/articles/fork-a-repo/ 58 | [branch]: 59 | https://help.github.com/articles/creating-and-deleting-branches-within-your-repository/ 60 | [pr]: https://help.github.com/articles/creating-a-pull-request-from-a-fork/ 61 | 62 | Inspired by 63 | https://github.com/thoughtbot/factory_bot/blob/master/CONTRIBUTING.md 64 | 65 | -------------------------------------------------------------------------------- /hub/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "io" 11 | "net/http" 12 | "os" 13 | 14 | notelib "github.com/blues/note/lib" 15 | ) 16 | 17 | // Address/ports of our server, used to spawn listers and to assign handlers to devices 18 | var ( 19 | serverAddress string 20 | serverPortTCP string 21 | serverPortTCPS string 22 | serverPortHTTP string 23 | serverHTTPReqTopic string 24 | ) 25 | 26 | // Main service entry point 27 | func main() { 28 | // If not specified on the command line, get our server address and ports 29 | if len(os.Args) < 2 { 30 | rsp, err := http.Get("http://checkip.amazonaws.com") 31 | if err != nil { 32 | fmt.Printf("can't get our own IP address: %s", err) 33 | return 34 | } 35 | defer rsp.Body.Close() 36 | var buf []byte 37 | buf, err = io.ReadAll(rsp.Body) 38 | if err != nil { 39 | fmt.Printf("error fetching IP addr: %s", err) 40 | return 41 | } 42 | serverAddress = string(bytes.TrimSpace(buf)) 43 | } else { 44 | serverAddress = os.Args[1] 45 | } 46 | serverHTTPReqTopic = "/req" 47 | serverPortHTTP = ":80" 48 | serverPortTCP = ":8081" 49 | serverPortTCPS = ":8086" 50 | 51 | // Initialize file system folders 52 | eventDir := os.Getenv("HOME") + "/note/events" 53 | eventLogInit(eventDir) 54 | notefileDir := os.Getenv("HOME") + "/note/notefiles" 55 | notelib.FileSetStorageLocation(notefileDir) 56 | 57 | // Set discovery callback 58 | notelib.HubSetDiscover(NotehubDiscover) 59 | 60 | // Spawn the TCP listeners 61 | go tcpHandler() 62 | go tcpsHandler() 63 | fmt.Printf("\nON DEVICE, SET HOST USING:\n'{\"req\":\"hub.set\",\"host\":\"tcp:%s%s|tcps:%s%s\",\"product\":\"\"}'\n\n", 64 | serverAddress, serverPortTCP, serverAddress, serverPortTCPS) 65 | fmt.Printf("TO RESTORE DEVICE'S HUB CONFIGURATION, USE:\n'{\"req\":\"hub.set\",\"host\":\"-\"}'\n\n") 66 | fmt.Printf("Your hub's data will be stored in: %s\n", notefileDir) 67 | fmt.Printf("Your hub's events will be stored in: %s\n", eventDir) 68 | fmt.Printf("\n") 69 | 70 | // Spawn HTTP for inbound web requests 71 | http.HandleFunc(serverHTTPReqTopic, httpReqHandler) 72 | http.HandleFunc(serverHTTPReqTopic+"/", httpReqHandler) 73 | err := http.ListenAndServe(serverPortHTTP, nil) 74 | if err != nil { 75 | fmt.Printf("Error running HTTP server on %s: %v\n", serverPortHTTP, err) 76 | os.Exit(1) 77 | } 78 | } 79 | 80 | // Get the directory containing keys 81 | func keyDirectory() string { 82 | return os.Getenv("HOME") + "/note/keys/" 83 | } 84 | -------------------------------------------------------------------------------- /lib/notehub.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | syntax = "proto2"; 6 | 7 | // Package notelib notehub.proto are the protocol definitions for notebox-notehub comms 8 | package notelib; 9 | 10 | message NotehubPB { 11 | optional int64 Version = 1; 12 | optional string MessageType = 2; 13 | optional string Error = 3; 14 | optional string DeviceUID = 4; 15 | optional string DeviceEndpointID = 5; 16 | optional int64 HubTimeNs = 6; 17 | optional string HubEndpointID = 7; 18 | optional string HubSessionTicket = 8; 19 | optional string HubSessionHandler = 9; 20 | optional int64 HubSessionTicketExpiresTimeSec = 10; 21 | optional string NotefileID = 11; 22 | optional string NotefileIDs = 12; 23 | optional int64 Since = 13; 24 | optional int64 Until = 14; 25 | optional int64 MaxChanges = 15; 26 | optional string DeviceSN = 16; 27 | optional string NoteID = 17; 28 | optional int64 SessionIDPrev = 18; 29 | optional int64 SessionIDNext = 19; 30 | optional bool SessionIDMismatch = 20; 31 | optional int64 Bytes1 = 21; 32 | optional int64 Bytes2 = 22; 33 | optional int64 Bytes3 = 23; 34 | optional int64 Bytes4 = 24; 35 | optional string ProductUID = 25; 36 | optional int64 UsageProvisioned = 26; 37 | optional uint32 UsageRcvdBytes = 27; 38 | optional uint32 UsageSentBytes = 28; 39 | optional uint32 UsageTCPSessions = 29; 40 | optional uint32 UsageTLSSessions = 30; 41 | optional uint32 UsageRcvdNotes = 31; 42 | optional uint32 UsageSentNotes = 32; 43 | optional string CellID = 33; 44 | optional bool NotificationSession = 34; 45 | optional int32 Voltage100 = 35; 46 | optional int32 Temp100 = 36; 47 | optional bool ContinuousSession = 37; 48 | optional int64 MotionSecs = 38; 49 | optional string MotionOrientation = 39; 50 | optional string SessionTrigger = 40; 51 | optional int32 Voltage1000 = 41; 52 | optional int32 Temp1000 = 42; 53 | optional string HubSessionFactoryResetID = 43; 54 | optional uint32 HighPowerSecsTotal = 44; 55 | optional uint32 HighPowerSecsData = 45; 56 | optional uint32 HighPowerSecsGPS = 46; 57 | optional uint32 HighPowerCyclesTotal = 47; 58 | optional uint32 HighPowerCyclesData = 48; 59 | optional uint32 HighPowerCyclesGPS = 49; 60 | optional string DeviceSKU = 50; 61 | optional int64 DeviceFirmware = 51; 62 | optional string DevicePIN = 52; 63 | optional string DeviceOrderingCode = 53; 64 | optional uint32 UsageRcvdBytesSecondary = 54; 65 | optional uint32 UsageSentBytesSecondary = 55; 66 | optional bool SuppressResponse = 56; 67 | optional string Where = 57; 68 | optional int64 WhereWhen = 58; 69 | optional string HubPacketHandler = 59; 70 | optional uint32 PowerSource = 60; 71 | optional double PowerMahUsed = 61; 72 | optional uint32 PenaltySecs = 62; 73 | optional uint32 FailedConnects = 63; 74 | optional string SocketAlias = 64; 75 | 76 | } 77 | 78 | -------------------------------------------------------------------------------- /lib/fileio.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Package notelib fileio.go is the lowest level I/O driver underlying the file transport 6 | package notelib 7 | 8 | import ( 9 | "context" 10 | "os" 11 | "path/filepath" 12 | "strings" 13 | ) 14 | 15 | // FileioExistsFunc checks for file existence 16 | type FileioExistsFunc func(ctx context.Context, path string) (exists bool, err error) 17 | 18 | // FileioDeleteFunc deletes a file 19 | type FileioDeleteFunc func(ctx context.Context, path string) (err error) 20 | 21 | // FileioCreateFunc creates a file 22 | type FileioCreateFunc func(ctx context.Context, path string) (err error) 23 | 24 | // FileioWriteJSONFunc writes a JSON file 25 | type FileioWriteJSONFunc func(ctx context.Context, path string, data []byte) (err error) 26 | 27 | // FileioReadJSONFunc reads a JSON file 28 | type FileioReadJSONFunc func(ctx context.Context, path string) (data []byte, err error) 29 | 30 | // Fileio defines a set of functions for alternative file I/O 31 | type Fileio struct { 32 | Exists FileioExistsFunc 33 | Create FileioCreateFunc 34 | Delete FileioDeleteFunc 35 | ReadJSON FileioReadJSONFunc 36 | WriteJSON FileioWriteJSONFunc 37 | } 38 | 39 | var fileioDefault = Fileio{ 40 | Exists: fileioExists, 41 | Create: fileioCreate, 42 | Delete: fileioDelete, 43 | ReadJSON: fileioReadJSON, 44 | WriteJSON: fileioWriteJSON, 45 | } 46 | 47 | // See if a file exists 48 | func fileioExists(ctx context.Context, path string) (exists bool, err error) { 49 | exists = true 50 | _, err = os.Stat(path) 51 | if err != nil { 52 | exists = false 53 | if os.IsNotExist(err) { 54 | err = nil 55 | } 56 | } 57 | return 58 | } 59 | 60 | // Delete a file 61 | func fileioDelete(ctx context.Context, path string) (err error) { 62 | return os.Remove(path) 63 | } 64 | 65 | // Create a file and write it 66 | func fileioCreate(ctx context.Context, path string) (err error) { 67 | str := strings.Split(path, "/") 68 | if len(str) > 1 { 69 | folder := strings.Join(str[0:len(str)-1], "/") 70 | _ = os.MkdirAll(folder, 0o777) 71 | } 72 | 73 | return fileioWriteJSON(ctx, path, []byte("{}")) 74 | } 75 | 76 | // Write an existing JSON file 77 | func fileioWriteJSON(ctx context.Context, path string, data []byte) (err error) { 78 | _ = os.MkdirAll(filepath.Dir(path), 0777) 79 | fd, err2 := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0o666) 80 | if err2 != nil { 81 | err = err2 82 | logError(ctx, "fileioWrite: error creating %s: %s", path, err) 83 | return 84 | } 85 | 86 | // Write it 87 | _, err = fd.Write(data) 88 | if err != nil { 89 | logError(ctx, "fileioWrite: error writing %s: %s", path, err) 90 | fd.Close() 91 | return 92 | } 93 | 94 | // Done 95 | fd.Close() 96 | return 97 | } 98 | 99 | // Read an existing file 100 | func fileioReadJSON(ctx context.Context, path string) (data []byte, err error) { 101 | data, err = os.ReadFile(path) 102 | return 103 | } 104 | -------------------------------------------------------------------------------- /lib/notelib.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Package notelib notelib.go has certain internal definitions, placed here so that 6 | // they parallel the clang version. 7 | package notelib 8 | 9 | import ( 10 | "sync" 11 | "time" 12 | 13 | "github.com/blues/note-go/note" 14 | ) 15 | 16 | // noteboxBody is the is the Body field within each note 17 | // within the notebox. Each note's unique ID is constructed 18 | // as a structured name of the form |. 19 | // EndpointID is a unique string assigned to each endpoint. 20 | // NotefileID is a unique string assigned to each notefile. 21 | // All NoteID's without the | separator are reserved for 22 | // future use, and at that time we will augment the Body 23 | // data structure to accomodate the new type of data. 24 | 25 | const TemplateFlagClearAfterSync = 0x00000001 26 | 27 | type notefileDesc struct { 28 | // Optional metadata about the notefile 29 | Info *note.NotefileInfo `json:"i,omitempty"` 30 | // Storage method and physical location 31 | Storage string `json:"s,omitempty"` 32 | // Template info 33 | BodyTemplate string `json:"B,omitempty"` 34 | PayloadTemplate uint32 `json:"P,omitempty"` 35 | TemplateFormat uint32 `json:"f,omitempty"` 36 | TemplatePort uint16 `json:"X,omitempty"` 37 | TemplateFlags uint32 `json:"d,omitempty"` 38 | } 39 | 40 | type noteboxBody struct { 41 | // Used when Note ID is of the form "endpointID|notefileID" 42 | Notefile notefileDesc `json:"n,omitempty"` 43 | } 44 | 45 | // OpenNotefile is the in-memory data structure for an open notefile 46 | type OpenNotefile struct { 47 | // Locks access to just this data structure 48 | lock sync.RWMutex 49 | // This notefile's containing box 50 | box *Notebox 51 | // Number of current users of the notefile who are 52 | // counting on the notefile address to be stable 53 | openCount int32 54 | // The time of last close where refcnt wne to 0 55 | closeTime time.Time 56 | // Modification count at point of last checkpoint 57 | modCountAfterCheckpoint int 58 | // The address of the open notefile 59 | notefile *Notefile 60 | // This notefile's storage object 61 | storage string 62 | // Whether or not this notefile has been deleted 63 | deleted bool 64 | } 65 | 66 | // NoteboxInstance is the in-memory data structure for an open notebox 67 | type NoteboxInstance struct { 68 | // Map of POINTERS to OpenNotefiles, indexed by storage object. 69 | // These must be pointers so that we can look it up and bump refcnt 70 | // atomically without a lock. 71 | openfiles sync.Map 72 | // This notebox's storage object 73 | storage string 74 | // The endpoint ID that is to be used for all operations on the notebox 75 | endpointID string 76 | } 77 | 78 | // Notebox is the in-memory data structure for an open notebox 79 | type Notebox struct { 80 | // Map of the Notefiles, indexed by storage object 81 | instance *NoteboxInstance 82 | } 83 | 84 | // If it takes more than 20 seconds, we are at risk of hitting the 30-second 85 | // transaction timeout, which could cause massive batches of notes to be 86 | // re-sent and re-enqueued. Make sure this is flagged as an error so that 87 | // the code or operational infrastructure can be fixed. 88 | const transactionErrorDuration = (20 * time.Second) 89 | -------------------------------------------------------------------------------- /hub/session.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Inbound TCP support 6 | package main 7 | 8 | import ( 9 | "context" 10 | "fmt" 11 | "net" 12 | 13 | notelib "github.com/blues/note/lib" 14 | ) 15 | 16 | // Process requests for the duration of a session being open 17 | func sessionHandler(connSession net.Conn, secure bool) { 18 | 19 | // Keep track of this, from a resource consumption perspective 20 | fmt.Printf("Opened session\n") 21 | 22 | var sessionSource string 23 | if secure { 24 | sessionSource = "tcps:" + connSession.RemoteAddr().String() 25 | } else { 26 | sessionSource = "tcp:" + connSession.RemoteAddr().String() 27 | } 28 | 29 | // Always start with a blank, inactive Session Context 30 | sessionContext := notelib.NewHubSession(sessionSource, secure) 31 | 32 | // Set up golang context with session context stored within 33 | ctx := context.Background() 34 | 35 | for { 36 | var request, response []byte 37 | var err error 38 | firstTransaction := sessionContext.Transactions == 0 39 | 40 | // Extract a request from the wire, and exit if error 41 | _, request, err = notelib.WireReadRequest(ctx, connSession, true) 42 | if err != nil { 43 | if !notelib.ErrorContains(err, "{closed}") { 44 | fmt.Printf("session: error reading request: %s\n", err) 45 | } 46 | break 47 | } 48 | 49 | // Do special processing on the first transaction 50 | if firstTransaction { 51 | 52 | // Extract session context info from the wire format even before processing the transaction 53 | _, err = notelib.WireExtractSessionContext(ctx, request, &sessionContext) 54 | if err != nil { 55 | fmt.Printf("session: error extracting session context from request: %s", err) 56 | break 57 | } 58 | 59 | // Exit if no DeviceUID, at a minimum because this is needed for authentication 60 | if sessionContext.Session.DeviceUID == "" { 61 | fmt.Printf("session: device UID is missing from request\n") 62 | break 63 | } 64 | 65 | // Make sure that this device is provisioned 66 | device, err2 := deviceGetOrProvision(sessionContext.Session.DeviceUID, sessionContext.Session.DeviceSN, sessionContext.Session.ProductUID) 67 | if err2 != nil { 68 | fmt.Printf("session: can't get or provision device: %s\n", err2) 69 | break 70 | } 71 | 72 | // If TLS, validate that the client device hasn't changed. 73 | if secure { 74 | err2 = tlsAuthenticate(connSession, device) 75 | if err2 != nil { 76 | fmt.Printf("session: %s\n", err2) 77 | break 78 | } 79 | } 80 | 81 | } 82 | 83 | // Process the request 84 | fmt.Printf("\nReceived %d-byte message from %s\n", len(request), sessionContext.Session.DeviceUID) 85 | var reqtype string 86 | var suppressResponse bool 87 | reqtype, response, suppressResponse, err = notelib.HubRequest(ctx, request, notehubEvent, &sessionContext) 88 | if err != nil { 89 | fmt.Printf("session: error processing '%s' request: %s\n", reqtype, err) 90 | break 91 | } 92 | 93 | // Write the response 94 | if !suppressResponse { 95 | connSession.Write(response) 96 | } 97 | sessionContext.Transactions++ 98 | 99 | } 100 | 101 | // Close the connection 102 | connSession.Close() 103 | fmt.Printf("\nClosed session\n\n") 104 | } 105 | -------------------------------------------------------------------------------- /hub/event.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "context" 9 | "fmt" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "github.com/blues/note-go/note" 15 | notelib "github.com/blues/note/lib" 16 | golc "github.com/google/open-location-code/go" 17 | ) 18 | 19 | // Event log directory 20 | var eventLogDirectory string 21 | 22 | // Initialize the event log 23 | func eventLogInit(dir string) { 24 | eventLogDirectory = dir 25 | _ = os.MkdirAll(eventLogDirectory, 0o777) 26 | } 27 | 28 | // Event handling procedure 29 | func notehubEvent(ctx context.Context, session *notelib.HubSession, local bool, event *note.Event) (err error) { 30 | // Retrieve the session context 31 | 32 | // If this is a queue and this is a template note, recursively expand it to multiple notifications 33 | if event.Bulk { 34 | return eventBulk(session, local, *event) 35 | } 36 | 37 | // Don't record events for environment variable updates 38 | if event.NotefileID == "_env.dbs" { 39 | return 40 | } 41 | 42 | // Add info about session and when routed 43 | event.TowerID = session.Session.CellID 44 | 45 | // Marshal the event in a tightly-compressed manner, preparing to output it as Newline-Delimited JSON (NDJSON) 46 | eventJSON, err := note.JSONMarshal(event) 47 | if err != nil { 48 | return err 49 | } 50 | eventNDJSON := string(eventJSON) + "\r\n" 51 | 52 | // Generate a valid log file name 53 | unPrefixedDeviceUID := strings.TrimPrefix(event.DeviceUID, "dev:") 54 | filename := fmt.Sprintf("%s-%s", time.Now().UTC().Format("2006-01-02"), unPrefixedDeviceUID) 55 | filename = filename + ".json" 56 | 57 | // Append the JSON to the file 58 | f, err := os.OpenFile(eventLogDirectory+"/"+filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644) 59 | if err == nil { 60 | f.WriteString(eventNDJSON) 61 | f.Close() 62 | } 63 | 64 | // Done 65 | fmt.Printf("event: appended to %s\n", filename) 66 | return 67 | } 68 | 69 | // For bulk data, process the template and payload, generating recursive notifications 70 | func eventBulk(session *notelib.HubSession, local bool, event note.Event) (err error) { 71 | // Get the template from the note 72 | bodyJSON, err := note.JSONMarshal(event.Body) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | // Begin decode of payload using this template 78 | bdc, err := notelib.BulkDecodeTemplate(bodyJSON, event.Payload) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | // Parse each entry within the payload 84 | for { 85 | 86 | // Get the next entry 87 | body, payload, when, wherewhen, olc, _, success := bdc.BulkDecodeNextEntry() 88 | if !success { 89 | break 90 | } 91 | 92 | // Generate a new notification request with a unique EventUID 93 | nn := event 94 | nn.Req = note.EventAdd 95 | nn.When = when 96 | nn.Where = olc 97 | if nn.Where != "" { 98 | area, err := golc.Decode(nn.Where) 99 | if err == nil { 100 | nn.WhereLat, nn.WhereLon = area.Center() 101 | nn.WhereWhen = wherewhen 102 | } 103 | } 104 | nn.Updates = 1 105 | nn.Bulk = false 106 | nn.Body = &body 107 | nn.Payload = payload 108 | nn.EventUID = notelib.GenerateEventUid(&nn) 109 | _ = notehubEvent(context.Background(), session, local, &nn) 110 | 111 | } 112 | 113 | // Done 114 | return 115 | } 116 | -------------------------------------------------------------------------------- /lib/discover.go: -------------------------------------------------------------------------------- 1 | // Copyright 2018 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // Package notelib discover.go is the notehub discovery handling support 6 | package notelib 7 | 8 | import ( 9 | "fmt" 10 | ) 11 | 12 | // DiscoverInfo is the information returned by DiscoverFunc 13 | type DiscoverInfo struct { 14 | HubEndpointID string 15 | HubSessionHandler string 16 | HubSessionTicket string 17 | HubPacketHandler string 18 | HubDeviceStorageObject string 19 | HubDeviceAppUID string 20 | HubTimeNs int64 21 | HubSessionTicketExpiresTimeNs int64 22 | HubCert []byte 23 | } 24 | 25 | // DiscoverFunc is the func to retrieve discovery info for this server 26 | type DiscoverFunc func(edgeUID string, deviceSN string, productUID string, appUID string, needHandlerInfo bool, hostname string, packetHandlerVersion string) (info DiscoverInfo, err error) 27 | 28 | var fnDiscover DiscoverFunc 29 | 30 | // HubSetDiscover sets the global discovery function 31 | func HubSetDiscover(fn DiscoverFunc) { 32 | // Remember the discovery function 33 | fnDiscover = fn 34 | 35 | // Initialize debugging if we've not done so before 36 | debugEnvInit() 37 | } 38 | 39 | // HubDiscover ensures that we've read the local server's discover info, and return the Hub's Endpoint ID 40 | func HubDiscover(deviceUID string, deviceSN string, productUID string, appUID string) (hubEndpointID string, retAppUID string, deviceStorageObject string, err error) { 41 | if fnDiscover == nil { 42 | err = fmt.Errorf("no discovery function is available") 43 | return 44 | } 45 | 46 | // Call the discover func with the null edge UID just to get basic server info 47 | discinfo, err := fnDiscover(deviceUID, deviceSN, productUID, appUID, false, "*", "") 48 | if err != nil { 49 | err = fmt.Errorf("error from discovery handler for %s: %s", deviceUID, err) 50 | return 51 | } 52 | 53 | return discinfo.HubEndpointID, discinfo.HubDeviceAppUID, discinfo.HubDeviceStorageObject, nil 54 | } 55 | 56 | // HubDiscoverSessionTicket gets the session ticket for a session 57 | func HubDiscoverSessionTicket(deviceUID string, deviceSN string, productUID string, appUID string) (hubSessionTicket string, err error) { 58 | if fnDiscover == nil { 59 | err = fmt.Errorf("no discovery function is available") 60 | return 61 | } 62 | 63 | // Call the discover func with the null edge UID just to get basic server info 64 | discinfo, err := fnDiscover(deviceUID, deviceSN, productUID, appUID, true, "*", "") 65 | if err != nil { 66 | err = fmt.Errorf("error from discovery handler for %s: %s", deviceUID, err) 67 | return 68 | } 69 | 70 | return discinfo.HubSessionTicket, nil 71 | } 72 | 73 | // HubProcessDiscoveryRequest calls the discover function, and return discovery info 74 | func hubProcessDiscoveryRequest(deviceUID string, deviceSN string, productUID string, hostname string, packetHandlerVersion string) (info DiscoverInfo, err error) { 75 | if fnDiscover == nil { 76 | err = fmt.Errorf("no discovery function is available") 77 | return 78 | } 79 | 80 | // Call the discover func 81 | info, err = fnDiscover(deviceUID, deviceSN, productUID, "", true, hostname, packetHandlerVersion) 82 | if err != nil { 83 | err = fmt.Errorf("error from discovery handler for %s: %s", deviceUID, err) 84 | return 85 | } 86 | 87 | // Done 88 | return 89 | } 90 | -------------------------------------------------------------------------------- /hub/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | // This handler is used to submit JSON requests into the Golang client API for Notecard-like request handling. 6 | 7 | package main 8 | 9 | import ( 10 | "fmt" 11 | "io" 12 | "net/http" 13 | "net/url" 14 | "strings" 15 | 16 | "github.com/blues/note-go/note" 17 | "github.com/blues/note-go/notehub" 18 | notelib "github.com/blues/note/lib" 19 | ) 20 | 21 | // Handle inbound HTTP request to the "req" topic 22 | func httpReqHandler(httpRsp http.ResponseWriter, httpReq *http.Request) { 23 | ctx := httpReq.Context() 24 | var err error 25 | 26 | // Allow the default values for these parameters to be set in the URL. 27 | _, args := httpArgs(httpReq, serverHTTPReqTopic) 28 | deviceUID := args["device"] 29 | appUID := args["app"] 30 | projectUID := args["project"] 31 | if projectUID != "" { 32 | appUID = projectUID 33 | } 34 | 35 | // Get the requestbody 36 | var reqJSON []byte 37 | reqJSON, err = io.ReadAll(httpReq.Body) 38 | if err != nil { 39 | err = fmt.Errorf("please supply a JSON request in the HTTP body") 40 | httpRsp.Write(notelib.ErrorResponse(err)) 41 | return 42 | } 43 | 44 | // Debug 45 | fmt.Printf("http req: %s\n", string(reqJSON)) 46 | 47 | // Attempt to extract the appUID and deviceUID from the request, giving these 48 | // priority over what was in the URL 49 | req := notehub.HubRequest{} 50 | err = note.JSONUnmarshal(reqJSON, &req) 51 | if err == nil { 52 | if req.DeviceUID != "" { 53 | deviceUID = req.DeviceUID 54 | } 55 | if req.AppUID != "" { 56 | appUID = req.AppUID 57 | } 58 | } 59 | 60 | // Look up the device 61 | var device DeviceState 62 | device, err = deviceGet(deviceUID) 63 | 64 | // Get the hub endpoint ID and storage object 65 | var hubEndpointID, deviceStorageObject string 66 | if err == nil { 67 | hubEndpointID, _, deviceStorageObject, err = notelib.HubDiscover(deviceUID, "", device.ProductUID, appUID) 68 | } 69 | 70 | // Process the request 71 | var rspJSON []byte 72 | if err == nil { 73 | var box *notelib.Notebox 74 | box, err = notelib.OpenEndpointNotebox(ctx, hubEndpointID, deviceStorageObject, false) 75 | if err == nil { 76 | ei := notelib.EventInfo(deviceUID, device.DeviceSN, device.ProductUID, appUID, "api:http", notehubEvent, nil) 77 | rspJSON = box.Request(ctx, ei, hubEndpointID, reqJSON) 78 | box.Close(ctx) 79 | } 80 | } 81 | 82 | // Handle errors 83 | if err != nil { 84 | rspJSON = notelib.ErrorResponse(err) 85 | } 86 | 87 | // Debug 88 | fmt.Printf("http rsp: %s\n", string(rspJSON)) 89 | 90 | // Write the response 91 | httpRsp.Write(rspJSON) 92 | } 93 | 94 | // httpArgs parses the request URI and returns interesting things 95 | func httpArgs(req *http.Request, topic string) (target string, args map[string]string) { 96 | args = map[string]string{} 97 | 98 | // Trim the request URI 99 | target = req.RequestURI[len(topic):] 100 | 101 | // If nothing left, there were no args 102 | if len(target) == 0 { 103 | return 104 | } 105 | 106 | // Make sure that the prefix is "/", else the pattern matcher is matching something we don't want 107 | target = strings.TrimPrefix(target, "/") 108 | 109 | // See if there is a query, and if so process it 110 | str := strings.SplitN(target, "?", 2) 111 | if len(str) == 1 { 112 | return 113 | } 114 | 115 | // Now that we know we have args, parse them 116 | target = str[0] 117 | values, err := url.ParseQuery(str[1]) 118 | if err != nil { 119 | return 120 | } 121 | 122 | // Generate the return arg in the format we expect 123 | for k, v := range values { 124 | if len(v) == 1 { 125 | args[k] = strings.TrimSuffix(strings.TrimPrefix(v[0], "\""), "\"") 126 | } 127 | } 128 | 129 | // Done 130 | return 131 | } 132 | -------------------------------------------------------------------------------- /hub/device.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "time" 10 | 11 | "github.com/google/uuid" 12 | ) 13 | 14 | // Lifetime of a session handler assignment, before device comes back and asks for a new handler 15 | const ticketExpirationMinutes = 60 * 24 * 3 16 | 17 | // DeviceState is a session state for a device, and when the state was last updated 18 | type DeviceState struct { 19 | // Provisioning Info 20 | DeviceUID string `json:"device,omitempty"` 21 | ProductUID string `json:"product,omitempty"` 22 | AppUID string `json:"app,omitempty"` 23 | DeviceSN string `json:"sn,omitempty"` 24 | DeviceDN string `json:"dn,omitempty"` 25 | 26 | // Session State 27 | Handler string `json:"handler,omitempty"` 28 | Ticket string `json:"ticket,omitempty"` 29 | TicketExpiresTimeSec int64 `json:"ticket_expires,omitempty"` 30 | } 31 | 32 | // Simulation of persistent device storage 33 | var ( 34 | deviceCacheInitialized = false 35 | deviceCache map[string]DeviceState 36 | ) 37 | 38 | // Simulate loading a specific device into the device cache 39 | func deviceCacheLoad(deviceUID string) (device DeviceState, present bool, err error) { 40 | // Initialize the device cache if not yet initialized 41 | if !deviceCacheInitialized { 42 | deviceCacheInitialized = true 43 | deviceCache = map[string]DeviceState{} 44 | } 45 | 46 | // Since we don't have persistence in this hub implementation, just see if it's in the cache 47 | device, present = deviceCache[deviceUID] 48 | return 49 | } 50 | 51 | // Look up a device, provisioning it if necessary 52 | func deviceGetOrProvision(deviceUID string, deviceSN string, productUID string) (device DeviceState, err error) { 53 | // Load the device 54 | device, present, err2 := deviceCacheLoad(deviceUID) 55 | if err2 != nil { 56 | err = err2 57 | return 58 | } 59 | 60 | // Provision the device if not present in the cache 61 | if !present || productUID != device.ProductUID { 62 | 63 | // Provision the device 64 | device.DeviceUID = deviceUID 65 | device.ProductUID = productUID 66 | device.DeviceSN = deviceSN 67 | 68 | } 69 | 70 | // Look up which app this product is currently assigned to 71 | device.AppUID, err = appOwningProduct(productUID) 72 | if err != nil { 73 | return 74 | } 75 | 76 | // If the ticket expired, assign a new handler. 77 | if device.TicketExpiresTimeSec == 0 || time.Now().Unix() >= device.TicketExpiresTimeSec { 78 | 79 | device.Handler = "tcp:" + serverAddress + serverPortTCP 80 | device.Handler += "|" 81 | device.Handler += "tcps:" + serverAddress + serverPortTCPS 82 | 83 | // Assign a new ticket whose duration will be constrained 84 | device.Ticket = uuid.New().String() 85 | device.TicketExpiresTimeSec = time.Now().Add(time.Duration(ticketExpirationMinutes) * time.Minute).Unix() 86 | 87 | } 88 | 89 | // Update the cached data structure 90 | deviceCache[deviceUID] = device 91 | 92 | // We've successfully found or provisioned the device 93 | return 94 | } 95 | 96 | // Look up, in persistent storage, which app currently 'owns' the reservation for this product UID 97 | func appOwningProduct(productUID string) (appUID string, err error) { 98 | return 99 | } 100 | 101 | // Look up device in the cache 102 | func deviceGet(deviceUID string) (device DeviceState, err error) { 103 | device, present, err2 := deviceCacheLoad(deviceUID) 104 | if err2 != nil { 105 | err = err2 106 | return 107 | } 108 | if !present { 109 | err = fmt.Errorf("deviceGet: not found") 110 | return 111 | } 112 | return 113 | } 114 | 115 | // Look up a device, provisioning it if necessary 116 | func deviceSet(device DeviceState) (err error) { 117 | // Set in cache 118 | deviceCache[device.DeviceUID] = device 119 | 120 | // This is where we'd write-through to persistent storage 121 | return 122 | } 123 | -------------------------------------------------------------------------------- /jsonxt/indent.go: -------------------------------------------------------------------------------- 1 | // Copyright 2010 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package jsonxt 6 | 7 | import "bytes" 8 | 9 | // Compact appends to dst the JSON-encoded src with 10 | // insignificant space characters elided. 11 | func Compact(dst *bytes.Buffer, src []byte) error { 12 | return compact(dst, src, false) 13 | } 14 | 15 | func compact(dst *bytes.Buffer, src []byte, escape bool) error { 16 | origLen := dst.Len() 17 | var scan scanner 18 | scan.reset() 19 | start := 0 20 | for i, c := range src { 21 | if escape && (c == '<' || c == '>' || c == '&') { 22 | if start < i { 23 | dst.Write(src[start:i]) 24 | } 25 | dst.WriteString(`\u00`) 26 | dst.WriteByte(hex[c>>4]) 27 | dst.WriteByte(hex[c&0xF]) 28 | start = i + 1 29 | } 30 | // Convert U+2028 and U+2029 (E2 80 A8 and E2 80 A9). 31 | if c == 0xE2 && i+2 < len(src) && src[i+1] == 0x80 && src[i+2]&^1 == 0xA8 { 32 | if start < i { 33 | dst.Write(src[start:i]) 34 | } 35 | dst.WriteString(`\u202`) 36 | dst.WriteByte(hex[src[i+2]&0xF]) 37 | start = i + 3 38 | } 39 | v := scan.step(&scan, c) 40 | if v >= scanSkipSpace { 41 | if v == scanError { 42 | break 43 | } 44 | if start < i { 45 | dst.Write(src[start:i]) 46 | } 47 | start = i + 1 48 | } 49 | } 50 | if scan.eof() == scanError { 51 | dst.Truncate(origLen) 52 | return scan.err 53 | } 54 | if start < len(src) { 55 | dst.Write(src[start:]) 56 | } 57 | return nil 58 | } 59 | 60 | func newline(dst *bytes.Buffer, prefix, indent string, depth int) { 61 | dst.WriteByte('\n') 62 | dst.WriteString(prefix) 63 | for i := 0; i < depth; i++ { 64 | dst.WriteString(indent) 65 | } 66 | } 67 | 68 | // Indent appends to dst an indented form of the JSON-encoded src. 69 | // Each element in a JSON object or array begins on a new, 70 | // indented line beginning with prefix followed by one or more 71 | // copies of indent according to the indentation nesting. 72 | // The data appended to dst does not begin with the prefix nor 73 | // any indentation, to make it easier to embed inside other formatted JSON data. 74 | // Although leading space characters (space, tab, carriage return, newline) 75 | // at the beginning of src are dropped, trailing space characters 76 | // at the end of src are preserved and copied to dst. 77 | // For example, if src has no trailing spaces, neither will dst; 78 | // if src ends in a trailing newline, so will dst. 79 | func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error { 80 | origLen := dst.Len() 81 | var scan scanner 82 | scan.reset() 83 | needIndent := false 84 | depth := 0 85 | for _, c := range src { 86 | scan.bytes++ 87 | v := scan.step(&scan, c) 88 | if v == scanSkipSpace { 89 | continue 90 | } 91 | if v == scanError { 92 | break 93 | } 94 | if needIndent && v != scanEndObject && v != scanEndArray { 95 | needIndent = false 96 | depth++ 97 | newline(dst, prefix, indent, depth) 98 | } 99 | 100 | // Emit semantically uninteresting bytes 101 | // (in particular, punctuation in strings) unmodified. 102 | if v == scanContinue { 103 | dst.WriteByte(c) 104 | continue 105 | } 106 | 107 | // Add spacing around real punctuation. 108 | switch c { 109 | case '{', '[': 110 | // delay indent so that empty object and array are formatted as {} and []. 111 | needIndent = true 112 | dst.WriteByte(c) 113 | 114 | case ',': 115 | dst.WriteByte(c) 116 | newline(dst, prefix, indent, depth) 117 | 118 | case ':': 119 | dst.WriteByte(c) 120 | dst.WriteByte(' ') 121 | 122 | case '}', ']': 123 | if needIndent { 124 | // suppress indent in empty object/array 125 | needIndent = false 126 | } else { 127 | depth-- 128 | newline(dst, prefix, indent, depth) 129 | } 130 | dst.WriteByte(c) 131 | 132 | default: 133 | dst.WriteByte(c) 134 | } 135 | } 136 | if scan.eof() == scanError { 137 | dst.Truncate(origLen) 138 | return scan.err 139 | } 140 | return nil 141 | } 142 | -------------------------------------------------------------------------------- /jsonxt/fold.go: -------------------------------------------------------------------------------- 1 | // Copyright 2013 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package jsonxt 6 | 7 | import ( 8 | "bytes" 9 | "unicode/utf8" 10 | ) 11 | 12 | const ( 13 | caseMask = ^byte(0x20) // Mask to ignore case in ASCII. 14 | kelvin = '\u212a' 15 | smallLongEss = '\u017f' 16 | ) 17 | 18 | // foldFunc returns one of four different case folding equivalence 19 | // functions, from most general (and slow) to fastest: 20 | // 21 | // 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 22 | // 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') 23 | // 3) asciiEqualFold, no special, but includes non-letters (including _) 24 | // 4) simpleLetterEqualFold, no specials, no non-letters. 25 | // 26 | // The letters S and K are special because they map to 3 runes, not just 2: 27 | // - S maps to s and to U+017F 'ſ' Latin small letter long s 28 | // - k maps to K and to U+212A 'K' Kelvin sign 29 | // 30 | // See https://play.golang.org/p/tTxjOc0OGo 31 | // 32 | // The returned function is specialized for matching against s and 33 | // should only be given s. It's not curried for performance reasons. 34 | func foldFunc(s []byte) func(s, t []byte) bool { 35 | nonLetter := false 36 | special := false // special letter 37 | for _, b := range s { 38 | if b >= utf8.RuneSelf { 39 | return bytes.EqualFold 40 | } 41 | upper := b & caseMask 42 | if upper < 'A' || upper > 'Z' { 43 | nonLetter = true 44 | } else if upper == 'K' || upper == 'S' { 45 | // See above for why these letters are special. 46 | special = true 47 | } 48 | } 49 | if special { 50 | return equalFoldRight 51 | } 52 | if nonLetter { 53 | return asciiEqualFold 54 | } 55 | return simpleLetterEqualFold 56 | } 57 | 58 | // equalFoldRight is a specialization of bytes.EqualFold when s is 59 | // known to be all ASCII (including punctuation), but contains an 's', 60 | // 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. 61 | // See comments on foldFunc. 62 | func equalFoldRight(s, t []byte) bool { 63 | for _, sb := range s { 64 | if len(t) == 0 { 65 | return false 66 | } 67 | tb := t[0] 68 | if tb < utf8.RuneSelf { 69 | if sb != tb { 70 | sbUpper := sb & caseMask 71 | if 'A' <= sbUpper && sbUpper <= 'Z' { 72 | if sbUpper != tb&caseMask { 73 | return false 74 | } 75 | } else { 76 | return false 77 | } 78 | } 79 | t = t[1:] 80 | continue 81 | } 82 | // sb is ASCII and t is not. t must be either kelvin 83 | // sign or long s; sb must be s, S, k, or K. 84 | tr, size := utf8.DecodeRune(t) 85 | switch sb { 86 | case 's', 'S': 87 | if tr != smallLongEss { 88 | return false 89 | } 90 | case 'k', 'K': 91 | if tr != kelvin { 92 | return false 93 | } 94 | default: 95 | return false 96 | } 97 | t = t[size:] 98 | 99 | } 100 | return len(t) == 0 101 | } 102 | 103 | // asciiEqualFold is a specialization of bytes.EqualFold for use when 104 | // s is all ASCII (but may contain non-letters) and contains no 105 | // special-folding letters. 106 | // See comments on foldFunc. 107 | func asciiEqualFold(s, t []byte) bool { 108 | if len(s) != len(t) { 109 | return false 110 | } 111 | for i, sb := range s { 112 | tb := t[i] 113 | if sb == tb { 114 | continue 115 | } 116 | if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { 117 | if sb&caseMask != tb&caseMask { 118 | return false 119 | } 120 | } else { 121 | return false 122 | } 123 | } 124 | return true 125 | } 126 | 127 | // simpleLetterEqualFold is a specialization of bytes.EqualFold for 128 | // use when s is all ASCII letters (no underscores, etc) and also 129 | // doesn't contain 'k', 'K', 's', or 'S'. 130 | // See comments on foldFunc. 131 | func simpleLetterEqualFold(s, t []byte) bool { 132 | if len(s) != len(t) { 133 | return false 134 | } 135 | for i, b := range s { 136 | if b&caseMask != t[i]&caseMask { 137 | return false 138 | } 139 | } 140 | return true 141 | } 142 | -------------------------------------------------------------------------------- /lib/event.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Blues Inc. All rights reserved. 2 | // Use of this source code is governed by licenses granted by the 3 | // copyright holder including that found in the LICENSE file. 4 | 5 | package notelib 6 | 7 | import ( 8 | "bytes" 9 | "context" 10 | "crypto/md5" 11 | "encoding/binary" 12 | "encoding/json" 13 | "fmt" 14 | "sort" 15 | 16 | "github.com/blues/note-go/note" 17 | "github.com/google/uuid" 18 | ) 19 | 20 | // EventFunc is the func to get called whenever there is a note add/update/delete 21 | type EventFunc func(ctx context.Context, sess *HubSession, local bool, data *note.Event) (err error) 22 | 23 | // Generate a UUID for an event in a way that will allow detection of duplicates of "user data" 24 | func GenerateEventUid(event *note.Event) string { 25 | 26 | // If the captured date is unknown, we can't make any assumptions about duplication 27 | if event == nil || event.When == 0 { 28 | return uuid.New().String() 29 | } 30 | 31 | // Create a new MD5 hash instance 32 | var h = md5.New() 33 | 34 | // Hash deviceUID so we don't conflict with another device in the app 35 | h.Write([]byte(event.DeviceUID)) 36 | 37 | // Convert the 'when' to a byte slice and hash it 38 | whenBytes := make([]byte, 8) 39 | binary.LittleEndian.PutUint64(whenBytes, uint64(event.When)) 40 | h.Write(whenBytes) 41 | 42 | // Hash notefileID and noteID 43 | h.Write([]byte(event.NotefileID)) 44 | h.Write([]byte(event.NoteID)) 45 | 46 | // If body is available, hash it 47 | if event.Body != nil { 48 | h.Write(marshalSortedJSON(*event.Body)) 49 | } 50 | 51 | // Hash the payload 52 | h.Write(event.Payload) 53 | 54 | // If details is available, hash it 55 | if event.Details != nil { 56 | h.Write(marshalSortedJSON(*event.Details)) 57 | } 58 | 59 | // Copy the MD5 checksum intoa 16-byte array for UUID conversion 60 | var array16 [16]byte 61 | copy(array16[:], h.Sum(nil)) 62 | 63 | // Convert to a compliant UUID 64 | return makeCustomUuid(array16) 65 | 66 | } 67 | 68 | // Generate a cusstom (type 8) UUID from a 16-byte array and return it as a formatted string 69 | // https://www.ietf.org/archive/id/draft-peabody-dispatch-new-uuid-format-04.html#name-uuid-version-8 70 | func makeCustomUuid(input [16]byte) string { 71 | uuid := input // Copy the entire input into the uuid array 72 | 73 | // Set the version (type 8 UUID, which is 0b1000) 74 | uuid[6] = (uuid[6] & 0x0F) | 0x80 75 | 76 | // Set the variant (the two most significant bits: 0b10 for RFC 4122) 77 | uuid[8] = (uuid[8] & 0x3F) | 0x80 78 | 79 | // Format the UUID as 8-4-4-4-12 directly with %x 80 | return fmt.Sprintf("%x-%x-%x-%x-%x", 81 | uuid[0:4], // First 4 bytes (8 hex characters) 82 | uuid[4:6], // Next 2 bytes (4 hex characters) 83 | uuid[6:8], // Next 2 bytes (4 hex characters) 84 | uuid[8:10], // Next 2 bytes (4 hex characters) 85 | uuid[10:16]) // Final 6 bytes (12 hex characters) 86 | } 87 | 88 | // This method recursively sorts JSON fields and marshals them. Thie purpose of 89 | // this method is to account for the fact that golang maps always come back in an 90 | // intentionally-arbitrary order, and for hashing we need them in a deterministic order. 91 | func marshalSortedJSON(jsonObject interface{}) []byte { 92 | 93 | switch v := jsonObject.(type) { 94 | 95 | case map[string]interface{}: 96 | 97 | // Sort map keys 98 | keys := make([]string, 0, len(v)) 99 | for key := range v { 100 | keys = append(keys, key) 101 | } 102 | sort.Strings(keys) 103 | 104 | // Create a buffer to hold the marshaled JSON 105 | var buffer bytes.Buffer 106 | buffer.WriteString("{") 107 | 108 | // Recursively marshal the value 109 | for i, key := range keys { 110 | valueBytes := marshalSortedJSON(v[key]) 111 | if i > 0 { 112 | buffer.WriteString(",") 113 | } 114 | buffer.WriteString(fmt.Sprintf("\"%s\":%s", key, valueBytes)) 115 | } 116 | 117 | buffer.WriteString("}") 118 | return buffer.Bytes() 119 | 120 | case []interface{}: 121 | 122 | // Handle JSON arrays 123 | var buffer bytes.Buffer 124 | buffer.WriteString("[") 125 | 126 | // Recursively marshal the elements 127 | for i, elem := range v { 128 | elemBytes := marshalSortedJSON(elem) 129 | if i > 0 { 130 | buffer.WriteString(",") 131 | } 132 | buffer.Write(elemBytes) 133 | } 134 | 135 | buffer.WriteString("]") 136 | return buffer.Bytes() 137 | 138 | default: 139 | j, err := json.Marshal(v) 140 | if err != nil { 141 | return []byte{} 142 | } 143 | return j 144 | } 145 | 146 | } 147 | -------------------------------------------------------------------------------- /lib/notebox_bug_demo_test.go: -------------------------------------------------------------------------------- 1 | package notelib 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | // TestNoteboxConcurrencyBugDemo demonstrates the specific bug: 12 | // Multiple goroutines calling OpenNotebox/Close on the same endpoint:localStorage 13 | // can cause race conditions due to insufficient concurrency protection 14 | func TestNoteboxConcurrencyBugDemo(t *testing.T) { 15 | longRunningTest(t) 16 | ctx := context.Background() 17 | 18 | // Clean up any leftover noteboxes from previous tests 19 | _ = Purge(ctx) 20 | forceCleanupAllNoteboxes() // Force cleanup to ensure test isolation 21 | 22 | // Create temp directory for test 23 | tmpDir := t.TempDir() 24 | 25 | // Set file storage directory 26 | FileSetStorageLocation(tmpDir) 27 | 28 | const ( 29 | endpoint = "bug-demo-endpoint" 30 | numGoroutines = 20 31 | numOperations = 50 32 | ) 33 | 34 | // Create a single storage location that all goroutines will share 35 | testStorage := FileStorageObject("shared_notebox_storage") 36 | 37 | // Create the notebox once 38 | err := CreateNotebox(ctx, endpoint, testStorage) 39 | if err != nil { 40 | t.Fatalf("Failed to create notebox: %v", err) 41 | } 42 | 43 | // Track errors and panics 44 | var ( 45 | errors sync.Map 46 | panics sync.Map 47 | wg sync.WaitGroup 48 | ) 49 | 50 | wg.Add(numGoroutines) 51 | 52 | // Launch N goroutines that all try to open/work/close the same notebox 53 | for i := 0; i < numGoroutines; i++ { 54 | go func(id int) { 55 | defer wg.Done() 56 | 57 | for op := 0; op < numOperations; op++ { 58 | // Catch panics 59 | func() { 60 | defer func() { 61 | if r := recover(); r != nil { 62 | panics.Store(fmt.Sprintf("goroutine-%d-op-%d", id, op), r) 63 | } 64 | }() 65 | 66 | // Open the notebox 67 | box, err := OpenNotebox(ctx, endpoint, testStorage) 68 | if err != nil { 69 | errors.Store(fmt.Sprintf("open-%d-%d", id, op), err) 70 | return 71 | } 72 | 73 | // Do some work 74 | notefileID := fmt.Sprintf("test-%d.db", id) 75 | err = box.AddNotefile(ctx, notefileID, nil) 76 | if err != nil { 77 | errors.Store(fmt.Sprintf("addnotefile-%d-%d", id, op), err) 78 | } 79 | 80 | // Close the notebox 81 | err = box.Close(ctx) 82 | if err != nil { 83 | errors.Store(fmt.Sprintf("close-%d-%d", id, op), err) 84 | } 85 | }() 86 | 87 | // Small delay to increase race likelihood 88 | if op%10 == 0 { 89 | time.Sleep(time.Microsecond) 90 | } 91 | } 92 | }(i) 93 | } 94 | 95 | wg.Wait() 96 | 97 | // Report results 98 | var errorCount, panicCount int 99 | errors.Range(func(key, value interface{}) bool { 100 | errorCount++ 101 | if errorCount <= 5 { 102 | t.Logf("Error %s: %v", key, value) 103 | } 104 | return true 105 | }) 106 | 107 | panics.Range(func(key, value interface{}) bool { 108 | panicCount++ 109 | t.Errorf("PANIC in %s: %v", key, value) 110 | return true 111 | }) 112 | 113 | t.Logf("\nSummary:") 114 | t.Logf("- Total operations: %d", numGoroutines*numOperations) 115 | t.Logf("- Total errors: %d", errorCount) 116 | t.Logf("- Total panics: %d", panicCount) 117 | 118 | if errorCount > 0 || panicCount > 0 { 119 | t.Fail() 120 | } 121 | 122 | // Force cleanup of closed noteboxes 123 | time.Sleep(100 * time.Millisecond) 124 | err = Purge(ctx) 125 | if err != nil { 126 | t.Logf("Purge error: %v", err) 127 | } 128 | 129 | // Check for leaked noteboxes 130 | time.Sleep(100 * time.Millisecond) 131 | checkForLeakedNoteboxes(t) 132 | } 133 | 134 | // checkForLeakedNoteboxes verifies no noteboxes remain open 135 | func checkForLeakedNoteboxes(t *testing.T) { 136 | count := 0 137 | openboxes.Range(func(key, value interface{}) bool { 138 | count++ 139 | if nbi, ok := value.(*NoteboxInstance); ok { 140 | openfiles := 0 141 | nbi.openfiles.Range(func(k, v interface{}) bool { 142 | if of, ok := v.(*OpenNotefile); ok { 143 | if of.openCount > 0 { 144 | openfiles++ 145 | } 146 | } 147 | return true 148 | }) 149 | if openfiles > 0 { 150 | t.Errorf("Leaked notebox: storage=%v, endpoint=%s, openfiles=%d", 151 | key, nbi.endpointID, openfiles) 152 | } 153 | } 154 | return true 155 | }) 156 | 157 | if count > 0 { 158 | t.Logf("Total noteboxes in cache: %d", count) 159 | } 160 | } 161 | 162 | // forceCleanupAllNoteboxes forcefully removes all noteboxes from the openboxes map 163 | // This is used for test cleanup to ensure test isolation 164 | func forceCleanupAllNoteboxes() { 165 | openboxes.Range(func(key, value interface{}) bool { 166 | openboxes.Delete(key) 167 | return true 168 | }) 169 | } 170 | -------------------------------------------------------------------------------- /jsonxt/tables.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 The Go Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style 3 | // license that can be found in the LICENSE file. 4 | 5 | package jsonxt 6 | 7 | import "unicode/utf8" 8 | 9 | // safeSet holds the value true if the ASCII character with the given array 10 | // position can be represented inside a JSON string without any further 11 | // escaping. 12 | // 13 | // All values are true except for the ASCII control characters (0-31), the 14 | // double quote ("), and the backslash character ("\"). 15 | var safeSet = [utf8.RuneSelf]bool{ 16 | ' ': true, 17 | '!': true, 18 | '"': false, 19 | '#': true, 20 | '$': true, 21 | '%': true, 22 | '&': true, 23 | '\'': true, 24 | '(': true, 25 | ')': true, 26 | '*': true, 27 | '+': true, 28 | ',': true, 29 | '-': true, 30 | '.': true, 31 | '/': true, 32 | '0': true, 33 | '1': true, 34 | '2': true, 35 | '3': true, 36 | '4': true, 37 | '5': true, 38 | '6': true, 39 | '7': true, 40 | '8': true, 41 | '9': true, 42 | ':': true, 43 | ';': true, 44 | '<': true, 45 | '=': true, 46 | '>': true, 47 | '?': true, 48 | '@': true, 49 | 'A': true, 50 | 'B': true, 51 | 'C': true, 52 | 'D': true, 53 | 'E': true, 54 | 'F': true, 55 | 'G': true, 56 | 'H': true, 57 | 'I': true, 58 | 'J': true, 59 | 'K': true, 60 | 'L': true, 61 | 'M': true, 62 | 'N': true, 63 | 'O': true, 64 | 'P': true, 65 | 'Q': true, 66 | 'R': true, 67 | 'S': true, 68 | 'T': true, 69 | 'U': true, 70 | 'V': true, 71 | 'W': true, 72 | 'X': true, 73 | 'Y': true, 74 | 'Z': true, 75 | '[': true, 76 | '\\': false, 77 | ']': true, 78 | '^': true, 79 | '_': true, 80 | '`': true, 81 | 'a': true, 82 | 'b': true, 83 | 'c': true, 84 | 'd': true, 85 | 'e': true, 86 | 'f': true, 87 | 'g': true, 88 | 'h': true, 89 | 'i': true, 90 | 'j': true, 91 | 'k': true, 92 | 'l': true, 93 | 'm': true, 94 | 'n': true, 95 | 'o': true, 96 | 'p': true, 97 | 'q': true, 98 | 'r': true, 99 | 's': true, 100 | 't': true, 101 | 'u': true, 102 | 'v': true, 103 | 'w': true, 104 | 'x': true, 105 | 'y': true, 106 | 'z': true, 107 | '{': true, 108 | '|': true, 109 | '}': true, 110 | '~': true, 111 | '\u007f': true, 112 | } 113 | 114 | // htmlSafeSet holds the value true if the ASCII character with the given 115 | // array position can be safely represented inside a JSON string, embedded 116 | // inside of HTML