├── .github └── workflows │ └── go.yml ├── .gitignore ├── .nowignore ├── LICENSE ├── README.md ├── VERSION ├── account ├── db.go ├── db_test.go ├── list.go ├── records.go ├── records_test.go └── tempdb.go ├── api ├── diff-service.go └── signup-service.go ├── cmd └── diffs │ ├── main.go │ └── main_test.go ├── db ├── commit.go ├── commit_test.go ├── db.go ├── db_test.go ├── diff.go ├── diff_test.go └── tempdb.go ├── go.mod ├── go.sum ├── kv ├── checksum.go ├── checksum_test.go ├── map.go ├── map_test.go ├── patch.go ├── patch_test.go ├── testing.go └── testing_test.go ├── licenses ├── APL.txt └── BSL.txt ├── now.json ├── serve ├── client_view.go ├── client_view_test.go ├── hello.go ├── hello_test.go ├── inject.go ├── inject_test.go ├── middleware.go ├── pull.go ├── pull_test.go ├── service.go ├── service_test.go ├── signup │ ├── get.go │ ├── post_failure.go │ ├── post_success.go │ ├── service.go │ └── service_test.go └── types │ └── types.go ├── tool └── release.sh └── util ├── chk └── chk.go ├── countingreader └── reader.go ├── gid └── gid.go ├── kp ├── db.go ├── doc.go ├── hash.go └── spec.go ├── log └── log.go ├── loghttp ├── filter.go ├── filter_test.go ├── loghttp.go └── loghttp_test.go ├── noms ├── diff │ └── nomsdiff.go ├── json │ ├── doc.go │ ├── from_json.go │ ├── from_json_test.go │ ├── hash.go │ ├── list.go │ ├── list_test.go │ ├── spec.go │ ├── to_json.go │ ├── to_json_test.go │ ├── value.go │ └── value_test.go ├── memstore │ └── memstore.go └── union │ ├── union.go │ └── union_test.go ├── tbl ├── tbl.go └── tbl_test.go ├── time ├── time.go └── time_test.go └── version └── version.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | if [ -f Gopkg.toml ]; then 29 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 30 | dep ensure 31 | fi 32 | 33 | - name: Build 34 | run: go build -v ./... 35 | 36 | - name: Test 37 | run: go test -v ./... 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .now 3 | .vscode 4 | repl 5 | 6 | # build artifacts 7 | build 8 | samples/hello-replicant-android-java/.idea 9 | samples/hello-replicant-android-java/repm/build/ 10 | repm/repm-sources.jar 11 | repm/repm.aar 12 | repm/Repm.framework 13 | vendor 14 | diffs 15 | -------------------------------------------------------------------------------- /.nowignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | 3 | # build artifacts 4 | build 5 | repl 6 | repm/repm-sources.jar 7 | repm/repm.aar 8 | 9 | # Directories we don't deploy 10 | samples 11 | bind 12 | vendor 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Source code in this repository is variously licensed under the Business Source 2 | License 1.1 (BSL), the MIT license, and BSD-style licenses. A copy of each 3 | license can be found in the licenses directory. Source code in a given file is 4 | licensed under the BSL and the copyright belongs to Rocicorp, LLC unless 5 | otherwise noted at the beginning of the file. 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Go](https://github.com/rocicorp/diff-server/workflows/Go/badge.svg) 2 | 3 | # *** DEPRECATED **** 4 | 5 | The Replicache Diff Server has been deprecated: its use is no longer recommended and it will not be supported at all after March'21. 6 | 7 | See [https://js.replicache.dev/](https://js.replicache.dev/) for how to set Replicache up without the Diff Server. 8 | 9 | # Replicache Diff Server 10 | 11 | This repository implements the Replicache Diff Server. See [Replicache](https://github.com/rocicorp/replicache) for more information. See the [contributing guide](https://github.com/rocicorp/replicache/blob/master/contributing.md) there for contributing information. 12 | 13 | ## Build 14 | 15 | ``` 16 | cd ~/work 17 | git clone https://github.com/rocicorp/diff-server 18 | cd diff-server 19 | go build ./cmd/diffs 20 | go test ./... 21 | ``` 22 | 23 | ## Run (Development Mode) 24 | 25 | ``` 26 | ./diffs serve --db=/tmp/diffs-data --account-db=/tmp/diffs-accounts --enable-inject 27 | 28 | # Pull from a Client View served from http://localhost:8000/replicache-client-view (replace clientViewURL as appropriate): 29 | curl -H "Authorization: sandbox" -d '{"version": 3, "clientID":"c1", "baseStateID":"00000000000000000000000000000000", "checksum":"00000000", "clientViewURL": "http://localhost:8000/replicache-client-view"}' http://localhost:7001/pull 30 | 31 | # Or inject a clientview... 32 | curl -H "Authorization: sandbox" -d '{"clientID":"c1", "clientViewResponse":{"clientView":{"foo":"bar"},"lastTransactionID":"2"}}' http://localhost:7001/inject 33 | # ... and then pull it (allowing localhost clientview fetch to fail): 34 | curl -H "Authorization: sandbox" -d '{"version": 3, "clientID":"c1", "baseStateID":"00000000000000000000000000000000", "checksum":"00000000", "clientViewURL": "http://localhost:8000/replicache-client-view"}' http://localhost:7001/pull 35 | ``` 36 | 37 | ## Deploy 38 | 39 | ``` 40 | now deploy 41 | now deploy --prod 42 | ``` 43 | 44 | ... or just check in a new commit, it will autodeploy. 45 | 46 | ## Release 47 | 48 | 1. Bump version: 49 | 50 | ``` 51 | go get github.com/rocicorp/repc/tool/bump 52 | bump --root=. diff-server 53 | # push to github and merge 54 | # pull merged commit 55 | git tag v 56 | git push origin v 57 | # update release notes on github 58 | ``` 59 | 60 | 2. Build release binaries: 61 | 62 | ``` 63 | ./tools/release.sh 64 | ``` 65 | 66 | 3. Find the new tag on [https://github.com/rocicorp/diff-server/releases](https://github.com/rocicorp/diff-server/releases) and edit it. 67 | 4. Upload `diffs` and `noms` artifacts generated in previous step (found in `build/`). 68 | 5. Save the release. 69 | 70 | Done. Customers can now run `tools/build.sh` to get the new version [as described here](https://github.com/rocicorp/replicache-sdk-js#get-binaries). 71 | 72 | ## Debug in production 73 | 74 | ``` 75 | # Get noms: https://github.com/attic-labs/noms#install 76 | # db spec below is something like aws:replicant/aa-replicant2/ 77 | # This only works if you have proper AWS credentials, obv. 78 | 79 | # delete client 80 | noms ds -d :: 81 | 82 | # chain diffs 83 | noms log ::client/ 84 | 85 | # chain 86 | noms log --oneline ::client/ | cut -d' ' -f1 | xargs -I{} noms show ::#{} 87 | 88 | # see the value 89 | noms log ::client/.value.data@target 90 | ``` 91 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.1.0 -------------------------------------------------------------------------------- /account/db.go: -------------------------------------------------------------------------------- 1 | // Package account is a lightweight account system for Replicache customers. 2 | package account 3 | 4 | import ( 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/attic-labs/noms/go/datas" 9 | "github.com/attic-labs/noms/go/marshal" 10 | "github.com/attic-labs/noms/go/spec" 11 | "github.com/attic-labs/noms/go/types" 12 | ) 13 | 14 | // DB represents the Replicache account database. It is modeled on the pattern 15 | // established by the main db (db/db.go). 16 | type DB struct { 17 | ds datas.Dataset 18 | 19 | mu sync.Mutex 20 | head Commit 21 | } 22 | 23 | // Commit is the Git-like commit structure Noms uses to store values. 24 | // The account database keeps a single Commit at the head of its dataset 25 | // containing all current entries. 26 | type Commit struct { 27 | // Parents and Meta are unused. 28 | Parents []types.Ref `noms:",set"` 29 | Meta struct { 30 | } 31 | 32 | Value Records 33 | } 34 | 35 | // NewDB returns a new account.DB. If we want the flexibility of using DB 36 | // with multiple Noms databases or datasets we could break those out as 37 | // parameters, but for now keeping it simpler. 38 | func NewDB(storageRoot string) (*DB, error) { 39 | sp, err := spec.ForDatabase(fmt.Sprintf("%s/%s", storageRoot, DatabaseName)) 40 | if err != nil { 41 | return nil, err 42 | } 43 | noms := sp.GetDatabase() 44 | ds := noms.GetDataset(DatasetName) 45 | r := DB{ 46 | ds: ds, 47 | } 48 | defer r.lock()() 49 | err = r.initLocked() 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | return &r, nil 55 | } 56 | 57 | const DatabaseName = "accounts" 58 | const DatasetName = "websignup" 59 | 60 | func (db *DB) initLocked() error { 61 | if !db.ds.HasHead() { 62 | accounts := Records{ 63 | NextASID: LowestASID, 64 | Record: make(map[uint32]Record), 65 | } 66 | return db.setHeadLocked(Commit{Value: accounts}) 67 | } 68 | 69 | var head Commit 70 | err := marshal.Unmarshal(db.ds.Head(), &head) 71 | if err != nil { 72 | return err 73 | } 74 | // Noms roundtrips empty maps as nil, so ensure we have a map. 75 | if head.Value.Record == nil { 76 | head.Value.Record = map[uint32]Record{} 77 | } 78 | 79 | db.head = head 80 | return nil 81 | } 82 | 83 | func (db *DB) Noms() datas.Database { 84 | return db.ds.Database() 85 | } 86 | 87 | // HeadValue returns the value at head. Note that the Records returned contains 88 | // pointer types (eg, a map) so any changes to the pointer members of the Records 89 | // returned will be visible to any other caller to whom this Records has been 90 | // returned. Use CopyRecords to get a value that is safe to change. This is 91 | // not a good pattern, especially because Noms might require retry on write, 92 | // but luckily this is a temporary thing. (Reader two years in the future: .) 93 | func (db *DB) HeadValue() Records { 94 | defer db.lock()() 95 | return db.head.Value 96 | } 97 | 98 | // SetHeadWithValue creates a new Commit with accounts as its value and sets head to it. 99 | // If setHead returns a RetryError, caller should reload head, re-apply changes, and 100 | // try again (up to a few times). 101 | func (db *DB) SetHeadWithValue(accounts Records) error { 102 | defer db.lock()() 103 | return db.setHeadLocked(Commit{Value: accounts}) 104 | } 105 | 106 | func (db *DB) setHeadLocked(newHead Commit) error { 107 | v, err := marshal.Marshal(db.Noms(), newHead) 108 | if err != nil { 109 | return err 110 | } 111 | ref := db.Noms().WriteValue(v) 112 | var ds datas.Dataset 113 | if ds, err = db.Noms().SetHead(db.ds, ref); err != nil { 114 | // We could save the caller a reload of head in this case by returning 115 | // the new ds on error. It kinda complicates the caller tho, so didn't do it. 116 | return RetryError{err} 117 | } 118 | db.ds = ds 119 | db.head = newHead 120 | return nil 121 | } 122 | 123 | // RetryError indicates someone set head out from under us and the operation 124 | // should be retried (re-load the new head, re-apply the changes, and attempt to 125 | // set head again). 126 | type RetryError struct { 127 | wrapped error 128 | } 129 | 130 | func (e RetryError) Error() string { 131 | return fmt.Sprintf("RetryError: %v", e.wrapped) 132 | } 133 | 134 | func (e *RetryError) Unwrap() error { return e.wrapped } 135 | 136 | // Reload reloads the latest state from the underlying noms db. 137 | func (db *DB) Reload() error { 138 | db.lock()() 139 | db.ds.Database().Rebase() 140 | db.ds = db.ds.Database().GetDataset(db.ds.ID()) 141 | return db.initLocked() 142 | } 143 | 144 | func (db *DB) lock() func() { 145 | db.mu.Lock() 146 | return func() { 147 | db.mu.Unlock() 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /account/db_test.go: -------------------------------------------------------------------------------- 1 | package account_test 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "sync" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "roci.dev/diff-server/account" 11 | ) 12 | 13 | func TestInit(t *testing.T) { 14 | assert := assert.New(t) 15 | db, dir := account.LoadTempDB(assert) 16 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 17 | 18 | assert.Equal(account.LowestASID, db.HeadValue().NextASID) 19 | assert.Equal(0, len(db.HeadValue().Record)) 20 | } 21 | 22 | func TestReload(t *testing.T) { 23 | assert := assert.New(t) 24 | db, dir := account.LoadTempDB(assert) 25 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 26 | 27 | // Use db2 to change head behind db's back. 28 | db2 := account.LoadTempDBWithPath(assert, dir) 29 | accounts := db2.HeadValue() 30 | accounts.NextASID = 2 31 | accounts.Record[2] = account.Record{ID: 2} 32 | assert.NoError(db2.SetHeadWithValue(accounts)) 33 | 34 | // Now ensure that if we reload db we see the changes from db2. 35 | assert.NoError(db.Reload()) 36 | assert.Equal(uint32(2), db.HeadValue().NextASID) 37 | _, exists := db.HeadValue().Record[2] 38 | assert.True(exists) 39 | } 40 | 41 | func TestSetHead(t *testing.T) { 42 | assert := assert.New(t) 43 | db, dir := account.LoadTempDB(assert) 44 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 45 | 46 | accounts := db.HeadValue() 47 | accounts.NextASID = 123 48 | accounts.Record[123] = account.Record{ID: 123} 49 | assert.NoError(db.SetHeadWithValue(accounts)) 50 | 51 | gotDB := account.LoadTempDBWithPath(assert, dir) 52 | assert.Equal(accounts, gotDB.HeadValue()) 53 | } 54 | 55 | func TestConcurrentSetHead(t *testing.T) { 56 | assert := assert.New(t) 57 | db, dir := account.LoadTempDB(assert) 58 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 59 | 60 | // Set head behind db's back. 61 | var wg sync.WaitGroup 62 | var err error 63 | wg.Add(1) 64 | go func() { 65 | otherDB := account.LoadTempDBWithPath(assert, dir) 66 | accounts := otherDB.HeadValue() 67 | accounts.NextASID = 1 68 | accounts.Record[1] = account.Record{ID: 1} 69 | err = otherDB.SetHeadWithValue(accounts) 70 | wg.Done() 71 | }() 72 | wg.Wait() 73 | assert.NoError(err) 74 | 75 | // Head has been set behind our back so we expect to get a RetryError 76 | // when we try to set head. 77 | accounts := db.HeadValue() 78 | accounts.NextASID = 2 79 | accounts.Record[2] = account.Record{ID: 2} 80 | err = db.SetHeadWithValue(accounts) 81 | assert.Error(err) 82 | var retryError account.RetryError 83 | assert.True(errors.As(err, &retryError)) 84 | } 85 | -------------------------------------------------------------------------------- /account/list.go: -------------------------------------------------------------------------------- 1 | package account 2 | 3 | var ( 4 | RegularAccounts = []Record{ 5 | { 6 | ID: 0, 7 | Name: "Sandbox", 8 | ClientViewHosts: []string{"localhost"}, 9 | ClientViewURLs: []string{"http://localhost:8000/replicache-client-view"}, 10 | }, 11 | { 12 | ID: 1, 13 | Name: "Replicache Sample TODO", 14 | ClientViewHosts: []string{"replicache-sample-todo.now.sh"}, 15 | ClientViewURLs: []string{"https://replicache-sample-todo.now.sh/serve/replicache-client-view"}, 16 | }, 17 | // Inactive 18 | // { 19 | // ID: 2, 20 | // Name: "Cron", 21 | // ClientViewURLs: []string{"https://api.cron.app/replicache-client-view"}, 22 | // }, 23 | { 24 | ID: 3, 25 | Name: "Songbook Studio", 26 | ClientViewHosts: []string{"us-central1-songbookstudio.cloudfunctions.net"}, 27 | ClientViewURLs: []string{"https://us-central1-songbookstudio.cloudfunctions.net/repliclient/4rzcWwvc83dlTz3CoX9WY8NHUxV2"}, 28 | }, 29 | { 30 | ID: 4, 31 | Name: "Songbook Studio (Vercel)", 32 | ClientViewHosts: []string{"songbook.studio"}, 33 | ClientViewURLs: []string{"https://songbook.studio/api/repliclient"}, 34 | }, 35 | } 36 | ) 37 | -------------------------------------------------------------------------------- /account/records.go: -------------------------------------------------------------------------------- 1 | package account 2 | 3 | import ( 4 | "net/url" 5 | "strconv" 6 | 7 | zl "github.com/rs/zerolog" 8 | ) 9 | 10 | // Records contains the set of Replicache account records (all of them 11 | // if read with ReadAllRecords, just those from the DB if read with 12 | // ReadRecords). 13 | type Records struct { 14 | NextASID uint32 15 | Record map[uint32]Record // Map key is the ID. 16 | } 17 | 18 | // CopyRecords deep copies Records (it contains a pointer type). 19 | func CopyRecords(records Records) Records { 20 | copy := Records{ 21 | NextASID: records.NextASID, 22 | Record: make(map[uint32]Record, len(records.Record)), 23 | } 24 | for _, record := range records.Record { 25 | copy.Record[record.ID] = CopyRecord(record) 26 | } 27 | return copy 28 | } 29 | 30 | // Record represents a single account record. 31 | type Record struct { 32 | ID uint32 33 | Name string 34 | Email string 35 | ClientViewHosts []string 36 | DateCreated string 37 | 38 | // ClientViewURLS is used only by Version 2 clients. It is DEPRECATED 39 | // and will go away when Version 2 is no longer supported. 40 | // TODO remove when Version 2 is no longer supported. 41 | ClientViewURLs []string 42 | } 43 | 44 | // CopyRecord deep copies a Record (it contains a pointer type). 45 | func CopyRecord(record Record) Record { 46 | copy := Record{ 47 | ID: record.ID, 48 | Name: record.Name, 49 | Email: record.Email, 50 | ClientViewHosts: make([]string, 0, len(record.ClientViewHosts)), 51 | DateCreated: record.DateCreated, 52 | ClientViewURLs: make([]string, 0, len(record.ClientViewURLs)), 53 | } 54 | for _, url := range record.ClientViewHosts { 55 | copy.ClientViewHosts = append(copy.ClientViewHosts, url) 56 | } 57 | for _, url := range record.ClientViewURLs { 58 | copy.ClientViewURLs = append(copy.ClientViewURLs, url) 59 | } 60 | return copy 61 | } 62 | 63 | // ASIDs are issued in a separate range from regular accounts. 64 | // See RFC: https://github.com/rocicorp/repc/issues/269 65 | const LowestASID uint32 = 1000000 66 | 67 | // We limit the number of auto-added client view hosts for auto-signup accounts. 68 | const MaxASClientViewHosts int = 5 69 | 70 | // ReadAllRecords returns the full set of Replicache account records. Reading 71 | // of Records is separate from Lookup so the caller can cache Records if they 72 | // so desire (it doesn't change very often). 73 | func ReadAllRecords(db *DB) (Records, error) { 74 | dbRecords, err := ReadRecords(db) 75 | if err != nil { 76 | return Records{}, err 77 | } 78 | 79 | // Now overlay the hard-coded regular accounts, removing any stale regular 80 | // account records that might have been saved. Since we are mutating records 81 | // we make a copy of it first :( Otherwise others who have a handle on it 82 | // will see our changes. 83 | // 84 | // And yes ugh: records are iterated in random order so this iterates 85 | // ALL our account records. 86 | records := CopyRecords(dbRecords) 87 | for _, record := range records.Record { 88 | if record.ID < LowestASID { 89 | delete(records.Record, record.ID) 90 | } 91 | } 92 | for _, record := range RegularAccounts { 93 | records.Record[record.ID] = record 94 | } 95 | 96 | return records, nil 97 | } 98 | 99 | // ReadRecords reads records from the db WITHOUT overlaying the production 100 | // account records. 101 | func ReadRecords(db *DB) (Records, error) { 102 | if err := db.Reload(); err != nil { 103 | return Records{}, err 104 | } 105 | return db.HeadValue(), nil 106 | } 107 | 108 | // Lookup returns the account record for the given authorization string 109 | // and true, or the empty Record and false if it does not exist. 110 | func Lookup(records Records, authorization string) (Record, bool) { 111 | // We have a special-case account where we send an auth string 112 | // instead of an ID in the Authorization header, so here do the 113 | // mapping manually. We could clean this up if we wanted, but it's 114 | // still not a bad idea to have indirection between the Authorization 115 | // header and an account Record eg if we wanted to hand out actual 116 | // authorization tokens that would need to be decoded and validated. 117 | if authorization == "sandbox" { 118 | authorization = "0" 119 | } 120 | id, err := strconv.ParseUint(authorization, 10, 32) 121 | if err != nil { 122 | return Record{}, false 123 | } 124 | r, found := records.Record[uint32(id)] 125 | return r, found 126 | } 127 | 128 | // WriteRecords writes the given records to the underlying db. It might 129 | // return an RetryError in which case the caller should retry the entire 130 | // operation: re-read Records with ReadRecords, copy it, apply changes, 131 | // and call WriteRecords again. Do not retry if the returned error cannot be 132 | // converted to a RetryError (via errors.As). 133 | func WriteRecords(db *DB, records Records) error { 134 | return db.SetHeadWithValue(records) 135 | } 136 | 137 | // ClientViewURLAuthorized returns a bool indicating whether the URL the client 138 | // is attempting to fetch from is authorized. We allow auto-signup accounts to 139 | // fetch their client view from any URL from up to some number of unique hosts. We 140 | // limit this number to prevent spamming and require fixed, explicitly configured 141 | // hosts for the non-ASID case for security. 142 | // 143 | // ClientViewURLAuthorized assumes that records is mutable. If the caller doesn't 144 | // want to see changes from ClientViewURLAuthorized it should pass in a copy from 145 | // CopyRecords(). 146 | func ClientViewURLAuthorized(maxASClientViewHosts int, db *DB, records Records, ID uint32, url string, l zl.Logger) (bool, error) { 147 | record, exists := records.Record[ID] 148 | if !exists { 149 | return false, nil 150 | } 151 | 152 | clientViewHost, err := host(url) 153 | if err != nil { 154 | return false, err 155 | } 156 | 157 | for _, authorizedHost := range record.ClientViewHosts { 158 | if clientViewHost == authorizedHost { 159 | return true, nil 160 | } 161 | } 162 | // Regular accounts have a fixed list of authorized hosts. 163 | if !isASID(record.ID) { 164 | return false, nil 165 | } 166 | 167 | // Here we know this is an auto-signup account and the host is not in the list. 168 | if len(record.ClientViewHosts) >= maxASClientViewHosts { 169 | return false, nil 170 | } 171 | 172 | record.ClientViewHosts = append(record.ClientViewHosts, clientViewHost) 173 | records.Record[record.ID] = record 174 | // TODO retry 175 | if err := WriteRecords(db, records); err != nil { 176 | return false, err 177 | } 178 | l.Debug().Msgf("Added clientViewHost %s for account %d (now %v)", clientViewHost, ID, record.ClientViewHosts) 179 | return true, nil 180 | } 181 | 182 | func isASID(id uint32) bool { 183 | return id >= LowestASID 184 | } 185 | 186 | func host(rawurl string) (string, error) { 187 | u, err := url.Parse(rawurl) 188 | if err != nil { 189 | return "", err 190 | } 191 | return u.Hostname(), nil 192 | } 193 | -------------------------------------------------------------------------------- /account/records_test.go: -------------------------------------------------------------------------------- 1 | package account_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "roci.dev/diff-server/account" 11 | "roci.dev/diff-server/util/log" 12 | ) 13 | 14 | func TestReadAllRecordsAndLookup(t *testing.T) { 15 | assert := assert.New(t) 16 | db, dir := account.LoadTempDB(assert) 17 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 18 | 19 | // Add an ASID account. 20 | newAccount := account.Record{ID: account.LowestASID + 42, Name: "Larry"} 21 | accounts := db.HeadValue() 22 | accounts.Record[newAccount.ID] = newAccount 23 | assert.NoError(db.SetHeadWithValue(accounts)) 24 | 25 | // Make sure unittest-account-adding function works. 26 | account.AddUnittestAccount(assert, db) 27 | 28 | tests := []struct { 29 | name string 30 | db *account.DB 31 | auth string 32 | wantFound bool 33 | wantName string 34 | }{ 35 | { 36 | "no such account", 37 | db, 38 | "nosuchaccount", 39 | false, 40 | "", 41 | }, 42 | { 43 | "sandbox regular account (mapped auth string)", 44 | db, 45 | "sandbox", 46 | true, 47 | "Sandbox", 48 | }, 49 | { 50 | "unittest account (added with test helper)", 51 | db, 52 | fmt.Sprintf("%d", account.UnittestID), 53 | true, 54 | "Unittest", 55 | }, 56 | { 57 | "sample app", 58 | db, 59 | "1", 60 | true, 61 | "Replicache Sample TODO", 62 | }, 63 | { 64 | "new autosignup account", 65 | db, 66 | fmt.Sprintf("%d", newAccount.ID), 67 | true, 68 | "Larry", 69 | }, 70 | } 71 | for _, tt := range tests { 72 | t.Run(tt.name, func(t *testing.T) { 73 | accounts, err := account.ReadAllRecords(tt.db) 74 | assert.NoError(err) 75 | got, found := account.Lookup(accounts, tt.auth) 76 | assert.Equal(tt.wantFound, found, "%s", tt.name) 77 | if tt.wantFound { 78 | assert.Equal(tt.wantName, got.Name, "%s", tt.name) 79 | } 80 | }) 81 | } 82 | } 83 | 84 | func TestReadRecordsDoesNotAlias(t *testing.T) { 85 | assert := assert.New(t) 86 | db, dir := account.LoadTempDB(assert) 87 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 88 | 89 | accounts, err := account.ReadAllRecords(db) 90 | assert.NoError(err) 91 | accounts2, err := account.ReadAllRecords(db) 92 | assert.NoError(err) 93 | newAccount := account.Record{ID: account.LowestASID + 42, Name: "Larry"} 94 | accounts.Record[newAccount.ID] = newAccount 95 | _, found := accounts2.Record[newAccount.ID] 96 | assert.False(found) 97 | } 98 | 99 | func TestWriteRecords(t *testing.T) { 100 | assert := assert.New(t) 101 | db, dir := account.LoadTempDB(assert) 102 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 103 | 104 | accounts, err := account.ReadAllRecords(db) 105 | assert.NoError(err) 106 | newAccount := account.Record{ID: account.LowestASID + 42, Name: "Larry", ClientViewHosts: []string{"host.com"}} 107 | accounts.Record[newAccount.ID] = newAccount 108 | assert.NoError(account.WriteRecords(db, accounts)) 109 | accounts, err = account.ReadAllRecords(db) 110 | assert.NoError(err) 111 | got, found := accounts.Record[newAccount.ID] 112 | assert.True(found) 113 | assert.Equal(newAccount.ID, got.ID) 114 | assert.Equal(newAccount.Name, got.Name) 115 | assert.True(reflect.DeepEqual(newAccount.ClientViewHosts, got.ClientViewHosts)) 116 | } 117 | 118 | func TestClientViewURLAuthorized(t *testing.T) { 119 | assert := assert.New(t) 120 | tests := []struct { 121 | name string 122 | ID uint32 123 | url string 124 | wantAuthorized bool 125 | wantErr string 126 | wantAdded bool 127 | }{ 128 | { 129 | "no such account", 130 | 123, 131 | "http://authorized.com", 132 | false, 133 | "", 134 | false, 135 | }, 136 | { 137 | "regular account, unauthorized url", 138 | 0, 139 | "http://UNauthorized.com", 140 | false, 141 | "", 142 | false, 143 | }, 144 | { 145 | "regular account, authorized url", 146 | 0, 147 | "http://authorized.com", 148 | true, 149 | "", 150 | false, 151 | }, 152 | { 153 | "regular account, authorized url includes port", 154 | 0, 155 | "http://authorized.com:1234/somepath", 156 | true, 157 | "", 158 | false, 159 | }, 160 | { 161 | "auto account, authorized url", 162 | account.LowestASID, 163 | "http://authorized.com", 164 | true, 165 | "", 166 | false, 167 | }, 168 | { 169 | "auto account, new url", 170 | account.LowestASID, 171 | "http://newhost.shouldbeauthorized.com", 172 | true, 173 | "", 174 | true, 175 | }, 176 | } 177 | for _, tt := range tests { 178 | t.Run(tt.name, func(t *testing.T) { 179 | db, dir := account.LoadTempDB(assert) 180 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 181 | records := account.Records{ 182 | 0, 183 | map[uint32]account.Record{ 184 | 0: {ID: 0, ClientViewHosts: []string{"authorized.com"}}, 185 | account.LowestASID: {ID: account.LowestASID, ClientViewHosts: []string{"authorized.com"}}, 186 | }, 187 | } 188 | assert.NoError(account.WriteRecords(db, records)) 189 | recordsCopy := account.CopyRecords(records) 190 | 191 | gotAuthorized, err := account.ClientViewURLAuthorized(account.MaxASClientViewHosts, db, recordsCopy, tt.ID, tt.url, log.Default()) 192 | if tt.wantErr != "" { 193 | assert.Error(err) 194 | assert.Contains(err.Error(), tt.wantErr) 195 | } else { 196 | assert.NoError(err) 197 | assert.Equal(tt.wantAuthorized, gotAuthorized, tt.name) 198 | 199 | originalRecord, exists := records.Record[tt.ID] 200 | if exists { 201 | recordsAfter, err := account.ReadRecords(db) 202 | assert.NoError(err) 203 | urlAdded := len(originalRecord.ClientViewHosts) != len(recordsAfter.Record[tt.ID].ClientViewHosts) 204 | assert.Equal(tt.wantAdded, urlAdded, "%s: URLs before: %v, URLs after: %v", tt.name, originalRecord.ClientViewHosts, recordsAfter.Record[tt.ID].ClientViewHosts) 205 | } 206 | } 207 | }) 208 | } 209 | } 210 | func TestClientViewURLAuthorizedWithMaxedURLs(t *testing.T) { 211 | assert := assert.New(t) 212 | db, dir := account.LoadTempDB(assert) 213 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 214 | records := account.Records{ 215 | 0, 216 | map[uint32]account.Record{ 217 | account.LowestASID: {ID: account.LowestASID, ClientViewHosts: []string{}}, 218 | }, 219 | } 220 | record := records.Record[account.LowestASID] 221 | for i := 0; i < account.MaxASClientViewHosts; i++ { 222 | record.ClientViewHosts = append(record.ClientViewHosts, fmt.Sprintf("%d.com", i)) 223 | } 224 | records.Record[account.LowestASID] = record 225 | assert.NoError(account.WriteRecords(db, records)) 226 | 227 | gotAuthorized, err := account.ClientViewURLAuthorized(account.MaxASClientViewHosts, db, records, account.LowestASID, "http://somenewhost.com", log.Default()) 228 | assert.NoError(err) 229 | assert.False(gotAuthorized) 230 | } 231 | 232 | func TestCopyRecord(t *testing.T) { 233 | assert := assert.New(t) 234 | 235 | record := account.Record{ 236 | ID: 1, 237 | Name: "name", 238 | Email: "email", 239 | ClientViewHosts: []string{"host1"}, 240 | DateCreated: "date", 241 | ClientViewURLs: []string{"url1"}, 242 | } 243 | copy := account.CopyRecord(record) 244 | assert.True(reflect.DeepEqual(record, copy)) 245 | 246 | // Ensure no aliasing. 247 | copy.ClientViewHosts = append(copy.ClientViewHosts, "host2") 248 | assert.NotEqual(len(record.ClientViewHosts), len(copy.ClientViewHosts)) 249 | copy.ClientViewURLs = append(copy.ClientViewURLs, "url2") 250 | assert.NotEqual(len(record.ClientViewURLs), len(copy.ClientViewURLs)) 251 | } 252 | -------------------------------------------------------------------------------- /account/tempdb.go: -------------------------------------------------------------------------------- 1 | package account 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func LoadTempDB(assert *assert.Assertions) (r *DB, dir string) { 10 | td, err := ioutil.TempDir("", "") 11 | assert.NoError(err) 12 | 13 | r = LoadTempDBWithPath(assert, td) 14 | return r, td 15 | } 16 | 17 | func LoadTempDBWithPath(assert *assert.Assertions, td string) (r *DB) { 18 | r, err := NewDB(td) 19 | assert.NoError(err) 20 | return r 21 | } 22 | 23 | const UnittestID = 0xFFFFFFFF 24 | 25 | func AddUnittestAccount(assert *assert.Assertions, db *DB) { 26 | accounts, err := ReadAllRecords(db) 27 | assert.NoError(err) 28 | record := Record{ID: UnittestID, Name: "Unittest", ClientViewHosts: []string{}} 29 | accounts.Record[record.ID] = record 30 | assert.NoError(WriteRecords(db, accounts)) 31 | } 32 | 33 | func AddUnittestAccountHost(assert *assert.Assertions, db *DB, host string) { 34 | accounts, err := ReadAllRecords(db) 35 | assert.NoError(err) 36 | record, exists := accounts.Record[UnittestID] 37 | assert.True(exists) 38 | record.ClientViewHosts = append(record.ClientViewHosts, host) 39 | accounts.Record[UnittestID] = record 40 | assert.NoError(WriteRecords(db, accounts)) 41 | } 42 | 43 | func AddUnittestAccountURL(assert *assert.Assertions, db *DB, url string) { 44 | accounts, err := ReadAllRecords(db) 45 | assert.NoError(err) 46 | record, exists := accounts.Record[UnittestID] 47 | assert.True(exists) 48 | record.ClientViewURLs = append(record.ClientViewURLs, url) 49 | accounts.Record[UnittestID] = record 50 | assert.NoError(WriteRecords(db, accounts)) 51 | } 52 | -------------------------------------------------------------------------------- /api/diff-service.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "net/http" 5 | "os" 6 | 7 | "github.com/attic-labs/noms/go/spec" 8 | "github.com/aws/aws-sdk-go/aws" 9 | "github.com/aws/aws-sdk-go/aws/credentials" 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | "github.com/gorilla/mux" 12 | zl "github.com/rs/zerolog" 13 | zlog "github.com/rs/zerolog/log" 14 | 15 | "roci.dev/diff-server/account" 16 | "roci.dev/diff-server/serve" 17 | "roci.dev/diff-server/util/loghttp" 18 | ) 19 | 20 | const ( 21 | aws_access_key_id = "REPLICANT_AWS_ACCESS_KEY_ID" 22 | aws_secret_access_key = "REPLICANT_AWS_SECRET_ACCESS_KEY" 23 | aws_region = "us-west-2" 24 | 25 | storageRoot = "aws:replicant/aa-replicant2" 26 | ) 27 | 28 | var ( 29 | diffServiceHandler http.Handler 30 | headerLogAllowlist = []string{"Authorization", "Content-Type", "Host", "X-Replicache-SyncID"} 31 | ) 32 | 33 | func init() { 34 | // Zeit now has a 4kb log limit per request, so set up some aggressive HTTP log filters. 35 | loghttp.Filters = append(loghttp.Filters, loghttp.NewBodyElider(500).Filter) 36 | loghttp.Filters = append(loghttp.Filters, loghttp.NewHeaderAllowlist(headerLogAllowlist).Filter) 37 | 38 | zl.SetGlobalLevel(zl.DebugLevel) 39 | zlog.Logger = zlog.Output(zl.ConsoleWriter{Out: os.Stderr, TimeFormat: "02 Jan 06 15:04:05.000 -0700", NoColor: true}) 40 | spec.GetAWSSession = func() *session.Session { 41 | return session.Must(session.NewSession( 42 | aws.NewConfig().WithRegion(aws_region).WithCredentials( 43 | // Have to do this wackiness because not allowed to set AWS env variables in Now for some reason. 44 | credentials.NewStaticCredentials( 45 | os.Getenv(aws_access_key_id), 46 | os.Getenv(aws_secret_access_key), "")))) 47 | } 48 | 49 | accountDB, err := account.NewDB(storageRoot) 50 | if err != nil { 51 | panic(err) 52 | } 53 | 54 | svc := serve.NewService(storageRoot, account.MaxASClientViewHosts, accountDB, false, serve.ClientViewGetter{}, false) 55 | mux := mux.NewRouter() 56 | serve.RegisterHandlers(svc, mux) 57 | diffServiceHandler = mux 58 | } 59 | 60 | // DiffServiceHandler implements the Vercel entrypoint for the DiffService. 61 | func DiffServiceHandler(w http.ResponseWriter, r *http.Request) { 62 | diffServiceHandler.ServeHTTP(w, r) 63 | } 64 | -------------------------------------------------------------------------------- /api/signup-service.go: -------------------------------------------------------------------------------- 1 | package api 2 | 3 | import ( 4 | "html/template" 5 | "net/http" 6 | "os" 7 | 8 | "github.com/attic-labs/noms/go/spec" 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/credentials" 11 | "github.com/aws/aws-sdk-go/aws/session" 12 | "github.com/gorilla/mux" 13 | zl "github.com/rs/zerolog" 14 | zlog "github.com/rs/zerolog/log" 15 | 16 | "roci.dev/diff-server/serve/signup" 17 | "roci.dev/diff-server/util/log" 18 | ) 19 | 20 | var ( 21 | signupHandler http.Handler 22 | ) 23 | 24 | func init() { 25 | zl.SetGlobalLevel(zl.DebugLevel) 26 | zlog.Logger = zlog.Output(zl.ConsoleWriter{Out: os.Stderr, TimeFormat: "02 Jan 06 15:04:05.000 -0700", NoColor: true}) 27 | spec.GetAWSSession = func() *session.Session { 28 | return session.Must(session.NewSession( 29 | aws.NewConfig().WithRegion(aws_region).WithCredentials( 30 | // Have to do this wackiness because not allowed to set AWS env variables in Now for some reason. 31 | credentials.NewStaticCredentials( 32 | os.Getenv(aws_access_key_id), 33 | os.Getenv(aws_secret_access_key), "")))) 34 | } 35 | 36 | // TODO should probably be sharing a mux with DiffService. 37 | mux := mux.NewRouter() 38 | 39 | // Set up signup service. 40 | tmpl := template.Must(signup.ParseTemplates(signup.Templates())) 41 | service := signup.NewService(log.Default(), tmpl, storageRoot) 42 | signup.RegisterHandlers(service, mux) 43 | 44 | signupHandler = mux 45 | } 46 | 47 | // SignupHandler implements the Vercel entrypoint for the signup service. 48 | func SignupHandler(w http.ResponseWriter, r *http.Request) { 49 | signupHandler.ServeHTTP(w, r) 50 | } 51 | -------------------------------------------------------------------------------- /cmd/diffs/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "html/template" 6 | "io" 7 | "net/http" 8 | "os" 9 | "os/signal" 10 | "runtime/pprof" 11 | "runtime/trace" 12 | "syscall" 13 | "time" 14 | 15 | "github.com/attic-labs/noms/go/spec" 16 | "github.com/gorilla/mux" 17 | zl "github.com/rs/zerolog" 18 | zlog "github.com/rs/zerolog/log" 19 | kingpin "gopkg.in/alecthomas/kingpin.v2" 20 | 21 | "roci.dev/diff-server/account" 22 | servepkg "roci.dev/diff-server/serve" 23 | "roci.dev/diff-server/serve/signup" 24 | "roci.dev/diff-server/util/log" 25 | "roci.dev/diff-server/util/version" 26 | ) 27 | 28 | const ( 29 | dropWarning = "This command deletes an entire database and its history. This operations is not recoverable. Proceed? y/n\n" 30 | ) 31 | 32 | type opt struct { 33 | Args []string 34 | OutField string 35 | } 36 | 37 | func main() { 38 | impl(os.Args[1:], os.Stdin, os.Stdout, os.Stderr, os.Exit) 39 | } 40 | 41 | func impl(args []string, in io.Reader, out, errs io.Writer, exit func(int)) { 42 | zlog.Logger = zlog.Output(zl.ConsoleWriter{Out: os.Stderr, TimeFormat: "02 Jan 06 15:04:05.000 -0700"}) 43 | l := log.Default() 44 | 45 | app := kingpin.New("diffs", "") 46 | app.ErrorWriter(errs) 47 | app.UsageWriter(errs) 48 | app.Terminate(exit) 49 | 50 | v := app.Flag("version", "Prints the version of diffs - same as the 'version' command.").Short('v').Bool() 51 | sps := app.Flag("db", "The prefix to use for databases managed. Both local and remote databases are supported. For local databases, specify a directory path to store the database in. For remote databases, specify the http(s) URL to the database (usually https://serve.replicate.to/).").PlaceHolder("/path/to/db").Required().String() 52 | ads := app.Flag("account-db", "Prefix for the account database. Both local and remote databases are supported. For local databases, this is a directory path.").PlaceHolder("/path/to/db").Required().String() 53 | tf := app.Flag("trace", "Name of a file to write a trace to").OpenFile(os.O_RDWR|os.O_CREATE, 0644) 54 | cpu := app.Flag("cpu", "Name of file to write CPU profile to").OpenFile(os.O_RDWR|os.O_CREATE, 0644) 55 | lv := app.Flag("log-level", "Verbosity of logging to print").Default("info").Enum("error", "info", "debug") 56 | 57 | app.PreAction(func(pc *kingpin.ParseContext) error { 58 | if *v { 59 | fmt.Println(version.Version()) 60 | exit(0) 61 | } 62 | return log.SetGlobalLevelFromString(*lv) 63 | }) 64 | 65 | stopCPUProfile := func() { 66 | if *cpu != nil { 67 | pprof.StopCPUProfile() 68 | } 69 | } 70 | stopTrace := func() { 71 | if *tf != nil { 72 | trace.Stop() 73 | } 74 | } 75 | defer stopTrace() 76 | defer stopCPUProfile() 77 | 78 | app.Action(func(pc *kingpin.ParseContext) error { 79 | if pc.SelectedCommand == nil { 80 | return nil 81 | } 82 | 83 | if *tf != nil { 84 | err := trace.Start(*tf) 85 | if err != nil { 86 | return err 87 | } 88 | } 89 | if *cpu != nil { 90 | err := pprof.StartCPUProfile(*tf) 91 | if err != nil { 92 | return err 93 | } 94 | } 95 | c := make(chan os.Signal) 96 | signal.Notify(c, os.Interrupt, syscall.SIGTERM) 97 | go func() { 98 | <-c 99 | stopTrace() 100 | stopCPUProfile() 101 | os.Exit(1) 102 | }() 103 | 104 | return nil 105 | }) 106 | 107 | serve(app, sps, ads, errs, l) 108 | 109 | if len(args) == 0 { 110 | app.Usage(args) 111 | return 112 | } 113 | 114 | _, err := app.Parse(args) 115 | if err != nil { 116 | fmt.Fprintln(errs, err.Error()) 117 | exit(1) 118 | } 119 | } 120 | 121 | type gsp func() (spec.Spec, error) 122 | 123 | func serve(parent *kingpin.Application, sps *string, ads *string, errs io.Writer, l zl.Logger) { 124 | kc := parent.Command("serve", "Starts a local diff-server.") 125 | port := kc.Flag("port", "The port to run on").Default("7001").Int() 126 | enableInject := kc.Flag("enable-inject", "Enable /inject endpoint which writes directly to the database for testing").Default("false").Bool() 127 | disableAuth := parent.Flag("disable-auth", "Disable auth check in pull").Default("false").Bool() 128 | kc.Action(func(_ *kingpin.ParseContext) error { 129 | l.Info().Msgf("Listening on %d...", *port) 130 | 131 | // Set up diffserver service (pull, inject, etc). 132 | if *disableAuth { 133 | l.Info().Msg("Pull auth check disabled") 134 | } 135 | 136 | accountDB, err := account.NewDB(*ads) 137 | if err != nil { 138 | panic(err) 139 | } 140 | 141 | svc := servepkg.NewService(*sps, account.MaxASClientViewHosts, accountDB, *disableAuth, servepkg.ClientViewGetter{}, *enableInject) 142 | mux := mux.NewRouter() 143 | servepkg.RegisterHandlers(svc, mux) 144 | 145 | // Set up signup service. 146 | tmpl := template.Must(signup.ParseTemplates(signup.Templates())) 147 | service := signup.NewService(l, tmpl, *ads) 148 | signup.RegisterHandlers(service, mux) 149 | 150 | server := &http.Server{ 151 | Addr: fmt.Sprintf(":%d", *port), 152 | Handler: mux, 153 | ReadTimeout: 10 * time.Second, 154 | WriteTimeout: 10 * time.Second, 155 | } 156 | return server.ListenAndServe() 157 | }) 158 | } 159 | -------------------------------------------------------------------------------- /cmd/diffs/main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httptest" 10 | "os" 11 | "strings" 12 | "testing" 13 | gt "time" 14 | 15 | "github.com/stretchr/testify/assert" 16 | 17 | "roci.dev/diff-server/account" 18 | "roci.dev/diff-server/db" 19 | "roci.dev/diff-server/util/time" 20 | ) 21 | 22 | func TestServe(t *testing.T) { 23 | assert := assert.New(t) 24 | dir, err := ioutil.TempDir("", "") 25 | assert.NoError(err) 26 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 27 | fmt.Println(dir) 28 | 29 | accountDBDir, err := ioutil.TempDir("", "") 30 | assert.NoError(err) 31 | defer func() { assert.NoError(os.RemoveAll(accountDBDir)) }() 32 | accountDB := account.LoadTempDBWithPath(assert, accountDBDir) 33 | account.AddUnittestAccount(assert, accountDB) 34 | 35 | cvServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 36 | w.WriteHeader(200) 37 | w.Write([]byte(`{"clientView": {}}`)) 38 | })) 39 | 40 | defer time.SetFake()() 41 | 42 | args := append([]string{"--db=" + dir, "--account-db=" + accountDBDir, "serve", "--port=8674"}) 43 | go impl(args, strings.NewReader(""), os.Stdout, os.Stderr, func(_ int) {}) 44 | 45 | // Wait for server to start... 46 | i := 0 47 | for { 48 | i++ 49 | gt.Sleep(100 * gt.Millisecond) 50 | resp, err := http.Get("http://localhost:8674/") 51 | if err == nil && resp.StatusCode == http.StatusOK { 52 | break 53 | } 54 | if i > 20 { 55 | panic("server never started") 56 | } 57 | } 58 | 59 | const code = `function add(id, d) { var v = db.get(id) || 0; v += d; db.put(id, v); return v; }` 60 | tc := []struct { 61 | rpc string 62 | req string 63 | authHeader string 64 | expectedResponse string 65 | expectedError string 66 | }{ 67 | {"pull", 68 | fmt.Sprintf(`{"baseStateID": "00000000000000000000000000000000", "checksum": "00000000", "clientID": "clientid", "clientViewURL": "%s", "version": 3}`, cvServer.URL), 69 | fmt.Sprintf("%d", account.UnittestID), 70 | `{"stateID":"r0d74qu25vi4dr8fmf58oike0cj4jpth","lastMutationID":0,"patch":[{"op":"replace","path":"","valueString":"{}"}],"checksum":"00000000","clientViewInfo":{"httpStatusCode":200,"errorMessage":""}}`, 71 | ""}, 72 | } 73 | 74 | for i, t := range tc { 75 | msg := fmt.Sprintf("test case %d: %s: %s", i, t.rpc, t.req) 76 | httpReq, err := http.NewRequest("POST", fmt.Sprintf("http://localhost:8674/%s", t.rpc), strings.NewReader(t.req)) 77 | assert.NoError(err) 78 | httpReq.Header.Add("Authorization", t.authHeader) 79 | httpReq.Header.Add("Content-type", "application/json") 80 | resp, err := http.DefaultClient.Do(httpReq) 81 | assert.NoError(err, msg) 82 | assert.Equal("application/json", resp.Header.Get("Content-type")) 83 | body := bytes.Buffer{} 84 | _, err = io.Copy(&body, resp.Body) 85 | assert.NoError(err, msg) 86 | assert.Equal(t.expectedResponse+"\n", string(body.Bytes()), msg) 87 | } 88 | 89 | cvServer.Close() 90 | } 91 | 92 | func TestEmptyInput(t *testing.T) { 93 | assert := assert.New(t) 94 | db.LoadTempDB(assert) 95 | var args []string 96 | 97 | // Just testing that they don't crash. 98 | // See https://github.com/aboodman/replicant/issues/120 99 | impl(args, strings.NewReader(""), ioutil.Discard, ioutil.Discard, func(_ int) {}) 100 | args = []string{"--db=/tmp/foo"} 101 | impl(args, strings.NewReader(""), ioutil.Discard, ioutil.Discard, func(_ int) {}) 102 | } 103 | -------------------------------------------------------------------------------- /db/commit.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "github.com/attic-labs/noms/go/marshal" 5 | "github.com/attic-labs/noms/go/nomdl" 6 | "github.com/attic-labs/noms/go/types" 7 | "github.com/attic-labs/noms/go/util/datetime" 8 | "roci.dev/diff-server/kv" 9 | ) 10 | 11 | var ( 12 | schema = nomdl.MustParseType(` 13 | Struct Commit { 14 | parents: Set>>, 15 | meta: Struct { 16 | date: Struct DateTime { 17 | secSinceEpoch: Number, 18 | }, 19 | }, 20 | value: Struct { 21 | checksum: String, 22 | lastMutationID: Number, 23 | data: Ref>, 24 | }, 25 | }`) 26 | ) 27 | 28 | type Commit struct { 29 | Parents []types.Ref `noms:",set"` 30 | Meta struct { 31 | Date datetime.DateTime 32 | } 33 | Value struct { 34 | Checksum types.String 35 | LastMutationID types.Number 36 | Data types.Ref `noms:",omitempty"` 37 | } 38 | NomsStruct types.Struct `noms:",original"` 39 | } 40 | 41 | func (c Commit) Ref() types.Ref { 42 | return types.NewRef(c.NomsStruct) 43 | } 44 | 45 | func (c Commit) Data(noms types.ValueReadWriter) kv.Map { 46 | return kv.FromNoms(noms, c.Value.Data.TargetValue(noms).(types.Map), kv.MustChecksumFromString(string(c.Value.Checksum))) 47 | } 48 | 49 | // Basis returns the basis (parent) of the Commit. 50 | func (c Commit) Basis(noms types.ValueReadWriter) (Commit, error) { 51 | return Read(noms, c.Parents[0].TargetHash()) 52 | } 53 | 54 | func makeCommit(noms types.ValueReadWriter, basis types.Ref, d datetime.DateTime, newData types.Ref, checksum types.String, lastMutationID uint64) Commit { 55 | c := Commit{} 56 | if !basis.IsZeroValue() { 57 | c.Parents = []types.Ref{basis} 58 | } 59 | c.Meta.Date = d 60 | c.Value.Checksum = checksum 61 | c.Value.LastMutationID = types.Number(lastMutationID) // Warning: potentially lossy! 62 | c.Value.Data = newData 63 | c.NomsStruct = marshal.MustMarshal(noms, c).(types.Struct) 64 | return c 65 | } 66 | -------------------------------------------------------------------------------- /db/commit_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/attic-labs/noms/go/chunks" 8 | "github.com/attic-labs/noms/go/marshal" 9 | "github.com/attic-labs/noms/go/types" 10 | "github.com/attic-labs/noms/go/util/datetime" 11 | "github.com/stretchr/testify/assert" 12 | 13 | "roci.dev/diff-server/kv" 14 | "roci.dev/diff-server/util/noms/diff" 15 | ) 16 | 17 | func TestBasis(t *testing.T) { 18 | assert := assert.New(t) 19 | db, _ := LoadTempDB(assert) 20 | genesis := db.Head() 21 | c, err := db.MaybePutData(kv.NewMap(db.Noms()), 2) 22 | assert.NoError(err) 23 | if err == nil { 24 | assert.False(c.NomsStruct.IsZeroValue()) 25 | } 26 | basis, err := c.Basis(db.Noms()) 27 | assert.NoError(err) 28 | assert.True(genesis.NomsStruct.Equals(basis.NomsStruct)) 29 | } 30 | 31 | func TestMarshal(t *testing.T) { 32 | assert := assert.New(t) 33 | 34 | noms := types.NewValueStore((&chunks.TestStorage{}).NewView()) 35 | emptyMap := noms.WriteValue(types.NewMap(noms)) 36 | checksum1 := types.String("1") 37 | lastMutationID1 := uint64(1) 38 | 39 | d := datetime.Now() 40 | dr := noms.WriteValue(types.NewMap(noms, types.String("foo"), types.String("bar"))) 41 | checksum2 := types.String("2") 42 | lastMutationID2 := uint64(2) 43 | c1 := makeCommit(noms, types.Ref{}, d, noms.WriteValue(types.NewMap(noms)), checksum1, lastMutationID1) 44 | c2 := makeCommit(noms, noms.WriteValue(c1.NomsStruct), d, dr, checksum2, lastMutationID2) 45 | noms.WriteValue(c2.NomsStruct) 46 | 47 | tc := []struct { 48 | in Commit 49 | exp types.Value 50 | }{ 51 | { 52 | c1, 53 | types.NewStruct("Commit", types.StructData{ 54 | "meta": types.NewStruct("", types.StructData{ 55 | "date": marshal.MustMarshal(noms, d), 56 | }), 57 | "parents": types.NewSet(noms), 58 | "value": types.NewStruct("", types.StructData{ 59 | "checksum": types.String("1"), 60 | "data": emptyMap, 61 | "lastMutationID": types.Number(lastMutationID1), 62 | }), 63 | }), 64 | }, 65 | { 66 | c2, 67 | types.NewStruct("Commit", types.StructData{ 68 | "parents": types.NewSet(noms, c1.Ref()), 69 | "meta": types.NewStruct("", types.StructData{ 70 | "date": marshal.MustMarshal(noms, d), 71 | }), 72 | "value": types.NewStruct("", types.StructData{ 73 | "checksum": types.String("2"), 74 | "data": dr, 75 | "lastMutationID": types.Number(lastMutationID2), 76 | }), 77 | }), 78 | }, 79 | } 80 | 81 | for i, t := range tc { 82 | act, err := marshal.Marshal(noms, t.in) 83 | assert.NoError(err, "test case: %d", i) 84 | assert.True(t.exp.Equals(act), "test case: %d - %s", i, diff.Diff(t.exp, act)) 85 | 86 | var roundtrip Commit 87 | err = marshal.Unmarshal(act, &roundtrip) 88 | assert.NoError(err) 89 | 90 | remarshalled, err := marshal.Marshal(noms, roundtrip) 91 | assert.NoError(err) 92 | assert.True(act.Equals(remarshalled), fmt.Sprintf("test case %d", i)) 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | // Package db implements the core database abstraction of Replicant. It provides facilities to import 2 | // transaction bundles, execute transactions, and synchronize Replicant databases. 3 | package db 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "sync" 9 | 10 | "github.com/attic-labs/noms/go/datas" 11 | "github.com/attic-labs/noms/go/hash" 12 | "github.com/attic-labs/noms/go/marshal" 13 | "github.com/attic-labs/noms/go/types" 14 | "roci.dev/diff-server/kv" 15 | "roci.dev/diff-server/util/time" 16 | ) 17 | 18 | type DB struct { 19 | ds datas.Dataset 20 | 21 | mu sync.Mutex 22 | head Commit 23 | } 24 | 25 | func New(ds datas.Dataset) (*DB, error) { 26 | r := DB{ 27 | ds: ds, 28 | } 29 | defer r.lock()() 30 | err := r.initLocked() 31 | if err != nil { 32 | return nil, err 33 | } 34 | 35 | return &r, nil 36 | } 37 | 38 | func (db *DB) initLocked() error { 39 | if !db.ds.HasHead() { 40 | m := kv.NewMap(db.Noms()) 41 | genesis := makeCommit(db.Noms(), 42 | types.Ref{}, 43 | time.DateTime(), 44 | db.Noms().WriteValue(m.NomsMap()), 45 | m.NomsChecksum(), 46 | 0 /*lastMutationID*/) 47 | db.Noms().WriteValue(genesis.NomsStruct) 48 | return db.setHeadLocked(genesis) 49 | } 50 | 51 | headType := types.TypeOf(db.ds.Head()) 52 | if !types.IsSubtype(schema, headType) { 53 | return fmt.Errorf("Cannot load database. Specified head has non-Replicache data of type: %s", headType.Describe()) 54 | } 55 | 56 | var head Commit 57 | err := marshal.Unmarshal(db.ds.Head(), &head) 58 | if err != nil { 59 | return err 60 | } 61 | 62 | db.head = head 63 | return nil 64 | } 65 | 66 | func (db *DB) Noms() datas.Database { 67 | return db.ds.Database() 68 | } 69 | 70 | func (db *DB) Head() Commit { 71 | defer db.lock()() 72 | return db.head 73 | } 74 | 75 | // setHead sets the head commit to newHead and fast-forwards the underlying dataset. 76 | func (db *DB) setHead(newHead Commit) error { 77 | defer db.lock()() 78 | return db.setHeadLocked(newHead) 79 | } 80 | 81 | func (db *DB) setHeadLocked(newHead Commit) error { 82 | ds, err := db.Noms().FastForward(db.ds, newHead.Ref()) 83 | if err != nil { 84 | return err 85 | } 86 | db.ds = ds 87 | db.head = newHead 88 | return nil 89 | } 90 | 91 | func (db *DB) Hash() hash.Hash { 92 | return db.Head().NomsStruct.Hash() 93 | } 94 | 95 | func (db *DB) Reload() error { 96 | db.lock()() 97 | db.ds.Database().Rebase() 98 | db.ds = db.ds.Database().GetDataset(db.ds.ID()) 99 | return db.initLocked() 100 | } 101 | 102 | // Read reads the Commit with the given hash from the db. 103 | func Read(noms types.ValueReadWriter, hash hash.Hash) (Commit, error) { 104 | if hash.IsEmpty() { 105 | return Commit{}, errors.New("commit (empty hash) not found") 106 | } 107 | v := noms.ReadValue(hash) 108 | if v == nil { 109 | return Commit{}, fmt.Errorf("commit %s not found", hash) 110 | } 111 | var c Commit 112 | err := marshal.Unmarshal(v, &c) 113 | return c, err 114 | } 115 | 116 | // MaybePutData creates a new commit with the given map and lastMutationID if 117 | // they are different from what is currently at head. It returns the new Commit 118 | // if written or a zero value Commit if not (commit.NomsStruct.IsZeroValue() will be true). 119 | func (db *DB) MaybePutData(m kv.Map, lastMutationID uint64) (Commit, error) { 120 | defer db.lock()() 121 | 122 | hv := db.head.Value 123 | hvc, err := kv.ChecksumFromString(string(hv.Checksum)) 124 | if err != nil { 125 | return Commit{}, fmt.Errorf("couldnt parse checksum from commit: %w", err) 126 | } 127 | if lastMutationID == uint64(hv.LastMutationID) && m.Checksum() == hvc.String() { 128 | return Commit{}, nil 129 | } 130 | basis := types.NewRef(db.head.NomsStruct) 131 | commit := makeCommit(db.Noms(), basis, time.DateTime(), db.Noms().WriteValue(m.NomsMap()), m.NomsChecksum(), lastMutationID) 132 | db.Noms().WriteValue(commit.NomsStruct) 133 | if err := db.setHeadLocked(commit); err != nil { 134 | return Commit{}, err 135 | } 136 | return commit, nil 137 | } 138 | 139 | func (db *DB) lock() func() { 140 | db.mu.Lock() 141 | return func() { 142 | db.mu.Unlock() 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /db/db_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/attic-labs/noms/go/types" 8 | "github.com/stretchr/testify/assert" 9 | "roci.dev/diff-server/kv" 10 | "roci.dev/diff-server/util/time" 11 | ) 12 | 13 | func TestReload(t *testing.T) { 14 | assert := assert.New(t) 15 | db, dir := LoadTempDB(assert) 16 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 17 | genesis := db.Head() 18 | 19 | // Change head behind db's back. 20 | db2 := LoadTempDBWithPath(assert, dir) 21 | me := kv.NewMap(db2.Noms()).Edit() 22 | assert.NoError(me.Set("key", types.Bool(true))) 23 | m := me.Build() 24 | valueRef := db2.Noms().WriteValue(m.NomsMap()) 25 | newCommit := makeCommit(db2.Noms(), types.NewRef(genesis.NomsStruct), time.DateTime(), valueRef, m.NomsChecksum(), 123) 26 | db2.Noms().WriteValue(newCommit.NomsStruct) 27 | err := db2.setHead(newCommit) 28 | assert.NoError(err) 29 | assert.False(genesis.NomsStruct.Equals(newCommit.NomsStruct)) 30 | assert.True(newCommit.NomsStruct.Equals(db2.Head().NomsStruct)) 31 | 32 | // Now check that db picks up the change. 33 | assert.NoError(db.Reload()) 34 | assert.True(newCommit.NomsStruct.Equals(db.Head().NomsStruct)) 35 | } 36 | 37 | func TestGenesis(t *testing.T) { 38 | assert := assert.New(t) 39 | db, _ := LoadTempDB(assert) 40 | assert.False(db.Hash().IsEmpty()) 41 | assert.True(db.Head().Data(db.Noms()).Empty()) 42 | } 43 | 44 | func TestRead(t *testing.T) { 45 | assert := assert.New(t) 46 | db, _ := LoadTempDB(assert) 47 | c, err := Read(db.Noms(), db.Hash()) 48 | assert.NoError(err) 49 | assert.True(db.Head().NomsStruct.Equals(c.NomsStruct)) 50 | } 51 | 52 | func TestMaybePutData(t *testing.T) { 53 | assert := assert.New(t) 54 | db, _ := LoadTempDB(assert) 55 | genesis := db.Head() 56 | me := kv.NewMap(db.Noms()).Edit() 57 | assert.NoError(me.Set("key", types.Bool(true))) 58 | m := me.Build() 59 | 60 | c1, err := db.MaybePutData(m, 1) 61 | assert.NoError(err) 62 | assert.False(genesis.NomsStruct.Equals(c1.NomsStruct)) 63 | assert.True(c1.NomsStruct.Equals(db.Head().NomsStruct)) 64 | assert.True(m.NomsMap().Value().Equals(c1.Data(db.Noms()))) 65 | assert.True(types.Number(1).Equals(c1.Value.LastMutationID)) 66 | 67 | c2, err := db.MaybePutData(m, 2) 68 | assert.NoError(err) 69 | assert.True(c2.NomsStruct.Equals(db.Head().NomsStruct)) 70 | assert.True(types.Number(2).Equals(c2.Value.LastMutationID)) 71 | 72 | c3, err := db.MaybePutData(m, 2) 73 | assert.NoError(err) 74 | assert.True(c3.NomsStruct.IsZeroValue()) 75 | assert.True(c2.NomsStruct.Equals(db.Head().NomsStruct)) 76 | } 77 | 78 | // hmmm.. we seem to have removed most tests. 79 | -------------------------------------------------------------------------------- /db/diff.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/attic-labs/noms/go/hash" 7 | "github.com/attic-labs/noms/go/marshal" 8 | "github.com/attic-labs/noms/go/types" 9 | "github.com/attic-labs/noms/go/util/datetime" 10 | zl "github.com/rs/zerolog" 11 | 12 | "roci.dev/diff-server/kv" 13 | ) 14 | 15 | func fullSync(version uint32, db *DB, from hash.Hash, l zl.Logger) ([]kv.Operation, Commit) { 16 | l.Debug().Msgf("Requested sync %s basis could not be found - sending a full sync", from.String()) 17 | 18 | var op kv.Operation 19 | // version 2 20 | op = 21 | kv.Operation{ 22 | Op: kv.OpReplace, 23 | Path: "", 24 | ValueString: "{}", 25 | } 26 | 27 | m := kv.NewMap(db.Noms()) 28 | return []kv.Operation{op}, makeCommit(db.Noms(), types.Ref{}, datetime.Epoch, db.ds.Database().WriteValue(m.NomsMap()), m.NomsChecksum(), 0 /*lastMutationID*/) 29 | } 30 | 31 | func maybeDecodeCommit(v types.Value, h hash.Hash, expectedChecksum kv.Checksum, l zl.Logger) (Commit, error) { 32 | var c Commit 33 | err := marshal.Unmarshal(v, &c) 34 | if err != nil { 35 | return Commit{}, fmt.Errorf("could not decode basis %s: %w", h, err) 36 | } 37 | checksum, err := kv.ChecksumFromString(string(c.Value.Checksum)) 38 | if err != nil { 39 | return Commit{}, fmt.Errorf("couldn't parse checksum from basis %s: %s", h, string(c.Value.Checksum)) 40 | } 41 | if !checksum.Equal(expectedChecksum) { 42 | return Commit{}, fmt.Errorf("checksum mismatch: %s from client, %s in db", expectedChecksum, checksum) 43 | } 44 | return c, nil 45 | } 46 | 47 | func (db *DB) Diff(version uint32, fromHash hash.Hash, fromChecksum kv.Checksum, to Commit, l zl.Logger) ([]kv.Operation, error) { 48 | r := []kv.Operation{} 49 | var fc Commit 50 | var err error 51 | v := db.Noms().ReadValue(fromHash) 52 | if v == nil { 53 | // Unknown basis is not really en error: maybe it's really old 54 | // or we're starting up cold. But it is an interesting situation 55 | // so for now (small number of clients, still early in integration) 56 | // we report it as an error so we can take a look and verify that 57 | // nothing is amiss. We can turn it back into an info once we have 58 | // more confidence. 59 | l.Error().Msgf("Sending full sync: unknown basis %s", fromHash) 60 | r, fc = fullSync(version, db, fromHash, l) 61 | } else { 62 | fc, err = maybeDecodeCommit(v, fromHash, fromChecksum, l) 63 | if err != nil { 64 | // Inability to decode a Commit or getting the wrong checksum is an error. 65 | l.Error().Msgf("Sending full sync: cannot diff from basis %s: %s", fromHash, err) 66 | r, fc = fullSync(version, db, fromHash, l) 67 | } 68 | } 69 | 70 | if !fc.Value.Data.Equals(to.Value.Data) { 71 | fm := fc.Data(db.Noms()) 72 | tm := to.Data(db.Noms()) 73 | r, err = kv.Diff(version, fm, tm, r) 74 | if err != nil { 75 | return nil, err 76 | } 77 | } 78 | 79 | return r, nil 80 | } 81 | -------------------------------------------------------------------------------- /db/diff_test.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/attic-labs/noms/go/hash" 9 | "github.com/attic-labs/noms/go/types" 10 | "github.com/stretchr/testify/assert" 11 | "roci.dev/diff-server/kv" 12 | "roci.dev/diff-server/util/log" 13 | ) 14 | 15 | func TestDiff(t *testing.T) { 16 | assert := assert.New(t) 17 | db, dir := LoadTempDB(assert) 18 | fmt.Println(dir) 19 | 20 | var fromID hash.Hash 21 | var fromChecksum string 22 | tc := []struct { 23 | label string 24 | f func() 25 | expectedDiff []kv.Operation 26 | expectedError string 27 | }{ 28 | { 29 | "same-commit", 30 | func() {}, 31 | []kv.Operation{}, 32 | "", 33 | }, 34 | { 35 | "change-1", 36 | func() { 37 | m := kv.NewMapForTest(db.Noms(), "foo", `"bar"`, "hot", `"dog"`) 38 | c, err := db.MaybePutData(m, 0 /*lastMutationID*/) 39 | assert.False(c.NomsStruct.IsZeroValue()) 40 | assert.NoError(err) 41 | }, 42 | []kv.Operation{ 43 | { 44 | Op: kv.OpAdd, 45 | Path: "/foo", 46 | ValueString: "\"bar\"", 47 | }, 48 | { 49 | Op: kv.OpAdd, 50 | Path: "/hot", 51 | ValueString: "\"dog\"", 52 | }, 53 | }, 54 | "", 55 | }, 56 | { 57 | "change-2", 58 | func() { 59 | m := kv.NewMapForTest(db.Noms(), "foo", `"baz"`, "mon", `"key"`) 60 | c, err := db.MaybePutData(m, 0 /*lastMutationID*/) 61 | assert.False(c.NomsStruct.IsZeroValue()) 62 | assert.NoError(err) 63 | }, 64 | []kv.Operation{ 65 | { 66 | Op: kv.OpReplace, 67 | Path: "/foo", 68 | ValueString: "\"baz\"", 69 | }, 70 | { 71 | Op: kv.OpRemove, 72 | Path: "/hot", 73 | }, 74 | { 75 | Op: kv.OpAdd, 76 | Path: "/mon", 77 | ValueString: "\"key\"", 78 | }, 79 | }, 80 | "", 81 | }, 82 | { 83 | "no-diff", 84 | func() {}, 85 | []kv.Operation{}, 86 | "", 87 | }, 88 | { 89 | "fresh-non-existing-commit", 90 | func() { 91 | db, dir = LoadTempDB(assert) 92 | fmt.Println("newdir", dir) 93 | me := kv.NewMapForTest(db.Noms()).Edit() 94 | for _, s := range []string{"a", "b", "c"} { 95 | assert.NoError(me.Set(types.String(s), types.String(s))) 96 | } 97 | m := me.Build() 98 | c, err := db.MaybePutData(m, 0 /*lastMutationID*/) 99 | assert.False(c.NomsStruct.IsZeroValue()) 100 | assert.NoError(err) 101 | }, 102 | []kv.Operation{ 103 | kv.Operation{ 104 | Op: kv.OpReplace, 105 | Path: "", 106 | ValueString: "{}", 107 | }, 108 | kv.Operation{ 109 | Op: kv.OpAdd, 110 | Path: "/a", 111 | ValueString: `"a"`, 112 | }, 113 | kv.Operation{ 114 | Op: kv.OpAdd, 115 | Path: "/b", 116 | ValueString: `"b"`, 117 | }, 118 | kv.Operation{ 119 | Op: kv.OpAdd, 120 | Path: "/c", 121 | ValueString: `"c"`, 122 | }, 123 | }, 124 | "", 125 | }, 126 | { 127 | "fresh-empty-commit", 128 | func() { 129 | fromID = hash.Hash{} 130 | }, 131 | []kv.Operation{ 132 | kv.Operation{ 133 | Op: kv.OpReplace, 134 | Path: "", 135 | ValueString: "{}", 136 | }, 137 | kv.Operation{ 138 | Op: kv.OpAdd, 139 | Path: "/a", 140 | ValueString: `"a"`, 141 | }, 142 | kv.Operation{ 143 | Op: kv.OpAdd, 144 | Path: "/b", 145 | ValueString: `"b"`, 146 | }, 147 | kv.Operation{ 148 | Op: kv.OpAdd, 149 | Path: "/c", 150 | ValueString: `"c"`, 151 | }, 152 | }, 153 | "", 154 | }, 155 | { 156 | "invalid-checksum", 157 | func() { 158 | m := kv.NewMapForTest(db.Noms(), "foo", `"bar"`) 159 | c, err := db.MaybePutData(m, 0 /*lastMutationID*/) 160 | assert.False(c.NomsStruct.IsZeroValue()) 161 | assert.NoError(err) 162 | fromChecksum = "00000000" 163 | }, 164 | []kv.Operation{ 165 | { 166 | Op: kv.OpReplace, 167 | Path: "", 168 | ValueString: "{}", 169 | }, 170 | { 171 | Op: kv.OpAdd, 172 | Path: "/foo", 173 | ValueString: "\"bar\"", 174 | }, 175 | }, 176 | "", 177 | }, 178 | { 179 | "same-commit-invalid-checksum", 180 | func() { 181 | fromChecksum = "00000000" 182 | }, 183 | []kv.Operation{ 184 | { 185 | Op: kv.OpReplace, 186 | Path: "", 187 | ValueString: "{}", 188 | }, 189 | { 190 | Op: kv.OpAdd, 191 | Path: "/foo", 192 | ValueString: "\"bar\"", 193 | }, 194 | }, 195 | "", 196 | }, 197 | { 198 | "invalid-commit-id", 199 | func() { 200 | r := db.Noms().WriteValue(types.String("not a commit")) 201 | fromID = r.TargetHash() 202 | }, 203 | []kv.Operation{ 204 | { 205 | Op: kv.OpReplace, 206 | Path: "", 207 | ValueString: "{}", 208 | }, 209 | { 210 | Op: kv.OpAdd, 211 | Path: "/foo", 212 | ValueString: "\"bar\"", 213 | }, 214 | }, 215 | "", 216 | }, 217 | } 218 | 219 | for _, t := range tc { 220 | fromID = db.Head().NomsStruct.Hash() 221 | var err error 222 | fromChecksum = string(db.Head().Value.Checksum) 223 | t.f() 224 | c, err := kv.ChecksumFromString(fromChecksum) 225 | assert.NoError(err) 226 | r, err := db.Diff(2, fromID, *c, db.Head(), log.Default()) 227 | if t.expectedError == "" { 228 | assert.NoError(err, t.label) 229 | expected, err := json.Marshal(t.expectedDiff) 230 | assert.NoError(err, t.label) 231 | actual, err := json.Marshal(r) 232 | assert.NoError(err, t.label) 233 | assert.Equal(string(expected), string(actual), t.label) 234 | } else { 235 | assert.Nil(r, t.label) 236 | assert.EqualError(err, t.expectedError, t.label) 237 | } 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /db/tempdb.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "io/ioutil" 5 | 6 | "github.com/attic-labs/noms/go/spec" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func LoadTempDB(assert *assert.Assertions) (r *DB, dir string) { 11 | td, err := ioutil.TempDir("", "") 12 | assert.NoError(err) 13 | 14 | r = LoadTempDBWithPath(assert, td) 15 | return r, td 16 | } 17 | 18 | func LoadTempDBWithPath(assert *assert.Assertions, td string) (r *DB) { 19 | sp, err := spec.ForDatabase(td) 20 | assert.NoError(err) 21 | 22 | noms := sp.GetDatabase() 23 | r, err = New(noms.GetDataset("foo")) 24 | assert.NoError(err) 25 | 26 | return r 27 | } 28 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module roci.dev/diff-server 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect 7 | github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 // indirect 8 | github.com/attic-labs/noms v0.0.0-20200622153158-26620a34bc8c 9 | github.com/aws/aws-sdk-go v1.36.29 10 | github.com/davecgh/go-spew v1.1.1 // indirect 11 | github.com/gibson042/canonicaljson-go v1.0.3 12 | github.com/golang/protobuf v1.3.1 // indirect 13 | github.com/golang/snappy v0.0.2 // indirect 14 | github.com/gorilla/mux v1.8.0 15 | github.com/jpillora/backoff v1.0.0 // indirect 16 | github.com/julienschmidt/httprouter v1.3.0 // indirect 17 | github.com/justinas/alice v1.2.0 18 | github.com/motemen/go-loghttp v0.0.0-20170804080138-974ac5ceac27 19 | github.com/motemen/go-nuts v0.0.0-20200601065735-3df31f16cb2f // indirect 20 | github.com/onsi/ginkgo v1.8.0 // indirect 21 | github.com/onsi/gomega v1.5.0 // indirect 22 | github.com/pkg/errors v0.9.1 23 | github.com/rs/zerolog v1.20.0 24 | github.com/stretchr/testify v1.7.0 25 | github.com/zenazn/goji v0.9.0 // indirect 26 | golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect 27 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 // indirect 28 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 29 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 30 | gopkg.in/yaml.v2 v2.4.0 // indirect 31 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/aboodman/noms-gx v0.0.0-20180714061401-d6cb97cb040b/go.mod h1:ni7quUEZfdz5Q36a9VJgeUlTaYfwY3fS3j/v5WIz8zs= 5 | github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= 6 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= 7 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 8 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= 9 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 10 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= 11 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 12 | github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 h1:EBTWhcAX7rNQ80RLwLCpHZBBrJuzallFHnF+yMXo928= 13 | github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= 14 | github.com/attic-labs/graphql v0.0.0-20190507195614-b6552d20145f h1:WMEteRGdJItAZfxaCPyL6SEfyh4+bE+LsN50UKz46EA= 15 | github.com/attic-labs/graphql v0.0.0-20190507195614-b6552d20145f/go.mod h1:1U3eDKPYQXn3o4jpC2rAlH9THIo+ZOKWSI0FyeG1SEI= 16 | github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible h1:wd5mq8xSfwCYd1JpQ309s+3tTlP/gifcG2awOA3x5Vk= 17 | github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible/go.mod h1:Cp18FeDCvsK+cD2QAGkqerGjrgSXLiJWnjHeY2mneBc= 18 | github.com/attic-labs/noms v0.0.0-20191214023511-2a57d6783c14 h1:C5eSE1iNyi1Oqon7wS4wYA+FIubvGbgJFuuAaQyLSv4= 19 | github.com/attic-labs/noms v0.0.0-20191214023511-2a57d6783c14/go.mod h1:qhMSThkSBnQ2QGe2Y5QAlTNjPtpyV2abaKPJRVmpr4A= 20 | github.com/attic-labs/noms v0.0.0-20200622153158-26620a34bc8c h1:BkFbxT/qwcNkK6tU1FASNLsIjllrivqTp4dd4/0T9ec= 21 | github.com/attic-labs/noms v0.0.0-20200622153158-26620a34bc8c/go.mod h1:qhMSThkSBnQ2QGe2Y5QAlTNjPtpyV2abaKPJRVmpr4A= 22 | github.com/aws/aws-sdk-go v1.19.26 h1:GavKlzJDfYQGoS4jn2F+KYYZlR8QEhrLPfpf8+oJhS4= 23 | github.com/aws/aws-sdk-go v1.19.26/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 24 | github.com/aws/aws-sdk-go v1.19.28 h1:u0KMC+Qv0YVyz8YR6mREEtslSPkdUMzXgDJFD5196O8= 25 | github.com/aws/aws-sdk-go v1.19.28/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= 26 | github.com/aws/aws-sdk-go v1.36.29 h1:lM1G3AF1+7vzFm0n7hfH8r2+750BTo+6Lo6FtPB7kzk= 27 | github.com/aws/aws-sdk-go v1.36.29/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= 28 | github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= 29 | github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf/go.mod h1:BO2rLUAZMrpgh6GBVKi0Gjdqw2MgCtJrtmUdDeZRKjY= 30 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 31 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 32 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 33 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 34 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 35 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 36 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 37 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 38 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 39 | github.com/gibson042/canonicaljson-go v1.0.3 h1:EAyF8L74AWabkyUmrvEFHEt/AGFQeD6RfwbAuf0j1bI= 40 | github.com/gibson042/canonicaljson-go v1.0.3/go.mod h1:DsLpJTThXyGNO+KZlI85C1/KDcImpP67k/RKVjcaEqo= 41 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 42 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 43 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 44 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 45 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 46 | github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= 47 | github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 48 | github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= 49 | github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 50 | github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= 51 | github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 52 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 53 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 54 | github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok= 55 | github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= 56 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 57 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= 58 | github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= 59 | github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= 60 | github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= 61 | github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= 62 | github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= 63 | github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= 64 | github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= 65 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 66 | github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d h1:c93kUJDtVAXFEhsCh5jSxyOJmFHuzcihnslQiX8Urwo= 67 | github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= 68 | github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g= 69 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 70 | github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= 71 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 72 | github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo= 73 | github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNEe7i7qA= 74 | github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 h1:l6Y3mFnF46A+CeZsTrT8kVIuhayq1266oxWpDKE7hnQ= 75 | github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6/go.mod h1:UtDV9qK925GVmbdjR+e1unqoo+wGWNHHC6XB1Eu6wpE= 76 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 77 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 78 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 79 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 80 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 81 | github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= 82 | github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= 83 | github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 84 | github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= 85 | github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= 86 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= 87 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= 88 | github.com/motemen/go-loghttp v0.0.0-20170804080138-974ac5ceac27 h1:uAI3rnOT1OSSY4PUtI/M1orb3q0ewkovwd3wr8xSno4= 89 | github.com/motemen/go-loghttp v0.0.0-20170804080138-974ac5ceac27/go.mod h1:6eu9CfGt5kfrMVgeu9MfB9PRUnpc47I+udLswiTszI8= 90 | github.com/motemen/go-nuts v0.0.0-20190725124253-1d2432db96b0 h1:CnSVrlMNAZMWI1+uH6ldpXRv2pe7t50IQX448EJrJhw= 91 | github.com/motemen/go-nuts v0.0.0-20190725124253-1d2432db96b0/go.mod h1:vfh/NPxHgDwggXit20W1llPsXcz39xJ7I8vo7kVrOCk= 92 | github.com/motemen/go-nuts v0.0.0-20200601065735-3df31f16cb2f h1:+LOUyKIw83Rm7cj3nwulz+EFXWaWH0AQNBZP6MV/Ip0= 93 | github.com/motemen/go-nuts v0.0.0-20200601065735-3df31f16cb2f/go.mod h1:1tCxZmJDqO6FrbHD/8LZ8ELaccFziVjn1Y/tNaABMTg= 94 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 95 | github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= 96 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 97 | github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= 98 | github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 99 | github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= 100 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 101 | github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= 102 | github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 103 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 104 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 105 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 106 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 107 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 108 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= 109 | github.com/rs/zerolog v1.18.0 h1:CbAm3kP2Tptby1i9sYy2MGRg0uxIN9cyDb59Ys7W8z8= 110 | github.com/rs/zerolog v1.18.0/go.mod h1:9nvC1axdVrAHcu/s9taAVfBuIdTZLVQmKQyvrUjF5+I= 111 | github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs= 112 | github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= 113 | github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= 114 | github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e h1:VAzdS5Nw68fbf5RZ8RDVlUvPXNU6Z3jtPCK/qvm4FoQ= 115 | github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= 116 | github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= 117 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 118 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 119 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 120 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 121 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 122 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 123 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 124 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 125 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 126 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 127 | github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= 128 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 129 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 130 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 131 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 132 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 133 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 134 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c h1:uOCk1iQW6Vc18bnC13MfzScl+wdKBmM9Y9kU7Z83/lw= 135 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 136 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 137 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= 138 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 139 | golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 140 | golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 141 | golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= 142 | golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 143 | golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 144 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= 145 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 146 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 147 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 148 | golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 149 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 150 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 151 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 152 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 153 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 154 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= 155 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 156 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 157 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 158 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 159 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= 160 | golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 161 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 162 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 163 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 164 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 165 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 166 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 167 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e h1:FDhOuMEY4JVRztM/gsbk+IKUQ8kj74bxZrgw87eMMVc= 168 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 169 | golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 170 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 171 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 172 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= 173 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 174 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 175 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 176 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 177 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 178 | gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= 179 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 180 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= 181 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 182 | gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= 183 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 184 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 185 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 186 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 187 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 188 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 189 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 190 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 191 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 192 | -------------------------------------------------------------------------------- /kv/checksum.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "fmt" 5 | "hash/crc32" 6 | "strconv" 7 | 8 | "roci.dev/diff-server/util/chk" 9 | ) 10 | 11 | // Checksum represents a fast, incrementally computable, 12 | // non-cryptographic checksum of the contents of a kv store. 13 | type Checksum struct { 14 | value uint32 15 | } 16 | 17 | func (c Checksum) String() string { 18 | return fmt.Sprintf("%08x", c.value) 19 | } 20 | 21 | // ChecksumFromString parses a checksum value from a string. 22 | func ChecksumFromString(s string) (*Checksum, error) { 23 | v, err := strconv.ParseUint(s, 16, 32) 24 | if err != nil { 25 | return &Checksum{}, fmt.Errorf("Unable to parse '%s' as a Checksum", s) 26 | } 27 | return &Checksum{uint32(v)}, nil 28 | } 29 | 30 | // MustChecksumFromString panics if it cannot parse a Checksum from s. 31 | func MustChecksumFromString(s string) Checksum { 32 | c, err := ChecksumFromString(s) 33 | chk.NoError(err) 34 | return *c 35 | } 36 | 37 | func hashEntry(key string, value []byte) uint32 { 38 | keyLen := []byte(fmt.Sprintf("%d", len(key))) 39 | valLen := []byte(fmt.Sprintf("%d", len(value))) 40 | keyBytes := []byte(key) 41 | totalLen := len(keyLen) + len(keyBytes) + len(valLen) + len(value) 42 | input := make([]byte, totalLen) 43 | var i int 44 | i += copy(input[i:], keyLen) 45 | i += copy(input[i:], keyBytes) 46 | i += copy(input[i:], valLen) 47 | copy(input[i:], value) 48 | // Note: we could probably avoid the above copies using crc32.Update. 49 | return crc32.ChecksumIEEE(input) 50 | } 51 | 52 | // Add adds an entry to the checksum. 53 | func (c *Checksum) Add(key string, value []byte) { 54 | c.value ^= hashEntry(key, value) 55 | } 56 | 57 | // Remove removes an entry from the checksum. 58 | func (c *Checksum) Remove(key string, value []byte) { 59 | c.value ^= hashEntry(key, value) 60 | } 61 | 62 | // Replace replaces a key's value in the checksum. 63 | func (c *Checksum) Replace(key string, oldValue, newValue []byte) { 64 | c.Remove(key, oldValue) 65 | c.Add(key, newValue) 66 | } 67 | 68 | // Equal returns true if two checksums are equal. 69 | func (c Checksum) Equal(c2 Checksum) bool { 70 | return c.value == c2.value 71 | } 72 | 73 | // Reset resets the checksum to zero. 74 | func (c *Checksum) Reset() { 75 | c.value = 0 76 | } 77 | -------------------------------------------------------------------------------- /kv/checksum_test.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "fmt" 5 | "hash/crc32" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestChecksumComputeAndValue(t *testing.T) { 12 | assert := assert.New(t) 13 | 14 | var c Checksum 15 | assert.Equal("00000000", c.String()) 16 | 17 | // Might look like a dumb test but it caught two errors in the 18 | // original implementation. 19 | k := "key⌘" // ⌘ is 3 bytes, ensuring code is counting bytes not runes 20 | v := []byte{0x01, 0x02} 21 | expectedInput := []byte{ 22 | 0x36, // '6' 23 | 0x6B, 0x65, 0x79, 0xe2, 0x8c, 0x98, // 'k''e''y''⌘' 24 | 0x32, // '2' 25 | 0x01, 0x02, // {0x01, 0x02} 26 | } 27 | c.Add(k, v) 28 | assert.Equal(fmt.Sprintf("%08x", crc32.ChecksumIEEE(expectedInput)), c.String()) 29 | } 30 | 31 | func TestChecksumOperations(t *testing.T) { 32 | assert := assert.New(t) 33 | 34 | k1, v1 := "1", []byte{0x01} 35 | k2, v2 := "2", []byte{0x02} 36 | var c1, c2 Checksum 37 | 38 | c1.Add(k1, v1) 39 | assert.True(c1.Equal(c1)) 40 | assert.False(c1.Equal(c2)) 41 | c1.Reset() 42 | assert.True(c1.Equal(c2)) 43 | 44 | c1.Add(k1, v1) 45 | c2.Add(k2, v2) 46 | c2.Add(k1, v1) 47 | assert.False(c2.Equal(c1)) 48 | c2.Remove(k2, v2) 49 | assert.True(c1.Equal(c2)) 50 | 51 | c1.Replace(k1, v1, v2) 52 | var c3 Checksum 53 | c3.Add(k1, v2) 54 | assert.True(c3.Equal(c1)) 55 | } 56 | 57 | func TestChecksumFromString(t *testing.T) { 58 | tests := []struct { 59 | name string 60 | s string 61 | wantVal uint32 62 | wantErr bool 63 | }{ 64 | {"parses", "00cf3d55", 13581653, false}, 65 | {"empty", "", 0, true}, 66 | {"not a hex number", "00ps", 0, true}, 67 | } 68 | for _, tt := range tests { 69 | t.Run(tt.name, func(t *testing.T) { 70 | c, err := ChecksumFromString(tt.s) 71 | 72 | if (err != nil) != tt.wantErr { 73 | t.Errorf("ChecksumFromString() error = %v, wantErr %v", err, tt.wantErr) 74 | } 75 | if !tt.wantErr && c.value != tt.wantVal { 76 | t.Errorf("ChecksumFromString() got = %v, want %v", c.value, tt.wantVal) 77 | } 78 | }) 79 | } 80 | } 81 | 82 | func TestMustChecksumFromString(t *testing.T) { 83 | tests := []struct { 84 | name string 85 | s string 86 | wantPanic bool 87 | }{ 88 | {"parses", "00cf3d55", false}, 89 | {"panics", "boom", true}, 90 | } 91 | for _, tt := range tests { 92 | t.Run(tt.name, func(t *testing.T) { 93 | if tt.wantPanic { 94 | assert.PanicsWithValue(t, `Unexpected error: &errors.errorString{s:"Unable to parse 'boom' as a Checksum"}`, func() { MustChecksumFromString(tt.s) }) 95 | } else { 96 | assert.NotPanics(t, func() { MustChecksumFromString(tt.s) }) 97 | } 98 | }) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /kv/map.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "roci.dev/diff-server/util/chk" 8 | nomsjson "roci.dev/diff-server/util/noms/json" 9 | 10 | "github.com/attic-labs/noms/go/types" 11 | ) 12 | 13 | // Map embeds types.Map, adding a few bits of logic like checksumming. 14 | // Map is NOT threadsafe. 15 | type Map struct { 16 | noms types.ValueReadWriter 17 | types.Map 18 | sum Checksum 19 | } 20 | 21 | // NewMap returns a new Map. 22 | func NewMap(noms types.ValueReadWriter) Map { 23 | return Map{noms, types.NewMap(noms), Checksum{0}} 24 | } 25 | 26 | // FromNoms creates a map from an existing Noms Map and Checksum. 27 | func FromNoms(noms types.ValueReadWriter, nm types.Map, c Checksum) Map { 28 | return Map{noms, nm, c} 29 | } 30 | 31 | // ComputeChecksum iterates a noms map and computes its checksum. The noms map is 32 | // assumed to be canonicalized. 33 | func ComputeChecksum(nm types.Map) Checksum { 34 | c := Checksum{} 35 | for mi := nm.Iterator(); mi.Valid(); mi.Next() { 36 | k := string(mi.Key().(types.String)) 37 | v, err := toJSON(mi.Value()) 38 | if err != nil { 39 | chk.Fail("Failed to serialize value to json.") 40 | } 41 | c.Add(k, v) 42 | } 43 | return c 44 | } 45 | 46 | // NomsMap returns the underlying noms map. 47 | func (m Map) NomsMap() types.Map { 48 | return m.Map 49 | } 50 | 51 | func toJSON(value types.Valuable) ([]byte, error) { 52 | var b bytes.Buffer 53 | if err := nomsjson.ToJSON(value.Value(), &b); err != nil { 54 | return []byte{}, err 55 | } 56 | return b.Bytes(), nil 57 | } 58 | 59 | // Checksum is the checksum of the Map. 60 | func (m Map) Checksum() string { 61 | return m.sum.String() 62 | } 63 | 64 | // NomsChecksum returns the checksum as a types.String. 65 | func (m Map) NomsChecksum() types.String { 66 | return types.String(m.Checksum()) 67 | } 68 | 69 | // Edit returns a MapEditor allowing mutation of the Map. The original 70 | // Map is not affected. 71 | func (m Map) Edit() *MapEditor { 72 | return &MapEditor{m.noms, m.Map.Edit(), m.sum} 73 | } 74 | 75 | // DebugString returns a nice string value of the Map, including the full underlying noms map. 76 | func (m Map) DebugString() string { 77 | return fmt.Sprintf("Checksum: %s, noms Map: %v\n", m.Checksum(), types.EncodedValue(m.NomsMap())) 78 | } 79 | 80 | // MapEditor embeds a types.MapEditor, enabling mutations. 81 | type MapEditor struct { 82 | noms types.ValueReadWriter 83 | *types.MapEditor 84 | sum Checksum 85 | } 86 | 87 | // Get changes the signature of MapEditor's Get to match that of Map. 88 | func (me *MapEditor) Get(key types.Value) types.Value { 89 | v := me.MapEditor.Get(key) 90 | if v == nil { 91 | return nil 92 | } 93 | return v.Value() 94 | } 95 | 96 | // Set sets the value for a given key. Set requires that the value has been 97 | // be parsed from canonical json, otherwise we might parse two different 98 | // values for the same canonical json. 99 | func (me *MapEditor) Set(key types.String, value types.Value) error { 100 | if me.MapEditor.Has(key) { 101 | // Have to do this in order to properly update checksum. 102 | if err := me.Remove(key); err != nil { 103 | return err 104 | } 105 | } 106 | 107 | JSON, err := toJSON(value) 108 | if err != nil { 109 | return err 110 | } 111 | me.MapEditor.Set(key, value) 112 | me.sum.Add(string(key), JSON) 113 | return nil 114 | } 115 | 116 | // Remove removes a key from the Map. 117 | func (me *MapEditor) Remove(key types.String) error { 118 | // Need the old value to update the checksum. 119 | // Note: Noms MapEditor.Get can return a value that has been removed 120 | // so here we check Has, which works correctly. Once 121 | // https://github.com/attic-labs/noms/pull/3872 is released this can 122 | // just Get directly. 123 | if me.MapEditor.Has(key) { 124 | oldValue := me.MapEditor.Get(key) 125 | oldValueJSON, err := toJSON(oldValue.Value()) 126 | if err != nil { 127 | return err 128 | } 129 | me.sum.Remove(string(key), oldValueJSON) 130 | } 131 | 132 | me.MapEditor.Remove(key) 133 | return nil 134 | } 135 | 136 | // Build converts back into a Map. 137 | func (me *MapEditor) Build() Map { 138 | return Map{me.noms, me.MapEditor.Map(), me.sum} 139 | } 140 | 141 | // Checksum is the Cheksum over the Map of k/vs. 142 | func (me MapEditor) Checksum() Checksum { 143 | return me.sum 144 | } 145 | 146 | // DebugString returns a nice string value of the MapEditor, including the full underlying noms map. 147 | func (me MapEditor) DebugString() string { 148 | m := me.Build() 149 | return fmt.Sprintf("Checksum: %s, noms Map: %v\n", m.Checksum(), types.EncodedValue(m.NomsMap())) 150 | } 151 | -------------------------------------------------------------------------------- /kv/map_test.go: -------------------------------------------------------------------------------- 1 | package kv_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/attic-labs/noms/go/nomdl" 7 | "github.com/attic-labs/noms/go/types" 8 | "github.com/stretchr/testify/assert" 9 | "roci.dev/diff-server/kv" 10 | nomsjson "roci.dev/diff-server/util/noms/json" 11 | "roci.dev/diff-server/util/noms/memstore" 12 | ) 13 | 14 | func s(s string) types.String { 15 | return types.String(s) 16 | } 17 | 18 | func TestComputeChecksum(t *testing.T) { 19 | assert := assert.New(t) 20 | noms := memstore.New() 21 | 22 | // Ensure it matches when built. 23 | me := kv.NewMap(noms).Edit() 24 | assert.NoError(me.Set(s("foo"), types.Bool(true))) 25 | assert.NoError(me.Set(s("bar"), types.Bool(true))) 26 | assert.NoError(me.Remove(s("foo"))) 27 | m := me.Build() 28 | assert.Equal(m.Checksum(), kv.ComputeChecksum(m.NomsMap()).String()) 29 | 30 | // Ensure it matches a noms map separately constructed. 31 | nm := types.NewMap(noms, s("bar"), types.Bool(true)) 32 | assert.Equal(m.Checksum(), kv.ComputeChecksum(nm).String()) 33 | } 34 | 35 | type getter interface { 36 | Get(types.Value) types.Value 37 | } 38 | 39 | func assertGetEqual(assert *assert.Assertions, m getter, key types.String, expected types.Value) { 40 | got := m.Get(key) 41 | if expected == nil { 42 | assert.Nil(got) 43 | } else { 44 | assert.True(expected.Equals(got)) 45 | } 46 | } 47 | 48 | func TestMapGetSetRemove(t *testing.T) { 49 | assert := assert.New(t) 50 | noms := memstore.New() 51 | 52 | k1 := s("k1") 53 | v1, v2 := s("1"), s("2") 54 | 55 | em := kv.NewMap(noms) 56 | assertGetEqual(assert, em, k1, nil) 57 | 58 | m1 := kv.NewMap(noms) 59 | m1e := m1.Edit() 60 | assert.NoError(m1e.Set(k1, v1)) 61 | assertGetEqual(assert, m1e, k1, v1) 62 | m1 = m1e.Build() 63 | assert.NotEqual(em.Checksum(), m1.Checksum()) 64 | assertGetEqual(assert, m1, k1, v1) 65 | m1e = m1.Edit() 66 | m1e.Set(k1, v2) 67 | assertGetEqual(assert, m1e, k1, v2) 68 | assertGetEqual(assert, m1, k1, v1) 69 | m2 := m1e.Build() 70 | assertGetEqual(assert, m2, k1, v2) 71 | assert.NotEqual(m2.Checksum(), m1.Checksum()) 72 | 73 | m2e := m2.Edit() 74 | assert.NoError(m2e.Remove(k1)) 75 | // Uncomment when https://github.com/attic-labs/noms/pull/3872 is released. 76 | // assertGetEqual(assert, m2e, k1, nil) 77 | assert.NoError(m2e.Remove(k1)) 78 | m2got := m2e.Build() 79 | assertGetEqual(assert, m2got, k1, nil) 80 | assert.NotEqual(m2.Checksum(), m2got.Checksum(), "got=%s, want=%s", m2got.DebugString(), m2.DebugString()) 81 | assert.Equal(em.Checksum(), m2got.Checksum(), "got=%s, want=%s", m2got.DebugString(), em.DebugString()) 82 | 83 | // Test that if we do two edit operations both stick. 84 | k2 := s("k2") 85 | m1 = kv.NewMap(noms) 86 | m1e = m1.Edit() 87 | assert.NoError(m1e.Set(k1, v1)) 88 | assert.NoError(m1e.Set(k2, v2)) 89 | assertGetEqual(assert, m1e, k1, v1) 90 | assertGetEqual(assert, m1e, k2, v2) 91 | m1 = m1e.Build() 92 | assertGetEqual(assert, m1, k1, v1) 93 | assertGetEqual(assert, m1, k2, v2) 94 | } 95 | func TestNull(t *testing.T) { 96 | assert := assert.New(t) 97 | noms := memstore.New() 98 | m1 := kv.NewMap(noms) 99 | m1e := m1.Edit() 100 | err := m1e.Set(s("foo"), nomsjson.Null()) 101 | m1 = m1e.Build() 102 | assert.NoError(err) 103 | act := m1.Get(s("foo")) 104 | assert.NotNil(act) 105 | assert.True(nomsjson.Null().Equals(act)) 106 | } 107 | 108 | func TestEmptyKey(t *testing.T) { 109 | assert := assert.New(t) 110 | noms := memstore.New() 111 | me := kv.NewMap(noms).Edit() 112 | assert.NoError(me.Set(s(""), types.Bool(true))) 113 | m := me.Build() 114 | expected := nomdl.MustParse(noms, "map {\"\": true}").(types.Map) 115 | assert.Equal(types.EncodedValue(m.NomsMap()), 116 | types.EncodedValue(expected)) 117 | } 118 | 119 | func TestEmpty(t *testing.T) { 120 | assert := assert.New(t) 121 | noms := memstore.New() 122 | 123 | m := kv.NewMap(noms) 124 | assert.True(m.Empty()) 125 | me := kv.NewMap(noms).Edit() 126 | assert.NoError(me.Set(s("foo"), types.Bool(true))) 127 | m = me.Build() 128 | assert.False(m.Empty()) 129 | } 130 | -------------------------------------------------------------------------------- /kv/patch.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | // This file implements JSON Patch format for Noms-based Maps. 4 | // See http://jsonpatch.com/ 5 | // 6 | // Notes: 7 | // - only currently supports the "add", "remove", and "replace" operations. 8 | // - can only compute diffs on Noms values that are Boolean|Number|String, or Lists and Maps containing those types. 9 | 10 | import ( 11 | "bytes" 12 | "encoding/json" 13 | "fmt" 14 | "runtime" 15 | "sort" 16 | "strings" 17 | "sync" 18 | 19 | "github.com/attic-labs/noms/go/types" 20 | "roci.dev/diff-server/util/chk" 21 | nomsjson "roci.dev/diff-server/util/noms/json" 22 | ) 23 | 24 | const ( 25 | // OpAdd is the JSONPatch "add" operation. 26 | OpAdd = "add" 27 | // OpRemove is the JSONPatch "add" operation. 28 | OpRemove = "remove" 29 | // OpReplace is the JSONPatch "replace" operation. 30 | OpReplace = "replace" 31 | ) 32 | 33 | // Operation is a single JSONPatch change. 34 | type Operation struct { 35 | Op string `json:"op"` 36 | Path string `json:"path"` 37 | Value json.RawMessage `json:"value,omitempty"` 38 | ValueString string `json:"valueString,omitempty"` 39 | } 40 | 41 | func jsonPointerEscape(s string) string { 42 | return strings.ReplaceAll(strings.ReplaceAll(s, "~", "~0"), "/", "~1") 43 | } 44 | 45 | func jsonPointerUnescape(s string) string { 46 | return strings.ReplaceAll(strings.ReplaceAll(s, "~1", "/"), "~0", "~") 47 | } 48 | 49 | // Diff calculates the difference between two maps as a JSON patch. Presently only 50 | // creates ops at the top level, at the level of keys, so not super efficient. 51 | func Diff(version uint32, from, to Map, r []Operation) ([]Operation, error) { 52 | dChan := make(chan types.ValueChanged) 53 | sChan := make(chan struct{}) 54 | out := make(chan Operation) 55 | 56 | go func() { 57 | defer close(dChan) 58 | // Diffing is delegated to the underlying noms maps. 59 | to.NomsMap().Diff(from.NomsMap(), dChan, sChan) 60 | }() 61 | 62 | wg := &sync.WaitGroup{} 63 | var err error 64 | 65 | // We do this in parallel because ToJSON() below can end up requiring fetching more data, which we don't want 66 | // serialized. 67 | for i := 0; i < runtime.NumCPU()*2; i++ { 68 | wg.Add(1) 69 | go func() { 70 | defer wg.Done() 71 | for d := range dChan { 72 | chk.Equal(types.StringKind, d.Key.Kind()) 73 | 74 | op := Operation{ 75 | Path: fmt.Sprintf("/%s", jsonPointerEscape(string(d.Key.(types.String)))), 76 | } 77 | switch d.ChangeType { 78 | case types.DiffChangeRemoved: 79 | op.Op = OpRemove 80 | case types.DiffChangeAdded, types.DiffChangeModified: 81 | b := &bytes.Buffer{} 82 | err = nomsjson.ToJSON(d.NewValue, b) 83 | if err != nil { 84 | // Would be nice to return an error out of here but there is no plumbing 85 | // for it. If you have time feel free. 86 | chk.Fail("Couldn't convert noms value to json: %#v", d) 87 | } 88 | if d.ChangeType == types.DiffChangeAdded { 89 | op.Op = OpAdd 90 | } else { 91 | op.Op = OpReplace 92 | } 93 | if version == 0 { 94 | op.Value = json.RawMessage(b.Bytes()) 95 | } else { 96 | op.ValueString = string(b.Bytes()) 97 | } 98 | default: 99 | chk.Fail("Unexpected ChangeType: %#v", d) 100 | } 101 | out <- op 102 | } 103 | }() 104 | } 105 | 106 | go func() { 107 | wg.Wait() 108 | close(out) 109 | }() 110 | 111 | for op := range out { 112 | r = append(r, op) 113 | } 114 | 115 | if err != nil { 116 | return nil, err 117 | } 118 | 119 | sort.Slice(r, func(i, j int) bool { 120 | return r[i].Path < r[j].Path 121 | }) 122 | 123 | return r, nil 124 | } 125 | 126 | // ApplyPatch applies the given series of ops to the input Map. 127 | func ApplyPatch(version uint32, vrw types.ValueReadWriter, to Map, patch []Operation) (Map, error) { 128 | if len(patch) == 0 { 129 | return to, nil 130 | } 131 | ed := to.Edit() 132 | for _, op := range patch { 133 | if version >= 2 && 134 | op.Path == "" && op.Op == OpReplace && op.ValueString == "{}" { 135 | // Clear map 136 | emptyMap := NewMap(ed.noms) 137 | ed = emptyMap.Edit() 138 | continue 139 | } 140 | 141 | if !strings.HasPrefix(op.Path, "/") { 142 | return Map{}, fmt.Errorf("Invalid path %s - must start with /", op.Path) 143 | } 144 | p := types.String(jsonPointerUnescape(op.Path[1:])) 145 | switch op.Op { 146 | case OpAdd, OpReplace: 147 | var v types.Value 148 | var err error 149 | if version == 0 { 150 | v, err = nomsjson.FromJSON(op.Value, vrw) 151 | } else { 152 | v, err = nomsjson.FromJSON([]byte(op.ValueString), vrw) 153 | } 154 | if err != nil { 155 | return Map{}, fmt.Errorf("couldnt parse value from JSON '%s': %w", op.Value, err) 156 | } 157 | if err := ed.Set(p, v); err != nil { 158 | return Map{}, err 159 | } 160 | case OpRemove: 161 | if len(p) == 0 { // Remove("/") 162 | emptyMap := NewMap(ed.noms) 163 | ed = emptyMap.Edit() 164 | } else if err := ed.Remove(p); err != nil { 165 | return Map{}, err 166 | } 167 | default: 168 | return Map{}, fmt.Errorf("Unknown JSON Patch operation: %s", op.Op) 169 | } 170 | } 171 | return ed.Build(), nil 172 | } 173 | -------------------------------------------------------------------------------- /kv/patch_test.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "encoding/json" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/attic-labs/noms/go/nomdl" 9 | "github.com/attic-labs/noms/go/types" 10 | "github.com/stretchr/testify/assert" 11 | "roci.dev/diff-server/util/noms/memstore" 12 | ) 13 | 14 | func TestDiffV0(t *testing.T) { 15 | assert := assert.New(t) 16 | 17 | tc := []struct { 18 | label string 19 | from string 20 | to string 21 | expectedResult []string 22 | expectedError string 23 | }{ 24 | {"insert", 25 | `map {}`, `map{"foo":"bar"}`, []string{`{"op":"add","path":"/foo","value":"bar"}`}, ""}, 26 | {"remove", 27 | `map{"foo":"bar"}`, `map {}`, []string{`{"op":"remove","path":"/foo"}`}, ""}, 28 | {"replace", 29 | `map{"foo":"bar"}`, `map {"foo":"baz"}`, []string{`{"op":"replace","path":"/foo","value":"baz"}`}, ""}, 30 | {"escape-1", 31 | `map {}`, `map{"/":"foo"}`, []string{`{"op":"add","path":"/~1","value":"foo"}`}, ""}, 32 | {"escape-2", 33 | `map {}`, `map{"~":"foo"}`, []string{`{"op":"add","path":"/~0","value":"foo"}`}, ""}, 34 | {"deep", 35 | `map {"foo":map{"bar":"baz"}}`, `map {"foo":map{"bar":"quux"}}`, 36 | []string{`{"op":"replace","path":"/foo","value":{"bar":"quux"}}`}, ""}, 37 | {"all-types", 38 | `map{}`, `map {"foo":map{"b":true,"i":42,"f":88.8,"s":"monkey","a":[],"a2":[true,42,8.88E1],"o":map{}}}`, 39 | []string{`{"op":"add","path":"/foo","value":{"a":[],"a2":[true,42,8.88E1],"b":true,"f":8.88E1,"i":42,"o":{},"s":"monkey"}}`}, ""}, 40 | {"multiple", 41 | `map {"a":"a","b":"b"}`, `map {"b":"bb","c":"c"}`, 42 | []string{ 43 | `{"op":"remove","path":"/a"}`, 44 | `{"op":"replace","path":"/b","value":"bb"}`, 45 | `{"op":"add","path":"/c","value":"c"}`, 46 | }, ""}, 47 | } 48 | 49 | noms := memstore.New() 50 | for _, t := range tc { 51 | nm := nomdl.MustParse(noms, t.from).(types.Map) 52 | from := FromNoms(noms, nm, ComputeChecksum(nm)) 53 | nm = nomdl.MustParse(noms, t.to).(types.Map) 54 | to := FromNoms(noms, nm, ComputeChecksum(nm)) 55 | r := []Operation{} 56 | r, err := Diff(0, from, to, r) 57 | if t.expectedError == "" { 58 | assert.NoError(err, t.label) 59 | j, err := json.Marshal(r) 60 | assert.NoError(err, t.label) 61 | assert.Equal("["+strings.Join(t.expectedResult, ",")+"]", string(j), t.label) 62 | got, err := ApplyPatch(0, noms, from, r) 63 | es, gots := types.EncodedValue(to.NomsMap()), types.EncodedValue(got.NomsMap()) 64 | assert.Equal(es, gots, "%s expected %s got %s", t.label, es, gots) 65 | assert.Equal(to.Checksum(), got.Checksum(), "%s expected %s got %s", t.label, es, gots) 66 | } else { 67 | assert.EqualError(err, t.expectedError, t.label) 68 | // buf might have arbitrary data, not part of the contract 69 | } 70 | } 71 | } 72 | 73 | func TestDiffV1(t *testing.T) { 74 | assert := assert.New(t) 75 | 76 | tc := []struct { 77 | label string 78 | from string 79 | to string 80 | expectedResult []string 81 | expectedError string 82 | }{ 83 | {"insert", 84 | `map {}`, `map{"foo":"bar"}`, []string{`{"op":"add","path":"/foo","valueString":"\"bar\""}`}, ""}, 85 | {"remove", 86 | `map{"foo":"bar"}`, `map {}`, []string{`{"op":"remove","path":"/foo"}`}, ""}, 87 | {"replace", 88 | `map{"foo":"bar"}`, `map {"foo":"baz"}`, []string{`{"op":"replace","path":"/foo","valueString":"\"baz\""}`}, ""}, 89 | {"escape-1", 90 | `map {}`, `map{"/":"foo"}`, []string{`{"op":"add","path":"/~1","valueString":"\"foo\""}`}, ""}, 91 | {"escape-2", 92 | `map {}`, `map{"~":"foo"}`, []string{`{"op":"add","path":"/~0","valueString":"\"foo\""}`}, ""}, 93 | {"deep", 94 | `map {"foo":map{"bar":"baz"}}`, `map {"foo":map{"bar":"quux"}}`, 95 | []string{`{"op":"replace","path":"/foo","valueString":"{\"bar\":\"quux\"}"}`}, ""}, 96 | {"all-types", 97 | `map{}`, `map {"foo":map{"b":true,"i":42,"f":88.8,"s":"monkey","a":[],"a2":[true,42,8.88E1],"o":map{}}}`, 98 | []string{`{"op":"add","path":"/foo","valueString":"{\"a\":[],\"a2\":[true,42,8.88E1],\"b\":true,\"f\":8.88E1,\"i\":42,\"o\":{},\"s\":\"monkey\"}"}`}, ""}, 99 | {"multiple", 100 | `map {"a":"a","b":"b"}`, `map {"b":"bb","c":"c"}`, 101 | []string{ 102 | `{"op":"remove","path":"/a"}`, 103 | `{"op":"replace","path":"/b","valueString":"\"bb\""}`, 104 | `{"op":"add","path":"/c","valueString":"\"c\""}`, 105 | }, ""}, 106 | } 107 | 108 | noms := memstore.New() 109 | for _, t := range tc { 110 | nm := nomdl.MustParse(noms, t.from).(types.Map) 111 | from := FromNoms(noms, nm, ComputeChecksum(nm)) 112 | nm = nomdl.MustParse(noms, t.to).(types.Map) 113 | to := FromNoms(noms, nm, ComputeChecksum(nm)) 114 | r := []Operation{} 115 | r, err := Diff(1, from, to, r) 116 | if t.expectedError == "" { 117 | assert.NoError(err, t.label) 118 | j, err := json.Marshal(r) 119 | assert.NoError(err, t.label) 120 | assert.Equal("["+strings.Join(t.expectedResult, ",")+"]", string(j), t.label) 121 | got, err := ApplyPatch(1, noms, from, r) 122 | es, gots := types.EncodedValue(to.NomsMap()), types.EncodedValue(got.NomsMap()) 123 | assert.Equal(es, gots, "%s expected %s got %s", t.label, es, gots) 124 | assert.Equal(to.Checksum(), got.Checksum(), "%s expected %s got %s", t.label, es, gots) 125 | } else { 126 | assert.EqualError(err, t.expectedError, t.label) 127 | // buf might have arbitrary data, not part of the contract 128 | } 129 | } 130 | } 131 | 132 | func TestTopLevelRemoveV0(t *testing.T) { 133 | // Diff doesn't currently generate a top level remove, so test here. 134 | assert := assert.New(t) 135 | noms := memstore.New() 136 | 137 | fs, ts := `map {"a":"a","b":"b"}`, `map {"b":"bb"}` 138 | nm := nomdl.MustParse(noms, fs).(types.Map) 139 | from := FromNoms(noms, nm, ComputeChecksum(nm)) 140 | nm = nomdl.MustParse(noms, ts).(types.Map) 141 | to := FromNoms(noms, nm, ComputeChecksum(nm)) 142 | 143 | ops := []Operation{ 144 | Operation{OpRemove, "/", []byte{}, ""}, 145 | Operation{OpReplace, "/b", []byte("\"bb\""), ""}, 146 | } 147 | r, err := ApplyPatch(0, noms, from, ops) 148 | assert.NoError(err) 149 | assert.Equal(types.EncodedValue(r.NomsMap()), types.EncodedValue(to.NomsMap())) 150 | assert.Equal(to.Checksum(), r.Checksum(), "expected %s, got %s", to.DebugString(), r.DebugString()) 151 | } 152 | 153 | func TestTopLevelRemoveV1(t *testing.T) { 154 | // Diff doesn't currently generate a top level remove, so test here. 155 | assert := assert.New(t) 156 | noms := memstore.New() 157 | 158 | fs, ts := `map {"a":"a","b":"b"}`, `map {"b":"bb"}` 159 | nm := nomdl.MustParse(noms, fs).(types.Map) 160 | from := FromNoms(noms, nm, ComputeChecksum(nm)) 161 | nm = nomdl.MustParse(noms, ts).(types.Map) 162 | to := FromNoms(noms, nm, ComputeChecksum(nm)) 163 | 164 | ops := []Operation{ 165 | Operation{OpRemove, "/", nil, ""}, 166 | Operation{OpReplace, "/b", nil, "\"bb\""}, 167 | } 168 | r, err := ApplyPatch(1, noms, from, ops) 169 | assert.NoError(err) 170 | assert.Equal(types.EncodedValue(r.NomsMap()), types.EncodedValue(to.NomsMap())) 171 | assert.Equal(to.Checksum(), r.Checksum(), "expected %s, got %s", to.DebugString(), r.DebugString()) 172 | } 173 | 174 | func TestTopLevelRemoveV2(t *testing.T) { 175 | // Diff doesn't currently generate a top level remove, so test here. 176 | assert := assert.New(t) 177 | noms := memstore.New() 178 | 179 | fs, ts := `map {"a":"a","b":"b"}`, `map {"":"d","b":"bb"}` 180 | nm := nomdl.MustParse(noms, fs).(types.Map) 181 | from := FromNoms(noms, nm, ComputeChecksum(nm)) 182 | nm = nomdl.MustParse(noms, ts).(types.Map) 183 | to := FromNoms(noms, nm, ComputeChecksum(nm)) 184 | 185 | ops := []Operation{ 186 | Operation{OpReplace, "", nil, "{}"}, 187 | Operation{OpReplace, "/b", nil, "\"bb\""}, 188 | Operation{OpAdd, "/", nil, "\"c\""}, 189 | Operation{OpReplace, "/", nil, "\"d\""}, 190 | } 191 | r, err := ApplyPatch(2, noms, from, ops) 192 | assert.NoError(err) 193 | assert.Equal(types.EncodedValue(r.NomsMap()), types.EncodedValue(to.NomsMap())) 194 | assert.Equal(to.Checksum(), r.Checksum(), "expected %s, got %s", to.DebugString(), r.DebugString()) 195 | } 196 | 197 | // There was a bug where we were including trailing newlines in values. 198 | func TestDiffDoesntIncludeNewlines(t *testing.T) { 199 | assert := assert.New(t) 200 | noms := memstore.New() 201 | 202 | from := NewMap(noms) 203 | to := NewMapForTest(noms, "key", "true") 204 | ops, err := Diff(1, from, to, []Operation{}) 205 | assert.NoError(err) 206 | assert.True(len(ops) == 1) 207 | assert.NotContains(string(ops[0].Value), "\n") 208 | } 209 | -------------------------------------------------------------------------------- /kv/testing.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "github.com/attic-labs/noms/go/types" 5 | "roci.dev/diff-server/util/chk" 6 | nomsjson "roci.dev/diff-server/util/noms/json" 7 | ) 8 | 9 | // NewMap returns a new Map with the given keys and values. 10 | func NewMapForTest(noms types.ValueReadWriter, kvs ...string) Map { 11 | me := NewMap(noms).Edit() 12 | for i := 0; i < len(kvs); i += 2 { 13 | v, err := nomsjson.FromJSON([]byte(kvs[i+1]), noms) 14 | chk.NoError(err) 15 | err = me.Set(types.String(kvs[i]), v) 16 | chk.NoError(err) 17 | } 18 | return me.Build() 19 | } 20 | -------------------------------------------------------------------------------- /kv/testing_test.go: -------------------------------------------------------------------------------- 1 | package kv_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | "roci.dev/diff-server/kv" 8 | "roci.dev/diff-server/util/noms/memstore" 9 | ) 10 | 11 | func TestNewMap(t *testing.T) { 12 | assert := assert.New(t) 13 | 14 | noms := memstore.New() 15 | 16 | // Ensure checksum matches if constructed vs built. 17 | constructed := kv.NewMapForTest(noms, "key1", `"1"`, "key2", `"2"`) 18 | me := kv.NewMap(noms).Edit() 19 | assert.NoError(me.Set("key1", s("1"))) 20 | assert.NoError(me.Set("key2", s("2"))) 21 | built := me.Build() 22 | assert.Equal(constructed.Checksum(), built.Checksum(), "constructed %v, built %v", constructed.DebugString(), built.DebugString()) 23 | } 24 | -------------------------------------------------------------------------------- /licenses/APL.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /licenses/BSL.txt: -------------------------------------------------------------------------------- 1 | Business Source License 1.1 2 | 3 | Parameters 4 | 5 | Licensor: Rocicorp, LLC 6 | 7 | Licensed Work: diff-server 1.1.0 8 | The Licensed Work is (c) 2020 Rocicorp, LLC. 9 | 10 | Additional Use Grant: You may use the Licensed Work when your application uses 11 | the Licensed Work with a total of less than 100 client 12 | instances for any purpose. 13 | 14 | Change Date: 2023-02-02 15 | 16 | Change License: Apache License, Version 2.0 17 | 18 | For information about alternative licensing arrangements for the Software, 19 | please visit: https://replicache.dev/ 20 | 21 | Notice 22 | 23 | The Business Source License (this document, or the “License”) is not an Open 24 | Source license. However, the Licensed Work will eventually be made available 25 | under an Open Source License, as stated in this License. 26 | 27 | License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. 28 | “Business Source License” is a trademark of MariaDB Corporation Ab. 29 | 30 | ----------------------------------------------------------------------------- 31 | 32 | Business Source License 1.1 33 | 34 | Terms 35 | 36 | The Licensor hereby grants you the right to copy, modify, create derivative 37 | works, redistribute, and make non-production use of the Licensed Work. The 38 | Licensor may make an Additional Use Grant, above, permitting limited 39 | production use. 40 | 41 | Effective on the Change Date, or the fourth anniversary of the first publicly 42 | available distribution of a specific version of the Licensed Work under this 43 | License, whichever comes first, the Licensor hereby grants you rights under 44 | the terms of the Change License, and the rights granted in the paragraph 45 | above terminate. 46 | 47 | If your use of the Licensed Work does not comply with the requirements 48 | currently in effect as described in this License, you must purchase a 49 | commercial license from the Licensor, its affiliated entities, or authorized 50 | resellers, or you must refrain from using the Licensed Work. 51 | 52 | All copies of the original and modified Licensed Work, and derivative works 53 | of the Licensed Work, are subject to this License. This License applies 54 | separately for each version of the Licensed Work and the Change Date may vary 55 | for each version of the Licensed Work released by Licensor. 56 | 57 | You must conspicuously display this License on each original or modified copy 58 | of the Licensed Work. If you receive the Licensed Work in original or 59 | modified form from a third party, the terms and conditions set forth in this 60 | License apply to your use of that work. 61 | 62 | Any use of the Licensed Work in violation of this License will automatically 63 | terminate your rights under this License for the current and all other 64 | versions of the Licensed Work. 65 | 66 | This License does not grant you any right in any trademark or logo of 67 | Licensor or its affiliates (provided that you may use a trademark or logo of 68 | Licensor as expressly required by this License). 69 | 70 | TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON 71 | AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, 72 | EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF 73 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND 74 | TITLE. 75 | 76 | MariaDB hereby grants you permission to use this License’s text to license 77 | your works, and to refer to it using the trademark “Business Source License”, 78 | as long as you comply with the Covenants of Licensor below. 79 | 80 | Covenants of Licensor 81 | 82 | In consideration of the right to use this License’s text and the “Business 83 | Source License” name and trademark, Licensor covenants to MariaDB, and to all 84 | other recipients of the licensed work to be provided by Licensor: 85 | 86 | 1. To specify as the Change License the GPL Version 2.0 or any later version, 87 | or a license that is compatible with GPL Version 2.0 or a later version, 88 | where “compatible” means that software provided under the Change License can 89 | be included in a program with software provided under GPL Version 2.0 or a 90 | later version. Licensor may specify additional Change Licenses without 91 | limitation. 92 | 93 | 2. To either: (a) specify an additional grant of rights to use that does not 94 | impose any additional restriction on the right granted in this License, as 95 | the Additional Use Grant; or (b) insert the text “None”. 96 | 97 | 3. To specify a Change Date. 98 | 99 | 4. Not to modify this License in any other way. -------------------------------------------------------------------------------- /now.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 2, 3 | "rewrites": [ 4 | { 5 | "source": "/inject", 6 | "destination": "/api/diff-service" 7 | }, 8 | { 9 | "source": "/pull", 10 | "destination": "/api/diff-service" 11 | }, 12 | { 13 | "source": "/signup", 14 | "destination": "/api/signup-service" 15 | } 16 | ], 17 | "env": { 18 | "REPLICANT_AWS_ACCESS_KEY_ID": "@aws_access_key_id", 19 | "REPLICANT_AWS_SECRET_ACCESS_KEY": "@aws_secret_access_key" 20 | } 21 | } -------------------------------------------------------------------------------- /serve/client_view.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | 10 | servetypes "roci.dev/diff-server/serve/types" 11 | ) 12 | 13 | type ClientViewGetter struct{} 14 | 15 | // Get fetches a client view. It returns an error if the response from the data layer doesn't have 16 | // a lastMutationID. 17 | func (g ClientViewGetter) Get(url string, req servetypes.ClientViewRequest, authToken string, syncID string) (servetypes.ClientViewResponse, int, error) { 18 | reqBody, err := json.Marshal(req) 19 | if err != nil { 20 | return servetypes.ClientViewResponse{}, 0, fmt.Errorf("could not marshal ClientViewRequest: %w", err) 21 | } 22 | httpReq, err := http.NewRequest("POST", url, bytes.NewReader(reqBody)) 23 | if err != nil { 24 | return servetypes.ClientViewResponse{}, 0, fmt.Errorf("could not create client view http request: %w", err) 25 | } 26 | httpReq.Header.Add("Content-type", "application/json") 27 | httpReq.Header.Add("Authorization", authToken) 28 | httpReq.Header.Add("X-Replicache-SyncID", syncID) 29 | httpResp, err := http.DefaultClient.Do(httpReq) 30 | if err != nil { 31 | return servetypes.ClientViewResponse{}, 0, fmt.Errorf("error sending client view http request: %w", err) 32 | } 33 | if httpResp.StatusCode != http.StatusOK { 34 | return servetypes.ClientViewResponse{}, httpResp.StatusCode, fmt.Errorf("client view fetch http request returned %s", httpResp.Status) 35 | } 36 | var resp servetypes.ClientViewResponse 37 | var r io.Reader = httpResp.Body 38 | defer httpResp.Body.Close() 39 | err = json.NewDecoder(r).Decode(&resp) 40 | if err != nil { 41 | return servetypes.ClientViewResponse{}, httpResp.StatusCode, fmt.Errorf("couldnt decode client view response: %w", err) 42 | } 43 | return resp, httpResp.StatusCode, nil 44 | } 45 | -------------------------------------------------------------------------------- /serve/client_view_test.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "net/http/httptest" 7 | "reflect" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | servetypes "roci.dev/diff-server/serve/types" 12 | ) 13 | 14 | func b(s string) []byte { 15 | return []byte(s) 16 | } 17 | func TestClientViewGetter_Get(t *testing.T) { 18 | assert := assert.New(t) 19 | 20 | type args struct { 21 | } 22 | tests := []struct { 23 | name string 24 | req servetypes.ClientViewRequest 25 | clientViewAuth string 26 | respCode int 27 | respBody string 28 | want servetypes.ClientViewResponse 29 | wantCode int 30 | wantErr string 31 | }{ 32 | { 33 | "ok", 34 | servetypes.ClientViewRequest{}, 35 | "authtoken", 36 | http.StatusOK, 37 | `{"clientView": {"key": "value"}, "lastMutationID": 2}`, 38 | servetypes.ClientViewResponse{ClientView: map[string]json.RawMessage{"key": b(`"value"`)}, LastMutationID: 2}, 39 | http.StatusOK, 40 | "", 41 | }, 42 | { 43 | "error", 44 | servetypes.ClientViewRequest{}, 45 | "authtoken", 46 | http.StatusBadRequest, 47 | ``, 48 | servetypes.ClientViewResponse{}, 49 | http.StatusBadRequest, 50 | "400", 51 | }, 52 | { 53 | "missing last mutation id", 54 | servetypes.ClientViewRequest{}, 55 | "authtoken", 56 | http.StatusOK, 57 | `{"clientView": {"foo": "bar"}}`, 58 | servetypes.ClientViewResponse{ClientView: map[string]json.RawMessage{"foo": b(`"bar"`)}, LastMutationID: 0}, 59 | http.StatusOK, 60 | "", 61 | }, 62 | } 63 | for _, tt := range tests { 64 | t.Run(tt.name, func(t *testing.T) { 65 | 66 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 67 | var reqBody servetypes.ClientViewRequest 68 | err := json.NewDecoder(r.Body).Decode(&reqBody) 69 | assert.NoError(err, tt.name) 70 | assert.Equal("application/json", r.Header.Get("Content-type"), tt.name) 71 | assert.Equal(tt.clientViewAuth, r.Header.Get("Authorization"), tt.name) 72 | assert.Equal("syncID", r.Header.Get("X-Replicache-SyncID"), tt.name) 73 | w.WriteHeader(tt.respCode) 74 | w.Write([]byte(tt.respBody)) 75 | })) 76 | 77 | g := ClientViewGetter{} 78 | got, gotCode, err := g.Get(server.URL, tt.req, tt.clientViewAuth, "syncID") 79 | assert.Equal(tt.wantCode, gotCode) 80 | if tt.wantErr == "" { 81 | assert.NoError(err) 82 | } else { 83 | assert.Error(err) 84 | assert.Regexp(tt.wantErr, err.Error(), tt.name) 85 | } 86 | if !reflect.DeepEqual(got, tt.want) { 87 | t.Errorf("ClientViewGetter.Get() case %s got %v (clientview=%v), want %v (clientview=%v)", tt.name, got, got.ClientView, tt.want, tt.want.ClientView) 88 | } 89 | }) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /serve/hello.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | 7 | "roci.dev/diff-server/util/version" 8 | ) 9 | 10 | // hello prints a hello message to let users know the server is running. 11 | func (s *Service) hello(w http.ResponseWriter, r *http.Request) { 12 | l := logger(r) 13 | if r.Method != "GET" { 14 | unsupportedMethodError(w, r.Method, l) 15 | return 16 | } 17 | w.Header().Add("Content-type", "text/plain") 18 | w.Write([]byte("Hello from Replicache\n")) 19 | w.Write([]byte(fmt.Sprintf("Version: %s\n", version.Version()))) 20 | } 21 | -------------------------------------------------------------------------------- /serve/hello_test.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httptest" 10 | "os" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | 15 | "roci.dev/diff-server/account" 16 | "roci.dev/diff-server/util/time" 17 | ) 18 | 19 | func TestHello(t *testing.T) { 20 | assert := assert.New(t) 21 | defer time.SetFake()() 22 | 23 | tc := []struct { 24 | method string 25 | wantRespCode int 26 | wantRespBody string 27 | }{ 28 | // Invalid method 29 | {"POST", http.StatusMethodNotAllowed, `Unsupported method: POST`}, 30 | 31 | // OK 32 | {"GET", http.StatusOK, `Hello from Replicache`}, 33 | } 34 | 35 | for i, t := range tc { 36 | td, _ := ioutil.TempDir("", "") 37 | defer func() { assert.NoError(os.RemoveAll(td)) }() 38 | 39 | adb, adir := account.LoadTempDB(assert) 40 | defer func() { assert.NoError(os.RemoveAll(adir)) }() 41 | 42 | s := NewService(td, account.MaxASClientViewHosts, adb, false, nil, true) 43 | 44 | msg := fmt.Sprintf("test case %d", i) 45 | req := httptest.NewRequest(t.method, "/hello", nil) 46 | resp := httptest.NewRecorder() 47 | s.hello(resp, req) 48 | 49 | body := bytes.Buffer{} 50 | _, err := io.Copy(&body, resp.Result().Body) 51 | assert.NoError(err, msg) 52 | assert.Equal(t.wantRespCode, resp.Result().StatusCode, msg) 53 | assert.Regexp(t.wantRespBody, string(body.Bytes())) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /serve/inject.go: -------------------------------------------------------------------------------- 1 | // Package serve implements the Replicant http server. This includes all the Noms endpoints, 2 | // plus a Replicant-specific sync endpoint that implements the server-side of the Replicant sync protocol. 3 | package serve 4 | 5 | import ( 6 | "encoding/json" 7 | "net/http" 8 | 9 | "github.com/pkg/errors" 10 | "roci.dev/diff-server/account" 11 | servetypes "roci.dev/diff-server/serve/types" 12 | ) 13 | 14 | // inject inserts a client view into the cache. This is primarily useful for testing without 15 | // having to have a data layer running. 16 | func (s *Service) inject(w http.ResponseWriter, r *http.Request) { 17 | l := logger(r) 18 | 19 | if !s.enableInject { 20 | w.WriteHeader(http.StatusNotFound) 21 | return 22 | } 23 | 24 | if r.Method != "POST" { 25 | unsupportedMethodError(w, r.Method, l) 26 | return 27 | } 28 | 29 | var req servetypes.InjectRequest 30 | err := json.NewDecoder(r.Body).Decode(&req) 31 | if err != nil { 32 | clientError(w, http.StatusBadRequest, errors.Wrap(err, "Bad request payload").Error(), l) 33 | return 34 | } 35 | 36 | if req.AccountID == "" { 37 | clientError(w, http.StatusBadRequest, "Missing accountID", l) 38 | return 39 | } 40 | 41 | // This check seems kind of useless given that much account info is public. 42 | records, err := account.ReadAllRecords(s.accountDB) 43 | if err != nil { 44 | serverError(w, err, l) 45 | } 46 | _, ok := account.Lookup(records, req.AccountID) 47 | if !ok { 48 | clientError(w, http.StatusBadRequest, "Unknown accountID", l) 49 | return 50 | } 51 | 52 | // TODO: auth 53 | 54 | if req.ClientID == "" { 55 | clientError(w, http.StatusBadRequest, "Missing clientID", l) 56 | return 57 | } 58 | 59 | db, err := s.GetDB(req.AccountID, req.ClientID) 60 | if err != nil { 61 | serverError(w, err, l) 62 | return 63 | } 64 | 65 | err = storeClientView(db, req.ClientViewResponse, l) 66 | if err != nil { 67 | serverError(w, err, l) 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /serve/inject_test.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httptest" 10 | "os" 11 | "strings" 12 | "testing" 13 | 14 | "github.com/attic-labs/noms/go/types" 15 | "github.com/stretchr/testify/assert" 16 | 17 | "roci.dev/diff-server/account" 18 | "roci.dev/diff-server/util/time" 19 | ) 20 | 21 | func TestInject(t *testing.T) { 22 | assert := assert.New(t) 23 | defer time.SetFake()() 24 | 25 | tc := []struct { 26 | injectEnabled bool 27 | method string 28 | req string 29 | wantRespCode int 30 | wantRespBody string 31 | wantChange bool 32 | }{ 33 | // Inject not enabled 34 | {false, "POST", fmt.Sprintf(`{"accountID": "%d", "clientID": "clientID", "clientViewResponse": {"clientView":{"foo": "bar"}, "lastTransactionID":"1"}}`, account.UnittestID), http.StatusNotFound, ``, false}, 35 | 36 | // Invalid method 37 | {true, "GET", ``, http.StatusMethodNotAllowed, `Unsupported method: GET`, false}, 38 | 39 | // Empty request 40 | {true, "POST", ``, http.StatusBadRequest, `Bad request payload: EOF`, false}, 41 | 42 | // Invalid JSON request 43 | {true, "POST", `!!`, http.StatusBadRequest, `Bad request payload: invalid character '!' looking for beginning of value`, false}, 44 | 45 | // Missing account ID 46 | {true, "POST", `{"clientID": "clientID", "clientViewResponse": {"clientView":{}, "lastTransactionID":"1"}}`, http.StatusBadRequest, `Missing accountID`, false}, 47 | 48 | // Unknown accountID 49 | {true, "POST", `{"accountID": "bonk", "clientID": "clientID", "clientViewResponse": {"clientView":{}, "lastTransactionID":"1"}}`, http.StatusBadRequest, `Unknown accountID`, false}, 50 | 51 | // OK 52 | {true, "POST", fmt.Sprintf(`{"accountID": "%d", "clientID": "clientID", "clientViewResponse": {"clientView":{"foo": "bar"}, "lastTransactionID":"1"}}`, account.UnittestID), http.StatusOK, ``, true}, 53 | } 54 | 55 | for i, t := range tc { 56 | td, _ := ioutil.TempDir("", "") 57 | defer func() { assert.NoError(os.RemoveAll(td)) }() 58 | 59 | adb, adir := account.LoadTempDB(assert) 60 | defer func() { assert.NoError(os.RemoveAll(adir)) }() 61 | account.AddUnittestAccount(assert, adb) 62 | 63 | s := NewService(td, account.MaxASClientViewHosts, adb, false, nil, t.injectEnabled) 64 | 65 | msg := fmt.Sprintf("test case %d", i) 66 | req := httptest.NewRequest(t.method, "/inject", strings.NewReader(t.req)) 67 | req.Header.Set("Content-type", "application/json") 68 | resp := httptest.NewRecorder() 69 | s.inject(resp, req) 70 | 71 | body := bytes.Buffer{} 72 | _, err := io.Copy(&body, resp.Result().Body) 73 | assert.NoError(err, msg) 74 | assert.Equal(t.wantRespCode, resp.Result().StatusCode, msg) 75 | assert.Equal(t.wantRespBody, string(body.Bytes()), msg) 76 | 77 | if t.wantChange { 78 | db, err := s.GetDB(fmt.Sprintf("%d", account.UnittestID), "clientID") 79 | assert.NoError(err, msg) 80 | m := db.Head().Data(db.Noms()) 81 | v, got := m.MaybeGet(types.String("foo")) 82 | assert.True(got, msg) 83 | assert.True(types.String("bar").Equals(v), msg) 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /serve/middleware.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "sync/atomic" 7 | 8 | zl "github.com/rs/zerolog" 9 | "roci.dev/diff-server/util/log" 10 | "roci.dev/diff-server/util/loghttp" 11 | ) 12 | 13 | // contectLogger is http middleware that inserts a contextual logger into 14 | // the http.Request's Context. 15 | func contextLogger(next http.Handler) http.Handler { 16 | return &contextLoggerHandler{next: next} 17 | } 18 | 19 | type contextLoggerHandler struct { 20 | next http.Handler 21 | reqID uint64 22 | } 23 | 24 | func (c *contextLoggerHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 25 | lc := log.Default().With().Str("req", r.URL.String()).Uint64("rid", atomic.AddUint64(&c.reqID, 1)) 26 | syncID := r.Header.Get("X-Replicache-SyncID") 27 | if syncID != "" { 28 | lc = lc.Str("syncID", syncID) 29 | } 30 | l := lc.Logger() 31 | ctx := context.WithValue(r.Context(), loggerKey{}, l) 32 | r = r.WithContext(ctx) 33 | c.next.ServeHTTP(w, r) 34 | } 35 | 36 | type loggerKey struct{} 37 | 38 | func logger(r *http.Request) zl.Logger { 39 | i := r.Context().Value(loggerKey{}) 40 | if i != nil { 41 | l, ok := i.(zl.Logger) 42 | if ok { 43 | return l 44 | } 45 | } 46 | l := log.Default() 47 | l.Error().Msgf("zlogger missing from request context for %s (this is expected in unit tests)", r.URL) 48 | return l 49 | } 50 | 51 | // panicCatcher is http middleware that recovers from panics, logs them, and 52 | // turns them into 500s. 53 | func panicCatcher(next http.Handler) http.Handler { 54 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 55 | l := logger(r) 56 | defer func() { 57 | err := recover() 58 | if err != nil { 59 | w.WriteHeader(http.StatusInternalServerError) 60 | l.Error().Msgf("Handler panicked: %#v", err) 61 | } 62 | }() 63 | next.ServeHTTP(w, r) 64 | }) 65 | } 66 | 67 | // logHTTP is http middleware that dumps HTTP requests and responses via loghttp. 68 | func logHTTP(next http.Handler) http.Handler { 69 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 70 | handler := loghttp.Wrap(next, logger(r)) 71 | handler.ServeHTTP(w, r) 72 | }) 73 | } 74 | -------------------------------------------------------------------------------- /serve/pull.go: -------------------------------------------------------------------------------- 1 | // Package serve implements the Replicant http server. This includes all the Noms endpoints, 2 | // plus a Replicant-specific sync endpoint that implements the server-side of the Replicant sync protocol. 3 | package serve 4 | 5 | import ( 6 | "bytes" 7 | "compress/gzip" 8 | "encoding/json" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "net/http" 13 | "strconv" 14 | "strings" 15 | 16 | "github.com/attic-labs/noms/go/hash" 17 | "github.com/attic-labs/noms/go/types" 18 | zl "github.com/rs/zerolog" 19 | 20 | "roci.dev/diff-server/account" 21 | "roci.dev/diff-server/db" 22 | "roci.dev/diff-server/kv" 23 | servetypes "roci.dev/diff-server/serve/types" 24 | nomsjson "roci.dev/diff-server/util/noms/json" 25 | ) 26 | 27 | func (s *Service) pull(rw http.ResponseWriter, r *http.Request) { 28 | l := logger(r) 29 | if r.Method != "OPTIONS" && r.Method != "POST" { 30 | unsupportedMethodError(rw, r.Method, l) 31 | return 32 | } 33 | rw.Header().Set("Access-Control-Allow-Origin", "*") 34 | rw.Header().Set("Access-Control-Allow-Methods", "*") 35 | rw.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-type, Referer, User-agent, X-Replicache-SyncID") 36 | if r.Method == "OPTIONS" { 37 | rw.WriteHeader(200) 38 | return 39 | } 40 | 41 | body := bytes.Buffer{} 42 | _, err := io.Copy(&body, r.Body) 43 | if err != nil { 44 | serverError(rw, fmt.Errorf("could not read body: %w", err), l) 45 | return 46 | } 47 | 48 | var preq servetypes.PullRequest 49 | err = json.Unmarshal(body.Bytes(), &preq) 50 | if err != nil { 51 | serverError(rw, fmt.Errorf("could not unmarshal body to json: %w", err), l) 52 | return 53 | } 54 | 55 | if preq.Version < 2 { 56 | clientError(rw, http.StatusBadRequest, "Unsupported PullRequest version", l) 57 | return 58 | } 59 | 60 | accountName := r.Header.Get("Authorization") 61 | if accountName == "" { 62 | clientError(rw, http.StatusBadRequest, "Missing Authorization header", l) 63 | return 64 | } 65 | accounts, err := account.ReadAllRecords(s.accountDB) 66 | if err != nil { 67 | serverError(rw, err, l) 68 | return 69 | } 70 | var acct account.Record 71 | if !s.disableAuth { 72 | var ok bool 73 | acct, ok = account.Lookup(accounts, accountName) 74 | if !ok { 75 | clientError(rw, http.StatusBadRequest, fmt.Sprintf("Unknown account: %s", accountName), l) 76 | return 77 | } 78 | } 79 | 80 | if preq.ClientID == "" { 81 | clientError(rw, http.StatusBadRequest, "Missing clientID", l) 82 | return 83 | } 84 | 85 | db, err := s.GetDB(accountName, preq.ClientID) 86 | if err != nil { 87 | serverError(rw, err, l) 88 | return 89 | } 90 | 91 | fromHash, ok := hash.MaybeParse(preq.BaseStateID) 92 | if preq.BaseStateID != "" && !ok { 93 | clientError(rw, http.StatusBadRequest, "Invalid baseStateID", l) 94 | return 95 | } 96 | fromChecksum, err := kv.ChecksumFromString(preq.Checksum) 97 | if err != nil { 98 | clientError(rw, http.StatusBadRequest, "Invalid checksum", l) 99 | return 100 | } 101 | 102 | clientViewURL := "" 103 | if preq.Version >= 3 { 104 | if preq.ClientViewURL == "" { 105 | clientError(rw, http.StatusBadRequest, "clientViewURL not provided in request", l) 106 | return 107 | } 108 | clientViewURL = preq.ClientViewURL 109 | 110 | var authorized bool 111 | if s.disableAuth { 112 | l.Info().Msg("Ignoring auth for this request (--disable-auth=true)") 113 | authorized = true 114 | } else { 115 | var err error 116 | authorized, err = account.ClientViewURLAuthorized(s.maxASClientViewURLs, s.accountDB, accounts, acct.ID, clientViewURL, l) 117 | if err != nil { 118 | serverError(rw, err, l) 119 | return 120 | } 121 | } 122 | if !authorized { 123 | clientError(rw, http.StatusForbidden, "clientViewURL is not authorized; please contact support@replicache.dev", l) 124 | return 125 | } 126 | } else { 127 | // TODO remove this block when Version 2 is deprecated. 128 | if len(acct.ClientViewURLs) > 0 { 129 | clientViewURL = acct.ClientViewURLs[0] 130 | } 131 | } 132 | 133 | cvReq := servetypes.ClientViewRequest{ 134 | ClientID: preq.ClientID, 135 | } 136 | syncID := r.Header.Get("X-Replicache-SyncID") 137 | head := db.Head() 138 | // minLastMutationID is the smallest last mutation id we will accept from the client view 139 | minLastMutationID := uint64(head.Value.LastMutationID) 140 | if preq.LastMutationID > minLastMutationID { 141 | minLastMutationID = preq.LastMutationID 142 | } 143 | cvInfo := maybeGetAndStoreNewClientView(db, preq.ClientViewAuth, clientViewURL, s.clientViewGetter, cvReq, minLastMutationID, syncID, l) 144 | 145 | head = db.Head() // head could have changed in maybeGetAndStoreNewClientView 146 | var presp servetypes.PullResponse 147 | if uint64(head.Value.LastMutationID) < preq.LastMutationID { 148 | // Refuse to send the client backwards in time. 149 | presp = nopPull(&preq, &cvInfo) 150 | } else { 151 | patch, err := db.Diff(preq.Version, fromHash, *fromChecksum, head, l) 152 | if err != nil { 153 | serverError(rw, err, l) 154 | return 155 | } 156 | presp = servetypes.PullResponse{ 157 | StateID: head.NomsStruct.Hash().String(), 158 | LastMutationID: uint64(head.Value.LastMutationID), 159 | Patch: patch, 160 | Checksum: string(head.Value.Checksum), 161 | ClientViewInfo: cvInfo, 162 | } 163 | } 164 | resp, err := json.Marshal(presp) 165 | if err != nil { 166 | serverError(rw, err, l) 167 | return 168 | } 169 | // Add a newline to make output to console etc nicer. 170 | resp = append(resp, byte('\n')) 171 | rw.Header().Set("Content-type", "application/json") 172 | rw.Header().Set("Entity-length", strconv.Itoa(len(resp))) 173 | 174 | w := io.Writer(rw) 175 | if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { 176 | rw.Header().Set("Content-encoding", "gzip") 177 | gzw := gzip.NewWriter(rw) 178 | defer gzw.Close() 179 | w = gzw 180 | } 181 | _, err = io.Copy(w, bytes.NewReader(resp)) 182 | if err != nil { 183 | serverError(rw, err, l) 184 | return 185 | } 186 | } 187 | 188 | func nopPull(pullReq *servetypes.PullRequest, cvInfo *servetypes.ClientViewInfo) servetypes.PullResponse { 189 | return servetypes.PullResponse{ 190 | StateID: pullReq.BaseStateID, 191 | LastMutationID: pullReq.LastMutationID, 192 | Patch: make([]kv.Operation, 0), 193 | Checksum: pullReq.Checksum, 194 | ClientViewInfo: *cvInfo, 195 | } 196 | } 197 | 198 | func maybeGetAndStoreNewClientView(db *db.DB, clientViewAuth string, url string, cvg clientViewGetter, cvReq servetypes.ClientViewRequest, minLastMutationID uint64, syncID string, l zl.Logger) servetypes.ClientViewInfo { 199 | clientViewInfo := servetypes.ClientViewInfo{} 200 | var err error 201 | defer func() { 202 | if err != nil { 203 | l.Info().Msgf("got error fetching clientview: %s", err) 204 | clientViewInfo.ErrorMessage = err.Error() 205 | } 206 | }() 207 | 208 | if url == "" { 209 | err = errors.New("not fetching new client view: no url provided via account or --client-view") 210 | return clientViewInfo 211 | } 212 | cvResp, cvCode, err := cvg.Get(url, cvReq, clientViewAuth, syncID) 213 | clientViewInfo.HTTPStatusCode = cvCode 214 | if err != nil { 215 | return clientViewInfo 216 | } 217 | 218 | // Refuse to go backwards in time. minLastMutationID is the greater of 219 | // the last mutation id of the client and head, the minimum lmid we will 220 | // accept from the client view. 221 | if cvResp.LastMutationID >= minLastMutationID { 222 | err = storeClientView(db, cvResp, l) 223 | } 224 | return clientViewInfo 225 | } 226 | 227 | func storeClientView(db *db.DB, cvResp servetypes.ClientViewResponse, l zl.Logger) error { 228 | me := kv.NewMap(db.Noms()).Edit() 229 | for k, JSON := range cvResp.ClientView { 230 | v, err := nomsjson.FromJSON(JSON, db.Noms()) 231 | if err != nil { 232 | return fmt.Errorf("error parsing clientview: %w", err) 233 | } 234 | if err := me.Set(types.String(k), v); err != nil { 235 | return fmt.Errorf("error setting value '%s' in clientview: %w", JSON, err) 236 | } 237 | } 238 | m := me.Build() 239 | c, err := db.MaybePutData(m, cvResp.LastMutationID) 240 | if err != nil { 241 | return fmt.Errorf("error writing new commit: %w", err) 242 | } 243 | if c.NomsStruct.IsZeroValue() { 244 | l.Debug().Msgf("Did not write a new commit (lastMutationID %d and checksum %s are identical to head); nop", cvResp.LastMutationID, m.Checksum()) 245 | } else { 246 | basis, err := c.Basis(db.Noms()) 247 | if err != nil { 248 | return err 249 | } 250 | l.Debug().Msgf("Wrote new commit %s with lastMutationID %d and checksum %s (previous commit %s had lastMutationID %d and checksum %s)", c.Ref().TargetHash(), cvResp.LastMutationID, m.Checksum(), basis.Ref().TargetHash(), uint64(basis.Value.LastMutationID), basis.Value.Checksum) 251 | } 252 | return nil 253 | } 254 | -------------------------------------------------------------------------------- /serve/service.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "regexp" 8 | "strings" 9 | "sync" 10 | 11 | "github.com/attic-labs/noms/go/datas" 12 | "github.com/attic-labs/noms/go/spec" 13 | "github.com/gorilla/mux" 14 | "github.com/justinas/alice" 15 | 16 | "roci.dev/diff-server/account" 17 | "roci.dev/diff-server/db" 18 | servetypes "roci.dev/diff-server/serve/types" 19 | 20 | zl "github.com/rs/zerolog" 21 | 22 | // Log all HTTP requests 23 | 24 | _ "roci.dev/diff-server/util/loghttp" 25 | ) 26 | 27 | var ( 28 | // /// 29 | pathRegex = regexp.MustCompile(`^\/([\w-]+)\/([\w-]+)\/([\w-]+)\/?$`) 30 | ) 31 | 32 | // Service is an instance of the Replicache Diffserver services. 33 | type Service struct { 34 | storageRoot string 35 | urlPrefix string 36 | maxASClientViewURLs int 37 | accountDB *account.DB 38 | nomsen map[string]datas.Database 39 | disableAuth bool 40 | enableInject bool 41 | mu sync.Mutex 42 | 43 | // cvg may be nil, in which case the server skips the client view request in pull, which is 44 | // useful if you are populating the db directly or in tests. 45 | clientViewGetter clientViewGetter 46 | } 47 | 48 | type clientViewGetter interface { 49 | Get(url string, req servetypes.ClientViewRequest, authToken string, syncID string) (servetypes.ClientViewResponse, int, error) 50 | } 51 | 52 | // NewService creates a new instances of the Replicant web service. 53 | func NewService(storageRoot string, maxASClientViewURLs int, accountDB *account.DB, disableAuth bool, cvg clientViewGetter, enableInject bool) *Service { 54 | return &Service{ 55 | storageRoot: storageRoot, 56 | maxASClientViewURLs: maxASClientViewURLs, 57 | accountDB: accountDB, 58 | nomsen: map[string]datas.Database{}, 59 | disableAuth: disableAuth, 60 | enableInject: enableInject, 61 | mu: sync.Mutex{}, 62 | clientViewGetter: cvg, 63 | } 64 | } 65 | 66 | // RegisterHandlers register's Service's handlers on the given router. 67 | func RegisterHandlers(s *Service, router *mux.Router) { 68 | router.SkipClean(true) 69 | router.HandleFunc("/", s.hello) 70 | inject := alice.New(panicCatcher).ThenFunc(s.inject) 71 | router.Handle("/inject", inject) 72 | pull := alice.New(contextLogger, panicCatcher, logHTTP).ThenFunc(s.pull) 73 | router.Handle("/pull", pull) 74 | } 75 | 76 | func (s *Service) GetDB(accountID, clientID string) (*db.DB, error) { 77 | noms, err := s.getNoms(accountID) 78 | if err != nil { 79 | return nil, err 80 | } 81 | dsName := fmt.Sprintf("client/%s", clientID) 82 | db, err := db.New(noms.GetDataset(dsName)) 83 | if err != nil { 84 | return nil, err 85 | } 86 | return db, nil 87 | } 88 | 89 | func (s *Service) getNoms(accountID string) (datas.Database, error) { 90 | s.mu.Lock() 91 | defer s.mu.Unlock() 92 | 93 | n := s.nomsen[accountID] 94 | if n == nil { 95 | sp, err := spec.ForDatabase(fmt.Sprintf("%s/%s", s.storageRoot, accountID)) 96 | if err != nil { 97 | return nil, err 98 | } 99 | n = sp.GetDatabase() 100 | s.nomsen[accountID] = n 101 | } else { 102 | n.Rebase() 103 | } 104 | return n, nil 105 | } 106 | 107 | func unsupportedMethodError(w http.ResponseWriter, m string, l zl.Logger) { 108 | clientError(w, http.StatusMethodNotAllowed, fmt.Sprintf("Unsupported method: %s", m), l) 109 | } 110 | 111 | func clientError(w http.ResponseWriter, code int, body string, l zl.Logger) { 112 | w.WriteHeader(code) 113 | l.Info().Int("status", code).Msg(body) 114 | io.Copy(w, strings.NewReader(body)) 115 | } 116 | 117 | func serverError(w http.ResponseWriter, err error, l zl.Logger) { 118 | w.WriteHeader(http.StatusInternalServerError) 119 | l.Info().Int("status", http.StatusInternalServerError).Err(err).Send() 120 | } 121 | -------------------------------------------------------------------------------- /serve/service_test.go: -------------------------------------------------------------------------------- 1 | package serve 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/httptest" 8 | "os" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/gorilla/mux" 13 | "github.com/stretchr/testify/assert" 14 | "roci.dev/diff-server/account" 15 | "roci.dev/diff-server/serve/types" 16 | ) 17 | 18 | func TestConcurrentAccessUsingMultipleServices(t *testing.T) { 19 | assert := assert.New(t) 20 | td, _ := ioutil.TempDir("", "") 21 | defer func() { assert.NoError(os.RemoveAll(td)) }() 22 | 23 | adb, adir := account.LoadTempDB(assert) 24 | defer func() { assert.NoError(os.RemoveAll(adir)) }() 25 | 26 | fcvg := &fakeClientViewGet{resp: types.ClientViewResponse{}, code: 200, err: nil} 27 | svc1 := NewService(td, account.MaxASClientViewHosts, adb, false, fcvg, true) 28 | svc2 := NewService(td, account.MaxASClientViewHosts, adb, false, fcvg, true) 29 | 30 | res := []*httptest.ResponseRecorder{ 31 | httptest.NewRecorder(), 32 | httptest.NewRecorder(), 33 | httptest.NewRecorder(), 34 | } 35 | 36 | reqBody := `{"baseStateID": "00000000000000000000000000000000", "checksum": "00000000", "clientID": "clientid", "clientViewURL": "http://localhost:8000/client-view", "version": 3}` 37 | 38 | req1 := httptest.NewRequest("POST", "/pull", strings.NewReader(reqBody)) 39 | req1.Header.Add("Authorization", "sandbox") 40 | req2 := httptest.NewRequest("POST", "/pull", strings.NewReader(reqBody)) 41 | req2.Header.Add("Authorization", "sandbox") 42 | req3 := httptest.NewRequest("POST", "/pull", strings.NewReader(reqBody)) 43 | req3.Header.Add("Authorization", "sandbox") 44 | mux1 := mux.NewRouter() 45 | RegisterHandlers(svc1, mux1) 46 | mux1.ServeHTTP(res[0], req1) 47 | mux2 := mux.NewRouter() 48 | RegisterHandlers(svc2, mux2) 49 | mux2.ServeHTTP(res[1], req2) 50 | mux1.ServeHTTP(res[2], req3) 51 | 52 | for i, r := range res { 53 | assert.Equal(http.StatusOK, r.Code, fmt.Sprintf("response %d: %s", i, string(r.Body.Bytes()))) 54 | } 55 | } 56 | 57 | func TestNo301(t *testing.T) { 58 | assert := assert.New(t) 59 | td, _ := ioutil.TempDir("", "") 60 | defer func() { assert.NoError(os.RemoveAll(td)) }() 61 | 62 | adb, adir := account.LoadTempDB(assert) 63 | defer func() { assert.NoError(os.RemoveAll(adir)) }() 64 | 65 | svc := NewService(td, account.MaxASClientViewHosts, adb, false, nil, true) 66 | r := httptest.NewRecorder() 67 | 68 | mux := mux.NewRouter() 69 | RegisterHandlers(svc, mux) 70 | mux.ServeHTTP(r, httptest.NewRequest("POST", "//pull", strings.NewReader(`{"accountID": "sandbox", "baseStateID": "00000000000000000000000000000000", "checksum": "00000000", "clientID": "clientid"}`))) 71 | assert.Equal(http.StatusNotFound, r.Code) 72 | assert.Equal("404 page not found\n", string(r.Body.Bytes())) 73 | } 74 | -------------------------------------------------------------------------------- /serve/signup/get.go: -------------------------------------------------------------------------------- 1 | package signup 2 | 3 | const GetTemplateName = "get" 4 | 5 | // GetTemplate is the HTML template for the page a customer uses to sign up. 6 | // The template is included statically because I cannot figure out how 7 | // to get Vercel to include template files so they are accessible at function 8 | // startup time. (I tried both "functions" with includeFiles and static 9 | // build rules.) 10 | const GetTemplate = ` 11 | 12 | 13 | 14 | 15 | Replicache Account Signup 16 | 17 | 18 | 19 |

Replicache Account Signup

20 | 21 |

Please fill out the form below to generate an Account ID for Replicache.
22 | You need to include your Account ID in the diffServerAuth field when
23 | you instantiate Replicache 24 | in your JavaScript application. 25 | 26 |

Note that your account will be suitable for evaluation purposes, but
27 | due to licensing and default account limitations, it will not be suitable for
28 | deployment to end users in production "for real." In order to deploy Replicache
29 | to end users in production for non-evaluation purposes, please email us
30 | at support@replicache.dev to upgrade your account.

31 | 32 |

33 |
34 |
35 | 36 |
37 | 38 | 39 | 40 | 41 | ` 42 | -------------------------------------------------------------------------------- /serve/signup/post_failure.go: -------------------------------------------------------------------------------- 1 | package signup 2 | 3 | const PostFailureTemplateName = "post_failure" 4 | 5 | // PostFailureTemplate is the HTML template rendered in response to an 6 | // unsuccessful customer signup. 7 | const PostFailureTemplate = ` 8 | 9 | 10 | 11 | Replicache Account Signup: Oops! 12 | 13 | 14 | 15 |

Could not create account :(

16 | 17 |

Your form submission had the following problem(s): 18 |

    19 | {{range .Reasons}} 20 |
  • {{.}}
    • 21 | {{end}} 22 |
23 | 24 |

Please hit "back", correct the problem(s), and submit again. If you feel that you've reached this
25 | page in error, our apologies, please email support@replicache.dev.
26 | Thanks! 27 | 28 | 29 | 30 | ` 31 | -------------------------------------------------------------------------------- /serve/signup/post_success.go: -------------------------------------------------------------------------------- 1 | package signup 2 | 3 | const PostSuccessTemplateName = "post_success" 4 | 5 | // PostSuccessTemplate is the HTML template rendered in response to a 6 | // successful customer signup. 7 | const PostSuccessTemplate = ` 8 | 9 | 10 | 11 | Replicache Account Signup: Success! 12 | 13 | 14 | 15 |

Success!

16 | 17 |

Your Account ID is {{ .ID }}

18 | 19 | Please note: 20 |
    21 |
  • Your account is suitable for evaluation purposes. To deploy to
    22 | end users in production for non-evaluation purposes, please contact us at support@replicache.dev.
    23 | (We just need to lift some default limits for you and ensure you agree to our BSL license.)

    24 | 25 |
  • You need to include your Account ID in the diffServerAuth field when
    26 | you instantiate Replicache 27 | in your JavaScript application. 28 | 29 |
30 | 31 |

Potential next steps: Replicache README, 32 | Replicache JS Quick start, or 33 | contact us at support@replicache.dev. 34 | 35 | 36 | 37 | ` 38 | -------------------------------------------------------------------------------- /serve/signup/service.go: -------------------------------------------------------------------------------- 1 | package signup 2 | 3 | import ( 4 | "fmt" 5 | "html/template" 6 | "io" 7 | "net/http" 8 | "strings" 9 | "time" 10 | 11 | "github.com/gorilla/mux" 12 | zl "github.com/rs/zerolog" 13 | "roci.dev/diff-server/account" 14 | ) 15 | 16 | // Templates returns the list of Templates the signup service needs. 17 | // Add new templates to this list! 18 | func Templates() []Template { 19 | return []Template{ 20 | {Name: GetTemplateName, Content: GetTemplate}, 21 | {Name: PostFailureTemplateName, Content: PostFailureTemplate}, 22 | {Name: PostSuccessTemplateName, Content: PostSuccessTemplate}, 23 | } 24 | } 25 | 26 | // Template contains a string with the template content. Normally we'd 27 | // have the content in a file but I can't figure out how to access files 28 | // at runtime with Vercel. 29 | type Template struct { 30 | Name string 31 | Content string 32 | } 33 | 34 | func ParseTemplates(templates []Template) (t *template.Template, err error) { 35 | t = template.New("") 36 | 37 | for _, tmpl := range templates { 38 | if _, err = t.New(tmpl.Name).Parse(tmpl.Content); err != nil { 39 | return 40 | } 41 | } 42 | 43 | return 44 | } 45 | 46 | // Service is an instance of the signup service. It returns a little form 47 | // to fill out with account information, accepts a POST from the form, and creates 48 | // the account in an account.DB. 49 | type Service struct { 50 | logger zl.Logger 51 | tmpl *template.Template 52 | storageRoot string 53 | } 54 | 55 | // NewService instantiates the signup service. Handlers need to be registered with 56 | // RegisterHandlers. 57 | // TODO NewService should probably take an account.DB instead of its storageroot 58 | func NewService(logger zl.Logger, tmpl *template.Template, storageRoot string) *Service { 59 | return &Service{logger, tmpl, storageRoot} 60 | } 61 | 62 | // Path is the URL path at which to serve. It is used when running locally. 63 | // When running on Vercel we serve from under /api/, but there is a rewrite 64 | // rule in now.json that maps /signup to the service api path. 65 | const Path = "/signup" 66 | 67 | // RegisterHandlers registers Service's handlers on the given router. 68 | func RegisterHandlers(s *Service, router *mux.Router) { 69 | router.HandleFunc(Path, s.handle) 70 | } 71 | 72 | func (s *Service) handle(w http.ResponseWriter, r *http.Request) { 73 | if r.Method == "GET" { 74 | if err := s.tmpl.ExecuteTemplate(w, GetTemplateName, getTemplateArgs{GetTemplateNameField, GetTemplateEmailField}); err != nil { 75 | serverError(w, err, s.logger) 76 | } 77 | return 78 | 79 | } else if r.Method == "POST" { 80 | name := r.FormValue(GetTemplateNameField) 81 | email := r.FormValue(GetTemplateEmailField) 82 | 83 | // The lightest of all possible lightweight form validations. 84 | validationFailures := []string{} 85 | if name == "" { 86 | validationFailures = append(validationFailures, "Please enter a Name (either your personal name or an entity, eg your company).") 87 | } 88 | if strings.Index(email, "@") == -1 { 89 | validationFailures = append(validationFailures, "Please enter a valid Email Address so we can contact you in the event of problems.") 90 | } 91 | if len(validationFailures) > 0 { 92 | templateArgs := postFailureTemplateArgs{Reasons: validationFailures} 93 | if err := s.tmpl.ExecuteTemplate(w, PostFailureTemplateName, templateArgs); err != nil { 94 | serverError(w, err, s.logger) 95 | } 96 | return 97 | } 98 | 99 | db, err := account.NewDB(s.storageRoot) 100 | if err != nil { 101 | serverError(w, err, s.logger) 102 | return 103 | } 104 | accounts, err := account.ReadAllRecords(db) 105 | if err != nil { 106 | serverError(w, err, s.logger) 107 | return 108 | } 109 | id := accounts.NextASID 110 | accounts.Record[id] = account.Record{ 111 | ID: id, 112 | Name: name, 113 | Email: email, 114 | DateCreated: time.Now().String(), 115 | } 116 | accounts.NextASID++ 117 | if err := account.WriteRecords(db, accounts); err != nil { 118 | // TODO: retry if head was changed from under us. 119 | serverError(w, err, s.logger) 120 | return 121 | } 122 | templateArgs := postSuccessTemplateArgs{ID: fmt.Sprintf("%d", id)} 123 | if err := s.tmpl.ExecuteTemplate(w, PostSuccessTemplateName, templateArgs); err != nil { 124 | serverError(w, err, s.logger) 125 | } 126 | s.logger.Info().Msgf("Created auto-signup account: %#v", accounts.Record[id]) 127 | return 128 | 129 | } else { 130 | unsupportedMethodError(w, r.Method, s.logger) 131 | } 132 | } 133 | 134 | const GetTemplateNameField = "name" 135 | const GetTemplateEmailField = "email" 136 | 137 | // getTemplateArgs holds the names of the form fields to use in the form. 138 | // They're extracted into the constants above so they are easy to change if need be. 139 | type getTemplateArgs struct { 140 | Name string 141 | Email string 142 | } 143 | 144 | type postSuccessTemplateArgs struct { 145 | ID string // The newly created account id. 146 | } 147 | 148 | type postFailureTemplateArgs struct { 149 | Reasons []string 150 | } 151 | 152 | func unsupportedMethodError(w http.ResponseWriter, m string, l zl.Logger) { 153 | clientError(w, http.StatusMethodNotAllowed, fmt.Sprintf("Unsupported method: %s", m), l) 154 | } 155 | 156 | func clientError(w http.ResponseWriter, code int, body string, l zl.Logger) { 157 | w.WriteHeader(code) 158 | l.Info().Int("status", code).Msg(body) 159 | io.Copy(w, strings.NewReader(body)) 160 | } 161 | 162 | func serverError(w http.ResponseWriter, err error, l zl.Logger) { 163 | w.WriteHeader(http.StatusInternalServerError) 164 | l.Error().Int("status", http.StatusInternalServerError).Err(err).Send() 165 | } 166 | -------------------------------------------------------------------------------- /serve/signup/service_test.go: -------------------------------------------------------------------------------- 1 | package signup_test 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "html/template" 7 | "io/ioutil" 8 | "net/http/httptest" 9 | "net/url" 10 | "os" 11 | "strings" 12 | "testing" 13 | 14 | "github.com/gorilla/mux" 15 | "github.com/stretchr/testify/assert" 16 | "roci.dev/diff-server/account" 17 | "roci.dev/diff-server/serve/signup" 18 | "roci.dev/diff-server/util/log" 19 | ) 20 | 21 | func TestGET(t *testing.T) { 22 | assert := assert.New(t) 23 | dir, err := ioutil.TempDir("", "") 24 | assert.NoError(err) 25 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 26 | 27 | tmpl := template.Must(signup.ParseTemplates(signup.Templates())) 28 | service := signup.NewService(log.Default(), tmpl, dir) 29 | m := mux.NewRouter() 30 | signup.RegisterHandlers(service, m) 31 | 32 | getForm := httptest.NewRequest("GET", signup.Path, nil) 33 | getFormRecorder := httptest.NewRecorder() 34 | m.ServeHTTP(getFormRecorder, getForm) 35 | resp := getFormRecorder.Result() 36 | bodyBytes, err := ioutil.ReadAll(resp.Body) 37 | assert.NoError(err) 38 | assert.Equal(200, resp.StatusCode) 39 | body := string(bodyBytes) 40 | assert.True(strings.Contains(body, fmt.Sprintf(`name="%s"`, signup.GetTemplateNameField))) 41 | assert.True(strings.Contains(body, fmt.Sprintf(`name="%s"`, signup.GetTemplateEmailField))) 42 | assert.True(strings.Contains(body, `type="submit"`)) 43 | } 44 | 45 | func TestPOSTSuccess(t *testing.T) { 46 | assert := assert.New(t) 47 | dir, err := ioutil.TempDir("", "") 48 | assert.NoError(err) 49 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 50 | 51 | tmpl := template.Must(signup.ParseTemplates(signup.Templates())) 52 | service := signup.NewService(log.Default(), tmpl, dir) 53 | m := mux.NewRouter() 54 | signup.RegisterHandlers(service, m) 55 | db, err := account.NewDB(dir) 56 | assert.NoError(err) 57 | expectedASID := db.HeadValue().NextASID 58 | 59 | postData := url.Values{} 60 | postData.Set(signup.GetTemplateNameField, "Larry") 61 | postData.Set(signup.GetTemplateEmailField, "larry@example.com") 62 | postForm := httptest.NewRequest("POST", signup.Path, bytes.NewBufferString(postData.Encode())) 63 | postForm.Header.Set("Content-Type", "application/x-www-form-urlencoded") 64 | postFormRecorder := httptest.NewRecorder() 65 | m.ServeHTTP(postFormRecorder, postForm) 66 | resp := postFormRecorder.Result() 67 | bodyBytes, err := ioutil.ReadAll(resp.Body) 68 | assert.NoError(err) 69 | 70 | // Ensure repsonse is what we expect. 71 | assert.Equal(200, resp.StatusCode) 72 | body := string(bodyBytes) 73 | assert.True(strings.Contains(body, fmt.Sprintf("ID is %d", expectedASID))) 74 | 75 | // Ensure the account db was updated. 76 | assert.NoError(db.Reload()) 77 | hv := db.HeadValue() 78 | assert.Equal(expectedASID+1, hv.NextASID) 79 | assert.Equal("Larry", hv.Record[expectedASID].Name) 80 | assert.Equal("larry@example.com", hv.Record[expectedASID].Email) 81 | assert.NotEqual("", hv.Record[expectedASID].DateCreated) 82 | } 83 | 84 | func TestPOSTFailure(t *testing.T) { 85 | assert := assert.New(t) 86 | dir, err := ioutil.TempDir("", "") 87 | assert.NoError(err) 88 | defer func() { assert.NoError(os.RemoveAll(dir)) }() 89 | 90 | tmpl := template.Must(signup.ParseTemplates(signup.Templates())) 91 | service := signup.NewService(log.Default(), tmpl, dir) 92 | m := mux.NewRouter() 93 | signup.RegisterHandlers(service, m) 94 | db, err := account.NewDB(dir) 95 | assert.NoError(err) 96 | expectedNextASID := db.HeadValue().NextASID 97 | 98 | postData := url.Values{} 99 | postData.Set(signup.GetTemplateNameField, "") // Empty name 100 | postData.Set(signup.GetTemplateEmailField, "larry") // No @ 101 | postForm := httptest.NewRequest("POST", signup.Path, bytes.NewBufferString(postData.Encode())) 102 | postForm.Header.Set("Content-Type", "application/x-www-form-urlencoded") 103 | postFormRecorder := httptest.NewRecorder() 104 | m.ServeHTTP(postFormRecorder, postForm) 105 | resp := postFormRecorder.Result() 106 | bodyBytes, err := ioutil.ReadAll(resp.Body) 107 | assert.NoError(err) 108 | 109 | // Ensure repsonse is what we expect. 110 | assert.Equal(200, resp.StatusCode) 111 | body := string(bodyBytes) 112 | assert.True(strings.Contains(body, "enter a Name")) 113 | assert.True(strings.Contains(body, "enter a valid Email Address")) 114 | assert.False(strings.Contains(body, "Success")) 115 | 116 | // Ensure the account db was updated. 117 | assert.NoError(db.Reload()) 118 | hv := db.HeadValue() 119 | assert.Equal(expectedNextASID, hv.NextASID) 120 | } 121 | -------------------------------------------------------------------------------- /serve/types/types.go: -------------------------------------------------------------------------------- 1 | package types 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "roci.dev/diff-server/kv" 7 | ) 8 | 9 | type PullRequest struct { 10 | // Version 0 -> uses raw json kv.Operation.Value 11 | // Version 1 -> uses stringified json kv.Operation.ValueString 12 | // Version 2 -> top-level remove uses replace path="" value="{}" instead of remove path="/" 13 | // Version 3 -> request explicitly specifies client view URL 14 | Version uint32 `json:"version"` 15 | ClientViewURL string `json:"clientViewURL"` 16 | ClientViewAuth string `json:"clientViewAuth"` 17 | ClientID string `json:"clientID"` 18 | BaseStateID string `json:"baseStateID"` 19 | Checksum string `json:"checksum"` 20 | LastMutationID uint64 `json:"lastMutationID"` 21 | } 22 | 23 | type PullResponse struct { 24 | StateID string `json:"stateID"` 25 | LastMutationID uint64 `json:"lastMutationID"` 26 | Patch []kv.Operation `json:"patch"` 27 | Checksum string `json:"checksum"` 28 | ClientViewInfo ClientViewInfo `json:"clientViewInfo"` 29 | } 30 | 31 | type ClientViewInfo struct { 32 | HTTPStatusCode int `json:"httpStatusCode"` 33 | ErrorMessage string `json:"errorMessage"` 34 | } 35 | 36 | type ClientViewRequest struct { 37 | ClientID string `json:"clientID"` 38 | } 39 | 40 | type ClientViewResponse struct { 41 | ClientView map[string]json.RawMessage `json:"clientView"` 42 | LastMutationID uint64 `json:"lastMutationID"` 43 | } 44 | 45 | type InjectRequest struct { 46 | AccountID string `json:"accountID"` 47 | ClientID string `json:"clientID"` 48 | ClientViewResponse ClientViewResponse `json:"clientViewResponse"` 49 | } 50 | -------------------------------------------------------------------------------- /tool/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" 4 | ROOT=$DIR/../ 5 | set -x 6 | 7 | HEAD_HASH=`git rev-parse HEAD | cut -c1-6` 8 | 9 | BUILDDIR=build 10 | rm -rf build 11 | mkdir build 12 | 13 | cd $ROOT 14 | 15 | # diffs 16 | echo "Building diffs..." 17 | 18 | GOOS=darwin GOARCH=amd64 go build -ldflags "-X roci.dev/diff-server/util/version.h=$HEAD_HASH" -o build/diffs-osx ./cmd/diffs 19 | GOOS=linux GOARCH=amd64 go build -ldflags "-X roci.dev/diff-server/util/version.h=$HEAD_HASH" -o build/diffs-linux ./cmd/diffs 20 | 21 | # noms tool 22 | echo "Building noms..." 23 | NOMS_VERSION=`go mod graph | grep '^github.com/attic-labs/noms@' | cut -d' ' -f1 | head -n1` 24 | go get $NOMS_VERSION 25 | GOOS=darwin GOARCH=amd64 go build -o build/noms-osx github.com/attic-labs/noms/cmd/noms 26 | GOOS=linux GOARCH=amd64 go build -o build/noms-linux github.com/attic-labs/noms/cmd/noms 27 | -------------------------------------------------------------------------------- /util/chk/chk.go: -------------------------------------------------------------------------------- 1 | // Package chk implements a handful of runtime assertions. 2 | package chk 3 | 4 | import "fmt" 5 | 6 | func Fail(msg string, params ...interface{}) { 7 | panic(fmt.Sprintf(msg, params...)) 8 | } 9 | 10 | func True(cond bool, msg string, params ...interface{}) { 11 | if !cond { 12 | Fail(msg, params...) 13 | } 14 | } 15 | 16 | func False(cond bool, msg string, params ...interface{}) { 17 | True(!cond, msg, params...) 18 | } 19 | 20 | func Equal(expected interface{}, actual interface{}) { 21 | if expected != actual { 22 | Fail("Expected %#v, got: %#v", expected, actual) 23 | } 24 | } 25 | 26 | func NotNil(v interface{}, msgAndParams ...interface{}) { 27 | if v == nil { 28 | var msg string 29 | if len(msgAndParams) > 0 { 30 | msg = fmt.Sprintf(msgAndParams[0].(string), msgAndParams[1:]...) 31 | } else { 32 | msg = "Expected non-nil value" 33 | } 34 | Fail(msg) 35 | } 36 | } 37 | 38 | func NoError(err error) { 39 | if err != nil { 40 | Fail("Unexpected error: %#v", err) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /util/countingreader/reader.go: -------------------------------------------------------------------------------- 1 | package countingreader 2 | 3 | import "io" 4 | 5 | type Callback func() 6 | 7 | type Reader struct { 8 | R io.Reader 9 | Callback Callback 10 | Count uint64 11 | } 12 | 13 | func (r *Reader) Read(p []byte) (n int, err error) { 14 | n, err = r.R.Read(p) 15 | r.Count += uint64(n) 16 | r.Callback() 17 | return n, err 18 | } 19 | -------------------------------------------------------------------------------- /util/gid/gid.go: -------------------------------------------------------------------------------- 1 | package gid 2 | 3 | import ( 4 | "bytes" 5 | "runtime" 6 | "strconv" 7 | 8 | "github.com/rs/zerolog" 9 | ) 10 | 11 | func Get() uint64 { 12 | b := make([]byte, 64) 13 | b = b[:runtime.Stack(b, false)] 14 | b = bytes.TrimPrefix(b, []byte("goroutine ")) 15 | b = b[:bytes.IndexByte(b, ' ')] 16 | n, _ := strconv.ParseUint(string(b), 10, 64) 17 | return n 18 | } 19 | 20 | type ZLogHook struct{} 21 | 22 | func (h ZLogHook) Run(e *zerolog.Event, level zerolog.Level, msg string) { 23 | e.Uint64("gr", Get()) 24 | } 25 | -------------------------------------------------------------------------------- /util/kp/db.go: -------------------------------------------------------------------------------- 1 | package kp 2 | 3 | import ( 4 | "github.com/attic-labs/noms/go/datas" 5 | "github.com/attic-labs/noms/go/spec" 6 | kingpin "gopkg.in/alecthomas/kingpin.v2" 7 | ) 8 | 9 | type DatabaseValue struct { 10 | db *datas.Database 11 | } 12 | 13 | func (db DatabaseValue) Set(value string) error { 14 | sp, err := spec.ForDatabase(value) 15 | if err != nil { 16 | return err 17 | } 18 | t := sp.GetDatabase() 19 | db.db = &t 20 | return nil 21 | } 22 | 23 | func (db DatabaseValue) String() string { 24 | return "" 25 | } 26 | 27 | func Database(s kingpin.Settings) (target *datas.Database) { 28 | s.SetValue(DatabaseValue{target}) 29 | return 30 | } 31 | -------------------------------------------------------------------------------- /util/kp/doc.go: -------------------------------------------------------------------------------- 1 | // Package kp implements integration of a variety of Noms and Replicant types for the Kingpin CLI library. 2 | package kp 3 | -------------------------------------------------------------------------------- /util/kp/hash.go: -------------------------------------------------------------------------------- 1 | package kp 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/attic-labs/noms/go/hash" 7 | kingpin "gopkg.in/alecthomas/kingpin.v2" 8 | ) 9 | 10 | type HashValue hash.Hash 11 | 12 | func (h *HashValue) Set(value string) error { 13 | v, ok := hash.MaybeParse(value) 14 | if !ok { 15 | return errors.New("Invalid hash string") 16 | } 17 | *h = (HashValue)(v) 18 | return nil 19 | } 20 | 21 | func (h *HashValue) String() string { 22 | return (*hash.Hash)(h).String() 23 | } 24 | 25 | func Hash(s kingpin.Settings, target *hash.Hash) { 26 | s.SetValue((*HashValue)(target)) 27 | return 28 | } 29 | -------------------------------------------------------------------------------- /util/kp/spec.go: -------------------------------------------------------------------------------- 1 | package kp 2 | 3 | import ( 4 | "github.com/attic-labs/noms/go/spec" 5 | kingpin "gopkg.in/alecthomas/kingpin.v2" 6 | ) 7 | 8 | type SpecValue spec.Spec 9 | 10 | func (s *SpecValue) Set(value string) error { 11 | sp, err := spec.ForDatabase(value) 12 | if err != nil { 13 | return err 14 | } 15 | *s = (SpecValue)(sp) 16 | return nil 17 | } 18 | 19 | func (s *SpecValue) String() string { 20 | return (*spec.Spec)(s).String() 21 | } 22 | 23 | func DatabaseSpec(s kingpin.Settings) (target *spec.Spec) { 24 | target = &spec.Spec{} 25 | s.SetValue((*SpecValue)(target)) 26 | return 27 | } 28 | -------------------------------------------------------------------------------- /util/log/log.go: -------------------------------------------------------------------------------- 1 | package log 2 | 3 | import ( 4 | "fmt" 5 | 6 | zl "github.com/rs/zerolog" 7 | zlog "github.com/rs/zerolog/log" 8 | 9 | "roci.dev/diff-server/util/gid" 10 | ) 11 | 12 | func Default() zl.Logger { 13 | return zlog.Hook(gid.ZLogHook{}).With().Timestamp().Logger() 14 | } 15 | 16 | func SetGlobalLevelFromString(s string) error { 17 | switch s { 18 | case "debug": 19 | zl.SetGlobalLevel(zl.DebugLevel) 20 | case "info": 21 | zl.SetGlobalLevel(zl.InfoLevel) 22 | case "error": 23 | zl.SetGlobalLevel(zl.ErrorLevel) 24 | default: 25 | return fmt.Errorf("Unknown log level: %s", s) 26 | } 27 | return nil 28 | } 29 | -------------------------------------------------------------------------------- /util/loghttp/filter.go: -------------------------------------------------------------------------------- 1 | package loghttp 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "regexp" 7 | ) 8 | 9 | // FilterFunc filters an HTTP request or response dump, eg to remove or redact 10 | // header lines. 11 | type FilterFunc func([]byte) []byte 12 | 13 | // HeaderAllowlist is a FilterFunc that removes all headers not on 14 | // its allowlist. An empty allowlist filters all headers. 15 | type HeaderAllowlist struct { 16 | re *regexp.Regexp 17 | } 18 | 19 | // NewHeaderAllowlist returns a new HeaderAllowlist that filters 20 | // all but the headers listed in allowlist. 21 | func NewHeaderAllowlist(allolist []string) HeaderAllowlist { 22 | // Build a regexp that will match header lines on the allowlist. 23 | // Default to matching none. 24 | reStr := "^$" 25 | if len(allolist) > 0 { 26 | reStr = fmt.Sprintf(`^(%s`, allolist[0]) 27 | for i := 1; i < len(allolist); i++ { 28 | reStr = fmt.Sprintf("%s|%s", reStr, allolist[i]) 29 | } 30 | reStr = fmt.Sprintf("%s):", reStr) 31 | } 32 | return HeaderAllowlist{regexp.MustCompile(reStr)} 33 | } 34 | 35 | // Filter filters the given HTTP request/response dump. 36 | func (hw HeaderAllowlist) Filter(httpReq []byte) []byte { 37 | endHeadersIndex := bytes.Index(httpReq, []byte("\r\n\r\n")) 38 | if endHeadersIndex == -1 { 39 | return httpReq 40 | } 41 | headerLines := bytes.Split(httpReq[:endHeadersIndex], []byte("\r\n")) 42 | if len(headerLines) == 0 { 43 | return httpReq 44 | } 45 | 46 | filtered := make([]byte, len(httpReq)) 47 | 48 | // Copy the request or status line to output. 49 | l := copy(filtered, headerLines[0]) 50 | l += copy(filtered[l:], []byte("\r\n")) 51 | 52 | // Filter the header lines. 53 | for i := 1; i < len(headerLines); i++ { 54 | if hw.re.Match(headerLines[i]) { 55 | l += copy(filtered[l:], headerLines[i]) 56 | l += copy(filtered[l:], []byte("\r\n")) 57 | } 58 | } 59 | l += copy(filtered[l:], []byte("\r\n")) 60 | 61 | // If no body we're done. Else copy the body. 62 | if endHeadersIndex+3 == len(httpReq) { 63 | return filtered[:l] 64 | } 65 | l += copy(filtered[l:], httpReq[endHeadersIndex+4:]) 66 | 67 | return filtered[:l] 68 | } 69 | 70 | // BodyElider is a FilterFunc that limits the size of the HTTP 71 | // request/response body to max bytes. Bytes are clipped from the 72 | // middle of the body, replaced with "...". 73 | type BodyElider struct { 74 | max int 75 | } 76 | 77 | // NewBodyElider returns a new BodyElider. The minimum value for 78 | // max is 6 to avoid annoying checks on negative indexes when eliding. 79 | func NewBodyElider(max int) BodyElider { 80 | if max < 6 { 81 | max = 6 82 | } 83 | return BodyElider{max} 84 | } 85 | 86 | // Filter filters an HTTP request or response body to max size. 87 | func (be BodyElider) Filter(httpReq []byte) []byte { 88 | endHeadersIndex := bytes.Index(httpReq, []byte("\r\n\r\n")) 89 | beginBodyIndex := 0 90 | // The dump code doesn't currently dump HTTP response headers. 91 | // In that case beginBodyIndex is zero. If we do have headers, 92 | // the body begins after the CRLFCRLF. 93 | if endHeadersIndex != -1 { 94 | beginBodyIndex = endHeadersIndex + 4 95 | } 96 | // If no body, we're done. 97 | if beginBodyIndex > len(httpReq) { 98 | return httpReq 99 | } 100 | 101 | body := httpReq[beginBodyIndex:] 102 | if len(body) <= be.max { 103 | return httpReq 104 | } 105 | size := beginBodyIndex + be.max 106 | filtered := make([]byte, size) 107 | l := copy(filtered, httpReq[:beginBodyIndex]) 108 | l += copy(filtered[l:], body[:be.max/2]) 109 | l += copy(filtered[l:], []byte("...")) 110 | l += copy(filtered[l:], body[len(body)-(size-l):]) 111 | return filtered[:l] 112 | } 113 | -------------------------------------------------------------------------------- /util/loghttp/filter_test.go: -------------------------------------------------------------------------------- 1 | package loghttp 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestHeaderAllowlist_Filter(t *testing.T) { 8 | tests := []struct { 9 | name string 10 | allowlist []string 11 | httpReq string 12 | want string 13 | }{ 14 | { 15 | "not http: empty input", 16 | []string{}, 17 | "", 18 | "", 19 | }, 20 | { 21 | "not http: random text", 22 | []string{}, 23 | "This is not HTTP", 24 | "This is not HTTP", 25 | }, 26 | { 27 | "request: empty allowlist, no headers or body", 28 | []string{}, 29 | "GET / HTTP/1.0\r\n\r\n", 30 | "GET / HTTP/1.0\r\n\r\n", 31 | }, 32 | { 33 | "request: empty allowlist, no headers or body", 34 | []string{}, 35 | "GET / HTTP/1.0\r\n\r\n", 36 | "GET / HTTP/1.0\r\n\r\n", 37 | }, 38 | { 39 | "request: empty allowlist, headers no body", 40 | []string{}, 41 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\n", 42 | "GET / HTTP/1.0\r\n\r\n", 43 | }, 44 | { 45 | "request: empty allowlist, headers and body", 46 | []string{}, 47 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\nbody", 48 | "GET / HTTP/1.0\r\n\r\nbody", 49 | }, 50 | { 51 | "request: filters no headers", 52 | []string{"Bar", "Foo"}, 53 | "GET / HTTP/1.0\r\nFoo: bar\r\nBar: baz\r\n\r\nbody", 54 | "GET / HTTP/1.0\r\nFoo: bar\r\nBar: baz\r\n\r\nbody", 55 | }, 56 | { 57 | "request: filters some headers", 58 | []string{"No-Such-Header", "Bar"}, 59 | "GET / HTTP/1.0\r\nFoo: bar\r\nBar: baz\r\nBonk: boof\r\n\r\nbody", 60 | "GET / HTTP/1.0\r\nBar: baz\r\n\r\nbody", 61 | }, 62 | { 63 | "request: filters all headers", 64 | []string{"No-Such-Header"}, 65 | "GET / HTTP/1.0\r\nFoo: bar\r\nBar: baz\r\nBonk: boof\r\n\r\nbody", 66 | "GET / HTTP/1.0\r\n\r\nbody", 67 | }, 68 | { 69 | "response: filters some headers", 70 | []string{"No-Such-Header", "Bar"}, 71 | "HTTP/1.0 200 OK\r\nFoo: bar\r\nBar: baz\r\nBonk: boof\r\n\r\nbody", 72 | "HTTP/1.0 200 OK\r\nBar: baz\r\n\r\nbody", 73 | }, 74 | } 75 | for _, tt := range tests { 76 | t.Run(tt.name, func(t *testing.T) { 77 | hw := NewHeaderAllowlist(tt.allowlist) 78 | got := string(hw.Filter([]byte(tt.httpReq))) 79 | if got != tt.want { 80 | t.Errorf("HeaderAllowlist.Filter() = %q, want %q", got, tt.want) 81 | } 82 | }) 83 | } 84 | } 85 | 86 | func TestBodyElider_Filter(t *testing.T) { 87 | tests := []struct { 88 | name string 89 | max int 90 | httpReq string 91 | want string 92 | }{ 93 | { 94 | "not http: empty input", 95 | 10, 96 | "", 97 | "", 98 | }, 99 | { 100 | "just a body", 101 | 10, 102 | "012345678901234567890", 103 | "01234...90", 104 | }, 105 | { 106 | "request: no body", 107 | 10, 108 | "GET / HTTP/1.0\r\n\r\n", 109 | "GET / HTTP/1.0\r\n\r\n", 110 | }, 111 | { 112 | "response: no body", 113 | 10, 114 | "HTTP/1.0 200 OK\r\n\r\n", 115 | "HTTP/1.0 200 OK\r\n\r\n", 116 | }, 117 | { 118 | "request: body size smaller than max", 119 | 10, 120 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\nbody", 121 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\nbody", 122 | }, 123 | { 124 | "request: body size equal to max", 125 | 10, 126 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\n0123456789", 127 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\n0123456789", 128 | }, 129 | { 130 | "request: body size larger than max", 131 | 10, 132 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\n012345678901234567890", 133 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\n01234...90", 134 | }, 135 | { 136 | "request: max too small", 137 | 1, 138 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\n012345", 139 | "GET / HTTP/1.0\r\nFoo: bar\r\n\r\n012345", 140 | }, 141 | } 142 | for _, tt := range tests { 143 | t.Run(tt.name, func(t *testing.T) { 144 | be := NewBodyElider(tt.max) 145 | got := string(be.Filter([]byte(tt.httpReq))) 146 | if got != tt.want { 147 | t.Errorf("BodyElider.Filter() = %q, want %q", got, tt.want) 148 | } 149 | }) 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /util/loghttp/loghttp.go: -------------------------------------------------------------------------------- 1 | package loghttp 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "fmt" 7 | "io/ioutil" 8 | "net/http" 9 | "net/http/httputil" 10 | "strings" 11 | 12 | lh "github.com/motemen/go-loghttp" 13 | zl "github.com/rs/zerolog" 14 | zlog "github.com/rs/zerolog/log" 15 | 16 | // Import go-loghttp/global to override default http transport. 17 | _ "github.com/motemen/go-loghttp/global" 18 | ) 19 | 20 | func init() { 21 | lh.DefaultLogRequest = func(req *http.Request) { 22 | // TODO respect log level setting 23 | var dump []byte 24 | var err error 25 | if strings.Index(req.URL.String(), "dynamodb") != -1 { 26 | dump = []byte("") 27 | } else { 28 | dump, err = httputil.DumpRequest(req, true) 29 | if err != nil { 30 | zlog.Err(err).Stack().Msg("Could not dump request") 31 | return 32 | } 33 | dump = filter(dump) 34 | } 35 | // TODO: Properly contextualize these logs. 36 | zlog.Debug(). 37 | Timestamp(). 38 | Str("method", req.Method). 39 | Str("url", req.URL.String()). 40 | Bytes("dump", dump). 41 | Msg("Outgoing request -->") 42 | } 43 | 44 | lh.DefaultLogResponse = func(resp *http.Response) { 45 | var dump []byte 46 | var err error 47 | if strings.Index(resp.Request.URL.String(), "dynamodb") != -1 { 48 | dump = []byte("") 49 | } else { 50 | dump, err = httputil.DumpResponse(resp, true) 51 | if err != nil { 52 | zlog.Err(err).Stack().Msg("Could not dump response") 53 | return 54 | } 55 | dump = filter(dump) 56 | } 57 | zlog.Debug(). 58 | Timestamp(). 59 | Str("method", resp.Request.Method). 60 | Str("url", resp.Request.URL.String()). 61 | Int("status", resp.StatusCode). 62 | Bytes("dump", dump). 63 | Msg("Outgoing request <--") 64 | } 65 | } 66 | 67 | // Filters are called on the HTTP dump before it is logged. 68 | var Filters []FilterFunc 69 | 70 | func filter(httpReq []byte) []byte { 71 | for _, f := range Filters { 72 | httpReq = f(httpReq) 73 | } 74 | return httpReq 75 | } 76 | 77 | // Wrap wraps the given handler with a Handler that logs HTTP requests 78 | // and responses. 79 | func Wrap(handler http.Handler, l zl.Logger) Handler { 80 | return Handler{wrapped: handler, l: l} 81 | } 82 | 83 | // Handler is a wrapper for http.Handlers that logs the HTTP request and 84 | // response. It logs full request headers but logging full response headers 85 | // seems like more work (eg 86 | // https://stackoverflow.com/questions/29319783/logging-responses-to-incoming-http-requests-inside-http-handlefunc) 87 | // so we settle for logging the response status code and response body for now. 88 | type Handler struct { 89 | wrapped http.Handler 90 | l zl.Logger 91 | } 92 | 93 | // ServeHTTP logs the request, calls the underlying handler, and logs the response. 94 | func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { 95 | dump, err := httputil.DumpRequest(r, true) 96 | if err != nil { 97 | h.l.Err(err).Stack().Msg("Could not dump request") 98 | http.Error(w, fmt.Sprint(err), http.StatusInternalServerError) 99 | return 100 | } 101 | dump = filter(dump) 102 | 103 | ll := h.l.With(). 104 | Str("method", r.Method). 105 | Str("req", r.URL.String()). 106 | Logger() 107 | 108 | ll.Debug(). 109 | Bytes("dump", dump). 110 | Msg("Incoming request -->") 111 | 112 | rl := &responseLogger{ResponseWriter: w, status: 200, l: ll} 113 | h.wrapped.ServeHTTP(rl, r) 114 | body := rl.responseBody.Bytes() 115 | maybeUnzippedBody, err := maybeUnzip(body) 116 | if err == nil { 117 | body = maybeUnzippedBody 118 | } else { 119 | ll.Err(err).Stack().Msgf("Error maybe-unzipping response of size %d with status %d; body: '%s'", len(body), rl.status, string(body)) 120 | } 121 | body = filter(body) 122 | ll.Debug(). 123 | Int("status", rl.status). 124 | Bytes("body", body). 125 | Msg("Incoming request <--") 126 | } 127 | 128 | func maybeUnzip(b []byte) ([]byte, error) { 129 | if http.DetectContentType(b) != "application/x-gzip" { 130 | return b, nil 131 | } 132 | zr, err := gzip.NewReader(bytes.NewReader(b)) 133 | if err != nil { 134 | return nil, err 135 | } 136 | return ioutil.ReadAll(zr) 137 | } 138 | 139 | type responseLogger struct { 140 | http.ResponseWriter 141 | responseBody bytes.Buffer 142 | status int 143 | l zl.Logger 144 | } 145 | 146 | func (r *responseLogger) WriteHeader(status int) { 147 | r.status = status 148 | r.ResponseWriter.WriteHeader(status) 149 | } 150 | 151 | func (r *responseLogger) Write(b []byte) (int, error) { 152 | _, err := r.responseBody.Write(b) 153 | if err != nil { 154 | r.l.Err(err).Msgf("Could not capture response body of length %d with status %d for logging; body: '%s'", len(b), r.status, string(b)) 155 | } 156 | n, err := r.ResponseWriter.Write(b) 157 | if err != nil { 158 | r.l.Err(err).Msgf("Error writing response body of length %d with status %d; body: '%s'", len(b), r.status, string(b)) 159 | } 160 | return n, err 161 | } 162 | -------------------------------------------------------------------------------- /util/loghttp/loghttp_test.go: -------------------------------------------------------------------------------- 1 | package loghttp 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "reflect" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test_maybeUnzip(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | var buf bytes.Buffer 16 | zw := gzip.NewWriter(&buf) 17 | _, err := zw.Write([]byte("this is gzipped")) 18 | assert.NoError(err) 19 | assert.NoError(zw.Close()) 20 | 21 | tests := []struct { 22 | in []byte 23 | want []byte 24 | }{ 25 | { 26 | []byte{}, 27 | []byte{}, 28 | }, 29 | { 30 | []byte("\n"), 31 | []byte("\n"), 32 | }, 33 | { 34 | []byte("not gzipped"), 35 | []byte("not gzipped"), 36 | }, 37 | { 38 | buf.Bytes(), 39 | []byte("this is gzipped"), 40 | }, 41 | } 42 | for i, tt := range tests { 43 | got, err := maybeUnzip(tt.in) 44 | assert.NoError(err) 45 | if !reflect.DeepEqual(got, tt.want) { 46 | t.Errorf("%d: maybeUnzip() = %v, want %v", i, got, tt.want) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /util/noms/diff/nomsdiff.go: -------------------------------------------------------------------------------- 1 | package diff 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/attic-labs/noms/go/diff" 7 | "github.com/attic-labs/noms/go/types" 8 | ) 9 | 10 | func Diff(v1, v2 types.Value) string { 11 | buf := &bytes.Buffer{} 12 | diff.PrintDiff(buf, v1, v2, false) 13 | return string(buf.Bytes()) 14 | } 15 | -------------------------------------------------------------------------------- /util/noms/json/doc.go: -------------------------------------------------------------------------------- 1 | // Package json implements integration for some Noms and Replicant types with Canonical JSON marshaling infrastructure. 2 | package json 3 | -------------------------------------------------------------------------------- /util/noms/json/from_json.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package json 6 | 7 | import ( 8 | "bytes" 9 | "io" 10 | "reflect" 11 | 12 | "github.com/attic-labs/noms/go/d" 13 | "github.com/attic-labs/noms/go/types" 14 | cjson "github.com/gibson042/canonicaljson-go" 15 | ) 16 | 17 | var ( 18 | null = types.NewStruct("Null", types.StructData{}) 19 | ) 20 | 21 | func Null() types.Struct { 22 | return null 23 | } 24 | 25 | func nomsValueFromDecodedJSONBase(vrw types.ValueReadWriter, o interface{}) types.Value { 26 | switch o := o.(type) { 27 | case string: 28 | return types.String(o) 29 | case bool: 30 | return types.Bool(o) 31 | case float64: 32 | return types.Number(o) 33 | case nil: 34 | return null 35 | case []interface{}: 36 | items := make([]types.Value, 0, len(o)) 37 | for _, v := range o { 38 | nv := nomsValueFromDecodedJSONBase(vrw, v) 39 | if nv != nil { 40 | items = append(items, nv) 41 | } 42 | } 43 | return types.NewList(vrw, items...) 44 | case map[string]interface{}: 45 | var v types.Value 46 | kv := make([]types.Value, 0, len(o)*2) 47 | for k, v := range o { 48 | nv := nomsValueFromDecodedJSONBase(vrw, v) 49 | if nv != nil { 50 | kv = append(kv, types.String(k), nv) 51 | } 52 | } 53 | v = types.NewMap(vrw, kv...) 54 | return v 55 | 56 | default: 57 | d.Chk.Fail("Nomsification failed.", "I don't understand %+v, which is of type %s!\n", o, reflect.TypeOf(o).String()) 58 | } 59 | return nil 60 | } 61 | 62 | // NomsValueFromDecodedJSON takes a generic Go interface{} and recursively 63 | // tries to resolve the types within so that it can build up and return 64 | // a Noms Value with the same structure. 65 | // 66 | // Currently, the only types supported are the Go versions of legal JSON types: 67 | // Primitives: 68 | // - float64 69 | // - bool 70 | // - string 71 | // - nil 72 | // 73 | // Composites: 74 | // - []interface{} 75 | // - map[string]interface{} 76 | func NomsValueFromDecodedJSON(vrw types.ValueReadWriter, o interface{}) types.Value { 77 | return nomsValueFromDecodedJSONBase(vrw, o) 78 | } 79 | 80 | // FromJSON canonicalizes the input JSON and parses a Noms Value from it. The input 81 | // slice is untouched. Canonicalization involves an extra round trip through Noms. 82 | // This process is, uh, ripe for optimization. 83 | func FromJSON(JSON []byte, vrw types.ValueReadWriter) (types.Value, error) { 84 | c, err := Canonicalize(JSON) 85 | if err != nil { 86 | return nil, err 87 | } 88 | return parseValue(bytes.NewReader(c), vrw) 89 | } 90 | 91 | func parseValue(r io.Reader, vrw types.ValueReadWriter) (types.Value, error) { 92 | dec := cjson.NewDecoder(r) 93 | // TODO: This is pretty inefficient. It would be better to parse the JSON directly into Noms values, 94 | // rather than going through a pile of Go interfaces. 95 | var pile interface{} 96 | err := dec.Decode(&pile) 97 | if err != nil { 98 | return nil, err 99 | } 100 | return NomsValueFromDecodedJSON(vrw, pile), nil 101 | } 102 | -------------------------------------------------------------------------------- /util/noms/json/from_json_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package json 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/attic-labs/noms/go/chunks" 12 | "github.com/attic-labs/noms/go/types" 13 | "github.com/stretchr/testify/assert" 14 | "github.com/stretchr/testify/suite" 15 | ) 16 | 17 | func TestLibTestSuite(t *testing.T) { 18 | suite.Run(t, &LibTestSuite{}) 19 | } 20 | 21 | type LibTestSuite struct { 22 | suite.Suite 23 | vs *types.ValueStore 24 | } 25 | 26 | func (suite *LibTestSuite) SetupTest() { 27 | st := &chunks.TestStorage{} 28 | suite.vs = types.NewValueStore(st.NewView()) 29 | } 30 | 31 | func (suite *LibTestSuite) TearDownTest() { 32 | suite.vs.Close() 33 | } 34 | 35 | func (suite *LibTestSuite) TestPrimitiveTypes() { 36 | vs := suite.vs 37 | suite.EqualValues(types.String("expected"), NomsValueFromDecodedJSON(vs, "expected")) 38 | suite.EqualValues(types.Bool(false), NomsValueFromDecodedJSON(vs, false)) 39 | suite.EqualValues(types.Number(1.7), NomsValueFromDecodedJSON(vs, 1.7)) 40 | suite.True(NomsValueFromDecodedJSON(vs, nil).Equals(Null())) 41 | suite.False(NomsValueFromDecodedJSON(vs, 1.7).Equals(types.Bool(true))) 42 | } 43 | 44 | func (suite *LibTestSuite) TestCompositeTypes() { 45 | vs := suite.vs 46 | 47 | // [false true null] 48 | suite.True( 49 | types.NewList(vs).Edit().Append(types.Bool(false)).Append(types.Bool(true)).Append(Null()).List().Equals( 50 | NomsValueFromDecodedJSON(vs, []interface{}{false, true, nil}))) 51 | 52 | // [[false true null]] 53 | suite.True( 54 | types.NewList(vs).Edit().Append( 55 | types.NewList(vs).Edit().Append(types.Bool(false)).Append(types.Bool(true)).Append(Null()).List()).List().Equals( 56 | NomsValueFromDecodedJSON(vs, []interface{}{[]interface{}{false, true, nil}}))) 57 | 58 | // {"string": "string", 59 | // "list": [false true], 60 | // "map": {"nested": "string"} 61 | // } 62 | m := types.NewMap( 63 | vs, 64 | types.String("string"), 65 | types.String("string"), 66 | types.String("list"), 67 | types.NewList(vs).Edit().Append(types.Bool(false)).Append(types.Bool(true)).List(), 68 | types.String("map"), 69 | types.NewMap( 70 | vs, 71 | types.String("nested"), 72 | types.String("string"))) 73 | o := NomsValueFromDecodedJSON(vs, map[string]interface{}{ 74 | "string": "string", 75 | "list": []interface{}{false, true}, 76 | "map": map[string]interface{}{"nested": "string"}, 77 | }) 78 | 79 | suite.True(m.Equals(o)) 80 | } 81 | 82 | func (suite *LibTestSuite) TestPanicOnUnsupportedType() { 83 | vs := suite.vs 84 | suite.Panics(func() { NomsValueFromDecodedJSON(vs, map[int]string{1: "one"}) }, "Should panic on map[int]string!") 85 | } 86 | 87 | func TestFromJSON(t *testing.T) { 88 | assert := assert.New(t) 89 | noms := types.NewValueStore((&chunks.TestStorage{}).NewView()) 90 | 91 | tests := []struct { 92 | name string 93 | in string 94 | want types.Value 95 | wantErr string 96 | }{ 97 | { 98 | "string", 99 | `"foo"`, 100 | types.String("foo"), 101 | "", 102 | }, 103 | { 104 | "ensure canonicalizes", 105 | `"\u000b"`, 106 | types.String("\u000B"), 107 | "", 108 | }, 109 | { 110 | "map", 111 | `{"foo": "bar"}`, 112 | types.NewMap(noms, types.String("foo"), types.String("bar")), 113 | "", 114 | }, 115 | { 116 | "error: empty value", 117 | ``, 118 | nil, 119 | "couldn't parse value '' as json", 120 | }, 121 | } 122 | for _, tt := range tests { 123 | t.Run(tt.name, func(t *testing.T) { 124 | got, err := FromJSON([]byte(tt.in), noms) 125 | if tt.wantErr != "" { 126 | assert.Error(err) 127 | assert.Regexp(tt.wantErr, err.Error()) 128 | } else { 129 | assert.NoError(err, tt.name) 130 | gotVal := "" 131 | if got != nil { 132 | gotVal = fmt.Sprintf("%s", types.EncodedValue(got)) 133 | } 134 | assert.True(tt.want.Equals(got), "%s: want %s got %s", tt.name, types.EncodedValue(tt.want), gotVal) 135 | } 136 | }) 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /util/noms/json/hash.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | ej "encoding/json" 5 | "errors" 6 | "fmt" 7 | 8 | "github.com/attic-labs/noms/go/hash" 9 | ) 10 | 11 | type Hash struct { 12 | hash.Hash 13 | } 14 | 15 | func (h Hash) MarshalJSON() ([]byte, error) { 16 | return []byte(fmt.Sprintf(`"%s"`, h.String())), nil 17 | } 18 | 19 | func (h *Hash) UnmarshalJSON(data []byte) (err error) { 20 | var str string 21 | err = ej.Unmarshal(data, &str) 22 | if err != nil { 23 | return err 24 | } 25 | hash, ok := hash.MaybeParse(str) 26 | if !ok { 27 | return errors.New("Invaild hash string") 28 | } 29 | h.Hash = hash 30 | return nil 31 | } 32 | -------------------------------------------------------------------------------- /util/noms/json/list.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/attic-labs/noms/go/types" 7 | 8 | "roci.dev/diff-server/util/chk" 9 | ) 10 | 11 | // TODO: test 12 | type List struct { 13 | Value 14 | } 15 | 16 | func MakeList(noms types.ValueReadWriter, v types.Value) List { 17 | if v != nil { 18 | chk.Equal(types.ListKind, v.Kind()) 19 | } 20 | return List{ 21 | Value: Make(noms, v), 22 | } 23 | } 24 | 25 | func (l *List) UnmarshalJSON(data []byte) error { 26 | temp := l.Value 27 | err := temp.UnmarshalJSON(data) 28 | if err != nil { 29 | return err 30 | } 31 | if temp.Kind() != types.ListKind { 32 | return fmt.Errorf("Unexpected Noms type: %s", types.TypeOf(temp.Value).Describe()) 33 | } 34 | l.Value = temp 35 | return nil 36 | } 37 | 38 | func (l *List) List() types.List { 39 | return l.Value.Value.(types.List) 40 | } 41 | -------------------------------------------------------------------------------- /util/noms/json/list_test.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/attic-labs/noms/go/types" 7 | "github.com/stretchr/testify/assert" 8 | 9 | "roci.dev/diff-server/util/noms/memstore" 10 | ) 11 | 12 | func TestListUnmarshal(t *testing.T) { 13 | assert := assert.New(t) 14 | vs := memstore.New() 15 | 16 | var l List 17 | l.Noms = vs 18 | 19 | err := l.UnmarshalJSON([]byte("42")) 20 | assert.EqualError(err, "Unexpected Noms type: Number") 21 | 22 | assert.Nil(l.Value.Value) 23 | assert.Panics(func() { 24 | l.List() 25 | }) 26 | 27 | err = l.UnmarshalJSON([]byte("[42]")) 28 | assert.NoError(err) 29 | expected := types.NewList(vs, types.Number(42)) 30 | assert.True(expected.Equals(l.Value.Value)) 31 | assert.True(expected.Equals(l.List())) 32 | } 33 | -------------------------------------------------------------------------------- /util/noms/json/spec.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/attic-labs/noms/go/spec" 7 | ) 8 | 9 | type Spec struct { 10 | spec.Spec 11 | } 12 | 13 | func (s Spec) MarshalJSON() ([]byte, error) { 14 | return json.Marshal(s.Spec.String()) 15 | } 16 | 17 | func (s *Spec) UnmarshalJSON(data []byte) error { 18 | var str string 19 | err := json.Unmarshal(data, &str) 20 | if err != nil { 21 | return err 22 | } 23 | sp, err := spec.ForDatabase(str) 24 | if err != nil { 25 | return err 26 | } 27 | s.Spec = sp 28 | return nil 29 | } 30 | -------------------------------------------------------------------------------- /util/noms/json/to_json.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package json 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | 11 | "github.com/attic-labs/noms/go/types" 12 | cjson "github.com/gibson042/canonicaljson-go" 13 | ) 14 | 15 | // noNewlineWriter writes to an underlying io.Writer, omitting any trailing newline. 16 | type noNewlineWriter struct { 17 | w io.Writer 18 | } 19 | 20 | // Helper broken out for testing. 21 | func hasNewline(s string) bool { 22 | for _, runeValue := range s { 23 | if string(runeValue) == "\n" { 24 | return true 25 | } 26 | } 27 | return false 28 | } 29 | 30 | // Write implements the io.Writer interface. 31 | func (w *noNewlineWriter) Write(p []byte) (int, error) { 32 | if len(p) == 0 { 33 | return w.w.Write(p) 34 | } 35 | 36 | var trailingNewline bool 37 | // Note: canonical json never has internal newlines. 38 | if hasNewline(string(p[len(p)-1])) { 39 | trailingNewline = true 40 | p = p[:len(p)-1] 41 | } 42 | n, err := w.w.Write(p) 43 | if trailingNewline && n == len(p) { 44 | n = n + 1 45 | } 46 | return n, err 47 | } 48 | 49 | // Canonicalize round-trips the json to canonicalize it. 50 | func Canonicalize(JSON []byte) ([]byte, error) { 51 | var v interface{} 52 | if err := cjson.Unmarshal(JSON, &v); err != nil { 53 | return nil, fmt.Errorf("couldn't parse value '%s' as json: %w", string(JSON), err) 54 | } 55 | return cjson.Marshal(v) 56 | } 57 | 58 | // ToJSON encodes a Noms value as canonical JSON. 59 | // It would be nice to have an option like the original noms 60 | // ops.Indent which would enable pretty printing via the default json library. 61 | func ToJSON(v types.Value, w io.Writer) error { 62 | // TODO: This is a quick hack that is expedient. We should marshal directly to the writer without 63 | // allocating a bunch of Go values. 64 | p, err := toPile(v) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | enc := cjson.NewEncoder(&noNewlineWriter{w}) 70 | return enc.Encode(p) 71 | } 72 | 73 | func toPile(v types.Value) (ret interface{}, err error) { 74 | switch v := v.(type) { 75 | case types.Bool: 76 | return bool(v), nil 77 | case types.Number: 78 | return float64(v), nil 79 | case types.String: 80 | return string(v), nil 81 | case types.Struct: 82 | if !Null().Equals(v) { 83 | return nil, fmt.Errorf("Unsupported struct type: %s", types.TypeOf(v).Describe()) 84 | } 85 | return nil, nil 86 | case types.Map: 87 | r := make(map[string]interface{}, v.Len()) 88 | v.Iter(func(k, cv types.Value) (stop bool) { 89 | sk, ok := k.(types.String) 90 | if !ok { 91 | err = fmt.Errorf("Map key kind %s not supported", types.KindToString[k.Kind()]) 92 | return true 93 | } 94 | var cp interface{} 95 | cp, err = toPile(cv) 96 | if err != nil { 97 | return true 98 | } 99 | r[string(sk)] = cp 100 | return false 101 | }) 102 | return r, err 103 | case types.List: 104 | r := make([]interface{}, v.Len()) 105 | v.Iter(func(cv types.Value, i uint64) (stop bool) { 106 | var cp interface{} 107 | cp, err = toPile(cv) 108 | if err != nil { 109 | return true 110 | } 111 | r[i] = cp 112 | return false 113 | }) 114 | return r, err 115 | } 116 | return nil, fmt.Errorf("Unsupported kind: %s", types.KindToString[v.Kind()]) 117 | } 118 | -------------------------------------------------------------------------------- /util/noms/json/to_json_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2019 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package json 6 | 7 | import ( 8 | "bytes" 9 | "reflect" 10 | "testing" 11 | 12 | "github.com/attic-labs/noms/go/chunks" 13 | "github.com/attic-labs/noms/go/types" 14 | "github.com/stretchr/testify/assert" 15 | "github.com/stretchr/testify/suite" 16 | ) 17 | 18 | func TestToJSONSuite(t *testing.T) { 19 | suite.Run(t, &ToJSONSuite{}) 20 | } 21 | 22 | type ToJSONSuite struct { 23 | suite.Suite 24 | vs *types.ValueStore 25 | } 26 | 27 | func (suite *ToJSONSuite) SetupTest() { 28 | st := &chunks.TestStorage{} 29 | suite.vs = types.NewValueStore(st.NewView()) 30 | } 31 | 32 | func (suite *ToJSONSuite) TearDownTest() { 33 | suite.vs.Close() 34 | } 35 | 36 | func (suite *ToJSONSuite) TestToJSON() { 37 | tc := []struct { 38 | desc string 39 | in types.Value 40 | exp string 41 | expError string 42 | }{ 43 | {"null", Null(), `null`, ""}, 44 | {"true", types.Bool(true), "true", ""}, 45 | {"false", types.Bool(false), "false", ""}, 46 | {"42", types.Number(42), "42", ""}, 47 | {"88.8", types.Number(88.8), "8.88E1", ""}, 48 | {"empty string", types.String(""), `""`, ""}, 49 | {"foobar", types.String("foobar"), `"foobar"`, ""}, 50 | {"strings with escaped newlines", types.String(`"\nmonkey`), `"\"\\nmonkey"`, ""}, 51 | {"strings with newlines", types.String("\nmonkey"), `"\nmonkey"`, ""}, 52 | {"strings with newline bytes", types.String("\x0amonkey"), `"\nmonkey"`, ""}, // U+000A is newline and its UTF-8 representation is '0a' 53 | {"unnamed struct", types.NewStruct("", types.StructData{}), "", "Unsupported struct type: Struct {}"}, 54 | {"named struct", types.NewStruct("Person", types.StructData{}), "", "Unsupported struct type: Struct Person {}"}, 55 | {"bad null struct", types.NewStruct("Null", types.StructData{"foo": types.String("bar")}), "", "Unsupported struct type: Struct Null {\n foo: String,\n}"}, 56 | {"empty list", types.NewList(suite.vs), "[]", ""}, 57 | {"non-empty list", types.NewList(suite.vs, types.Number(42), types.String("foo")), `[42,"foo"]`, ""}, 58 | {"sets", types.NewSet(suite.vs), "", "Unsupported kind: Set"}, 59 | {"map non-string key", types.NewMap(suite.vs, types.Number(42), types.Number(42)), "", "Map key kind Number not supported"}, 60 | {"empty map", types.NewMap(suite.vs), "{}", ""}, 61 | {"non-empty map", types.NewMap(suite.vs, types.String("foo"), types.String("bar"), types.String("baz"), types.Number(42)), `{"baz":42,"foo":"bar"}`, ""}, 62 | {"map with newlines in strings", types.NewMap(suite.vs, types.String("foo\n"), types.String("ba\nr")), `{"foo\n":"ba\nr"}`, ""}, 63 | {"complex value", types.NewMap(suite.vs, 64 | types.String("list"), types.NewList(suite.vs, 65 | types.NewMap(suite.vs, 66 | types.String("foo"), types.String("bar"), 67 | types.String("hot"), types.Number(42), 68 | types.String("null"), Null()))), `{"list":[{"foo":"bar","hot":42,"null":null}]}`, ""}, 69 | } 70 | 71 | for _, t := range tc { 72 | buf := &bytes.Buffer{} 73 | err := ToJSON(t.in, buf) 74 | if t.expError != "" { 75 | suite.EqualError(err, t.expError, t.desc) 76 | suite.Equal("", string(buf.Bytes()), t.desc) 77 | } else { 78 | suite.NoError(err) 79 | suite.Equal(t.exp, string(buf.Bytes()), t.desc) 80 | } 81 | } 82 | } 83 | 84 | // We have this test to convince ourselves that the canonical json never has 85 | // internal newlines, thus that our newlie-filtering is not going to strip 86 | // newlines from json strings. 87 | func Test_hasNewline(t *testing.T) { 88 | assert.True(t, hasNewline("foo\n")) 89 | 90 | type args struct { 91 | s string 92 | } 93 | tests := []struct { 94 | input string 95 | }{ 96 | {"no newline when encoded to json"}, 97 | {"no newlines when encoded to json \\n"}, 98 | {`"no newlines when encoded to json"`}, 99 | {`"no newlines when encoded to json\n"`}, 100 | {`"no newlines when encoded to json 101 | even like this"`}, 102 | {"no newlines when encoded to json\n"}, 103 | {"no newlines when encoded to json\x0a"}, 104 | } 105 | for _, tt := range tests { 106 | t.Run(tt.input, func(t *testing.T) { 107 | v := types.String(tt.input) 108 | var b bytes.Buffer 109 | assert.NoError(t, ToJSON(v, &b)) 110 | if got := hasNewline(string(b.Bytes())); got { 111 | t.Errorf("hasNewline(%q) = %v, want false", tt.input, got) 112 | } 113 | }) 114 | } 115 | } 116 | 117 | func TestCanonicalize(t *testing.T) { 118 | tests := []struct { 119 | name string 120 | JSON []byte 121 | want []byte 122 | wantErr bool 123 | }{ 124 | { 125 | "object", 126 | []byte(" { \"z\" : 1, \n \"a\": 2 } \r"), 127 | []byte("{\"a\":2,\"z\":1}"), 128 | false, 129 | }, 130 | } 131 | for _, tt := range tests { 132 | t.Run(tt.name, func(t *testing.T) { 133 | got, err := Canonicalize(tt.JSON) 134 | if (err != nil) != tt.wantErr { 135 | t.Errorf("Canonicalize() error = %v, wantErr %v", err, tt.wantErr) 136 | return 137 | } 138 | if !reflect.DeepEqual(got, tt.want) { 139 | t.Errorf("Canonicalize() = %v, want %v", got, tt.want) 140 | } 141 | }) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /util/noms/json/value.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "bytes" 5 | 6 | "github.com/attic-labs/noms/go/types" 7 | "roci.dev/diff-server/util/chk" 8 | ) 9 | 10 | type Value struct { 11 | types.Value 12 | Noms types.ValueReadWriter 13 | } 14 | 15 | func New(noms types.ValueReadWriter, v types.Value) *Value { 16 | r := Make(noms, v) 17 | return &r 18 | } 19 | 20 | func Make(noms types.ValueReadWriter, v types.Value) Value { 21 | return Value{ 22 | Noms: noms, 23 | Value: v, 24 | } 25 | } 26 | 27 | func (v Value) MarshalJSON() ([]byte, error) { 28 | buf := &bytes.Buffer{} 29 | err := ToJSON(v.Value, buf) 30 | if err != nil { 31 | return nil, err 32 | } 33 | return buf.Bytes(), nil 34 | } 35 | 36 | func (v *Value) UnmarshalJSON(data []byte) error { 37 | chk.NotNil(v.Noms, "Need to set Noms field to unmarshal from JSON") 38 | r, err := FromJSON(data, v.Noms) 39 | if err != nil { 40 | return err 41 | } 42 | v.Value = r 43 | return nil 44 | } 45 | -------------------------------------------------------------------------------- /util/noms/json/value_test.go: -------------------------------------------------------------------------------- 1 | package json 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/attic-labs/noms/go/chunks" 8 | "github.com/attic-labs/noms/go/types" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestValueJSONMarshal(t *testing.T) { 13 | assert := assert.New(t) 14 | noms := types.NewValueStore((&chunks.TestStorage{}).NewView()) 15 | 16 | tc := []struct { 17 | n types.Value 18 | j string 19 | }{ 20 | {types.Bool(true), "true"}, 21 | {types.Bool(false), "false"}, 22 | {types.Number(42), "42"}, 23 | {types.String("foo"), "\"foo\""}, 24 | {types.NewList(noms, types.Bool(true)), "[true]"}, 25 | {types.NewMap(noms, types.String("foo"), types.Bool(true)), "{\"foo\":true}"}, 26 | {Null(), "null"}, 27 | } 28 | 29 | for i, t := range tc { 30 | msg := fmt.Sprintf("test case %d", i) 31 | v := New(noms, t.n) 32 | marshaled, err := v.MarshalJSON() 33 | assert.NoError(err, msg) 34 | assert.Equal(t.j, string(marshaled), msg) 35 | v = New(noms, nil) 36 | err = v.UnmarshalJSON(marshaled) 37 | assert.NoError(err, msg) 38 | assert.True(v.Value.Equals(t.n), msg) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /util/noms/memstore/memstore.go: -------------------------------------------------------------------------------- 1 | package memstore 2 | 3 | import ( 4 | "github.com/attic-labs/noms/go/chunks" 5 | "github.com/attic-labs/noms/go/types" 6 | ) 7 | 8 | func New() *types.ValueStore { 9 | ts := &chunks.TestStorage{} 10 | return types.NewValueStore(ts.NewView()) 11 | } 12 | -------------------------------------------------------------------------------- /util/noms/union/union.go: -------------------------------------------------------------------------------- 1 | package union 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | 8 | "github.com/attic-labs/noms/go/marshal" 9 | "github.com/attic-labs/noms/go/types" 10 | 11 | "roci.dev/diff-server/util/chk" 12 | ) 13 | 14 | func Marshal(st interface{}, noms types.ValueReadWriter) (types.Value, error) { 15 | t := reflect.TypeOf(st) 16 | v := reflect.ValueOf(st) 17 | chk.Equal(reflect.Struct, v.Kind()) 18 | var r types.Value 19 | for i := 0; i < v.NumField(); i++ { 20 | fv := v.Field(i).Interface() 21 | chk.Equal(reflect.Struct, v.Kind()) 22 | if reflect.DeepEqual(reflect.Zero(t.Field(i).Type).Interface(), fv) { 23 | continue 24 | } 25 | nfv, err := marshal.Marshal(noms, fv) 26 | if err != nil { 27 | return nil, fmt.Errorf("Could not marshal field %s: %v", t.Field(i).PkgPath, err) 28 | } 29 | if r != nil { 30 | return nil, errors.New("At most one field of a union may be set") 31 | } 32 | r = nfv 33 | } 34 | return r, nil 35 | } 36 | 37 | func Unmarshal(in types.Value, out interface{}) error { 38 | if in.Kind() != types.StructKind { 39 | return errors.New("Can only unmarshal Noms structs into unions") 40 | } 41 | v := reflect.ValueOf(out) 42 | if v.Kind() != reflect.Ptr { 43 | return errors.New("Can only unmarshal unions into pointer to struct") 44 | } 45 | v = reflect.Indirect(v) 46 | if v.Kind() != reflect.Struct { 47 | return errors.New("Can only unmarshal unions into pointer to struct") 48 | } 49 | t := v.Type() 50 | fn := in.(types.Struct).Name() 51 | _, ok := t.FieldByName(fn) 52 | if !ok { 53 | return fmt.Errorf("Could not get field: %s", fn) 54 | } 55 | err := marshal.Unmarshal(in, v.FieldByName(fn).Addr().Interface()) 56 | if err != nil { 57 | return fmt.Errorf("Cannot unmarshal noms value onto field %s: %v", fn, err) 58 | } 59 | return nil 60 | } 61 | -------------------------------------------------------------------------------- /util/noms/union/union_test.go: -------------------------------------------------------------------------------- 1 | package union 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/attic-labs/noms/go/types" 7 | "github.com/stretchr/testify/assert" 8 | 9 | "roci.dev/diff-server/util/noms/memstore" 10 | ) 11 | 12 | func TestMarshalUnion(t *testing.T) { 13 | assert := assert.New(t) 14 | 15 | type Bar struct { 16 | A string 17 | } 18 | 19 | type Baz struct { 20 | B string 21 | } 22 | 23 | type Foo struct { 24 | Bar Bar 25 | Baz Baz 26 | } 27 | 28 | f := Foo{} 29 | f.Baz.B = "monkey" 30 | 31 | ms := memstore.New() 32 | 33 | exp := types.NewStruct("Baz", types.StructData{"b": types.String("monkey")}) 34 | 35 | v, err := Marshal(f, ms) 36 | assert.NoError(err) 37 | assert.True(exp.Equals(v)) 38 | 39 | act := Foo{} 40 | err = Unmarshal(v, &act) 41 | assert.NoError(err) 42 | assert.Equal(act, f) 43 | 44 | f.Bar.A = "a" 45 | v, err = Marshal(f, ms) 46 | assert.EqualError(err, "At most one field of a union may be set") 47 | assert.Nil(v) 48 | } 49 | -------------------------------------------------------------------------------- /util/tbl/tbl.go: -------------------------------------------------------------------------------- 1 | package tbl 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strconv" 7 | ) 8 | 9 | // Table implements a simple two-column ascii table. 10 | type Table struct { 11 | rows []row 12 | } 13 | 14 | type row struct { 15 | key string 16 | value string 17 | } 18 | 19 | // Add a row to the table. 20 | func (t *Table) Add(key, value string) *Table { 21 | t.rows = append(t.rows, row{key, value}) 22 | return t 23 | } 24 | 25 | // WriteTo writes the table out. 26 | func (t *Table) WriteTo(w io.Writer) (int64, error) { 27 | keyWidth := 0 28 | for _, r := range t.rows { 29 | kl := len(r.key) 30 | if kl > keyWidth { 31 | keyWidth = kl 32 | } 33 | } 34 | res := 0 35 | for _, r := range t.rows { 36 | n, err := fmt.Fprintf(w, "%-"+strconv.Itoa(keyWidth)+"s%s\n", r.key, r.value) 37 | if err != nil { 38 | return 0, err 39 | } 40 | res += n 41 | } 42 | return int64(res), nil 43 | } 44 | -------------------------------------------------------------------------------- /util/tbl/tbl_test.go: -------------------------------------------------------------------------------- 1 | package tbl 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestBasics(t *testing.T) { 11 | assert := assert.New(t) 12 | tc := []struct { 13 | in *Table 14 | expected string 15 | }{ 16 | {&Table{}, ""}, 17 | {(&Table{}).Add("foo", "bar"), "foobar\n"}, 18 | {(&Table{}).Add("a", "a").Add("bb", "bb"), "a a\nbbbb\n"}, 19 | {(&Table{}).Add("a", "a").Add("ccc", "ccc").Add("bb", "bb"), "a a\ncccccc\nbb bb\n"}, 20 | } 21 | 22 | for _, t := range tc { 23 | sb := &strings.Builder{} 24 | n, err := t.in.WriteTo(sb) 25 | assert.NoError(err) 26 | assert.Equal(int64(len(t.expected)), n) 27 | assert.Equal(t.expected, sb.String()) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /util/time/time.go: -------------------------------------------------------------------------------- 1 | package time 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/attic-labs/noms/go/util/datetime" 7 | ) 8 | 9 | var ( 10 | fakeTime *time.Time 11 | ) 12 | 13 | func Now() time.Time { 14 | if fakeTime != nil { 15 | return *fakeTime 16 | } 17 | return time.Now() 18 | } 19 | 20 | func DateTime() datetime.DateTime { 21 | return datetime.DateTime{Now()} 22 | } 23 | 24 | func SetFake() (undo func()) { 25 | loc, err := time.LoadLocation("US/Hawaii") 26 | if err != nil { 27 | panic(err) 28 | } 29 | f := time.Date(2014, 1, 24, 0, 0, 0, 0, loc) 30 | fakeTime = &f 31 | return ClearFake 32 | } 33 | 34 | func ClearFake() { 35 | fakeTime = nil 36 | } 37 | 38 | func String(t time.Time) string { 39 | if fakeTime != nil { 40 | t = t.In(fakeTime.Location()) 41 | } 42 | return t.String() 43 | } 44 | -------------------------------------------------------------------------------- /util/time/time_test.go: -------------------------------------------------------------------------------- 1 | package time 2 | 3 | import ( 4 | "testing" 5 | gt "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestBasics(t *testing.T) { 11 | assert := assert.New(t) 12 | SetFake() 13 | f := Now() 14 | assert.NotEqual(f, gt.Now()) 15 | assert.NotEmpty(f) 16 | ClearFake() 17 | assert.NotEqual(f, Now()) 18 | func() { 19 | defer SetFake()() 20 | assert.Equal(f, Now()) 21 | }() 22 | assert.NotEqual(f, Now()) 23 | } 24 | -------------------------------------------------------------------------------- /util/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | // This is updated when we bump the version, by the 'bump' command from repc. 4 | const v = "1.1.0" 5 | 6 | // This is injected with the correct value when building a release. 7 | var h = "devbuild" 8 | 9 | // Version returns the current version of Replicant. 10 | func Version() string { 11 | return v + "+" + h 12 | } 13 | --------------------------------------------------------------------------------