├── .github ├── PULL_REQUEST_TEMPLATE.md └── ISSUE_TEMPLATE.md ├── go.mod ├── go.sum ├── docs └── contributing.md ├── cmd └── rvcs │ └── rvcs.go ├── publish ├── verify.go ├── sign.go ├── helper.go ├── push.go └── pull.go ├── extensions ├── rvcs-pull-file ├── rvcs-sign-ssh ├── rvcs-verify-ssh └── rvcs-push-file ├── snapshot ├── identity_test.go ├── hash_test.go ├── identity.go ├── tree.go ├── tree_test.go ├── hash.go ├── file_test.go ├── file.go └── snapshot.go ├── command ├── merge.go ├── remove-mirror.go ├── import.go ├── publish.go ├── add-mirror.go ├── log.go ├── snapshot.go ├── export.go └── commands.go ├── config ├── config_test.go └── config.go ├── merge ├── base.go ├── helper.go ├── helper_test.go ├── base_test.go ├── checkout.go ├── merge.go └── merge_test.go ├── log ├── log_test.go └── log.go ├── bundle ├── bundle_test.go └── bundle.go ├── LICENSE ├── README.md └── storage └── storage_test.go /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Fixes # 2 | 3 | > It's a good idea to open an issue first for discussion. 4 | 5 | - [ ] Tests pass 6 | - [ ] Appropriate changes to README are included in PR -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Expected Behavior 2 | 3 | 4 | ## Actual Behavior 5 | 6 | 7 | ## Steps to Reproduce the Problem 8 | 9 | 1. 10 | 1. 11 | 1. 12 | 13 | ## Specifications 14 | 15 | - Version: 16 | - Platform: -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/google/recursive-version-control-system 2 | 3 | go 1.18 4 | 5 | require ( 6 | filippo.io/age v1.1.1 7 | github.com/google/go-cmp v0.5.7 8 | golang.org/x/sys v0.8.0 9 | golang.org/x/term v0.3.0 10 | ) 11 | 12 | require golang.org/x/crypto v0.4.0 // indirect 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | filippo.io/age v1.1.1 h1:pIpO7l151hCnQ4BdyBujnGP2YlUo0uj6sAVNHGBvXHg= 2 | filippo.io/age v1.1.1/go.mod h1:l03SrzDUrBkdBx8+IILdnn2KZysqQdbEBUQ4p3sqEQE= 3 | github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= 4 | github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= 5 | golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= 6 | golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= 7 | golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= 8 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 9 | golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= 10 | golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= 11 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 12 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 13 | -------------------------------------------------------------------------------- /docs/contributing.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code Reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows [Google's Open Source Community 28 | Guidelines](https://opensource.google/conduct/). 29 | -------------------------------------------------------------------------------- /cmd/rvcs/rvcs.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "log" 20 | "os" 21 | "path/filepath" 22 | 23 | "github.com/google/recursive-version-control-system/command" 24 | "github.com/google/recursive-version-control-system/storage" 25 | ) 26 | 27 | func main() { 28 | home, err := os.UserHomeDir() 29 | if err != nil { 30 | log.Fatalf("failure resolving the user's home dir: %v\n", err) 31 | } 32 | s := &storage.LocalFiles{filepath.Join(home, ".rvcs/archive")} 33 | ctx := context.Background() 34 | 35 | ret := command.Run(ctx, s, os.Args) 36 | os.Exit(ret) 37 | } 38 | -------------------------------------------------------------------------------- /publish/verify.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package publish defines methods for publishing rvcs snapshots. 16 | package publish 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "fmt" 22 | 23 | "github.com/google/recursive-version-control-system/snapshot" 24 | "github.com/google/recursive-version-control-system/storage" 25 | ) 26 | 27 | func Verify(ctx context.Context, s *storage.LocalFiles, id *snapshot.Identity, signatureHash *snapshot.Hash) (*snapshot.Hash, error) { 28 | if id == nil { 29 | return nil, errors.New("identity must not be nil") 30 | } 31 | if signatureHash == nil { 32 | // This is always the case for a new identity, so we treat 33 | // the nil hash as a special case that can alwasy be verified. 34 | return nil, nil 35 | } 36 | args := []string{id.String(), signatureHash.String()} 37 | h, err := runHelper(ctx, "verify", id.Algorithm(), args) 38 | if err != nil { 39 | return nil, fmt.Errorf("failure invoking the verify helper for %q: %v", id.Algorithm(), err) 40 | } 41 | return h, nil 42 | } 43 | -------------------------------------------------------------------------------- /publish/sign.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package publish defines methods for publishing rvcs snapshots. 16 | package publish 17 | 18 | import ( 19 | "context" 20 | "errors" 21 | "fmt" 22 | 23 | "github.com/google/recursive-version-control-system/snapshot" 24 | "github.com/google/recursive-version-control-system/storage" 25 | ) 26 | 27 | func Sign(ctx context.Context, s *storage.LocalFiles, id *snapshot.Identity, h *snapshot.Hash, prevSignature *snapshot.Hash) (*snapshot.Hash, error) { 28 | if id == nil { 29 | return nil, errors.New("identity must not be nil") 30 | } 31 | if h == nil { 32 | // Signing a nil hash is a no-op 33 | return nil, nil 34 | } 35 | args := []string{id.String(), h.String(), prevSignature.String()} 36 | h, err := runHelper(ctx, "sign", id.Algorithm(), args) 37 | if err != nil { 38 | return nil, fmt.Errorf("failure invoking the sign helper for %q: %v", id.Algorithm(), err) 39 | } 40 | if err := s.UpdateSignatureForIdentity(ctx, id, h); err != nil { 41 | return nil, fmt.Errorf("failure updating the latest snapshot for %q to %q: %v", id, h, err) 42 | } 43 | return h, nil 44 | } 45 | -------------------------------------------------------------------------------- /extensions/rvcs-pull-file: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Copyright 2022 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | path="${1#file://}" 18 | identity="${2}" 19 | prevHash="${3}" 20 | outFile="${4}" 21 | 22 | if [ -z "${path}" ]; then 23 | echo "No file path provided." >&2 24 | echo "Usage:" >&2 25 | echo " rvcs-pull-file file:// " >&2 26 | exit 1 27 | fi 28 | 29 | if [ -z "${identity}" ]; then 30 | echo "No identity provided." >&2 31 | echo "Usage:" >&2 32 | echo " rvcs-pull-file file:// " >&2 33 | exit 1 34 | fi 35 | 36 | if [ -z "${outFile}" ]; then 37 | echo "No output file provided." >&2 38 | echo "Usage:" >&2 39 | echo " rvcs-pull-file file:// " >&2 40 | exit 1 41 | fi 42 | 43 | bundleName="$(echo "${identity}" | shasum -a 256 | cut -d ' ' -f 1)-bundle.zip" 44 | bundlePath="${path}/${bundleName}" 45 | if [ ! -f "${bundlePath}" ]; then 46 | echo "File ${bundlePath} does not exist..." >&2 47 | exit 0 48 | fi 49 | 50 | rvcs import "${bundlePath}" >&2 51 | unzip -p "${bundlePath}" "metadata/signature" | tr -d "[:space:]" > "${outFile}" 52 | for previousBundle in $(unzip -p "${bundlePath}" "metadata/previous"); do 53 | additional=$(rvcs import -v "${previousBundle}") 54 | if [ "${additional}" == "" ]; then 55 | exit 0 56 | fi 57 | done 58 | -------------------------------------------------------------------------------- /snapshot/identity_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snapshot 16 | 17 | import "testing" 18 | 19 | func TestParseIdentityRoundTrip(t *testing.T) { 20 | testCases := []struct { 21 | Description string 22 | Serialized string 23 | WantError bool 24 | }{ 25 | { 26 | Description: "empty identity string", 27 | }, 28 | { 29 | Description: "missing colon", 30 | Serialized: "a", 31 | WantError: true, 32 | }, 33 | { 34 | Description: "not enough colons", 35 | Serialized: "a:b", 36 | WantError: true, 37 | }, 38 | { 39 | Description: "valid format", 40 | Serialized: "ed25519::0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF", 41 | }, 42 | } 43 | for _, testCase := range testCases { 44 | parsed, err := ParseIdentity(testCase.Serialized) 45 | if testCase.WantError { 46 | if err == nil { 47 | t.Errorf("unexpected response for test case %q: %+v", testCase.Description, parsed) 48 | } 49 | } else if err != nil { 50 | t.Errorf("unexpected failure parsing the serialized identity %q for the test case %q: %v", testCase.Serialized, testCase.Description, err) 51 | } else if got, want := parsed.String(), testCase.Serialized; got != want { 52 | t.Errorf("unexpected result for identity parsing roundtrip of %q; got %q, want %q", testCase.Description, got, want) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /command/merge.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | "path/filepath" 23 | 24 | "github.com/google/recursive-version-control-system/merge" 25 | "github.com/google/recursive-version-control-system/snapshot" 26 | "github.com/google/recursive-version-control-system/storage" 27 | ) 28 | 29 | const mergeUsage = `Usage: %s merge 30 | 31 | Where is a local file path, and is one of: 32 | 33 | The hash of a known snapshot. 34 | A local file path which has previously been snapshotted. 35 | ` 36 | 37 | func mergeCommand(ctx context.Context, s *storage.LocalFiles, cmd string, args []string) (int, error) { 38 | if len(args) != 2 { 39 | fmt.Fprintf(flag.CommandLine.Output(), mergeUsage, cmd) 40 | return 1, nil 41 | } 42 | h, err := resolveSnapshot(ctx, s, args[0]) 43 | if err != nil { 44 | return 1, fmt.Errorf("failure resolving the snapshot hash for %q: %v", args[0], err) 45 | } 46 | abs, err := filepath.Abs(args[1]) 47 | if err != nil { 48 | return 1, fmt.Errorf("failure determining the absolute path of %q: %v", args[1], err) 49 | } 50 | if err := merge.Merge(ctx, s, h, snapshot.Path(abs)); err != nil { 51 | return 1, fmt.Errorf("failure merging %q into %q: %v", h, abs, err) 52 | } 53 | return 0, nil 54 | } 55 | -------------------------------------------------------------------------------- /snapshot/hash_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snapshot 16 | 17 | import "testing" 18 | 19 | func TestParseHashRoundTrip(t *testing.T) { 20 | testCases := []struct { 21 | Description string 22 | Serialized string 23 | WantError bool 24 | }{ 25 | { 26 | Description: "empty hash string", 27 | }, 28 | { 29 | Description: "missing colon", 30 | Serialized: "a", 31 | WantError: true, 32 | }, 33 | { 34 | Description: "too many colons", 35 | Serialized: "a:b:c", 36 | WantError: true, 37 | }, 38 | { 39 | Description: "unknown hash function", 40 | Serialized: "a:b", 41 | WantError: true, 42 | }, 43 | { 44 | Description: "non-hex contents", 45 | Serialized: "sha256:qwerty", 46 | WantError: true, 47 | }, 48 | { 49 | Description: "valid SHA-256", 50 | Serialized: "sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245", 51 | }, 52 | } 53 | for _, testCase := range testCases { 54 | parsed, err := ParseHash(testCase.Serialized) 55 | if testCase.WantError { 56 | if err == nil { 57 | t.Errorf("unexpected response for test case %q: %+v", testCase.Description, parsed) 58 | } 59 | } else if err != nil { 60 | t.Errorf("unexpected failure parsing the serialized hash %q for the test case %q: %v", testCase.Serialized, testCase.Description, err) 61 | } else if got, want := parsed.String(), testCase.Serialized; got != want { 62 | t.Errorf("unexpected result for hash parsing roundtrip of %q; got %q, want %q", testCase.Description, got, want) 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /command/remove-mirror.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | "net/url" 23 | 24 | "github.com/google/recursive-version-control-system/config" 25 | "github.com/google/recursive-version-control-system/snapshot" 26 | "github.com/google/recursive-version-control-system/storage" 27 | ) 28 | 29 | const removeMirrorUsage = `Usage: %s remove-mirror [] 30 | 31 | Where is the optional identity to mirror (omit to apply to all identities), and is the URL of the mirror. 32 | ` 33 | 34 | func removeMirrorCommand(ctx context.Context, s *storage.LocalFiles, cmd string, args []string) (int, error) { 35 | if len(args) < 1 { 36 | fmt.Fprintf(flag.CommandLine.Output(), removeMirrorUsage, cmd) 37 | return 1, nil 38 | } 39 | var id *snapshot.Identity 40 | var err error 41 | if len(args) > 1 { 42 | id, err = snapshot.ParseIdentity(args[0]) 43 | if err != nil { 44 | return 1, fmt.Errorf("failure parsing the identity %q: %v", args[0], err) 45 | } 46 | args = args[1:] 47 | } 48 | mirrorURL, err := url.Parse(args[0]) 49 | if err != nil { 50 | return 1, fmt.Errorf("failure parsing the mirror URL %q: %v", args[0], err) 51 | } 52 | settings, err := config.Read() 53 | if err != nil { 54 | return 1, fmt.Errorf("failure reading the existing config settings: %v", err) 55 | } 56 | if id == nil { 57 | settings = settings.WithoutAdditionalMirror(mirrorURL) 58 | } else { 59 | settings = settings.WithoutMirrorForIdentity(id.String(), mirrorURL) 60 | } 61 | if err := settings.Write(); err != nil { 62 | return 1, fmt.Errorf("failure writing the updated config settings: %v", err) 63 | } 64 | return 0, nil 65 | } 66 | -------------------------------------------------------------------------------- /config/config_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package config defines the configuration options for rvcs. 16 | package config 17 | 18 | import ( 19 | "encoding/json" 20 | "net/url" 21 | "testing" 22 | 23 | "github.com/google/go-cmp/cmp" 24 | ) 25 | 26 | func TestParseSettings(t *testing.T) { 27 | testCases := []struct { 28 | Description string 29 | Serialized string 30 | Want *Settings 31 | }{ 32 | { 33 | Description: "Empty settings", 34 | Serialized: "{}", 35 | Want: &Settings{}, 36 | }, 37 | { 38 | Description: "Empty top-level fields", 39 | Serialized: "{\"Identities\": [], \"AdditionalMirrors\": []}", 40 | Want: &Settings{ 41 | Identities: []*Identity{}, 42 | AdditionalMirrors: []*Mirror{}, 43 | }, 44 | }, 45 | { 46 | Description: "Non-empty mirrors", 47 | Serialized: "{\"AdditionalMirrors\": [{\"url\": \"gcs://example.com/some-path\", \"helperFlags\": [\"--foo\", \"--bar\"], \"readOnly\": true}]}", 48 | Want: &Settings{ 49 | AdditionalMirrors: []*Mirror{ 50 | &Mirror{ 51 | URL: &url.URL{ 52 | Scheme: "gcs", 53 | Host: "example.com", 54 | Path: "/some-path", 55 | }, 56 | HelperFlags: []string{ 57 | "--foo", 58 | "--bar", 59 | }, 60 | ReadOnly: true, 61 | }, 62 | }, 63 | }, 64 | }, 65 | } 66 | for _, testCase := range testCases { 67 | var s Settings 68 | if err := json.Unmarshal([]byte(testCase.Serialized), &s); err != nil { 69 | t.Errorf("Error parsing the settings for %q: %v", testCase.Description, err) 70 | } else if got, want := &s, testCase.Want; !cmp.Equal(got, want) { 71 | t.Errorf("Wrong value unmarshalling config settings; got %+v, want %+v, diff %q", got, want, cmp.Diff(got, want)) 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /extensions/rvcs-sign-ssh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Copyright 2022 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Usage: 18 | # 19 | # rvcs publish ssh:: 20 | # 21 | # ... where is the base64-encoded contents of one of 22 | # the "id_....pub" files under your "~/.ssh" directory. 23 | 24 | pubkey="${1#ssh::}" 25 | hash="${2}" 26 | previousSignature="${3:-}" 27 | outFile="${4}" 28 | 29 | if [ -z "${pubkey}" ]; then 30 | echo "No public key provided." >&2 31 | echo "Usage:" >&2 32 | echo " rvcs-sign-ssh ssh:: ( | \"\") " >&2 33 | exit 1 34 | fi 35 | 36 | if [ -z "${hash}" ]; then 37 | echo "No snapshot hashcode provided." >&2 38 | echo "Usage:" >&2 39 | echo " rvcs-sign-ssh ssh:: ( | \"\") " >&2 40 | exit 1 41 | fi 42 | 43 | if [ -z "${outFile}" ]; then 44 | echo "No output file provided." >&2 45 | echo "Usage:" >&2 46 | echo " rvcs-sign-ssh ssh:: ( | \"\") " >&2 47 | exit 1 48 | fi 49 | 50 | keyfile="$(grep "${pubkey}" ~/.ssh/*.pub | cut -d ":" -f 1)" 51 | if [ -z "${keyfile}" ]; then 52 | echo "Could not find the public key file for ${pubkey} under ~/.ssh/" >&2 53 | exit 1 54 | fi 55 | 56 | signingKey="${keyfile%.pub}" 57 | dir=$(mktemp -d) 58 | echo "${hash}" > "${dir}/signed.txt" 59 | echo "${previousSignature}" >> "${dir}/signed.txt" 60 | rvcs snapshot --additional-parents="${hash}" "${dir}/signed.txt" >/dev/null 61 | ssh-keygen -q -Y sign -f "${signingKey}" -n "github.com/google/recursive-version-control-system" "${dir}/signed.txt" 62 | rvcs snapshot "--additional-parents=${previousSignature}" "${dir}" | cut -d " " -f 1 | tr -d "[:space:]" > "${outFile}" 63 | rm -rf "${dir}" 64 | -------------------------------------------------------------------------------- /snapshot/identity.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snapshot 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | ) 21 | 22 | // Identity represents an identity that can sign a hash. 23 | type Identity struct { 24 | // algorithm is the name of the signing algorithm used (e.g. `ed25519`, etc). 25 | algorithm string 26 | 27 | // contents is the name of the identity. The semantics of this will vary by signature. 28 | contents string 29 | } 30 | 31 | // ParseIdentity parses the string encoding of an identity. 32 | func ParseIdentity(str string) (*Identity, error) { 33 | if len(str) == 0 { 34 | return nil, nil 35 | } 36 | if !strings.Contains(str, "::") { 37 | return nil, fmt.Errorf("malformed identity string %q", str) 38 | } 39 | parts := strings.SplitN(str, "::", 2) 40 | if len(parts) != 2 { 41 | return nil, fmt.Errorf("internal programming error in snapshot.ParseIdentity(%q)", str) 42 | } 43 | return &Identity{ 44 | algorithm: parts[0], 45 | contents: parts[1], 46 | }, nil 47 | } 48 | 49 | // Algorithm returns the name of the signing algorithm used (e.g. `ed25519`, etc). 50 | func (h *Identity) Algorithm() string { 51 | return h.algorithm 52 | } 53 | 54 | // Contents returns the identity contents. 55 | func (h *Identity) Contents() string { 56 | return h.contents 57 | } 58 | 59 | // Equal reports whether or not two hash objects are equal. 60 | func (h *Identity) Equal(other *Identity) bool { 61 | if h == nil || other == nil { 62 | return h == nil && other == nil 63 | } 64 | return h.algorithm == other.algorithm && h.contents == other.contents 65 | } 66 | 67 | // String implements the `fmt.Stringer` interface. 68 | // 69 | // The resulting value is used when serializing objects holding a hash. 70 | func (h *Identity) String() string { 71 | if h == nil { 72 | return "" 73 | } 74 | return h.algorithm + "::" + h.contents 75 | } 76 | -------------------------------------------------------------------------------- /publish/helper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package publish defines methods for publishing rvcs snapshots. 16 | package publish 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "os" 22 | "strings" 23 | 24 | "github.com/google/recursive-version-control-system/snapshot" 25 | exec "golang.org/x/sys/execabs" 26 | ) 27 | 28 | // runHelper invokes the external helper tool for the given command/namespace. 29 | // 30 | // The stdin and stderr are connected to the corresponding stdin/stderr of 31 | // the rvcs tool, while the stdout is captured. 32 | // 33 | // If the external helper tool exits with a 0 status and outputs the hash 34 | // of a snapshot, then this method returns that hash. Otherwise, this returns 35 | // an error. 36 | func runHelper(ctx context.Context, cmd, namespace string, args []string) (*snapshot.Hash, error) { 37 | helperCommand := fmt.Sprintf("rvcs-%s-%s", cmd, namespace) 38 | outFile, err := os.CreateTemp("", helperCommand+"*") 39 | if err != nil { 40 | return nil, fmt.Errorf("failure creating a temporary file for the helper command %q: %v", cmd, err) 41 | } 42 | defer os.Remove(outFile.Name()) 43 | 44 | args = append(args, outFile.Name()) 45 | helper := exec.CommandContext(ctx, helperCommand, args...) 46 | helper.Stdin = os.Stdin 47 | helper.Stdout = os.Stdout 48 | helper.Stderr = os.Stderr 49 | if err := helper.Run(); err != nil { 50 | return nil, fmt.Errorf("failure running the helper command %q: %v", helperCommand, err) 51 | } 52 | 53 | out, err := os.ReadFile(outFile.Name()) 54 | if err != nil { 55 | return nil, fmt.Errorf("failure reading the output temporary file for the helper command %q: %v", cmd, err) 56 | } 57 | 58 | h, err := snapshot.ParseHash(strings.TrimSpace(string(out))) 59 | if err != nil { 60 | return nil, fmt.Errorf("failure parsing the stdout of the helper %q: %v", helperCommand, err) 61 | } 62 | return h, nil 63 | } 64 | -------------------------------------------------------------------------------- /publish/push.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package publish defines methods for publishing rvcs snapshots. 16 | package publish 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | 22 | "github.com/google/recursive-version-control-system/config" 23 | "github.com/google/recursive-version-control-system/snapshot" 24 | "github.com/google/recursive-version-control-system/storage" 25 | ) 26 | 27 | func pushTo(ctx context.Context, m *config.Mirror, s *storage.LocalFiles, id *snapshot.Identity, h *snapshot.Hash) (*snapshot.Hash, error) { 28 | if m == nil || m.URL == nil { 29 | return h, nil 30 | } 31 | args := m.HelperFlags 32 | args = append(args, m.URL.String(), id.String(), h.String()) 33 | h, err := runHelper(ctx, "push", m.URL.Scheme, args) 34 | if err != nil { 35 | return nil, fmt.Errorf("failure invoking the push helper for %q: %v", m.URL.Scheme, err) 36 | } 37 | return h, nil 38 | } 39 | 40 | func Push(ctx context.Context, settings *config.Settings, s *storage.LocalFiles, id *snapshot.Identity, signature *snapshot.Hash) (*snapshot.Hash, error) { 41 | pushed := signature 42 | var mirrors []*config.Mirror 43 | for _, idSetting := range settings.Identities { 44 | if idSetting.Name == id.String() { 45 | for _, mirror := range idSetting.Mirrors { 46 | if !mirror.ReadOnly { 47 | mirrors = append(mirrors, mirror) 48 | } 49 | } 50 | } 51 | } 52 | mirrors = append(mirrors, settings.AdditionalMirrors...) 53 | for _, mirror := range mirrors { 54 | pushed, err := pushTo(ctx, mirror, s, id, pushed) 55 | if !pushed.Equal(signature) { 56 | if _, err := Verify(ctx, s, id, signature); err != nil { 57 | return nil, fmt.Errorf("failure verifying the upstream signature for %q at %+v: %v", id, mirror, err) 58 | } 59 | } 60 | if err != nil { 61 | return nil, fmt.Errorf("failure pushing the latest snapshot for %q to %q: %v", id, mirror.URL, err) 62 | } 63 | } 64 | if !pushed.Equal(signature) { 65 | if err := s.UpdateSignatureForIdentity(ctx, id, pushed); err != nil { 66 | return nil, fmt.Errorf("failure updating the latest snapshot for %q to %q: %v", id, pushed, err) 67 | } 68 | } 69 | return pushed, nil 70 | } 71 | -------------------------------------------------------------------------------- /command/import.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | "path/filepath" 23 | 24 | "github.com/google/recursive-version-control-system/bundle" 25 | "github.com/google/recursive-version-control-system/storage" 26 | ) 27 | 28 | const importUsage = `Usage: %s import []* 29 | 30 | Where is a local filesystem path for the bundle to import, and are one of: 31 | 32 | ` 33 | 34 | var ( 35 | importFlags = flag.NewFlagSet("import", flag.ContinueOnError) 36 | 37 | importExcludeFlag = importFlags.String( 38 | "exclude", "", 39 | "comma separated list of objects to exclude from the import") 40 | importExcludeFromFileFlag = importFlags.String( 41 | "exclude-from-file", "", 42 | "path to a file containing a newline separated list of objects to exclude from the import") 43 | 44 | importVerboseFlag = importFlags.Bool( 45 | "v", false, 46 | "verbose output. Print the hash of every object imported") 47 | ) 48 | 49 | func importCommand(ctx context.Context, s *storage.LocalFiles, cmd string, args []string) (int, error) { 50 | importFlags.Usage = func() { 51 | fmt.Fprintf(flag.CommandLine.Output(), importUsage, cmd) 52 | importFlags.PrintDefaults() 53 | } 54 | if err := importFlags.Parse(args); err != nil { 55 | return 1, nil 56 | } 57 | args = importFlags.Args() 58 | if len(args) < 1 { 59 | fmt.Fprintf(flag.CommandLine.Output(), importUsage, cmd) 60 | importFlags.PrintDefaults() 61 | return 1, nil 62 | } 63 | exclude, err := hashesFromFileAndFlag(ctx, *importExcludeFromFileFlag, *importExcludeFlag) 64 | if err != nil { 65 | return 1, err 66 | } 67 | 68 | path, err := filepath.Abs(args[0]) 69 | if err != nil { 70 | return 1, fmt.Errorf("failure resolving the absolute path of %q: %v", args[0], err) 71 | } 72 | 73 | included, err := bundle.Import(ctx, s, path, exclude) 74 | if err != nil { 75 | return 1, fmt.Errorf("failure importing the bundle: %v\n", err) 76 | } 77 | if *importVerboseFlag { 78 | for _, h := range included { 79 | fmt.Println(h.String()) 80 | } 81 | } 82 | return 0, nil 83 | } 84 | -------------------------------------------------------------------------------- /command/publish.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | 23 | "github.com/google/recursive-version-control-system/config" 24 | "github.com/google/recursive-version-control-system/publish" 25 | "github.com/google/recursive-version-control-system/snapshot" 26 | "github.com/google/recursive-version-control-system/storage" 27 | ) 28 | 29 | const publishUsage = `Usage: %s merge 30 | 31 | Where is an identity, and is one of: 32 | 33 | The hash of a known snapshot. 34 | A local file path which has previously been snapshotted. 35 | A different identity for which a snapshot has already been published. 36 | ` 37 | 38 | func publishCommand(ctx context.Context, s *storage.LocalFiles, cmd string, args []string) (int, error) { 39 | settings, err := config.Read() 40 | if err != nil { 41 | return 1, fmt.Errorf("failure reading the config settings: %v", err) 42 | } 43 | if len(args) != 2 { 44 | fmt.Fprintf(flag.CommandLine.Output(), publishUsage, cmd) 45 | return 1, nil 46 | } 47 | h, err := resolveSnapshot(ctx, s, args[0]) 48 | if err != nil { 49 | return 1, fmt.Errorf("failure resolving the snapshot hash for %q: %v", args[0], err) 50 | } 51 | id, err := snapshot.ParseIdentity(args[1]) 52 | if err != nil { 53 | return 1, fmt.Errorf("failure parsing the identity %q: %v", args[1], err) 54 | } 55 | signature, signed, err := resolveIdentitySnapshot(ctx, s, id) 56 | if err != nil { 57 | return 1, fmt.Errorf("failure resolving the previous signature for %q: %v", id, err) 58 | } 59 | if !signed.Equal(h) { 60 | // The hash has not already been signed for this identity, so 61 | // we must do that now. 62 | signature, err = publish.Sign(ctx, s, id, h, signature) 63 | if err != nil { 64 | return 1, fmt.Errorf("failure signing %q with %q: %v", h, id, err) 65 | } 66 | } 67 | signature, err = publish.Push(ctx, settings, s, id, signature) 68 | if err != nil { 69 | return 1, fmt.Errorf("failure pushing the latest signature for %q: %v", id, err) 70 | } 71 | fmt.Printf("%s %s\n", signature, id) 72 | return 0, nil 73 | } 74 | -------------------------------------------------------------------------------- /merge/base.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package merge defines methods for merging two snapshots together. 16 | package merge 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | 22 | "github.com/google/recursive-version-control-system/log" 23 | "github.com/google/recursive-version-control-system/snapshot" 24 | "github.com/google/recursive-version-control-system/storage" 25 | ) 26 | 27 | // Base identifies the "merge base" between two snapshots; the most recent 28 | // common ancestor of both. 29 | // 30 | // Ancestry for snapshots is defined as follows: 31 | // 32 | // 1. The nil snapshot is an ancestor of every snapshot 33 | // 2. Every snapshot is an ancestor of itself 34 | // 3. If a snapshot has parents, then every ancestor of one of the parents 35 | // is also an ancestor of the snapshot. 36 | // 37 | // This means there is always a common ancestor for any two given snapshots, 38 | // because the nil hash/snapshot is considered an ancestor for all snapshots. 39 | // 40 | // Regardless, this method can still return an error in cases where the 41 | // snapshot storage is incomplete and some snapshots are missing. 42 | func Base(ctx context.Context, s *storage.LocalFiles, lhs, rhs *snapshot.Hash) (*snapshot.Hash, error) { 43 | if lhs.Equal(rhs) { 44 | return lhs, nil 45 | } 46 | if lhs == nil || rhs == nil { 47 | return nil, nil 48 | } 49 | lhsLog, err := log.ReadLog(ctx, s, lhs, -1) 50 | if err != nil { 51 | return nil, fmt.Errorf("failure reading the log for %q: %v", lhs, err) 52 | } 53 | lhsAncestors := make(map[snapshot.Hash]struct{}) 54 | for _, e := range lhsLog { 55 | lhsAncestors[*e.Hash] = struct{}{} 56 | } 57 | rhsLog, err := log.ReadLog(ctx, s, rhs, -1) 58 | if err != nil { 59 | return nil, fmt.Errorf("failure reading the log for %q: %v", rhs, err) 60 | } 61 | rhsAncestors := make(map[snapshot.Hash]struct{}) 62 | for _, e := range rhsLog { 63 | rhsAncestors[*e.Hash] = struct{}{} 64 | } 65 | for len(lhsLog) > 0 && len(rhsLog) > 0 { 66 | if _, ok := rhsAncestors[*lhsLog[0].Hash]; ok { 67 | return lhsLog[0].Hash, nil 68 | } 69 | if _, ok := lhsAncestors[*rhsLog[0].Hash]; ok { 70 | return rhsLog[0].Hash, nil 71 | } 72 | lhsLog = lhsLog[1:] 73 | rhsLog = rhsLog[1:] 74 | } 75 | // There are no common ancestors other than the nil snapshot 76 | return nil, nil 77 | } 78 | -------------------------------------------------------------------------------- /snapshot/tree.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snapshot 16 | 17 | import ( 18 | "encoding/base64" 19 | "fmt" 20 | "path/filepath" 21 | "sort" 22 | "strings" 23 | ) 24 | 25 | // Path represents the filesystem path of a file. 26 | // 27 | // This can be either an absolute or relative path. 28 | type Path string 29 | 30 | // Join returns the path corresponding to joining this path with the supplied child path. 31 | func (p Path) Join(child Path) Path { 32 | return Path(filepath.Join(string(p), string(child))) 33 | } 34 | 35 | func (p Path) encode() string { 36 | return base64.RawStdEncoding.EncodeToString([]byte(p)) 37 | } 38 | 39 | func decodePath(encoded string) (Path, error) { 40 | decoded, err := base64.RawStdEncoding.DecodeString(encoded) 41 | if err != nil { 42 | return Path(""), fmt.Errorf("failure decoding the encoded path string %q: %v", encoded, err) 43 | } 44 | return Path(decoded), nil 45 | } 46 | 47 | // Tree represents the contents of a directory. 48 | // 49 | // The keys are relative paths of the directory children, and the values 50 | // are the hashes of each child's latest snapshot. 51 | type Tree map[Path]*Hash 52 | 53 | // String implements the `fmt.Stringer` interface. 54 | // 55 | // The resulting value is suitable for serialization. 56 | func (t Tree) String() string { 57 | var lines []string 58 | for p, h := range t { 59 | if h != nil { 60 | line := p.encode() + " " + h.String() 61 | lines = append(lines, line) 62 | } 63 | } 64 | sort.Strings(lines) 65 | return strings.Join(lines, "\n") 66 | } 67 | 68 | // ParseTree parses a `Tree` object from its encoded form. 69 | // 70 | // The input string must match the form returned by the `Tree.String` method. 71 | func ParseTree(encoded string) (Tree, error) { 72 | t := make(Tree) 73 | lines := strings.Split(encoded, "\n") 74 | for _, line := range lines { 75 | if len(line) == 0 { 76 | continue 77 | } 78 | parts := strings.SplitN(line, " ", 2) 79 | if len(parts) != 2 { 80 | return nil, fmt.Errorf("malformed entry %q in encoded tree %q", line, encoded) 81 | } 82 | p, err := decodePath(parts[0]) 83 | if err != nil { 84 | return nil, fmt.Errorf("failure parsing encoded path %q: %v", parts[0], err) 85 | } 86 | h, err := ParseHash(parts[1]) 87 | if err != nil { 88 | return nil, fmt.Errorf("failure parsing encoded hash %q: %v", parts[1], err) 89 | } 90 | t[p] = h 91 | } 92 | return t, nil 93 | } 94 | -------------------------------------------------------------------------------- /snapshot/tree_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snapshot 16 | 17 | import "testing" 18 | 19 | func TestParseTreeRoundTrip(t *testing.T) { 20 | testCases := []struct { 21 | Description string 22 | Serialized string 23 | Want string 24 | WantError bool 25 | }{ 26 | { 27 | Description: "empty tree", 28 | }, 29 | { 30 | Description: "missing hash", 31 | Serialized: "abcde", 32 | WantError: true, 33 | }, 34 | { 35 | Description: "malformed hash", 36 | Serialized: "abcde sha256:oops", 37 | WantError: true, 38 | }, 39 | { 40 | Description: "invalid encoded path", 41 | Serialized: ":foo:bar:baz sha256:oops", 42 | WantError: true, 43 | }, 44 | { 45 | Description: "too many hashes", 46 | Serialized: "abcd sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245 sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245", 47 | WantError: true, 48 | }, 49 | { 50 | Description: "valid encoded tree", 51 | Serialized: "abcd sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245\nefgh sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245", 52 | Want: "abcd sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245\nefgh sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245", 53 | }, 54 | { 55 | Description: "valid encoded tree with empty lines", 56 | Serialized: "abcd sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245\n\nefgh sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245\n", 57 | Want: "abcd sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245\nefgh sha256:d897f1f67a26ce92b59937134d467131537360a63b39316e5c847114a142c245", 58 | }, 59 | } 60 | for _, testCase := range testCases { 61 | parsed, err := ParseTree(testCase.Serialized) 62 | if testCase.WantError { 63 | if err == nil { 64 | t.Errorf("unexpected response for test case %q: %+v", testCase.Description, parsed) 65 | } 66 | } else if err != nil { 67 | t.Errorf("unexpected failure parsing the serialized tree %q for the test case %q: %v", testCase.Serialized, testCase.Description, err) 68 | } else if got, want := parsed.String(), testCase.Want; got != want { 69 | t.Errorf("unexpected result for tree parsing roundtrip of %q; got %q, want %q", testCase.Description, got, want) 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /command/add-mirror.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | "net/url" 23 | 24 | "github.com/google/recursive-version-control-system/config" 25 | "github.com/google/recursive-version-control-system/snapshot" 26 | "github.com/google/recursive-version-control-system/storage" 27 | ) 28 | 29 | const addMirrorUsage = `Usage: %s add-mirror []* [] 30 | 31 | Where is the optional identity to mirror (omit to apply to all identities), is the URL of the mirror, and are one of: 32 | 33 | ` 34 | 35 | var ( 36 | addMirrorFlags = flag.NewFlagSet("add-mirror", flag.ContinueOnError) 37 | addMirrorReadOnlyFlag = addMirrorFlags.Bool( 38 | "read-only", false, 39 | "if true, then snapshots are only read from the mirror, and not pushed to it") 40 | ) 41 | 42 | func addMirrorCommand(ctx context.Context, s *storage.LocalFiles, cmd string, args []string) (int, error) { 43 | addMirrorFlags.Usage = func() { 44 | fmt.Fprintf(flag.CommandLine.Output(), addMirrorUsage, cmd) 45 | addMirrorFlags.PrintDefaults() 46 | } 47 | if err := addMirrorFlags.Parse(args); err != nil { 48 | return 1, nil 49 | } 50 | args = addMirrorFlags.Args() 51 | if len(args) < 1 { 52 | fmt.Fprintf(flag.CommandLine.Output(), addMirrorUsage, cmd) 53 | addMirrorFlags.PrintDefaults() 54 | return 1, nil 55 | } 56 | var id *snapshot.Identity 57 | var err error 58 | if len(args) > 1 { 59 | id, err = snapshot.ParseIdentity(args[0]) 60 | if err != nil { 61 | return 1, fmt.Errorf("failure parsing the identity %q: %v", args[0], err) 62 | } 63 | args = args[1:] 64 | } 65 | mirrorURL, err := url.Parse(args[0]) 66 | if err != nil { 67 | return 1, fmt.Errorf("failure parsing the mirror URL %q: %v", args[0], err) 68 | } 69 | m := &config.Mirror{ 70 | URL: mirrorURL, 71 | ReadOnly: *addMirrorReadOnlyFlag, 72 | } 73 | settings, err := config.Read() 74 | if err != nil { 75 | return 1, fmt.Errorf("failure reading the existing config settings: %v", err) 76 | } 77 | if id == nil { 78 | settings = settings.WithAdditionalMirror(m) 79 | } else { 80 | settings = settings.WithMirrorForIdentity(id.String(), m) 81 | } 82 | if err := settings.Write(); err != nil { 83 | return 1, fmt.Errorf("failure writing the updated config settings: %v", err) 84 | } 85 | return 0, nil 86 | } 87 | -------------------------------------------------------------------------------- /command/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | 23 | "github.com/google/recursive-version-control-system/log" 24 | "github.com/google/recursive-version-control-system/storage" 25 | ) 26 | 27 | const logUsage = `Usage: %s log []* 28 | 29 | Where is one of: 30 | 31 | The hash of a known snapshot. 32 | A local file path which has previously been snapshotted. 33 | 34 | ` 35 | 36 | var ( 37 | logFlags = flag.NewFlagSet("log", flag.ContinueOnError) 38 | 39 | logShort bool 40 | logDepthFlag = logFlags.Int( 41 | "depth", -1, 42 | "maximum depth of the history to traverse. If less than 0, then there is no limit.") 43 | ) 44 | 45 | func init() { 46 | logFlags.BoolVar(&logShort, "short", false, 47 | "print short output, consisting of just the hash for each snapshot") 48 | logFlags.BoolVar(&logShort, "s", false, 49 | "print short output, consisting of just the hash for each snapshot") 50 | } 51 | 52 | func logCommand(ctx context.Context, s *storage.LocalFiles, cmd string, args []string) (int, error) { 53 | logFlags.Usage = func() { 54 | fmt.Fprintf(flag.CommandLine.Output(), logUsage, cmd) 55 | logFlags.PrintDefaults() 56 | } 57 | if err := logFlags.Parse(args); err != nil { 58 | return 1, nil 59 | } 60 | args = logFlags.Args() 61 | if len(args) != 1 { 62 | fmt.Fprintf(flag.CommandLine.Output(), logUsage, cmd) 63 | logFlags.PrintDefaults() 64 | return 1, nil 65 | } 66 | h, err := resolveSnapshot(ctx, s, args[0]) 67 | if err != nil { 68 | return 1, fmt.Errorf("failure resolving the snapshot hash for %q: %v", args[0], err) 69 | } 70 | entries, err := log.ReadLog(ctx, s, h, *logDepthFlag) 71 | if err != nil { 72 | return 1, fmt.Errorf("failure reading the log for %q: %v", args[0], err) 73 | } 74 | if logShort { 75 | for _, e := range entries { 76 | fmt.Println(e.Hash) 77 | } 78 | return 0, nil 79 | } 80 | summaries, err := log.SummarizeLog(ctx, s, entries) 81 | if err != nil { 82 | return 1, fmt.Errorf("failure summarizing log entries for %q: %v", args[0], err) 83 | } 84 | for i, e := range entries { 85 | if i > 0 { 86 | // Separate log entries for each change with a newline to make the output more readable. 87 | fmt.Println() 88 | } 89 | summary, ok := summaries[*e.Hash] 90 | if !ok { 91 | return 1, fmt.Errorf("internal error reading log summaries: entry %q is missing", e.Hash) 92 | } 93 | for _, line := range summary { 94 | fmt.Println(line) 95 | } 96 | } 97 | return 0, nil 98 | } 99 | -------------------------------------------------------------------------------- /log/log_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package log 16 | 17 | import ( 18 | "context" 19 | "os" 20 | "path/filepath" 21 | "testing" 22 | 23 | "github.com/google/recursive-version-control-system/snapshot" 24 | "github.com/google/recursive-version-control-system/storage" 25 | ) 26 | 27 | func TestLog(t *testing.T) { 28 | dir := t.TempDir() 29 | abs, err := filepath.Abs(dir) 30 | if err != nil { 31 | t.Fatalf("failure resolving the absolute path of %q: %v", dir, err) 32 | } 33 | dir = abs 34 | archiveDir := filepath.Join(dir, "archive") 35 | s := &storage.LocalFiles{ 36 | ArchiveDir: archiveDir, 37 | } 38 | workingDir := filepath.Join(dir, "working-dir") 39 | if err := os.MkdirAll(workingDir, os.FileMode(0700)); err != nil { 40 | t.Fatalf("failure creating the temporary working dir: %v", err) 41 | } 42 | ctx, cancel := context.WithCancel(context.Background()) 43 | defer cancel() 44 | 45 | // Take an initial snapshot 46 | h1, _, err := snapshot.Current(ctx, s, snapshot.Path(workingDir)) 47 | if err != nil { 48 | t.Fatalf("failure creating the initial (empty) snapshot: %v", err) 49 | } 50 | 51 | // Write a file and take another snapshot 52 | file := filepath.Join(workingDir, "example.txt") 53 | if err := os.WriteFile(file, []byte("Hello, World!"), 0700); err != nil { 54 | t.Fatalf("failure creating the example file to snapshot: %v", err) 55 | } 56 | h2, _, err := snapshot.Current(ctx, s, snapshot.Path(workingDir)) 57 | if err != nil { 58 | t.Fatalf("failure creating the updated snapshot: %v", err) 59 | } 60 | 61 | if entries, err := ReadLog(ctx, s, h2, 0); err != nil { 62 | t.Errorf("failure reading the log with a depth of 0: %v", err) 63 | } else if len(entries) != 0 { 64 | t.Errorf("unexpected log entries with a depth of 0: %+v", entries) 65 | } 66 | 67 | if entries, err := ReadLog(ctx, s, h2, 1); err != nil { 68 | t.Errorf("failure reading the log with a depth of 1: %v", err) 69 | } else if len(entries) != 1 { 70 | t.Errorf("unexpected log entries with a depth of 1: %+v", entries) 71 | } else if !entries[0].Hash.Equal(h2) { 72 | t.Errorf("unexpected log entry hash with a depth of 1: %+v", entries[0].Hash) 73 | } 74 | 75 | if entries, err := ReadLog(ctx, s, h2, -1); err != nil { 76 | t.Errorf("failure reading the log with a depth of -1: %v", err) 77 | } else if len(entries) != 2 { 78 | t.Errorf("unexpected log entries with a depth of -1: %+v", entries) 79 | } else if !entries[0].Hash.Equal(h2) { 80 | t.Errorf("unexpected first log entry hash with a depth of -1: %+v", entries[0].Hash) 81 | } else if !entries[1].Hash.Equal(h1) { 82 | t.Errorf("unexpected second log entry hash with a depth of -1: %+v", entries[0].Hash) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /command/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | "strings" 25 | 26 | "github.com/google/recursive-version-control-system/snapshot" 27 | "github.com/google/recursive-version-control-system/storage" 28 | ) 29 | 30 | const snapshotUsage = `Usage: %s snapshot []* 31 | 32 | Where is a local filesystem path, and are one of: 33 | 34 | ` 35 | 36 | var ( 37 | snapshotFlags = flag.NewFlagSet("snapshot", flag.ContinueOnError) 38 | 39 | snapshotAdditionalParentsFlag = snapshotFlags.String( 40 | "additional-parents", "", 41 | "comma separated list of additional parents for the generated snapshot") 42 | ) 43 | 44 | func snapshotCommand(ctx context.Context, s *storage.LocalFiles, cmd string, args []string) (int, error) { 45 | snapshotFlags.Usage = func() { 46 | fmt.Fprintf(flag.CommandLine.Output(), snapshotUsage, cmd) 47 | snapshotFlags.PrintDefaults() 48 | } 49 | if err := snapshotFlags.Parse(args); err != nil { 50 | return 1, nil 51 | } 52 | args = snapshotFlags.Args() 53 | 54 | var additionalParents []*snapshot.Hash 55 | for _, parent := range strings.Split(*snapshotAdditionalParentsFlag, ",") { 56 | parentHash, err := resolveSnapshot(ctx, s, parent) 57 | if err != nil { 58 | return 1, fmt.Errorf("failure resolving the additional parent %q: %v", parent, err) 59 | } 60 | if parentHash != nil { 61 | additionalParents = append(additionalParents, parentHash) 62 | } 63 | } 64 | 65 | var path string 66 | if len(args) > 0 { 67 | path = args[0] 68 | } else { 69 | wd, err := os.Getwd() 70 | if err != nil { 71 | return 1, fmt.Errorf("failure determining the current working directory: %v\n", err) 72 | } 73 | path = wd 74 | } 75 | abs, err := filepath.Abs(path) 76 | if err != nil { 77 | return 1, fmt.Errorf("failure resolving the absolute path of %q: %v", path, err) 78 | } 79 | path = abs 80 | 81 | h, f, err := snapshot.Current(ctx, s, snapshot.Path(path)) 82 | if err != nil { 83 | return 1, fmt.Errorf("failure snapshotting the directory %q: %v\n", path, err) 84 | } else if h == nil || f == nil { 85 | fmt.Printf("Did not generate a snapshot as %q does not exist\n", path) 86 | return 1, nil 87 | } 88 | if len(additionalParents) > 0 { 89 | f.Parents = append(f.Parents, additionalParents...) 90 | h, err = s.StoreSnapshot(ctx, snapshot.Path(path), f) 91 | if err != nil { 92 | return 1, fmt.Errorf("failure updating the snapshot of %q to include the additional parents %v: %v", path, additionalParents, err) 93 | } 94 | } 95 | 96 | fmt.Printf("%s %s\n", h, path) 97 | return 0, nil 98 | } 99 | -------------------------------------------------------------------------------- /snapshot/hash.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snapshot 16 | 17 | import ( 18 | "crypto/sha256" 19 | "encoding/hex" 20 | "fmt" 21 | "hash" 22 | "io" 23 | "strings" 24 | ) 25 | 26 | var ( 27 | defaultHashFunction = "sha256" 28 | supportedHashFunctions = map[string]func() hash.Hash{ 29 | "sha256": sha256.New, 30 | } 31 | ) 32 | 33 | // Hash represents a hash/fingerprint of a blob. 34 | type Hash struct { 35 | // function is the name of the hash function used (e.g. `sha256`, etc). 36 | function string 37 | 38 | // hexContents is the hash value serialized as a hexadecimal string. 39 | hexContents string 40 | } 41 | 42 | // NewHash constructs a new hash by calculating the checksum of the provided reader. 43 | // 44 | // The caller is responsible for closing the reader. 45 | func NewHash(reader io.Reader) (*Hash, error) { 46 | sum := supportedHashFunctions[defaultHashFunction]() 47 | if _, err := io.Copy(sum, reader); err != nil { 48 | return nil, fmt.Errorf("failure hashing an object: %v", err) 49 | } 50 | return &Hash{ 51 | function: defaultHashFunction, 52 | hexContents: fmt.Sprintf("%x", sum.Sum(nil)), 53 | }, nil 54 | } 55 | 56 | // ParseHash parses the string encoding of a hash. 57 | func ParseHash(str string) (*Hash, error) { 58 | if len(str) == 0 { 59 | return nil, nil 60 | } 61 | if !strings.Contains(str, ":") { 62 | return nil, fmt.Errorf("malformed hash string %q", str) 63 | } 64 | parts := strings.SplitN(str, ":", 2) 65 | if len(parts) != 2 { 66 | return nil, fmt.Errorf("internal programming error in snapshot.ParseHash(%q)", str) 67 | } 68 | if _, ok := supportedHashFunctions[parts[0]]; !ok { 69 | return nil, fmt.Errorf("unsupported hash function %q", parts[0]) 70 | } 71 | if _, err := hex.DecodeString(parts[1]); err != nil { 72 | return nil, fmt.Errorf("malformed hash contents %q: %v", parts[1], err) 73 | } 74 | return &Hash{ 75 | function: parts[0], 76 | hexContents: parts[1], 77 | }, nil 78 | } 79 | 80 | // Function returns the name of the hash function used (e.g. `sha256`, etc). 81 | func (h *Hash) Function() string { 82 | return h.function 83 | } 84 | 85 | // HexContents returns the hash value serialized as a hexadecimal string. 86 | func (h *Hash) HexContents() string { 87 | return h.hexContents 88 | } 89 | 90 | // Equal reports whether or not two hash objects are equal. 91 | func (h *Hash) Equal(other *Hash) bool { 92 | if h == nil || other == nil { 93 | return h == nil && other == nil 94 | } 95 | return h.function == other.function && h.hexContents == other.hexContents 96 | } 97 | 98 | // String implements the `fmt.Stringer` interface. 99 | // 100 | // The resulting value is used when serializing objects holding a hash. 101 | func (h *Hash) String() string { 102 | if h == nil { 103 | return "" 104 | } 105 | return h.function + ":" + h.hexContents 106 | } 107 | -------------------------------------------------------------------------------- /extensions/rvcs-verify-ssh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Copyright 2022 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Usage: 18 | # 19 | # rvcs merge ssh:: 20 | # 21 | # ... where is the base64-encoded contents of one of 22 | # the "id_....pub" files under your "~/.ssh" directory. 23 | # 24 | # Alternatively, you can invoke this helper directly using 25 | # 26 | # rvcs-verify-ssh ssh:: 27 | # 28 | # If the signature can be verified, the tool will write to the output file 29 | # the hash of the snapshot that was signed by it. 30 | 31 | pubkey="${1#ssh::}" 32 | hash="${2}" 33 | outFile="${3}" 34 | 35 | if [ -z "${pubkey}" ]; then 36 | echo "No public key provided." >&2 37 | echo "Usage:" >&2 38 | echo " rvcs-verify-ssh ssh:: " >&2 39 | exit 1 40 | fi 41 | 42 | if [ -z "${hash}" ]; then 43 | echo "No signature hashcode provided." >&2 44 | echo "Usage:" >&2 45 | echo " rvcs-verify-ssh ssh:: " >&2 46 | exit 1 47 | fi 48 | 49 | if [ -z "${outFile}" ]; then 50 | echo "No output file provided." >&2 51 | echo "Usage:" >&2 52 | echo " rvcs-verify-ssh ssh:: " >&2 53 | exit 1 54 | fi 55 | 56 | keyfile="$(grep "${pubkey}" ~/.ssh/*.pub | cut -d ":" -f 1)" 57 | if [ -z "${keyfile}" ]; then 58 | echo "Could not find the public key file for ${pubkey} under ~/.ssh/" >&2 59 | exit 1 60 | fi 61 | 62 | signerEmail="$(cat "${keyfile}" | cut -d " " -f 3)" 63 | if [ -z "${signerEmail}" ]; then 64 | echo "Could not parse the public key file at '${keyfile}'" >&2 65 | exit 1 66 | fi 67 | allowedSigner="${signerEmail} $(cat "${keyfile}" | cut -d " " -f 1) $(cat "${keyfile}" | cut -d " " -f 2)" 68 | 69 | dir=$(mktemp -d) 70 | function fail() { 71 | echo "${1}" >&2 72 | rm -rf "${dir}" 73 | exit 1 74 | } 75 | 76 | # Fetch the signature and verify that it includes all the required contents... 77 | rvcs merge "${hash}" "${dir}/signature" >&2 78 | for PARENT in `rvcs log -depth 2 -s "${dir}/signature" | tail -n +2`; do 79 | grep -q "${PARENT}" "${dir}/signature/signed.txt" || fail " ${PARENT} is not in the signature." 80 | done 81 | for FILE in `ls "${dir}/signature" | grep -v "signed.txt"`; do 82 | fileHash="$(rvcs log -s -depth 1 "${dir}/signature/${FILE}" | tr -d "[:space:]")" 83 | grep -q "${fileHash}" "${dir}/signature/signed.txt" || fail "${FILE} hash '${fileHash}' is not in the signature." 84 | done 85 | 86 | # Verify the signature matches the key... 87 | ssh-keygen -Y verify -f <(echo "${allowedSigner}") -I "${signerEmail}" -n "github.com/google/recursive-version-control-system" -s "${dir}/signature/signed.txt.sig" < "${dir}/signature/signed.txt" >&2 || fail "Failed to verify that the signature matches the key." 88 | 89 | # If we got this far then the signature is fully verified and we should 90 | # print the hash of the signed snapshot. 91 | cat "${dir}/signature/signed.txt" | head -n 1 | tr -d "[:space:]" > "${outFile}" 92 | rm -rf "${dir}" 93 | -------------------------------------------------------------------------------- /snapshot/file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snapshot 16 | 17 | import "testing" 18 | 19 | func TestParseFileRoundTrip(t *testing.T) { 20 | testCases := []struct { 21 | Description string 22 | Serialized string 23 | Want string 24 | WantError bool 25 | }{ 26 | { 27 | Description: "empty file string", 28 | }, 29 | { 30 | Description: "missing contents", 31 | Serialized: "drwxr-x---", 32 | WantError: true, 33 | }, 34 | { 35 | Description: "empty contents", 36 | Serialized: "drwxr-x---\n", 37 | WantError: true, 38 | }, 39 | { 40 | Description: "empty directory", 41 | Serialized: "drwxr-x---\nsha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 42 | Want: "drwxr-x---\nsha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 43 | }, 44 | { 45 | Description: "empty lines in parents", 46 | Serialized: "drwxr-x---\nsha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\n\n", 47 | Want: "drwxr-x---\nsha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 48 | }, 49 | } 50 | for _, testCase := range testCases { 51 | parsed, err := ParseFile(testCase.Serialized) 52 | if testCase.WantError { 53 | if err == nil { 54 | t.Errorf("unexpected response for test case %q: %+v", testCase.Description, parsed) 55 | } 56 | } else if err != nil { 57 | t.Errorf("unexpected failure parsing the serialized file %q for the test case %q: %v", testCase.Serialized, testCase.Description, err) 58 | } else if got, want := parsed.String(), testCase.Want; got != want { 59 | t.Errorf("unexpected result for file parsing roundtrip of %q; got %q, want %q", testCase.Description, got, want) 60 | } 61 | } 62 | } 63 | 64 | func TestFilePermissions(t *testing.T) { 65 | testCases := []struct { 66 | Description string 67 | File *File 68 | Want string 69 | }{ 70 | { 71 | Description: "nil file", 72 | Want: "-rwx------", 73 | }, 74 | { 75 | Description: "empty mode", 76 | File: &File{ 77 | Mode: "", 78 | }, 79 | Want: "-rwx------", 80 | }, 81 | { 82 | Description: "permissions only", 83 | File: &File{ 84 | Mode: "rw-rw-rw-", 85 | }, 86 | Want: "-rw-rw-rw-", 87 | }, 88 | { 89 | Description: "regular file", 90 | File: &File{ 91 | Mode: "-r-xr--r--", 92 | }, 93 | Want: "-r-xr--r--", 94 | }, 95 | { 96 | Description: "directory", 97 | File: &File{ 98 | Mode: "dr-xr-xr--", 99 | }, 100 | Want: "-r-xr-xr--", 101 | }, 102 | { 103 | Description: "multiple file type descriptors", 104 | File: &File{ 105 | Mode: "dLTrwxr-xr-x", 106 | }, 107 | Want: "-rwxr-xr-x", 108 | }, 109 | } 110 | for _, testCase := range testCases { 111 | if got, want := testCase.File.Permissions().String(), testCase.Want; got != want { 112 | t.Errorf("unexpected permissions for %q: got %q, want %q", testCase.Description, got, want) 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /publish/pull.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package publish defines methods for publishing rvcs snapshots. 16 | package publish 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | 22 | "github.com/google/recursive-version-control-system/config" 23 | "github.com/google/recursive-version-control-system/snapshot" 24 | "github.com/google/recursive-version-control-system/storage" 25 | ) 26 | 27 | func pullFrom(ctx context.Context, m *config.Mirror, s *storage.LocalFiles, id *snapshot.Identity, prev *snapshot.Hash) (*snapshot.Hash, error) { 28 | if m == nil || m.URL == nil { 29 | return prev, nil 30 | } 31 | args := m.HelperFlags 32 | args = append(args, m.URL.String(), id.String(), prev.String()) 33 | h, err := runHelper(ctx, "pull", m.URL.Scheme, args) 34 | if err != nil { 35 | return nil, fmt.Errorf("failure invoking the pull helper for %q: %v", m.URL.Scheme, err) 36 | } 37 | return h, nil 38 | } 39 | 40 | func pullFromAndVerify(ctx context.Context, m *config.Mirror, s *storage.LocalFiles, id *snapshot.Identity, prevSignature *snapshot.Hash, prevSigned *snapshot.Hash) (signature *snapshot.Hash, signed *snapshot.Hash, err error) { 41 | signature, err = pullFrom(ctx, m, s, id, prevSignature) 42 | if err != nil { 43 | return nil, nil, fmt.Errorf("failure pulling the latest snapshot for %q from %q: %v", id, m.URL, err) 44 | } 45 | if signature == nil { 46 | // This identity is not known on that mirror 47 | return nil, nil, nil 48 | } 49 | if signature.Equal(prevSignature) { 50 | return prevSignature, prevSigned, nil 51 | } 52 | signed, err = Verify(ctx, s, id, signature) 53 | if err != nil { 54 | return nil, nil, fmt.Errorf("failure verifying the signature for %q from %q: %v", id, m.URL, err) 55 | } 56 | return signature, signed, nil 57 | } 58 | 59 | func Pull(ctx context.Context, settings *config.Settings, s *storage.LocalFiles, id *snapshot.Identity) (signature *snapshot.Hash, signed *snapshot.Hash, err error) { 60 | signature, err = s.LatestSignatureForIdentity(ctx, id) 61 | if err != nil { 62 | return nil, nil, fmt.Errorf("failure looking up the previous signature for %q: %v", id, err) 63 | } 64 | signed, err = Verify(ctx, s, id, signature) 65 | if err != nil { 66 | return nil, nil, fmt.Errorf("failure verifying the previous signature for %q: %v", id, err) 67 | } 68 | for _, idSetting := range settings.Identities { 69 | if idSetting.Name == id.String() { 70 | for _, mirror := range idSetting.Mirrors { 71 | signature, signed, err = pullFromAndVerify(ctx, mirror, s, id, signature, signed) 72 | if err != nil { 73 | return nil, nil, fmt.Errorf("failure pulling the latest snapshot for %q from %+v: %v", id, mirror, err) 74 | } 75 | } 76 | } 77 | } 78 | for _, mirror := range settings.AdditionalMirrors { 79 | signature, signed, err = pullFromAndVerify(ctx, mirror, s, id, signature, signed) 80 | if err != nil { 81 | return nil, nil, fmt.Errorf("failure pulling the latest snapshot for %q from %+v: %v", id, mirror, err) 82 | } 83 | } 84 | if signature == nil { 85 | return nil, nil, nil 86 | } 87 | if err := s.UpdateSignatureForIdentity(ctx, id, signature); err != nil { 88 | return nil, nil, fmt.Errorf("failure updating the latest snapshot for %q to %q: %v", id, signature, err) 89 | } 90 | return signature, signed, nil 91 | } 92 | -------------------------------------------------------------------------------- /bundle/bundle_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package bundle defines methods for bundling snapshots together so they can be imported and/or exported. 16 | package bundle 17 | 18 | import ( 19 | "context" 20 | "os" 21 | "path/filepath" 22 | "testing" 23 | 24 | "github.com/google/recursive-version-control-system/snapshot" 25 | "github.com/google/recursive-version-control-system/storage" 26 | ) 27 | 28 | func TestRoundtrip(t *testing.T) { 29 | archiveDir := filepath.Join(t.TempDir(), "archive") 30 | s := &storage.LocalFiles{archiveDir} 31 | 32 | workDir := filepath.Join(t.TempDir(), "workDir") 33 | if err := os.MkdirAll(workDir, os.FileMode(0700)); err != nil { 34 | t.Fatalf("failure creating the work dir: %v", err) 35 | } 36 | 37 | // Take an initial snapshot 38 | file := filepath.Join(workDir, "hello.txt") 39 | p := snapshot.Path(file) 40 | if err := os.WriteFile(file, []byte("Hello, World!"), 0700); err != nil { 41 | t.Fatalf("failure creating the example file to snapshot: %v", err) 42 | } 43 | h1, f1, err := snapshot.Current(context.Background(), s, p) 44 | if err != nil { 45 | t.Fatalf("failure creating the initial snapshot for the file: %v", err) 46 | } 47 | if err := os.WriteFile(file, []byte("Goodbye, World!"), 0700); err != nil { 48 | t.Fatalf("failure updating the example file to snapshot: %v", err) 49 | } 50 | h2, f2, err := snapshot.Current(context.Background(), s, p) 51 | if err != nil { 52 | t.Fatalf("failure creating the updated snapshot for the file: %v", err) 53 | } 54 | 55 | bundleFile := filepath.Join(t.TempDir(), "bundle.zip") 56 | included, err := Export(context.Background(), s, bundleFile, []*snapshot.Hash{h2}, nil, nil, true) 57 | if err != nil { 58 | t.Fatalf("failure creating the bundle %q: %v", bundleFile, err) 59 | } 60 | 61 | includedMap := make(map[snapshot.Hash]struct{}) 62 | for _, i := range included { 63 | includedMap[*i] = struct{}{} 64 | } 65 | if _, ok := includedMap[*h2]; !ok { 66 | t.Errorf("bundle export does not include the specified hash %q: got %v", h2, included) 67 | } 68 | if _, ok := includedMap[*h1]; !ok { 69 | t.Errorf("bundle export does not include the parent of the specified hash %q: got %v", h1, included) 70 | } 71 | 72 | archive2Dir := filepath.Join(t.TempDir(), "archive2") 73 | s2 := &storage.LocalFiles{archive2Dir} 74 | imported, err := Import(context.Background(), s2, bundleFile, nil) 75 | if err != nil { 76 | t.Fatalf("failure importing the bundle %q: %v", bundleFile, err) 77 | } 78 | importedMap := make(map[snapshot.Hash]struct{}) 79 | for _, i := range imported { 80 | importedMap[*i] = struct{}{} 81 | } 82 | if _, ok := importedMap[*h2]; !ok { 83 | t.Errorf("bundle import does not include the specified hash %q: got %v", h2, imported) 84 | } 85 | if _, ok := importedMap[*h1]; !ok { 86 | t.Errorf("bundle import does not include the parent of the specified hash %q: got %v", h1, imported) 87 | } 88 | if f1v2, err := s2.ReadSnapshot(context.Background(), h1); err != nil { 89 | t.Errorf("error finding imported snapshot %q: %v", h1, err) 90 | } else if got, want := f1v2.String(), f1.String(); got != want { 91 | t.Errorf("unexpected contents for snapshot %q: got %q, want %q", h1, got, want) 92 | } 93 | if f2v2, err := s2.ReadSnapshot(context.Background(), h2); err != nil { 94 | t.Errorf("error finding imported snapshot %q: %v", h2, err) 95 | } else if got, want := f2v2.String(), f2.String(); got != want { 96 | t.Errorf("unexpected contents for snapshot %q: got %q, want %q", h1, got, want) 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /extensions/rvcs-push-file: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | 3 | # Copyright 2022 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | path="${1#file://}" 18 | identity="${2}" 19 | hash="${3}" 20 | outFile="${4}" 21 | 22 | if [ -z "${path}" ]; then 23 | echo "No file path provided." >&2 24 | echo "Usage:" >&2 25 | echo " rvcs-push-file file:// " >&2 26 | exit 1 27 | fi 28 | 29 | if [ -z "${identity}" ]; then 30 | echo "No identity provided." >&2 31 | echo "Usage:" >&2 32 | echo " rvcs-push-file file:// " >&2 33 | exit 1 34 | fi 35 | 36 | if [ -z "${hash}" ]; then 37 | echo "No snapshot hashcode provided." >&2 38 | echo "Usage:" >&2 39 | echo " rvcs-push-file file:// " >&2 40 | exit 1 41 | fi 42 | 43 | if [ -z "${outFile}" ]; then 44 | echo "No output file provided." >&2 45 | echo "Usage:" >&2 46 | echo " rvcs-push-file file:// " >&2 47 | exit 1 48 | fi 49 | 50 | mkdir -p "${path}" 51 | bundleName="$(echo "${identity}" | shasum -a 256 | cut -d ' ' -f 1)-bundle.zip" 52 | bundlePath="${path}/${bundleName}" 53 | 54 | tempDir=$(mktemp -d) 55 | touch "${tempDir}/exclude.txt" 56 | touch "${tempDir}/previous.txt" 57 | if [ -f "${bundlePath}" ]; then 58 | # An older version of the signature was previously pushed. 59 | # 60 | # Save that one off to a new location and then make sure 61 | # the new bundle only includes the incremental changes since 62 | # the previous bundle(s). 63 | 64 | # ... but first, double check whether or not anything has changed... 65 | prevSig="$(unzip -p "${bundlePath}" "metadata/signature" | tr -d "[:space:]")" 66 | if [ "${prevSig}" == "${hash}" ]; then 67 | echo -n "${hash}" 68 | exit 0 69 | fi 70 | 71 | # First, move the previous bundle to its new location... 72 | highest="0" 73 | prefix="${bundlePath%.zip}-" 74 | for prev in $(unzip -p "${bundlePath}" "metadata/previous"); do 75 | suffix="${prev#${prefix}}" 76 | count="${suffix%.zip}" 77 | if [[ "${highest}" < "${count}" ]]; then 78 | highest="${count}" 79 | fi 80 | done 81 | replacement="${prefix}$(expr "${highest}" "+" "1").zip" 82 | mv "${bundlePath}" "${replacement}" 83 | 84 | # Next, update the excludes to include all the objects in the previous 85 | # bundle... 86 | additionalExcludeFiles=$(unzip -Z1 "${replacement}" objects/*) 87 | for additionalExclude in ${additionalExcludeFiles}; do 88 | hashAlgorithm="$(echo "${additionalExclude}" | cut -d '/' -f 2)" 89 | hashContents="$(echo "${additionalExclude}" | cut -d '/' -f 3)$(echo "${additionalExclude}" | cut -d '/' -f 4)$(echo "${additionalExclude}" | cut -d '/' -f 5)" 90 | echo "${hashAlgorithm}:${hashContents}" >> "${tempDir}/exclude.txt" 91 | done 92 | 93 | # Then, update the previous to start with this previous bundle... 94 | echo "${replacement}" > "${tempDir}/previous.txt" 95 | 96 | # Finally, append to the excludes and previous lists everything from 97 | # the previous bundle... 98 | unzip -p "${replacement}" "metadata/exclude" >> "${tempDir}/exclude.txt" 99 | unzip -p "${replacement}" "metadata/previous" >> "${tempDir}/previous.txt" 100 | fi 101 | 102 | rvcs export --exclude-from-file "${tempDir}/exclude.txt" --include-parents --snapshots="${hash}" --metadata="signature=${hash}" --metadata-from-files="exclude=${tempDir}/exclude.txt,previous=${tempDir}/previous.txt" "${bundlePath}" >&2 103 | rm -rf "${tempDir}" 104 | echo -n "${hash}" > "${outFile}" 105 | -------------------------------------------------------------------------------- /merge/helper.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package merge defines methods for merging two snapshots together. 16 | package merge 17 | 18 | import ( 19 | "bytes" 20 | "context" 21 | "encoding/json" 22 | "fmt" 23 | "os" 24 | "os/exec" 25 | "path/filepath" 26 | "time" 27 | 28 | "github.com/google/recursive-version-control-system/snapshot" 29 | "github.com/google/recursive-version-control-system/storage" 30 | ) 31 | 32 | const ( 33 | HelperEnvironmentVariable = "RVCS_MERGE_HELPER_COMMAND" 34 | HelperArgsEnvironmentVariable = "RVCS_MERGE_HELPER_ARGS" 35 | ) 36 | 37 | func mergeWithHelper(ctx context.Context, s *storage.LocalFiles, p snapshot.Path, mode string, base, src, dest *snapshot.Hash) (*snapshot.Hash, error) { 38 | helperCmd := os.Getenv(HelperEnvironmentVariable) 39 | helperArgs := os.Getenv(HelperArgsEnvironmentVariable) 40 | if len(helperCmd) == 0 { 41 | helperCmd = "diff3" 42 | if len(helperArgs) == 0 { 43 | helperArgs = "[\"-m\"]" 44 | } 45 | } 46 | var args []string 47 | if err := json.Unmarshal([]byte(helperArgs), &args); err != nil { 48 | return nil, fmt.Errorf("failure parsing the helper args %q: %v", helperArgs, err) 49 | } 50 | tmpDir, err := os.MkdirTemp("", fmt.Sprintf("rvcs-merge-helper-%q", helperCmd)) 51 | if err != nil { 52 | return nil, fmt.Errorf("failure generating the temporary working directory for the merge helper %q: %v", helperCmd, err) 53 | } 54 | defer os.RemoveAll(tmpDir) 55 | 56 | tmpPath := snapshot.Path(tmpDir) 57 | srcPath := tmpPath.Join(snapshot.Path("src")).Join(p) 58 | if err := Checkout(ctx, s, src, srcPath); err != nil { 59 | return nil, fmt.Errorf("failure checking out %q to a temporary path for the merge helper: %v", src, err) 60 | } 61 | basePath := tmpPath.Join(snapshot.Path("base")).Join(p) 62 | if base == nil { 63 | // Simply create an empty file to serve as the merge base. 64 | // 65 | // With the default merge helper of `diff3`, this will always 66 | // result in unresolvable conflicts, but the user might have 67 | // configured a more intelligent merge helper that knows how 68 | // to resolve some cases of this, so we give it a chance to 69 | // try. 70 | parent := filepath.Dir(string(basePath)) 71 | if err := os.MkdirAll(parent, os.FileMode(0700)); err != nil { 72 | return nil, fmt.Errorf("failure ensuring the parent directory of %q exists: %v", basePath, err) 73 | } 74 | if _, err := os.Create(string(basePath)); err != nil { 75 | return nil, fmt.Errorf("failure creating an empty temporary file to serve as the merge base for the merge helper: %v", err) 76 | } 77 | } else if err := Checkout(ctx, s, base, basePath); err != nil { 78 | return nil, fmt.Errorf("failure checking out %q to a temporary path for the merge helper: %v", base, err) 79 | } 80 | destPath := tmpPath.Join(snapshot.Path("dest")).Join(p) 81 | if err := Checkout(ctx, s, dest, destPath); err != nil { 82 | return nil, fmt.Errorf("failure checking out %q to a temporary path for the merge helper: %v", dest, err) 83 | } 84 | args = append(args, string(srcPath), string(basePath), string(destPath)) 85 | 86 | helperCtx, cancel := context.WithTimeout(ctx, time.Second) 87 | defer cancel() 88 | 89 | out, err := exec.CommandContext(helperCtx, helperCmd, args...).Output() 90 | if err != nil { 91 | return nil, fmt.Errorf("merge helper %q failed: %v", helperCmd, err) 92 | } 93 | contentsHash, err := s.StoreObject(ctx, int64(len(out)), bytes.NewReader(out)) 94 | if err != nil { 95 | return nil, fmt.Errorf("failure hashing the merged contents: %v", err) 96 | } 97 | mergedFile := &snapshot.File{ 98 | Mode: mode, 99 | Contents: contentsHash, 100 | Parents: []*snapshot.Hash{src, dest}, 101 | } 102 | fileBytes := []byte(mergedFile.String()) 103 | h, err := s.StoreObject(ctx, int64(len(fileBytes)), bytes.NewReader(fileBytes)) 104 | if err != nil { 105 | return nil, fmt.Errorf("failure storing the merged snapshot: %v", err) 106 | } 107 | return h, nil 108 | } 109 | -------------------------------------------------------------------------------- /snapshot/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package snapshot 16 | 17 | import ( 18 | "fmt" 19 | "io/fs" 20 | "os" 21 | "strings" 22 | ) 23 | 24 | // File is the top-level object in a snapshot. 25 | // 26 | // File encodes the entire, transitive history of a file. If the file is 27 | // a directory, then this history also includes the histories for all 28 | // of the children of that directory. 29 | type File struct { 30 | // Mode is the string representation of a Posix-style file mode. 31 | // 32 | // It should be of the form []+. 33 | // 34 | // is a single character indicating the type of the 35 | // file, such as `d` for a directory or `L` for a symbolic link, etc. 36 | // 37 | // is a sequence of 9 characters representing the 38 | // Unix permission bits. 39 | Mode string 40 | 41 | // Contents is the hash of the contents for the snapshotted file. 42 | // 43 | // If the file is a directory (the mode line starts with `d`), then 44 | // this will be the hash of a `Tree` object. 45 | // 46 | // If the file is a symbolic link (the mode line starts with a `L`), 47 | // then this will be the hash of another `File` object, unless the 48 | // link is broken in which case the contents will be nil. 49 | // 50 | // In all other cases, the contents is a hash of the sequence of 51 | // bytes read from the file. 52 | Contents *Hash 53 | 54 | // Parents stores the hashes for the previous snapshots that 55 | // immediately preceeded this one. 56 | Parents []*Hash 57 | } 58 | 59 | // IsDir reports whether or not the file is the snapshot of a directory. 60 | func (f *File) IsDir() bool { 61 | if f == nil { 62 | return false 63 | } 64 | return strings.HasPrefix(f.Mode, "d") 65 | } 66 | 67 | // IsLink reports whether or not the file is the snapshot of a symbolic link. 68 | func (f *File) IsLink() bool { 69 | if f == nil { 70 | return false 71 | } 72 | return strings.HasPrefix(f.Mode, "L") 73 | } 74 | 75 | // String implements the `fmt.Stringer` interface. 76 | // 77 | // The resulting value is suitable for serialization. 78 | func (f *File) String() string { 79 | if f == nil { 80 | return "" 81 | } 82 | var contentsStr string 83 | if f.Contents != nil { 84 | contentsStr = f.Contents.String() 85 | } 86 | lines := []string{f.Mode, contentsStr} 87 | for _, parent := range f.Parents { 88 | if parent != nil { 89 | lines = append(lines, parent.String()) 90 | } 91 | } 92 | return strings.Join(lines, "\n") 93 | } 94 | 95 | // ParseFile parses a `File` object from its encoded form. 96 | // 97 | // The input string must match the form returned by the `File.String` method. 98 | func ParseFile(encoded string) (*File, error) { 99 | if len(encoded) == 0 { 100 | return nil, nil 101 | } 102 | lines := strings.Split(string(encoded), "\n") 103 | if len(lines) < 2 { 104 | return nil, fmt.Errorf("malformed file metadata: %q", encoded) 105 | } 106 | var hashes []*Hash 107 | for i, line := range lines[1:] { 108 | hash, err := ParseHash(line) 109 | if err != nil { 110 | return nil, fmt.Errorf("failure parsing the hash %q: %v", line, err) 111 | } 112 | if hash != nil { 113 | hashes = append(hashes, hash) 114 | } else if i == 0 { 115 | return nil, fmt.Errorf("missing contents for the encoded file %q", encoded) 116 | } 117 | } 118 | f := &File{ 119 | Mode: lines[0], 120 | Contents: hashes[0], 121 | Parents: hashes[1:], 122 | } 123 | return f, nil 124 | } 125 | 126 | // Permissions returns the permission subset of the file mode. 127 | // 128 | // The returned `os.FileMode` object does not include any information 129 | // on the file type (e.g. directory vs. link, etc). 130 | func (f *File) Permissions() os.FileMode { 131 | if f == nil || len(f.Mode) < 9 { 132 | // This is not a Posix-style mode line; default to 0700 133 | return os.FileMode(0700) 134 | } 135 | permStr := f.Mode[len(f.Mode)-9:] 136 | perm := fs.ModePerm 137 | for i, c := range permStr { 138 | if c == '-' { 139 | perm ^= (1 << uint(8-i)) 140 | } 141 | } 142 | return perm 143 | } 144 | -------------------------------------------------------------------------------- /command/export.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | "io" 23 | "os" 24 | "path/filepath" 25 | "strings" 26 | 27 | "github.com/google/recursive-version-control-system/bundle" 28 | "github.com/google/recursive-version-control-system/storage" 29 | ) 30 | 31 | const exportUsage = `Usage: %s export []* 32 | 33 | Where is a local filesystem path for the newly generated bundle, and are one of: 34 | 35 | ` 36 | 37 | var ( 38 | exportFlags = flag.NewFlagSet("export", flag.ContinueOnError) 39 | 40 | exportSnapshotsFlag = exportFlags.String( 41 | "snapshots", "", 42 | "comma separated list of snapshots to include in the exported bundle") 43 | exportSnapshotsFromFileFlag = exportFlags.String( 44 | "snapshots-from-file", "", 45 | "path to a file containing a newline separated list of snapshots to include in the exported bundle") 46 | 47 | exportExcludeFlag = exportFlags.String( 48 | "exclude", "", 49 | ("comma separated list of objects to exclude from the exported bundle." + 50 | "This takes precedence over the `snapshots` flag, so a hash specified " + 51 | "in both flags will not be included in the bundle.")) 52 | exportExcludeFromFileFlag = exportFlags.String( 53 | "exclude-from-file", "", 54 | "path to a file containing a newline separated list of objects to exclude in the exported bundle") 55 | 56 | exportMetadataFlag = exportFlags.String( 57 | "metadata", "", 58 | "comma separated list of key=value pairs to include in the exported bundle") 59 | exportMetadataFromFilesFlag = exportFlags.String( 60 | "metadata-from-files", "", 61 | "comma separated list of key= pairs to include in the exported bundle. The entries must be local files whose contents will be what is included.") 62 | 63 | exportIncludeParentsFlag = exportFlags.Bool( 64 | "include-parents", false, 65 | "if true, then the exported bundle will recursively include the parents of selected snapshots") 66 | exportVerboseFlag = exportFlags.Bool( 67 | "v", false, 68 | "verbose output. Print the hash of every object included in the exported bundle") 69 | ) 70 | 71 | func metadataFromFilesAndFlag(ctx context.Context, fromFiles, fromFlag string) (map[string]io.ReadCloser, error) { 72 | metadata := make(map[string]io.ReadCloser) 73 | for _, pair := range strings.Split(fromFiles, ",") { 74 | if len(pair) == 0 { 75 | continue 76 | } 77 | parts := strings.Split(pair, "=") 78 | if len(parts) != 2 { 79 | return nil, fmt.Errorf("malformed key=value pair %q", pair) 80 | } 81 | f, err := os.Open(parts[1]) 82 | if err != nil { 83 | return nil, fmt.Errorf("failure opening the metadata file %q: %v", parts[1], err) 84 | } 85 | metadata[parts[0]] = f 86 | } 87 | for _, pair := range strings.Split(fromFlag, ",") { 88 | if len(pair) == 0 { 89 | continue 90 | } 91 | parts := strings.Split(pair, "=") 92 | if len(parts) != 2 { 93 | return nil, fmt.Errorf("malformed key=value pair %q", pair) 94 | } 95 | metadata[parts[0]] = io.NopCloser(strings.NewReader(parts[1])) 96 | } 97 | return metadata, nil 98 | } 99 | 100 | func exportCommand(ctx context.Context, s *storage.LocalFiles, cmd string, args []string) (int, error) { 101 | exportFlags.Usage = func() { 102 | fmt.Fprintf(flag.CommandLine.Output(), exportUsage, cmd) 103 | exportFlags.PrintDefaults() 104 | } 105 | if err := exportFlags.Parse(args); err != nil { 106 | return 1, nil 107 | } 108 | args = exportFlags.Args() 109 | if len(args) < 1 { 110 | fmt.Fprintf(flag.CommandLine.Output(), exportUsage, cmd) 111 | exportFlags.PrintDefaults() 112 | return 1, nil 113 | } 114 | 115 | snapshots, err := hashesFromFileAndFlag(ctx, *exportSnapshotsFromFileFlag, *exportSnapshotsFlag) 116 | if err != nil { 117 | return 1, err 118 | } 119 | exclude, err := hashesFromFileAndFlag(ctx, *exportExcludeFromFileFlag, *exportExcludeFlag) 120 | if err != nil { 121 | return 1, err 122 | } 123 | metadata, err := metadataFromFilesAndFlag(ctx, *exportMetadataFromFilesFlag, *exportMetadataFlag) 124 | if err != nil { 125 | return 1, err 126 | } 127 | 128 | path, err := filepath.Abs(args[0]) 129 | if err != nil { 130 | return 1, fmt.Errorf("failure resolving the absolute path of %q: %v", args[0], err) 131 | } 132 | 133 | included, err := bundle.Export(ctx, s, path, snapshots, exclude, metadata, *exportIncludeParentsFlag) 134 | if err != nil { 135 | return 1, fmt.Errorf("failure creating the bundle: %v\n", err) 136 | } 137 | if *exportVerboseFlag { 138 | for _, h := range included { 139 | fmt.Println(h.String()) 140 | } 141 | } 142 | return 0, nil 143 | } 144 | -------------------------------------------------------------------------------- /command/commands.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package command defines the command line interface for rvcs 16 | package command 17 | 18 | import ( 19 | "context" 20 | "flag" 21 | "fmt" 22 | "os" 23 | "path/filepath" 24 | "strings" 25 | 26 | "github.com/google/recursive-version-control-system/config" 27 | "github.com/google/recursive-version-control-system/publish" 28 | "github.com/google/recursive-version-control-system/snapshot" 29 | "github.com/google/recursive-version-control-system/storage" 30 | ) 31 | 32 | type command func(context.Context, *storage.LocalFiles, string, []string) (int, error) 33 | 34 | var ( 35 | commandMap = map[string]command{ 36 | "add-mirror": addMirrorCommand, 37 | "export": exportCommand, 38 | "import": importCommand, 39 | "log": logCommand, 40 | "merge": mergeCommand, 41 | "publish": publishCommand, 42 | "remove-mirror": removeMirrorCommand, 43 | "snapshot": snapshotCommand, 44 | } 45 | 46 | usage = `Usage: %s 47 | 48 | Where is one of: 49 | 50 | add-mirror 51 | export 52 | import 53 | log 54 | merge 55 | publish 56 | remove-mirror 57 | snapshot 58 | ` 59 | ) 60 | 61 | func resolveIdentitySnapshot(ctx context.Context, s *storage.LocalFiles, id *snapshot.Identity) (signature *snapshot.Hash, signed *snapshot.Hash, err error) { 62 | settings, err := config.Read() 63 | if err != nil { 64 | return nil, nil, fmt.Errorf("failure reading the config settings: %v", err) 65 | } 66 | signature, signed, err = publish.Pull(ctx, settings, s, id) 67 | if err != nil { 68 | return nil, nil, fmt.Errorf("failure pulling the latest snapshot for %q: %v", id, err) 69 | } 70 | return signature, signed, nil 71 | } 72 | 73 | func resolveSnapshot(ctx context.Context, s *storage.LocalFiles, name string) (*snapshot.Hash, error) { 74 | h, err := snapshot.ParseHash(name) 75 | if err == nil { 76 | return h, nil 77 | } 78 | id, err := snapshot.ParseIdentity(name) 79 | if err == nil { 80 | _, signed, err := resolveIdentitySnapshot(ctx, s, id) 81 | return signed, err 82 | } 83 | abs, err := filepath.Abs(name) 84 | if err != nil { 85 | return nil, fmt.Errorf("failure resolving the absolute path of %q: %v", name, err) 86 | } 87 | h, _, err = s.FindSnapshot(ctx, snapshot.Path(abs)) 88 | if err == nil { 89 | return h, nil 90 | } 91 | return nil, fmt.Errorf("unable to resolve the hash corresponding to %q", name) 92 | } 93 | 94 | func readHashesFromFile(ctx context.Context, path string) ([]*snapshot.Hash, error) { 95 | if path == "" { 96 | return nil, nil 97 | } 98 | contents, err := os.ReadFile(path) 99 | if err != nil { 100 | return nil, fmt.Errorf("failure reading hashes from the file %q: %v", path, err) 101 | } 102 | var hashes []*snapshot.Hash 103 | for _, line := range strings.Split(string(contents), "\n") { 104 | line = strings.TrimSpace(line) 105 | if len(line) == 0 { 106 | continue 107 | } 108 | h, err := snapshot.ParseHash(line) 109 | if err != nil { 110 | return nil, fmt.Errorf("failure parsing file hash entry %q: %v", line, err) 111 | } 112 | if h != nil { 113 | hashes = append(hashes, h) 114 | } 115 | } 116 | return hashes, nil 117 | } 118 | 119 | func hashesFromFileAndFlag(ctx context.Context, fromFile, fromFlag string) ([]*snapshot.Hash, error) { 120 | hashes, err := readHashesFromFile(ctx, fromFile) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | for _, s := range strings.Split(fromFlag, ",") { 126 | if len(s) == 0 { 127 | continue 128 | } 129 | h, err := snapshot.ParseHash(s) 130 | if err != nil { 131 | return nil, fmt.Errorf("failure parsing flag hash entry %q: %v", s, err) 132 | } 133 | if h != nil { 134 | hashes = append(hashes, h) 135 | } 136 | } 137 | return hashes, nil 138 | } 139 | 140 | // Run implements the subcommands of the `rvcs` CLI. 141 | // 142 | // The passed in `args` should be the value returned by `os.Args` 143 | // 144 | // The returned value is the exit code of the command; 0 for success 145 | // and non-zero for any form of failure. 146 | func Run(ctx context.Context, s *storage.LocalFiles, args []string) (exitCode int) { 147 | if len(args) < 2 { 148 | fmt.Fprintf(flag.CommandLine.Output(), usage, args[0]) 149 | return 1 150 | } 151 | subcommand, ok := commandMap[args[1]] 152 | if !ok { 153 | fmt.Fprintf(flag.CommandLine.Output(), "Unknown subcommand %q\n", args[1]) 154 | fmt.Fprintf(flag.CommandLine.Output(), usage, args[0]) 155 | return 1 156 | } 157 | retcode, err := subcommand(ctx, s, args[0], args[2:]) 158 | if err != nil { 159 | fmt.Fprintf(flag.CommandLine.Output(), "Failure running the %q subcommand: %v\n", args[1], err) 160 | } 161 | return retcode 162 | } 163 | -------------------------------------------------------------------------------- /merge/helper_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package merge defines methods for merging two snapshots together. 16 | package merge 17 | 18 | import ( 19 | "context" 20 | "os" 21 | "path/filepath" 22 | "testing" 23 | 24 | "github.com/google/recursive-version-control-system/snapshot" 25 | "github.com/google/recursive-version-control-system/storage" 26 | ) 27 | 28 | func TestMergeWithHelperNoConflict(t *testing.T) { 29 | dir := t.TempDir() 30 | archive := filepath.Join(dir, "archive") 31 | s := &storage.LocalFiles{ArchiveDir: archive} 32 | 33 | original := filepath.Join(dir, "original.txt") 34 | originalPath := snapshot.Path(original) 35 | 36 | // Take an initial snapshot 37 | if err := os.WriteFile(original, []byte("A\nB\nC\nD\nE\n"), 0700); err != nil { 38 | t.Fatalf("failure creating the example file to snapshot: %v", err) 39 | } 40 | h1, f1, err := snapshot.Current(context.Background(), s, originalPath) 41 | if err != nil { 42 | t.Fatalf("failure creating the initial snapshot for the file: %v", err) 43 | } else if h1 == nil { 44 | t.Fatalf("unexpected nil hash for the file") 45 | } else if f1 == nil { 46 | t.Fatalf("unexpected nil snapshot for the file") 47 | } 48 | 49 | clone1 := filepath.Join(dir, "clone1.txt") 50 | clone1Path := snapshot.Path(clone1) 51 | if err := Checkout(context.Background(), s, h1, clone1Path); err != nil { 52 | t.Fatalf("failure checking out the file snapshot %q to %q: %v", h1, clone1Path, err) 53 | } 54 | if err := os.WriteFile(clone1, []byte("A\nX\nB\nY\nC\nD\nE\n"), 0700); err != nil { 55 | t.Fatalf("failure updating the first clone of the example file to snapshot: %v", err) 56 | } 57 | h2, _, err := snapshot.Current(context.Background(), s, clone1Path) 58 | if err != nil { 59 | t.Fatalf("failure creating the first updated snapshot for the file: %v", err) 60 | } else if h2 == nil { 61 | t.Fatalf("unexpected nil hash for the file") 62 | } 63 | 64 | clone2 := filepath.Join(dir, "clone2.txt") 65 | clone2Path := snapshot.Path(clone2) 66 | if err := Checkout(context.Background(), s, h1, clone2Path); err != nil { 67 | t.Fatalf("failure checking out the file snapshot %q to %q: %v", h1, clone2Path, err) 68 | } 69 | if err := os.WriteFile(clone2, []byte("A\nB\nC\nZ\nD\nE\n"), 0700); err != nil { 70 | t.Fatalf("failure updating the second clone of the example file to snapshot: %v", err) 71 | } 72 | h3, _, err := snapshot.Current(context.Background(), s, clone2Path) 73 | if err != nil { 74 | t.Fatalf("failure creating the second updated snapshot for the file: %v", err) 75 | } else if h3 == nil { 76 | t.Fatalf("unexpected nil hash for the file") 77 | } 78 | 79 | h4, err := mergeWithHelper(context.Background(), s, originalPath, "-rwx------", h1, h2, h3) 80 | if err != nil { 81 | t.Fatalf("failure merging non-conflicting changes with the helper: %v", err) 82 | } 83 | 84 | merged := filepath.Join(dir, "merged.txt") 85 | mergedPath := snapshot.Path(merged) 86 | if err := Checkout(context.Background(), s, h4, mergedPath); err != nil { 87 | t.Fatalf("failure checking out the merged snapshot %q: %v", h4, err) 88 | } 89 | mergedBytes, err := os.ReadFile(merged) 90 | if err != nil { 91 | t.Fatalf("failure reading the contents of the checked out merged snapshot: %v", err) 92 | } 93 | if got, want := string(mergedBytes), "A\nX\nB\nY\nC\nZ\nD\nE\n"; got != want { 94 | t.Errorf("unexpected results of merging non-conflicting changes with the helper: got %q, want %q", got, want) 95 | } 96 | } 97 | 98 | func TestMergeWithHelperNilBaseNoConflict(t *testing.T) { 99 | dir := t.TempDir() 100 | archive := filepath.Join(dir, "archive") 101 | s := &storage.LocalFiles{ArchiveDir: archive} 102 | 103 | original := filepath.Join(dir, "original.txt") 104 | originalPath := snapshot.Path(original) 105 | 106 | version1 := filepath.Join(dir, "version1.txt") 107 | version1Path := snapshot.Path(version1) 108 | if err := os.WriteFile(version1, []byte("A\nB\nC\nD\nE\n"), 0700); err != nil { 109 | t.Fatalf("failure writing the first version of the example file to snapshot: %v", err) 110 | } 111 | v1Hash, _, err := snapshot.Current(context.Background(), s, version1Path) 112 | if err != nil { 113 | t.Fatalf("failure creating the first updated snapshot for the file: %v", err) 114 | } else if v1Hash == nil { 115 | t.Fatalf("unexpected nil hash for the file") 116 | } 117 | 118 | version2 := filepath.Join(dir, "version2.txt") 119 | version2Path := snapshot.Path(version2) 120 | if err := os.WriteFile(version2, []byte("A\nB\nC\nD\nE\n"), 0700); err != nil { 121 | t.Fatalf("failure writing the second version of the example file to snapshot: %v", err) 122 | } 123 | v2Hash, _, err := snapshot.Current(context.Background(), s, version2Path) 124 | if err != nil { 125 | t.Fatalf("failure creating the second updated snapshot for the file: %v", err) 126 | } else if v2Hash == nil { 127 | t.Fatalf("unexpected nil hash for the file") 128 | } 129 | 130 | if mergedHash, err := mergeWithHelper(context.Background(), s, originalPath, "-rwx------", nil, v1Hash, v2Hash); err == nil { 131 | t.Errorf("unexpected result from merging unrelated files with the default merge helper: %v", mergedHash) 132 | } else if got, want := err.Error(), "merge helper \"diff3\" failed: exit status 1"; got != want { 133 | t.Errorf("unexexpected error message from merging unrelated files with the default merge helper: got %q, want %q", got, want) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /merge/base_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package merge defines methods for merging two snapshots together. 16 | package merge 17 | 18 | import ( 19 | "context" 20 | "os" 21 | "path/filepath" 22 | "testing" 23 | 24 | "github.com/google/recursive-version-control-system/snapshot" 25 | "github.com/google/recursive-version-control-system/storage" 26 | ) 27 | 28 | func setupSnapshots(t *testing.T) (s *storage.LocalFiles, parent *snapshot.Hash, child1 *snapshot.Hash, child2 *snapshot.Hash) { 29 | dir := t.TempDir() 30 | archive := filepath.Join(dir, "archive") 31 | s = &storage.LocalFiles{ArchiveDir: archive} 32 | 33 | filename := filepath.Join(dir, "example.txt") 34 | p := snapshot.Path(filename) 35 | 36 | // Take an initial snapshot 37 | if err := os.WriteFile(filename, []byte("Hello, World!"), 0700); err != nil { 38 | t.Fatalf("failure creating the example file to snapshot: %v", err) 39 | } 40 | h1, f1, err := snapshot.Current(context.Background(), s, p) 41 | if err != nil { 42 | t.Fatalf("failure creating the initial snapshot for the file: %v", err) 43 | } else if h1 == nil { 44 | t.Fatalf("unexpected nil hash for the file") 45 | } else if f1 == nil { 46 | t.Fatalf("unexpected nil snapshot for the file") 47 | } 48 | 49 | if err := os.WriteFile(filename, []byte("Goodbye, World!"), 0700); err != nil { 50 | t.Fatalf("failure updating the example file to snapshot: %v", err) 51 | } 52 | h2, f2, err := snapshot.Current(context.Background(), s, p) 53 | if err != nil { 54 | t.Fatalf("failure creating the updated snapshot for the file: %v", err) 55 | } else if h2 == nil { 56 | t.Fatalf("unexpected nil hash for the file") 57 | } else if f2 == nil { 58 | t.Fatalf("unexpected nil snapshot for the file") 59 | } 60 | 61 | if err := os.RemoveAll(filename); err != nil { 62 | t.Fatalf("failure removing the example file: %v", err) 63 | } 64 | if err := Checkout(context.Background(), s, h1, snapshot.Path(filename)); err != nil { 65 | t.Fatalf("failure checking out the initial snapshot: %v", err) 66 | } 67 | if err := os.WriteFile(filename, []byte("Hello again, World!"), 0700); err != nil { 68 | t.Fatalf("failure updating the example file to a second snapshot: %v", err) 69 | } 70 | h3, f3, err := snapshot.Current(context.Background(), s, p) 71 | if err != nil { 72 | t.Fatalf("failure creating the second updated snapshot for the file: %v", err) 73 | } else if h3 == nil { 74 | t.Fatalf("unexpected nil hash for the file") 75 | } else if f3 == nil { 76 | t.Fatalf("unexpected nil snapshot for the file") 77 | } 78 | return s, h1, h2, h3 79 | } 80 | 81 | func TestNilBase(t *testing.T) { 82 | s, h1, h2, _ := setupSnapshots(t) 83 | if base, err := Base(context.Background(), s, nil, nil); err != nil { 84 | t.Errorf("failure computing the mergebase of nil and itself: %v", err) 85 | } else if base != nil { 86 | t.Errorf("unexpected mergebase for nil and itself: %q", base) 87 | } 88 | if base, err := Base(context.Background(), s, nil, h1); err != nil { 89 | t.Errorf("failure computing the mergebase of nil and an initial snapshot: %v", err) 90 | } else if base != nil { 91 | t.Errorf("unexpected mergebase for nil and an initial snapshot: %q", base) 92 | } 93 | if base, err := Base(context.Background(), s, h1, nil); err != nil { 94 | t.Errorf("failure computing the mergebase of an initial snapshot and nil: %v", err) 95 | } else if base != nil { 96 | t.Errorf("unexpected mergebase for an initial snapshot and nil: %q", base) 97 | } 98 | if base, err := Base(context.Background(), s, nil, h2); err != nil { 99 | t.Errorf("failure computing the mergebase of nil and an updated snapshot: %v", err) 100 | } else if base != nil { 101 | t.Errorf("unexpected mergebase for nil and an updated snapshot: %q", base) 102 | } 103 | if base, err := Base(context.Background(), s, h2, nil); err != nil { 104 | t.Errorf("failure computing the mergebase of an updated snapshot and nil: %v", err) 105 | } else if base != nil { 106 | t.Errorf("unexpected mergebase for an updated snapshot and nil: %q", base) 107 | } 108 | } 109 | 110 | func TestTrivialBase(t *testing.T) { 111 | s, h1, h2, _ := setupSnapshots(t) 112 | if base, err := Base(context.Background(), s, h1, h1); err != nil { 113 | t.Errorf("failure computing the mergebase of an initial snapshot and itself: %v", err) 114 | } else if !base.Equal(h1) { 115 | t.Errorf("unexpected mergebase for an initial snapshot and itself: %q", base) 116 | } 117 | if base, err := Base(context.Background(), s, h2, h2); err != nil { 118 | t.Errorf("failure computing the mergebase of an updated snapshot and itself: %v", err) 119 | } else if !base.Equal(h2) { 120 | t.Errorf("unexpected mergebase for an updated snapshot and itself: %q", base) 121 | } 122 | } 123 | 124 | func TestDirectBase(t *testing.T) { 125 | s, h1, h2, _ := setupSnapshots(t) 126 | if base, err := Base(context.Background(), s, h1, h2); err != nil { 127 | t.Errorf("failure computing the mergebase of an initial snapshot and its child: %v", err) 128 | } else if !base.Equal(h1) { 129 | t.Errorf("unexpected mergebase for an initial snapshot and its child: %q", base) 130 | } 131 | if base, err := Base(context.Background(), s, h2, h1); err != nil { 132 | t.Errorf("failure computing the mergebase of a child and its parent: %v", err) 133 | } else if !base.Equal(h1) { 134 | t.Errorf("unexpected mergebase for a child and its parent: %q", base) 135 | } 136 | } 137 | 138 | func TestMutualParentBase(t *testing.T) { 139 | s, h1, h2, h3 := setupSnapshots(t) 140 | if base, err := Base(context.Background(), s, h2, h3); err != nil { 141 | t.Errorf("failure computing the mergebase of two sibling snapshots: %v", err) 142 | } else if !base.Equal(h1) { 143 | t.Errorf("unexpected mergebase for two sibling snapshots: %v", base) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /merge/checkout.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package merge defines methods for merging two snapshots together. 16 | package merge 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "io" 22 | "os" 23 | "path/filepath" 24 | 25 | "github.com/google/recursive-version-control-system/snapshot" 26 | "github.com/google/recursive-version-control-system/storage" 27 | ) 28 | 29 | func recreateLink(ctx context.Context, s *storage.LocalFiles, h *snapshot.Hash, f *snapshot.File, p snapshot.Path) error { 30 | contentsReader, err := s.ReadObject(ctx, f.Contents) 31 | if err != nil { 32 | return fmt.Errorf("failure opening the contents of the link snapshot %q: %v", h, err) 33 | } 34 | contents, err := io.ReadAll(contentsReader) 35 | if err != nil { 36 | return fmt.Errorf("failure reading the contents of the link snapshot %q: %v", h, err) 37 | } 38 | if target, err := os.Readlink(string(p)); err == nil && target == string(contents) { 39 | // The link already exists and points at the correct target 40 | return nil 41 | } 42 | if err := os.RemoveAll(string(p)); err != nil { 43 | return fmt.Errorf("failure removing the old file at %q: %v", p, err) 44 | } 45 | if err := os.Symlink(string(contents), string(p)); err != nil { 46 | return fmt.Errorf("failure recreating the symlink %q: %v", h, err) 47 | } 48 | return nil 49 | } 50 | 51 | func ensureDirExistsWithPermissions(ctx context.Context, path string, perm os.FileMode) error { 52 | if err := os.Mkdir(path, perm); err == nil { 53 | return nil 54 | } else if !os.IsExist(err) { 55 | return fmt.Errorf("failure creating the directory %q: %v", path, err) 56 | } 57 | if info, err := os.Lstat(path); err != nil { 58 | return fmt.Errorf("failure reading file metadata for the path %q: %v", path, err) 59 | } else if info.IsDir() { 60 | return os.Chmod(path, perm) 61 | } 62 | if err := os.RemoveAll(path); err != nil { 63 | return fmt.Errorf("failure removing the old file at %q: %v", path, err) 64 | } 65 | return os.Mkdir(path, perm) 66 | } 67 | 68 | func recreateDir(ctx context.Context, s *storage.LocalFiles, h *snapshot.Hash, f *snapshot.File, p snapshot.Path) error { 69 | perm := f.Permissions() 70 | if err := ensureDirExistsWithPermissions(ctx, string(p), perm); err != nil { 71 | return fmt.Errorf("failure creating the directory %q: %v", p, err) 72 | } 73 | 74 | tree, err := s.ListDirectorySnapshotContents(ctx, h, f) 75 | if err != nil { 76 | return fmt.Errorf("failure reading the contents of the directory snapshot %q: %v", h, err) 77 | } 78 | 79 | contents, err := os.Open(string(p)) 80 | if err != nil { 81 | return fmt.Errorf("failure opening the directory %q: %v", p, err) 82 | } 83 | entries, err := contents.ReadDir(0) 84 | if err != nil { 85 | return fmt.Errorf("failure reading the filesystem contents of the directory %q: %v", p, err) 86 | } 87 | for _, entry := range entries { 88 | child := snapshot.Path(entry.Name()) 89 | if _, ok := tree[child]; ok { 90 | continue 91 | } 92 | // The child does not exist in the snapshot to checkout 93 | childPath := p.Join(child) 94 | if s.Exclude(childPath) { 95 | // The child path is meant to be excluded from 96 | // snapshotting, so it is expected that it would not 97 | // be in the snapshot. 98 | continue 99 | } 100 | if err := os.RemoveAll(string(childPath)); err != nil { 101 | return fmt.Errorf("failure removing the deleted file %q: %v", childPath, err) 102 | } 103 | } 104 | for child, childHash := range tree { 105 | childPath := p.Join(child) 106 | if s.Exclude(childPath) { 107 | // The child path is meant to be excluded from 108 | // snapshotting, so it should also be excluded from 109 | // being updated when checking out a snapshot. 110 | continue 111 | } 112 | if err := Checkout(ctx, s, childHash, childPath); err != nil { 113 | return fmt.Errorf("failure checking out the child path %q: %v", childPath, err) 114 | } 115 | } 116 | return nil 117 | } 118 | 119 | func ensureFileExistsWithPermissions(ctx context.Context, path string, perm os.FileMode) (*os.File, error) { 120 | out, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm) 121 | if err != nil { 122 | if err := os.RemoveAll(path); err != nil { 123 | return nil, fmt.Errorf("failure removing the old file at %q: %v", path, err) 124 | } 125 | out, err = os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, perm) 126 | } 127 | if err != nil { 128 | return nil, err 129 | } 130 | if err := out.Chmod(perm); err != nil { 131 | return nil, fmt.Errorf("failure changing the permissions of %q: %v", path, err) 132 | } 133 | return out, nil 134 | } 135 | 136 | func recreateFile(ctx context.Context, s *storage.LocalFiles, h *snapshot.Hash, f *snapshot.File, p snapshot.Path) error { 137 | if f.IsLink() { 138 | return recreateLink(ctx, s, h, f, p) 139 | } 140 | if f.IsDir() { 141 | return recreateDir(ctx, s, h, f, p) 142 | } 143 | perm := f.Permissions() 144 | contentsReader, err := s.ReadObject(ctx, f.Contents) 145 | if err != nil { 146 | return fmt.Errorf("failure opening the contents of the link snapshot %q: %v", h, err) 147 | } 148 | out, err := ensureFileExistsWithPermissions(ctx, string(p), perm) 149 | if err != nil { 150 | return fmt.Errorf("failure opening the file %q: %v", p, err) 151 | } 152 | if _, err := io.Copy(out, contentsReader); err != nil { 153 | return fmt.Errorf("failure writing the contents of %q: %v", p, err) 154 | } 155 | if err := out.Close(); err != nil { 156 | return fmt.Errorf("failure closing the file %q: %v", p, err) 157 | } 158 | return nil 159 | } 160 | 161 | // Checkout "checks out" the given snapshot to a new file location. 162 | // 163 | // If any files already exist at the given location, they will be overwritten. 164 | // 165 | // If there are any nested files under the given location that do not exist 166 | // in the checked out snapshot, then they will be removed. 167 | // 168 | // For regular files and directories, the checked out file permissions will 169 | // match what the corresponding permissions are in the snapshot. However, 170 | // for symbolic links, the file permissions from the snapshot are ignored. 171 | // 172 | // If there are any errors during the checkout, then the applied filesystem 173 | // changes are not rolled back and the local file system can be left in an 174 | // inconsistent state. 175 | func Checkout(ctx context.Context, s *storage.LocalFiles, h *snapshot.Hash, p snapshot.Path) error { 176 | f, err := s.ReadSnapshot(ctx, h) 177 | if err != nil { 178 | return fmt.Errorf("failure reading the file snapshot for %q: %v", h, err) 179 | } 180 | if f == nil { 181 | // The source file does not exist; nothing for us to do. 182 | return nil 183 | } 184 | parent := filepath.Dir(string(p)) 185 | if err := os.MkdirAll(parent, os.FileMode(0700)); err != nil { 186 | return fmt.Errorf("failure ensuring the parent directory of %q exists: %v", p, err) 187 | } 188 | if err := recreateFile(ctx, s, h, f, p); err != nil { 189 | return fmt.Errorf("failure checking out the snapshot %q to the path %q: %v", h, p, err) 190 | } 191 | if _, err := s.StoreSnapshot(ctx, p, f); err != nil { 192 | return fmt.Errorf("failure updating the snapshot for %q to %q: %v", p, h, err) 193 | } 194 | return nil 195 | } 196 | -------------------------------------------------------------------------------- /log/log.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package log provides methods for extracting the log of changes for a snapshot. 16 | package log 17 | 18 | import ( 19 | "context" 20 | "fmt" 21 | "path/filepath" 22 | "sort" 23 | "syscall" 24 | 25 | "github.com/google/recursive-version-control-system/snapshot" 26 | "github.com/google/recursive-version-control-system/storage" 27 | "golang.org/x/term" 28 | ) 29 | 30 | type LogEntry struct { 31 | // Hash is the hash of the file snapshot 32 | Hash *snapshot.Hash 33 | 34 | // File is the file snapshot 35 | File *snapshot.File 36 | 37 | // summary is a list of strings that describe what changed 38 | // between the file snapshot and its first parent. 39 | // 40 | // This is empty until the first time the `SummarizeLog` method 41 | // has been successfully called with this LogEntry. 42 | summary []string 43 | 44 | // nestedPaths is an ordered slice of all subpaths of the file. 45 | // 46 | // This is only ever populated for snapshots of directories, 47 | // and only if the `SummarizeLog` method has been called. 48 | nestedPaths []string 49 | 50 | // nestedContents is a map from all subpaths of the file to 51 | // the corresponding nested file snapshots. 52 | // 53 | // This is only ever populated for snapshots of directories, 54 | // and only if the `SummarizeLog` method has been called. 55 | nestedContents map[string]*snapshot.Hash 56 | } 57 | 58 | func dirContents(ctx context.Context, s *storage.LocalFiles, h *snapshot.Hash, f *snapshot.File, subpath string, includeDirectories bool, contentsMap map[string]*snapshot.Hash) error { 59 | tree, err := s.ListDirectorySnapshotContents(ctx, h, f) 60 | if err != nil { 61 | return fmt.Errorf("failure listing the directory contents of the snapshot %q: %v", h, err) 62 | } 63 | for p, ph := range tree { 64 | child, err := s.ReadSnapshot(ctx, ph) 65 | if err != nil { 66 | return fmt.Errorf("failure reading the file snapshot for %q: %v", p, err) 67 | } 68 | childPath := filepath.Join(subpath, string(p)) 69 | if child.IsDir() { 70 | if includeDirectories { 71 | contentsMap[childPath] = ph 72 | } 73 | if err := dirContents(ctx, s, ph, child, childPath, includeDirectories, contentsMap); err != nil { 74 | return fmt.Errorf("failure enumerating the contents of %q: %v", p, err) 75 | } 76 | } else { 77 | contentsMap[childPath] = ph 78 | } 79 | } 80 | return nil 81 | } 82 | 83 | // NestedContents returns a map from subpaths of the log entry's file to 84 | // the corresponding (hashes of the) file snapshots for the nested files. 85 | // 86 | // This is only defined for snapshots of directories, and for all other 87 | // cases the return value will be nil. 88 | func (e *LogEntry) NestedContents(ctx context.Context, s *storage.LocalFiles, includeDirectories bool) ([]string, map[string]*snapshot.Hash, error) { 89 | if e.nestedPaths != nil && e.nestedContents != nil { 90 | return e.nestedPaths, e.nestedContents, nil 91 | } 92 | if !e.File.IsDir() { 93 | return nil, nil, nil 94 | } 95 | paths := []string{} 96 | contentsMap := make(map[string]*snapshot.Hash) 97 | if err := dirContents(ctx, s, e.Hash, e.File, "", includeDirectories, contentsMap); err != nil { 98 | return nil, nil, fmt.Errorf("failure reading the nested contents for %q: %v", e.Hash, err) 99 | } 100 | for path, _ := range contentsMap { 101 | paths = append(paths, path) 102 | } 103 | sort.Strings(paths) 104 | e.nestedPaths = paths 105 | e.nestedContents = contentsMap 106 | return e.nestedPaths, e.nestedContents, nil 107 | } 108 | 109 | func deleteLine(deletedPath string, deletedHash *snapshot.Hash) string { 110 | coreText := fmt.Sprintf(" -%s(%s)", deletedPath, deletedHash) 111 | if !term.IsTerminal(syscall.Stdout) { 112 | return coreText 113 | } 114 | // Add ascii color escape codes if running in a terminal 115 | return fmt.Sprintf("\033[31m%s\033[0m", coreText) 116 | } 117 | 118 | func insertLine(insertedPath string, insertedHash *snapshot.Hash) string { 119 | coreText := fmt.Sprintf(" +%s(%s)", insertedPath, insertedHash) 120 | if !term.IsTerminal(syscall.Stdout) { 121 | return coreText 122 | } 123 | // Add ascii color escape codes if running in a terminal 124 | return fmt.Sprintf("\033[32m%s\033[0m", coreText) 125 | } 126 | 127 | func describeChanged(paths, previousPaths []string, contents, previousContents map[string]*snapshot.Hash) []string { 128 | changes := []string{} 129 | for _, p := range paths { 130 | h := contents[p] 131 | for len(previousPaths) > 0 && previousPaths[0] < p { 132 | deletedPath := previousPaths[0] 133 | previousPaths = previousPaths[1:] 134 | changes = append(changes, deleteLine(deletedPath, previousContents[deletedPath])) 135 | } 136 | var previousHash *snapshot.Hash 137 | if len(previousPaths) > 0 && previousPaths[0] == p { 138 | previousHash = previousContents[p] 139 | previousPaths = previousPaths[1:] 140 | } 141 | if previousHash.Equal(h) { 142 | continue 143 | } 144 | if previousHash != nil { 145 | changes = append(changes, deleteLine(p, previousHash)) 146 | } 147 | changes = append(changes, insertLine(p, h)) 148 | } 149 | for _, deletedPath := range previousPaths { 150 | previousHash := previousContents[deletedPath] 151 | changes = append(changes, deleteLine(deletedPath, previousHash)) 152 | } 153 | return changes 154 | } 155 | 156 | func SummarizeLog(ctx context.Context, s *storage.LocalFiles, entries []*LogEntry) (map[snapshot.Hash][]string, error) { 157 | pathsMap := make(map[snapshot.Hash][]string) 158 | contentsMap := make(map[snapshot.Hash]map[string]*snapshot.Hash) 159 | for _, e := range entries { 160 | paths, contents, err := e.NestedContents(ctx, s, false) 161 | if err != nil { 162 | return nil, fmt.Errorf("failure reading the nested contents of snapshot %q: %v", e.Hash, err) 163 | } 164 | if paths != nil && contents != nil { 165 | pathsMap[*e.Hash] = paths 166 | contentsMap[*e.Hash] = contents 167 | } 168 | } 169 | result := make(map[snapshot.Hash][]string) 170 | for _, e := range entries { 171 | var prevPaths []string 172 | var prevContents map[string]*snapshot.Hash 173 | if len(e.File.Parents) > 0 { 174 | firstParent := e.File.Parents[0] 175 | prevPaths = pathsMap[*firstParent] 176 | prevContents = contentsMap[*firstParent] 177 | } 178 | summary := []string{e.Hash.String()} 179 | contents, contentsOk := contentsMap[*e.Hash] 180 | paths, pathsOk := pathsMap[*e.Hash] 181 | if contentsOk && pathsOk { 182 | summary = append(summary, describeChanged(paths, prevPaths, contents, prevContents)...) 183 | } 184 | result[*e.Hash] = summary 185 | } 186 | return result, nil 187 | } 188 | 189 | func ReadLog(ctx context.Context, s *storage.LocalFiles, h *snapshot.Hash, maxDepth int) ([]*LogEntry, error) { 190 | visited := make(map[snapshot.Hash]*snapshot.File) 191 | queue := []*snapshot.Hash{h} 192 | result := []*LogEntry{} 193 | var depth int 194 | for len(queue) > 0 && depth != maxDepth { 195 | var next []*snapshot.Hash 196 | for _, h := range queue { 197 | f, err := s.ReadSnapshot(ctx, h) 198 | if err != nil { 199 | return nil, fmt.Errorf("failure reading the snapshot for %q: %v", h, err) 200 | } 201 | visited[*h] = f 202 | result = append(result, &LogEntry{ 203 | Hash: h, 204 | File: f, 205 | }) 206 | for _, p := range f.Parents { 207 | if _, ok := visited[*p]; !ok { 208 | next = append(next, p) 209 | } 210 | } 211 | } 212 | queue = next 213 | depth++ 214 | } 215 | return result, nil 216 | } 217 | -------------------------------------------------------------------------------- /snapshot/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package snapshot defines the model for snapshots of a file's history. 16 | package snapshot 17 | 18 | import ( 19 | "bytes" 20 | "context" 21 | "fmt" 22 | "io" 23 | "io/fs" 24 | "os" 25 | "path/filepath" 26 | "strings" 27 | "time" 28 | ) 29 | 30 | // Storage defines persistent storage of snapshots. 31 | type Storage interface { 32 | // StoreObject persists the contents of the given reader, returning the resulting hash of those contents. 33 | // 34 | // This is used for persistently storing the contents of individual files. 35 | StoreObject(context.Context, int64, io.Reader) (*Hash, error) 36 | 37 | // Exclude reports whether or not the given path should be excluded from storage. 38 | Exclude(Path) bool 39 | 40 | // FindSnapshot reads the latest snapshot (if any) for the given path. 41 | FindSnapshot(context.Context, Path) (*Hash, *File, error) 42 | 43 | // StoreSnapshot stores a mapping from the given path to the given snapshot. 44 | StoreSnapshot(context.Context, Path, *File) (*Hash, error) 45 | 46 | // CachePathInfo caches the file information for the given path. 47 | // 48 | // This is used to avoid rehashing the contents of files that have 49 | // not changed since the last time they were snapshotted. 50 | CachePathInfo(context.Context, Path, os.FileInfo) error 51 | 52 | // PathInfoMatchesCache reports whether or not the given file 53 | // information matches the file information that was previously cached 54 | // for the given path. 55 | PathInfoMatchesCache(context.Context, Path, os.FileInfo) bool 56 | } 57 | 58 | func snapshotFileMetadata(ctx context.Context, s Storage, p Path, info os.FileInfo, contentsHash *Hash) (*Hash, *File, error) { 59 | modeLine := info.Mode().String() 60 | prevFileHash, prev, err := s.FindSnapshot(ctx, p) 61 | if err != nil && !os.IsNotExist(err) { 62 | return nil, nil, fmt.Errorf("failure looking up the previous file snapshot: %v", err) 63 | } 64 | if prev != nil && prev.Mode == modeLine && prev.Contents.Equal(contentsHash) { 65 | // The file is unchanged from the last snapshot... 66 | return prevFileHash, prev, nil 67 | } 68 | f := &File{ 69 | Contents: contentsHash, 70 | Mode: modeLine, 71 | } 72 | if prev != nil { 73 | f.Parents = []*Hash{prevFileHash} 74 | } 75 | h, err := s.StoreSnapshot(ctx, p, f) 76 | if err != nil { 77 | return nil, nil, fmt.Errorf("failure saving the latest file metadata for %q: %v", p, err) 78 | } 79 | return h, f, nil 80 | } 81 | 82 | func readCached(ctx context.Context, s Storage, p Path, info os.FileInfo) (*Hash, *File, bool) { 83 | if !s.PathInfoMatchesCache(ctx, p, info) { 84 | return nil, nil, false 85 | } 86 | cachedHash, cachedFile, err := s.FindSnapshot(ctx, p) 87 | if err != nil { 88 | return nil, nil, false 89 | } 90 | return cachedHash, cachedFile, true 91 | } 92 | 93 | // timeNow is a handle on `time.Now` that lets us replace it for simulating the passage of time in unit tests. 94 | var timeNow func() time.Time = time.Now 95 | 96 | func snapshotRegularFile(ctx context.Context, s Storage, p Path, info os.FileInfo, contents io.Reader) (h *Hash, f *File, err error) { 97 | startTimeSec := timeNow().Truncate(time.Second) 98 | if cachedHash, cachedFile, ok := readCached(ctx, s, p, info); ok { 99 | return cachedHash, cachedFile, nil 100 | } 101 | defer func() { 102 | // Cache the path info if appropriate... 103 | if err != nil || h == nil { 104 | // We did not construct a snapshot, so nothing to cache 105 | return 106 | } 107 | latestInfo, err := os.Lstat(string(p)) 108 | if err != nil { 109 | // We could not determine if the file has changed during snapshotting, so don't cache. 110 | return 111 | } 112 | if !latestInfo.ModTime().Equal(info.ModTime()) { 113 | // The file changed while we were snapshotting it; don't cache anything 114 | return 115 | } 116 | if !latestInfo.ModTime().Before(startTimeSec.Add(-1 * time.Second)) { 117 | // The file timestamp matches when we started, so there's a potential 118 | // race condition where it might have updated after we snapshotted, 119 | // and we should not cache it. 120 | return 121 | } 122 | s.CachePathInfo(ctx, p, info) 123 | }() 124 | h, err = s.StoreObject(ctx, info.Size(), contents) 125 | if err != nil { 126 | return nil, nil, fmt.Errorf("failure storing an object: %v", err) 127 | } 128 | return snapshotFileMetadata(ctx, s, p, info, h) 129 | } 130 | 131 | func snapshotDirectory(ctx context.Context, s Storage, p Path, info os.FileInfo, contents *os.File) (*Hash, *File, error) { 132 | entries, err := contents.ReadDir(0) 133 | if err != nil { 134 | return nil, nil, fmt.Errorf("failure reading the filesystem contents of the directory %q: %v", p, err) 135 | } 136 | childHashes := make(Tree) 137 | for _, entry := range entries { 138 | childPath := Path(filepath.Join(string(p), entry.Name())) 139 | childHash, _, err := Current(ctx, s, childPath) 140 | if err != nil { 141 | return nil, nil, fmt.Errorf("failure hashing the child dir %q: %v", childPath, err) 142 | } 143 | if childHash != nil { 144 | childHashes[Path(entry.Name())] = childHash 145 | } 146 | } 147 | contentsJson := []byte(childHashes.String()) 148 | contentsHash, err := s.StoreObject(ctx, int64(len(contentsJson)), bytes.NewReader(contentsJson)) 149 | return snapshotFileMetadata(ctx, s, p, info, contentsHash) 150 | } 151 | 152 | func snapshotLink(ctx context.Context, s Storage, p Path, info os.FileInfo) (*Hash, *File, error) { 153 | target, err := os.Readlink(string(p)) 154 | if err != nil { 155 | return nil, nil, fmt.Errorf("failure reading the link target for %q: %v", p, err) 156 | } 157 | 158 | h, err := s.StoreObject(ctx, int64(len(target)), strings.NewReader(target)) 159 | if err != nil { 160 | return nil, nil, fmt.Errorf("failure storing an object: %v", err) 161 | } 162 | return snapshotFileMetadata(ctx, s, p, info, h) 163 | } 164 | 165 | // Current generates a snapshot for the given path, stored in the given store. 166 | // 167 | // The passed in path must be an absolute path. 168 | // 169 | // The returned value is the hash of the generated `snapshot.File` object. 170 | func Current(ctx context.Context, s Storage, p Path) (*Hash, *File, error) { 171 | if s.Exclude(p) { 172 | // We are not supposed to store snapshots for the given path, so pretend it does not exist. 173 | return nil, nil, nil 174 | } 175 | stat, err := os.Lstat(string(p)) 176 | if os.IsNotExist(err) { 177 | // The referenced file does not exist, so the corresponding 178 | // hash should be nil. 179 | return nil, nil, nil 180 | } 181 | if err != nil { 182 | return nil, nil, fmt.Errorf("failure reading the file stat for %q: %v", p, err) 183 | } 184 | if stat.Mode()&fs.ModeSymlink != 0 { 185 | return snapshotLink(ctx, s, p, stat) 186 | } 187 | contents, err := os.Open(string(p)) 188 | if os.IsNotExist(err) { 189 | // The file we tried to open no longer exists. 190 | // 191 | // This could happen if there is a race condition where the 192 | // file was deleted before we could read it. In that case, 193 | // return an empty snapshot. 194 | return nil, nil, nil 195 | } 196 | if err != nil { 197 | return nil, nil, fmt.Errorf("failure reading the file %q: %v", p, err) 198 | } 199 | defer contents.Close() 200 | 201 | info, err := contents.Stat() 202 | if err != nil { 203 | return nil, nil, fmt.Errorf("failure reading the filesystem metadata for %q: %v", p, err) 204 | } 205 | if info.IsDir() { 206 | return snapshotDirectory(ctx, s, p, info, contents) 207 | } else { 208 | return snapshotRegularFile(ctx, s, p, info, contents) 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package config defines the configuration options for rvcs. 16 | package config 17 | 18 | import ( 19 | "encoding/json" 20 | "fmt" 21 | "net/url" 22 | "os" 23 | "path/filepath" 24 | ) 25 | 26 | // Mirror defines the configuration for a single mirror 27 | type Mirror struct { 28 | // URL is the location of the mirror 29 | URL *url.URL 30 | 31 | // HelperFlags are command line arguments to pass to the mirror helper tool 32 | HelperFlags []string 33 | 34 | // ReadOnly indicates that the mirror should only be pulled from, and 35 | // not pushed to. 36 | ReadOnly bool 37 | } 38 | 39 | // MarshalJSON implements the json.Marshaler interface. 40 | func (m *Mirror) MarshalJSON() ([]byte, error) { 41 | var rawMap struct { 42 | URL string `json:"url"` 43 | HelperFlags []string `json:"helperFlags,omitempty"` 44 | ReadOnly bool `json:"readOnly,omitempty"` 45 | } 46 | rawMap.URL = m.URL.String() 47 | rawMap.HelperFlags = m.HelperFlags 48 | rawMap.ReadOnly = m.ReadOnly 49 | return json.Marshal(rawMap) 50 | } 51 | 52 | // UnmarshalJSON implements the json.Unmarshaler interface. 53 | func (m *Mirror) UnmarshalJSON(text []byte) error { 54 | var rawMap struct { 55 | URL string `json:"url"` 56 | HelperFlags []string `json:"helperFlags,omitempty"` 57 | ReadOnly bool `json:"readOnly,omitempty"` 58 | } 59 | if err := json.Unmarshal(text, &rawMap); err != nil { 60 | return fmt.Errorf("failure parsing the raw JSON for a mirror config: %v", err) 61 | } 62 | u, err := url.Parse(rawMap.URL) 63 | if err != nil { 64 | return fmt.Errorf("failure parsing the URL for a mirror config: %v", err) 65 | } 66 | m.URL = u 67 | m.HelperFlags = rawMap.HelperFlags 68 | m.ReadOnly = rawMap.ReadOnly 69 | return nil 70 | } 71 | 72 | func removeMirror(ms []*Mirror, u *url.URL) []*Mirror { 73 | var remaining []*Mirror 74 | target := u.String() 75 | for _, m := range ms { 76 | if m.URL.String() == target { 77 | continue 78 | } 79 | remaining = append(remaining, m) 80 | } 81 | return remaining 82 | } 83 | 84 | func addOrOverwriteMirror(ms []*Mirror, m *Mirror) []*Mirror { 85 | return append(removeMirror(ms, m.URL), m) 86 | } 87 | 88 | // Identity holds the config for a single identity used to sign and/or verify snapshots. 89 | type Identity struct { 90 | // Name is the name of the identity and must be able to be parsed by 91 | // the `snapshot.ParseIdentity` method. 92 | Name string `json:"name"` 93 | 94 | // Mirrors are a list of mirrors that we pull snapshots from, and 95 | // potentially push to (if they are not read-only) for the given 96 | // identity. 97 | Mirrors []*Mirror `json:"mirrors,omitempty"` 98 | } 99 | 100 | // Settings defines configuration settings for the rvcs tool. 101 | type Settings struct { 102 | // Identities is a list of configurations for each of the identities we keep track of. 103 | Identities []*Identity `json:"identities,omitempty"` 104 | 105 | // AdditionalMirrors is a list of mirrors which we will try to use for 106 | // any identities that do not have a matching entry in the 107 | // `identities` field. 108 | AdditionalMirrors []*Mirror `json:"additionalMirrors,omitempty"` 109 | } 110 | 111 | // Read reads in the configuration saved in the user's config directory. 112 | func Read() (*Settings, error) { 113 | cfgDir, err := os.UserConfigDir() 114 | if err != nil { 115 | return nil, fmt.Errorf("failure identifying the user config dir: %v", err) 116 | } 117 | rvcsCfgDir := filepath.Join(cfgDir, "rvcs") 118 | if err := os.MkdirAll(rvcsCfgDir, 0700); err != nil { 119 | return nil, fmt.Errorf("failure ensuring the config dir exists: %v", err) 120 | } 121 | cfgFile := filepath.Join(rvcsCfgDir, "config.json") 122 | 123 | var s Settings 124 | bs, err := os.ReadFile(cfgFile) 125 | if os.IsNotExist(err) { 126 | // The config file does not exist, so return an empty config. 127 | return &s, nil 128 | } 129 | if err := json.Unmarshal(bs, &s); err != nil { 130 | return nil, fmt.Errorf("failure parsing the config file contents: %v", err) 131 | } 132 | return &s, nil 133 | } 134 | 135 | // Write writes the given settings to the configuration saved in the user's config directory. 136 | func (s *Settings) Write() error { 137 | cfgDir, err := os.UserConfigDir() 138 | if err != nil { 139 | return fmt.Errorf("failure identifying the user config dir: %v", err) 140 | } 141 | rvcsCfgDir := filepath.Join(cfgDir, "rvcs") 142 | if err := os.MkdirAll(rvcsCfgDir, 0700); err != nil { 143 | return fmt.Errorf("failure ensuring the config dir exists: %v", err) 144 | } 145 | cfgFile := filepath.Join(rvcsCfgDir, "config.json") 146 | 147 | jsonBytes, err := json.Marshal(s) 148 | if err != nil { 149 | return err 150 | } 151 | return os.WriteFile(cfgFile, jsonBytes, 0700) 152 | } 153 | 154 | // WithAdditionalMirror returns a new Settings instance with the given additional mirror. 155 | // 156 | // If a mirror with the same URL already exists in the settings 157 | // `AdditionalMirrors` field, then it is replaced with the specified mirror. 158 | // 159 | // WithAdditionalMirror always returns a new Settings instance even if it is 160 | // identical to the original instance. 161 | func (s *Settings) WithAdditionalMirror(m *Mirror) *Settings { 162 | return &Settings{ 163 | Identities: s.Identities, 164 | AdditionalMirrors: addOrOverwriteMirror(s.AdditionalMirrors, m), 165 | } 166 | } 167 | 168 | // WithMirrorForIdentity returns a new Settings instance with the given mirror for the named identity. 169 | // 170 | // If the identity already has a mirror with the same URL, then that mirror 171 | // is replaced with the specified one. 172 | // 173 | // WithMirrorForIdentity always returns a new Settings instance even if it is 174 | // identical to the original instance. 175 | func (s *Settings) WithMirrorForIdentity(idName string, m *Mirror) *Settings { 176 | res := &Settings{ 177 | AdditionalMirrors: s.AdditionalMirrors, 178 | } 179 | for i, existingID := range s.Identities { 180 | if existingID.Name != idName { 181 | continue 182 | } 183 | updatedID := &Identity{ 184 | Name: idName, 185 | Mirrors: addOrOverwriteMirror(existingID.Mirrors, m), 186 | } 187 | res.Identities = append(append(s.Identities[:i], updatedID), s.Identities[i+1:]...) 188 | return res 189 | } 190 | res.Identities = append(s.Identities, &Identity{ 191 | Name: idName, 192 | Mirrors: []*Mirror{m}, 193 | }) 194 | return res 195 | } 196 | 197 | // WithoutAdditionalMirror returns a new Settings instance without the given mirror in the `AdditionalMirrors` field. 198 | // 199 | // WithoutAdditionalMirror always returns a new Settings instance even if it is 200 | // identical to the original instance. 201 | func (s *Settings) WithoutAdditionalMirror(u *url.URL) *Settings { 202 | return &Settings{ 203 | Identities: s.Identities, 204 | AdditionalMirrors: removeMirror(s.AdditionalMirrors, u), 205 | } 206 | } 207 | 208 | // WithoutMirrorForIdentity returns a new Settings instance without the given mirror for the named identity. 209 | // 210 | // WithoutMirrorForIdentity always returns a new Settings instance even if it 211 | // is identical to the original instance. 212 | func (s *Settings) WithoutMirrorForIdentity(idName string, u *url.URL) *Settings { 213 | res := &Settings{ 214 | AdditionalMirrors: s.AdditionalMirrors, 215 | } 216 | for i, existingID := range s.Identities { 217 | if existingID.Name != idName { 218 | continue 219 | } 220 | updatedID := &Identity{ 221 | Name: idName, 222 | Mirrors: removeMirror(existingID.Mirrors, u), 223 | } 224 | res.Identities = append(append(s.Identities[:i], updatedID), s.Identities[i+1:]...) 225 | return res 226 | } 227 | res.Identities = s.Identities 228 | return res 229 | } 230 | -------------------------------------------------------------------------------- /merge/merge.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package merge defines methods for merging two snapshots together. 16 | package merge 17 | 18 | import ( 19 | "bytes" 20 | "context" 21 | "errors" 22 | "fmt" 23 | "os" 24 | "path/filepath" 25 | "strings" 26 | 27 | "github.com/google/recursive-version-control-system/log" 28 | "github.com/google/recursive-version-control-system/snapshot" 29 | "github.com/google/recursive-version-control-system/storage" 30 | ) 31 | 32 | func IsAncestor(ctx context.Context, s *storage.LocalFiles, base, h *snapshot.Hash) (bool, error) { 33 | if base == nil { 34 | // The nil snapshot is an ancestor of all other snapshots. 35 | return true, nil 36 | } 37 | snapshotLog, err := log.ReadLog(ctx, s, h, -1) 38 | if err != nil { 39 | return false, fmt.Errorf("failure reading the log for %q: %v", h, err) 40 | } 41 | for _, e := range snapshotLog { 42 | if e.Hash.Equal(base) { 43 | return true, nil 44 | } 45 | } 46 | return false, nil 47 | } 48 | 49 | func mergeWithBase(ctx context.Context, s *storage.LocalFiles, subPath snapshot.Path, base, src, dest *snapshot.Hash, forceKeepMode bool) (*snapshot.Hash, error) { 50 | // First we handle the trivial cases where the merge result should 51 | // just be one of the two provided snapshots. 52 | if src.Equal(dest) { 53 | return src, nil 54 | } 55 | if src.Equal(base) { 56 | return dest, nil 57 | } 58 | if dest.Equal(base) { 59 | return src, nil 60 | } 61 | 62 | // If either the source or destination do not have the base as an 63 | // ancestor, then that means the changes in the base were rolled back 64 | // in that version. In that case, we have to ask the user to manually 65 | // merge the two versions. 66 | if src == nil || dest == nil { 67 | return nil, fmt.Errorf("the nested snapshot under the path %q was deleted in either the source or destination snapshot, so the two snapshots have to be manually merged", subPath) 68 | } 69 | if isAncestor, err := IsAncestor(ctx, s, base, src); err != nil { 70 | return nil, err 71 | } else if !isAncestor { 72 | // The changes from the base snapshot were rolled back in 73 | // the source... 74 | return nil, fmt.Errorf("nested changes under the path %q were rolled back in the source snapshot, so the two snapshots have to be manually merged", subPath) 75 | } 76 | if isAncestor, err := IsAncestor(ctx, s, base, dest); err != nil { 77 | return nil, err 78 | } else if !isAncestor { 79 | // The changes from the base snapshot were rolled back in 80 | // the destination... 81 | return nil, fmt.Errorf("nested changes under the path %q were rolled back in the destination snapshot, so the two snapshots have to be manually merged", subPath) 82 | } 83 | 84 | // For everything else we have to compare the actual snapshots, so 85 | // we first have to read both snapshots. 86 | srcFile, err := s.ReadSnapshot(ctx, src) 87 | if err != nil { 88 | return nil, fmt.Errorf("failure reading the file snapshot for %q: %v", src, err) 89 | } 90 | destFile, err := s.ReadSnapshot(ctx, dest) 91 | if err != nil { 92 | return nil, fmt.Errorf("failure reading the file snapshot for %q: %v", dest, err) 93 | } 94 | var baseFile *snapshot.File 95 | if base != nil { 96 | baseFile, err = s.ReadSnapshot(ctx, base) 97 | if err != nil { 98 | return nil, fmt.Errorf("failure reading the file snapshot for %q: %v", base, err) 99 | } 100 | } 101 | 102 | // If either the source or the destination are symbolic links, then 103 | // the user has to manually merge them. 104 | if srcFile.IsLink() || destFile.IsLink() { 105 | return nil, fmt.Errorf("one or both versions of the snapshot at %q represent a symlink, so the two snapshots for that path have to be manually merged", subPath) 106 | } 107 | 108 | if !(srcFile.IsDir() && destFile.IsDir()) { 109 | return mergeWithHelper(ctx, s, subPath, destFile.Mode, base, src, dest) 110 | } 111 | 112 | // Both source and destination are directories, so we recursively 113 | // merge every nested path under either of them using the corresponding 114 | // nested path from the base as a reference point. 115 | srcTree, err := s.ListDirectorySnapshotContents(ctx, src, srcFile) 116 | if err != nil { 117 | return nil, fmt.Errorf("failure reading the tree for the snapshot %q: %v", src, err) 118 | } 119 | destTree, err := s.ListDirectorySnapshotContents(ctx, dest, destFile) 120 | if err != nil { 121 | return nil, fmt.Errorf("failure reading the tree for the snapshot %q: %v", dest, err) 122 | } 123 | var baseTree snapshot.Tree 124 | if baseFile.IsDir() { 125 | baseTree, err = s.ListDirectorySnapshotContents(ctx, base, baseFile) 126 | if err != nil { 127 | return nil, fmt.Errorf("failure reading the tree for the snapshot %q: %v", base, err) 128 | } 129 | } else { 130 | // The base was a different type, so each subpath of it should 131 | // just be nil 132 | baseTree = make(snapshot.Tree) 133 | } 134 | 135 | mergedTree := make(snapshot.Tree) 136 | subpaths := make(map[snapshot.Path]struct{}) 137 | for p, _ := range srcTree { 138 | subpaths[p] = struct{}{} 139 | } 140 | for p, _ := range destTree { 141 | subpaths[p] = struct{}{} 142 | } 143 | var nestedErrors []string 144 | for p, _ := range subpaths { 145 | childSubPath := subPath.Join(p) 146 | childBase := baseTree[p] 147 | childSrc := srcTree[p] 148 | childDest := destTree[p] 149 | mergedChild, err := mergeWithBase(ctx, s, childSubPath, childBase, childSrc, childDest, forceKeepMode) 150 | if err != nil { 151 | nestedErrors = append(nestedErrors, err.Error()) 152 | } 153 | if mergedChild != nil { 154 | mergedTree[p] = mergedChild 155 | } 156 | } 157 | if srcFile.Mode != destFile.Mode && !forceKeepMode { 158 | nestedErrors = append(nestedErrors, fmt.Sprintf("file permissions for %q do not match between versions; source mode line: %q, destination mode line %q. Manually update the permissions for the source to match what you want for the merge result, and then re-run the merge with the option to force using the source permissions", subPath, srcFile.Mode, destFile.Mode)) 159 | } 160 | if len(nestedErrors) > 0 { 161 | return nil, errors.New(strings.Join(nestedErrors, "\n")) 162 | } 163 | 164 | contentsBytes := []byte(mergedTree.String()) 165 | contentsHash, err := s.StoreObject(ctx, int64(len(contentsBytes)), bytes.NewReader(contentsBytes)) 166 | if err != nil { 167 | return nil, fmt.Errorf("failure storing the contents of a merged tree: %v", err) 168 | } 169 | mergedFile := &snapshot.File{ 170 | Mode: srcFile.Mode, 171 | Contents: contentsHash, 172 | Parents: []*snapshot.Hash{src, dest}, 173 | } 174 | fileBytes := []byte(mergedFile.String()) 175 | h, err := s.StoreObject(ctx, int64(len(fileBytes)), bytes.NewReader(fileBytes)) 176 | if err != nil { 177 | return nil, fmt.Errorf("failure storing the merged snapshot: %v", err) 178 | } 179 | return h, nil 180 | } 181 | 182 | // Merge attempts to automatically merge the given snapshot into the local 183 | // filesystem at the specified destination path. 184 | // 185 | // If there are any conflicts between the specified snapshot and the local 186 | // filesystem contents, then the `Merge` method retursn an error without 187 | // modifying the local filesystem. 188 | // 189 | // In case there are no conflicts but the local storage is missing some 190 | // referenced snapshots, then it is possible for this method to both modify 191 | // the local filesystem contents *and* to also return an error. In that case 192 | // the previous version of the local filesystem contents will be retrievable 193 | // using the `rvcs log` command. 194 | func Merge(ctx context.Context, s *storage.LocalFiles, src *snapshot.Hash, dest snapshot.Path) error { 195 | destParent := filepath.Dir(string(dest)) 196 | if err := os.MkdirAll(destParent, os.FileMode(0700)); err != nil { 197 | return fmt.Errorf("failure ensuring the parent directory of %q exists: %v", dest, err) 198 | } 199 | destPrevHash, _, err := snapshot.Current(ctx, s, dest) 200 | if err != nil { 201 | return fmt.Errorf("failure generating snapshot of destination %q prior to merging: %v", dest, err) 202 | } 203 | if destPrevHash == nil { 204 | // The destination does not exist; simply check out the source hash there. 205 | return Checkout(ctx, s, src, dest) 206 | } 207 | mergeBase, err := Base(ctx, s, src, destPrevHash) 208 | if err != nil { 209 | return fmt.Errorf("failure determining the merge base for %q and %q: %v", src, destPrevHash, err) 210 | } 211 | if mergeBase.Equal(src) { 212 | // The source has already been merged in 213 | return nil 214 | } 215 | 216 | mergedHash, err := mergeWithBase(ctx, s, dest, mergeBase, src, destPrevHash, false) 217 | if err != nil { 218 | return fmt.Errorf("unable to automatically merge the two snapshots: %v", err) 219 | } 220 | 221 | // Update the destination to point to the merged snapshot 222 | if err := os.RemoveAll(string(dest)); err != nil { 223 | return fmt.Errorf("failure updating %q to point to newer snapshot %q; failure removing old files: %v", dest, mergedHash, err) 224 | } 225 | return Checkout(ctx, s, mergedHash, dest) 226 | } 227 | -------------------------------------------------------------------------------- /bundle/bundle.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package bundle defines methods for bundling snapshots together so they can be imported and/or exported. 16 | package bundle 17 | 18 | import ( 19 | "archive/zip" 20 | "context" 21 | "fmt" 22 | "io" 23 | "os" 24 | "path" 25 | "strings" 26 | "sync" 27 | 28 | "github.com/google/recursive-version-control-system/snapshot" 29 | "github.com/google/recursive-version-control-system/storage" 30 | ) 31 | 32 | func bundleEntryPath(h *snapshot.Hash) string { 33 | if len(h.HexContents()) > 4 { 34 | return path.Join("objects", h.Function(), h.HexContents()[0:2], h.HexContents()[2:4], h.HexContents()[4:]) 35 | } 36 | if len(h.HexContents()) > 2 { 37 | return path.Join("objects", h.Function(), h.HexContents()[0:2], h.HexContents()[2:]) 38 | } 39 | return path.Join("objects", h.Function(), h.HexContents()) 40 | } 41 | 42 | func bundlePathHash(path string) (*snapshot.Hash, error) { 43 | if !strings.HasPrefix(path, "objects/") { 44 | return nil, fmt.Errorf("Path %q is not an object path", path) 45 | } 46 | p := strings.TrimPrefix(path, "objects/") 47 | parts := strings.Split(p, "/") 48 | if len(parts) < 2 { 49 | return nil, fmt.Errorf("Path %q does not correspond to a valid hash", path) 50 | } 51 | f := parts[0] 52 | c := strings.Join(parts[1:], "") 53 | return snapshot.ParseHash(fmt.Sprintf("%s:%s", f, c)) 54 | } 55 | 56 | type ZipWriter struct { 57 | nested *zip.Writer 58 | visited map[snapshot.Hash]struct{} 59 | exclude map[snapshot.Hash]struct{} 60 | recurseParents bool 61 | 62 | mu sync.Mutex 63 | included []*snapshot.Hash 64 | } 65 | 66 | func NewZipWriter(w io.Writer, exclude []*snapshot.Hash, metadata map[string]io.ReadCloser, recurseParents bool) (*ZipWriter, error) { 67 | excludeMap := make(map[snapshot.Hash]struct{}) 68 | for _, h := range exclude { 69 | excludeMap[*h] = struct{}{} 70 | } 71 | nested := zip.NewWriter(w) 72 | for name, r := range metadata { 73 | fw, err := nested.Create(path.Join("metadata", name)) 74 | if err != nil { 75 | return nil, fmt.Errorf("failure creating a zip file entry for metadata key %q: %v", name, err) 76 | } 77 | if _, err := io.Copy(fw, r); err != nil { 78 | return nil, fmt.Errorf("failure writing the zip file entry for metadata key %q: %v", name, err) 79 | } 80 | if err := r.Close(); err != nil { 81 | return nil, fmt.Errorf("failure closing the metadata reader: %v", err) 82 | } 83 | } 84 | return &ZipWriter{ 85 | nested: nested, 86 | visited: make(map[snapshot.Hash]struct{}), 87 | exclude: excludeMap, 88 | recurseParents: recurseParents, 89 | }, nil 90 | } 91 | 92 | func (w *ZipWriter) Close() error { 93 | return w.nested.Close() 94 | } 95 | 96 | func (w *ZipWriter) AddObject(ctx context.Context, s *storage.LocalFiles, h *snapshot.Hash) error { 97 | if _, ok := w.exclude[*h]; ok { 98 | // We are explicitly excluding this object. 99 | return nil 100 | } 101 | if _, ok := w.visited[*h]; ok { 102 | // We already added this to the zip writer. 103 | return nil 104 | } 105 | w.mu.Lock() 106 | defer w.mu.Unlock() 107 | w.visited[*h] = struct{}{} 108 | r, err := s.ReadObject(ctx, h) 109 | if err != nil { 110 | return fmt.Errorf("failure opening the contents of the object %q: %v", h, err) 111 | } 112 | defer r.Close() 113 | fw, err := w.nested.Create(bundleEntryPath(h)) 114 | if err != nil { 115 | return fmt.Errorf("failure creating the zip file entry for %q: %v", h, err) 116 | } 117 | if _, err := io.Copy(fw, r); err != nil { 118 | return fmt.Errorf("failure writing the zip file entry for %q: %v", h, err) 119 | } 120 | w.included = append(w.included, h) 121 | return nil 122 | } 123 | 124 | func (w *ZipWriter) AddFile(ctx context.Context, s *storage.LocalFiles, h *snapshot.Hash, f *snapshot.File) (err error) { 125 | if err := w.AddObject(ctx, s, h); err != nil { 126 | return fmt.Errorf("failure adding the snapshot %q to the bundle: %v", h, err) 127 | } 128 | defer func() { 129 | if err != nil { 130 | return 131 | } 132 | if !w.recurseParents { 133 | return 134 | } 135 | for _, parentHash := range f.Parents { 136 | parent, err := s.ReadSnapshot(ctx, parentHash) 137 | if err != nil { 138 | // The history is incomplete 139 | continue 140 | } 141 | err = w.AddFile(ctx, s, parentHash, parent) 142 | if err != nil { 143 | err = fmt.Errorf("failure adding the parent %q to the bundle: %v", parentHash, err) 144 | return 145 | } 146 | } 147 | }() 148 | if f.Contents == nil { 149 | return nil 150 | } 151 | if err := w.AddObject(ctx, s, f.Contents); err != nil { 152 | return fmt.Errorf("failure adding the contents of the snapshot %q to the bundle: %v", h, err) 153 | } 154 | if !f.IsDir() { 155 | return nil 156 | } 157 | tree, err := s.ListDirectorySnapshotContents(ctx, h, f) 158 | if err != nil { 159 | return fmt.Errorf("failure reading the contents of the directory snapshot %q: %v", h, err) 160 | } 161 | for _, childHash := range tree { 162 | if _, ok := w.exclude[*childHash]; ok { 163 | continue 164 | } 165 | if _, ok := w.visited[*childHash]; ok { 166 | continue 167 | } 168 | child, err := s.ReadSnapshot(ctx, childHash) 169 | if err != nil { 170 | return fmt.Errorf("failure reading the snapshot %q: %v", childHash, err) 171 | } 172 | if err := w.AddFile(ctx, s, childHash, child); err != nil { 173 | return fmt.Errorf("failure adding the child %q to the bundle: %v", childHash, err) 174 | } 175 | } 176 | return nil 177 | } 178 | 179 | // Export writes a bundle with the specified snapshots to the given writer. 180 | // 181 | // If the returned error is nil, then the written bundle will include the 182 | // specified snapshots, and their contents. For any snapshots of a directory, 183 | // the bundle will also recursively include the snapshots for the children 184 | // of that directory. 185 | // 186 | // The `exclude` argument specifies a list of objects (by hash) that will 187 | // not be included in the resulting bundle even if they otherwise would 188 | // have been. 189 | // 190 | // The `metadata` argument specifies an additional map of key/value pairs 191 | // to include in the bundle in a separate subpath from the bundled objects. 192 | func Export(ctx context.Context, s *storage.LocalFiles, path string, snapshots []*snapshot.Hash, exclude []*snapshot.Hash, metadata map[string]io.ReadCloser, recurseParents bool) (included []*snapshot.Hash, err error) { 193 | w, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0700) 194 | if err != nil { 195 | return nil, fmt.Errorf("failure opening the file %q: %v", path, err) 196 | } 197 | defer w.Close() 198 | zw, err := NewZipWriter(w, exclude, metadata, recurseParents) 199 | if err != nil { 200 | return nil, fmt.Errorf("failure creating the zip writer for the bundle: %v", err) 201 | } 202 | defer func() { 203 | ce := zw.Close() 204 | if err == nil { 205 | err = ce 206 | } 207 | }() 208 | 209 | for _, h := range snapshots { 210 | f, err := s.ReadSnapshot(ctx, h) 211 | if err != nil { 212 | return nil, fmt.Errorf("failure reading the snapshot %q: %v", h, err) 213 | } 214 | if err := zw.AddFile(ctx, s, h, f); err != nil { 215 | return nil, fmt.Errorf("failure adding %q to the zip file: %v", h, err) 216 | } 217 | } 218 | return zw.included, nil 219 | } 220 | 221 | func validateZipEntry(ctx context.Context, f *zip.File) error { 222 | h, err := bundlePathHash(f.Name) 223 | if err != nil { 224 | // We allow additional/non-object files in bundles 225 | return nil 226 | } 227 | r, err := f.Open() 228 | if err != nil { 229 | return fmt.Errorf("failure reading entry %q: %v", f.Name, err) 230 | } 231 | realHash, err := snapshot.NewHash(r) 232 | if err != nil { 233 | return fmt.Errorf("failure hashing the entry %q: %v", f.Name, err) 234 | } 235 | if !realHash.Equal(h) { 236 | return fmt.Errorf("mismatched hash for entry %q: got %q, want %q", f.Name, realHash, h) 237 | } 238 | return nil 239 | } 240 | 241 | func Import(ctx context.Context, s *storage.LocalFiles, path string, exclude []*snapshot.Hash) (included []*snapshot.Hash, err error) { 242 | r, err := zip.OpenReader(path) 243 | if err != nil { 244 | return nil, fmt.Errorf("failure opening the zip file %q: %v", path, err) 245 | } 246 | defer r.Close() 247 | // We first validate that the bundle only includes valid object contents... 248 | for _, f := range r.File { 249 | if err := validateZipEntry(ctx, f); err != nil { 250 | return nil, fmt.Errorf("failure validating the zip entry %q: %v", f.Name, err) 251 | } 252 | } 253 | for _, f := range r.File { 254 | h, err := bundlePathHash(f.Name) 255 | if err != nil { 256 | // We allow additional/non-object files in bundles 257 | continue 258 | } 259 | if _, err := s.ReadObject(ctx, h); err == nil { 260 | // We already have this object and can skip importing it. 261 | continue 262 | } 263 | r, err := f.Open() 264 | if err != nil { 265 | return nil, fmt.Errorf("failure reading entry %q: %v", f.Name, err) 266 | } 267 | if h, err := s.StoreObject(ctx, int64(f.FileInfo().Size()), r); err != nil { 268 | return nil, fmt.Errorf("failure importing the zip entry %q: %v", f.Name, err) 269 | } else { 270 | included = append(included, h) 271 | } 272 | } 273 | return included, nil 274 | } 275 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /merge/merge_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package merge defines methods for merging two snapshots together. 16 | package merge 17 | 18 | import ( 19 | "context" 20 | "os" 21 | "path/filepath" 22 | "testing" 23 | 24 | "github.com/google/recursive-version-control-system/snapshot" 25 | "github.com/google/recursive-version-control-system/storage" 26 | ) 27 | 28 | func TestMergeRegularFile(t *testing.T) { 29 | dir := t.TempDir() 30 | archive := filepath.Join(dir, "archive") 31 | s := &storage.LocalFiles{ArchiveDir: archive} 32 | 33 | original := filepath.Join(dir, "original.txt") 34 | originalPath := snapshot.Path(original) 35 | 36 | // Take an initial snapshot 37 | if err := os.WriteFile(original, []byte("Hello, World!"), 0700); err != nil { 38 | t.Fatalf("failure creating the example file to snapshot: %v", err) 39 | } 40 | h1, f1, err := snapshot.Current(context.Background(), s, originalPath) 41 | if err != nil { 42 | t.Fatalf("failure creating the initial snapshot for the file: %v", err) 43 | } else if h1 == nil { 44 | t.Fatalf("unexpected nil hash for the file") 45 | } else if f1 == nil { 46 | t.Fatalf("unexpected nil snapshot for the file") 47 | } 48 | 49 | clone := filepath.Join(dir, "clone.txt") 50 | clonePath := snapshot.Path(clone) 51 | 52 | if err := Merge(context.Background(), s, h1, clonePath); err != nil { 53 | t.Fatalf("failure checking out the file snapshot %q: %v", h1, err) 54 | } 55 | 56 | // Validate that the cloned file matches the original... 57 | verifyFilesMatch(t, original, clone) 58 | h2, f2, err := snapshot.Current(context.Background(), s, clonePath) 59 | if err != nil { 60 | t.Errorf("failure creating the cloned snapshot for the file: %v", err) 61 | } else if got, want := h2, h1; !got.Equal(want) { 62 | t.Errorf("unexpected hash for the cloned file; got %q, want %q", got, want) 63 | } else if got, want := f2.String(), f1.String(); got != want { 64 | t.Errorf("unexpected contents for the cloned snapshot for the file: got %q, want %q", got, want) 65 | } 66 | } 67 | 68 | func TestMergeSymlink(t *testing.T) { 69 | dir := t.TempDir() 70 | archive := filepath.Join(dir, "archive") 71 | s := &storage.LocalFiles{ArchiveDir: archive} 72 | 73 | target := filepath.Join(dir, "target.txt") 74 | if err := os.WriteFile(target, []byte("Hello, World!"), 0700); err != nil { 75 | t.Fatalf("failure creating the example file to target: %v", err) 76 | } 77 | 78 | original := filepath.Join(dir, "original.txt") 79 | originalPath := snapshot.Path(original) 80 | if err := os.Symlink(target, original); err != nil { 81 | t.Fatalf("failure creating the example symlink: %v", err) 82 | } 83 | 84 | // Take an initial snapshot 85 | h1, f1, err := snapshot.Current(context.Background(), s, originalPath) 86 | if err != nil { 87 | t.Fatalf("failure creating the initial snapshot for the symlink: %v", err) 88 | } else if h1 == nil { 89 | t.Fatalf("unexpected nil hash for the symlink") 90 | } else if f1 == nil { 91 | t.Fatalf("unexpected nil snapshot for the symlink") 92 | } 93 | 94 | clone := filepath.Join(dir, "clone.txt") 95 | clonePath := snapshot.Path(clone) 96 | if err := Merge(context.Background(), s, h1, clonePath); err != nil { 97 | t.Fatalf("failure checking out the symlink snapshot %q: %v", h1, err) 98 | } 99 | 100 | // Validate that the cloned file matches the original... 101 | if originalTarget, err := os.Readlink(original); err != nil { 102 | t.Errorf("failure reading the original symlink target: %v", err) 103 | } else if clonedTarget, err := os.Readlink(clone); err != nil { 104 | t.Errorf("failure reading the cloned symlink target: %v", err) 105 | } else if got, want := originalTarget, clonedTarget; got != want { 106 | t.Errorf("unexpected target for cloned symlink; got %q, want %q", got, want) 107 | } 108 | h2, f2, err := snapshot.Current(context.Background(), s, clonePath) 109 | if err != nil { 110 | t.Errorf("failure creating the cloned snapshot for the symlink: %v", err) 111 | } else if got, want := h2, h1; !got.Equal(want) { 112 | t.Errorf("unexpected hash for the cloned symlink; got %q, want %q", got, want) 113 | } else if got, want := f2.String(), f1.String(); got != want { 114 | t.Errorf("unexpected contents for the cloned snapshot for the symlink: got %q, want %q", got, want) 115 | } 116 | } 117 | 118 | func TestMergeExcludedDir(t *testing.T) { 119 | dir := t.TempDir() 120 | archive := filepath.Join(dir, "archive") 121 | s := &storage.LocalFiles{ArchiveDir: archive} 122 | 123 | file := filepath.Join(dir, "example.txt") 124 | if err := os.WriteFile(file, []byte("Hello, World!"), 0700); err != nil { 125 | t.Fatalf("failure creating the example file: %v", err) 126 | } 127 | 128 | dirPath := snapshot.Path(dir) 129 | h1, f1, err := snapshot.Current(context.Background(), s, dirPath) 130 | if err != nil { 131 | t.Fatalf("failure creating the initial snapshot for the directory: %v", err) 132 | } else if h1 == nil { 133 | t.Fatalf("unexpected nil hash for the directory") 134 | } else if f1 == nil { 135 | t.Fatalf("unexpected nil snapshot for the directory") 136 | } 137 | 138 | // Verify that the snapshot does not include the storage archive... 139 | if tree, err := s.ListDirectorySnapshotContents(context.Background(), h1, f1); err != nil { 140 | t.Fatalf("failure reading the contents of the directory snapshot %q: %v", h1, err) 141 | } else if _, ok := tree[snapshot.Path("archive")]; ok { 142 | t.Error("unexpectedly included the storage archive in the snapshot") 143 | } 144 | 145 | if err := Merge(context.Background(), s, h1, dirPath); err != nil { 146 | t.Fatalf("failure checking out the directory snapshot %q: %v", h1, err) 147 | } 148 | 149 | // Verify that the storage archive has not been removed... 150 | if f2, err := s.ReadSnapshot(context.Background(), h1); err != nil { 151 | t.Errorf("failure reading the snapshot back from storage: %v", err) 152 | } else if got, want := f2.String(), f1.String(); got != want { 153 | t.Errorf("unexpected snapshot read back from storage: got %q, want %q", got, want) 154 | } 155 | } 156 | 157 | func TestMergeDir(t *testing.T) { 158 | dir := t.TempDir() 159 | archive := filepath.Join(dir, "archive") 160 | s := &storage.LocalFiles{ArchiveDir: archive} 161 | 162 | workingDir := filepath.Join(dir, "working-dir") 163 | if err := os.Mkdir(workingDir, 0700); err != nil { 164 | t.Fatalf("failure creating the working directory for the test: %v", err) 165 | } 166 | dirPath := snapshot.Path(workingDir) 167 | file1 := filepath.Join(workingDir, "example1.txt") 168 | file2 := filepath.Join(workingDir, "example2.txt") 169 | file3 := filepath.Join(workingDir, "example3.txt") 170 | 171 | if err := os.WriteFile(file1, []byte("Hello, World 1!"), 0700); err != nil { 172 | t.Fatalf("failure creating the example file 1: %v", err) 173 | } 174 | if err := os.WriteFile(file2, []byte("Hello, World 2!"), 0700); err != nil { 175 | t.Fatalf("failure creating the example file 2: %v", err) 176 | } 177 | if err := os.WriteFile(file3, []byte("Hello, World 3!"), 0700); err != nil { 178 | t.Fatalf("failure creating the example file 3: %v", err) 179 | } 180 | 181 | h1, f1, err := snapshot.Current(context.Background(), s, dirPath) 182 | if err != nil { 183 | t.Fatalf("failure creating the initial snapshot for the directory: %v", err) 184 | } else if h1 == nil { 185 | t.Fatalf("unexpected nil hash for the directory") 186 | } else if f1 == nil { 187 | t.Fatalf("unexpected nil snapshot for the directory") 188 | } 189 | 190 | cloneDir := filepath.Join(dir, "clone-dir") 191 | cloneDirPath := snapshot.Path(cloneDir) 192 | if err := Merge(context.Background(), s, h1, cloneDirPath); err != nil { 193 | t.Fatalf("failure checking out the directory snapshot %q: %v", h1, err) 194 | } 195 | verifyFilesMatch(t, file1, filepath.Join(cloneDir, "example1.txt")) 196 | verifyFilesMatch(t, file2, filepath.Join(cloneDir, "example2.txt")) 197 | verifyFilesMatch(t, file3, filepath.Join(cloneDir, "example3.txt")) 198 | } 199 | 200 | func TestMergeNonConflictingChangesDir(t *testing.T) { 201 | dir := t.TempDir() 202 | archive := filepath.Join(dir, "archive") 203 | s := &storage.LocalFiles{ArchiveDir: archive} 204 | 205 | workingDir := filepath.Join(dir, "working-dir") 206 | if err := os.Mkdir(workingDir, 0700); err != nil { 207 | t.Fatalf("failure creating the working directory for the test: %v", err) 208 | } 209 | dirPath := snapshot.Path(workingDir) 210 | file1 := filepath.Join(workingDir, "example1.txt") 211 | file2 := filepath.Join(workingDir, "example2.txt") 212 | file3 := filepath.Join(workingDir, "example3.txt") 213 | 214 | if err := os.WriteFile(file1, []byte("Hello, World 1!"), 0700); err != nil { 215 | t.Fatalf("failure creating the example file 1: %v", err) 216 | } 217 | if err := os.WriteFile(file2, []byte("Hello, World 2!"), 0700); err != nil { 218 | t.Fatalf("failure creating the example file 2: %v", err) 219 | } 220 | if err := os.WriteFile(file3, []byte("Hello, World 3!"), 0700); err != nil { 221 | t.Fatalf("failure creating the example file 3: %v", err) 222 | } 223 | 224 | h1, f1, err := snapshot.Current(context.Background(), s, dirPath) 225 | if err != nil { 226 | t.Fatalf("failure creating the initial snapshot for the directory: %v", err) 227 | } else if h1 == nil { 228 | t.Fatalf("unexpected nil hash for the directory") 229 | } else if f1 == nil { 230 | t.Fatalf("unexpected nil snapshot for the directory") 231 | } 232 | 233 | cloneDir := filepath.Join(dir, "clone-dir") 234 | cloneDirPath := snapshot.Path(cloneDir) 235 | if err := Merge(context.Background(), s, h1, cloneDirPath); err != nil { 236 | t.Fatalf("failure checking out the directory snapshot %q: %v", h1, err) 237 | } 238 | verifyFilesMatch(t, file1, filepath.Join(cloneDir, "example1.txt")) 239 | verifyFilesMatch(t, file2, filepath.Join(cloneDir, "example2.txt")) 240 | verifyFilesMatch(t, file3, filepath.Join(cloneDir, "example3.txt")) 241 | 242 | if err := os.WriteFile(file1, []byte("Hello, World 1, v2!"), 0700); err != nil { 243 | t.Fatalf("failure updating the example file 1: %v", err) 244 | } 245 | if err := os.WriteFile(filepath.Join(cloneDir, "example2.txt"), []byte("Hello, World 2, v2!"), 0700); err != nil { 246 | t.Fatalf("failure updating the example file 2: %v", err) 247 | } 248 | 249 | h2, f2, err := snapshot.Current(context.Background(), s, dirPath) 250 | if err != nil { 251 | t.Fatalf("failure creating the updated snapshot for the directory: %v", err) 252 | } else if h2 == nil { 253 | t.Fatalf("unexpected nil hash for the directory") 254 | } else if f2 == nil { 255 | t.Fatalf("unexpected nil snapshot for the directory") 256 | } 257 | h3, f3, err := snapshot.Current(context.Background(), s, cloneDirPath) 258 | if err != nil { 259 | t.Fatalf("failure creating the updated snapshot for the cloned directory: %v", err) 260 | } else if h3 == nil { 261 | t.Fatalf("unexpected nil hash for the directory") 262 | } else if f3 == nil { 263 | t.Fatalf("unexpected nil snapshot for the directory") 264 | } 265 | 266 | mergeDir := filepath.Join(dir, "merge-dir") 267 | mergeDirPath := snapshot.Path(mergeDir) 268 | if err := Checkout(context.Background(), s, h2, mergeDirPath); err != nil { 269 | t.Fatalf("failure checking out the directory snapshot %q: %v", h2, err) 270 | } 271 | if err := Merge(context.Background(), s, h3, mergeDirPath); err != nil { 272 | t.Fatalf("failure checking out the directory snapshot %q: %v", h1, err) 273 | } 274 | verifyFilesMatch(t, file1, filepath.Join(mergeDir, "example1.txt")) 275 | verifyFilesMatch(t, filepath.Join(cloneDir, "example2.txt"), filepath.Join(mergeDir, "example2.txt")) 276 | verifyFilesMatch(t, file3, filepath.Join(mergeDir, "example3.txt")) 277 | } 278 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Recursive Version Control System 2 | 3 | This repository contains an *EXPERIMENTAL* version control system. 4 | 5 | The aim of this experiment is to explore an alternative object model for 6 | distributed version control systems. That model is designed to be as simple 7 | as possible and has fewer concepts than existing DVCS's like git and Mercurial. 8 | 9 | ## Disclaimer 10 | 11 | This is not an officially supported Google product. 12 | 13 | ## Overview 14 | 15 | The recursive version control system (rvcs) tracks the version history of 16 | individual files and directories. For a directory, this history includes the 17 | histories of all the files in that directory. 18 | 19 | That hierarchical structure means you can share your files at any level 20 | and can share different files/directories with different audiences. 21 | 22 | To share a file with others, you publish it by signing its history. 23 | 24 | Published files are automatically copied to and from a set of mirrors that 25 | you configure. 26 | 27 | The recursive nature of the history tracking means that you can use the same 28 | tool for tracking the history of a single file, an entire directory, or even 29 | your entire file system. 30 | 31 | ## Usage 32 | 33 | Snapshot the current contents of a file: 34 | 35 | ```shell 36 | rvcs snapshot 37 | ``` 38 | 39 | Publish the most recent snapshot of a file by signing it: 40 | 41 | ```shell 42 | rvcs publish 43 | ``` 44 | 45 | Merge in changes from the most recent snapshot signed by someone: 46 | 47 | ```shell 48 | rvcs merge 49 | ``` 50 | 51 | ## Getting Started 52 | 53 | ### Installation 54 | 55 | If you have the [Go tools installed](https://golang.org/doc/install), you can 56 | install the `rvcs` tool by running the following command: 57 | 58 | go install github.com/google/recursive-version-control-system/cmd/...@latest 59 | 60 | Then, make sure that `${GOPATH}/bin` is in your PATH. 61 | 62 | Optionally, you can also copy the files from the `extensions` directory into some directory in your PATH to use them for publishing snapshots. 63 | 64 | ### Example Setup And Usage 65 | 66 | The `extensions` directory includes helpers for using SSH public keys as 67 | identities and local filesystem paths as mirrors. So, if you have them 68 | installed then you can set up an example identity and mirror using the 69 | following commands: 70 | 71 | ```shell 72 | ssh-keygen -t ed25519 -f ~/.ssh/rvcs_example -C "Example identity for RVCS" 73 | export RVCS_EXAMPLE_IDENTITY="ssh::$(cat ~/.ssh/rvcs_example.pub | cut -d ' ' -f 2)" 74 | mkdir -p ${HOME}/rvcs-example/local-filesystem-mirror 75 | export RVCS_EXAMPLE_MIRROR="file://${HOME}/rvcs-example/local-filesystem-mirror" 76 | rvcs add-mirror ${RVCS_EXAMPLE_IDENTITY} ${RVCS_EXAMPLE_MIRROR} 77 | ``` 78 | 79 | Then you can publish an example directory to that identity with the following: 80 | 81 | ```shell 82 | mkdir -p ${HOME}/rvcs-example/dir-to-publish 83 | echo "Hello, World\!" > ${HOME}/rvcs-example/dir-to-publish/hello.txt 84 | rvcs snapshot ${HOME}/rvcs-example/dir-to-publish 85 | rvcs publish ${HOME}/rvcs-example/dir-to-publish ${RVCS_EXAMPLE_IDENTITY} 86 | ``` 87 | 88 | If you share the local mirror (the directory you created at 89 | `${HOME}/rvcs-example/local-filesystem-mirror`) with another user, they 90 | can then retrieve your published snapshot with: 91 | 92 | ```shell 93 | rvcs add-mirror --read-only ${RVCS_EXAMPLE_IDENTITY} ${RVCS_EXAMPLE_MIRROR} 94 | rvcs merge ${RVCS_EXAMPLE_IDENTITY} ~/rvcs-example-merge 95 | ``` 96 | 97 | After you are done with the example, you can clean up by removing the mirror: 98 | 99 | ```shell 100 | rvcs remove-mirror ${RVCS_EXAMPLE_IDENTITY} ${RVCS_EXAMPLE_MIRROR} 101 | ``` 102 | 103 | ## Status 104 | 105 | This is *experimental* and very much a work-in-progress. 106 | 107 | In particular, the tool is still subject to change and there is no guarantee 108 | of backwards compatibility. 109 | 110 | The `snapshot` command is fully implemented, and no changes are currently 111 | planned for it, but that is subject to change. 112 | 113 | The `publish` and `merge` commands are both implemented, but rely on external 114 | helper commands in order to actually use them. 115 | 116 | The `merge` command defaults to using the widely-available `diff3` command if 117 | no helper is provided. For the `publish` command, there are proof of concept 118 | helpers provided in the `extensions` directory. 119 | 120 | ## Model 121 | 122 | The core concept in rvcs is a `snapshot`. A snapshot describes a point-in-time 123 | in the history of a file, where the file might be a regular file or a directory 124 | containing other files. 125 | 126 | Each snapshot contains a fixed set of metadata about the file (such as whether 127 | or not it is a directory), a link to the contents of the file at that point, 128 | and links to any other snapshots that came immediately before it. 129 | 130 | These links in a snapshot are of the form `:`, 131 | where `` is the name of a specific 132 | [function](https://en.wikipedia.org/wiki/Hash_function) used to generate 133 | a hash, and `` is the generated hash of the thing being 134 | referenced. Currently, the only supported hash function is 135 | [sha256](https://en.wikipedia.org/wiki/SHA-2). 136 | 137 | When the snapshot is for a directory, the contents are a plain text file 138 | listing the names of each file contained in that directory, and that file's 139 | corresponding snapshot. 140 | 141 | ## Publishing Snapshots 142 | 143 | You share snapshots with others by "publishing" them. This consists of signing 144 | the snapshot by generating a signature for it tied to some identity you 145 | control. 146 | 147 | The rvcs tool does not mandate a specific format or type for signatures. 148 | Instead, it allows you to configure external tools used for generating and 149 | validating signatures. 150 | 151 | That, in turn, is the primary extension mechanism for rvcs, as signature 152 | types can be defined to hold any data you want. 153 | 154 | ### Sign and Verify Helpers 155 | 156 | Identities are of the form `::`. In order to be able 157 | to publish a snapshot with a given identity, you must have "sign" and "verify" 158 | helpers located somewhere in your local system path. 159 | 160 | These helpers will always be named of the form `rvcs-sign-` and 161 | `rvcs-verify-`, where `` is the prefix of the identity 162 | that comes before the first pair of colons. 163 | 164 | So, for example, to publish a snapshot with the identity `example::user`, 165 | you must have two programs in your system path named `rvcs-sign-example` and 166 | `rvcs-verify-example`. 167 | 168 | The sign helper takes four arguments; the full contents of the 169 | identity (e.g. `example::user` for the example above), the hash of the 170 | snapshot to sign, the hash of the previous signature created for that 171 | identity (or the empty string if there is none), and a file to which it 172 | writes its output. 173 | 174 | If it is successful, then it writes to the output file the hash of the 175 | snapshot of the generated signature and exits with a status code of `0`. 176 | 177 | The verify helper does the reverse of that. It takes three arguments; the 178 | identity, the hash of the generated signature, and a file to write output. 179 | It then verifies that this signature is valid for the specified identity. 180 | 181 | If it is, then the verify helper outputs the hash of the signed snapshot 182 | and exits with a status code of `0`. 183 | 184 | There are example sign and verify helpers in the `extensions` directory that 185 | demonstrate how to sign and verify signatures using SSH keys. 186 | 187 | ## Mirrors 188 | 189 | The rvcs tool also does not mandate a specific mechanism for copying snapshots 190 | between different machines, or among different users. 191 | 192 | Instead, you configure a set of URLs as "mirrors". 193 | 194 | When you sign a snapshot to publish it, that snapshot is automatically pushed 195 | to these mirrors, and when you try to lookup a signed snapshot the tool 196 | automatically reads any updated values from the mirrors. 197 | 198 | The actual communication with each mirror is performed by an external tool 199 | chosen based on the URL of the mirror. 200 | 201 | ### Push and Pull Helpers 202 | 203 | Similarly to the sign and verify helpers, the rvcs tool relies on push and 204 | pull helpers to push snapshots to and pull them from mirrors. 205 | 206 | The helper tools are named of the form `rvcs-push-` and 207 | `rvcs-pull-`, where `` is the scheme portion of the mirror's 208 | URL. 209 | 210 | So, for example, if a mirror has the URL `file:///some/local/path`, then 211 | rvcs will try to invoke a tool named `rvcs-push-file` to push to that mirror 212 | and one named `rvcs-pull-file` to pull from it. 213 | 214 | The pull helper tool takes the full URL of the mirror (including the scheme), 215 | the fully specified identity (including the namespace), the hash of the most 216 | recently-known signature for that identity, and a file for it to write output. 217 | 218 | When successfull it outputs the hash of the latest signature for that 219 | identity that it pulled from the mirror and exits with a status code of `0`. 220 | 221 | The push helper takes the full URL of the mirror, the fully specified 222 | identity, the hash of the latest, updated signature for that identity, and 223 | a file for it to write output. 224 | 225 | If it successfully pushes that update to the mirror then it outputs the 226 | hash of the signature that was pushed and exits with a status code of `0`. 227 | 228 | There are example push and pull helpers in the `extensions` directory that 229 | demonstrate how to use a local file path as a mirror. 230 | 231 | ## Merging 232 | 233 | The `rvcs` provides a `merge` subcommand to automatically merge different 234 | snapshots together and then checkout the result into some local file path. 235 | 236 | The `merge` command takes two arguments, the "left hand side" of the merge 237 | and the "right hand side". 238 | 239 | The left hand side can be any type of reference to a snapshot, while the 240 | right hand side must be a local file system path. 241 | 242 | If the merge is successful, then the file system contents of the path 243 | provided for the right hand side are updated to match the merged snapshot. 244 | 245 | ### Merge Helpers 246 | 247 | The `rvcs` tool will do its best to automatically merge changes to directories, 248 | but if there are conflicting changes to individual files it relies on an 249 | external helper command to try to automatically merge them. 250 | 251 | By default, it uses the `diff3` command to perform this merge. That can be 252 | changed by specifying the name of the command to use in the 253 | `RVCS_MERGE_HELPER_COMMAND` environment variable. 254 | 255 | If the supplied merge helper requires extra arguments, then they can be 256 | specified in a JSON-encoded list using the `RVCS_MERGE_HELPER_ARGS` environment 257 | variable. 258 | 259 | The `rvcs` tool invokes the specified merge helper with the specified args, 260 | followed by the paths to three files: 261 | 262 | 1. A file with the same contents and history as the left-hand side of the 263 | merge. 264 | 2. A file with the same contents and history as the most recent common 265 | ancestor of both sides of the merge. 266 | 3. A file with the same contents and history as the right-hand side of the 267 | merge. 268 | 269 | If the merge helper exits with a status of `0`, then its standard output is 270 | taken as the contents of the successfully-merged file. 271 | 272 | Otherwise, the automatic merge fails and you have to manually merge the 273 | changes. 274 | 275 | ### Manual Merges 276 | 277 | The `rvcs` tool enables version control for files located anywhere on your 278 | computer. This fact enables a workflow that we refer to as manual merging. 279 | 280 | This is the fallback that you can use to merge conflicting changes in the 281 | event that `rvcs` is not able to automatically merge them. 282 | 283 | To perform a manual merge you create a temporary directory where you will 284 | merge the conflicting versions. 285 | 286 | You then check out the right-hand-side of your conflicting merge into this 287 | temporary directory: 288 | 289 | ```shell 290 | rvcs merge ${RIGHT_HAND_SIDE} ${TEMPORARY_DIRECTORY}/${FILENAME} 291 | ``` 292 | 293 | Optionally, you can also check out the left-hand-side into this same 294 | directory if you want to see the changes side-by-side: 295 | 296 | ```shell 297 | rvcs merge ${LEFT_HAND_SIDE} ${TEMPORARY_DIRECTORY}/other_${FILENAME} 298 | ``` 299 | 300 | Next, you manually edit the contents of the checked out temporary file 301 | to look the way you want. 302 | 303 | After that you can create a manual merge of the two sides using the 304 | `rvcs snapshot` command with the `--additional-parents` flag: 305 | 306 | ```shell 307 | rvcs snapshot --additional-parents=${LEFT_HAND_SIDE} ${TEMPORARY_DIRECTORY}/${FILENAME} 308 | ``` 309 | 310 | This new snapshot will have both sides of the merge as its parents, so 311 | you can then merge it into your destination path: 312 | 313 | ```shell 314 | rvcs merge ${TEMPORARY_DIRECTORY}/${FILENAME} ${RIGHT_HAND_SIDE} 315 | ``` 316 | 317 | Finally, you can clean up by removing the temporary directory. 318 | -------------------------------------------------------------------------------- /storage/storage_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package storage defines the persistent storage of snapshots. 16 | package storage 17 | 18 | import ( 19 | "bytes" 20 | "context" 21 | "os" 22 | "path/filepath" 23 | "strings" 24 | "testing" 25 | 26 | "github.com/google/go-cmp/cmp" 27 | "github.com/google/recursive-version-control-system/snapshot" 28 | ) 29 | 30 | func TestSnapshotCurrent(t *testing.T) { 31 | dir := t.TempDir() 32 | archive := filepath.Join(dir, "archive") 33 | s := &LocalFiles{ArchiveDir: archive} 34 | 35 | workingDir := filepath.Join(dir, "working-dir") 36 | if err := os.Mkdir(workingDir, 0700); err != nil { 37 | t.Fatalf("failure creating the working directory for the test: %v", err) 38 | } 39 | file := filepath.Join(workingDir, "example.txt") 40 | dirPath := snapshot.Path(workingDir) 41 | p := snapshot.Path(file) 42 | 43 | // Take an initial snapshot 44 | if err := os.WriteFile(file, []byte("Hello, World!"), 0700); err != nil { 45 | t.Fatalf("failure creating the example file to snapshot: %v", err) 46 | } 47 | h1, f1, err := snapshot.Current(context.Background(), s, p) 48 | if err != nil { 49 | t.Errorf("failure creating the initial snapshot for the file: %v", err) 50 | } else if h1 == nil { 51 | t.Error("unexpected nil hash for the file") 52 | } else if f1 == nil { 53 | t.Error("unexpected nil snapshot for the file") 54 | } 55 | 56 | // Verify that we can take the snapshot again without it changing 57 | h2, f2, err := snapshot.Current(context.Background(), s, p) 58 | if err != nil { 59 | t.Errorf("failure replicating the initial snapshot for the file: %v", err) 60 | } else if got, want := h2, h1; !got.Equal(want) { 61 | t.Errorf("unexpected hash for the file; got %q, want %q", got, want) 62 | } else if got, want := f2.String(), f1.String(); got != want { 63 | t.Errorf("unexpected snapshot for the file; got %q, want %q", got, want) 64 | } 65 | 66 | // Modify the file and verify that the snapshot both changes and points to the parent 67 | if err := os.WriteFile(file, []byte("Goodbye, World!"), 0700); err != nil { 68 | t.Fatalf("failure updating the example file to snapshot: %v", err) 69 | } 70 | h3, f3, err := snapshot.Current(context.Background(), s, p) 71 | if err != nil { 72 | t.Errorf("failure creating the updated snapshot for the file: %v", err) 73 | } else if h3 == nil { 74 | t.Error("unexpected nil hash for the updated file") 75 | } else if f3 == nil { 76 | t.Error("unexpected nil snapshot for the updated file") 77 | } else if h3.Equal(h1) { 78 | t.Error("failed to update the snapshot") 79 | } else if !f3.Parents[0].Equal(h1) { 80 | t.Errorf("updated snapshot did not include the original as its parent: %q", f3) 81 | } 82 | 83 | // Write a large file (> 1 MB) and verify that we can snapshot it 84 | // and read it back 85 | largeObjSize := 2 * 1024 * 1024 86 | var largeBytes bytes.Buffer 87 | largeBytes.Grow(largeObjSize) 88 | for i := 0; i < largeObjSize; i++ { 89 | largeBytes.WriteString(" ") 90 | } 91 | largeFile := filepath.Join(workingDir, "largeFile.txt") 92 | p2 := snapshot.Path(largeFile) 93 | if err := os.WriteFile(largeFile, largeBytes.Bytes(), 0700); err != nil { 94 | t.Fatalf("failure writing a large file: %v", err) 95 | } 96 | h4, f4, err := snapshot.Current(context.Background(), s, dirPath) 97 | if err != nil { 98 | t.Errorf("failure creating the updated snapshot containing a large file: %v", err) 99 | } else if h4 == nil { 100 | t.Error("unexpected nil hash for the working directory") 101 | } else if f4 == nil { 102 | t.Error("unexpected nil snapshot for the working directory") 103 | } 104 | 105 | var readLargeBytes bytes.Buffer 106 | h5, f5, err := snapshot.Current(context.Background(), s, p2) 107 | if err != nil { 108 | t.Errorf("failure getting the current snapshot for a large file: %v", err) 109 | } else if h5 == nil { 110 | t.Error("unexpected nil hash for the large file") 111 | } else if f5 == nil { 112 | t.Error("unexpected nil snapshot for the large file") 113 | } else if largeBytesReader, err := s.ReadObject(context.Background(), f5.Contents); err != nil { 114 | t.Errorf("failure opening the contents reader of a large file: %v", err) 115 | } else if _, err := readLargeBytes.ReadFrom(largeBytesReader); err != nil { 116 | t.Errorf("failure reading back the contents of a large file: %v", err) 117 | } else if diff := cmp.Diff(string(largeBytes.Bytes()), string(readLargeBytes.Bytes())); len(diff) > 0 { 118 | t.Errorf("wrong contents read back for a large file: diff %s", diff) 119 | } 120 | 121 | // Confirm that the stored large object contents are encrypted. 122 | objPath, objName := objectName(f5.Contents, filepath.Join(s.ArchiveDir, largeObjectStorageDir), true) 123 | if _, err := os.Stat(filepath.Join(objPath, objName)); err != nil { 124 | t.Errorf("failure finding the stored object contents in the expected location: %v", err) 125 | } else { 126 | var readRawBytes bytes.Buffer 127 | reader, err := os.Open(filepath.Join(objPath, objName)) 128 | if err != nil { 129 | t.Errorf("failure opening the stored object contents: %v", err) 130 | } else if _, err := readRawBytes.ReadFrom(reader); err != nil { 131 | t.Errorf("failure reading the raw stored object contents: %v", err) 132 | } else if diff := cmp.Diff(string(readRawBytes.Bytes()), string(largeBytes.Bytes())); len(diff) == 0 { 133 | t.Error("failed to encrypt the large object") 134 | } 135 | } 136 | } 137 | 138 | func TestLinkSnapshot(t *testing.T) { 139 | dir := t.TempDir() 140 | archive := filepath.Join(dir, "archive") 141 | s := &LocalFiles{ArchiveDir: archive} 142 | 143 | workingDir := filepath.Join(dir, "working-dir") 144 | if err := os.Mkdir(workingDir, 0700); err != nil { 145 | t.Fatalf("failure creating the working directory for the test: %v", err) 146 | } 147 | 148 | file1 := filepath.Join(workingDir, "example1.txt") 149 | file2 := filepath.Join(workingDir, "example2.txt") 150 | link := filepath.Join(workingDir, "link") 151 | p := snapshot.Path(link) 152 | 153 | // Take an initial snapshot 154 | if err := os.WriteFile(file1, []byte("Hello, World!"), 0700); err != nil { 155 | t.Fatalf("failure creating the first example file: %v", err) 156 | } 157 | if err := os.WriteFile(file2, []byte("Also hello, World!"), 0700); err != nil { 158 | t.Fatalf("failure creating the second example file: %v", err) 159 | } 160 | if err := os.Symlink(file1, link); err != nil { 161 | t.Fatalf("failure creating the example symlink to snapshot: %v", err) 162 | } 163 | 164 | h1, f1, err := snapshot.Current(context.Background(), s, p) 165 | if err != nil { 166 | t.Errorf("failure creating the initial snapshot for a symlink: %v", err) 167 | } else if h1 == nil { 168 | t.Error("unexpected nil hash for the symlink") 169 | } else if f1 == nil { 170 | t.Error("unexpected nil snapshot for the symlink") 171 | } else if !f1.IsLink() { 172 | t.Error("unexpected snapshot type for the symlink") 173 | } 174 | 175 | // Verify that we can take the snapshot again without it changing 176 | h2, f2, err := snapshot.Current(context.Background(), s, p) 177 | if err != nil { 178 | t.Errorf("failure replicating the initial snapshot for the symlink: %v", err) 179 | } else if got, want := h2, h1; !got.Equal(want) { 180 | t.Errorf("unexpected hash for the symlink; got %q, want %q", got, want) 181 | } else if got, want := f2.String(), f1.String(); got != want { 182 | t.Errorf("unexpected snapshot for the symlink; got %q, want %q", got, want) 183 | } 184 | 185 | // Modify the contents of the linked file and verify that the snapshot of the symlink does not change 186 | if err := os.WriteFile(file1, []byte("Goodbye, World!"), 0700); err != nil { 187 | t.Fatalf("failure updating the contents of the example symlink target: %v", err) 188 | } 189 | h3, f3, err := snapshot.Current(context.Background(), s, p) 190 | if err != nil { 191 | t.Errorf("failure replicating the initial snapshot for the symlink: %v", err) 192 | } else if got, want := h3, h1; !got.Equal(want) { 193 | t.Errorf("unexpected hash for the symlink; got %q, want %q", got, want) 194 | } else if got, want := f3.String(), f1.String(); got != want { 195 | t.Errorf("unexpected snapshot for the symlink; got %q, want %q", got, want) 196 | } 197 | 198 | if err := os.Remove(link); err != nil { 199 | t.Fatalf("error removing the symlink: %v", err) 200 | } else if err := os.Symlink(file2, link); err != nil { 201 | t.Fatalf("error recreating the symlink: %v", err) 202 | } 203 | 204 | h4, f4, err := snapshot.Current(context.Background(), s, p) 205 | if err != nil { 206 | t.Errorf("failure creating the updated snapshot for the symlink: %v", err) 207 | } else if h4 == nil { 208 | t.Error("unexpected nil hash for the updated symlink") 209 | } else if f4 == nil { 210 | t.Error("unexpected nil snapshot for the updated symlink") 211 | } else if h4.Equal(h1) { 212 | t.Error("failed to update the snapshot") 213 | } else if !f4.Parents[0].Equal(h1) { 214 | t.Errorf("updated snapshot did not include the original as its parent: %q", f3) 215 | } 216 | } 217 | 218 | func TestDirSnapshot(t *testing.T) { 219 | dir := t.TempDir() 220 | archive := filepath.Join(dir, "archive") 221 | s := &LocalFiles{ArchiveDir: archive} 222 | 223 | workingDir := filepath.Join(dir, "working-dir") 224 | if err := os.Mkdir(workingDir, 0700); err != nil { 225 | t.Fatalf("failure creating the working directory for the test: %v", err) 226 | } 227 | 228 | // Setup multiple levels of directory that we will snapshot 229 | containerDir := filepath.Join(workingDir, "container") 230 | containerPath := snapshot.Path(containerDir) 231 | nestedDir := filepath.Join(containerDir, "nested") 232 | nestedPath := snapshot.Path(nestedDir) 233 | if err := os.MkdirAll(nestedDir, 0700); err != nil { 234 | t.Fatalf("failure creating the test directories: %v", err) 235 | } 236 | file1 := filepath.Join(nestedDir, "example1.txt") 237 | file1Path := snapshot.Path(file1) 238 | file2 := filepath.Join(nestedDir, "example2.txt") 239 | file2Path := snapshot.Path(file2) 240 | // Take an initial snapshot 241 | if err := os.WriteFile(file1, []byte("Hello, World!"), 0700); err != nil { 242 | t.Fatalf("failure creating the first example file to snapshot: %v", err) 243 | } 244 | if err := os.WriteFile(file2, []byte("Also... hello, World!"), 0700); err != nil { 245 | t.Fatalf("failure creating the second example file to snapshot: %v", err) 246 | } 247 | link := filepath.Join(nestedDir, "link") 248 | linkPath := snapshot.Path(link) 249 | if err := os.Symlink(file1, link); err != nil { 250 | t.Fatalf("failure creating the example symlink to snapshot: %v", err) 251 | } 252 | containerHash, containerFile, err := snapshot.Current(context.Background(), s, containerPath) 253 | if err != nil { 254 | t.Errorf("failure creating the initial snapshot for the dir: %v", err) 255 | } else if containerHash == nil { 256 | t.Error("unexpected nil hash for the dir") 257 | } else if containerFile == nil { 258 | t.Error("unexpected nil snapshot for the dir") 259 | } else if !containerFile.IsDir() { 260 | t.Errorf("unexpected type for the dir snapshot: %q", containerFile.Mode) 261 | } 262 | 263 | nestedHash, nestedFile, err := snapshot.Current(context.Background(), s, nestedPath) 264 | if err != nil { 265 | t.Errorf("failure creating the initial snapshot for the nested dir: %v", err) 266 | } else if nestedHash == nil { 267 | t.Error("unexpected nil hash for the nested dir") 268 | } else if nestedFile == nil { 269 | t.Error("unexpected nil snapshot for the nested dir") 270 | } else if !nestedFile.IsDir() { 271 | t.Errorf("unexpected type for the nested dir snapshot: %q", nestedFile.Mode) 272 | } 273 | 274 | expectedTree := make(snapshot.Tree) 275 | expectedTree[snapshot.Path("nested")] = nestedHash 276 | expectedHash, err := snapshot.NewHash(strings.NewReader(expectedTree.String())) 277 | if err != nil { 278 | t.Errorf("failure hashing the expected tree: %v", err) 279 | } else if got, want := containerFile.Contents, expectedHash; !got.Equal(want) { 280 | t.Errorf("unexpected contents hash for the containing dir: got %q, want %q", got, want) 281 | } 282 | 283 | // Take a second snapshot and verify that it remains unchanged... 284 | containerHash2, containerFile2, err := snapshot.Current(context.Background(), s, containerPath) 285 | if err != nil { 286 | t.Errorf("failure creating the initial snapshot for the dir: %v", err) 287 | } else if got, want := containerHash2, containerHash; !got.Equal(want) { 288 | t.Errorf("unexpected hash for an unchanged directory; got %q, want %q", got, want) 289 | } else if got, want := containerFile2.String(), containerFile.String(); got != want { 290 | t.Errorf("unexpected snapshot for an unchanged directory; got %q, want %q", got, want) 291 | } 292 | 293 | // Look up the initial snapshots for all the nested files for later checks... 294 | file1Hash, file1Snapshot, err := s.FindSnapshot(context.Background(), file1Path) 295 | if err != nil { 296 | t.Errorf("failure looking up the snapshot for the first nested file: %v", err) 297 | } else if file1Hash == nil || file1Snapshot == nil { 298 | t.Errorf("missing snapshot for the first nested file: %q, %+v", file1Hash, file1Snapshot) 299 | } 300 | file2Hash, file2Snapshot, err := s.FindSnapshot(context.Background(), file2Path) 301 | if err != nil { 302 | t.Errorf("failure looking up the snapshot for the second nested file: %v", err) 303 | } else if file2Hash == nil || file2Snapshot == nil { 304 | t.Errorf("missing snapshot for the second nested file: %q, %+v", file2Hash, file2Snapshot) 305 | } 306 | linkHash, linkSnapshot, err := s.FindSnapshot(context.Background(), linkPath) 307 | if err != nil { 308 | t.Errorf("failure looking up the snapshot for the nested symlink: %v", err) 309 | } else if linkHash == nil || linkSnapshot == nil { 310 | t.Errorf("missing snapshot for the nested symlink: %q, %+v", linkHash, linkSnapshot) 311 | } 312 | 313 | // Perform a single nested update, and then re-snapshot... 314 | if err := os.WriteFile(file2, []byte("Goodbye, World!"), 0700); err != nil { 315 | t.Fatalf("failure updating the second example file to snapshot: %v", err) 316 | } 317 | containerHash3, containerFile3, err := snapshot.Current(context.Background(), s, containerPath) 318 | if err != nil { 319 | t.Errorf("failure creating the updated snapshot for the dir: %v", err) 320 | } else if containerHash3.Equal(containerHash) { 321 | t.Errorf("failed to update the hash for a nested change; got %q", containerHash3) 322 | } else if containerFile3.String() == containerFile.String() { 323 | t.Errorf("failed to update the snapshot for a nested change; got %+v", containerFile3) 324 | } 325 | 326 | // Compare the nested file snapshots for unchanged files to verify they are the same... 327 | if file1Hash2, file1Snapshot2, err := s.FindSnapshot(context.Background(), file1Path); err != nil { 328 | t.Errorf("failure looking up the snapshot for the first nested file: %v", err) 329 | } else if got, want := file1Hash2, file1Hash; !got.Equal(want) { 330 | t.Errorf("unexpected hash for an unchanged nested file: got %q, want %q", got, want) 331 | } else if got, want := file1Snapshot2.String(), file1Snapshot.String(); got != want { 332 | t.Errorf("unexpected snapshot for an unchanged nested file: got %q, want %q", got, want) 333 | } 334 | if linkHash2, linkSnapshot2, err := s.FindSnapshot(context.Background(), linkPath); err != nil { 335 | t.Errorf("failure looking up the snapshot for the nested link: %v", err) 336 | } else if got, want := linkHash2, linkHash; !got.Equal(want) { 337 | t.Errorf("unexpected hash for an unchanged nested symlink: got %q, want %q", got, want) 338 | } else if got, want := linkSnapshot2.String(), linkSnapshot.String(); got != want { 339 | t.Errorf("unexpected snapshot for an unchanged nested symlink: got %q, want %q", got, want) 340 | } 341 | 342 | // Compare the nested file snapshot for the changed file to verify that it has been updated... 343 | if file2Hash2, file2Snapshot2, err := s.FindSnapshot(context.Background(), file2Path); err != nil { 344 | t.Errorf("failure looking up the snapshot for the updated nested file: %v", err) 345 | } else if file2Hash2.Equal(file2Hash) { 346 | t.Errorf("unexpectedly unchanged hash for a changed nested file: got %q", file2Hash2) 347 | } else if file2Snapshot2.String() == file2Snapshot.String() { 348 | t.Errorf("unexpectedly unchanged snapshot for a changed nested file: got %+v", file2Snapshot2) 349 | } 350 | 351 | // Remove the nested file and verify that updated snapshots are correct... 352 | if err := os.Remove(file2); err != nil { 353 | t.Fatalf("failure removing a nested file: %v", err) 354 | } 355 | containerHash4, containerFile4, err := snapshot.Current(context.Background(), s, containerPath) 356 | if err != nil { 357 | t.Errorf("failure creating the updated snapshot for the dir after removing a nested file: %v", err) 358 | } else if containerHash4.Equal(containerHash3) { 359 | t.Errorf("failed to update the hash for a nested file removal; got %q", containerHash4) 360 | } else if containerFile4.String() == containerFile3.String() { 361 | t.Errorf("failed to update the snapshot for a nested file removal; got %+v", containerFile4) 362 | } 363 | if file2Hash3, file2Snapshot3, err := s.FindSnapshot(context.Background(), file2Path); err == nil { 364 | t.Errorf("unexpected hash and/or snapshot for a removed file: hash %q, snapshot %+v", file2Hash3, file2Snapshot3) 365 | } 366 | } 367 | --------------------------------------------------------------------------------