├── .github ├── ISSUE_TEMPLATE │ ├── config.yml │ └── open_an_issue.md ├── config.yml └── workflows │ ├── generated-pr.yml │ ├── go-check.yml │ ├── go-test.yml │ ├── release-check.yml │ ├── releaser.yml │ ├── stale.yml │ └── tagpush.yml ├── .gx ├── lastpubver └── post-install ├── CODEOWNERS ├── LICENSE ├── README.md ├── coding.go ├── coding_test.go ├── dagutils ├── diff.go ├── diff_test.go ├── diffenum.go ├── diffenum_test.go ├── utils.go └── utils_test.go ├── errservice.go ├── go.mod ├── go.sum ├── merkledag.go ├── merkledag_test.go ├── node.go ├── node_test.go ├── pb ├── compat_test.go ├── merkledag.pb.go ├── merkledag.proto ├── merkledagpb_test.go ├── stability_test.go └── upgrade_check.go ├── prime.go ├── raw.go ├── readonly.go ├── readonly_test.go ├── rwservice.go ├── session.go ├── test └── utils.go ├── traverse ├── traverse.go └── traverse_test.go └── version.json /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | blank_issues_enabled: false 2 | contact_links: 3 | - name: Getting Help on IPFS 4 | url: https://ipfs.io/help 5 | about: All information about how and where to get help on IPFS. 6 | - name: IPFS Official Forum 7 | url: https://discuss.ipfs.io 8 | about: Please post general questions, support requests, and discussions here. 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/open_an_issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Open an issue 3 | about: Only for actionable issues relevant to this repository. 4 | title: '' 5 | labels: need/triage 6 | assignees: '' 7 | 8 | --- 9 | 20 | -------------------------------------------------------------------------------- /.github/config.yml: -------------------------------------------------------------------------------- 1 | # Configuration for welcome - https://github.com/behaviorbot/welcome 2 | 3 | # Configuration for new-issue-welcome - https://github.com/behaviorbot/new-issue-welcome 4 | # Comment to be posted to on first time issues 5 | newIssueWelcomeComment: > 6 | Thank you for submitting your first issue to this repository! A maintainer 7 | will be here shortly to triage and review. 8 | 9 | In the meantime, please double-check that you have provided all the 10 | necessary information to make this process easy! Any information that can 11 | help save additional round trips is useful! We currently aim to give 12 | initial feedback within **two business days**. If this does not happen, feel 13 | free to leave a comment. 14 | 15 | Please keep an eye on how this issue will be labeled, as labels give an 16 | overview of priorities, assignments and additional actions requested by the 17 | maintainers: 18 | 19 | - "Priority" labels will show how urgent this is for the team. 20 | - "Status" labels will show if this is ready to be worked on, blocked, or in progress. 21 | - "Need" labels will indicate if additional input or analysis is required. 22 | 23 | Finally, remember to use https://discuss.ipfs.io if you just need general 24 | support. 25 | 26 | # Configuration for new-pr-welcome - https://github.com/behaviorbot/new-pr-welcome 27 | # Comment to be posted to on PRs from first time contributors in your repository 28 | newPRWelcomeComment: > 29 | Thank you for submitting this PR! 30 | 31 | A maintainer will be here shortly to review it. 32 | 33 | We are super grateful, but we are also overloaded! Help us by making sure 34 | that: 35 | 36 | * The context for this PR is clear, with relevant discussion, decisions 37 | and stakeholders linked/mentioned. 38 | 39 | * Your contribution itself is clear (code comments, self-review for the 40 | rest) and in its best form. Follow the [code contribution 41 | guidelines](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md#code-contribution-guidelines) 42 | if they apply. 43 | 44 | Getting other community members to do a review would be great help too on 45 | complex PRs (you can ask in the chats/forums). If you are unsure about 46 | something, just leave us a comment. 47 | 48 | Next steps: 49 | 50 | * A maintainer will triage and assign priority to this PR, commenting on 51 | any missing things and potentially assigning a reviewer for high 52 | priority items. 53 | 54 | * The PR gets reviews, discussed and approvals as needed. 55 | 56 | * The PR is merged by maintainers when it has been approved and comments addressed. 57 | 58 | We currently aim to provide initial feedback/triaging within **two business 59 | days**. Please keep an eye on any labelling actions, as these will indicate 60 | priorities and status of your contribution. 61 | 62 | We are very grateful for your contribution! 63 | 64 | 65 | # Configuration for first-pr-merge - https://github.com/behaviorbot/first-pr-merge 66 | # Comment to be posted to on pull requests merged by a first time user 67 | # Currently disabled 68 | #firstPRMergeComment: "" 69 | -------------------------------------------------------------------------------- /.github/workflows/generated-pr.yml: -------------------------------------------------------------------------------- 1 | name: Close Generated PRs 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | stale: 14 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-generated-pr.yml@v1 15 | -------------------------------------------------------------------------------- /.github/workflows/go-check.yml: -------------------------------------------------------------------------------- 1 | name: Go Checks 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: ["master"] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | go-check: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/go-check.yml@v1.0 19 | -------------------------------------------------------------------------------- /.github/workflows/go-test.yml: -------------------------------------------------------------------------------- 1 | name: Go Test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: ["master"] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: read 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | go-test: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/go-test.yml@v1.0 19 | secrets: 20 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 21 | -------------------------------------------------------------------------------- /.github/workflows/release-check.yml: -------------------------------------------------------------------------------- 1 | name: Release Checker 2 | 3 | on: 4 | pull_request_target: 5 | paths: [ 'version.json' ] 6 | types: [ opened, synchronize, reopened, labeled, unlabeled ] 7 | workflow_dispatch: 8 | 9 | permissions: 10 | contents: write 11 | pull-requests: write 12 | 13 | concurrency: 14 | group: ${{ github.workflow }}-${{ github.ref }} 15 | cancel-in-progress: true 16 | 17 | jobs: 18 | release-check: 19 | uses: ipdxco/unified-github-workflows/.github/workflows/release-check.yml@v1.0 20 | -------------------------------------------------------------------------------- /.github/workflows/releaser.yml: -------------------------------------------------------------------------------- 1 | name: Releaser 2 | 3 | on: 4 | push: 5 | paths: [ 'version.json' ] 6 | workflow_dispatch: 7 | 8 | permissions: 9 | contents: write 10 | 11 | concurrency: 12 | group: ${{ github.workflow }}-${{ github.sha }} 13 | cancel-in-progress: true 14 | 15 | jobs: 16 | releaser: 17 | uses: ipdxco/unified-github-workflows/.github/workflows/releaser.yml@v1.0 18 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | name: Close Stale Issues 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * *' 6 | workflow_dispatch: 7 | 8 | permissions: 9 | issues: write 10 | pull-requests: write 11 | 12 | jobs: 13 | stale: 14 | uses: ipdxco/unified-github-workflows/.github/workflows/reusable-stale-issue.yml@v1 15 | -------------------------------------------------------------------------------- /.github/workflows/tagpush.yml: -------------------------------------------------------------------------------- 1 | name: Tag Push Checker 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | permissions: 9 | contents: read 10 | issues: write 11 | 12 | concurrency: 13 | group: ${{ github.workflow }}-${{ github.ref }} 14 | cancel-in-progress: true 15 | 16 | jobs: 17 | releaser: 18 | uses: ipdxco/unified-github-workflows/.github/workflows/tagpush.yml@v1.0 19 | -------------------------------------------------------------------------------- /.gx/lastpubver: -------------------------------------------------------------------------------- 1 | 1.1.40: QmY6UwsN3D6uoxrRkYpJ8Wos8R66gwLmdn3wy7jM7CCRQ1 2 | -------------------------------------------------------------------------------- /.gx/post-install: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ipfs/go-merkledag/e233e4dd214b1f92a7cb826861a35c5a4fdd970a/.gx/post-install -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | @rvagg 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014-2018 Juan Batiz-Benet 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | go-merkledag 2 | ================== 3 | 4 | [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io) 5 | [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/) 6 | [![Coverage Status](https://codecov.io/gh/ipfs/go-merkledag/branch/master/graph/badge.svg)](https://codecov.io/gh/ipfs/go-merkledag/branch/master) 7 | 8 | > go-merkledag implements the 'DAGService' interface and adds two ipld node types, Protobuf and Raw 9 | 10 | ## Status 11 | 12 | ❗ This library is maintained, but not actively developed. It will continue to receive fixes and security updates for users that depend on it. However, it may be deprecated in the future and it is recommended that you use alternatives to the functionality in go-merkledag, including: 13 | 14 | * A fork of this library for use by Kubo is being maintained here: [github.com/ipfs/boxo/ipld/merkledag](https://pkg.go.dev/github.com/ipfs/boxo/ipld/merkledag) 15 | * Working directly with DAG-PB (ProtoNode) should directly use [github.com/ipld/go-codec-dagpb](https://pkg.go.dev/github.com/ipld/go-codec-dagpb) in conjunction with [github.com/ipld/go-ipld-prime](https://pkg.go.dev/github.com/ipld/go-ipld-prime) 16 | * Traversals / DAG walking should use [github.com/ipld/go-ipld-prime/traversal](https://pkg.go.dev/github.com/ipld/go-ipld-prime/traversal) 17 | 18 | ## License 19 | 20 | MIT © Juan Batiz-Benet 21 | -------------------------------------------------------------------------------- /coding.go: -------------------------------------------------------------------------------- 1 | package merkledag 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | "strings" 7 | 8 | blocks "github.com/ipfs/go-block-format" 9 | cid "github.com/ipfs/go-cid" 10 | format "github.com/ipfs/go-ipld-format" 11 | pb "github.com/ipfs/go-merkledag/pb" 12 | dagpb "github.com/ipld/go-codec-dagpb" 13 | ipld "github.com/ipld/go-ipld-prime" 14 | "github.com/ipld/go-ipld-prime/fluent/qp" 15 | cidlink "github.com/ipld/go-ipld-prime/linking/cid" 16 | ) 17 | 18 | // Make sure the user doesn't upgrade this file. 19 | // We need to check *here* as well as inside the `pb` package *just* in case the 20 | // user replaces *all* go files in that package. 21 | const _ = pb.DoNotUpgradeFileEverItWillChangeYourHashes 22 | 23 | // for now, we use a PBNode intermediate thing. 24 | // because native go objects are nice. 25 | 26 | // pbLinkSlice is a slice of pb.PBLink, similar to LinkSlice but for sorting the 27 | // PB form 28 | type pbLinkSlice []*pb.PBLink 29 | 30 | func (pbls pbLinkSlice) Len() int { return len(pbls) } 31 | func (pbls pbLinkSlice) Swap(a, b int) { pbls[a], pbls[b] = pbls[b], pbls[a] } 32 | func (pbls pbLinkSlice) Less(a, b int) bool { return *pbls[a].Name < *pbls[b].Name } 33 | 34 | // unmarshal decodes raw data into a *Node instance. 35 | // The conversion uses an intermediate PBNode. 36 | func unmarshal(encodedBytes []byte) (*ProtoNode, error) { 37 | nb := dagpb.Type.PBNode.NewBuilder() 38 | if err := dagpb.DecodeBytes(nb, encodedBytes); err != nil { 39 | return nil, err 40 | } 41 | nd := nb.Build() 42 | return fromImmutableNode(&immutableProtoNode{encodedBytes, nd.(dagpb.PBNode)}), nil 43 | } 44 | 45 | func fromImmutableNode(encoded *immutableProtoNode) *ProtoNode { 46 | n := new(ProtoNode) 47 | n.encoded = encoded 48 | if n.encoded.PBNode.Data.Exists() { 49 | n.data = n.encoded.PBNode.Data.Must().Bytes() 50 | } 51 | numLinks := n.encoded.PBNode.Links.Length() 52 | // links may not be sorted after deserialization, but we don't change 53 | // them until we mutate this node since we're representing the current, 54 | // as-serialized state 55 | n.links = make([]*format.Link, numLinks) 56 | linkAllocs := make([]format.Link, numLinks) 57 | for i := int64(0); i < numLinks; i++ { 58 | next := n.encoded.PBNode.Links.Lookup(i) 59 | name := "" 60 | if next.FieldName().Exists() { 61 | name = next.FieldName().Must().String() 62 | } 63 | c := next.FieldHash().Link().(cidlink.Link).Cid 64 | size := uint64(0) 65 | if next.FieldTsize().Exists() { 66 | size = uint64(next.FieldTsize().Must().Int()) 67 | } 68 | link := &linkAllocs[i] 69 | link.Name = name 70 | link.Size = size 71 | link.Cid = c 72 | n.links[i] = link 73 | } 74 | // we don't set n.linksDirty because the order of the links list from 75 | // serialized form needs to be stable, until we start mutating the ProtoNode 76 | return n 77 | } 78 | func (n *ProtoNode) marshalImmutable() (*immutableProtoNode, error) { 79 | links := n.Links() 80 | nd, err := qp.BuildMap(dagpb.Type.PBNode, 2, func(ma ipld.MapAssembler) { 81 | qp.MapEntry(ma, "Links", qp.List(int64(len(links)), func(la ipld.ListAssembler) { 82 | for _, link := range links { 83 | // it shouldn't be possible to get here with an undefined CID, but in 84 | // case it is we're going to drop this link from the encoded form 85 | // entirely 86 | if link.Cid.Defined() { 87 | qp.ListEntry(la, qp.Map(3, func(ma ipld.MapAssembler) { 88 | qp.MapEntry(ma, "Hash", qp.Link(cidlink.Link{Cid: link.Cid})) 89 | qp.MapEntry(ma, "Name", qp.String(link.Name)) 90 | sz := int64(link.Size) 91 | if sz < 0 { // overflow, >MaxInt64 is almost certainly an error 92 | sz = 0 93 | } 94 | qp.MapEntry(ma, "Tsize", qp.Int(sz)) 95 | })) 96 | } 97 | } 98 | })) 99 | if n.data != nil { 100 | qp.MapEntry(ma, "Data", qp.Bytes(n.data)) 101 | } 102 | }) 103 | if err != nil { 104 | return nil, err 105 | } 106 | 107 | // 1KiB can be allocated on the stack, and covers most small nodes 108 | // without having to grow the buffer and cause allocations. 109 | enc := make([]byte, 0, 1024) 110 | 111 | enc, err = dagpb.AppendEncode(enc, nd) 112 | if err != nil { 113 | return nil, err 114 | } 115 | return &immutableProtoNode{enc, nd.(dagpb.PBNode)}, nil 116 | } 117 | 118 | // Marshal encodes a *Node instance into a new byte slice. 119 | // The conversion uses an intermediate PBNode. 120 | func (n *ProtoNode) Marshal() ([]byte, error) { 121 | enc, err := n.marshalImmutable() 122 | if err != nil { 123 | return nil, err 124 | } 125 | return enc.encoded, nil 126 | } 127 | 128 | // GetPBNode converts *ProtoNode into it's protocol buffer variant. 129 | // If you plan on mutating the data of the original node, it is recommended 130 | // that you call ProtoNode.Copy() before calling ProtoNode.GetPBNode() 131 | func (n *ProtoNode) GetPBNode() *pb.PBNode { 132 | pbn := &pb.PBNode{} 133 | if len(n.links) > 0 { 134 | pbn.Links = make([]*pb.PBLink, len(n.links)) 135 | } 136 | 137 | for i, l := range n.links { 138 | pbn.Links[i] = &pb.PBLink{} 139 | pbn.Links[i].Name = &l.Name 140 | pbn.Links[i].Tsize = &l.Size 141 | if l.Cid.Defined() { 142 | pbn.Links[i].Hash = l.Cid.Bytes() 143 | } 144 | } 145 | 146 | // Ensure links are sorted prior to encode, regardless of `linksDirty`. They 147 | // may not have come sorted if we deserialized a badly encoded form that 148 | // didn't have links already sorted. 149 | sort.Stable(pbLinkSlice(pbn.Links)) 150 | 151 | if len(n.data) > 0 { 152 | pbn.Data = n.data 153 | } 154 | return pbn 155 | } 156 | 157 | // EncodeProtobuf returns the encoded raw data version of a Node instance. 158 | // It may use a cached encoded version, unless the force flag is given. 159 | func (n *ProtoNode) EncodeProtobuf(force bool) ([]byte, error) { 160 | if n.encoded == nil || n.linksDirty || force { 161 | if n.linksDirty { 162 | // there was a mutation involving links, make sure we sort before we build 163 | // and cache a `Node` form that captures the current state 164 | sort.Stable(LinkSlice(n.links)) 165 | n.linksDirty = false 166 | } 167 | n.cached = cid.Undef 168 | var err error 169 | n.encoded, err = n.marshalImmutable() 170 | if err != nil { 171 | return nil, err 172 | } 173 | } 174 | 175 | if !n.cached.Defined() { 176 | c, err := n.CidBuilder().Sum(n.encoded.encoded) 177 | if err != nil { 178 | return nil, err 179 | } 180 | 181 | n.cached = c 182 | } 183 | 184 | return n.encoded.encoded, nil 185 | } 186 | 187 | // DecodeProtobuf decodes raw data and returns a new Node instance. 188 | func DecodeProtobuf(encoded []byte) (*ProtoNode, error) { 189 | n, err := unmarshal(encoded) 190 | if err != nil { 191 | return nil, fmt.Errorf("incorrectly formatted merkledag node: %s", err) 192 | } 193 | return n, nil 194 | } 195 | 196 | // DecodeProtobufBlock is a block decoder for protobuf IPLD nodes conforming to 197 | // node.DecodeBlockFunc 198 | func DecodeProtobufBlock(b blocks.Block) (format.Node, error) { 199 | c := b.Cid() 200 | if c.Type() != cid.DagProtobuf { 201 | return nil, fmt.Errorf("this function can only decode protobuf nodes") 202 | } 203 | 204 | decnd, err := DecodeProtobuf(b.RawData()) 205 | if err != nil { 206 | if strings.Contains(err.Error(), "Unmarshal failed") { 207 | return nil, fmt.Errorf("the block referred to by '%s' was not a valid merkledag node", c) 208 | } 209 | return nil, fmt.Errorf("failed to decode Protocol Buffers: %v", err) 210 | } 211 | 212 | decnd.cached = c 213 | decnd.builder = c.Prefix() 214 | return decnd, nil 215 | } 216 | 217 | // Type assertion 218 | var _ format.DecodeBlockFunc = DecodeProtobufBlock 219 | -------------------------------------------------------------------------------- /coding_test.go: -------------------------------------------------------------------------------- 1 | package merkledag_test 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | cid "github.com/ipfs/go-cid" 9 | ipld "github.com/ipfs/go-ipld-format" 10 | "github.com/ipfs/go-merkledag" 11 | ) 12 | 13 | var benchInput []byte 14 | 15 | func init() { 16 | someData := bytes.Repeat([]byte("some plaintext data\n"), 10) 17 | // make a test CID -- doesn't matter just to add as a link 18 | someCid, _ := cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4}) 19 | 20 | node := &merkledag.ProtoNode{} 21 | node.SetData(someData) 22 | for i := 0; i < 10; i++ { 23 | node.AddRawLink(fmt.Sprintf("%d", i), &ipld.Link{ 24 | Size: 10, 25 | Cid: someCid, 26 | }) 27 | } 28 | 29 | enc, err := node.EncodeProtobuf(true) 30 | if err != nil { 31 | panic(err) 32 | } 33 | benchInput = enc 34 | } 35 | 36 | func BenchmarkRoundtrip(b *testing.B) { 37 | b.ReportAllocs() 38 | b.RunParallel(func(pb *testing.PB) { 39 | for pb.Next() { 40 | node, err := merkledag.DecodeProtobuf(benchInput) 41 | if err != nil { 42 | b.Fatal(err) 43 | } 44 | 45 | enc, err := node.EncodeProtobuf(true) 46 | if err != nil { 47 | b.Fatal(err) 48 | } 49 | _ = enc 50 | } 51 | }) 52 | } 53 | -------------------------------------------------------------------------------- /dagutils/diff.go: -------------------------------------------------------------------------------- 1 | package dagutils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "path" 7 | 8 | "github.com/ipfs/go-cid" 9 | ipld "github.com/ipfs/go-ipld-format" 10 | 11 | dag "github.com/ipfs/go-merkledag" 12 | ) 13 | 14 | // ChangeType denotes type of change in Change 15 | type ChangeType int 16 | 17 | // These constants define the changes that can be applied to a DAG. 18 | const ( 19 | Add ChangeType = iota 20 | Remove 21 | Mod 22 | ) 23 | 24 | // Change represents a change to a DAG and contains a reference to the old and 25 | // new CIDs. 26 | type Change struct { 27 | Type ChangeType 28 | Path string 29 | Before cid.Cid 30 | After cid.Cid 31 | } 32 | 33 | // String prints a human-friendly line about a change. 34 | func (c *Change) String() string { 35 | switch c.Type { 36 | case Add: 37 | return fmt.Sprintf("Added %s at %s", c.After.String(), c.Path) 38 | case Remove: 39 | return fmt.Sprintf("Removed %s from %s", c.Before.String(), c.Path) 40 | case Mod: 41 | return fmt.Sprintf("Changed %s to %s at %s", c.Before.String(), c.After.String(), c.Path) 42 | default: 43 | panic("nope") 44 | } 45 | } 46 | 47 | // ApplyChange applies the requested changes to the given node in the given dag. 48 | func ApplyChange(ctx context.Context, ds ipld.DAGService, nd *dag.ProtoNode, cs []*Change) (*dag.ProtoNode, error) { 49 | e := NewDagEditor(nd, ds) 50 | for _, c := range cs { 51 | switch c.Type { 52 | case Add: 53 | child, err := ds.Get(ctx, c.After) 54 | if err != nil { 55 | return nil, err 56 | } 57 | 58 | childpb, ok := child.(*dag.ProtoNode) 59 | if !ok { 60 | return nil, dag.ErrNotProtobuf 61 | } 62 | 63 | err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | case Remove: 69 | err := e.RmLink(ctx, c.Path) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | case Mod: 75 | err := e.RmLink(ctx, c.Path) 76 | if err != nil { 77 | return nil, err 78 | } 79 | child, err := ds.Get(ctx, c.After) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | childpb, ok := child.(*dag.ProtoNode) 85 | if !ok { 86 | return nil, dag.ErrNotProtobuf 87 | } 88 | 89 | err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil) 90 | if err != nil { 91 | return nil, err 92 | } 93 | } 94 | } 95 | 96 | return e.Finalize(ctx, ds) 97 | } 98 | 99 | // Diff returns a set of changes that transform node 'a' into node 'b'. 100 | // It only traverses links in the following cases: 101 | // 1. two node's links number are greater than 0. 102 | // 2. both of two nodes are ProtoNode. 103 | // Otherwise, it compares the cid and emits a Mod change object. 104 | func Diff(ctx context.Context, ds ipld.DAGService, a, b ipld.Node) ([]*Change, error) { 105 | if a.Cid() == b.Cid() { 106 | return []*Change{}, nil 107 | } 108 | 109 | cleanA, okA := a.Copy().(*dag.ProtoNode) 110 | cleanB, okB := b.Copy().(*dag.ProtoNode) 111 | 112 | linksA := a.Links() 113 | linksB := b.Links() 114 | 115 | if !okA || !okB || (len(linksA) == 0 && len(linksB) == 0) { 116 | return []*Change{{Type: Mod, Before: a.Cid(), After: b.Cid()}}, nil 117 | } 118 | 119 | var out []*Change 120 | for _, linkA := range linksA { 121 | linkB, _, err := b.ResolveLink([]string{linkA.Name}) 122 | if err != nil { 123 | continue 124 | } 125 | 126 | cleanA.RemoveNodeLink(linkA.Name) 127 | cleanB.RemoveNodeLink(linkA.Name) 128 | 129 | if linkA.Cid == linkB.Cid { 130 | continue 131 | } 132 | 133 | nodeA, err := linkA.GetNode(ctx, ds) 134 | if err != nil { 135 | return nil, err 136 | } 137 | 138 | nodeB, err := linkB.GetNode(ctx, ds) 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | sub, err := Diff(ctx, ds, nodeA, nodeB) 144 | if err != nil { 145 | return nil, err 146 | } 147 | 148 | for _, c := range sub { 149 | c.Path = path.Join(linkA.Name, c.Path) 150 | } 151 | 152 | out = append(out, sub...) 153 | } 154 | 155 | for _, l := range cleanA.Links() { 156 | out = append(out, &Change{Type: Remove, Path: l.Name, Before: l.Cid}) 157 | } 158 | 159 | for _, l := range cleanB.Links() { 160 | out = append(out, &Change{Type: Add, Path: l.Name, After: l.Cid}) 161 | } 162 | 163 | return out, nil 164 | } 165 | 166 | // Conflict represents two incompatible changes and is returned by MergeDiffs(). 167 | type Conflict struct { 168 | A *Change 169 | B *Change 170 | } 171 | 172 | // MergeDiffs takes two slice of changes and adds them to a single slice. 173 | // When a Change from b happens to the same path of an existing change in a, 174 | // a conflict is created and b is not added to the merged slice. 175 | // A slice of Conflicts is returned and contains pointers to the 176 | // Changes involved (which share the same path). 177 | func MergeDiffs(a, b []*Change) ([]*Change, []Conflict) { 178 | paths := make(map[string]*Change) 179 | for _, c := range b { 180 | paths[c.Path] = c 181 | } 182 | 183 | var changes []*Change 184 | var conflicts []Conflict 185 | 186 | // NOTE: we avoid iterating over maps here to ensure iteration order is determistic. We 187 | // include changes from a first, then b. 188 | for _, changeA := range a { 189 | if changeB, ok := paths[changeA.Path]; ok { 190 | conflicts = append(conflicts, Conflict{changeA, changeB}) 191 | } else { 192 | changes = append(changes, changeA) 193 | } 194 | delete(paths, changeA.Path) 195 | } 196 | 197 | for _, c := range b { 198 | if _, ok := paths[c.Path]; ok { 199 | changes = append(changes, c) 200 | } 201 | } 202 | 203 | return changes, conflicts 204 | } 205 | -------------------------------------------------------------------------------- /dagutils/diff_test.go: -------------------------------------------------------------------------------- 1 | package dagutils 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | cid "github.com/ipfs/go-cid" 8 | ipld "github.com/ipfs/go-ipld-format" 9 | dag "github.com/ipfs/go-merkledag" 10 | mdtest "github.com/ipfs/go-merkledag/test" 11 | ) 12 | 13 | func TestMergeDiffs(t *testing.T) { 14 | node1 := dag.NodeWithData([]byte("one")) 15 | node2 := dag.NodeWithData([]byte("two")) 16 | node3 := dag.NodeWithData([]byte("three")) 17 | node4 := dag.NodeWithData([]byte("four")) 18 | 19 | changesA := []*Change{ 20 | {Add, "one", cid.Cid{}, node1.Cid()}, 21 | {Remove, "two", node2.Cid(), cid.Cid{}}, 22 | {Mod, "three", node3.Cid(), node4.Cid()}, 23 | } 24 | 25 | changesB := []*Change{ 26 | {Mod, "two", node2.Cid(), node3.Cid()}, 27 | {Add, "four", cid.Cid{}, node4.Cid()}, 28 | } 29 | 30 | changes, conflicts := MergeDiffs(changesA, changesB) 31 | if len(changes) != 3 { 32 | t.Fatal("unexpected merge changes") 33 | } 34 | 35 | expect := []*Change{ 36 | changesA[0], 37 | changesA[2], 38 | changesB[1], 39 | } 40 | 41 | for i, change := range changes { 42 | if change.Type != expect[i].Type { 43 | t.Error("unexpected diff change type") 44 | } 45 | 46 | if change.Path != expect[i].Path { 47 | t.Error("unexpected diff change path") 48 | } 49 | 50 | if change.Before != expect[i].Before { 51 | t.Error("unexpected diff change before") 52 | } 53 | 54 | if change.After != expect[i].After { 55 | t.Error("unexpected diff change before") 56 | } 57 | } 58 | 59 | if len(conflicts) != 1 { 60 | t.Fatal("unexpected merge conflicts") 61 | } 62 | 63 | if conflicts[0].A != changesA[1] { 64 | t.Error("unexpected merge conflict a") 65 | } 66 | 67 | if conflicts[0].B != changesB[0] { 68 | t.Error("unexpected merge conflict b") 69 | } 70 | } 71 | 72 | func TestDiff(t *testing.T) { 73 | ctx := context.Background() 74 | ds := mdtest.Mock() 75 | 76 | rootA := &dag.ProtoNode{} 77 | rootB := &dag.ProtoNode{} 78 | 79 | child1 := dag.NodeWithData([]byte("one")) 80 | child2 := dag.NodeWithData([]byte("two")) 81 | child3 := dag.NodeWithData([]byte("three")) 82 | child4 := dag.NodeWithData([]byte("four")) 83 | 84 | rootA.AddNodeLink("one", child1) 85 | rootA.AddNodeLink("two", child2) 86 | 87 | rootB.AddNodeLink("one", child3) 88 | rootB.AddNodeLink("four", child4) 89 | 90 | nodes := []ipld.Node{child1, child2, child3, child4, rootA, rootB} 91 | if err := ds.AddMany(ctx, nodes); err != nil { 92 | t.Fatal("failed to add nodes") 93 | } 94 | 95 | changes, err := Diff(ctx, ds, rootA, rootB) 96 | if err != nil { 97 | t.Fatal("unexpected diff error") 98 | } 99 | 100 | if len(changes) != 3 { 101 | t.Fatal("unexpected diff changes") 102 | } 103 | 104 | expect := []Change{ 105 | {Mod, "one", child1.Cid(), child3.Cid()}, 106 | {Remove, "two", child2.Cid(), cid.Cid{}}, 107 | {Add, "four", cid.Cid{}, child4.Cid()}, 108 | } 109 | 110 | for i, change := range changes { 111 | if change.Type != expect[i].Type { 112 | t.Error("unexpected diff change type") 113 | } 114 | 115 | if change.Path != expect[i].Path { 116 | t.Error("unexpected diff change path") 117 | } 118 | 119 | if change.Before != expect[i].Before { 120 | t.Error("unexpected diff change before") 121 | } 122 | 123 | if change.After != expect[i].After { 124 | t.Error("unexpected diff change before") 125 | } 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /dagutils/diffenum.go: -------------------------------------------------------------------------------- 1 | package dagutils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | cid "github.com/ipfs/go-cid" 8 | ipld "github.com/ipfs/go-ipld-format" 9 | 10 | mdag "github.com/ipfs/go-merkledag" 11 | ) 12 | 13 | // DiffEnumerate fetches every object in the graph pointed to by 'to' that is 14 | // not in 'from'. This can be used to more efficiently fetch a graph if you can 15 | // guarantee you already have the entirety of 'from' 16 | func DiffEnumerate(ctx context.Context, dserv ipld.NodeGetter, from, to cid.Cid) error { 17 | fnd, err := dserv.Get(ctx, from) 18 | if err != nil { 19 | return fmt.Errorf("get %s: %s", from, err) 20 | } 21 | 22 | tnd, err := dserv.Get(ctx, to) 23 | if err != nil { 24 | return fmt.Errorf("get %s: %s", to, err) 25 | } 26 | 27 | diff := getLinkDiff(fnd, tnd) 28 | 29 | sset := cid.NewSet() 30 | for _, c := range diff { 31 | // Since we're already assuming we have everything in the 'from' graph, 32 | // add all those cids to our 'already seen' set to avoid potentially 33 | // enumerating them later 34 | if c.bef.Defined() { 35 | sset.Add(c.bef) 36 | } 37 | } 38 | for _, c := range diff { 39 | if !c.bef.Defined() { 40 | if sset.Has(c.aft) { 41 | continue 42 | } 43 | err := mdag.Walk(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit, mdag.Concurrent()) 44 | if err != nil { 45 | return err 46 | } 47 | } else { 48 | err := DiffEnumerate(ctx, dserv, c.bef, c.aft) 49 | if err != nil { 50 | return err 51 | } 52 | } 53 | } 54 | 55 | return nil 56 | } 57 | 58 | // if both bef and aft are not nil, then that signifies bef was replaces with aft. 59 | // if bef is nil and aft is not, that means aft was newly added 60 | // if aft is nil and bef is not, that means bef was deleted 61 | type diffpair struct { 62 | bef, aft cid.Cid 63 | } 64 | 65 | // getLinkDiff returns a changeset between nodes 'a' and 'b'. Currently does 66 | // not log deletions as our usecase doesnt call for this. 67 | func getLinkDiff(a, b ipld.Node) []diffpair { 68 | ina := make(map[string]*ipld.Link) 69 | inb := make(map[string]*ipld.Link) 70 | var aonly []cid.Cid 71 | for _, l := range b.Links() { 72 | inb[l.Cid.KeyString()] = l 73 | } 74 | for _, l := range a.Links() { 75 | var key = l.Cid.KeyString() 76 | ina[key] = l 77 | if inb[key] == nil { 78 | aonly = append(aonly, l.Cid) 79 | } 80 | } 81 | 82 | var out []diffpair 83 | var aindex int 84 | 85 | for _, l := range b.Links() { 86 | if ina[l.Cid.KeyString()] != nil { 87 | continue 88 | } 89 | 90 | if aindex < len(aonly) { 91 | out = append(out, diffpair{bef: aonly[aindex], aft: l.Cid}) 92 | aindex++ 93 | } else { 94 | out = append(out, diffpair{aft: l.Cid}) 95 | continue 96 | } 97 | } 98 | return out 99 | } 100 | -------------------------------------------------------------------------------- /dagutils/diffenum_test.go: -------------------------------------------------------------------------------- 1 | package dagutils 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/ipfs/go-cid" 9 | ipld "github.com/ipfs/go-ipld-format" 10 | 11 | dag "github.com/ipfs/go-merkledag" 12 | mdtest "github.com/ipfs/go-merkledag/test" 13 | ) 14 | 15 | func buildNode(name string, desc map[string]ndesc, out map[string]ipld.Node) ipld.Node { 16 | this := desc[name] 17 | nd := new(dag.ProtoNode) 18 | nd.SetData([]byte(name)) 19 | for k, v := range this { 20 | child, ok := out[v] 21 | if !ok { 22 | child = buildNode(v, desc, out) 23 | out[v] = child 24 | } 25 | 26 | if err := nd.AddNodeLink(k, child); err != nil { 27 | panic(err) 28 | } 29 | } 30 | 31 | return nd 32 | } 33 | 34 | type ndesc map[string]string 35 | 36 | func mkGraph(desc map[string]ndesc) map[string]ipld.Node { 37 | out := make(map[string]ipld.Node) 38 | for name := range desc { 39 | if _, ok := out[name]; ok { 40 | continue 41 | } 42 | 43 | out[name] = buildNode(name, desc, out) 44 | } 45 | return out 46 | } 47 | 48 | var tg1 = map[string]ndesc{ 49 | "a1": { 50 | "foo": "b", 51 | }, 52 | "b": {}, 53 | "a2": { 54 | "foo": "b", 55 | "bar": "c", 56 | }, 57 | "c": {}, 58 | } 59 | 60 | var tg2 = map[string]ndesc{ 61 | "a1": { 62 | "foo": "b", 63 | }, 64 | "b": {}, 65 | "a2": { 66 | "foo": "b", 67 | "bar": "c", 68 | }, 69 | "c": {"baz": "d"}, 70 | "d": {}, 71 | } 72 | 73 | var tg3 = map[string]ndesc{ 74 | "a1": { 75 | "foo": "b", 76 | "bar": "c", 77 | }, 78 | "b": {}, 79 | "a2": { 80 | "foo": "b", 81 | "bar": "d", 82 | }, 83 | "c": {}, 84 | "d": {}, 85 | } 86 | 87 | var tg4 = map[string]ndesc{ 88 | "a1": { 89 | "key1": "b", 90 | "key2": "c", 91 | }, 92 | "a2": { 93 | "key1": "b", 94 | "key2": "d", 95 | }, 96 | } 97 | 98 | var tg5 = map[string]ndesc{ 99 | "a1": { 100 | "key1": "a", 101 | "key2": "b", 102 | }, 103 | "a2": { 104 | "key1": "c", 105 | "key2": "d", 106 | }, 107 | } 108 | 109 | func TestNameMatching(t *testing.T) { 110 | nds := mkGraph(tg4) 111 | 112 | diff := getLinkDiff(nds["a1"], nds["a2"]) 113 | if len(diff) != 1 { 114 | t.Fatal(fmt.Errorf("node diff didn't match by name")) 115 | } 116 | } 117 | 118 | func TestNameMatching2(t *testing.T) { 119 | nds := mkGraph(tg5) 120 | 121 | diff := getLinkDiff(nds["a1"], nds["a2"]) 122 | if len(diff) != 2 { 123 | t.Fatal(fmt.Errorf("incorrect number of link diff elements")) 124 | } 125 | if !(diff[0].bef.Equals(nds["a1"].Links()[0].Cid) && diff[0].aft.Equals(nds["a2"].Links()[0].Cid)) { 126 | t.Fatal(fmt.Errorf("node diff didn't match by name")) 127 | } 128 | } 129 | 130 | func TestDiffEnumBasic(t *testing.T) { 131 | ctx, cancel := context.WithCancel(context.Background()) 132 | defer cancel() 133 | nds := mkGraph(tg1) 134 | 135 | ds := mdtest.Mock() 136 | lgds := &getLogger{ds: ds} 137 | 138 | for _, nd := range nds { 139 | err := ds.Add(ctx, nd) 140 | if err != nil { 141 | t.Fatal(err) 142 | } 143 | } 144 | 145 | err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) 146 | if err != nil { 147 | t.Fatal(err) 148 | } 149 | 150 | err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()}) 151 | if err != nil { 152 | t.Fatal(err) 153 | } 154 | } 155 | 156 | type getLogger struct { 157 | ds ipld.NodeGetter 158 | log []cid.Cid 159 | } 160 | 161 | func (gl *getLogger) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) { 162 | nd, err := gl.ds.Get(ctx, c) 163 | if err != nil { 164 | return nil, err 165 | } 166 | gl.log = append(gl.log, c) 167 | return nd, nil 168 | } 169 | 170 | func (gl *getLogger) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption { 171 | outCh := make(chan *ipld.NodeOption, len(cids)) 172 | nds := gl.ds.GetMany(ctx, cids) 173 | for no := range nds { 174 | if no.Err == nil { 175 | gl.log = append(gl.log, no.Node.Cid()) 176 | } 177 | select { 178 | case outCh <- no: 179 | default: 180 | panic("too many responses") 181 | } 182 | } 183 | return nds 184 | } 185 | 186 | func assertCidList(a, b []cid.Cid) error { 187 | if len(a) != len(b) { 188 | return fmt.Errorf("got different number of cids than expected") 189 | } 190 | for i, c := range a { 191 | if !c.Equals(b[i]) { 192 | return fmt.Errorf("expected %s, got %s", c, b[i]) 193 | } 194 | } 195 | return nil 196 | } 197 | 198 | func TestDiffEnumFail(t *testing.T) { 199 | ctx, cancel := context.WithCancel(context.Background()) 200 | defer cancel() 201 | nds := mkGraph(tg2) 202 | 203 | ds := mdtest.Mock() 204 | lgds := &getLogger{ds: ds} 205 | 206 | for _, s := range []string{"a1", "a2", "b", "c"} { 207 | err := ds.Add(ctx, nds[s]) 208 | if err != nil { 209 | t.Fatal(err) 210 | } 211 | } 212 | 213 | err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) 214 | if !ipld.IsNotFound(err) { 215 | t.Fatal("expected err not found") 216 | } 217 | 218 | err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()}) 219 | if err != nil { 220 | t.Fatal(err) 221 | } 222 | 223 | } 224 | 225 | func TestDiffEnumRecurse(t *testing.T) { 226 | ctx, cancel := context.WithCancel(context.Background()) 227 | defer cancel() 228 | nds := mkGraph(tg3) 229 | 230 | ds := mdtest.Mock() 231 | lgds := &getLogger{ds: ds} 232 | 233 | for _, s := range []string{"a1", "a2", "b", "c", "d"} { 234 | err := ds.Add(ctx, nds[s]) 235 | if err != nil { 236 | t.Fatal(err) 237 | } 238 | } 239 | 240 | err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid()) 241 | if err != nil { 242 | t.Fatal(err) 243 | } 244 | 245 | err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid(), nds["d"].Cid()}) 246 | if err != nil { 247 | t.Fatal(err) 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /dagutils/utils.go: -------------------------------------------------------------------------------- 1 | package dagutils 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "strings" 7 | 8 | bserv "github.com/ipfs/go-blockservice" 9 | ds "github.com/ipfs/go-datastore" 10 | syncds "github.com/ipfs/go-datastore/sync" 11 | bstore "github.com/ipfs/go-ipfs-blockstore" 12 | offline "github.com/ipfs/go-ipfs-exchange-offline" 13 | ipld "github.com/ipfs/go-ipld-format" 14 | 15 | dag "github.com/ipfs/go-merkledag" 16 | ) 17 | 18 | // Editor represents a ProtoNode tree editor and provides methods to 19 | // modify it. 20 | type Editor struct { 21 | root *dag.ProtoNode 22 | 23 | // tmp is a temporary in memory (for now) dagstore for all of the 24 | // intermediary nodes to be stored in 25 | tmp ipld.DAGService 26 | 27 | // src is the dagstore with *all* of the data on it, it is used to pull 28 | // nodes from for modification (nil is a valid value) 29 | src ipld.DAGService 30 | } 31 | 32 | // NewMemoryDagService returns a new, thread-safe in-memory DAGService. 33 | func NewMemoryDagService() ipld.DAGService { 34 | // build mem-datastore for editor's intermediary nodes 35 | bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore())) 36 | bsrv := bserv.New(bs, offline.Exchange(bs)) 37 | return dag.NewDAGService(bsrv) 38 | } 39 | 40 | // NewDagEditor returns an ProtoNode editor. 41 | // 42 | // * root is the node to be modified 43 | // * source is the dagstore to pull nodes from (optional) 44 | func NewDagEditor(root *dag.ProtoNode, source ipld.DAGService) *Editor { 45 | return &Editor{ 46 | root: root, 47 | tmp: NewMemoryDagService(), 48 | src: source, 49 | } 50 | } 51 | 52 | // GetNode returns the a copy of the root node being edited. 53 | func (e *Editor) GetNode() *dag.ProtoNode { 54 | return e.root.Copy().(*dag.ProtoNode) 55 | } 56 | 57 | // GetDagService returns the DAGService used by this editor. 58 | func (e *Editor) GetDagService() ipld.DAGService { 59 | return e.tmp 60 | } 61 | 62 | func addLink(ctx context.Context, ds ipld.DAGService, root *dag.ProtoNode, childname string, childnd ipld.Node) (*dag.ProtoNode, error) { 63 | if childname == "" { 64 | return nil, errors.New("cannot create link with no name") 65 | } 66 | 67 | // ensure that the node we are adding is in the dagservice 68 | err := ds.Add(ctx, childnd) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | _ = ds.Remove(ctx, root.Cid()) 74 | 75 | // ensure no link with that name already exists 76 | _ = root.RemoveNodeLink(childname) // ignore error, only option is ErrNotFound 77 | 78 | if err := root.AddNodeLink(childname, childnd); err != nil { 79 | return nil, err 80 | } 81 | 82 | if err := ds.Add(ctx, root); err != nil { 83 | return nil, err 84 | } 85 | return root, nil 86 | } 87 | 88 | // InsertNodeAtPath inserts a new node in the tree and replaces the current root with the new one. 89 | func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert ipld.Node, create func() *dag.ProtoNode) error { 90 | splpath := strings.Split(pth, "/") 91 | nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create) 92 | if err != nil { 93 | return err 94 | } 95 | e.root = nd 96 | return nil 97 | } 98 | 99 | func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert ipld.Node, create func() *dag.ProtoNode) (*dag.ProtoNode, error) { 100 | if len(path) == 1 { 101 | return addLink(ctx, e.tmp, root, path[0], toinsert) 102 | } 103 | 104 | nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0]) 105 | if err != nil { 106 | // if 'create' is true, we create directories on the way down as needed 107 | if err == dag.ErrLinkNotFound && create != nil { 108 | nd = create() 109 | err = nil // no longer an error case 110 | } else if ipld.IsNotFound(err) { 111 | // try finding it in our source dagstore 112 | nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0]) 113 | } 114 | 115 | // if we receive an ErrNotFound, then our second 'GetLinkedNode' call 116 | // also fails, we want to error out 117 | if err != nil { 118 | return nil, err 119 | } 120 | } 121 | 122 | ndprime, err := e.insertNodeAtPath(ctx, nd, path[1:], toinsert, create) 123 | if err != nil { 124 | return nil, err 125 | } 126 | 127 | _ = e.tmp.Remove(ctx, root.Cid()) 128 | 129 | _ = root.RemoveNodeLink(path[0]) 130 | err = root.AddNodeLink(path[0], ndprime) 131 | if err != nil { 132 | return nil, err 133 | } 134 | 135 | err = e.tmp.Add(ctx, root) 136 | if err != nil { 137 | return nil, err 138 | } 139 | 140 | return root, nil 141 | } 142 | 143 | // RmLink removes the link with the given name and updates the root node of 144 | // the editor. 145 | func (e *Editor) RmLink(ctx context.Context, pth string) error { 146 | splpath := strings.Split(pth, "/") 147 | nd, err := e.rmLink(ctx, e.root, splpath) 148 | if err != nil { 149 | return err 150 | } 151 | e.root = nd 152 | return nil 153 | } 154 | 155 | func (e *Editor) rmLink(ctx context.Context, root *dag.ProtoNode, path []string) (*dag.ProtoNode, error) { 156 | if len(path) == 1 { 157 | // base case, remove node in question 158 | err := root.RemoveNodeLink(path[0]) 159 | if err != nil { 160 | return nil, err 161 | } 162 | 163 | err = e.tmp.Add(ctx, root) 164 | if err != nil { 165 | return nil, err 166 | } 167 | 168 | return root, nil 169 | } 170 | 171 | // search for node in both tmp dagstore and source dagstore 172 | nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0]) 173 | if ipld.IsNotFound(err) { 174 | nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0]) 175 | } 176 | 177 | if err != nil { 178 | return nil, err 179 | } 180 | 181 | nnode, err := e.rmLink(ctx, nd, path[1:]) 182 | if err != nil { 183 | return nil, err 184 | } 185 | 186 | _ = e.tmp.Remove(ctx, root.Cid()) 187 | 188 | _ = root.RemoveNodeLink(path[0]) 189 | err = root.AddNodeLink(path[0], nnode) 190 | if err != nil { 191 | return nil, err 192 | } 193 | 194 | err = e.tmp.Add(ctx, root) 195 | if err != nil { 196 | return nil, err 197 | } 198 | 199 | return root, nil 200 | } 201 | 202 | // Finalize writes the new DAG to the given DAGService and returns the modified 203 | // root node. 204 | func (e *Editor) Finalize(ctx context.Context, ds ipld.DAGService) (*dag.ProtoNode, error) { 205 | nd := e.GetNode() 206 | err := copyDag(ctx, nd, e.tmp, ds) 207 | return nd, err 208 | } 209 | 210 | func copyDag(ctx context.Context, nd ipld.Node, from, to ipld.DAGService) error { 211 | // TODO(#4609): make this batch. 212 | err := to.Add(ctx, nd) 213 | if err != nil { 214 | return err 215 | } 216 | 217 | for _, lnk := range nd.Links() { 218 | child, err := lnk.GetNode(ctx, from) 219 | if err != nil { 220 | if ipld.IsNotFound(err) { 221 | // not found means we didnt modify it, and it should 222 | // already be in the target datastore 223 | continue 224 | } 225 | return err 226 | } 227 | 228 | err = copyDag(ctx, child, from, to) 229 | if err != nil { 230 | return err 231 | } 232 | } 233 | return nil 234 | } 235 | -------------------------------------------------------------------------------- /dagutils/utils_test.go: -------------------------------------------------------------------------------- 1 | package dagutils 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/ipfs/go-cid" 9 | ipld "github.com/ipfs/go-ipld-format" 10 | 11 | dag "github.com/ipfs/go-merkledag" 12 | mdtest "github.com/ipfs/go-merkledag/test" 13 | ) 14 | 15 | func TestAddLink(t *testing.T) { 16 | ctx, context := context.WithCancel(context.Background()) 17 | defer context() 18 | 19 | ds := mdtest.Mock() 20 | fishnode := dag.NodeWithData([]byte("fishcakes!")) 21 | 22 | err := ds.Add(ctx, fishnode) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | 27 | nd := new(dag.ProtoNode) 28 | nnode, err := addLink(ctx, ds, nd, "fish", fishnode) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | fnprime, err := nnode.GetLinkedNode(ctx, ds, "fish") 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | 38 | fnpkey := fnprime.Cid() 39 | if !fnpkey.Equals(fishnode.Cid()) { 40 | t.Fatal("wrong child node found!") 41 | } 42 | } 43 | 44 | func assertNodeAtPath(t *testing.T, ds ipld.DAGService, root *dag.ProtoNode, pth string, exp cid.Cid) { 45 | parts := strings.Split(pth, "/") 46 | cur := root 47 | for _, e := range parts { 48 | nxt, err := cur.GetLinkedProtoNode(context.Background(), ds, e) 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | 53 | cur = nxt 54 | } 55 | 56 | curc := cur.Cid() 57 | if !curc.Equals(exp) { 58 | t.Fatal("node not as expected at end of path") 59 | } 60 | } 61 | 62 | func TestInsertNode(t *testing.T) { 63 | root := new(dag.ProtoNode) 64 | e := NewDagEditor(root, nil) 65 | 66 | testInsert(t, e, "a", "anodefortesting", false, "") 67 | testInsert(t, e, "a/b", "data", false, "") 68 | testInsert(t, e, "a/b/c/d/e", "blah", false, "no link by that name") 69 | testInsert(t, e, "a/b/c/d/e", "foo", true, "") 70 | testInsert(t, e, "a/b/c/d/f", "baz", true, "") 71 | testInsert(t, e, "a/b/c/d/f", "bar", true, "") 72 | 73 | testInsert(t, e, "", "bar", true, "cannot create link with no name") 74 | testInsert(t, e, "////", "slashes", true, "cannot create link with no name") 75 | 76 | c := e.GetNode().Cid() 77 | 78 | if c.String() != "QmZ8yeT9uD6ouJPNAYt62XffYuXBT6b4mP4obRSE9cJrSt" { 79 | t.Fatal("output was different than expected: ", c) 80 | } 81 | } 82 | 83 | func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) { 84 | child := dag.NodeWithData([]byte(data)) 85 | err := e.tmp.Add(context.Background(), child) 86 | if err != nil { 87 | t.Fatal(err) 88 | } 89 | 90 | var c func() *dag.ProtoNode 91 | if create { 92 | c = func() *dag.ProtoNode { 93 | return &dag.ProtoNode{} 94 | } 95 | } 96 | 97 | err = e.InsertNodeAtPath(context.Background(), path, child, c) 98 | if experr != "" { 99 | var got string 100 | if err != nil { 101 | got = err.Error() 102 | } 103 | if got != experr { 104 | t.Fatalf("expected '%s' but got '%s'", experr, got) 105 | } 106 | return 107 | } 108 | 109 | if err != nil { 110 | t.Fatal(err, path, data, create, experr) 111 | } 112 | 113 | assertNodeAtPath(t, e.tmp, e.root, path, child.Cid()) 114 | } 115 | -------------------------------------------------------------------------------- /errservice.go: -------------------------------------------------------------------------------- 1 | package merkledag 2 | 3 | import ( 4 | "context" 5 | 6 | cid "github.com/ipfs/go-cid" 7 | ipld "github.com/ipfs/go-ipld-format" 8 | ) 9 | 10 | // ErrorService implements ipld.DAGService, returning 'Err' for every call. 11 | type ErrorService struct { 12 | Err error 13 | } 14 | 15 | var _ ipld.DAGService = (*ErrorService)(nil) 16 | 17 | // Add returns the cs.Err. 18 | func (cs *ErrorService) Add(ctx context.Context, nd ipld.Node) error { 19 | return cs.Err 20 | } 21 | 22 | // AddMany returns the cs.Err. 23 | func (cs *ErrorService) AddMany(ctx context.Context, nds []ipld.Node) error { 24 | return cs.Err 25 | } 26 | 27 | // Get returns the cs.Err. 28 | func (cs *ErrorService) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) { 29 | return nil, cs.Err 30 | } 31 | 32 | // GetMany many returns the cs.Err. 33 | func (cs *ErrorService) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption { 34 | ch := make(chan *ipld.NodeOption) 35 | close(ch) 36 | return ch 37 | } 38 | 39 | // Remove returns the cs.Err. 40 | func (cs *ErrorService) Remove(ctx context.Context, c cid.Cid) error { 41 | return cs.Err 42 | } 43 | 44 | // RemoveMany returns the cs.Err. 45 | func (cs *ErrorService) RemoveMany(ctx context.Context, cids []cid.Cid) error { 46 | return cs.Err 47 | } 48 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ipfs/go-merkledag 2 | 3 | go 1.23 4 | 5 | require ( 6 | github.com/gogo/protobuf v1.3.2 7 | github.com/ipfs/go-block-format v0.0.3 8 | github.com/ipfs/go-blockservice v0.5.0 9 | github.com/ipfs/go-cid v0.3.2 10 | github.com/ipfs/go-datastore v0.6.0 11 | github.com/ipfs/go-ipfs-blockstore v1.2.0 12 | github.com/ipfs/go-ipfs-exchange-offline v0.3.0 13 | github.com/ipfs/go-ipfs-util v0.0.2 14 | github.com/ipfs/go-ipld-format v0.5.0 15 | github.com/ipfs/go-ipld-legacy v0.2.1 16 | github.com/ipfs/go-log/v2 v2.5.1 17 | github.com/ipld/go-codec-dagpb v1.6.0 18 | github.com/ipld/go-ipld-prime v0.20.0 19 | github.com/multiformats/go-multihash v0.2.1 20 | ) 21 | 22 | require ( 23 | github.com/benbjohnson/clock v1.3.0 // indirect 24 | github.com/cskr/pubsub v1.0.2 // indirect 25 | github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect 26 | github.com/go-logr/logr v1.2.3 // indirect 27 | github.com/go-logr/stdr v1.2.2 // indirect 28 | github.com/google/gopacket v1.1.19 // indirect 29 | github.com/google/uuid v1.3.0 // indirect 30 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 // indirect 31 | github.com/hashicorp/golang-lru v0.5.4 // indirect 32 | github.com/huin/goupnp v1.0.3 // indirect 33 | github.com/ipfs/bbloom v0.0.4 // indirect 34 | github.com/ipfs/go-bitswap v0.11.0 // indirect 35 | github.com/ipfs/go-ipfs-delay v0.0.1 // indirect 36 | github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect 37 | github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect 38 | github.com/ipfs/go-ipfs-pq v0.0.2 // indirect 39 | github.com/ipfs/go-ipfs-routing v0.3.0 // indirect 40 | github.com/ipfs/go-log v1.0.5 // indirect 41 | github.com/ipfs/go-metrics-interface v0.0.1 // indirect 42 | github.com/ipfs/go-peertaskqueue v0.8.0 // indirect 43 | github.com/ipfs/go-verifcid v0.0.1 // indirect 44 | github.com/jackpal/go-nat-pmp v1.0.2 // indirect 45 | github.com/jbenet/goprocess v0.1.4 // indirect 46 | github.com/jtolds/gls v4.2.1+incompatible // indirect 47 | github.com/klauspost/cpuid/v2 v2.1.0 // indirect 48 | github.com/koron/go-ssdp v0.0.3 // indirect 49 | github.com/libp2p/go-buffer-pool v0.1.0 // indirect 50 | github.com/libp2p/go-cidranger v1.1.0 // indirect 51 | github.com/libp2p/go-libp2p v0.22.0 // indirect 52 | github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect 53 | github.com/libp2p/go-libp2p-record v0.2.0 // indirect 54 | github.com/libp2p/go-libp2p-testing v0.12.0 // indirect 55 | github.com/libp2p/go-msgio v0.2.0 // indirect 56 | github.com/libp2p/go-nat v0.1.0 // indirect 57 | github.com/libp2p/go-netroute v0.2.0 // indirect 58 | github.com/libp2p/go-openssl v0.1.0 // indirect 59 | github.com/mattn/go-isatty v0.0.16 // indirect 60 | github.com/mattn/go-pointer v0.0.1 // indirect 61 | github.com/miekg/dns v1.1.50 // indirect 62 | github.com/minio/sha256-simd v1.0.0 // indirect 63 | github.com/mr-tron/base58 v1.2.0 // indirect 64 | github.com/multiformats/go-base32 v0.0.4 // indirect 65 | github.com/multiformats/go-base36 v0.1.0 // indirect 66 | github.com/multiformats/go-multiaddr v0.7.0 // indirect 67 | github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect 68 | github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect 69 | github.com/multiformats/go-multibase v0.1.1 // indirect 70 | github.com/multiformats/go-multicodec v0.8.0 // indirect 71 | github.com/multiformats/go-multistream v0.3.3 // indirect 72 | github.com/multiformats/go-varint v0.0.6 // indirect 73 | github.com/opentracing/opentracing-go v1.2.0 // indirect 74 | github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect 75 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect 76 | github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect 77 | github.com/spaolacci/murmur3 v1.1.0 // indirect 78 | go.opentelemetry.io/otel v1.7.0 // indirect 79 | go.opentelemetry.io/otel/trace v1.7.0 // indirect 80 | go.uber.org/atomic v1.10.0 // indirect 81 | go.uber.org/multierr v1.8.0 // indirect 82 | go.uber.org/zap v1.22.0 // indirect 83 | golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect 84 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect 85 | golang.org/x/net v0.7.0 // indirect 86 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect 87 | golang.org/x/sys v0.5.0 // indirect 88 | golang.org/x/tools v0.1.12 // indirect 89 | google.golang.org/protobuf v1.28.1 // indirect 90 | lukechampine.com/blake3 v1.1.7 // indirect 91 | ) 92 | -------------------------------------------------------------------------------- /merkledag.go: -------------------------------------------------------------------------------- 1 | // Package merkledag implements the IPFS Merkle DAG data structures. 2 | package merkledag 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "sync" 8 | 9 | blocks "github.com/ipfs/go-block-format" 10 | bserv "github.com/ipfs/go-blockservice" 11 | cid "github.com/ipfs/go-cid" 12 | format "github.com/ipfs/go-ipld-format" 13 | legacy "github.com/ipfs/go-ipld-legacy" 14 | dagpb "github.com/ipld/go-codec-dagpb" 15 | 16 | // blank import is used to register the IPLD raw codec 17 | _ "github.com/ipld/go-ipld-prime/codec/raw" 18 | basicnode "github.com/ipld/go-ipld-prime/node/basic" 19 | ) 20 | 21 | var ipldLegacyDecoder *legacy.Decoder 22 | 23 | // TODO: Don't require global registries 24 | func init() { 25 | d := legacy.NewDecoder() 26 | d.RegisterCodec(cid.DagProtobuf, dagpb.Type.PBNode, ProtoNodeConverter) 27 | d.RegisterCodec(cid.Raw, basicnode.Prototype.Bytes, RawNodeConverter) 28 | ipldLegacyDecoder = d 29 | } 30 | 31 | // contextKey is a type to use as value for the ProgressTracker contexts. 32 | type contextKey string 33 | 34 | const progressContextKey contextKey = "progress" 35 | 36 | // NewDAGService constructs a new DAGService (using the default implementation). 37 | // Note that the default implementation is also an ipld.LinkGetter. 38 | func NewDAGService(bs bserv.BlockService) *dagService { 39 | return &dagService{ 40 | Blocks: bs, 41 | decoder: ipldLegacyDecoder, 42 | } 43 | } 44 | 45 | // dagService is an IPFS Merkle DAG service. 46 | // - the root is virtual (like a forest) 47 | // - stores nodes' data in a BlockService 48 | // TODO: should cache Nodes that are in memory, and be 49 | // 50 | // able to free some of them when vm pressure is high 51 | type dagService struct { 52 | Blocks bserv.BlockService 53 | decoder *legacy.Decoder 54 | } 55 | 56 | // Add adds a node to the dagService, storing the block in the BlockService 57 | func (n *dagService) Add(ctx context.Context, nd format.Node) error { 58 | if n == nil { // FIXME remove this assertion. protect with constructor invariant 59 | return fmt.Errorf("dagService is nil") 60 | } 61 | 62 | return n.Blocks.AddBlock(ctx, nd) 63 | } 64 | 65 | func (n *dagService) AddMany(ctx context.Context, nds []format.Node) error { 66 | blks := make([]blocks.Block, len(nds)) 67 | for i, nd := range nds { 68 | blks[i] = nd 69 | } 70 | return n.Blocks.AddBlocks(ctx, blks) 71 | } 72 | 73 | // Get retrieves a node from the dagService, fetching the block in the BlockService 74 | func (n *dagService) Get(ctx context.Context, c cid.Cid) (format.Node, error) { 75 | if n == nil { 76 | return nil, fmt.Errorf("dagService is nil") 77 | } 78 | 79 | ctx, cancel := context.WithCancel(ctx) 80 | defer cancel() 81 | 82 | b, err := n.Blocks.GetBlock(ctx, c) 83 | if err != nil { 84 | return nil, err 85 | } 86 | 87 | return n.decoder.DecodeNode(ctx, b) 88 | } 89 | 90 | // GetLinks return the links for the node, the node doesn't necessarily have 91 | // to exist locally. 92 | func (n *dagService) GetLinks(ctx context.Context, c cid.Cid) ([]*format.Link, error) { 93 | if c.Type() == cid.Raw { 94 | return nil, nil 95 | } 96 | node, err := n.Get(ctx, c) 97 | if err != nil { 98 | return nil, err 99 | } 100 | return node.Links(), nil 101 | } 102 | 103 | func (n *dagService) Remove(ctx context.Context, c cid.Cid) error { 104 | return n.Blocks.DeleteBlock(ctx, c) 105 | } 106 | 107 | // RemoveMany removes multiple nodes from the DAG. It will likely be faster than 108 | // removing them individually. 109 | // 110 | // This operation is not atomic. If it returns an error, some nodes may or may 111 | // not have been removed. 112 | func (n *dagService) RemoveMany(ctx context.Context, cids []cid.Cid) error { 113 | // TODO(#4608): make this batch all the way down. 114 | for _, c := range cids { 115 | if err := n.Blocks.DeleteBlock(ctx, c); err != nil { 116 | return err 117 | } 118 | } 119 | return nil 120 | } 121 | 122 | // GetLinksDirect creates a function to get the links for a node, from 123 | // the node, bypassing the LinkService. If the node does not exist 124 | // locally (and can not be retrieved) an error will be returned. 125 | func GetLinksDirect(serv format.NodeGetter) GetLinks { 126 | return func(ctx context.Context, c cid.Cid) ([]*format.Link, error) { 127 | nd, err := serv.Get(ctx, c) 128 | if err != nil { 129 | return nil, err 130 | } 131 | return nd.Links(), nil 132 | } 133 | } 134 | 135 | type sesGetter struct { 136 | bs *bserv.Session 137 | decoder *legacy.Decoder 138 | } 139 | 140 | // Get gets a single node from the DAG. 141 | func (sg *sesGetter) Get(ctx context.Context, c cid.Cid) (format.Node, error) { 142 | blk, err := sg.bs.GetBlock(ctx, c) 143 | 144 | if err != nil { 145 | return nil, err 146 | } 147 | 148 | return sg.decoder.DecodeNode(ctx, blk) 149 | } 150 | 151 | // GetMany gets many nodes at once, batching the request if possible. 152 | func (sg *sesGetter) GetMany(ctx context.Context, keys []cid.Cid) <-chan *format.NodeOption { 153 | return getNodesFromBG(ctx, sg.bs, keys, sg.decoder) 154 | } 155 | 156 | // WrapSession wraps a blockservice session to satisfy the format.NodeGetter interface 157 | func WrapSession(s *bserv.Session) format.NodeGetter { 158 | return &sesGetter{ 159 | bs: s, 160 | decoder: ipldLegacyDecoder, 161 | } 162 | } 163 | 164 | // Session returns a NodeGetter using a new session for block fetches. 165 | func (n *dagService) Session(ctx context.Context) format.NodeGetter { 166 | session := bserv.NewSession(ctx, n.Blocks) 167 | return &sesGetter{ 168 | bs: session, 169 | decoder: n.decoder, 170 | } 171 | } 172 | 173 | // FetchGraph fetches all nodes that are children of the given node 174 | func FetchGraph(ctx context.Context, root cid.Cid, serv format.DAGService) error { 175 | return FetchGraphWithDepthLimit(ctx, root, -1, serv) 176 | } 177 | 178 | // FetchGraphWithDepthLimit fetches all nodes that are children to the given 179 | // node down to the given depth. maxDepth=0 means "only fetch root", 180 | // maxDepth=1 means "fetch root and its direct children" and so on... 181 | // maxDepth=-1 means unlimited. 182 | func FetchGraphWithDepthLimit(ctx context.Context, root cid.Cid, depthLim int, serv format.DAGService) error { 183 | var ng format.NodeGetter = NewSession(ctx, serv) 184 | 185 | set := make(map[cid.Cid]int) 186 | 187 | // Visit function returns true when: 188 | // * The element is not in the set and we're not over depthLim 189 | // * The element is in the set but recorded depth is deeper 190 | // than currently seen (if we find it higher in the tree we'll need 191 | // to explore deeper than before). 192 | // depthLim = -1 means we only return true if the element is not in the 193 | // set. 194 | visit := func(c cid.Cid, depth int) bool { 195 | oldDepth, ok := set[c] 196 | 197 | if (ok && depthLim < 0) || (depthLim >= 0 && depth > depthLim) { 198 | return false 199 | } 200 | 201 | if !ok || oldDepth > depth { 202 | set[c] = depth 203 | return true 204 | } 205 | return false 206 | } 207 | 208 | // If we have a ProgressTracker, we wrap the visit function to handle it 209 | v, _ := ctx.Value(progressContextKey).(*ProgressTracker) 210 | if v == nil { 211 | return WalkDepth(ctx, GetLinksDirect(ng), root, visit, Concurrent()) 212 | } 213 | 214 | visitProgress := func(c cid.Cid, depth int) bool { 215 | if visit(c, depth) { 216 | v.Increment() 217 | return true 218 | } 219 | return false 220 | } 221 | return WalkDepth(ctx, GetLinksDirect(ng), root, visitProgress, Concurrent()) 222 | } 223 | 224 | // GetMany gets many nodes from the DAG at once. 225 | // 226 | // This method may not return all requested nodes (and may or may not return an 227 | // error indicating that it failed to do so. It is up to the caller to verify 228 | // that it received all nodes. 229 | func (n *dagService) GetMany(ctx context.Context, keys []cid.Cid) <-chan *format.NodeOption { 230 | return getNodesFromBG(ctx, n.Blocks, keys, n.decoder) 231 | } 232 | 233 | func dedupKeys(keys []cid.Cid) []cid.Cid { 234 | set := cid.NewSet() 235 | for _, c := range keys { 236 | set.Add(c) 237 | } 238 | if set.Len() == len(keys) { 239 | return keys 240 | } 241 | return set.Keys() 242 | } 243 | 244 | func getNodesFromBG(ctx context.Context, bs bserv.BlockGetter, keys []cid.Cid, decoder *legacy.Decoder) <-chan *format.NodeOption { 245 | keys = dedupKeys(keys) 246 | 247 | out := make(chan *format.NodeOption, len(keys)) 248 | blocks := bs.GetBlocks(ctx, keys) 249 | var count int 250 | 251 | go func() { 252 | defer close(out) 253 | for { 254 | select { 255 | case b, ok := <-blocks: 256 | if !ok { 257 | if count != len(keys) { 258 | out <- &format.NodeOption{Err: fmt.Errorf("failed to fetch all nodes")} 259 | } 260 | return 261 | } 262 | 263 | nd, err := decoder.DecodeNode(ctx, b) 264 | if err != nil { 265 | out <- &format.NodeOption{Err: err} 266 | return 267 | } 268 | 269 | out <- &format.NodeOption{Node: nd} 270 | count++ 271 | 272 | case <-ctx.Done(): 273 | out <- &format.NodeOption{Err: ctx.Err()} 274 | return 275 | } 276 | } 277 | }() 278 | return out 279 | } 280 | 281 | // GetLinks is the type of function passed to the EnumerateChildren function(s) 282 | // for getting the children of an IPLD node. 283 | type GetLinks func(context.Context, cid.Cid) ([]*format.Link, error) 284 | 285 | // GetLinksWithDAG returns a GetLinks function that tries to use the given 286 | // NodeGetter as a LinkGetter to get the children of a given IPLD node. This may 287 | // allow us to traverse the DAG without actually loading and parsing the node in 288 | // question (if we already have the links cached). 289 | func GetLinksWithDAG(ng format.NodeGetter) GetLinks { 290 | return func(ctx context.Context, c cid.Cid) ([]*format.Link, error) { 291 | return format.GetLinks(ctx, ng, c) 292 | } 293 | } 294 | 295 | // defaultConcurrentFetch is the default maximum number of concurrent fetches 296 | // that 'fetchNodes' will start at a time 297 | const defaultConcurrentFetch = 32 298 | 299 | // walkOptions represent the parameters of a graph walking algorithm 300 | type walkOptions struct { 301 | SkipRoot bool 302 | Concurrency int 303 | ErrorHandler func(c cid.Cid, err error) error 304 | } 305 | 306 | // WalkOption is a setter for walkOptions 307 | type WalkOption func(*walkOptions) 308 | 309 | func (wo *walkOptions) addHandler(handler func(c cid.Cid, err error) error) { 310 | if wo.ErrorHandler != nil { 311 | wo.ErrorHandler = func(c cid.Cid, err error) error { 312 | return handler(c, wo.ErrorHandler(c, err)) 313 | } 314 | } else { 315 | wo.ErrorHandler = handler 316 | } 317 | } 318 | 319 | // SkipRoot is a WalkOption indicating that the root node should skipped 320 | func SkipRoot() WalkOption { 321 | return func(walkOptions *walkOptions) { 322 | walkOptions.SkipRoot = true 323 | } 324 | } 325 | 326 | // Concurrent is a WalkOption indicating that node fetching should be done in 327 | // parallel, with the default concurrency factor. 328 | // NOTE: When using that option, the walk order is *not* guarantee. 329 | // NOTE: It *does not* make multiple concurrent calls to the passed `visit` function. 330 | func Concurrent() WalkOption { 331 | return func(walkOptions *walkOptions) { 332 | walkOptions.Concurrency = defaultConcurrentFetch 333 | } 334 | } 335 | 336 | // Concurrency is a WalkOption indicating that node fetching should be done in 337 | // parallel, with a specific concurrency factor. 338 | // NOTE: When using that option, the walk order is *not* guarantee. 339 | // NOTE: It *does not* make multiple concurrent calls to the passed `visit` function. 340 | func Concurrency(worker int) WalkOption { 341 | return func(walkOptions *walkOptions) { 342 | walkOptions.Concurrency = worker 343 | } 344 | } 345 | 346 | // IgnoreErrors is a WalkOption indicating that the walk should attempt to 347 | // continue even when an error occur. 348 | func IgnoreErrors() WalkOption { 349 | return func(walkOptions *walkOptions) { 350 | walkOptions.addHandler(func(c cid.Cid, err error) error { 351 | return nil 352 | }) 353 | } 354 | } 355 | 356 | // IgnoreMissing is a WalkOption indicating that the walk should continue when 357 | // a node is missing. 358 | func IgnoreMissing() WalkOption { 359 | return func(walkOptions *walkOptions) { 360 | walkOptions.addHandler(func(c cid.Cid, err error) error { 361 | if format.IsNotFound(err) { 362 | return nil 363 | } 364 | return err 365 | }) 366 | } 367 | } 368 | 369 | // OnMissing is a WalkOption adding a callback that will be triggered on a missing 370 | // node. 371 | func OnMissing(callback func(c cid.Cid)) WalkOption { 372 | return func(walkOptions *walkOptions) { 373 | walkOptions.addHandler(func(c cid.Cid, err error) error { 374 | if format.IsNotFound(err) { 375 | callback(c) 376 | } 377 | return err 378 | }) 379 | } 380 | } 381 | 382 | // OnError is a WalkOption adding a custom error handler. 383 | // If this handler return a nil error, the walk will continue. 384 | func OnError(handler func(c cid.Cid, err error) error) WalkOption { 385 | return func(walkOptions *walkOptions) { 386 | walkOptions.addHandler(handler) 387 | } 388 | } 389 | 390 | // WalkGraph will walk the dag in order (depth first) starting at the given root. 391 | func Walk(ctx context.Context, getLinks GetLinks, c cid.Cid, visit func(cid.Cid) bool, options ...WalkOption) error { 392 | visitDepth := func(c cid.Cid, depth int) bool { 393 | return visit(c) 394 | } 395 | 396 | return WalkDepth(ctx, getLinks, c, visitDepth, options...) 397 | } 398 | 399 | // WalkDepth walks the dag starting at the given root and passes the current 400 | // depth to a given visit function. The visit function can be used to limit DAG 401 | // exploration. 402 | func WalkDepth(ctx context.Context, getLinks GetLinks, c cid.Cid, visit func(cid.Cid, int) bool, options ...WalkOption) error { 403 | opts := &walkOptions{} 404 | for _, opt := range options { 405 | opt(opts) 406 | } 407 | 408 | if opts.Concurrency > 1 { 409 | return parallelWalkDepth(ctx, getLinks, c, visit, opts) 410 | } else { 411 | return sequentialWalkDepth(ctx, getLinks, c, 0, visit, opts) 412 | } 413 | } 414 | 415 | func sequentialWalkDepth(ctx context.Context, getLinks GetLinks, root cid.Cid, depth int, visit func(cid.Cid, int) bool, options *walkOptions) error { 416 | if !(options.SkipRoot && depth == 0) { 417 | if !visit(root, depth) { 418 | return nil 419 | } 420 | } 421 | 422 | links, err := getLinks(ctx, root) 423 | if err != nil && options.ErrorHandler != nil { 424 | err = options.ErrorHandler(root, err) 425 | } 426 | if err != nil { 427 | return err 428 | } 429 | 430 | for _, lnk := range links { 431 | if err := sequentialWalkDepth(ctx, getLinks, lnk.Cid, depth+1, visit, options); err != nil { 432 | return err 433 | } 434 | } 435 | return nil 436 | } 437 | 438 | // ProgressTracker is used to show progress when fetching nodes. 439 | type ProgressTracker struct { 440 | Total int 441 | lk sync.Mutex 442 | } 443 | 444 | // DeriveContext returns a new context with value "progress" derived from 445 | // the given one. 446 | func (p *ProgressTracker) DeriveContext(ctx context.Context) context.Context { 447 | return context.WithValue(ctx, progressContextKey, p) 448 | } 449 | 450 | // Increment adds one to the total progress. 451 | func (p *ProgressTracker) Increment() { 452 | p.lk.Lock() 453 | defer p.lk.Unlock() 454 | p.Total++ 455 | } 456 | 457 | // Value returns the current progress. 458 | func (p *ProgressTracker) Value() int { 459 | p.lk.Lock() 460 | defer p.lk.Unlock() 461 | return p.Total 462 | } 463 | 464 | func parallelWalkDepth(ctx context.Context, getLinks GetLinks, root cid.Cid, visit func(cid.Cid, int) bool, options *walkOptions) error { 465 | type cidDepth struct { 466 | cid cid.Cid 467 | depth int 468 | } 469 | 470 | type linksDepth struct { 471 | links []*format.Link 472 | depth int 473 | } 474 | 475 | feed := make(chan cidDepth) 476 | out := make(chan linksDepth) 477 | done := make(chan struct{}) 478 | 479 | var visitlk sync.Mutex 480 | var wg sync.WaitGroup 481 | 482 | errChan := make(chan error) 483 | fetchersCtx, cancel := context.WithCancel(ctx) 484 | defer wg.Wait() 485 | defer cancel() 486 | for i := 0; i < options.Concurrency; i++ { 487 | wg.Add(1) 488 | go func() { 489 | defer wg.Done() 490 | for cdepth := range feed { 491 | ci := cdepth.cid 492 | depth := cdepth.depth 493 | 494 | var shouldVisit bool 495 | 496 | // bypass the root if needed 497 | if !(options.SkipRoot && depth == 0) { 498 | visitlk.Lock() 499 | shouldVisit = visit(ci, depth) 500 | visitlk.Unlock() 501 | } else { 502 | shouldVisit = true 503 | } 504 | 505 | if shouldVisit { 506 | links, err := getLinks(ctx, ci) 507 | if err != nil && options.ErrorHandler != nil { 508 | err = options.ErrorHandler(root, err) 509 | } 510 | if err != nil { 511 | select { 512 | case errChan <- err: 513 | case <-fetchersCtx.Done(): 514 | } 515 | return 516 | } 517 | 518 | outLinks := linksDepth{ 519 | links: links, 520 | depth: depth + 1, 521 | } 522 | 523 | select { 524 | case out <- outLinks: 525 | case <-fetchersCtx.Done(): 526 | return 527 | } 528 | } 529 | select { 530 | case done <- struct{}{}: 531 | case <-fetchersCtx.Done(): 532 | } 533 | } 534 | }() 535 | } 536 | defer close(feed) 537 | 538 | send := feed 539 | var todoQueue []cidDepth 540 | var inProgress int 541 | 542 | next := cidDepth{ 543 | cid: root, 544 | depth: 0, 545 | } 546 | 547 | for { 548 | select { 549 | case send <- next: 550 | inProgress++ 551 | if len(todoQueue) > 0 { 552 | next = todoQueue[0] 553 | todoQueue = todoQueue[1:] 554 | } else { 555 | next = cidDepth{} 556 | send = nil 557 | } 558 | case <-done: 559 | inProgress-- 560 | if inProgress == 0 && !next.cid.Defined() { 561 | return nil 562 | } 563 | case linksDepth := <-out: 564 | for _, lnk := range linksDepth.links { 565 | cd := cidDepth{ 566 | cid: lnk.Cid, 567 | depth: linksDepth.depth, 568 | } 569 | 570 | if !next.cid.Defined() { 571 | next = cd 572 | send = feed 573 | } else { 574 | todoQueue = append(todoQueue, cd) 575 | } 576 | } 577 | case err := <-errChan: 578 | return err 579 | 580 | case <-ctx.Done(): 581 | return ctx.Err() 582 | } 583 | } 584 | } 585 | 586 | var _ format.LinkGetter = &dagService{} 587 | var _ format.NodeGetter = &dagService{} 588 | var _ format.NodeGetter = &sesGetter{} 589 | var _ format.DAGService = &dagService{} 590 | -------------------------------------------------------------------------------- /merkledag_test.go: -------------------------------------------------------------------------------- 1 | package merkledag_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "crypto/rand" 7 | "encoding/hex" 8 | "encoding/json" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "math" 13 | "strings" 14 | "sync" 15 | "testing" 16 | "time" 17 | 18 | . "github.com/ipfs/go-merkledag" 19 | mdpb "github.com/ipfs/go-merkledag/pb" 20 | dstest "github.com/ipfs/go-merkledag/test" 21 | 22 | blocks "github.com/ipfs/go-block-format" 23 | bserv "github.com/ipfs/go-blockservice" 24 | bstest "github.com/ipfs/go-blockservice/test" 25 | cid "github.com/ipfs/go-cid" 26 | offline "github.com/ipfs/go-ipfs-exchange-offline" 27 | u "github.com/ipfs/go-ipfs-util" 28 | ipld "github.com/ipfs/go-ipld-format" 29 | prime "github.com/ipld/go-ipld-prime" 30 | mh "github.com/multiformats/go-multihash" 31 | ) 32 | 33 | var someCid cid.Cid = func() cid.Cid { 34 | c, _ := cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4}) 35 | return c 36 | }() 37 | 38 | // makeDepthTestingGraph makes a small DAG with two levels. The level-two 39 | // nodes are both children of the root and of one of the level 1 nodes. 40 | // This is meant to test the Walk*Depth functions. 41 | func makeDepthTestingGraph(t *testing.T, ds ipld.DAGService) ipld.Node { 42 | root := NodeWithData(nil) 43 | l11 := NodeWithData([]byte("leve1_node1")) 44 | l12 := NodeWithData([]byte("leve1_node2")) 45 | l21 := NodeWithData([]byte("leve2_node1")) 46 | l22 := NodeWithData([]byte("leve2_node2")) 47 | l23 := NodeWithData([]byte("leve2_node3")) 48 | 49 | l11.AddNodeLink(l21.Cid().String(), l21) 50 | l11.AddNodeLink(l22.Cid().String(), l22) 51 | l11.AddNodeLink(l23.Cid().String(), l23) 52 | 53 | root.AddNodeLink(l11.Cid().String(), l11) 54 | root.AddNodeLink(l12.Cid().String(), l12) 55 | root.AddNodeLink(l23.Cid().String(), l23) 56 | 57 | ctx := context.Background() 58 | for _, n := range []ipld.Node{l23, l22, l21, l12, l11, root} { 59 | err := ds.Add(ctx, n) 60 | if err != nil { 61 | t.Fatal(err) 62 | } 63 | } 64 | 65 | return root 66 | } 67 | 68 | // Check that all children of root are in the given set and in the datastore 69 | func traverseAndCheck(t *testing.T, root ipld.Node, ds ipld.DAGService, hasF func(c cid.Cid) bool) { 70 | // traverse dag and check 71 | for _, lnk := range root.Links() { 72 | c := lnk.Cid 73 | if !hasF(c) { 74 | t.Fatal("missing key in set! ", lnk.Cid.String()) 75 | } 76 | child, err := ds.Get(context.Background(), c) 77 | if err != nil { 78 | t.Fatal(err) 79 | } 80 | traverseAndCheck(t, child, ds, hasF) 81 | } 82 | } 83 | 84 | type brokenBuilder struct{} 85 | 86 | func (brokenBuilder) Sum([]byte) (cid.Cid, error) { return cid.Undef, errors.New("Nope!") } 87 | func (brokenBuilder) GetCodec() uint64 { return 0 } 88 | func (b brokenBuilder) WithCodec(uint64) cid.Builder { return b } 89 | 90 | // builder that will pass the basic SetCidBuilder tests but fail otherwise 91 | type sneakyBrokenBuilder struct{} 92 | 93 | func (sneakyBrokenBuilder) Sum(data []byte) (cid.Cid, error) { 94 | if len(data) == 256 { 95 | return V1CidPrefix().Sum(data) 96 | } 97 | return cid.Undef, errors.New("Nope!") 98 | } 99 | func (sneakyBrokenBuilder) GetCodec() uint64 { return 0 } 100 | func (b sneakyBrokenBuilder) WithCodec(uint64) cid.Builder { return b } 101 | 102 | func TestBadBuilderEncode(t *testing.T) { 103 | n := NodeWithData([]byte("boop")) 104 | 105 | t.Run("good builder sanity check", func(t *testing.T) { 106 | if _, err := n.EncodeProtobuf(false); err != nil { 107 | t.Fatal(err) 108 | } 109 | if err := n.SetCidBuilder( 110 | &cid.Prefix{ 111 | MhType: mh.SHA2_256, 112 | MhLength: -1, 113 | Version: 1, 114 | Codec: cid.DagProtobuf, 115 | }, 116 | ); err != nil { 117 | t.Fatal(err) 118 | } 119 | }) 120 | 121 | t.Run("hasher we can't use, should error", func(t *testing.T) { 122 | if err := n.SetCidBuilder( 123 | &cid.Prefix{ 124 | MhType: mh.SHA2_256_TRUNC254_PADDED, 125 | MhLength: 256, 126 | Version: 1, 127 | Codec: cid.DagProtobuf, 128 | }, 129 | ); err == nil { 130 | t.Fatal("expected SetCidBuilder to error on unusable hasher") 131 | } 132 | if _, err := n.EncodeProtobuf(false); err != nil { 133 | t.Fatalf("expected EncodeProtobuf to use safe CidBuilder: %v", err) 134 | } 135 | }) 136 | 137 | t.Run("broken custom builder, should error", func(t *testing.T) { 138 | if err := n.SetCidBuilder(brokenBuilder{}); err == nil { 139 | t.Fatal("expected SetCidBuilder to error on unusable hasher") 140 | } 141 | if _, err := n.EncodeProtobuf(false); err != nil { 142 | t.Fatalf("expected EncodeProtobuf to use safe CidBuilder: %v", err) 143 | } 144 | }) 145 | 146 | t.Run("broken custom builder as pointer, should error", func(t *testing.T) { 147 | if err := n.SetCidBuilder(&brokenBuilder{}); err == nil { 148 | t.Fatal("expected SetCidBuilder to error on unusable hasher") 149 | } 150 | if _, err := n.EncodeProtobuf(false); err != nil { 151 | t.Fatalf("expected EncodeProtobuf to use safe CidBuilder: %v", err) 152 | } 153 | }) 154 | 155 | t.Run("broken sneaky custom builder, should error", func(t *testing.T) { 156 | if err := n.SetCidBuilder(sneakyBrokenBuilder{}); err != nil { 157 | t.Fatalf("expected SetCidBuilder to not error with sneaky custom builder: %v", err) 158 | } 159 | if _, err := n.EncodeProtobuf(false); err == nil { 160 | t.Fatal("expected EncodeProtobuf to fail using the sneaky custom builder") 161 | } 162 | if len(n.RawData()) != 0 { 163 | t.Fatal("expected RawData to return zero-byte slice") 164 | } 165 | if n.Cid().String() != "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" { 166 | t.Fatal("expected Cid to return the zero dag-pb CID") 167 | } 168 | if n.String() != "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" { 169 | t.Fatal("expected String to return the zero dag-pb CID string") 170 | } 171 | }) 172 | } 173 | 174 | func TestLinkChecking(t *testing.T) { 175 | cases := []struct { 176 | name string 177 | fn func(*ProtoNode) error 178 | }{ 179 | { 180 | name: "AddRawLink overflow Tsize", 181 | fn: func(n *ProtoNode) error { 182 | return n.AddRawLink("foo", &ipld.Link{Size: math.MaxUint64, Cid: someCid}) 183 | }, 184 | }, 185 | 186 | { 187 | name: "AddRawLink undefined CID", 188 | fn: func(n *ProtoNode) error { 189 | return n.AddRawLink("foo", &ipld.Link{Cid: cid.Undef}) 190 | }, 191 | }, 192 | 193 | { 194 | name: "SetLinks overflow Tsize", 195 | fn: func(n *ProtoNode) error { 196 | return n.SetLinks([]*ipld.Link{{Size: math.MaxUint64, Cid: someCid}}) 197 | }, 198 | }, 199 | 200 | { 201 | name: "SetLinks undefined CID", 202 | fn: func(n *ProtoNode) error { 203 | return n.SetLinks([]*ipld.Link{{Cid: cid.Undef}}) 204 | }, 205 | }, 206 | 207 | { 208 | name: "UnmarshalJSON overflow Tsize", 209 | fn: func(n *ProtoNode) error { 210 | return n.UnmarshalJSON([]byte(`{"data":null,"links":[{"Name":"","Size":18446744073709549568,"Cid":{"/":"QmNPWHBrVQiiV8FpyNuEPhB9E2rbvdy9Yx79EY1EJuyf9o"}}]}`)) 211 | }, 212 | }, 213 | 214 | { 215 | name: "UnmarshalJSON undefined CID", 216 | fn: func(n *ProtoNode) error { 217 | return n.UnmarshalJSON([]byte(`{"data":null,"links":[{"Name":"","Size":100}]}`)) 218 | }, 219 | }, 220 | } 221 | 222 | for _, tc := range cases { 223 | t.Run(tc.name, func(t *testing.T) { 224 | n := NodeWithData([]byte("boop")) 225 | err := tc.fn(n) 226 | if err == nil { 227 | t.Fatal("expected error") 228 | } 229 | }) 230 | } 231 | 232 | t.Run("round-trip block with bad Tsize", func(t *testing.T) { 233 | badblock, _ := hex.DecodeString("122f0a22122000bb3604d2ecd386227007c548249521fbb9a394e1e26460091d0a692888e7361880f0ffffffffffffff01") 234 | n, err := DecodeProtobuf(badblock) 235 | if err != nil { 236 | t.Fatal(err) 237 | } 238 | // sanity 239 | if len(n.Links()) != 1 { 240 | t.Fatal("expected a link") 241 | } 242 | // sanity 243 | if n.Links()[0].Size <= math.MaxInt64 { 244 | t.Fatal("expected link Tsize to be oversized") 245 | } 246 | 247 | // forced round-trip 248 | byts, err := n.EncodeProtobuf(true) 249 | if err != nil { 250 | t.Fatal(err) 251 | } 252 | n, err = DecodeProtobuf(byts) 253 | if err != nil { 254 | t.Fatal(err) 255 | } 256 | if len(n.Links()) != 1 { 257 | t.Fatal("expected a link") 258 | } 259 | if n.Links()[0].Size != 0 { 260 | t.Fatal("expected link Tsize to be truncated on reencode") 261 | } 262 | }) 263 | } 264 | 265 | func TestNode(t *testing.T) { 266 | 267 | n1 := NodeWithData([]byte("beep")) 268 | n2 := NodeWithData([]byte("boop")) 269 | n3 := NodeWithData([]byte("beep boop")) 270 | if err := n3.AddNodeLink("beep-link", n1); err != nil { 271 | t.Error(err) 272 | } 273 | if err := n3.AddNodeLink("boop-link", n2); err != nil { 274 | t.Error(err) 275 | } 276 | 277 | printn := func(name string, n *ProtoNode) { 278 | fmt.Println(">", name) 279 | fmt.Println("data:", string(n.Data())) 280 | 281 | fmt.Println("links:") 282 | for _, l := range n.Links() { 283 | fmt.Println("-", l.Name, l.Size, l.Cid) 284 | } 285 | 286 | e, err := n.EncodeProtobuf(false) 287 | if err != nil { 288 | t.Error(err) 289 | } else { 290 | fmt.Println("encoded:", e) 291 | } 292 | 293 | h := n.Multihash() 294 | k := n.Cid().Hash() 295 | if k.String() != h.String() { 296 | t.Error("Key is not equivalent to multihash") 297 | } else { 298 | fmt.Println("key: ", k) 299 | } 300 | 301 | SubtestNodeStat(t, n) 302 | } 303 | 304 | printn("beep", n1) 305 | printn("boop", n2) 306 | printn("beep boop", n3) 307 | } 308 | 309 | func SubtestNodeStat(t *testing.T, n *ProtoNode) { 310 | enc, err := n.EncodeProtobuf(true) 311 | if err != nil { 312 | t.Error("n.EncodeProtobuf(true) failed") 313 | return 314 | } 315 | 316 | cumSize, err := n.Size() 317 | if err != nil { 318 | t.Error("n.Size() failed") 319 | return 320 | } 321 | 322 | k := n.Cid() 323 | 324 | expected := ipld.NodeStat{ 325 | NumLinks: len(n.Links()), 326 | BlockSize: len(enc), 327 | LinksSize: len(enc) - len(n.Data()), // includes framing. 328 | DataSize: len(n.Data()), 329 | CumulativeSize: int(cumSize), 330 | Hash: k.String(), 331 | } 332 | 333 | actual, err := n.Stat() 334 | if err != nil { 335 | t.Error("n.Stat() failed") 336 | return 337 | } 338 | 339 | if expected != *actual { 340 | t.Errorf("n.Stat incorrect.\nexpect: %s\nactual: %s", expected, actual) 341 | } else { 342 | fmt.Printf("n.Stat correct: %s\n", actual) 343 | } 344 | } 345 | 346 | type devZero struct{} 347 | 348 | func (devZero) Read(b []byte) (int, error) { 349 | for i := range b { 350 | b[i] = 0 351 | } 352 | return len(b), nil 353 | } 354 | 355 | func TestBatchFetch(t *testing.T) { 356 | read := io.LimitReader(u.NewTimeSeededRand(), 1024*32) 357 | runBatchFetchTest(t, read) 358 | } 359 | 360 | func TestBatchFetchDupBlock(t *testing.T) { 361 | read := io.LimitReader(devZero{}, 1024*32) 362 | runBatchFetchTest(t, read) 363 | } 364 | 365 | // makeTestDAG creates a simple DAG from the data in a reader. 366 | // First, a node is created from each 512 bytes of data from the reader 367 | // (like a the Size chunker would do). Then all nodes are added as children 368 | // to a root node, which is returned. 369 | func makeTestDAG(t *testing.T, read io.Reader, ds ipld.DAGService) ipld.Node { 370 | p := make([]byte, 512) 371 | nodes := []*ProtoNode{} 372 | 373 | for { 374 | n, err := io.ReadFull(read, p) 375 | if err == io.EOF { 376 | break 377 | } 378 | 379 | if err != nil { 380 | t.Fatal(err) 381 | } 382 | 383 | if n != len(p) { 384 | t.Fatal("should have read 512 bytes from the reader") 385 | } 386 | 387 | protoNode := NodeWithData(p) 388 | nodes = append(nodes, protoNode) 389 | } 390 | 391 | ctx := context.Background() 392 | // Add a root referencing all created nodes 393 | root := NodeWithData(nil) 394 | for _, n := range nodes { 395 | err := root.AddNodeLink(n.Cid().String(), n) 396 | if err != nil { 397 | t.Fatal(err) 398 | } 399 | err = ds.Add(ctx, n) 400 | if err != nil { 401 | t.Fatal(err) 402 | } 403 | } 404 | err := ds.Add(ctx, root) 405 | if err != nil { 406 | t.Fatal(err) 407 | } 408 | return root 409 | } 410 | 411 | // makeTestDAGReader takes the root node as returned by makeTestDAG and 412 | // provides a reader that reads all the RawData from that node and its children. 413 | func makeTestDAGReader(t *testing.T, root ipld.Node, ds ipld.DAGService) io.Reader { 414 | ctx := context.Background() 415 | buf := new(bytes.Buffer) 416 | buf.Write(root.RawData()) 417 | for _, l := range root.Links() { 418 | n, err := ds.Get(ctx, l.Cid) 419 | if err != nil { 420 | t.Fatal(err) 421 | } 422 | _, err = buf.Write(n.RawData()) 423 | if err != nil { 424 | t.Fatal(err) 425 | } 426 | } 427 | return buf 428 | } 429 | 430 | func runBatchFetchTest(t *testing.T, read io.Reader) { 431 | ctx := context.Background() 432 | var dagservs []ipld.DAGService 433 | for _, bsi := range bstest.Mocks(5) { 434 | dagservs = append(dagservs, NewDAGService(bsi)) 435 | } 436 | 437 | root := makeTestDAG(t, read, dagservs[0]) 438 | 439 | t.Log("finished setup.") 440 | 441 | dagr := makeTestDAGReader(t, root, dagservs[0]) 442 | 443 | expected, err := io.ReadAll(dagr) 444 | if err != nil { 445 | t.Fatal(err) 446 | } 447 | 448 | err = dagservs[0].Add(ctx, root) 449 | if err != nil { 450 | t.Fatal(err) 451 | } 452 | 453 | t.Log("Added file to first node.") 454 | 455 | c := root.Cid() 456 | 457 | wg := sync.WaitGroup{} 458 | errs := make(chan error) 459 | 460 | for i := 1; i < len(dagservs); i++ { 461 | wg.Add(1) 462 | go func(i int) { 463 | defer wg.Done() 464 | first, err := dagservs[i].Get(ctx, c) 465 | if err != nil { 466 | errs <- err 467 | } 468 | fmt.Println("Got first node back.") 469 | 470 | firstpb, ok := first.(*ProtoNode) 471 | if !ok { 472 | errs <- ErrNotProtobuf 473 | } 474 | read := makeTestDAGReader(t, firstpb, dagservs[i]) 475 | datagot, err := io.ReadAll(read) 476 | if err != nil { 477 | errs <- err 478 | } 479 | 480 | if !bytes.Equal(datagot, expected) { 481 | errs <- errors.New("got bad data back") 482 | } 483 | }(i) 484 | } 485 | 486 | go func() { 487 | wg.Wait() 488 | close(errs) 489 | }() 490 | 491 | for err := range errs { 492 | if err != nil { 493 | t.Fatal(err) 494 | } 495 | } 496 | } 497 | 498 | func TestCantGet(t *testing.T) { 499 | ds := dstest.Mock() 500 | a := NodeWithData([]byte("A")) 501 | 502 | c := a.Cid() 503 | _, err := ds.Get(context.Background(), c) 504 | if !strings.Contains(err.Error(), "not found") { 505 | t.Fatal("expected err not found, got: ", err) 506 | } 507 | } 508 | 509 | func TestFetchGraph(t *testing.T) { 510 | var dservs []ipld.DAGService 511 | bsis := bstest.Mocks(2) 512 | for _, bsi := range bsis { 513 | dservs = append(dservs, NewDAGService(bsi)) 514 | } 515 | 516 | read := io.LimitReader(u.NewTimeSeededRand(), 1024*32) 517 | root := makeTestDAG(t, read, dservs[0]) 518 | 519 | err := FetchGraph(context.TODO(), root.Cid(), dservs[1]) 520 | if err != nil { 521 | t.Fatal(err) 522 | } 523 | 524 | // create an offline dagstore and ensure all blocks were fetched 525 | bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore())) 526 | 527 | offlineDS := NewDAGService(bs) 528 | 529 | err = Walk(context.Background(), offlineDS.GetLinks, root.Cid(), func(_ cid.Cid) bool { return true }) 530 | if err != nil { 531 | t.Fatal(err) 532 | } 533 | } 534 | 535 | func TestFetchGraphWithDepthLimit(t *testing.T) { 536 | type testcase struct { 537 | depthLim int 538 | setLen int 539 | } 540 | 541 | tests := []testcase{ 542 | {1, 4}, 543 | {0, 1}, 544 | {-1, 6}, 545 | {2, 6}, 546 | {3, 6}, 547 | } 548 | 549 | testF := func(t *testing.T, tc testcase) { 550 | var dservs []ipld.DAGService 551 | bsis := bstest.Mocks(2) 552 | for _, bsi := range bsis { 553 | dservs = append(dservs, NewDAGService(bsi)) 554 | } 555 | 556 | root := makeDepthTestingGraph(t, dservs[0]) 557 | 558 | err := FetchGraphWithDepthLimit(context.TODO(), root.Cid(), tc.depthLim, dservs[1]) 559 | if err != nil { 560 | t.Fatal(err) 561 | } 562 | 563 | // create an offline dagstore and ensure all blocks were fetched 564 | bs := bserv.New(bsis[1].Blockstore(), offline.Exchange(bsis[1].Blockstore())) 565 | 566 | offlineDS := NewDAGService(bs) 567 | 568 | set := make(map[string]int) 569 | visitF := func(c cid.Cid, depth int) bool { 570 | if tc.depthLim < 0 || depth <= tc.depthLim { 571 | set[string(c.Bytes())] = depth 572 | return true 573 | } 574 | return false 575 | 576 | } 577 | 578 | err = WalkDepth(context.Background(), offlineDS.GetLinks, root.Cid(), visitF) 579 | if err != nil { 580 | t.Fatal(err) 581 | } 582 | 583 | if len(set) != tc.setLen { 584 | t.Fatalf("expected %d nodes but visited %d", tc.setLen, len(set)) 585 | } 586 | } 587 | 588 | for _, tc := range tests { 589 | t.Run(fmt.Sprintf("depth limit %d", tc.depthLim), func(t *testing.T) { 590 | testF(t, tc) 591 | }) 592 | } 593 | } 594 | 595 | func TestWalk(t *testing.T) { 596 | bsi := bstest.Mocks(1) 597 | ds := NewDAGService(bsi[0]) 598 | 599 | read := io.LimitReader(u.NewTimeSeededRand(), 1024*1024) 600 | root := makeTestDAG(t, read, ds) 601 | 602 | set := cid.NewSet() 603 | 604 | err := Walk(context.Background(), ds.GetLinks, root.Cid(), set.Visit) 605 | if err != nil { 606 | t.Fatal(err) 607 | } 608 | 609 | traverseAndCheck(t, root, ds, set.Has) 610 | } 611 | 612 | func TestFetchFailure(t *testing.T) { 613 | ctx := context.Background() 614 | 615 | ds := dstest.Mock() 616 | ds_bad := dstest.Mock() 617 | 618 | top := new(ProtoNode) 619 | for i := 0; i < 10; i++ { 620 | nd := NodeWithData([]byte{byte('a' + i)}) 621 | err := ds.Add(ctx, nd) 622 | if err != nil { 623 | t.Fatal(err) 624 | } 625 | 626 | err = top.AddNodeLink(fmt.Sprintf("AA%d", i), nd) 627 | if err != nil { 628 | t.Fatal(err) 629 | } 630 | } 631 | 632 | for i := 0; i < 10; i++ { 633 | nd := NodeWithData([]byte{'f', 'a' + byte(i)}) 634 | err := ds_bad.Add(ctx, nd) 635 | if err != nil { 636 | t.Fatal(err) 637 | } 638 | 639 | err = top.AddNodeLink(fmt.Sprintf("BB%d", i), nd) 640 | if err != nil { 641 | t.Fatal(err) 642 | } 643 | } 644 | 645 | getters := ipld.GetDAG(ctx, ds, top) 646 | for i, getter := range getters { 647 | _, err := getter.Get(ctx) 648 | if err != nil && i < 10 { 649 | t.Fatal(err) 650 | } 651 | if err == nil && i >= 10 { 652 | t.Fatal("should have failed request") 653 | } 654 | } 655 | } 656 | 657 | func TestUnmarshalFailure(t *testing.T) { 658 | badData := []byte("hello world") 659 | 660 | _, err := DecodeProtobuf(badData) 661 | if err == nil { 662 | t.Fatal("shouldnt succeed to parse this") 663 | } 664 | 665 | // now with a bad link 666 | pbn := &mdpb.PBNode{Links: []*mdpb.PBLink{{Hash: []byte("not a multihash")}}} 667 | badlink, err := pbn.Marshal() 668 | if err != nil { 669 | t.Fatal(err) 670 | } 671 | 672 | _, err = DecodeProtobuf(badlink) 673 | if err == nil { 674 | t.Fatal("should have failed to parse node with bad link") 675 | } 676 | 677 | n := &ProtoNode{} 678 | n.Marshal() 679 | } 680 | 681 | func TestBasicAddGet(t *testing.T) { 682 | ctx := context.Background() 683 | 684 | ds := dstest.Mock() 685 | nd := new(ProtoNode) 686 | 687 | err := ds.Add(ctx, nd) 688 | if err != nil { 689 | t.Fatal(err) 690 | } 691 | 692 | out, err := ds.Get(ctx, nd.Cid()) 693 | if err != nil { 694 | t.Fatal(err) 695 | } 696 | 697 | if !nd.Cid().Equals(out.Cid()) { 698 | t.Fatal("output didnt match input") 699 | } 700 | } 701 | 702 | func TestGetRawNodes(t *testing.T) { 703 | ctx := context.Background() 704 | 705 | rn := NewRawNode([]byte("test")) 706 | 707 | ds := dstest.Mock() 708 | 709 | err := ds.Add(ctx, rn) 710 | if err != nil { 711 | t.Fatal(err) 712 | } 713 | 714 | if !rn.Cid().Equals(rn.Cid()) { 715 | t.Fatal("output cids didnt match") 716 | } 717 | 718 | out, err := ds.Get(ctx, rn.Cid()) 719 | if err != nil { 720 | t.Fatal(err) 721 | } 722 | 723 | if !bytes.Equal(out.RawData(), []byte("test")) { 724 | t.Fatal("raw block should match input data") 725 | } 726 | 727 | if out.Links() != nil { 728 | t.Fatal("raw blocks shouldnt have links") 729 | } 730 | 731 | if out.Tree("", -1) != nil { 732 | t.Fatal("tree should return no paths in a raw block") 733 | } 734 | 735 | size, err := out.Size() 736 | if err != nil { 737 | t.Fatal(err) 738 | } 739 | if size != 4 { 740 | t.Fatal("expected size to be 4") 741 | } 742 | 743 | ns, err := out.Stat() 744 | if err != nil { 745 | t.Fatal(err) 746 | } 747 | 748 | if ns.DataSize != 4 { 749 | t.Fatal("expected size to be 4, got: ", ns.DataSize) 750 | } 751 | 752 | _, _, err = out.Resolve([]string{"foo"}) 753 | if err != ErrLinkNotFound { 754 | t.Fatal("shouldnt find links under raw blocks") 755 | } 756 | } 757 | 758 | func TestProtoNodeResolve(t *testing.T) { 759 | 760 | nd := new(ProtoNode) 761 | nd.SetLinks([]*ipld.Link{{Name: "foo", Cid: someCid}}) 762 | 763 | lnk, left, err := nd.ResolveLink([]string{"foo", "bar"}) 764 | if err != nil { 765 | t.Fatal(err) 766 | } 767 | 768 | if len(left) != 1 || left[0] != "bar" { 769 | t.Fatal("expected the single path element 'bar' to remain") 770 | } 771 | 772 | if lnk.Name != "foo" { 773 | t.Fatal("how did we get anything else?") 774 | } 775 | 776 | tvals := nd.Tree("", -1) 777 | if len(tvals) != 1 || tvals[0] != "foo" { 778 | t.Fatal("expected tree to return []{\"foo\"}") 779 | } 780 | } 781 | 782 | func TestCidRetention(t *testing.T) { 783 | ctx := context.Background() 784 | 785 | nd := new(ProtoNode) 786 | nd.SetData([]byte("fooooo")) 787 | 788 | pref := nd.Cid().Prefix() 789 | pref.Version = 1 790 | 791 | c2, err := pref.Sum(nd.RawData()) 792 | if err != nil { 793 | t.Fatal(err) 794 | } 795 | 796 | blk, err := blocks.NewBlockWithCid(nd.RawData(), c2) 797 | if err != nil { 798 | t.Fatal(err) 799 | } 800 | 801 | bs := dstest.Bserv() 802 | err = bs.AddBlock(ctx, blk) 803 | if err != nil { 804 | t.Fatal(err) 805 | } 806 | 807 | ds := NewDAGService(bs) 808 | out, err := ds.Get(ctx, c2) 809 | if err != nil { 810 | t.Fatal(err) 811 | } 812 | 813 | if !out.Cid().Equals(c2) { 814 | t.Fatal("output cid didnt match") 815 | } 816 | } 817 | 818 | func TestCidRawDoesnNeedData(t *testing.T) { 819 | srv := NewDAGService(dstest.Bserv()) 820 | nd := NewRawNode([]byte("somedata")) 821 | 822 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 823 | defer cancel() 824 | 825 | // there is no data for this node in the blockservice 826 | // so dag service can't load it 827 | links, err := srv.GetLinks(ctx, nd.Cid()) 828 | if err != nil { 829 | t.Fatal(err) 830 | } 831 | if len(links) != 0 { 832 | t.Fatal("raw node shouldn't have any links") 833 | } 834 | } 835 | 836 | func TestRawToJson(t *testing.T) { 837 | rawData := []byte{1, 2, 3, 4} 838 | nd := NewRawNode(rawData) 839 | encoded, err := nd.MarshalJSON() 840 | if err != nil { 841 | t.Fatal(err) 842 | } 843 | var res interface{} 844 | err = json.Unmarshal(encoded, &res) 845 | if err != nil { 846 | t.Fatal(err) 847 | } 848 | resBytes, ok := res.(string) 849 | if !ok { 850 | t.Fatal("expected to marshal to a string") 851 | } 852 | if string(rawData) != resBytes { 853 | t.Fatal("failed to round-trip bytes") 854 | } 855 | } 856 | 857 | func TestGetManyDuplicate(t *testing.T) { 858 | ctx := context.Background() 859 | 860 | srv := NewDAGService(dstest.Bserv()) 861 | 862 | nd := NodeWithData([]byte("foo")) 863 | if err := srv.Add(ctx, nd); err != nil { 864 | t.Fatal(err) 865 | } 866 | nds := srv.GetMany(ctx, []cid.Cid{nd.Cid(), nd.Cid(), nd.Cid()}) 867 | out, ok := <-nds 868 | if !ok { 869 | t.Fatal("expecting node foo") 870 | } 871 | if out.Err != nil { 872 | t.Fatal(out.Err) 873 | } 874 | if !out.Node.Cid().Equals(nd.Cid()) { 875 | t.Fatal("got wrong node") 876 | } 877 | out, ok = <-nds 878 | if ok { 879 | if out.Err != nil { 880 | t.Fatal(out.Err) 881 | } else { 882 | t.Fatal("expecting no more nodes") 883 | } 884 | } 885 | } 886 | 887 | func TestEnumerateAsyncFailsNotFound(t *testing.T) { 888 | ctx := context.Background() 889 | 890 | a := NodeWithData([]byte("foo1")) 891 | b := NodeWithData([]byte("foo2")) 892 | c := NodeWithData([]byte("foo3")) 893 | d := NodeWithData([]byte("foo4")) 894 | e := NodeWithData([]byte("foo5")) 895 | 896 | ds := dstest.Mock() 897 | for _, n := range []ipld.Node{a, b, c} { 898 | err := ds.Add(ctx, n) 899 | if err != nil { 900 | t.Fatal(err) 901 | } 902 | } 903 | 904 | parent := new(ProtoNode) 905 | if err := parent.AddNodeLink("a", a); err != nil { 906 | t.Fatal(err) 907 | } 908 | 909 | if err := parent.AddNodeLink("b", b); err != nil { 910 | t.Fatal(err) 911 | } 912 | 913 | if err := parent.AddNodeLink("c", c); err != nil { 914 | t.Fatal(err) 915 | } 916 | 917 | if err := parent.AddNodeLink("d", d); err != nil { 918 | t.Fatal(err) 919 | } 920 | 921 | if err := parent.AddNodeLink("e", e); err != nil { 922 | t.Fatal(err) 923 | } 924 | 925 | err := ds.Add(ctx, parent) 926 | if err != nil { 927 | t.Fatal(err) 928 | } 929 | 930 | cset := cid.NewSet() 931 | err = Walk(ctx, GetLinksDirect(ds), parent.Cid(), cset.Visit) 932 | if err == nil { 933 | t.Fatal("this should have failed") 934 | } 935 | } 936 | 937 | func TestLinkSorting(t *testing.T) { 938 | az := "az" 939 | aaaa := "aaaa" 940 | bbbb := "bbbb" 941 | cccc := "cccc" 942 | 943 | azBlk := NewRawNode([]byte(az)) 944 | aaaaBlk := NewRawNode([]byte(aaaa)) 945 | bbbbBlk := NewRawNode([]byte(bbbb)) 946 | ccccBlk := NewRawNode([]byte(cccc)) 947 | pbn := &mdpb.PBNode{ 948 | Links: []*mdpb.PBLink{ 949 | {Hash: bbbbBlk.Cid().Bytes(), Name: &bbbb}, 950 | {Hash: azBlk.Cid().Bytes(), Name: &az}, 951 | {Hash: aaaaBlk.Cid().Bytes(), Name: &aaaa}, 952 | {Hash: ccccBlk.Cid().Bytes(), Name: &cccc}, 953 | }, 954 | } 955 | byts, err := pbn.Marshal() 956 | if err != nil { 957 | t.Fatal(err) 958 | } 959 | 960 | mustLookupNodeString := func(t *testing.T, node prime.Node, name string) prime.Node { 961 | subNode, err := node.LookupByString(name) 962 | if err != nil { 963 | t.Fatal(err) 964 | } 965 | return subNode 966 | } 967 | 968 | mustLookupNodeIndex := func(t *testing.T, node prime.Node, idx int64) prime.Node { 969 | subNode, err := node.LookupByIndex(idx) 970 | if err != nil { 971 | t.Fatal(err) 972 | } 973 | return subNode 974 | } 975 | 976 | mustNodeAsString := func(t *testing.T, node prime.Node) string { 977 | str, err := node.AsString() 978 | if err != nil { 979 | t.Fatal(err) 980 | } 981 | return str 982 | } 983 | 984 | verifyUnsortedNode := func(t *testing.T, node *ProtoNode) { 985 | links := node.Links() 986 | if len(links) != 4 { 987 | t.Errorf("wrong number of links, expected 4 but got %d", len(links)) 988 | } 989 | if links[0].Name != bbbb { 990 | t.Errorf("expected link 0 to be 'bbbb', got %s", links[0].Name) 991 | } 992 | if links[1].Name != az { 993 | t.Errorf("expected link 0 to be 'az', got %s", links[1].Name) 994 | } 995 | if links[2].Name != aaaa { 996 | t.Errorf("expected link 0 to be 'aaaa', got %s", links[2].Name) 997 | } 998 | if links[3].Name != cccc { 999 | t.Errorf("expected link 0 to be 'cccc', got %s", links[3].Name) 1000 | } 1001 | 1002 | // check the go-ipld-prime form 1003 | linksNode := mustLookupNodeString(t, node, "Links") 1004 | if linksNode.Length() != 4 { 1005 | t.Errorf("(Node) wrong number of links, expected 4 but got %d", len(links)) 1006 | } 1007 | if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 0), "Name")); name != bbbb { 1008 | t.Errorf("(Node) expected link 0 to be 'bbbb', got %s", name) 1009 | } 1010 | if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 1), "Name")); name != az { 1011 | t.Errorf("(Node) expected link 0 to be 'az', got %s", name) 1012 | } 1013 | if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 2), "Name")); name != aaaa { 1014 | t.Errorf("(Node) expected link 0 to be 'aaaa', got %s", name) 1015 | } 1016 | if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 3), "Name")); name != cccc { 1017 | t.Errorf("(Node) expected link 0 to be 'cccc', got %s", name) 1018 | } 1019 | } 1020 | 1021 | verifySortedNode := func(t *testing.T, node *ProtoNode) { 1022 | links := node.Links() 1023 | if len(links) != 4 { 1024 | t.Errorf("wrong number of links, expected 4 but got %d", len(links)) 1025 | } 1026 | if links[0].Name != aaaa { 1027 | t.Errorf("expected link 0 to be 'aaaa', got %s", links[0].Name) 1028 | } 1029 | if links[1].Name != az { 1030 | t.Errorf("expected link 0 to be 'az', got %s", links[1].Name) 1031 | } 1032 | if links[2].Name != bbbb { 1033 | t.Errorf("expected link 0 to be 'bbbb', got %s", links[2].Name) 1034 | } 1035 | if links[3].Name != cccc { 1036 | t.Errorf("expected link 0 to be 'cccc', got %s", links[3].Name) 1037 | } 1038 | 1039 | // check the go-ipld-prime form 1040 | linksNode := mustLookupNodeString(t, node, "Links") 1041 | if linksNode.Length() != 4 { 1042 | t.Errorf("(Node) wrong number of links, expected 4 but got %d", len(links)) 1043 | } 1044 | if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 0), "Name")); name != aaaa { 1045 | t.Errorf("(Node) expected link 0 to be 'aaaa', got %s", name) 1046 | } 1047 | if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 1), "Name")); name != az { 1048 | t.Errorf("(Node) expected link 0 to be 'az', got %s", name) 1049 | } 1050 | if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 2), "Name")); name != bbbb { 1051 | t.Errorf("(Node) expected link 0 to be 'bbbb', got %s", name) 1052 | } 1053 | if name := mustNodeAsString(t, mustLookupNodeString(t, mustLookupNodeIndex(t, linksNode, 3), "Name")); name != cccc { 1054 | t.Errorf("(Node) expected link 0 to be 'cccc', got %s", name) 1055 | } 1056 | } 1057 | 1058 | t.Run("decode", func(t *testing.T) { 1059 | node, err := DecodeProtobuf(byts) 1060 | if err != nil { 1061 | t.Fatal(err) 1062 | } 1063 | verifyUnsortedNode(t, node) 1064 | }) 1065 | 1066 | t.Run("RawData() should not mutate, should return original form", func(t *testing.T) { 1067 | node, err := DecodeProtobuf(byts) 1068 | if err != nil { 1069 | t.Fatal(err) 1070 | } 1071 | rawData := node.RawData() 1072 | verifyUnsortedNode(t, node) 1073 | if !bytes.Equal(rawData, byts) { 1074 | t.Error("RawData() did not return original bytes") 1075 | } 1076 | }) 1077 | 1078 | t.Run("Size() should not mutate", func(t *testing.T) { 1079 | node, err := DecodeProtobuf(byts) 1080 | if err != nil { 1081 | t.Fatal(err) 1082 | } 1083 | sz, err := node.Size() 1084 | if err != nil { 1085 | t.Fatal(err) 1086 | } 1087 | if sz != 182 { 1088 | t.Errorf("expected size to be 182, got %d", sz) 1089 | } 1090 | verifyUnsortedNode(t, node) 1091 | }) 1092 | 1093 | t.Run("GetPBNode() should not mutate, returned PBNode should be sorted", func(t *testing.T) { 1094 | node, err := DecodeProtobuf(byts) 1095 | if err != nil { 1096 | t.Fatal(err) 1097 | } 1098 | rtPBNode := node.GetPBNode() 1099 | rtByts, err := rtPBNode.Marshal() 1100 | if err != nil { 1101 | t.Fatal(err) 1102 | } 1103 | verifyUnsortedNode(t, node) 1104 | rtNode, err := DecodeProtobuf(rtByts) 1105 | if err != nil { 1106 | t.Fatal(err) 1107 | } 1108 | verifySortedNode(t, rtNode) 1109 | }) 1110 | 1111 | t.Run("add and remove link should mutate", func(t *testing.T) { 1112 | node, err := DecodeProtobuf(byts) 1113 | if err != nil { 1114 | t.Fatal(err) 1115 | } 1116 | if err = node.AddRawLink("foo", &ipld.Link{ 1117 | Size: 10, 1118 | Cid: someCid, 1119 | }); err != nil { 1120 | t.Fatal(err) 1121 | } 1122 | if err = node.RemoveNodeLink("foo"); err != nil { 1123 | t.Fatal(err) 1124 | } 1125 | verifySortedNode(t, node) 1126 | }) 1127 | 1128 | t.Run("update link should not mutate, returned ProtoNode should be sorted", func(t *testing.T) { 1129 | node, err := DecodeProtobuf(byts) 1130 | if err != nil { 1131 | t.Fatal(err) 1132 | } 1133 | newNode, err := node.UpdateNodeLink("self", node) 1134 | if err != nil { 1135 | t.Fatal(err) 1136 | } 1137 | if err = newNode.RemoveNodeLink("self"); err != nil { 1138 | t.Fatal(err) 1139 | } 1140 | verifySortedNode(t, newNode) 1141 | verifyUnsortedNode(t, node) 1142 | }) 1143 | 1144 | t.Run("SetLinks() should mutate", func(t *testing.T) { 1145 | node, err := DecodeProtobuf(byts) 1146 | if err != nil { 1147 | t.Fatal(err) 1148 | } 1149 | links := node.Links() // clone 1150 | node.SetLinks(links) 1151 | verifySortedNode(t, node) 1152 | }) 1153 | } 1154 | 1155 | func TestProgressIndicator(t *testing.T) { 1156 | testProgressIndicator(t, 5) 1157 | } 1158 | 1159 | func TestProgressIndicatorNoChildren(t *testing.T) { 1160 | testProgressIndicator(t, 0) 1161 | } 1162 | 1163 | func testProgressIndicator(t *testing.T, depth int) { 1164 | ds := dstest.Mock() 1165 | 1166 | top, numChildren := mkDag(ds, depth) 1167 | 1168 | v := new(ProgressTracker) 1169 | ctx := v.DeriveContext(context.Background()) 1170 | 1171 | err := FetchGraph(ctx, top, ds) 1172 | if err != nil { 1173 | t.Fatal(err) 1174 | } 1175 | 1176 | if v.Value() != numChildren+1 { 1177 | t.Errorf("wrong number of children reported in progress indicator, expected %d, got %d", 1178 | numChildren+1, v.Value()) 1179 | } 1180 | } 1181 | 1182 | func mkDag(ds ipld.DAGService, depth int) (cid.Cid, int) { 1183 | ctx := context.Background() 1184 | 1185 | totalChildren := 0 1186 | f := func() *ProtoNode { 1187 | p := new(ProtoNode) 1188 | buf := make([]byte, 16) 1189 | rand.Read(buf) 1190 | 1191 | p.SetData(buf) 1192 | err := ds.Add(ctx, p) 1193 | if err != nil { 1194 | panic(err) 1195 | } 1196 | return p 1197 | } 1198 | 1199 | for i := 0; i < depth; i++ { 1200 | thisf := f 1201 | f = func() *ProtoNode { 1202 | pn := mkNodeWithChildren(thisf, 10) 1203 | err := ds.Add(ctx, pn) 1204 | if err != nil { 1205 | panic(err) 1206 | } 1207 | totalChildren += 10 1208 | return pn 1209 | } 1210 | } 1211 | 1212 | nd := f() 1213 | err := ds.Add(ctx, nd) 1214 | if err != nil { 1215 | panic(err) 1216 | } 1217 | 1218 | return nd.Cid(), totalChildren 1219 | } 1220 | 1221 | func mkNodeWithChildren(getChild func() *ProtoNode, width int) *ProtoNode { 1222 | cur := new(ProtoNode) 1223 | 1224 | for i := 0; i < width; i++ { 1225 | c := getChild() 1226 | if err := cur.AddNodeLink(fmt.Sprint(i), c); err != nil { 1227 | panic(err) 1228 | } 1229 | } 1230 | 1231 | return cur 1232 | } 1233 | -------------------------------------------------------------------------------- /node.go: -------------------------------------------------------------------------------- 1 | package merkledag 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "math" 9 | "sort" 10 | 11 | blocks "github.com/ipfs/go-block-format" 12 | cid "github.com/ipfs/go-cid" 13 | format "github.com/ipfs/go-ipld-format" 14 | legacy "github.com/ipfs/go-ipld-legacy" 15 | logging "github.com/ipfs/go-log/v2" 16 | dagpb "github.com/ipld/go-codec-dagpb" 17 | ipld "github.com/ipld/go-ipld-prime" 18 | mh "github.com/multiformats/go-multihash" 19 | mhcore "github.com/multiformats/go-multihash/core" 20 | ) 21 | 22 | // Common errors 23 | var ( 24 | ErrNotProtobuf = fmt.Errorf("expected protobuf dag node") 25 | ErrNotRawNode = fmt.Errorf("expected raw bytes node") 26 | ErrLinkNotFound = fmt.Errorf("no link by that name") 27 | ) 28 | 29 | var log = logging.Logger("merkledag") 30 | 31 | // for testing custom CidBuilders 32 | var zeros [256]byte 33 | var zeroCid = mustZeroCid() 34 | 35 | type immutableProtoNode struct { 36 | encoded []byte 37 | dagpb.PBNode 38 | } 39 | 40 | // ProtoNode represents a node in the IPFS Merkle DAG. 41 | // nodes have opaque data and a set of navigable links. 42 | // ProtoNode is a go-ipld-legacy.UniversalNode, meaning it is both 43 | // a go-ipld-prime node and a go-ipld-format node. 44 | // ProtoNode maintains compatibility with it's original implementation 45 | // as a go-ipld-format only node, which included some mutability, namely the 46 | // the ability to add/remove links in place 47 | // 48 | // TODO: We should be able to eventually replace this implementation with 49 | // * go-codec-dagpb for basic DagPB encode/decode to go-ipld-prime 50 | // * go-unixfsnode ADLs for higher level DAGPB functionality 51 | // For the time being however, go-unixfsnode is read only and 52 | // this mutable protonode implementation is needed to support go-unixfs, 53 | // the only library that implements both read and write for UnixFS v1. 54 | type ProtoNode struct { 55 | links []*format.Link 56 | linksDirty bool 57 | data []byte 58 | 59 | // cache encoded/marshaled value, kept to make the go-ipld-prime Node interface 60 | // work (see prime.go), and to provide a cached []byte encoded form available 61 | encoded *immutableProtoNode 62 | cached cid.Cid 63 | 64 | // builder specifies cid version and hashing function 65 | builder cid.Builder 66 | } 67 | 68 | var v0CidPrefix = cid.Prefix{ 69 | Codec: cid.DagProtobuf, 70 | MhLength: -1, 71 | MhType: mh.SHA2_256, 72 | Version: 0, 73 | } 74 | 75 | var v1CidPrefix = cid.Prefix{ 76 | Codec: cid.DagProtobuf, 77 | MhLength: -1, 78 | MhType: mh.SHA2_256, 79 | Version: 1, 80 | } 81 | 82 | // V0CidPrefix returns a prefix for CIDv0 83 | func V0CidPrefix() cid.Prefix { return v0CidPrefix } 84 | 85 | // V1CidPrefix returns a prefix for CIDv1 with the default settings 86 | func V1CidPrefix() cid.Prefix { return v1CidPrefix } 87 | 88 | // PrefixForCidVersion returns the Protobuf prefix for a given CID version 89 | func PrefixForCidVersion(version int) (cid.Prefix, error) { 90 | switch version { 91 | case 0: 92 | return v0CidPrefix, nil 93 | case 1: 94 | return v1CidPrefix, nil 95 | default: 96 | return cid.Prefix{}, fmt.Errorf("unknown CID version: %d", version) 97 | } 98 | } 99 | 100 | // CidBuilder returns the CID Builder for this ProtoNode, it is never nil 101 | func (n *ProtoNode) CidBuilder() cid.Builder { 102 | if n.builder == nil { 103 | n.builder = v0CidPrefix 104 | } 105 | return n.builder 106 | } 107 | 108 | // SetCidBuilder sets the CID builder if it is non nil, if nil then it 109 | // is reset to the default value. An error will be returned if the builder 110 | // is not usable. 111 | func (n *ProtoNode) SetCidBuilder(builder cid.Builder) error { 112 | if builder == nil { 113 | n.builder = v0CidPrefix 114 | return nil 115 | } 116 | switch b := builder.(type) { 117 | case cid.Prefix: 118 | if err := checkHasher(b.MhType, b.MhLength); err != nil { 119 | return err 120 | } 121 | case *cid.Prefix: 122 | if err := checkHasher(b.MhType, b.MhLength); err != nil { 123 | return err 124 | } 125 | default: 126 | // We have to test it's a usable hasher by invoking it and checking it 127 | // doesn't error. This is only a basic check, there are still ways it may 128 | // break 129 | if _, err := builder.Sum(zeros[:]); err != nil { 130 | return err 131 | } 132 | } 133 | n.builder = builder.WithCodec(cid.DagProtobuf) 134 | n.cached = cid.Undef 135 | return nil 136 | } 137 | 138 | // check whether the hasher is likely to be a usable one 139 | func checkHasher(indicator uint64, sizeHint int) error { 140 | mhLen := sizeHint 141 | if mhLen <= 0 { 142 | mhLen = -1 143 | } 144 | _, err := mhcore.GetVariableHasher(indicator, mhLen) 145 | return err 146 | } 147 | 148 | // LinkSlice is a slice of format.Links 149 | type LinkSlice []*format.Link 150 | 151 | func (ls LinkSlice) Len() int { return len(ls) } 152 | func (ls LinkSlice) Swap(a, b int) { ls[a], ls[b] = ls[b], ls[a] } 153 | func (ls LinkSlice) Less(a, b int) bool { return ls[a].Name < ls[b].Name } 154 | 155 | // NodeWithData builds a new Protonode with the given data. 156 | func NodeWithData(d []byte) *ProtoNode { 157 | return &ProtoNode{data: d} 158 | } 159 | 160 | // AddNodeLink adds a link to another node. The link will be added in 161 | // sorted order. 162 | // 163 | // If sorting has not already been applied to this node (because 164 | // it was deserialized from a form that did not have sorted links), the links 165 | // list will be sorted. If a ProtoNode was deserialized from a badly encoded 166 | // form that did not already have its links sorted, calling AddNodeLink and then 167 | // RemoveNodeLink for the same link, will not result in an identically encoded 168 | // form as the links will have been sorted. 169 | func (n *ProtoNode) AddNodeLink(name string, that format.Node) error { 170 | lnk, err := format.MakeLink(that) 171 | if err != nil { 172 | return err 173 | } 174 | 175 | lnk.Name = name 176 | 177 | n.AddRawLink(name, lnk) 178 | 179 | return nil 180 | } 181 | 182 | // AddRawLink adds a copy of a link to this node. The link will be added in 183 | // sorted order. 184 | // 185 | // If sorting has not already been applied to this node (because 186 | // it was deserialized from a form that did not have sorted links), the links 187 | // list will be sorted. If a ProtoNode was deserialized from a badly encoded 188 | // form that did not already have its links sorted, calling AddRawLink and then 189 | // RemoveNodeLink for the same link, will not result in an identically encoded 190 | // form as the links will have been sorted. 191 | func (n *ProtoNode) AddRawLink(name string, l *format.Link) error { 192 | lnk := &format.Link{ 193 | Name: name, 194 | Size: l.Size, 195 | Cid: l.Cid, 196 | } 197 | if err := checkLink(lnk); err != nil { 198 | return err 199 | } 200 | n.links = append(n.links, lnk) 201 | n.linksDirty = true // needs a sort 202 | n.encoded = nil 203 | return nil 204 | } 205 | 206 | // RemoveNodeLink removes a link on this node by the given name. If there are 207 | // no links with this name, ErrLinkNotFound will be returned. If there are more 208 | // than one link with this name, they will all be removed. 209 | func (n *ProtoNode) RemoveNodeLink(name string) error { 210 | ref := n.links[:0] 211 | found := false 212 | 213 | for _, v := range n.links { 214 | if v.Name != name { 215 | ref = append(ref, v) 216 | } else { 217 | found = true 218 | } 219 | } 220 | 221 | if !found { 222 | return ErrLinkNotFound 223 | } 224 | 225 | n.links = ref 226 | // Even though a removal won't change sorting, this node may have come from 227 | // a deserialized state with badly sorted links. Now that we are mutating, 228 | // we need to ensure the resulting link list is sorted when it gets consumed. 229 | n.linksDirty = true 230 | n.encoded = nil 231 | 232 | return nil 233 | } 234 | 235 | // GetNodeLink returns a copy of the link with the given name. 236 | func (n *ProtoNode) GetNodeLink(name string) (*format.Link, error) { 237 | for _, l := range n.links { 238 | if l.Name == name { 239 | return &format.Link{ 240 | Name: l.Name, 241 | Size: l.Size, 242 | Cid: l.Cid, 243 | }, nil 244 | } 245 | } 246 | return nil, ErrLinkNotFound 247 | } 248 | 249 | // GetLinkedProtoNode returns a copy of the ProtoNode with the given name. 250 | func (n *ProtoNode) GetLinkedProtoNode(ctx context.Context, ds format.DAGService, name string) (*ProtoNode, error) { 251 | nd, err := n.GetLinkedNode(ctx, ds, name) 252 | if err != nil { 253 | return nil, err 254 | } 255 | 256 | pbnd, ok := nd.(*ProtoNode) 257 | if !ok { 258 | return nil, ErrNotProtobuf 259 | } 260 | 261 | return pbnd, nil 262 | } 263 | 264 | // GetLinkedNode returns a copy of the IPLD Node with the given name. 265 | func (n *ProtoNode) GetLinkedNode(ctx context.Context, ds format.DAGService, name string) (format.Node, error) { 266 | lnk, err := n.GetNodeLink(name) 267 | if err != nil { 268 | return nil, err 269 | } 270 | 271 | return lnk.GetNode(ctx, ds) 272 | } 273 | 274 | // Copy returns a copy of the node. The resulting node will have a properly 275 | // sorted Links list regardless of whether the original came from a badly 276 | // serialized form that didn't have a sorted list. 277 | // NOTE: This does not make copies of Node objects in the links. 278 | func (n *ProtoNode) Copy() format.Node { 279 | nnode := new(ProtoNode) 280 | if len(n.data) > 0 { 281 | nnode.data = make([]byte, len(n.data)) 282 | copy(nnode.data, n.data) 283 | } 284 | 285 | if len(n.links) > 0 { 286 | nnode.links = append([]*format.Link(nil), n.links...) 287 | // Sort links regardless of linksDirty state, this may have come from a 288 | // serialized form that had badly sorted links, in which case linksDirty 289 | // will not be true. 290 | sort.Stable(LinkSlice(nnode.links)) 291 | } 292 | 293 | nnode.builder = n.builder 294 | 295 | return nnode 296 | } 297 | 298 | // RawData returns the encoded byte form of this node. 299 | // 300 | // Note that this method may return an empty byte slice if there is an error 301 | // performing the encode. To check whether such an error may have occurred, use 302 | // node.EncodeProtobuf(false), instead (or prior to calling RawData) and check 303 | // for its returned error value; the result of EncodeProtobuf is cached so there 304 | // is minimal overhead when invoking both methods. 305 | func (n *ProtoNode) RawData() []byte { 306 | out, err := n.EncodeProtobuf(false) 307 | if err != nil { 308 | log.Errorf("failed to encode dag-pb block: %s", err.Error()) 309 | return nil 310 | } 311 | return out 312 | } 313 | 314 | // Data returns the data stored by this node. 315 | func (n *ProtoNode) Data() []byte { 316 | return n.data 317 | } 318 | 319 | // SetData stores data in this nodes. 320 | func (n *ProtoNode) SetData(d []byte) { 321 | n.encoded = nil 322 | n.cached = cid.Undef 323 | n.data = d 324 | } 325 | 326 | // UpdateNodeLink return a copy of the node with the link name set to point to 327 | // that. The link will be added in sorted order. If a link of the same name 328 | // existed, it is removed. 329 | // 330 | // If sorting has not already been applied to this node (because 331 | // it was deserialized from a form that did not have sorted links), the links 332 | // list will be sorted in the returned copy. 333 | func (n *ProtoNode) UpdateNodeLink(name string, that *ProtoNode) (*ProtoNode, error) { 334 | newnode := n.Copy().(*ProtoNode) 335 | _ = newnode.RemoveNodeLink(name) // ignore error 336 | err := newnode.AddNodeLink(name, that) 337 | return newnode, err 338 | } 339 | 340 | // Size returns the total size of the data addressed by node, 341 | // including the total sizes of references. 342 | func (n *ProtoNode) Size() (uint64, error) { 343 | b, err := n.EncodeProtobuf(false) 344 | if err != nil { 345 | return 0, err 346 | } 347 | 348 | s := uint64(len(b)) 349 | for _, l := range n.links { 350 | s += l.Size 351 | } 352 | return s, nil 353 | } 354 | 355 | // Stat returns statistics on the node. 356 | func (n *ProtoNode) Stat() (*format.NodeStat, error) { 357 | enc, err := n.EncodeProtobuf(false) 358 | if err != nil { 359 | return nil, err 360 | } 361 | 362 | cumSize, err := n.Size() 363 | if err != nil { 364 | return nil, err 365 | } 366 | 367 | return &format.NodeStat{ 368 | Hash: n.Cid().String(), 369 | NumLinks: len(n.links), 370 | BlockSize: len(enc), 371 | LinksSize: len(enc) - len(n.data), // includes framing. 372 | DataSize: len(n.data), 373 | CumulativeSize: int(cumSize), 374 | }, nil 375 | } 376 | 377 | // Loggable implements the ipfs/go-log.Loggable interface. 378 | func (n *ProtoNode) Loggable() map[string]interface{} { 379 | return map[string]interface{}{ 380 | "node": n.String(), 381 | } 382 | } 383 | 384 | // UnmarshalJSON reads the node fields from a JSON-encoded byte slice. 385 | func (n *ProtoNode) UnmarshalJSON(b []byte) error { 386 | s := struct { 387 | Data []byte `json:"data"` 388 | Links []*format.Link `json:"links"` 389 | }{} 390 | 391 | err := json.Unmarshal(b, &s) 392 | if err != nil { 393 | return err 394 | } 395 | 396 | n.data = s.Data 397 | // Links may not be sorted after deserialization, but we don't change 398 | // them until we mutate this node since we're representing the current, 399 | // as-serialized state. So n.linksDirty is not set here. 400 | n.links = s.Links 401 | for _, lnk := range s.Links { 402 | if err := checkLink(lnk); err != nil { 403 | return err 404 | } 405 | } 406 | 407 | n.encoded = nil 408 | return nil 409 | } 410 | 411 | func checkLink(lnk *format.Link) error { 412 | if lnk.Size > math.MaxInt64 { 413 | return fmt.Errorf("value of Tsize is too large: %d", lnk.Size) 414 | } 415 | if !lnk.Cid.Defined() { 416 | return errors.New("link must have a value Cid value") 417 | } 418 | return nil 419 | } 420 | 421 | // MarshalJSON returns a JSON representation of the node. 422 | func (n *ProtoNode) MarshalJSON() ([]byte, error) { 423 | if n.linksDirty { 424 | // there was a mutation involving links, make sure we sort 425 | sort.Stable(LinkSlice(n.links)) 426 | n.linksDirty = false 427 | n.encoded = nil 428 | } 429 | 430 | out := map[string]interface{}{ 431 | "data": n.data, 432 | "links": n.links, 433 | } 434 | 435 | return json.Marshal(out) 436 | } 437 | 438 | // Cid returns the node's Cid, calculated according to its prefix 439 | // and raw data contents. 440 | // 441 | // Note that this method may return a CID representing a zero-length byte slice 442 | // if there is an error performing the encode. To check whether such an error 443 | // may have occurred, use node.EncodeProtobuf(false), instead (or prior to 444 | // calling RawData) and check for its returned error value; the result of 445 | // EncodeProtobuf is cached so there is minimal overhead when invoking both 446 | // methods. 447 | func (n *ProtoNode) Cid() cid.Cid { 448 | // re-encode if necessary and we'll get a new cached CID 449 | if _, err := n.EncodeProtobuf(false); err != nil { 450 | log.Errorf("failed to encode dag-pb block: %s", err.Error()) 451 | // error, return a zero-CID 452 | c, err := n.CidBuilder().Sum([]byte{}) 453 | if err != nil { 454 | // CidBuilder was a source of error, return _the_ dag-pb zero CIDv1 455 | return zeroCid 456 | } 457 | return c 458 | } 459 | return n.cached 460 | } 461 | 462 | // String prints the node's Cid. 463 | // 464 | // Note that this method may return a CID representing a zero-length byte slice 465 | // if there is an error performing the encode. To check whether such an error 466 | // may have occurred, use node.EncodeProtobuf(false), instead (or prior to 467 | // calling RawData) and check for its returned error value; the result of 468 | // EncodeProtobuf is cached so there is minimal overhead when invoking both 469 | // methods. 470 | func (n *ProtoNode) String() string { 471 | return n.Cid().String() 472 | } 473 | 474 | // Multihash hashes the encoded data of this node. 475 | // 476 | // Note that this method may return a multihash representing a zero-length byte 477 | // slice if there is an error performing the encode. To check whether such an 478 | // error may have occurred, use node.EncodeProtobuf(false), instead (or prior to 479 | // calling RawData) and check for its returned error value; the result of 480 | // EncodeProtobuf is cached so there is minimal overhead when invoking both 481 | // methods. 482 | func (n *ProtoNode) Multihash() mh.Multihash { 483 | return n.Cid().Hash() 484 | } 485 | 486 | // Links returns a copy of the node's links. 487 | func (n *ProtoNode) Links() []*format.Link { 488 | if n.linksDirty { 489 | // there was a mutation involving links, make sure we sort 490 | sort.Stable(LinkSlice(n.links)) 491 | n.linksDirty = false 492 | n.encoded = nil 493 | } 494 | return append([]*format.Link(nil), n.links...) 495 | } 496 | 497 | // SetLinks replaces the node links with a copy of the provided links. Sorting 498 | // will be applied to the list. 499 | func (n *ProtoNode) SetLinks(links []*format.Link) error { 500 | for _, lnk := range links { 501 | if err := checkLink(lnk); err != nil { 502 | return err 503 | } 504 | } 505 | n.links = append([]*format.Link(nil), links...) 506 | n.linksDirty = true // needs a sort 507 | n.encoded = nil 508 | return nil 509 | } 510 | 511 | // Resolve is an alias for ResolveLink. 512 | func (n *ProtoNode) Resolve(path []string) (interface{}, []string, error) { 513 | return n.ResolveLink(path) 514 | } 515 | 516 | // ResolveLink consumes the first element of the path and obtains the link 517 | // corresponding to it from the node. It returns the link 518 | // and the path without the consumed element. 519 | func (n *ProtoNode) ResolveLink(path []string) (*format.Link, []string, error) { 520 | if len(path) == 0 { 521 | return nil, nil, fmt.Errorf("end of path, no more links to resolve") 522 | } 523 | 524 | lnk, err := n.GetNodeLink(path[0]) 525 | if err != nil { 526 | return nil, nil, err 527 | } 528 | 529 | return lnk, path[1:], nil 530 | } 531 | 532 | // Tree returns the link names of the ProtoNode. 533 | // ProtoNodes are only ever one path deep, so anything different than an empty 534 | // string for p results in nothing. The depth parameter is ignored. 535 | func (n *ProtoNode) Tree(p string, depth int) []string { 536 | if p != "" { 537 | return nil 538 | } 539 | 540 | if n.linksDirty { 541 | // there was a mutation involving links, make sure we sort 542 | sort.Stable(LinkSlice(n.links)) 543 | n.linksDirty = false 544 | n.encoded = nil 545 | } 546 | 547 | out := make([]string, 0, len(n.links)) 548 | for _, lnk := range n.links { 549 | out = append(out, lnk.Name) 550 | } 551 | return out 552 | } 553 | 554 | func ProtoNodeConverter(b blocks.Block, nd ipld.Node) (legacy.UniversalNode, error) { 555 | pbNode, ok := nd.(dagpb.PBNode) 556 | if !ok { 557 | return nil, ErrNotProtobuf 558 | } 559 | encoded := &immutableProtoNode{b.RawData(), pbNode} 560 | pn := fromImmutableNode(encoded) 561 | pn.cached = b.Cid() 562 | pn.builder = b.Cid().Prefix() 563 | return pn, nil 564 | } 565 | 566 | // TODO: replace with cid.MustParse() when we bump go-cid 567 | func mustZeroCid() cid.Cid { 568 | c, err := cid.Parse("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") 569 | if err != nil { 570 | panic(err) 571 | } 572 | return c 573 | } 574 | 575 | var _ legacy.UniversalNode = &ProtoNode{} 576 | -------------------------------------------------------------------------------- /node_test.go: -------------------------------------------------------------------------------- 1 | package merkledag_test 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | . "github.com/ipfs/go-merkledag" 9 | mdtest "github.com/ipfs/go-merkledag/test" 10 | 11 | cid "github.com/ipfs/go-cid" 12 | ipld "github.com/ipfs/go-ipld-format" 13 | ) 14 | 15 | var sampleCid cid.Cid 16 | 17 | func init() { 18 | var err error 19 | // make a test CID -- doesn't matter just to add as a link 20 | sampleCid, err = cid.Cast([]byte{1, 85, 0, 5, 0, 1, 2, 3, 4}) 21 | if err != nil { 22 | panic(err) 23 | } 24 | } 25 | 26 | func TestStableCID(t *testing.T) { 27 | nd := &ProtoNode{} 28 | nd.SetData([]byte("foobar")) 29 | nd.SetLinks([]*ipld.Link{ 30 | {Name: "a", Cid: sampleCid}, 31 | {Name: "b", Cid: sampleCid}, 32 | {Name: "c", Cid: sampleCid}, 33 | }) 34 | expected, err := cid.Decode("QmciCHWD9Q47VPX6naY3XsPZGnqVqbedAniGCcaHjBaCri") 35 | if err != nil { 36 | t.Fatal(err) 37 | } 38 | if !nd.Cid().Equals(expected) { 39 | t.Fatalf("Got CID %s, expected CID %s", nd.Cid(), expected) 40 | } 41 | } 42 | 43 | func TestRemoveLink(t *testing.T) { 44 | nd := &ProtoNode{} 45 | nd.SetLinks([]*ipld.Link{ 46 | {Name: "a", Cid: sampleCid}, 47 | {Name: "b", Cid: sampleCid}, 48 | {Name: "a", Cid: sampleCid}, 49 | {Name: "a", Cid: sampleCid}, 50 | {Name: "c", Cid: sampleCid}, 51 | {Name: "a", Cid: sampleCid}, 52 | }) 53 | 54 | err := nd.RemoveNodeLink("a") 55 | if err != nil { 56 | t.Fatal(err) 57 | } 58 | 59 | if len(nd.Links()) != 2 { 60 | t.Fatal("number of links incorrect") 61 | } 62 | 63 | if nd.Links()[0].Name != "b" { 64 | t.Fatal("link order wrong") 65 | } 66 | 67 | if nd.Links()[1].Name != "c" { 68 | t.Fatal("link order wrong") 69 | } 70 | 71 | // should fail 72 | err = nd.RemoveNodeLink("a") 73 | if err != ErrLinkNotFound { 74 | t.Fatal("should have failed to remove link") 75 | } 76 | 77 | // ensure nothing else got touched 78 | if len(nd.Links()) != 2 { 79 | t.Fatal("number of links incorrect") 80 | } 81 | 82 | if nd.Links()[0].Name != "b" { 83 | t.Fatal("link order wrong") 84 | } 85 | 86 | if nd.Links()[1].Name != "c" { 87 | t.Fatal("link order wrong") 88 | } 89 | } 90 | 91 | func TestFindLink(t *testing.T) { 92 | ctx := context.Background() 93 | 94 | ds := mdtest.Mock() 95 | ndEmpty := new(ProtoNode) 96 | err := ds.Add(ctx, ndEmpty) 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | 101 | kEmpty := ndEmpty.Cid() 102 | 103 | nd := &ProtoNode{} 104 | nd.SetLinks([]*ipld.Link{ 105 | {Name: "a", Cid: kEmpty}, 106 | {Name: "c", Cid: kEmpty}, 107 | {Name: "b", Cid: kEmpty}, 108 | }) 109 | 110 | err = ds.Add(ctx, nd) 111 | if err != nil { 112 | t.Fatal(err) 113 | } 114 | 115 | lnk, err := nd.GetNodeLink("b") 116 | if err != nil { 117 | t.Fatal(err) 118 | } 119 | 120 | if lnk.Name != "b" { 121 | t.Fatal("got wrong link back") 122 | } 123 | 124 | _, err = nd.GetNodeLink("f") 125 | if err != ErrLinkNotFound { 126 | t.Fatal("shouldnt have found link") 127 | } 128 | 129 | _, err = nd.GetLinkedNode(context.Background(), ds, "b") 130 | if err != nil { 131 | t.Fatal(err) 132 | } 133 | 134 | outnd, err := nd.UpdateNodeLink("b", nd) 135 | if err != nil { 136 | t.Fatal(err) 137 | } 138 | 139 | olnk, err := outnd.GetNodeLink("b") 140 | if err != nil { 141 | t.Fatal(err) 142 | } 143 | 144 | if olnk.Cid.String() == kEmpty.String() { 145 | t.Fatal("new link should have different hash") 146 | } 147 | } 148 | 149 | func TestNodeCopy(t *testing.T) { 150 | nd := &ProtoNode{} 151 | nd.SetLinks([]*ipld.Link{ 152 | {Name: "a", Cid: sampleCid}, 153 | {Name: "c", Cid: sampleCid}, 154 | {Name: "b", Cid: sampleCid}, 155 | }) 156 | 157 | nd.SetData([]byte("testing")) 158 | 159 | ond := nd.Copy().(*ProtoNode) 160 | ond.SetData(nil) 161 | 162 | if nd.Data() == nil { 163 | t.Fatal("should be different objects") 164 | } 165 | } 166 | 167 | func TestJsonRoundtrip(t *testing.T) { 168 | nd := new(ProtoNode) 169 | nd.SetLinks([]*ipld.Link{ 170 | {Name: "a", Cid: sampleCid}, 171 | {Name: "c", Cid: sampleCid}, 172 | {Name: "b", Cid: sampleCid}, 173 | }) 174 | nd.SetData([]byte("testing")) 175 | 176 | jb, err := nd.MarshalJSON() 177 | if err != nil { 178 | t.Fatal(err) 179 | } 180 | 181 | nn := new(ProtoNode) 182 | err = nn.UnmarshalJSON(jb) 183 | if err != nil { 184 | t.Fatal(err) 185 | } 186 | 187 | if !bytes.Equal(nn.Data(), nd.Data()) { 188 | t.Fatal("data wasnt the same") 189 | } 190 | 191 | if !nn.Cid().Equals(nd.Cid()) { 192 | t.Fatal("objects differed after marshaling") 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /pb/compat_test.go: -------------------------------------------------------------------------------- 1 | package merkledag_pb 2 | 3 | // mirrored in JavaScript @ https://github.com/ipld/js-dag-pb/blob/master/test/test-compat.js 4 | 5 | import ( 6 | "encoding/hex" 7 | "encoding/json" 8 | "testing" 9 | ) 10 | 11 | var dataZero []byte = make([]byte, 0) 12 | var dataSome []byte = []byte{0, 1, 2, 3, 4} 13 | var cidBytes []byte = []byte{1, 85, 0, 5, 0, 1, 2, 3, 4} 14 | var zeroName string = "" 15 | var someName string = "some name" 16 | var zeroTsize uint64 = 0 17 | var someTsize uint64 = 1010 18 | var largeTsize uint64 = 9007199254740991 // JavaScript Number.MAX_SAFE_INTEGER 19 | 20 | type testCase struct { 21 | name string 22 | node *PBNode 23 | expectedBytes string 24 | expectedForm string 25 | } 26 | 27 | var testCases = []testCase{ 28 | { 29 | name: "empty", 30 | node: &PBNode{}, 31 | expectedBytes: "", 32 | expectedForm: "{}", 33 | }, 34 | { 35 | name: "Data zero", 36 | node: &PBNode{Data: dataZero}, 37 | expectedBytes: "0a00", 38 | expectedForm: `{ 39 | "Data": "" 40 | }`, 41 | }, 42 | { 43 | name: "Data some", 44 | node: &PBNode{Data: dataSome}, 45 | expectedBytes: "0a050001020304", 46 | expectedForm: `{ 47 | "Data": "0001020304" 48 | }`, 49 | }, 50 | { 51 | name: "Links zero", 52 | node: &PBNode{Links: make([]*PBLink, 0)}, 53 | expectedBytes: "", 54 | expectedForm: "{}", 55 | }, 56 | { 57 | name: "Data some Links zero", 58 | node: &PBNode{Data: dataSome, Links: make([]*PBLink, 0)}, 59 | expectedBytes: "0a050001020304", 60 | expectedForm: `{ 61 | "Data": "0001020304" 62 | }`, 63 | }, 64 | { 65 | name: "Links empty", 66 | node: &PBNode{Links: []*PBLink{{}}}, 67 | expectedBytes: "1200", 68 | expectedForm: `{ 69 | "Links": [ 70 | {} 71 | ] 72 | }`, 73 | }, 74 | { 75 | name: "Data some Links empty", 76 | node: &PBNode{Data: dataSome, Links: []*PBLink{{}}}, 77 | expectedBytes: "12000a050001020304", 78 | expectedForm: `{ 79 | "Data": "0001020304", 80 | "Links": [ 81 | {} 82 | ] 83 | }`, 84 | }, 85 | { 86 | name: "Links Hash zero", 87 | node: &PBNode{Links: []*PBLink{{Hash: dataZero}}}, 88 | expectedBytes: "12020a00", 89 | expectedForm: `{ 90 | "Links": [ 91 | { 92 | "Hash": "" 93 | } 94 | ] 95 | }`, 96 | }, 97 | { 98 | name: "Links Hash some", 99 | node: &PBNode{Links: []*PBLink{{Hash: cidBytes}}}, 100 | expectedBytes: "120b0a09015500050001020304", 101 | expectedForm: `{ 102 | "Links": [ 103 | { 104 | "Hash": "015500050001020304" 105 | } 106 | ] 107 | }`, 108 | }, 109 | { 110 | name: "Links Name zero", 111 | node: &PBNode{Links: []*PBLink{{Name: &zeroName}}}, 112 | expectedBytes: "12021200", 113 | expectedForm: `{ 114 | "Links": [ 115 | { 116 | "Name": "" 117 | } 118 | ] 119 | }`, 120 | }, 121 | { 122 | name: "Links Hash some Name zero", 123 | node: &PBNode{Links: []*PBLink{{Hash: cidBytes, Name: &zeroName}}}, 124 | expectedBytes: "120d0a090155000500010203041200", 125 | expectedForm: `{ 126 | "Links": [ 127 | { 128 | "Hash": "015500050001020304", 129 | "Name": "" 130 | } 131 | ] 132 | }`, 133 | }, 134 | { 135 | name: "Links Name some", 136 | node: &PBNode{Links: []*PBLink{{Name: &someName}}}, 137 | expectedBytes: "120b1209736f6d65206e616d65", 138 | expectedForm: `{ 139 | "Links": [ 140 | { 141 | "Name": "some name" 142 | } 143 | ] 144 | }`, 145 | }, 146 | { 147 | name: "Links Hash some Name some", 148 | node: &PBNode{Links: []*PBLink{{Hash: cidBytes, Name: &someName}}}, 149 | expectedBytes: "12160a090155000500010203041209736f6d65206e616d65", 150 | expectedForm: `{ 151 | "Links": [ 152 | { 153 | "Hash": "015500050001020304", 154 | "Name": "some name" 155 | } 156 | ] 157 | }`, 158 | }, 159 | { 160 | name: "Links Tsize zero", 161 | node: &PBNode{Links: []*PBLink{{Tsize: &zeroTsize}}}, 162 | expectedBytes: "12021800", 163 | expectedForm: `{ 164 | "Links": [ 165 | { 166 | "Tsize": 0 167 | } 168 | ] 169 | }`, 170 | }, 171 | { 172 | name: "Links Hash some Tsize zero", 173 | node: &PBNode{Links: []*PBLink{{Hash: cidBytes, Tsize: &zeroTsize}}}, 174 | expectedBytes: "120d0a090155000500010203041800", 175 | expectedForm: `{ 176 | "Links": [ 177 | { 178 | "Hash": "015500050001020304", 179 | "Tsize": 0 180 | } 181 | ] 182 | }`, 183 | }, 184 | { 185 | name: "Links Tsize some", 186 | node: &PBNode{Links: []*PBLink{{Tsize: &someTsize}}}, 187 | expectedBytes: "120318f207", 188 | expectedForm: `{ 189 | "Links": [ 190 | { 191 | "Tsize": 1010 192 | } 193 | ] 194 | }`, 195 | }, 196 | { 197 | name: "Links Hash some Tsize some", 198 | node: &PBNode{Links: []*PBLink{{Hash: cidBytes, Tsize: &largeTsize}}}, 199 | expectedBytes: "12140a0901550005000102030418ffffffffffffff0f", 200 | expectedForm: `{ 201 | "Links": [ 202 | { 203 | "Hash": "015500050001020304", 204 | "Tsize": 9007199254740991 205 | } 206 | ] 207 | }`, 208 | }, 209 | } 210 | 211 | func TestCompat(t *testing.T) { 212 | for _, tc := range testCases { 213 | t.Run(tc.name, func(t *testing.T) { 214 | verifyRoundTrip(t, tc) 215 | }) 216 | } 217 | } 218 | 219 | func verifyRoundTrip(t *testing.T, tc testCase) { 220 | actualBytes, actualForm, err := nodeRoundTripToString(t, tc.node) 221 | if err != nil { 222 | t.Fatal(err) 223 | } 224 | 225 | if actualBytes != tc.expectedBytes { 226 | t.Logf( 227 | "Expected bytes: [%v]\nGot: [%v]\n", 228 | tc.expectedBytes, 229 | actualBytes) 230 | t.Error("Did not match") 231 | } 232 | 233 | if actualForm != tc.expectedForm { 234 | t.Logf( 235 | "Expected form: [%v]\nGot: [%v]\n", 236 | tc.expectedForm, 237 | actualForm) 238 | t.Error("Did not match") 239 | } 240 | } 241 | 242 | func nodeRoundTripToString(t *testing.T, n *PBNode) (string, string, error) { 243 | bytes, err := n.Marshal() 244 | if err != nil { 245 | return "", "", err 246 | } 247 | t.Logf("[%v]\n", hex.EncodeToString(bytes)) 248 | rt := new(PBNode) 249 | if err := rt.Unmarshal(bytes); err != nil { 250 | return "", "", err 251 | } 252 | str, err := json.MarshalIndent(cleanPBNode(t, rt), "", "\t") 253 | if err != nil { 254 | return "", "", err 255 | } 256 | return hex.EncodeToString(bytes), string(str), nil 257 | } 258 | 259 | // convert a PBLink into a map for clean JSON marshalling 260 | func cleanPBLink(t *testing.T, link *PBLink) map[string]interface{} { 261 | if link == nil { 262 | return nil 263 | } 264 | // this would be a bad pb decode 265 | if link.XXX_unrecognized != nil { 266 | t.Fatal("Got unexpected XXX_unrecognized") 267 | } 268 | nl := make(map[string]interface{}) 269 | if link.Hash != nil { 270 | nl["Hash"] = hex.EncodeToString(link.Hash) 271 | } 272 | if link.Name != nil { 273 | nl["Name"] = link.Name 274 | } 275 | if link.Tsize != nil { 276 | nl["Tsize"] = link.Tsize 277 | } 278 | return nl 279 | } 280 | 281 | // convert a PBNode into a map for clean JSON marshalling 282 | func cleanPBNode(t *testing.T, node *PBNode) map[string]interface{} { 283 | // this would be a bad pb decode 284 | if node.XXX_unrecognized != nil { 285 | t.Fatal("Got unexpected XXX_unrecognized") 286 | } 287 | nn := make(map[string]interface{}) 288 | if node.Data != nil { 289 | nn["Data"] = hex.EncodeToString(node.Data) 290 | } 291 | if node.Links != nil { 292 | links := make([]map[string]interface{}, len(node.Links)) 293 | for i, l := range node.Links { 294 | links[i] = cleanPBLink(t, l) 295 | } 296 | nn["Links"] = links 297 | } 298 | return nn 299 | } 300 | -------------------------------------------------------------------------------- /pb/merkledag.pb.go: -------------------------------------------------------------------------------- 1 | // Code originally generated by protoc-gen-gogo from merkledag.proto, 2 | // now manually managed 3 | 4 | package merkledag_pb 5 | 6 | import ( 7 | bytes "bytes" 8 | fmt "fmt" 9 | io "io" 10 | math "math" 11 | math_bits "math/bits" 12 | reflect "reflect" 13 | strings "strings" 14 | 15 | _ "github.com/gogo/protobuf/gogoproto" 16 | proto "github.com/gogo/protobuf/proto" 17 | ) 18 | 19 | // DoNotUpgradeFileEverItWillChangeYourHashes warns users about not breaking 20 | // their file hashes. 21 | const DoNotUpgradeFileEverItWillChangeYourHashes = ` 22 | This file does not produce canonical protobufs. Unfortunately, if we change it, 23 | we'll change the hashes of the files we produce. 24 | 25 | Do *not regenerate this file. 26 | ` 27 | 28 | // Reference imports to suppress errors if they are not otherwise used. 29 | var _ = proto.Marshal 30 | var _ = fmt.Errorf 31 | var _ = math.Inf 32 | 33 | // This is a compile-time assertion to ensure that this generated file 34 | // is compatible with the proto package it is being compiled against. 35 | // A compilation error at this line likely means your copy of the 36 | // proto package needs to be updated. 37 | const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package 38 | 39 | // An IPFS MerkleDAG Link 40 | type PBLink struct { 41 | // multihash of the target object 42 | Hash []byte `protobuf:"bytes,1,opt,name=Hash" json:"Hash,omitempty"` 43 | // utf string name. should be unique per object 44 | Name *string `protobuf:"bytes,2,opt,name=Name" json:"Name,omitempty"` 45 | // cumulative size of target object 46 | Tsize *uint64 `protobuf:"varint,3,opt,name=Tsize" json:"Tsize,omitempty"` 47 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 48 | XXX_unrecognized []byte `json:"-"` 49 | XXX_sizecache int32 `json:"-"` 50 | } 51 | 52 | func (m *PBLink) Reset() { *m = PBLink{} } 53 | func (*PBLink) ProtoMessage() {} 54 | func (*PBLink) Descriptor() ([]byte, []int) { 55 | return fileDescriptor_10837cc3557cec00, []int{0} 56 | } 57 | func (m *PBLink) XXX_Unmarshal(b []byte) error { 58 | return m.Unmarshal(b) 59 | } 60 | func (m *PBLink) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 61 | if deterministic { 62 | return xxx_messageInfo_PBLink.Marshal(b, m, deterministic) 63 | } else { 64 | b = b[:cap(b)] 65 | n, err := m.MarshalToSizedBuffer(b) 66 | if err != nil { 67 | return nil, err 68 | } 69 | return b[:n], nil 70 | } 71 | } 72 | func (m *PBLink) XXX_Merge(src proto.Message) { 73 | xxx_messageInfo_PBLink.Merge(m, src) 74 | } 75 | func (m *PBLink) XXX_Size() int { 76 | return m.Size() 77 | } 78 | func (m *PBLink) XXX_DiscardUnknown() { 79 | xxx_messageInfo_PBLink.DiscardUnknown(m) 80 | } 81 | 82 | var xxx_messageInfo_PBLink proto.InternalMessageInfo 83 | 84 | func (m *PBLink) GetHash() []byte { 85 | if m != nil { 86 | return m.Hash 87 | } 88 | return nil 89 | } 90 | 91 | func (m *PBLink) GetName() string { 92 | if m != nil && m.Name != nil { 93 | return *m.Name 94 | } 95 | return "" 96 | } 97 | 98 | func (m *PBLink) GetTsize() uint64 { 99 | if m != nil && m.Tsize != nil { 100 | return *m.Tsize 101 | } 102 | return 0 103 | } 104 | 105 | // An IPFS MerkleDAG Node 106 | type PBNode struct { 107 | // refs to other objects 108 | Links []*PBLink `protobuf:"bytes,2,rep,name=Links" json:"Links,omitempty"` 109 | // opaque user data 110 | Data []byte `protobuf:"bytes,1,opt,name=Data" json:"Data,omitempty"` 111 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 112 | XXX_unrecognized []byte `json:"-"` 113 | XXX_sizecache int32 `json:"-"` 114 | } 115 | 116 | func (m *PBNode) Reset() { *m = PBNode{} } 117 | func (*PBNode) ProtoMessage() {} 118 | func (*PBNode) Descriptor() ([]byte, []int) { 119 | return fileDescriptor_10837cc3557cec00, []int{1} 120 | } 121 | func (m *PBNode) XXX_Unmarshal(b []byte) error { 122 | return m.Unmarshal(b) 123 | } 124 | func (m *PBNode) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 125 | if deterministic { 126 | return xxx_messageInfo_PBNode.Marshal(b, m, deterministic) 127 | } else { 128 | b = b[:cap(b)] 129 | n, err := m.MarshalToSizedBuffer(b) 130 | if err != nil { 131 | return nil, err 132 | } 133 | return b[:n], nil 134 | } 135 | } 136 | func (m *PBNode) XXX_Merge(src proto.Message) { 137 | xxx_messageInfo_PBNode.Merge(m, src) 138 | } 139 | func (m *PBNode) XXX_Size() int { 140 | return m.Size() 141 | } 142 | func (m *PBNode) XXX_DiscardUnknown() { 143 | xxx_messageInfo_PBNode.DiscardUnknown(m) 144 | } 145 | 146 | var xxx_messageInfo_PBNode proto.InternalMessageInfo 147 | 148 | func (m *PBNode) GetLinks() []*PBLink { 149 | if m != nil { 150 | return m.Links 151 | } 152 | return nil 153 | } 154 | 155 | func (m *PBNode) GetData() []byte { 156 | if m != nil { 157 | return m.Data 158 | } 159 | return nil 160 | } 161 | 162 | func init() { 163 | proto.RegisterType((*PBLink)(nil), "merkledag.pb.PBLink") 164 | proto.RegisterType((*PBNode)(nil), "merkledag.pb.PBNode") 165 | } 166 | 167 | func init() { proto.RegisterFile("merkledag.proto", fileDescriptor_10837cc3557cec00) } 168 | 169 | var fileDescriptor_10837cc3557cec00 = []byte{ 170 | // 227 bytes of a gzipped FileDescriptorProto 171 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0xcf, 0x4d, 0x2d, 0xca, 172 | 0xce, 0x49, 0x4d, 0x49, 0x4c, 0xd7, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x41, 0x12, 0x48, 173 | 0x92, 0xd2, 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f, 174 | 0xcf, 0xd7, 0x07, 0x2b, 0x4a, 0x2a, 0x4d, 0x03, 0xf3, 0xc0, 0x1c, 0x30, 0x0b, 0xa2, 0x59, 0xc9, 175 | 0x8d, 0x8b, 0x2d, 0xc0, 0xc9, 0x27, 0x33, 0x2f, 0x5b, 0x48, 0x88, 0x8b, 0xc5, 0x23, 0xb1, 0x38, 176 | 0x43, 0x82, 0x51, 0x81, 0x51, 0x83, 0x27, 0x08, 0xcc, 0x06, 0x89, 0xf9, 0x25, 0xe6, 0xa6, 0x4a, 177 | 0x30, 0x29, 0x30, 0x6a, 0x70, 0x06, 0x81, 0xd9, 0x42, 0x22, 0x5c, 0xac, 0x21, 0xc5, 0x99, 0x55, 178 | 0xa9, 0x12, 0xcc, 0x0a, 0x8c, 0x1a, 0x2c, 0x41, 0x10, 0x8e, 0x92, 0x07, 0xc8, 0x1c, 0xbf, 0xfc, 179 | 0x94, 0x54, 0x21, 0x2d, 0x2e, 0x56, 0x90, 0x79, 0xc5, 0x12, 0x4c, 0x0a, 0xcc, 0x1a, 0xdc, 0x46, 180 | 0x22, 0x7a, 0xc8, 0xce, 0xd3, 0x83, 0x58, 0x16, 0x04, 0x51, 0x02, 0x32, 0xdf, 0x25, 0xb1, 0x24, 181 | 0x11, 0x66, 0x27, 0x88, 0xed, 0xa4, 0x73, 0xe3, 0xa1, 0x1c, 0xc3, 0x83, 0x87, 0x72, 0x8c, 0x1f, 182 | 0x1e, 0xca, 0x31, 0xfe, 0x78, 0x28, 0xc7, 0xd8, 0xf0, 0x48, 0x8e, 0x71, 0xc5, 0x23, 0x39, 0xc6, 183 | 0x1d, 0x8f, 0xe4, 0x18, 0x0f, 0x3c, 0x92, 0x63, 0x3c, 0xf1, 0x48, 0x8e, 0xf1, 0xc2, 0x23, 0x39, 184 | 0xc6, 0x07, 0x8f, 0xe4, 0x18, 0x01, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1d, 0x56, 0xb5, 0x6e, 0x0e, 185 | 0x01, 0x00, 0x00, 186 | } 187 | 188 | func (pbLink *PBLink) VerboseEqual(that interface{}) error { 189 | if that == nil { 190 | if pbLink == nil { 191 | return nil 192 | } 193 | return fmt.Errorf("that == nil && this != nil") 194 | } 195 | 196 | that1, ok := that.(*PBLink) 197 | if !ok { 198 | that2, ok := that.(PBLink) 199 | if ok { 200 | that1 = &that2 201 | } else { 202 | return fmt.Errorf("that is not of type *PBLink") 203 | } 204 | } 205 | if that1 == nil { 206 | if pbLink == nil { 207 | return nil 208 | } 209 | return fmt.Errorf("that is type *PBLink but is nil && this != nil") 210 | } else if pbLink == nil { 211 | return fmt.Errorf("that is type *PBLink but is not nil && this == nil") 212 | } 213 | if !bytes.Equal(pbLink.Hash, that1.Hash) { 214 | return fmt.Errorf("this.Hash(%v) is not equal to that.Hash(%v)", pbLink.Hash, that1.Hash) 215 | } 216 | if pbLink.Name != nil && that1.Name != nil { 217 | if *pbLink.Name != *that1.Name { 218 | return fmt.Errorf("this.Name(%v) is not equal to that.Name(%v)", *pbLink.Name, *that1.Name) 219 | } 220 | } else if pbLink.Name != nil { 221 | return fmt.Errorf("this.Name == nil && that.Name != nil") 222 | } else if that1.Name != nil { 223 | return fmt.Errorf("this.Name(%v) is not equal to that.Name(%v)", pbLink.Name, that1.Name) 224 | } 225 | if pbLink.Tsize != nil && that1.Tsize != nil { 226 | if *pbLink.Tsize != *that1.Tsize { 227 | return fmt.Errorf("this.Tsize(%v) is not equal to that.Tsize(%v)", *pbLink.Tsize, *that1.Tsize) 228 | } 229 | } else if pbLink.Tsize != nil { 230 | return fmt.Errorf("this.Tsize == nil && that.Tsize != nil") 231 | } else if that1.Tsize != nil { 232 | return fmt.Errorf("this.Tsize(%v) is not equal to that.Tsize(%v)", pbLink.Tsize, that1.Tsize) 233 | } 234 | if !bytes.Equal(pbLink.XXX_unrecognized, that1.XXX_unrecognized) { 235 | return fmt.Errorf("XXX_unrecognized this(%v) is not equal to that(%v)", pbLink.XXX_unrecognized, that1.XXX_unrecognized) 236 | } 237 | return nil 238 | } 239 | func (pbLink *PBLink) Equal(that interface{}) bool { 240 | if that == nil { 241 | return pbLink == nil 242 | } 243 | 244 | that1, ok := that.(*PBLink) 245 | if !ok { 246 | that2, ok := that.(PBLink) 247 | if ok { 248 | that1 = &that2 249 | } else { 250 | return false 251 | } 252 | } 253 | if that1 == nil { 254 | return pbLink == nil 255 | } else if pbLink == nil { 256 | return false 257 | } 258 | if !bytes.Equal(pbLink.Hash, that1.Hash) { 259 | return false 260 | } 261 | if pbLink.Name != nil && that1.Name != nil { 262 | if *pbLink.Name != *that1.Name { 263 | return false 264 | } 265 | } else if pbLink.Name != nil { 266 | return false 267 | } else if that1.Name != nil { 268 | return false 269 | } 270 | if pbLink.Tsize != nil && that1.Tsize != nil { 271 | if *pbLink.Tsize != *that1.Tsize { 272 | return false 273 | } 274 | } else if pbLink.Tsize != nil { 275 | return false 276 | } else if that1.Tsize != nil { 277 | return false 278 | } 279 | if !bytes.Equal(pbLink.XXX_unrecognized, that1.XXX_unrecognized) { 280 | return false 281 | } 282 | return true 283 | } 284 | func (pbLink *PBNode) VerboseEqual(that interface{}) error { 285 | if that == nil { 286 | if pbLink == nil { 287 | return nil 288 | } 289 | return fmt.Errorf("that == nil && this != nil") 290 | } 291 | 292 | that1, ok := that.(*PBNode) 293 | if !ok { 294 | that2, ok := that.(PBNode) 295 | if ok { 296 | that1 = &that2 297 | } else { 298 | return fmt.Errorf("that is not of type *PBNode") 299 | } 300 | } 301 | if that1 == nil { 302 | if pbLink == nil { 303 | return nil 304 | } 305 | return fmt.Errorf("that is type *PBNode but is nil && this != nil") 306 | } else if pbLink == nil { 307 | return fmt.Errorf("that is type *PBNode but is not nil && this == nil") 308 | } 309 | if len(pbLink.Links) != len(that1.Links) { 310 | return fmt.Errorf("len(this.Links)(%v) is not equal to len(that.Links)(%v)", len(pbLink.Links), len(that1.Links)) 311 | } 312 | for i := range pbLink.Links { 313 | if !pbLink.Links[i].Equal(that1.Links[i]) { 314 | return fmt.Errorf("this.Links[%v](%v) is not equal to that.Links[%v](%v)", i, pbLink.Links[i], i, that1.Links[i]) 315 | } 316 | } 317 | if !bytes.Equal(pbLink.Data, that1.Data) { 318 | return fmt.Errorf("this.Data(%v) is not equal to that.Data(%v)", pbLink.Data, that1.Data) 319 | } 320 | if !bytes.Equal(pbLink.XXX_unrecognized, that1.XXX_unrecognized) { 321 | return fmt.Errorf("this.XXX_unrecognized(%v) is not equal to that.XXX_unrecognized(%v)", pbLink.XXX_unrecognized, that1.XXX_unrecognized) 322 | } 323 | return nil 324 | } 325 | 326 | func (pbNode *PBNode) Equal(that interface{}) bool { 327 | if that == nil { 328 | return pbNode == nil 329 | } 330 | 331 | that1, ok := that.(*PBNode) 332 | if !ok { 333 | that2, ok := that.(PBNode) 334 | if ok { 335 | that1 = &that2 336 | } else { 337 | return false 338 | } 339 | } 340 | if that1 == nil { 341 | return pbNode == nil 342 | } else if pbNode == nil { 343 | return false 344 | } 345 | if len(pbNode.Links) != len(that1.Links) { 346 | return false 347 | } 348 | for i := range pbNode.Links { 349 | if !pbNode.Links[i].Equal(that1.Links[i]) { 350 | return false 351 | } 352 | } 353 | if !bytes.Equal(pbNode.Data, that1.Data) { 354 | return false 355 | } 356 | if !bytes.Equal(pbNode.XXX_unrecognized, that1.XXX_unrecognized) { 357 | return false 358 | } 359 | return true 360 | } 361 | func (pbLink *PBLink) GoString() string { 362 | if pbLink == nil { 363 | return "nil" 364 | } 365 | s := make([]string, 0, 7) 366 | s = append(s, "&merkledag_pb.PBLink{") 367 | if pbLink.Hash != nil { 368 | s = append(s, "Hash: "+valueToGoStringMerkledag(pbLink.Hash, "byte")+",\n") 369 | } 370 | if pbLink.Name != nil { 371 | s = append(s, "Name: "+valueToGoStringMerkledag(pbLink.Name, "string")+",\n") 372 | } 373 | if pbLink.Tsize != nil { 374 | s = append(s, "Tsize: "+valueToGoStringMerkledag(pbLink.Tsize, "uint64")+",\n") 375 | } 376 | if pbLink.XXX_unrecognized != nil { 377 | s = append(s, "XXX_unrecognized:"+fmt.Sprintf("%#v", pbLink.XXX_unrecognized)+",\n") 378 | } 379 | s = append(s, "}") 380 | return strings.Join(s, "") 381 | } 382 | func (pbNode *PBNode) GoString() string { 383 | if pbNode == nil { 384 | return "nil" 385 | } 386 | s := make([]string, 0, 6) 387 | s = append(s, "&merkledag_pb.PBNode{") 388 | if pbNode.Links != nil { 389 | s = append(s, "Links: "+fmt.Sprintf("%#v", pbNode.Links)+",\n") 390 | } 391 | if pbNode.Data != nil { 392 | s = append(s, "Data: "+valueToGoStringMerkledag(pbNode.Data, "byte")+",\n") 393 | } 394 | if pbNode.XXX_unrecognized != nil { 395 | s = append(s, "XXX_unrecognized:"+fmt.Sprintf("%#v", pbNode.XXX_unrecognized)+",\n") 396 | } 397 | s = append(s, "}") 398 | return strings.Join(s, "") 399 | } 400 | func valueToGoStringMerkledag(v interface{}, typ string) string { 401 | rv := reflect.ValueOf(v) 402 | if rv.IsNil() { 403 | return "nil" 404 | } 405 | pv := reflect.Indirect(rv).Interface() 406 | return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) 407 | } 408 | func (m *PBLink) Marshal() (dAtA []byte, err error) { 409 | size := m.Size() 410 | dAtA = make([]byte, size) 411 | n, err := m.MarshalToSizedBuffer(dAtA[:size]) 412 | if err != nil { 413 | return nil, err 414 | } 415 | return dAtA[:n], nil 416 | } 417 | 418 | func (m *PBLink) MarshalTo(dAtA []byte) (int, error) { 419 | size := m.Size() 420 | return m.MarshalToSizedBuffer(dAtA[:size]) 421 | } 422 | 423 | func (m *PBLink) MarshalToSizedBuffer(dAtA []byte) (int, error) { 424 | i := len(dAtA) 425 | _ = i 426 | var l int 427 | _ = l 428 | if m.XXX_unrecognized != nil { 429 | i -= len(m.XXX_unrecognized) 430 | copy(dAtA[i:], m.XXX_unrecognized) 431 | } 432 | if m.Tsize != nil { 433 | i = encodeVarintMerkledag(dAtA, i, uint64(*m.Tsize)) 434 | i-- 435 | dAtA[i] = 0x18 436 | } 437 | if m.Name != nil { 438 | i -= len(*m.Name) 439 | copy(dAtA[i:], *m.Name) 440 | i = encodeVarintMerkledag(dAtA, i, uint64(len(*m.Name))) 441 | i-- 442 | dAtA[i] = 0x12 443 | } 444 | if m.Hash != nil { 445 | i -= len(m.Hash) 446 | copy(dAtA[i:], m.Hash) 447 | i = encodeVarintMerkledag(dAtA, i, uint64(len(m.Hash))) 448 | i-- 449 | dAtA[i] = 0xa 450 | } 451 | return len(dAtA) - i, nil 452 | } 453 | 454 | func (m *PBNode) Marshal() (dAtA []byte, err error) { 455 | size := m.Size() 456 | dAtA = make([]byte, size) 457 | n, err := m.MarshalToSizedBuffer(dAtA[:size]) 458 | if err != nil { 459 | return nil, err 460 | } 461 | return dAtA[:n], nil 462 | } 463 | 464 | func (m *PBNode) MarshalTo(dAtA []byte) (int, error) { 465 | size := m.Size() 466 | return m.MarshalToSizedBuffer(dAtA[:size]) 467 | } 468 | 469 | func (m *PBNode) MarshalToSizedBuffer(dAtA []byte) (int, error) { 470 | i := len(dAtA) 471 | _ = i 472 | var l int 473 | _ = l 474 | if m.XXX_unrecognized != nil { 475 | i -= len(m.XXX_unrecognized) 476 | copy(dAtA[i:], m.XXX_unrecognized) 477 | } 478 | if m.Data != nil { 479 | i -= len(m.Data) 480 | copy(dAtA[i:], m.Data) 481 | i = encodeVarintMerkledag(dAtA, i, uint64(len(m.Data))) 482 | i-- 483 | dAtA[i] = 0xa 484 | } 485 | if len(m.Links) > 0 { 486 | for iNdEx := len(m.Links) - 1; iNdEx >= 0; iNdEx-- { 487 | { 488 | size, err := m.Links[iNdEx].MarshalToSizedBuffer(dAtA[:i]) 489 | if err != nil { 490 | return 0, err 491 | } 492 | i -= size 493 | i = encodeVarintMerkledag(dAtA, i, uint64(size)) 494 | } 495 | i-- 496 | dAtA[i] = 0x12 497 | } 498 | } 499 | return len(dAtA) - i, nil 500 | } 501 | 502 | func encodeVarintMerkledag(dAtA []byte, offset int, v uint64) int { 503 | offset -= sovMerkledag(v) 504 | base := offset 505 | for v >= 1<<7 { 506 | dAtA[offset] = uint8(v&0x7f | 0x80) 507 | v >>= 7 508 | offset++ 509 | } 510 | dAtA[offset] = uint8(v) 511 | return base 512 | } 513 | func NewPopulatedPBLink(r randyMerkledag, easy bool) *PBLink { 514 | this := &PBLink{} 515 | if r.Intn(5) != 0 { 516 | v1 := r.Intn(100) 517 | this.Hash = make([]byte, v1) 518 | for i := 0; i < v1; i++ { 519 | this.Hash[i] = byte(r.Intn(256)) 520 | } 521 | } 522 | if r.Intn(5) != 0 { 523 | v2 := string(randStringMerkledag(r)) 524 | this.Name = &v2 525 | } 526 | if r.Intn(5) != 0 { 527 | v3 := uint64(uint64(r.Uint32())) 528 | this.Tsize = &v3 529 | } 530 | if !easy && r.Intn(10) != 0 { 531 | this.XXX_unrecognized = randUnrecognizedMerkledag(r, 4) 532 | } 533 | return this 534 | } 535 | 536 | func NewPopulatedPBNode(r randyMerkledag, easy bool) *PBNode { 537 | this := &PBNode{} 538 | if r.Intn(5) != 0 { 539 | v4 := r.Intn(100) 540 | this.Data = make([]byte, v4) 541 | for i := 0; i < v4; i++ { 542 | this.Data[i] = byte(r.Intn(256)) 543 | } 544 | } 545 | if r.Intn(5) != 0 { 546 | v5 := r.Intn(5) 547 | this.Links = make([]*PBLink, v5) 548 | for i := 0; i < v5; i++ { 549 | this.Links[i] = NewPopulatedPBLink(r, easy) 550 | } 551 | } 552 | if !easy && r.Intn(10) != 0 { 553 | this.XXX_unrecognized = randUnrecognizedMerkledag(r, 3) 554 | } 555 | return this 556 | } 557 | 558 | type randyMerkledag interface { 559 | Float32() float32 560 | Float64() float64 561 | Int63() int64 562 | Int31() int32 563 | Uint32() uint32 564 | Intn(n int) int 565 | } 566 | 567 | func randUTF8RuneMerkledag(r randyMerkledag) rune { 568 | ru := r.Intn(62) 569 | if ru < 10 { 570 | return rune(ru + 48) 571 | } else if ru < 36 { 572 | return rune(ru + 55) 573 | } 574 | return rune(ru + 61) 575 | } 576 | func randStringMerkledag(r randyMerkledag) string { 577 | v6 := r.Intn(100) 578 | tmps := make([]rune, v6) 579 | for i := 0; i < v6; i++ { 580 | tmps[i] = randUTF8RuneMerkledag(r) 581 | } 582 | return string(tmps) 583 | } 584 | func randUnrecognizedMerkledag(r randyMerkledag, maxFieldNumber int) (dAtA []byte) { 585 | l := r.Intn(5) 586 | for i := 0; i < l; i++ { 587 | wire := r.Intn(4) 588 | if wire == 3 { 589 | wire = 5 590 | } 591 | fieldNumber := maxFieldNumber + r.Intn(100) 592 | dAtA = randFieldMerkledag(dAtA, r, fieldNumber, wire) 593 | } 594 | return dAtA 595 | } 596 | func randFieldMerkledag(dAtA []byte, r randyMerkledag, fieldNumber int, wire int) []byte { 597 | key := uint32(fieldNumber)<<3 | uint32(wire) 598 | switch wire { 599 | case 0: 600 | dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(key)) 601 | v7 := r.Int63() 602 | if r.Intn(2) == 0 { 603 | v7 *= -1 604 | } 605 | dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(v7)) 606 | case 1: 607 | dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(key)) 608 | dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) 609 | case 2: 610 | dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(key)) 611 | ll := r.Intn(100) 612 | dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(ll)) 613 | for j := 0; j < ll; j++ { 614 | dAtA = append(dAtA, byte(r.Intn(256))) 615 | } 616 | default: 617 | dAtA = encodeVarintPopulateMerkledag(dAtA, uint64(key)) 618 | dAtA = append(dAtA, byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256)), byte(r.Intn(256))) 619 | } 620 | return dAtA 621 | } 622 | func encodeVarintPopulateMerkledag(dAtA []byte, v uint64) []byte { 623 | for v >= 1<<7 { 624 | dAtA = append(dAtA, uint8(uint64(v)&0x7f|0x80)) 625 | v >>= 7 626 | } 627 | dAtA = append(dAtA, uint8(v)) 628 | return dAtA 629 | } 630 | func (m *PBLink) Size() (n int) { 631 | if m == nil { 632 | return 0 633 | } 634 | var l int 635 | _ = l 636 | if m.Hash != nil { 637 | l = len(m.Hash) 638 | n += 1 + l + sovMerkledag(uint64(l)) 639 | } 640 | if m.Name != nil { 641 | l = len(*m.Name) 642 | n += 1 + l + sovMerkledag(uint64(l)) 643 | } 644 | if m.Tsize != nil { 645 | n += 1 + sovMerkledag(uint64(*m.Tsize)) 646 | } 647 | if m.XXX_unrecognized != nil { 648 | n += len(m.XXX_unrecognized) 649 | } 650 | return n 651 | } 652 | 653 | func (m *PBNode) Size() (n int) { 654 | if m == nil { 655 | return 0 656 | } 657 | var l int 658 | _ = l 659 | if m.Data != nil { 660 | l = len(m.Data) 661 | n += 1 + l + sovMerkledag(uint64(l)) 662 | } 663 | if len(m.Links) > 0 { 664 | for _, e := range m.Links { 665 | l = e.Size() 666 | n += 1 + l + sovMerkledag(uint64(l)) 667 | } 668 | } 669 | if m.XXX_unrecognized != nil { 670 | n += len(m.XXX_unrecognized) 671 | } 672 | return n 673 | } 674 | 675 | func sovMerkledag(x uint64) (n int) { 676 | return (math_bits.Len64(x|1) + 6) / 7 677 | } 678 | func (pbLink *PBLink) String() string { 679 | if pbLink == nil { 680 | return "nil" 681 | } 682 | s := strings.Join([]string{`&PBLink{`, 683 | `Hash:` + valueToStringMerkledag(pbLink.Hash) + `,`, 684 | `Name:` + valueToStringMerkledag(pbLink.Name) + `,`, 685 | `Tsize:` + valueToStringMerkledag(pbLink.Tsize) + `,`, 686 | `XXX_unrecognized:` + fmt.Sprintf("%v", pbLink.XXX_unrecognized) + `,`, 687 | `}`, 688 | }, "") 689 | return s 690 | } 691 | func (pbNode *PBNode) String() string { 692 | if pbNode == nil { 693 | return "nil" 694 | } 695 | repeatedStringForLinks := "[]*PBLink{" 696 | for _, f := range pbNode.Links { 697 | repeatedStringForLinks += strings.Replace(f.String(), "PBLink", "PBLink", 1) + "," 698 | } 699 | repeatedStringForLinks += "}" 700 | s := strings.Join([]string{`&PBNode{`, 701 | `Data:` + valueToStringMerkledag(pbNode.Data) + `,`, 702 | `Links:` + repeatedStringForLinks + `,`, 703 | `XXX_unrecognized:` + fmt.Sprintf("%v", pbNode.XXX_unrecognized) + `,`, 704 | `}`, 705 | }, "") 706 | return s 707 | } 708 | func valueToStringMerkledag(v interface{}) string { 709 | rv := reflect.ValueOf(v) 710 | if rv.IsNil() { 711 | return "nil" 712 | } 713 | pv := reflect.Indirect(rv).Interface() 714 | return fmt.Sprintf("*%v", pv) 715 | } 716 | func (m *PBLink) Unmarshal(dAtA []byte) error { 717 | l := len(dAtA) 718 | iNdEx := 0 719 | for iNdEx < l { 720 | preIndex := iNdEx 721 | var wire uint64 722 | for shift := uint(0); ; shift += 7 { 723 | if shift >= 64 { 724 | return ErrIntOverflowMerkledag 725 | } 726 | if iNdEx >= l { 727 | return io.ErrUnexpectedEOF 728 | } 729 | b := dAtA[iNdEx] 730 | iNdEx++ 731 | wire |= uint64(b&0x7F) << shift 732 | if b < 0x80 { 733 | break 734 | } 735 | } 736 | fieldNum := int32(wire >> 3) 737 | wireType := int(wire & 0x7) 738 | if wireType == 4 { 739 | return fmt.Errorf("proto: PBLink: wiretype end group for non-group") 740 | } 741 | if fieldNum <= 0 { 742 | return fmt.Errorf("proto: PBLink: illegal tag %d (wire type %d)", fieldNum, wire) 743 | } 744 | switch fieldNum { 745 | case 1: 746 | if wireType != 2 { 747 | return fmt.Errorf("proto: wrong wireType = %d for field Hash", wireType) 748 | } 749 | var byteLen int 750 | for shift := uint(0); ; shift += 7 { 751 | if shift >= 64 { 752 | return ErrIntOverflowMerkledag 753 | } 754 | if iNdEx >= l { 755 | return io.ErrUnexpectedEOF 756 | } 757 | b := dAtA[iNdEx] 758 | iNdEx++ 759 | byteLen |= int(b&0x7F) << shift 760 | if b < 0x80 { 761 | break 762 | } 763 | } 764 | if byteLen < 0 { 765 | return ErrInvalidLengthMerkledag 766 | } 767 | postIndex := iNdEx + byteLen 768 | if postIndex < 0 { 769 | return ErrInvalidLengthMerkledag 770 | } 771 | if postIndex > l { 772 | return io.ErrUnexpectedEOF 773 | } 774 | m.Hash = append(m.Hash[:0], dAtA[iNdEx:postIndex]...) 775 | if m.Hash == nil { 776 | m.Hash = []byte{} 777 | } 778 | iNdEx = postIndex 779 | case 2: 780 | if wireType != 2 { 781 | return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) 782 | } 783 | var stringLen uint64 784 | for shift := uint(0); ; shift += 7 { 785 | if shift >= 64 { 786 | return ErrIntOverflowMerkledag 787 | } 788 | if iNdEx >= l { 789 | return io.ErrUnexpectedEOF 790 | } 791 | b := dAtA[iNdEx] 792 | iNdEx++ 793 | stringLen |= uint64(b&0x7F) << shift 794 | if b < 0x80 { 795 | break 796 | } 797 | } 798 | intStringLen := int(stringLen) 799 | if intStringLen < 0 { 800 | return ErrInvalidLengthMerkledag 801 | } 802 | postIndex := iNdEx + intStringLen 803 | if postIndex < 0 { 804 | return ErrInvalidLengthMerkledag 805 | } 806 | if postIndex > l { 807 | return io.ErrUnexpectedEOF 808 | } 809 | s := string(dAtA[iNdEx:postIndex]) 810 | m.Name = &s 811 | iNdEx = postIndex 812 | case 3: 813 | if wireType != 0 { 814 | return fmt.Errorf("proto: wrong wireType = %d for field Tsize", wireType) 815 | } 816 | var v uint64 817 | for shift := uint(0); ; shift += 7 { 818 | if shift >= 64 { 819 | return ErrIntOverflowMerkledag 820 | } 821 | if iNdEx >= l { 822 | return io.ErrUnexpectedEOF 823 | } 824 | b := dAtA[iNdEx] 825 | iNdEx++ 826 | v |= uint64(b&0x7F) << shift 827 | if b < 0x80 { 828 | break 829 | } 830 | } 831 | m.Tsize = &v 832 | default: 833 | iNdEx = preIndex 834 | skippy, err := skipMerkledag(dAtA[iNdEx:]) 835 | if err != nil { 836 | return err 837 | } 838 | if skippy < 0 { 839 | return ErrInvalidLengthMerkledag 840 | } 841 | if (iNdEx + skippy) < 0 { 842 | return ErrInvalidLengthMerkledag 843 | } 844 | if (iNdEx + skippy) > l { 845 | return io.ErrUnexpectedEOF 846 | } 847 | m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) 848 | iNdEx += skippy 849 | } 850 | } 851 | 852 | if iNdEx > l { 853 | return io.ErrUnexpectedEOF 854 | } 855 | return nil 856 | } 857 | func (m *PBNode) Unmarshal(dAtA []byte) error { 858 | l := len(dAtA) 859 | iNdEx := 0 860 | for iNdEx < l { 861 | preIndex := iNdEx 862 | var wire uint64 863 | for shift := uint(0); ; shift += 7 { 864 | if shift >= 64 { 865 | return ErrIntOverflowMerkledag 866 | } 867 | if iNdEx >= l { 868 | return io.ErrUnexpectedEOF 869 | } 870 | b := dAtA[iNdEx] 871 | iNdEx++ 872 | wire |= uint64(b&0x7F) << shift 873 | if b < 0x80 { 874 | break 875 | } 876 | } 877 | fieldNum := int32(wire >> 3) 878 | wireType := int(wire & 0x7) 879 | if wireType == 4 { 880 | return fmt.Errorf("proto: PBNode: wiretype end group for non-group") 881 | } 882 | if fieldNum <= 0 { 883 | return fmt.Errorf("proto: PBNode: illegal tag %d (wire type %d)", fieldNum, wire) 884 | } 885 | switch fieldNum { 886 | case 1: 887 | if wireType != 2 { 888 | return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) 889 | } 890 | var byteLen int 891 | for shift := uint(0); ; shift += 7 { 892 | if shift >= 64 { 893 | return ErrIntOverflowMerkledag 894 | } 895 | if iNdEx >= l { 896 | return io.ErrUnexpectedEOF 897 | } 898 | b := dAtA[iNdEx] 899 | iNdEx++ 900 | byteLen |= int(b&0x7F) << shift 901 | if b < 0x80 { 902 | break 903 | } 904 | } 905 | if byteLen < 0 { 906 | return ErrInvalidLengthMerkledag 907 | } 908 | postIndex := iNdEx + byteLen 909 | if postIndex < 0 { 910 | return ErrInvalidLengthMerkledag 911 | } 912 | if postIndex > l { 913 | return io.ErrUnexpectedEOF 914 | } 915 | m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) 916 | if m.Data == nil { 917 | m.Data = []byte{} 918 | } 919 | iNdEx = postIndex 920 | case 2: 921 | if wireType != 2 { 922 | return fmt.Errorf("proto: wrong wireType = %d for field Links", wireType) 923 | } 924 | var msglen int 925 | for shift := uint(0); ; shift += 7 { 926 | if shift >= 64 { 927 | return ErrIntOverflowMerkledag 928 | } 929 | if iNdEx >= l { 930 | return io.ErrUnexpectedEOF 931 | } 932 | b := dAtA[iNdEx] 933 | iNdEx++ 934 | msglen |= int(b&0x7F) << shift 935 | if b < 0x80 { 936 | break 937 | } 938 | } 939 | if msglen < 0 { 940 | return ErrInvalidLengthMerkledag 941 | } 942 | postIndex := iNdEx + msglen 943 | if postIndex < 0 { 944 | return ErrInvalidLengthMerkledag 945 | } 946 | if postIndex > l { 947 | return io.ErrUnexpectedEOF 948 | } 949 | m.Links = append(m.Links, &PBLink{}) 950 | if err := m.Links[len(m.Links)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { 951 | return err 952 | } 953 | iNdEx = postIndex 954 | default: 955 | iNdEx = preIndex 956 | skippy, err := skipMerkledag(dAtA[iNdEx:]) 957 | if err != nil { 958 | return err 959 | } 960 | if skippy < 0 { 961 | return ErrInvalidLengthMerkledag 962 | } 963 | if (iNdEx + skippy) < 0 { 964 | return ErrInvalidLengthMerkledag 965 | } 966 | if (iNdEx + skippy) > l { 967 | return io.ErrUnexpectedEOF 968 | } 969 | m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) 970 | iNdEx += skippy 971 | } 972 | } 973 | 974 | if iNdEx > l { 975 | return io.ErrUnexpectedEOF 976 | } 977 | return nil 978 | } 979 | func skipMerkledag(dAtA []byte) (n int, err error) { 980 | l := len(dAtA) 981 | iNdEx := 0 982 | depth := 0 983 | for iNdEx < l { 984 | var wire uint64 985 | for shift := uint(0); ; shift += 7 { 986 | if shift >= 64 { 987 | return 0, ErrIntOverflowMerkledag 988 | } 989 | if iNdEx >= l { 990 | return 0, io.ErrUnexpectedEOF 991 | } 992 | b := dAtA[iNdEx] 993 | iNdEx++ 994 | wire |= (uint64(b) & 0x7F) << shift 995 | if b < 0x80 { 996 | break 997 | } 998 | } 999 | wireType := int(wire & 0x7) 1000 | switch wireType { 1001 | case 0: 1002 | for shift := uint(0); ; shift += 7 { 1003 | if shift >= 64 { 1004 | return 0, ErrIntOverflowMerkledag 1005 | } 1006 | if iNdEx >= l { 1007 | return 0, io.ErrUnexpectedEOF 1008 | } 1009 | iNdEx++ 1010 | if dAtA[iNdEx-1] < 0x80 { 1011 | break 1012 | } 1013 | } 1014 | case 1: 1015 | iNdEx += 8 1016 | case 2: 1017 | var length int 1018 | for shift := uint(0); ; shift += 7 { 1019 | if shift >= 64 { 1020 | return 0, ErrIntOverflowMerkledag 1021 | } 1022 | if iNdEx >= l { 1023 | return 0, io.ErrUnexpectedEOF 1024 | } 1025 | b := dAtA[iNdEx] 1026 | iNdEx++ 1027 | length |= (int(b) & 0x7F) << shift 1028 | if b < 0x80 { 1029 | break 1030 | } 1031 | } 1032 | if length < 0 { 1033 | return 0, ErrInvalidLengthMerkledag 1034 | } 1035 | iNdEx += length 1036 | if iNdEx < 0 { 1037 | return 0, ErrInvalidLengthMerkledag 1038 | } 1039 | case 3: 1040 | depth++ 1041 | case 4: 1042 | if depth == 0 { 1043 | return 0, ErrUnexpectedEndOfGroupMerkledag 1044 | } 1045 | depth-- 1046 | case 5: 1047 | iNdEx += 4 1048 | default: 1049 | return 0, fmt.Errorf("proto: illegal wireType %d", wireType) 1050 | } 1051 | if depth == 0 { 1052 | return iNdEx, nil 1053 | } 1054 | } 1055 | return 0, io.ErrUnexpectedEOF 1056 | } 1057 | 1058 | var ( 1059 | ErrInvalidLengthMerkledag = fmt.Errorf("proto: negative length found during unmarshaling") 1060 | ErrIntOverflowMerkledag = fmt.Errorf("proto: integer overflow") 1061 | ErrUnexpectedEndOfGroupMerkledag = fmt.Errorf("proto: unexpected end of group") 1062 | ) 1063 | -------------------------------------------------------------------------------- /pb/merkledag.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package merkledag.pb; 4 | 5 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 6 | 7 | option (gogoproto.gostring_all) = true; 8 | option (gogoproto.equal_all) = true; 9 | option (gogoproto.verbose_equal_all) = true; 10 | option (gogoproto.goproto_stringer_all) = false; 11 | option (gogoproto.stringer_all) = true; 12 | option (gogoproto.populate_all) = true; 13 | option (gogoproto.testgen_all) = true; 14 | option (gogoproto.benchgen_all) = true; 15 | option (gogoproto.marshaler_all) = true; 16 | option (gogoproto.sizer_all) = true; 17 | option (gogoproto.unmarshaler_all) = true; 18 | 19 | 20 | // An IPFS MerkleDAG Link 21 | message PBLink { 22 | 23 | // multihash of the target object 24 | optional bytes Hash = 1; 25 | 26 | // utf string name. should be unique per object 27 | optional string Name = 2; 28 | 29 | // cumulative size of target object 30 | optional uint64 Tsize = 3; 31 | } 32 | 33 | // An IPFS MerkleDAG Node 34 | message PBNode { 35 | 36 | // refs to other objects 37 | repeated PBLink Links = 2; 38 | 39 | // opaque user data 40 | optional bytes Data = 1; 41 | } 42 | -------------------------------------------------------------------------------- /pb/merkledagpb_test.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-gogo. DO NOT EDIT. 2 | // source: merkledag.proto 3 | 4 | package merkledag_pb 5 | 6 | import ( 7 | fmt "fmt" 8 | go_parser "go/parser" 9 | math "math" 10 | math_rand "math/rand" 11 | testing "testing" 12 | time "time" 13 | 14 | _ "github.com/gogo/protobuf/gogoproto" 15 | github_com_gogo_protobuf_jsonpb "github.com/gogo/protobuf/jsonpb" 16 | github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" 17 | proto "github.com/gogo/protobuf/proto" 18 | ) 19 | 20 | // Reference imports to suppress errors if they are not otherwise used. 21 | var _ = proto.Marshal 22 | var _ = fmt.Errorf 23 | var _ = math.Inf 24 | 25 | func TestPBLinkProto(t *testing.T) { 26 | seed := time.Now().UnixNano() 27 | popr := math_rand.New(math_rand.NewSource(seed)) 28 | p := NewPopulatedPBLink(popr, false) 29 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) 30 | if err != nil { 31 | t.Fatalf("seed = %d, err = %v", seed, err) 32 | } 33 | msg := &PBLink{} 34 | if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { 35 | t.Fatalf("seed = %d, err = %v", seed, err) 36 | } 37 | littlefuzz := make([]byte, len(dAtA)) 38 | copy(littlefuzz, dAtA) 39 | for i := range dAtA { 40 | dAtA[i] = byte(popr.Intn(256)) 41 | } 42 | if err := p.VerboseEqual(msg); err != nil { 43 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 44 | } 45 | if !p.Equal(msg) { 46 | t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) 47 | } 48 | if len(littlefuzz) > 0 { 49 | fuzzamount := 100 50 | for i := 0; i < fuzzamount; i++ { 51 | littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) 52 | littlefuzz = append(littlefuzz, byte(popr.Intn(256))) 53 | } 54 | // shouldn't panic 55 | _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) 56 | } 57 | } 58 | 59 | func TestPBLinkMarshalTo(t *testing.T) { 60 | seed := time.Now().UnixNano() 61 | popr := math_rand.New(math_rand.NewSource(seed)) 62 | p := NewPopulatedPBLink(popr, false) 63 | size := p.Size() 64 | dAtA := make([]byte, size) 65 | for i := range dAtA { 66 | dAtA[i] = byte(popr.Intn(256)) 67 | } 68 | _, err := p.MarshalTo(dAtA) 69 | if err != nil { 70 | t.Fatalf("seed = %d, err = %v", seed, err) 71 | } 72 | msg := &PBLink{} 73 | if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { 74 | t.Fatalf("seed = %d, err = %v", seed, err) 75 | } 76 | for i := range dAtA { 77 | dAtA[i] = byte(popr.Intn(256)) 78 | } 79 | if err := p.VerboseEqual(msg); err != nil { 80 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 81 | } 82 | if !p.Equal(msg) { 83 | t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) 84 | } 85 | } 86 | 87 | func BenchmarkPBLinkProtoMarshal(b *testing.B) { 88 | popr := math_rand.New(math_rand.NewSource(616)) 89 | total := 0 90 | pops := make([]*PBLink, 10000) 91 | for i := 0; i < 10000; i++ { 92 | pops[i] = NewPopulatedPBLink(popr, false) 93 | } 94 | b.ResetTimer() 95 | for i := 0; i < b.N; i++ { 96 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) 97 | if err != nil { 98 | panic(err) 99 | } 100 | total += len(dAtA) 101 | } 102 | b.SetBytes(int64(total / b.N)) 103 | } 104 | 105 | func BenchmarkPBLinkProtoUnmarshal(b *testing.B) { 106 | popr := math_rand.New(math_rand.NewSource(616)) 107 | total := 0 108 | datas := make([][]byte, 10000) 109 | for i := 0; i < 10000; i++ { 110 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBLink(popr, false)) 111 | if err != nil { 112 | panic(err) 113 | } 114 | datas[i] = dAtA 115 | } 116 | msg := &PBLink{} 117 | b.ResetTimer() 118 | for i := 0; i < b.N; i++ { 119 | total += len(datas[i%10000]) 120 | if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { 121 | panic(err) 122 | } 123 | } 124 | b.SetBytes(int64(total / b.N)) 125 | } 126 | 127 | func TestPBNodeProto(t *testing.T) { 128 | seed := time.Now().UnixNano() 129 | popr := math_rand.New(math_rand.NewSource(seed)) 130 | p := NewPopulatedPBNode(popr, false) 131 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) 132 | if err != nil { 133 | t.Fatalf("seed = %d, err = %v", seed, err) 134 | } 135 | msg := &PBNode{} 136 | if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { 137 | t.Fatalf("seed = %d, err = %v", seed, err) 138 | } 139 | littlefuzz := make([]byte, len(dAtA)) 140 | copy(littlefuzz, dAtA) 141 | for i := range dAtA { 142 | dAtA[i] = byte(popr.Intn(256)) 143 | } 144 | if err := p.VerboseEqual(msg); err != nil { 145 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 146 | } 147 | if !p.Equal(msg) { 148 | t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) 149 | } 150 | if len(littlefuzz) > 0 { 151 | fuzzamount := 100 152 | for i := 0; i < fuzzamount; i++ { 153 | littlefuzz[popr.Intn(len(littlefuzz))] = byte(popr.Intn(256)) 154 | littlefuzz = append(littlefuzz, byte(popr.Intn(256))) 155 | } 156 | // shouldn't panic 157 | _ = github_com_gogo_protobuf_proto.Unmarshal(littlefuzz, msg) 158 | } 159 | } 160 | 161 | func TestPBNodeMarshalTo(t *testing.T) { 162 | seed := time.Now().UnixNano() 163 | popr := math_rand.New(math_rand.NewSource(seed)) 164 | p := NewPopulatedPBNode(popr, false) 165 | size := p.Size() 166 | dAtA := make([]byte, size) 167 | for i := range dAtA { 168 | dAtA[i] = byte(popr.Intn(256)) 169 | } 170 | _, err := p.MarshalTo(dAtA) 171 | if err != nil { 172 | t.Fatalf("seed = %d, err = %v", seed, err) 173 | } 174 | msg := &PBNode{} 175 | if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { 176 | t.Fatalf("seed = %d, err = %v", seed, err) 177 | } 178 | for i := range dAtA { 179 | dAtA[i] = byte(popr.Intn(256)) 180 | } 181 | if err := p.VerboseEqual(msg); err != nil { 182 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 183 | } 184 | if !p.Equal(msg) { 185 | t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) 186 | } 187 | } 188 | 189 | func BenchmarkPBNodeProtoMarshal(b *testing.B) { 190 | popr := math_rand.New(math_rand.NewSource(616)) 191 | total := 0 192 | pops := make([]*PBNode, 10000) 193 | for i := 0; i < 10000; i++ { 194 | pops[i] = NewPopulatedPBNode(popr, false) 195 | } 196 | b.ResetTimer() 197 | for i := 0; i < b.N; i++ { 198 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(pops[i%10000]) 199 | if err != nil { 200 | panic(err) 201 | } 202 | total += len(dAtA) 203 | } 204 | b.SetBytes(int64(total / b.N)) 205 | } 206 | 207 | func BenchmarkPBNodeProtoUnmarshal(b *testing.B) { 208 | popr := math_rand.New(math_rand.NewSource(616)) 209 | total := 0 210 | datas := make([][]byte, 10000) 211 | for i := 0; i < 10000; i++ { 212 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(NewPopulatedPBNode(popr, false)) 213 | if err != nil { 214 | panic(err) 215 | } 216 | datas[i] = dAtA 217 | } 218 | msg := &PBNode{} 219 | b.ResetTimer() 220 | for i := 0; i < b.N; i++ { 221 | total += len(datas[i%10000]) 222 | if err := github_com_gogo_protobuf_proto.Unmarshal(datas[i%10000], msg); err != nil { 223 | panic(err) 224 | } 225 | } 226 | b.SetBytes(int64(total / b.N)) 227 | } 228 | 229 | func TestPBLinkJSON(t *testing.T) { 230 | seed := time.Now().UnixNano() 231 | popr := math_rand.New(math_rand.NewSource(seed)) 232 | p := NewPopulatedPBLink(popr, true) 233 | marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} 234 | jsondata, err := marshaler.MarshalToString(p) 235 | if err != nil { 236 | t.Fatalf("seed = %d, err = %v", seed, err) 237 | } 238 | msg := &PBLink{} 239 | err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) 240 | if err != nil { 241 | t.Fatalf("seed = %d, err = %v", seed, err) 242 | } 243 | if err := p.VerboseEqual(msg); err != nil { 244 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 245 | } 246 | if !p.Equal(msg) { 247 | t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) 248 | } 249 | } 250 | func TestPBNodeJSON(t *testing.T) { 251 | seed := time.Now().UnixNano() 252 | popr := math_rand.New(math_rand.NewSource(seed)) 253 | p := NewPopulatedPBNode(popr, true) 254 | marshaler := github_com_gogo_protobuf_jsonpb.Marshaler{} 255 | jsondata, err := marshaler.MarshalToString(p) 256 | if err != nil { 257 | t.Fatalf("seed = %d, err = %v", seed, err) 258 | } 259 | msg := &PBNode{} 260 | err = github_com_gogo_protobuf_jsonpb.UnmarshalString(jsondata, msg) 261 | if err != nil { 262 | t.Fatalf("seed = %d, err = %v", seed, err) 263 | } 264 | if err := p.VerboseEqual(msg); err != nil { 265 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 266 | } 267 | if !p.Equal(msg) { 268 | t.Fatalf("seed = %d, %#v !Json Equal %#v", seed, msg, p) 269 | } 270 | } 271 | func TestPBLinkProtoText(t *testing.T) { 272 | seed := time.Now().UnixNano() 273 | popr := math_rand.New(math_rand.NewSource(seed)) 274 | p := NewPopulatedPBLink(popr, true) 275 | dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) 276 | msg := &PBLink{} 277 | if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { 278 | t.Fatalf("seed = %d, err = %v", seed, err) 279 | } 280 | if err := p.VerboseEqual(msg); err != nil { 281 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 282 | } 283 | if !p.Equal(msg) { 284 | t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) 285 | } 286 | } 287 | 288 | func TestPBLinkProtoCompactText(t *testing.T) { 289 | seed := time.Now().UnixNano() 290 | popr := math_rand.New(math_rand.NewSource(seed)) 291 | p := NewPopulatedPBLink(popr, true) 292 | dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) 293 | msg := &PBLink{} 294 | if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { 295 | t.Fatalf("seed = %d, err = %v", seed, err) 296 | } 297 | if err := p.VerboseEqual(msg); err != nil { 298 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 299 | } 300 | if !p.Equal(msg) { 301 | t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) 302 | } 303 | } 304 | 305 | func TestPBNodeProtoText(t *testing.T) { 306 | seed := time.Now().UnixNano() 307 | popr := math_rand.New(math_rand.NewSource(seed)) 308 | p := NewPopulatedPBNode(popr, true) 309 | dAtA := github_com_gogo_protobuf_proto.MarshalTextString(p) 310 | msg := &PBNode{} 311 | if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { 312 | t.Fatalf("seed = %d, err = %v", seed, err) 313 | } 314 | if err := p.VerboseEqual(msg); err != nil { 315 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 316 | } 317 | if !p.Equal(msg) { 318 | t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) 319 | } 320 | } 321 | 322 | func TestPBNodeProtoCompactText(t *testing.T) { 323 | seed := time.Now().UnixNano() 324 | popr := math_rand.New(math_rand.NewSource(seed)) 325 | p := NewPopulatedPBNode(popr, true) 326 | dAtA := github_com_gogo_protobuf_proto.CompactTextString(p) 327 | msg := &PBNode{} 328 | if err := github_com_gogo_protobuf_proto.UnmarshalText(dAtA, msg); err != nil { 329 | t.Fatalf("seed = %d, err = %v", seed, err) 330 | } 331 | if err := p.VerboseEqual(msg); err != nil { 332 | t.Fatalf("seed = %d, %#v !VerboseProto %#v, since %v", seed, msg, p, err) 333 | } 334 | if !p.Equal(msg) { 335 | t.Fatalf("seed = %d, %#v !Proto %#v", seed, msg, p) 336 | } 337 | } 338 | 339 | func TestPBLinkVerboseEqual(t *testing.T) { 340 | popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) 341 | p := NewPopulatedPBLink(popr, false) 342 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) 343 | if err != nil { 344 | panic(err) 345 | } 346 | msg := &PBLink{} 347 | if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { 348 | panic(err) 349 | } 350 | if err := p.VerboseEqual(msg); err != nil { 351 | t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err) 352 | } 353 | } 354 | func TestPBNodeVerboseEqual(t *testing.T) { 355 | popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) 356 | p := NewPopulatedPBNode(popr, false) 357 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) 358 | if err != nil { 359 | panic(err) 360 | } 361 | msg := &PBNode{} 362 | if err := github_com_gogo_protobuf_proto.Unmarshal(dAtA, msg); err != nil { 363 | panic(err) 364 | } 365 | if err := p.VerboseEqual(msg); err != nil { 366 | t.Fatalf("%#v !VerboseEqual %#v, since %v", msg, p, err) 367 | } 368 | } 369 | func TestPBLinkGoString(t *testing.T) { 370 | popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) 371 | p := NewPopulatedPBLink(popr, false) 372 | s1 := p.GoString() 373 | s2 := fmt.Sprintf("%#v", p) 374 | if s1 != s2 { 375 | t.Fatalf("GoString want %v got %v", s1, s2) 376 | } 377 | _, err := go_parser.ParseExpr(s1) 378 | if err != nil { 379 | t.Fatal(err) 380 | } 381 | } 382 | func TestPBNodeGoString(t *testing.T) { 383 | popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) 384 | p := NewPopulatedPBNode(popr, false) 385 | s1 := p.GoString() 386 | s2 := fmt.Sprintf("%#v", p) 387 | if s1 != s2 { 388 | t.Fatalf("GoString want %v got %v", s1, s2) 389 | } 390 | _, err := go_parser.ParseExpr(s1) 391 | if err != nil { 392 | t.Fatal(err) 393 | } 394 | } 395 | func TestPBLinkSize(t *testing.T) { 396 | seed := time.Now().UnixNano() 397 | popr := math_rand.New(math_rand.NewSource(seed)) 398 | p := NewPopulatedPBLink(popr, true) 399 | size2 := github_com_gogo_protobuf_proto.Size(p) 400 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) 401 | if err != nil { 402 | t.Fatalf("seed = %d, err = %v", seed, err) 403 | } 404 | size := p.Size() 405 | if len(dAtA) != size { 406 | t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) 407 | } 408 | if size2 != size { 409 | t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) 410 | } 411 | size3 := github_com_gogo_protobuf_proto.Size(p) 412 | if size3 != size { 413 | t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) 414 | } 415 | } 416 | 417 | func BenchmarkPBLinkSize(b *testing.B) { 418 | popr := math_rand.New(math_rand.NewSource(616)) 419 | total := 0 420 | pops := make([]*PBLink, 1000) 421 | for i := 0; i < 1000; i++ { 422 | pops[i] = NewPopulatedPBLink(popr, false) 423 | } 424 | b.ResetTimer() 425 | for i := 0; i < b.N; i++ { 426 | total += pops[i%1000].Size() 427 | } 428 | b.SetBytes(int64(total / b.N)) 429 | } 430 | 431 | func TestPBNodeSize(t *testing.T) { 432 | seed := time.Now().UnixNano() 433 | popr := math_rand.New(math_rand.NewSource(seed)) 434 | p := NewPopulatedPBNode(popr, true) 435 | size2 := github_com_gogo_protobuf_proto.Size(p) 436 | dAtA, err := github_com_gogo_protobuf_proto.Marshal(p) 437 | if err != nil { 438 | t.Fatalf("seed = %d, err = %v", seed, err) 439 | } 440 | size := p.Size() 441 | if len(dAtA) != size { 442 | t.Errorf("seed = %d, size %v != marshalled size %v", seed, size, len(dAtA)) 443 | } 444 | if size2 != size { 445 | t.Errorf("seed = %d, size %v != before marshal proto.Size %v", seed, size, size2) 446 | } 447 | size3 := github_com_gogo_protobuf_proto.Size(p) 448 | if size3 != size { 449 | t.Errorf("seed = %d, size %v != after marshal proto.Size %v", seed, size, size3) 450 | } 451 | } 452 | 453 | func BenchmarkPBNodeSize(b *testing.B) { 454 | popr := math_rand.New(math_rand.NewSource(616)) 455 | total := 0 456 | pops := make([]*PBNode, 1000) 457 | for i := 0; i < 1000; i++ { 458 | pops[i] = NewPopulatedPBNode(popr, false) 459 | } 460 | b.ResetTimer() 461 | for i := 0; i < b.N; i++ { 462 | total += pops[i%1000].Size() 463 | } 464 | b.SetBytes(int64(total / b.N)) 465 | } 466 | 467 | func TestPBLinkStringer(t *testing.T) { 468 | popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) 469 | p := NewPopulatedPBLink(popr, false) 470 | s1 := p.String() 471 | s2 := fmt.Sprintf("%v", p) 472 | if s1 != s2 { 473 | t.Fatalf("String want %v got %v", s1, s2) 474 | } 475 | } 476 | func TestPBNodeStringer(t *testing.T) { 477 | popr := math_rand.New(math_rand.NewSource(time.Now().UnixNano())) 478 | p := NewPopulatedPBNode(popr, false) 479 | s1 := p.String() 480 | s2 := fmt.Sprintf("%v", p) 481 | if s1 != s2 { 482 | t.Fatalf("String want %v got %v", s1, s2) 483 | } 484 | } 485 | 486 | //These tests are generated by github.com/gogo/protobuf/plugin/testgen 487 | -------------------------------------------------------------------------------- /pb/stability_test.go: -------------------------------------------------------------------------------- 1 | package merkledag_pb 2 | 3 | import ( 4 | bytes "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestStability(t *testing.T) { 9 | correct := []byte("\x12\x87\x01\n;\x81\x869\xacH\xa4Ư\xa2\xf1X\x1a\x8b\x95%\xe2\x0f\xdah\x92\u007f+/\xf86\xf75x\xdb\x0f\xa5L)\xf7\xfd\x92\x8d\x92\xcaC\xf1\x93\xde\xe4\u007fY\x15I\xf5\x97\xa8\x11\xc8\xfag\xab\x03\x1e\xbd\x12B|CMw`mHq{>?|vd{0F7>8m[C`HSg3UcXmGs-qp-z6{Kc.tGX->H07\x18\xeaئ\xd0\b\x129\x121LZ3,V9jnmk^veYEV71EMLt9;6]}bnkU2e7GXmqisoCPV0C+ni\x18Ա\xfe\x8b\f\x12u\n'\xf2#\xc1\xc0nQ\xf9\xb5\x19\x80\xcd\xf8\x06k1\xf6#\x84\x1c\xb6\xbf\xeaY\x9b\xd8O\x84\x04\xdbKq\xe4\xae\xf2\xd6\xe9*\x16B\x12D[gVeg4=t}EGSu82+dmgvQ+Tr>_sLUJ|iZ[P2y2T67ilvEikK}\\iru?IF?mVS[Mv9KG8+\x18\x92\xa0\xf9\xa1\n\n\x11?̎\v\x06ѣ\x80nH\x12\x00\xa7\xd2w͝") 10 | n := new(PBNode) 11 | err := n.Unmarshal(correct) 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | d, err := n.Marshal() 16 | if err != nil { 17 | t.Fatal(err) 18 | } 19 | if !bytes.Equal(d, correct) { 20 | t.Logf("%q", d) 21 | t.Fatal("protobuf not stable") 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /pb/upgrade_check.go: -------------------------------------------------------------------------------- 1 | package merkledag_pb 2 | 3 | // Make sure the user doesn't upgrade this package! 4 | // This will fail to build if the user does. 5 | const _ = DoNotUpgradeFileEverItWillChangeYourHashes 6 | -------------------------------------------------------------------------------- /prime.go: -------------------------------------------------------------------------------- 1 | package merkledag 2 | 3 | import ( 4 | dagpb "github.com/ipld/go-codec-dagpb" 5 | "github.com/ipld/go-ipld-prime" 6 | ) 7 | 8 | // Protonode was originally implemented as a go-ipld-format node, and included 9 | // functionality that does not fit well into the model for go-ipld-prime, namely 10 | // the ability ot modify the node in place. 11 | 12 | // In order to support the go-ipld-prime interface, all of these prime methods 13 | // serialize and rebuild the go-ipld-prime node as needed, so that it remains up 14 | // to date with mutations made via the add/remove link methods 15 | 16 | // Kind returns a value from the Kind enum describing what the 17 | // essential serializable kind of this node is (map, list, integer, etc). 18 | // Most other handling of a node requires first switching upon the kind. 19 | func (n *ProtoNode) Kind() ipld.Kind { 20 | _, _ = n.EncodeProtobuf(false) 21 | return n.encoded.Kind() 22 | } 23 | 24 | // LookupByString looks up a child object in this node and returns it. 25 | // The returned Node may be any of the Kind: 26 | // a primitive (string, int64, etc), a map, a list, or a link. 27 | // 28 | // If the Kind of this Node is not Kind_Map, a nil node and an error 29 | // will be returned. 30 | // 31 | // If the key does not exist, a nil node and an error will be returned. 32 | func (n *ProtoNode) LookupByString(key string) (ipld.Node, error) { 33 | _, err := n.EncodeProtobuf(false) 34 | if err != nil { 35 | return nil, err 36 | } 37 | return n.encoded.LookupByString(key) 38 | } 39 | 40 | // LookupByNode is the equivalent of LookupByString, but takes a reified Node 41 | // as a parameter instead of a plain string. 42 | // This mechanism is useful if working with typed maps (if the key types 43 | // have constraints, and you already have a reified `schema.TypedNode` value, 44 | // using that value can save parsing and validation costs); 45 | // and may simply be convenient if you already have a Node value in hand. 46 | // 47 | // (When writing generic functions over Node, a good rule of thumb is: 48 | // when handling a map, check for `schema.TypedNode`, and in this case prefer 49 | // the LookupByNode(Node) method; otherwise, favor LookupByString; typically 50 | // implementations will have their fastest paths thusly.) 51 | func (n *ProtoNode) LookupByNode(key ipld.Node) (ipld.Node, error) { 52 | _, err := n.EncodeProtobuf(false) 53 | if err != nil { 54 | return nil, err 55 | } 56 | return n.encoded.LookupByNode(key) 57 | } 58 | 59 | // LookupByIndex is the equivalent of LookupByString but for indexing into a list. 60 | // As with LookupByString, the returned Node may be any of the Kind: 61 | // a primitive (string, int64, etc), a map, a list, or a link. 62 | // 63 | // If the Kind of this Node is not Kind_List, a nil node and an error 64 | // will be returned. 65 | // 66 | // If idx is out of range, a nil node and an error will be returned. 67 | func (n *ProtoNode) LookupByIndex(idx int64) (ipld.Node, error) { 68 | _, err := n.EncodeProtobuf(false) 69 | if err != nil { 70 | return nil, err 71 | } 72 | return n.encoded.LookupByIndex(idx) 73 | } 74 | 75 | // LookupBySegment is will act as either LookupByString or LookupByIndex, 76 | // whichever is contextually appropriate. 77 | // 78 | // Using LookupBySegment may imply an "atoi" conversion if used on a list node, 79 | // or an "itoa" conversion if used on a map node. If an "itoa" conversion 80 | // takes place, it may error, and this method may return that error. 81 | func (n *ProtoNode) LookupBySegment(seg ipld.PathSegment) (ipld.Node, error) { 82 | _, err := n.EncodeProtobuf(false) 83 | if err != nil { 84 | return nil, err 85 | } 86 | return n.encoded.LookupBySegment(seg) 87 | } 88 | 89 | // Note that when using codegenerated types, there may be a fifth variant 90 | // of lookup method on maps: `Get($GeneratedTypeKey) $GeneratedTypeValue`! 91 | // MapIterator returns an iterator which yields key-value pairs 92 | // traversing the node. 93 | // If the node kind is anything other than a map, nil will be returned. 94 | // 95 | // The iterator will yield every entry in the map; that is, it 96 | // can be expected that itr.Next will be called node.Length times 97 | // before itr.Done becomes true. 98 | func (n *ProtoNode) MapIterator() ipld.MapIterator { 99 | _, _ = n.EncodeProtobuf(false) 100 | return n.encoded.MapIterator() 101 | } 102 | 103 | // ListIterator returns an iterator which yields key-value pairs 104 | // traversing the node. 105 | // If the node kind is anything other than a list, nil will be returned. 106 | // 107 | // The iterator will yield every entry in the list; that is, it 108 | // can be expected that itr.Next will be called node.Length times 109 | // before itr.Done becomes true. 110 | func (n *ProtoNode) ListIterator() ipld.ListIterator { 111 | _, _ = n.EncodeProtobuf(false) 112 | return n.encoded.ListIterator() 113 | } 114 | 115 | // Length returns the length of a list, or the number of entries in a map, 116 | // or -1 if the node is not of list nor map kind. 117 | func (n *ProtoNode) Length() int64 { 118 | _, _ = n.EncodeProtobuf(false) 119 | return n.encoded.Length() 120 | } 121 | 122 | // Absent nodes are returned when traversing a struct field that is 123 | // defined by a schema but unset in the data. (Absent nodes are not 124 | // possible otherwise; you'll only see them from `schema.TypedNode`.) 125 | // The absent flag is necessary so iterating over structs can 126 | // unambiguously make the distinction between values that are 127 | // present-and-null versus values that are absent. 128 | // 129 | // Absent nodes respond to `Kind()` as `ipld.Kind_Null`, 130 | // for lack of any better descriptive value; you should therefore 131 | // always check IsAbsent rather than just a switch on kind 132 | // when it may be important to handle absent values distinctly. 133 | func (n *ProtoNode) IsAbsent() bool { 134 | _, _ = n.EncodeProtobuf(false) 135 | return n.encoded.IsAbsent() 136 | } 137 | 138 | func (n *ProtoNode) IsNull() bool { 139 | _, _ = n.EncodeProtobuf(false) 140 | return n.encoded.IsNull() 141 | } 142 | 143 | func (n *ProtoNode) AsBool() (bool, error) { 144 | _, err := n.EncodeProtobuf(false) 145 | if err != nil { 146 | return false, err 147 | } 148 | return n.encoded.AsBool() 149 | } 150 | 151 | func (n *ProtoNode) AsInt() (int64, error) { 152 | _, err := n.EncodeProtobuf(false) 153 | if err != nil { 154 | return 0, err 155 | } 156 | return n.encoded.AsInt() 157 | } 158 | 159 | func (n *ProtoNode) AsFloat() (float64, error) { 160 | _, err := n.EncodeProtobuf(false) 161 | if err != nil { 162 | return 0, err 163 | } 164 | return n.encoded.AsFloat() 165 | } 166 | 167 | func (n *ProtoNode) AsString() (string, error) { 168 | _, err := n.EncodeProtobuf(false) 169 | if err != nil { 170 | return "", err 171 | } 172 | return n.encoded.AsString() 173 | } 174 | 175 | func (n *ProtoNode) AsBytes() ([]byte, error) { 176 | _, err := n.EncodeProtobuf(false) 177 | if err != nil { 178 | return nil, err 179 | } 180 | return n.encoded.AsBytes() 181 | } 182 | 183 | func (n *ProtoNode) AsLink() (ipld.Link, error) { 184 | _, err := n.EncodeProtobuf(false) 185 | if err != nil { 186 | return nil, err 187 | } 188 | return n.encoded.AsLink() 189 | } 190 | 191 | // Prototype returns a NodePrototype which can describe some properties of this node's implementation, 192 | // and also be used to get a NodeBuilder, 193 | // which can be use to create new nodes with the same implementation as this one. 194 | // 195 | // For typed nodes, the NodePrototype will also implement schema.Type. 196 | // 197 | // For Advanced Data Layouts, the NodePrototype will encapsulate any additional 198 | // parameters and configuration of the ADL, and will also (usually) 199 | // implement NodePrototypeSupportingAmend. 200 | // 201 | // Calling this method should not cause an allocation. 202 | func (n *ProtoNode) Prototype() ipld.NodePrototype { 203 | return dagpb.Type.PBNode 204 | } 205 | 206 | var _ ipld.Node = &ProtoNode{} 207 | -------------------------------------------------------------------------------- /raw.go: -------------------------------------------------------------------------------- 1 | package merkledag 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | 7 | blocks "github.com/ipfs/go-block-format" 8 | u "github.com/ipfs/go-ipfs-util" 9 | legacy "github.com/ipfs/go-ipld-legacy" 10 | ipld "github.com/ipld/go-ipld-prime" 11 | basicnode "github.com/ipld/go-ipld-prime/node/basic" 12 | 13 | cid "github.com/ipfs/go-cid" 14 | format "github.com/ipfs/go-ipld-format" 15 | ) 16 | 17 | // RawNode represents a node which only contains data. 18 | type RawNode struct { 19 | blocks.Block 20 | 21 | // Always a node/basic Bytes. 22 | // We can't reference a specific type, as it's not exposed there. 23 | // If we find that the interface indirection really matters, 24 | // then we could possibly use dagpb.Bytes. 25 | ipld.Node 26 | } 27 | 28 | var _ legacy.UniversalNode = &RawNode{} 29 | 30 | // NewRawNode creates a RawNode using the default sha2-256 hash function. 31 | func NewRawNode(data []byte) *RawNode { 32 | h := u.Hash(data) 33 | c := cid.NewCidV1(cid.Raw, h) 34 | blk, _ := blocks.NewBlockWithCid(data, c) 35 | return &RawNode{blk, basicnode.NewBytes(data)} 36 | } 37 | 38 | // DecodeRawBlock is a block decoder for raw IPLD nodes conforming to `node.DecodeBlockFunc`. 39 | func DecodeRawBlock(block blocks.Block) (format.Node, error) { 40 | if block.Cid().Type() != cid.Raw { 41 | return nil, fmt.Errorf("raw nodes cannot be decoded from non-raw blocks: %d", block.Cid().Type()) 42 | } 43 | // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is. 44 | return &RawNode{block, basicnode.NewBytes(block.RawData())}, nil 45 | } 46 | 47 | var _ format.DecodeBlockFunc = DecodeRawBlock 48 | 49 | // NewRawNodeWPrefix creates a RawNode using the provided cid builder 50 | func NewRawNodeWPrefix(data []byte, builder cid.Builder) (*RawNode, error) { 51 | builder = builder.WithCodec(cid.Raw) 52 | c, err := builder.Sum(data) 53 | if err != nil { 54 | return nil, err 55 | } 56 | blk, err := blocks.NewBlockWithCid(data, c) 57 | if err != nil { 58 | return nil, err 59 | } 60 | // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is. 61 | return &RawNode{blk, basicnode.NewBytes(data)}, nil 62 | } 63 | 64 | // Links returns nil. 65 | func (rn *RawNode) Links() []*format.Link { 66 | return nil 67 | } 68 | 69 | // ResolveLink returns an error. 70 | func (rn *RawNode) ResolveLink(path []string) (*format.Link, []string, error) { 71 | return nil, nil, ErrLinkNotFound 72 | } 73 | 74 | // Resolve returns an error. 75 | func (rn *RawNode) Resolve(path []string) (interface{}, []string, error) { 76 | return nil, nil, ErrLinkNotFound 77 | } 78 | 79 | // Tree returns nil. 80 | func (rn *RawNode) Tree(p string, depth int) []string { 81 | return nil 82 | } 83 | 84 | // Copy performs a deep copy of this node and returns it as an format.Node 85 | func (rn *RawNode) Copy() format.Node { 86 | copybuf := make([]byte, len(rn.RawData())) 87 | copy(copybuf, rn.RawData()) 88 | nblk, err := blocks.NewBlockWithCid(rn.RawData(), rn.Cid()) 89 | if err != nil { 90 | // programmer error 91 | panic("failure attempting to clone raw block: " + err.Error()) 92 | } 93 | // Once you "share" a block, it should be immutable. Therefore, we can just use this block as-is. 94 | return &RawNode{nblk, basicnode.NewBytes(nblk.RawData())} 95 | } 96 | 97 | // Size returns the size of this node 98 | func (rn *RawNode) Size() (uint64, error) { 99 | return uint64(len(rn.RawData())), nil 100 | } 101 | 102 | // Stat returns some Stats about this node. 103 | func (rn *RawNode) Stat() (*format.NodeStat, error) { 104 | return &format.NodeStat{ 105 | Hash: rn.Cid().String(), 106 | NumLinks: 0, 107 | BlockSize: len(rn.RawData()), 108 | LinksSize: 0, 109 | CumulativeSize: len(rn.RawData()), 110 | DataSize: len(rn.RawData()), 111 | }, nil 112 | } 113 | 114 | // MarshalJSON is required for our "ipfs dag" commands. 115 | func (rn *RawNode) MarshalJSON() ([]byte, error) { 116 | return json.Marshal(string(rn.RawData())) 117 | } 118 | 119 | func RawNodeConverter(b blocks.Block, nd ipld.Node) (legacy.UniversalNode, error) { 120 | if nd.Kind() != ipld.Kind_Bytes { 121 | return nil, ErrNotRawNode 122 | } 123 | return &RawNode{b, nd}, nil 124 | } 125 | 126 | var _ legacy.UniversalNode = (*RawNode)(nil) 127 | -------------------------------------------------------------------------------- /readonly.go: -------------------------------------------------------------------------------- 1 | package merkledag 2 | 3 | import ( 4 | "fmt" 5 | 6 | ipld "github.com/ipfs/go-ipld-format" 7 | ) 8 | 9 | // ErrReadOnly is used when a read-only datastructure is written to. 10 | var ErrReadOnly = fmt.Errorf("cannot write to readonly DAGService") 11 | 12 | // NewReadOnlyDagService takes a NodeGetter, and returns a full DAGService 13 | // implementation that returns ErrReadOnly when its 'write' methods are 14 | // invoked. 15 | func NewReadOnlyDagService(ng ipld.NodeGetter) ipld.DAGService { 16 | return &ComboService{ 17 | Read: ng, 18 | Write: &ErrorService{ErrReadOnly}, 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /readonly_test.go: -------------------------------------------------------------------------------- 1 | package merkledag_test 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | . "github.com/ipfs/go-merkledag" 8 | dstest "github.com/ipfs/go-merkledag/test" 9 | 10 | cid "github.com/ipfs/go-cid" 11 | ipld "github.com/ipfs/go-ipld-format" 12 | ) 13 | 14 | func TestReadonlyProperties(t *testing.T) { 15 | ds := dstest.Mock() 16 | ro := NewReadOnlyDagService(ds) 17 | 18 | ctx := context.Background() 19 | nds := []ipld.Node{ 20 | NewRawNode([]byte("foo1")), 21 | NewRawNode([]byte("foo2")), 22 | NewRawNode([]byte("foo3")), 23 | NewRawNode([]byte("foo4")), 24 | } 25 | cids := []cid.Cid{ 26 | nds[0].Cid(), 27 | nds[1].Cid(), 28 | nds[2].Cid(), 29 | nds[3].Cid(), 30 | } 31 | 32 | // add to the actual underlying datastore 33 | if err := ds.Add(ctx, nds[2]); err != nil { 34 | t.Fatal(err) 35 | } 36 | if err := ds.Add(ctx, nds[3]); err != nil { 37 | t.Fatal(err) 38 | } 39 | 40 | if err := ro.Add(ctx, nds[0]); err != ErrReadOnly { 41 | t.Fatal("expected ErrReadOnly") 42 | } 43 | if err := ro.Add(ctx, nds[2]); err != ErrReadOnly { 44 | t.Fatal("expected ErrReadOnly") 45 | } 46 | 47 | if err := ro.AddMany(ctx, nds[0:1]); err != ErrReadOnly { 48 | t.Fatal("expected ErrReadOnly") 49 | } 50 | 51 | if err := ro.Remove(ctx, cids[3]); err != ErrReadOnly { 52 | t.Fatal("expected ErrReadOnly") 53 | } 54 | if err := ro.RemoveMany(ctx, cids[1:2]); err != ErrReadOnly { 55 | t.Fatal("expected ErrReadOnly") 56 | } 57 | 58 | if _, err := ro.Get(ctx, cids[0]); !ipld.IsNotFound(err) { 59 | t.Fatal("expected ErrNotFound") 60 | } 61 | if _, err := ro.Get(ctx, cids[3]); err != nil { 62 | t.Fatal(err) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /rwservice.go: -------------------------------------------------------------------------------- 1 | package merkledag 2 | 3 | import ( 4 | "context" 5 | 6 | cid "github.com/ipfs/go-cid" 7 | ipld "github.com/ipfs/go-ipld-format" 8 | ) 9 | 10 | // ComboService implements ipld.DAGService, using 'Read' for all fetch methods, 11 | // and 'Write' for all methods that add new objects. 12 | type ComboService struct { 13 | Read ipld.NodeGetter 14 | Write ipld.DAGService 15 | } 16 | 17 | var _ ipld.DAGService = (*ComboService)(nil) 18 | 19 | // Add writes a new node using the Write DAGService. 20 | func (cs *ComboService) Add(ctx context.Context, nd ipld.Node) error { 21 | return cs.Write.Add(ctx, nd) 22 | } 23 | 24 | // AddMany adds nodes using the Write DAGService. 25 | func (cs *ComboService) AddMany(ctx context.Context, nds []ipld.Node) error { 26 | return cs.Write.AddMany(ctx, nds) 27 | } 28 | 29 | // Get fetches a node using the Read DAGService. 30 | func (cs *ComboService) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) { 31 | return cs.Read.Get(ctx, c) 32 | } 33 | 34 | // GetMany fetches nodes using the Read DAGService. 35 | func (cs *ComboService) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption { 36 | return cs.Read.GetMany(ctx, cids) 37 | } 38 | 39 | // Remove deletes a node using the Write DAGService. 40 | func (cs *ComboService) Remove(ctx context.Context, c cid.Cid) error { 41 | return cs.Write.Remove(ctx, c) 42 | } 43 | 44 | // RemoveMany deletes nodes using the Write DAGService. 45 | func (cs *ComboService) RemoveMany(ctx context.Context, cids []cid.Cid) error { 46 | return cs.Write.RemoveMany(ctx, cids) 47 | } 48 | -------------------------------------------------------------------------------- /session.go: -------------------------------------------------------------------------------- 1 | package merkledag 2 | 3 | import ( 4 | "context" 5 | 6 | ipld "github.com/ipfs/go-ipld-format" 7 | ) 8 | 9 | // SessionMaker is an object that can generate a new fetching session. 10 | type SessionMaker interface { 11 | Session(context.Context) ipld.NodeGetter 12 | } 13 | 14 | // NewSession returns a session backed NodeGetter if the given NodeGetter 15 | // implements SessionMaker. 16 | func NewSession(ctx context.Context, g ipld.NodeGetter) ipld.NodeGetter { 17 | if sm, ok := g.(SessionMaker); ok { 18 | return sm.Session(ctx) 19 | } 20 | return g 21 | } 22 | -------------------------------------------------------------------------------- /test/utils.go: -------------------------------------------------------------------------------- 1 | package mdutils 2 | 3 | import ( 4 | dag "github.com/ipfs/go-merkledag" 5 | 6 | bsrv "github.com/ipfs/go-blockservice" 7 | ds "github.com/ipfs/go-datastore" 8 | dssync "github.com/ipfs/go-datastore/sync" 9 | blockstore "github.com/ipfs/go-ipfs-blockstore" 10 | offline "github.com/ipfs/go-ipfs-exchange-offline" 11 | ipld "github.com/ipfs/go-ipld-format" 12 | ) 13 | 14 | // Mock returns a new thread-safe, mock DAGService. 15 | func Mock() ipld.DAGService { 16 | return dag.NewDAGService(Bserv()) 17 | } 18 | 19 | // Bserv returns a new, thread-safe, mock BlockService. 20 | func Bserv() bsrv.BlockService { 21 | bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) 22 | return bsrv.New(bstore, offline.Exchange(bstore)) 23 | } 24 | -------------------------------------------------------------------------------- /traverse/traverse.go: -------------------------------------------------------------------------------- 1 | // Package traverse provides merkledag traversal functions 2 | package traverse 3 | 4 | import ( 5 | "context" 6 | "errors" 7 | 8 | ipld "github.com/ipfs/go-ipld-format" 9 | ) 10 | 11 | // Order is an identifier for traversal algorithm orders 12 | type Order int 13 | 14 | // These constants define different traversing methods 15 | const ( 16 | // DFSPre defines depth-first pre-order 17 | DFSPre Order = iota 18 | // DFSPost defines depth-first post-order 19 | DFSPost 20 | // BFS defines breadth-first order 21 | BFS 22 | ) 23 | 24 | // Options specifies a series of traversal options 25 | type Options struct { 26 | DAG ipld.NodeGetter // the dagservice to fetch nodes 27 | Order Order // what order to traverse in 28 | Func Func // the function to perform at each step 29 | ErrFunc ErrFunc // see ErrFunc. Optional 30 | 31 | SkipDuplicates bool // whether to skip duplicate nodes 32 | } 33 | 34 | // State is a current traversal state 35 | type State struct { 36 | Node ipld.Node 37 | Depth int 38 | } 39 | 40 | type traversal struct { 41 | opts Options 42 | seen map[string]struct{} 43 | } 44 | 45 | func (t *traversal) shouldSkip(n ipld.Node) (bool, error) { 46 | if t.opts.SkipDuplicates { 47 | k := n.Cid() 48 | if _, found := t.seen[k.KeyString()]; found { 49 | return true, nil 50 | } 51 | t.seen[k.KeyString()] = struct{}{} 52 | } 53 | 54 | return false, nil 55 | } 56 | 57 | func (t *traversal) callFunc(next State) error { 58 | return t.opts.Func(next) 59 | } 60 | 61 | // getNode returns the node for link. If it return an error, 62 | // stop processing. if it returns a nil node, just skip it. 63 | // 64 | // the error handling is a little complicated. 65 | func (t *traversal) getNode(link *ipld.Link) (ipld.Node, error) { 66 | 67 | getNode := func(l *ipld.Link) (ipld.Node, error) { 68 | next, err := l.GetNode(context.TODO(), t.opts.DAG) 69 | if err != nil { 70 | return nil, err 71 | } 72 | 73 | skip, err := t.shouldSkip(next) 74 | if skip { 75 | next = nil 76 | } 77 | return next, err 78 | } 79 | 80 | next, err := getNode(link) 81 | if err != nil && t.opts.ErrFunc != nil { // attempt recovery. 82 | err = t.opts.ErrFunc(err) 83 | next = nil // skip regardless 84 | } 85 | return next, err 86 | } 87 | 88 | // Func is the type of the function called for each dag.Node visited by Traverse. 89 | // The traversal argument contains the current traversal state. 90 | // If an error is returned, processing stops. 91 | type Func func(current State) error 92 | 93 | // ErrFunc is provided to handle problems when walking to the Node. Traverse 94 | // will call ErrFunc with the error encountered. ErrFunc can decide how to 95 | // handle that error, and return an error back to Traversal with how to proceed: 96 | // - nil - skip the Node and its children, but continue processing 97 | // - all other errors halt processing immediately. 98 | // 99 | // If ErrFunc is nil, Traversal will stop, as if: 100 | // 101 | // opts.ErrFunc = func(err error) { return err } 102 | type ErrFunc func(err error) error 103 | 104 | // Traverse initiates a DAG traversal with the given options starting at 105 | // the given root. 106 | func Traverse(root ipld.Node, o Options) error { 107 | t := traversal{ 108 | opts: o, 109 | seen: map[string]struct{}{}, 110 | } 111 | 112 | state := State{ 113 | Node: root, 114 | Depth: 0, 115 | } 116 | 117 | switch o.Order { 118 | default: 119 | return dfsPreTraverse(state, &t) 120 | case DFSPre: 121 | return dfsPreTraverse(state, &t) 122 | case DFSPost: 123 | return dfsPostTraverse(state, &t) 124 | case BFS: 125 | return bfsTraverse(state, &t) 126 | } 127 | } 128 | 129 | type dfsFunc func(state State, t *traversal) error 130 | 131 | func dfsPreTraverse(state State, t *traversal) error { 132 | if err := t.callFunc(state); err != nil { 133 | return err 134 | } 135 | return dfsDescend(dfsPreTraverse, state, t) 136 | } 137 | 138 | func dfsPostTraverse(state State, t *traversal) error { 139 | if err := dfsDescend(dfsPostTraverse, state, t); err != nil { 140 | return err 141 | } 142 | return t.callFunc(state) 143 | } 144 | 145 | func dfsDescend(df dfsFunc, curr State, t *traversal) error { 146 | for _, l := range curr.Node.Links() { 147 | node, err := t.getNode(l) 148 | if err != nil { 149 | return err 150 | } 151 | if node == nil { // skip 152 | continue 153 | } 154 | 155 | next := State{ 156 | Node: node, 157 | Depth: curr.Depth + 1, 158 | } 159 | if err := df(next, t); err != nil { 160 | return err 161 | } 162 | } 163 | return nil 164 | } 165 | 166 | func bfsTraverse(root State, t *traversal) error { 167 | 168 | if skip, err := t.shouldSkip(root.Node); skip || err != nil { 169 | return err 170 | } 171 | 172 | var q queue 173 | q.enq(root) 174 | for q.len() > 0 { 175 | curr := q.deq() 176 | if curr.Node == nil { 177 | return errors.New("failed to dequeue though queue not empty") 178 | } 179 | 180 | // call user's func 181 | if err := t.callFunc(curr); err != nil { 182 | return err 183 | } 184 | 185 | for _, l := range curr.Node.Links() { 186 | node, err := t.getNode(l) 187 | if err != nil { 188 | return err 189 | } 190 | if node == nil { // skip 191 | continue 192 | } 193 | 194 | q.enq(State{ 195 | Node: node, 196 | Depth: curr.Depth + 1, 197 | }) 198 | } 199 | } 200 | return nil 201 | } 202 | 203 | type queue struct { 204 | s []State 205 | } 206 | 207 | func (q *queue) enq(n State) { 208 | q.s = append(q.s, n) 209 | } 210 | 211 | func (q *queue) deq() State { 212 | if len(q.s) < 1 { 213 | return State{} 214 | } 215 | n := q.s[0] 216 | q.s = q.s[1:] 217 | return n 218 | } 219 | 220 | func (q *queue) len() int { 221 | return len(q.s) 222 | } 223 | -------------------------------------------------------------------------------- /traverse/traverse_test.go: -------------------------------------------------------------------------------- 1 | package traverse 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "testing" 8 | 9 | mdag "github.com/ipfs/go-merkledag" 10 | mdagtest "github.com/ipfs/go-merkledag/test" 11 | 12 | ipld "github.com/ipfs/go-ipld-format" 13 | ) 14 | 15 | func TestDFSPreNoSkip(t *testing.T) { 16 | ds := mdagtest.Mock() 17 | opts := Options{Order: DFSPre, DAG: ds} 18 | 19 | testWalkOutputs(t, newFan(t, ds), opts, []byte(` 20 | 0 /a 21 | 1 /a/aa 22 | 1 /a/ab 23 | 1 /a/ac 24 | 1 /a/ad 25 | `)) 26 | 27 | testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` 28 | 0 /a 29 | 1 /a/aa 30 | 2 /a/aa/aaa 31 | 3 /a/aa/aaa/aaaa 32 | 4 /a/aa/aaa/aaaa/aaaaa 33 | `)) 34 | 35 | testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` 36 | 0 /a 37 | 1 /a/aa 38 | 2 /a/aa/aaa 39 | 2 /a/aa/aab 40 | 1 /a/ab 41 | 2 /a/ab/aba 42 | 2 /a/ab/abb 43 | `)) 44 | 45 | testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` 46 | 0 /a 47 | 1 /a/aa 48 | 2 /a/aa/aaa 49 | 3 /a/aa/aaa/aaaa 50 | 4 /a/aa/aaa/aaaa/aaaaa 51 | 4 /a/aa/aaa/aaaa/aaaaa 52 | 3 /a/aa/aaa/aaaa 53 | 4 /a/aa/aaa/aaaa/aaaaa 54 | 4 /a/aa/aaa/aaaa/aaaaa 55 | 2 /a/aa/aaa 56 | 3 /a/aa/aaa/aaaa 57 | 4 /a/aa/aaa/aaaa/aaaaa 58 | 4 /a/aa/aaa/aaaa/aaaaa 59 | 3 /a/aa/aaa/aaaa 60 | 4 /a/aa/aaa/aaaa/aaaaa 61 | 4 /a/aa/aaa/aaaa/aaaaa 62 | 1 /a/aa 63 | 2 /a/aa/aaa 64 | 3 /a/aa/aaa/aaaa 65 | 4 /a/aa/aaa/aaaa/aaaaa 66 | 4 /a/aa/aaa/aaaa/aaaaa 67 | 3 /a/aa/aaa/aaaa 68 | 4 /a/aa/aaa/aaaa/aaaaa 69 | 4 /a/aa/aaa/aaaa/aaaaa 70 | 2 /a/aa/aaa 71 | 3 /a/aa/aaa/aaaa 72 | 4 /a/aa/aaa/aaaa/aaaaa 73 | 4 /a/aa/aaa/aaaa/aaaaa 74 | 3 /a/aa/aaa/aaaa 75 | 4 /a/aa/aaa/aaaa/aaaaa 76 | 4 /a/aa/aaa/aaaa/aaaaa 77 | `)) 78 | } 79 | 80 | func TestDFSPreSkip(t *testing.T) { 81 | ds := mdagtest.Mock() 82 | opts := Options{Order: DFSPre, SkipDuplicates: true, DAG: ds} 83 | 84 | testWalkOutputs(t, newFan(t, ds), opts, []byte(` 85 | 0 /a 86 | 1 /a/aa 87 | 1 /a/ab 88 | 1 /a/ac 89 | 1 /a/ad 90 | `)) 91 | 92 | testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` 93 | 0 /a 94 | 1 /a/aa 95 | 2 /a/aa/aaa 96 | 3 /a/aa/aaa/aaaa 97 | 4 /a/aa/aaa/aaaa/aaaaa 98 | `)) 99 | 100 | testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` 101 | 0 /a 102 | 1 /a/aa 103 | 2 /a/aa/aaa 104 | 2 /a/aa/aab 105 | 1 /a/ab 106 | 2 /a/ab/aba 107 | 2 /a/ab/abb 108 | `)) 109 | 110 | testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` 111 | 0 /a 112 | 1 /a/aa 113 | 2 /a/aa/aaa 114 | 3 /a/aa/aaa/aaaa 115 | 4 /a/aa/aaa/aaaa/aaaaa 116 | `)) 117 | } 118 | 119 | func TestDFSPostNoSkip(t *testing.T) { 120 | ds := mdagtest.Mock() 121 | opts := Options{Order: DFSPost, DAG: ds} 122 | 123 | testWalkOutputs(t, newFan(t, ds), opts, []byte(` 124 | 1 /a/aa 125 | 1 /a/ab 126 | 1 /a/ac 127 | 1 /a/ad 128 | 0 /a 129 | `)) 130 | 131 | testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` 132 | 4 /a/aa/aaa/aaaa/aaaaa 133 | 3 /a/aa/aaa/aaaa 134 | 2 /a/aa/aaa 135 | 1 /a/aa 136 | 0 /a 137 | `)) 138 | 139 | testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` 140 | 2 /a/aa/aaa 141 | 2 /a/aa/aab 142 | 1 /a/aa 143 | 2 /a/ab/aba 144 | 2 /a/ab/abb 145 | 1 /a/ab 146 | 0 /a 147 | `)) 148 | 149 | testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` 150 | 4 /a/aa/aaa/aaaa/aaaaa 151 | 4 /a/aa/aaa/aaaa/aaaaa 152 | 3 /a/aa/aaa/aaaa 153 | 4 /a/aa/aaa/aaaa/aaaaa 154 | 4 /a/aa/aaa/aaaa/aaaaa 155 | 3 /a/aa/aaa/aaaa 156 | 2 /a/aa/aaa 157 | 4 /a/aa/aaa/aaaa/aaaaa 158 | 4 /a/aa/aaa/aaaa/aaaaa 159 | 3 /a/aa/aaa/aaaa 160 | 4 /a/aa/aaa/aaaa/aaaaa 161 | 4 /a/aa/aaa/aaaa/aaaaa 162 | 3 /a/aa/aaa/aaaa 163 | 2 /a/aa/aaa 164 | 1 /a/aa 165 | 4 /a/aa/aaa/aaaa/aaaaa 166 | 4 /a/aa/aaa/aaaa/aaaaa 167 | 3 /a/aa/aaa/aaaa 168 | 4 /a/aa/aaa/aaaa/aaaaa 169 | 4 /a/aa/aaa/aaaa/aaaaa 170 | 3 /a/aa/aaa/aaaa 171 | 2 /a/aa/aaa 172 | 4 /a/aa/aaa/aaaa/aaaaa 173 | 4 /a/aa/aaa/aaaa/aaaaa 174 | 3 /a/aa/aaa/aaaa 175 | 4 /a/aa/aaa/aaaa/aaaaa 176 | 4 /a/aa/aaa/aaaa/aaaaa 177 | 3 /a/aa/aaa/aaaa 178 | 2 /a/aa/aaa 179 | 1 /a/aa 180 | 0 /a 181 | `)) 182 | } 183 | 184 | func TestDFSPostSkip(t *testing.T) { 185 | ds := mdagtest.Mock() 186 | opts := Options{Order: DFSPost, SkipDuplicates: true, DAG: ds} 187 | 188 | testWalkOutputs(t, newFan(t, ds), opts, []byte(` 189 | 1 /a/aa 190 | 1 /a/ab 191 | 1 /a/ac 192 | 1 /a/ad 193 | 0 /a 194 | `)) 195 | 196 | testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` 197 | 4 /a/aa/aaa/aaaa/aaaaa 198 | 3 /a/aa/aaa/aaaa 199 | 2 /a/aa/aaa 200 | 1 /a/aa 201 | 0 /a 202 | `)) 203 | 204 | testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` 205 | 2 /a/aa/aaa 206 | 2 /a/aa/aab 207 | 1 /a/aa 208 | 2 /a/ab/aba 209 | 2 /a/ab/abb 210 | 1 /a/ab 211 | 0 /a 212 | `)) 213 | 214 | testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` 215 | 4 /a/aa/aaa/aaaa/aaaaa 216 | 3 /a/aa/aaa/aaaa 217 | 2 /a/aa/aaa 218 | 1 /a/aa 219 | 0 /a 220 | `)) 221 | } 222 | 223 | func TestBFSNoSkip(t *testing.T) { 224 | ds := mdagtest.Mock() 225 | opts := Options{Order: BFS, DAG: ds} 226 | 227 | testWalkOutputs(t, newFan(t, ds), opts, []byte(` 228 | 0 /a 229 | 1 /a/aa 230 | 1 /a/ab 231 | 1 /a/ac 232 | 1 /a/ad 233 | `)) 234 | 235 | testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` 236 | 0 /a 237 | 1 /a/aa 238 | 2 /a/aa/aaa 239 | 3 /a/aa/aaa/aaaa 240 | 4 /a/aa/aaa/aaaa/aaaaa 241 | `)) 242 | 243 | testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` 244 | 0 /a 245 | 1 /a/aa 246 | 1 /a/ab 247 | 2 /a/aa/aaa 248 | 2 /a/aa/aab 249 | 2 /a/ab/aba 250 | 2 /a/ab/abb 251 | `)) 252 | 253 | testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` 254 | 0 /a 255 | 1 /a/aa 256 | 1 /a/aa 257 | 2 /a/aa/aaa 258 | 2 /a/aa/aaa 259 | 2 /a/aa/aaa 260 | 2 /a/aa/aaa 261 | 3 /a/aa/aaa/aaaa 262 | 3 /a/aa/aaa/aaaa 263 | 3 /a/aa/aaa/aaaa 264 | 3 /a/aa/aaa/aaaa 265 | 3 /a/aa/aaa/aaaa 266 | 3 /a/aa/aaa/aaaa 267 | 3 /a/aa/aaa/aaaa 268 | 3 /a/aa/aaa/aaaa 269 | 4 /a/aa/aaa/aaaa/aaaaa 270 | 4 /a/aa/aaa/aaaa/aaaaa 271 | 4 /a/aa/aaa/aaaa/aaaaa 272 | 4 /a/aa/aaa/aaaa/aaaaa 273 | 4 /a/aa/aaa/aaaa/aaaaa 274 | 4 /a/aa/aaa/aaaa/aaaaa 275 | 4 /a/aa/aaa/aaaa/aaaaa 276 | 4 /a/aa/aaa/aaaa/aaaaa 277 | 4 /a/aa/aaa/aaaa/aaaaa 278 | 4 /a/aa/aaa/aaaa/aaaaa 279 | 4 /a/aa/aaa/aaaa/aaaaa 280 | 4 /a/aa/aaa/aaaa/aaaaa 281 | 4 /a/aa/aaa/aaaa/aaaaa 282 | 4 /a/aa/aaa/aaaa/aaaaa 283 | 4 /a/aa/aaa/aaaa/aaaaa 284 | 4 /a/aa/aaa/aaaa/aaaaa 285 | `)) 286 | } 287 | 288 | func TestBFSSkip(t *testing.T) { 289 | ds := mdagtest.Mock() 290 | opts := Options{Order: BFS, SkipDuplicates: true, DAG: ds} 291 | 292 | testWalkOutputs(t, newFan(t, ds), opts, []byte(` 293 | 0 /a 294 | 1 /a/aa 295 | 1 /a/ab 296 | 1 /a/ac 297 | 1 /a/ad 298 | `)) 299 | 300 | testWalkOutputs(t, newLinkedList(t, ds), opts, []byte(` 301 | 0 /a 302 | 1 /a/aa 303 | 2 /a/aa/aaa 304 | 3 /a/aa/aaa/aaaa 305 | 4 /a/aa/aaa/aaaa/aaaaa 306 | `)) 307 | 308 | testWalkOutputs(t, newBinaryTree(t, ds), opts, []byte(` 309 | 0 /a 310 | 1 /a/aa 311 | 1 /a/ab 312 | 2 /a/aa/aaa 313 | 2 /a/aa/aab 314 | 2 /a/ab/aba 315 | 2 /a/ab/abb 316 | `)) 317 | 318 | testWalkOutputs(t, newBinaryDAG(t, ds), opts, []byte(` 319 | 0 /a 320 | 1 /a/aa 321 | 2 /a/aa/aaa 322 | 3 /a/aa/aaa/aaaa 323 | 4 /a/aa/aaa/aaaa/aaaaa 324 | `)) 325 | } 326 | 327 | func testWalkOutputs(t *testing.T, root ipld.Node, opts Options, expect []byte) { 328 | expect = bytes.TrimLeft(expect, "\n") 329 | 330 | buf := new(bytes.Buffer) 331 | walk := func(current State) error { 332 | s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.(*mdag.ProtoNode).Data()) 333 | t.Logf("walk: %s", s) 334 | buf.Write([]byte(s)) 335 | return nil 336 | } 337 | 338 | opts.Func = walk 339 | if err := Traverse(root, opts); err != nil { 340 | t.Error(err) 341 | return 342 | } 343 | 344 | actual := buf.Bytes() 345 | if !bytes.Equal(actual, expect) { 346 | t.Error("error: outputs differ") 347 | t.Logf("expect:\n%s", expect) 348 | t.Logf("actual:\n%s", actual) 349 | } else { 350 | t.Logf("expect matches actual:\n%s", expect) 351 | } 352 | } 353 | 354 | func newFan(t *testing.T, ds ipld.DAGService) ipld.Node { 355 | a := mdag.NodeWithData([]byte("/a")) 356 | addLink(t, ds, a, child(t, ds, a, "aa")) 357 | addLink(t, ds, a, child(t, ds, a, "ab")) 358 | addLink(t, ds, a, child(t, ds, a, "ac")) 359 | addLink(t, ds, a, child(t, ds, a, "ad")) 360 | return a 361 | } 362 | 363 | func newLinkedList(t *testing.T, ds ipld.DAGService) ipld.Node { 364 | a := mdag.NodeWithData([]byte("/a")) 365 | aa := child(t, ds, a, "aa") 366 | aaa := child(t, ds, aa, "aaa") 367 | aaaa := child(t, ds, aaa, "aaaa") 368 | aaaaa := child(t, ds, aaaa, "aaaaa") 369 | addLink(t, ds, aaaa, aaaaa) 370 | addLink(t, ds, aaa, aaaa) 371 | addLink(t, ds, aa, aaa) 372 | addLink(t, ds, a, aa) 373 | return a 374 | } 375 | 376 | func newBinaryTree(t *testing.T, ds ipld.DAGService) ipld.Node { 377 | a := mdag.NodeWithData([]byte("/a")) 378 | aa := child(t, ds, a, "aa") 379 | ab := child(t, ds, a, "ab") 380 | addLink(t, ds, aa, child(t, ds, aa, "aaa")) 381 | addLink(t, ds, aa, child(t, ds, aa, "aab")) 382 | addLink(t, ds, ab, child(t, ds, ab, "aba")) 383 | addLink(t, ds, ab, child(t, ds, ab, "abb")) 384 | addLink(t, ds, a, aa) 385 | addLink(t, ds, a, ab) 386 | return a 387 | } 388 | 389 | func newBinaryDAG(t *testing.T, ds ipld.DAGService) ipld.Node { 390 | a := mdag.NodeWithData([]byte("/a")) 391 | aa := child(t, ds, a, "aa") 392 | aaa := child(t, ds, aa, "aaa") 393 | aaaa := child(t, ds, aaa, "aaaa") 394 | aaaaa := child(t, ds, aaaa, "aaaaa") 395 | addLink(t, ds, aaaa, aaaaa) 396 | addLink(t, ds, aaaa, aaaaa) 397 | addLink(t, ds, aaa, aaaa) 398 | addLink(t, ds, aaa, aaaa) 399 | addLink(t, ds, aa, aaa) 400 | addLink(t, ds, aa, aaa) 401 | addLink(t, ds, a, aa) 402 | addLink(t, ds, a, aa) 403 | return a 404 | } 405 | 406 | func addLink(t *testing.T, ds ipld.DAGService, a, b ipld.Node) { 407 | to := string(a.(*mdag.ProtoNode).Data()) + "2" + string(b.(*mdag.ProtoNode).Data()) 408 | if err := ds.Add(context.Background(), b); err != nil { 409 | t.Error(err) 410 | } 411 | if err := a.(*mdag.ProtoNode).AddNodeLink(to, b.(*mdag.ProtoNode)); err != nil { 412 | t.Error(err) 413 | } 414 | } 415 | 416 | func child(t *testing.T, ds ipld.DAGService, a ipld.Node, name string) ipld.Node { 417 | return mdag.NodeWithData([]byte(string(a.(*mdag.ProtoNode).Data()) + "/" + name)) 418 | } 419 | -------------------------------------------------------------------------------- /version.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "v0.11.0" 3 | } 4 | --------------------------------------------------------------------------------