├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── escape.go ├── fmtstr.go ├── fmtstr_test.go ├── main.go ├── package.json ├── repo.go ├── spec.md ├── test ├── pack-basic.sh ├── pack-serve.sh └── sharness │ ├── .gitignore │ ├── Makefile │ ├── lib │ ├── install-sharness.sh │ └── test-lib.sh │ ├── t0000-sharness.sh │ ├── t0010-basic-commands.sh │ └── t0020-make-command.sh ├── ui.go └── utils.go /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | *.swp 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: 2 | - linux 3 | - osx 4 | 5 | language: go 6 | 7 | go: 8 | - 1.7 9 | 10 | env: 11 | - TEST_VERBOSE=1 12 | 13 | install: 14 | - go get 15 | - go install 16 | 17 | script: 18 | - make test 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 IPFS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | export IPFS_API=ipfs.io 2 | 3 | all: deps 4 | 5 | install: deps 6 | go install 7 | 8 | build: deps 9 | go build 10 | 11 | deps: gx 12 | gx install 13 | 14 | gx: 15 | go get github.com/whyrusleeping/gx 16 | go get github.com/whyrusleeping/gx-go 17 | 18 | test: 19 | cd test/sharness && make 20 | 21 | .PHONY: test 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ipfs-pack - filesystem packing tool 2 | 3 | `ipfs-pack` is a tool and library to work with ipfs and large collections of data in UNIX/POSIX filesystems. 4 | 5 | - It identifies singular collections or bundles of data (the pack). 6 | - It creates a light-weight cryptographically secure manifest that preserves the integrity of the collection over time, and _travels with the data_ (PackManifest). 7 | - It helps use ipfs in a mode that references the filesystem files directly and avoids duplicating data (filestore). 8 | - It carries a standard dataset metadata file to capture and present information about the dataset (data-package.json). 9 | - It helps verify the authenticity of data through a file carrying cryptographic signatures (PackAuth). 10 | 11 | 12 | ## Installing 13 | 14 | Pre-built binaries are available on the [ipfs distributions page](https://dist.ipfs.io/#ipfs-pack). 15 | 16 | ### From source 17 | If there is not a pre-built binary for your system, or you'd like to try out 18 | unreleased features, or for any other reason you want to build from source, its 19 | relatively simple. First, make sure you have go installed and properly 20 | configured. [This guide](https://golang.org/doc/install) from the go team 21 | should help with that. Once thats done, simply run `make build`. 22 | 23 | ## Usage 24 | 25 | ``` 26 | $ ipfs-pack --help 27 | NAME: 28 | ipfs-pack - A filesystem packing tool. 29 | 30 | USAGE: 31 | ipfs-pack [global options] command [command options] [arguments...] 32 | 33 | VERSION: 34 | v0.1.0 35 | 36 | COMMANDS: 37 | make makes the package, overwriting the PackManifest file. 38 | verify verifies the ipfs-pack manifest file is correct. 39 | repo manipulate the ipfs repo cache associated with this pack. 40 | serve start an ipfs node to serve this pack's contents. 41 | help, h Shows a list of commands or help for one command 42 | 43 | GLOBAL OPTIONS: 44 | --help, -h show help 45 | --version, -v print the version 46 | ``` 47 | 48 | ### Make a pack 49 | ``` 50 | $ cd /path/to/data/dir 51 | $ ipfs-pack make 52 | wrote PackManifest 53 | ``` 54 | 55 | ### Verify a pack 56 | ``` 57 | $ ipfs-pack verify 58 | Pack verified successfully! 59 | ``` 60 | 61 | ## Testing 62 | Tests require the [random-files](https://github.com/jbenet/go-random-files) module 63 | 64 | ```bash 65 | go get -u github.com/jbenet/go-random-files/random-files 66 | ``` 67 | 68 | Run tests with 69 | ```bash 70 | ./test/pack-basic.sh 71 | ./test/pack-serve.sh 72 | ``` 73 | 74 | ## Spec 75 | 76 | Read the `ipfs-pack` work-in-progress "spec" here: [Spec (WIP)](./spec.md). 77 | 78 | -------------------------------------------------------------------------------- /escape.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strconv" 5 | "unicode/utf8" 6 | ) 7 | 8 | // unescape unescapes a strict subset of c-style escapes sequences: 9 | // \' or \" not allowed; octals (\034) must always be 3 digits; hexs 10 | // (\xAF) must always be 2 digits 11 | func unescape(str string) (string, error) { 12 | buf := make([]byte, len(str)) 13 | offset := 0 14 | for len(str) > 0 { 15 | val, _, tail, err := strconv.UnquoteChar(str, 0) 16 | if err != nil { 17 | return "", err 18 | } 19 | offset += utf8.EncodeRune(buf[offset:], val) 20 | str = tail 21 | } 22 | return string(buf[0:offset]), nil 23 | } 24 | 25 | // escape escapes problematic characters using c-style backslashes. 26 | // Currently escapes: newline, c.r., tab, and backslash 27 | func escape(str string) string { 28 | buf := make([]byte, 0, len(str)*2) 29 | for i := 0; i < len(str); i++ { 30 | switch str[i] { 31 | case '\n': 32 | buf = append(buf, '\\', 'n') 33 | case '\r': 34 | buf = append(buf, '\\', 'r') 35 | case '\t': 36 | buf = append(buf, '\\', 't') 37 | case '\\': 38 | buf = append(buf, '\\', '\\') 39 | default: 40 | buf = append(buf, str[i]) 41 | } 42 | } 43 | return string(buf) 44 | } 45 | -------------------------------------------------------------------------------- /fmtstr.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/hex" 6 | mh "gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash" 7 | ) 8 | 9 | const ( 10 | DefaultChunker = iota 11 | DefaultRaw 12 | SizeChunker 13 | RabinChunker 14 | ) 15 | 16 | const ( 17 | BalancedLayout = iota 18 | TrickleLayout 19 | ) 20 | 21 | const ( 22 | DefaultImporter = iota 23 | ) 24 | 25 | var DefaultImporterSettings = Importer{ 26 | Version: 0, 27 | Which: DefaultImporter, 28 | Args: ImportArgs{ 29 | Hash: mh.SHA2_256, 30 | Layout: BalancedLayout, 31 | Chunker: DefaultRaw, 32 | }, 33 | } 34 | 35 | type Importer struct { 36 | Version uint64 37 | Which uint64 38 | Args ImportArgs 39 | } 40 | 41 | type ImportArgs struct { 42 | Hash uint64 43 | Layout uint64 44 | Chunker uint64 45 | } 46 | 47 | func (i Importer) String() string { 48 | buf := make([]byte, 16) 49 | n := binary.PutUvarint(buf, i.Version) 50 | n += binary.PutUvarint(buf[n:], i.Which) 51 | n += binary.PutUvarint(buf[n:], i.Args.Hash) 52 | n += binary.PutUvarint(buf[n:], i.Args.Layout) 53 | n += binary.PutUvarint(buf[n:], i.Args.Chunker) 54 | 55 | return "f" + hex.EncodeToString(buf[:n]) 56 | } 57 | -------------------------------------------------------------------------------- /fmtstr_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestDefaultString(t *testing.T) { 8 | // since we don't actually do any parsing, this should suffice 9 | defstr := DefaultImporterSettings.String() 10 | if defstr != "f0000120001" { 11 | t.Fatal("importer string does not match expected") 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "os" 9 | "os/signal" 10 | "path/filepath" 11 | "runtime/pprof" 12 | "strings" 13 | "time" 14 | 15 | cli "gx/ipfs/QmVahSzvB3Upf5dAW15dpktF6PXb4z9V5LohmbcUqktyF4/cli" 16 | 17 | chunk "gx/ipfs/QmWo8jYc19ppG7YoTsrr2kEtLRbARTJho5oNXFTR6B7Peq/go-ipfs-chunker" 18 | core "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/core" 19 | cu "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/core/coreunix" 20 | bitswap "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/exchange/bitswap" 21 | filestore "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/filestore" 22 | balanced "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/importer/balanced" 23 | h "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/importer/helpers" 24 | dag "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/merkledag" 25 | fsrepo "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/repo/fsrepo" 26 | ft "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/unixfs" 27 | cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" 28 | files "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files" 29 | ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" 30 | 31 | human "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" 32 | ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore" 33 | node "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" 34 | 35 | pb "gx/ipfs/QmeWjRodbcZFKe5tMN7poEx3izym6osrLSnTLf9UjJZBbs/pb" 36 | ) 37 | 38 | const PackVersion = "v0.6.0" 39 | 40 | var ( 41 | cwd string 42 | ) 43 | 44 | func init() { 45 | d, err := os.Getwd() 46 | if err != nil { 47 | fmt.Println(err) 48 | os.Exit(1) 49 | } 50 | cwd = d 51 | } 52 | 53 | const ( 54 | ManifestFilename = "PackManifest" 55 | PackRepo = ".ipfs-pack" 56 | ) 57 | 58 | func main() { 59 | if err := doMain(); err != nil { 60 | fmt.Println(err) 61 | os.Exit(1) 62 | } 63 | } 64 | 65 | func setupProfiling() (func(), error) { 66 | halt := func() {} 67 | 68 | proffi := os.Getenv("IPFS_PACK_CPU_PROFILE") 69 | if proffi != "" { 70 | fi, err := os.Create(proffi) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | err = pprof.StartCPUProfile(fi) 76 | if err != nil { 77 | return nil, err 78 | } 79 | 80 | halt = func() { 81 | pprof.StopCPUProfile() 82 | fi.Close() 83 | } 84 | } 85 | 86 | memprofi := os.Getenv("IPFS_PACK_MEM_PROFILE") 87 | if memprofi != "" { 88 | go func() { 89 | for range time.NewTicker(time.Second * 5).C { 90 | fi, err := os.Create(memprofi) 91 | if err != nil { 92 | fmt.Fprintf(os.Stderr, "error writing heap profile: %s", err) 93 | return 94 | } 95 | 96 | if err := pprof.WriteHeapProfile(fi); err != nil { 97 | fmt.Fprintf(os.Stderr, "error writing heap profile: %s", err) 98 | return 99 | } 100 | 101 | fi.Close() 102 | } 103 | }() 104 | } 105 | 106 | return halt, nil 107 | } 108 | 109 | func doMain() error { 110 | if haltprof, err := setupProfiling(); err != nil { 111 | return err 112 | } else { 113 | defer haltprof() 114 | } 115 | 116 | app := cli.NewApp() 117 | app.Usage = "A filesystem packing tool" 118 | app.Version = PackVersion 119 | app.Commands = []cli.Command{ 120 | makePackCommand, 121 | verifyPackCommand, 122 | repoCommand, 123 | servePackCommand, 124 | } 125 | 126 | return app.Run(os.Args) 127 | } 128 | 129 | var makePackCommand = cli.Command{ 130 | Name: "make", 131 | Usage: "makes the package, overwriting the PackManifest file", 132 | ArgsUsage: "", 133 | Action: func(c *cli.Context) error { 134 | workdir := cwd 135 | if c.Args().Present() { 136 | argpath, err := filepath.Abs(c.Args().First()) 137 | if err != nil { 138 | return err 139 | } 140 | workdir = argpath 141 | } 142 | 143 | repo, err := getRepo(workdir) 144 | if err != nil { 145 | return err 146 | } 147 | 148 | adder, err := getAdder(repo.Datastore(), repo.FileManager()) 149 | if err != nil { 150 | return err 151 | } 152 | dirname := filepath.Base(workdir) 153 | 154 | output := make(chan interface{}) 155 | adder.Out = output 156 | adder.Progress = true 157 | 158 | done := make(chan struct{}) 159 | manifestName := filepath.Join(workdir, ManifestFilename) 160 | manifest, err := os.Create(manifestName + ".tmp") 161 | if err != nil { 162 | return err 163 | } 164 | 165 | imp := DefaultImporterSettings.String() 166 | 167 | fmt.Println("Building IPFS Pack") 168 | 169 | bar := pb.New64(-1) 170 | bar.Units = pb.U_BYTES 171 | bar.ShowSpeed = true 172 | bar.ShowPercent = false 173 | bar.Start() 174 | 175 | go func() { 176 | defer close(done) 177 | defer manifest.Close() 178 | var sizetotal int64 179 | var sizethis int64 180 | for v := range output { 181 | ao := v.(*cu.AddedObject) 182 | if ao.Bytes == 0 { 183 | sizetotal += sizethis 184 | sizethis = 0 185 | } else { 186 | sizethis = ao.Bytes 187 | bar.Set64(sizetotal + sizethis) 188 | } 189 | if ao.Hash == "" { 190 | continue 191 | } 192 | towrite := ao.Name[len(dirname):] 193 | if len(towrite) > 0 { 194 | towrite = towrite[1:] 195 | } else { 196 | towrite = "." 197 | } 198 | fmt.Fprintf(manifest, "%s\t%s\t%s\n", ao.Hash, imp, escape(towrite)) 199 | } 200 | }() 201 | 202 | sf, err := getFilteredDirFile(workdir) 203 | if err != nil { 204 | return err 205 | } 206 | 207 | go func() { 208 | sizer := sf.(files.SizeFile) 209 | size, err := sizer.Size() 210 | if err != nil { 211 | fmt.Println("warning: could not compute size:", err) 212 | return 213 | } 214 | bar.Total = size 215 | bar.ShowPercent = true 216 | }() 217 | 218 | err = adder.AddFile(sf) 219 | if err != nil { 220 | return err 221 | } 222 | 223 | _, err = adder.Finalize() 224 | if err != nil { 225 | return err 226 | } 227 | 228 | close(output) 229 | <-done 230 | err = os.Rename(manifestName+".tmp", manifestName) 231 | if err != nil { 232 | fmt.Printf("Pack creation completed sucessfully, but we failed to rename '%s' to '%s' due to the following error:\n", manifestName+".tmp", manifestName) 233 | fmt.Println(err) 234 | fmt.Println("To resolve the issue, manually rename the mentioned file.") 235 | os.Exit(1) 236 | } 237 | 238 | mes := "wrote PackManifest" 239 | clearBar(bar, mes) 240 | return nil 241 | }, 242 | } 243 | 244 | func clearBar(bar *pb.ProgressBar, mes string) { 245 | fmt.Printf("\r%s%s\n", mes, strings.Repeat(" ", bar.GetWidth()-len(mes))) 246 | } 247 | func getPackRoot(nd *core.IpfsNode, workdir string) (node.Node, error) { 248 | ctx := context.Background() 249 | root, err := getManifestRoot(workdir) 250 | if err != nil { 251 | return nil, err 252 | } 253 | 254 | proot, err := nd.DAG.Get(ctx, root) 255 | if err != nil { 256 | return nil, err 257 | } 258 | 259 | pfi, err := os.Open(filepath.Join(workdir, ManifestFilename)) 260 | if err != nil { 261 | return nil, err 262 | } 263 | 264 | manifhash, err := cu.Add(nd, pfi) 265 | if err != nil { 266 | return nil, err 267 | } 268 | 269 | manifcid, err := cid.Decode(manifhash) 270 | if err != nil { 271 | return nil, err 272 | } 273 | 274 | manifnode, err := nd.DAG.Get(context.Background(), manifcid) 275 | if err != nil { 276 | return nil, err 277 | } 278 | 279 | prootpb := proot.(*dag.ProtoNode) 280 | prootpb.AddNodeLinkClean(ManifestFilename, manifnode.(*dag.ProtoNode)) 281 | if err := nd.DAG.Add(ctx, prootpb); err != nil { 282 | return nil, err 283 | } 284 | return prootpb, nil 285 | } 286 | 287 | var servePackCommand = cli.Command{ 288 | Name: "serve", 289 | Usage: "start an ipfs node to serve this pack's contents", 290 | Flags: []cli.Flag{ 291 | cli.BoolTFlag{ 292 | Name: "verify", 293 | Usage: "verify integrity of pack before serving", 294 | }, 295 | }, 296 | Action: func(c *cli.Context) error { 297 | workdir := cwd 298 | if c.Args().Present() { 299 | argpath, err := filepath.Abs(c.Args().First()) 300 | if err != nil { 301 | return err 302 | } 303 | workdir = argpath 304 | } 305 | if _, err := os.Stat(workdir); os.IsNotExist(err) { 306 | fmt.Printf("No such directory: '%s'\n\nCOMMAND HELP:\n", workdir) 307 | return cli.ShowCommandHelp(c, "serve") 308 | } 309 | 310 | packpath := filepath.Join(workdir, PackRepo) 311 | if !fsrepo.IsInitialized(packpath) { 312 | return fmt.Errorf("No ipfs-pack found in '%s'\nplease run 'ipfs-pack make' before 'ipfs-pack serve'", cwd) 313 | } 314 | 315 | r, err := getRepo(workdir) 316 | if err != nil { 317 | return fmt.Errorf("error opening repo: %s", err) 318 | } 319 | 320 | verify := c.BoolT("verify") 321 | if verify { 322 | _, ds := buildDagserv(r.Datastore(), r.FileManager()) 323 | fi, err := os.Open(filepath.Join(workdir, ManifestFilename)) 324 | if err != nil { 325 | switch { 326 | case os.IsNotExist(err): 327 | return fmt.Errorf("error: no %s found in %s", ManifestFilename, workdir) 328 | default: 329 | return fmt.Errorf("error opening %s: %s", ManifestFilename, err) 330 | } 331 | } 332 | defer fi.Close() 333 | 334 | fmt.Print("Verifying pack contents before serving...") 335 | 336 | problem, err := verifyPack(ds, workdir, fi) 337 | if err != nil { 338 | return fmt.Errorf("error verifying pack: %s", err) 339 | } 340 | 341 | if problem { 342 | fmt.Println() 343 | return fmt.Errorf(`Pack verify failed, refusing to serve. 344 | If you meant to change the files, re-run 'ipfs-pack make' to rebuild the manifest 345 | Otherwise, replace the bad files with the originals and run 'ipfs-pack serve' again`) 346 | } else { 347 | fmt.Printf("\r" + QClrLine + "Verified pack, starting server...") 348 | } 349 | } 350 | 351 | cfg := &core.BuildCfg{ 352 | Online: true, 353 | Repo: r, 354 | Routing: core.DHTClientOption, 355 | } 356 | 357 | nd, err := core.NewNode(context.Background(), cfg) 358 | if err != nil { 359 | return err 360 | } 361 | 362 | proot, err := getPackRoot(nd, workdir) 363 | if err != nil { 364 | return err 365 | } 366 | 367 | totsize, err := proot.(*dag.ProtoNode).Size() 368 | if err != nil { 369 | return err 370 | } 371 | 372 | fmt.Print(QReset) 373 | putMessage(1, color(Cyan, "ipfs-pack")) 374 | padPrint(3, "Pack Status", color(Green, "serving globally")) 375 | padPrint(4, "Uptime", "0m") 376 | padPrint(5, "Version", PackVersion) 377 | padPrint(6, "Pack Size", human.Bytes(totsize)) 378 | padPrint(7, "Connections", "0 peers") 379 | padPrint(8, "PeerID", nd.Identity.Pretty()) 380 | padPrint(10, "Shared", "blocks total up rate up") 381 | padPrint(11, "", "0 0 0") 382 | 383 | padPrint(13, "Pack Root Hash", fmt.Sprintf("dweb:%s", color(Blue, "/ipfs/"+proot.Cid().String()))) 384 | 385 | addrs := nd.PeerHost.Addrs() 386 | putMessage(15, "Addresses") 387 | for i, a := range addrs { 388 | putMessage(16+i, a.String()+"/ipfs/"+nd.Identity.Pretty()) 389 | } 390 | 391 | bottom := 16 + len(addrs) 392 | lg := NewLog(bottom+3, 10) 393 | putMessage(bottom+1, "Activity Log") 394 | putMessage(bottom+2, "------------") 395 | putMessage(bottom+15, "[Press Ctrl+c to shutdown]") 396 | 397 | tick := time.NewTicker(time.Second) 398 | provdelay := time.After(time.Second * 5) 399 | start := time.Now() 400 | addlog := make(chan string, 16) 401 | nd.PeerHost.Network().Notify(&LogNotifee{addlog}) 402 | killed := make(chan os.Signal) 403 | signal.Notify(killed, os.Interrupt) 404 | 405 | var provinprogress bool 406 | for { 407 | putMessage(bottom+13, "") 408 | select { 409 | case <-nd.Context().Done(): 410 | return nil 411 | case <-tick.C: 412 | npeers := len(nd.PeerHost.Network().Peers()) 413 | padPrint(7, "Connections", fmt.Sprint(npeers)+" peers") 414 | 415 | st, err := nd.Exchange.(*bitswap.Bitswap).Stat() 416 | if err != nil { 417 | fmt.Println("error getting block stat: ", err) 418 | continue 419 | } 420 | 421 | bw := nd.Reporter.GetBandwidthTotals() 422 | printDataSharedLine(11, st.BlocksSent, bw.TotalOut, bw.RateOut) 423 | printTime(4, start) 424 | case <-provdelay: 425 | if !provinprogress { 426 | lg.Add("announcing pack content to the network") 427 | done := make(chan time.Time) 428 | provdelay = done 429 | go func() { 430 | ctx, cancel := context.WithTimeout(context.Background(), time.Minute) 431 | err := nd.Routing.Provide(ctx, proot.Cid(), true) 432 | if err != nil { 433 | lg.Add(fmt.Sprintf("error notifying network about our pack: %s", err)) 434 | } 435 | close(done) 436 | cancel() 437 | }() 438 | provinprogress = true 439 | } else { 440 | lg.Add("completed network announcement!") 441 | provdelay = time.After(time.Hour * 12) 442 | provinprogress = false 443 | } 444 | case mes := <-addlog: 445 | lg.Add(mes) 446 | lg.Print() 447 | case <-killed: 448 | fmt.Println(QReset) 449 | fmt.Println("Shutting down ipfs-pack node...") 450 | nd.Close() 451 | return nil 452 | } 453 | } 454 | 455 | return nil 456 | }, 457 | } 458 | 459 | func printTime(line int, start time.Time) { 460 | t := time.Since(start) 461 | h := int(t.Hours()) 462 | m := int(t.Minutes()) % 60 463 | s := int(t.Seconds()) % 60 464 | padPrint(line, "Uptime", fmt.Sprintf("%dh %dm %ds", h, m, s)) 465 | } 466 | 467 | var verifyPackCommand = cli.Command{ 468 | Name: "verify", 469 | Usage: "verifies the ipfs-pack manifest file is correct.", 470 | Action: func(c *cli.Context) error { 471 | workdir := cwd 472 | if c.Args().Present() { 473 | argpath, err := filepath.Abs(c.Args().First()) 474 | if err != nil { 475 | return err 476 | } 477 | workdir = argpath 478 | } 479 | 480 | // TODO: check for files in pack that arent in manifest 481 | fi, err := os.Open(filepath.Join(workdir, ManifestFilename)) 482 | if err != nil { 483 | switch { 484 | case os.IsNotExist(err): 485 | return fmt.Errorf("error: no %s found in %s", ManifestFilename, workdir) 486 | default: 487 | return fmt.Errorf("error opening %s: %s", ManifestFilename, err) 488 | } 489 | } 490 | 491 | var dstore ds.Batching 492 | var fm *filestore.FileManager 493 | packpath := filepath.Join(workdir, ".ipfs-pack") 494 | if fsrepo.IsInitialized(packpath) { 495 | r, err := getRepo(workdir) 496 | if err != nil { 497 | return err 498 | } 499 | dstore = r.Datastore() 500 | fm = r.FileManager() 501 | } else { 502 | dstore = ds.NewNullDatastore() 503 | } 504 | 505 | _, ds := buildDagserv(dstore, fm) 506 | 507 | issue, err := verifyPack(ds, workdir, fi) 508 | if err != nil { 509 | return err 510 | } 511 | 512 | if !issue { 513 | fmt.Println("pack verification succeeded") // nicola is an easter egg 514 | } else { 515 | return fmt.Errorf("error: pack verification failed") 516 | } 517 | return nil 518 | }, 519 | } 520 | 521 | func verifyPack(ds ipld.DAGService, workdir string, manif io.Reader) (bool, error) { 522 | imp := DefaultImporterSettings.String() 523 | 524 | var issue bool 525 | scan := bufio.NewScanner(manif) 526 | for scan.Scan() { 527 | parts := strings.SplitN(scan.Text(), "\t", 3) 528 | hash := parts[0] 529 | fmtstr := parts[1] 530 | path, err := unescape(parts[2]) 531 | if err != nil { 532 | fmt.Printf("%v\n", err) 533 | issue = true 534 | continue 535 | } 536 | 537 | if fmtstr != imp { 538 | if !issue { 539 | fmt.Println() 540 | } 541 | fmt.Printf("error: unsupported importer settings in manifest file: %s\n", fmtstr) 542 | issue = true 543 | continue 544 | } 545 | 546 | params := &h.DagBuilderParams{ 547 | Dagserv: ds, 548 | NoCopy: true, 549 | RawLeaves: true, 550 | Maxlinks: h.DefaultLinksPerBlock, 551 | } 552 | 553 | ok, mes, err := verifyItem(path, hash, workdir, params) 554 | if err != nil { 555 | return false, err 556 | } 557 | if !ok { 558 | if !issue { 559 | fmt.Println() 560 | } 561 | fmt.Println(mes) 562 | issue = true 563 | continue 564 | } 565 | } 566 | return issue, nil 567 | } 568 | 569 | func verifyItem(path, hash, workdir string, params *h.DagBuilderParams) (bool, string, error) { 570 | st, err := os.Lstat(filepath.Join(workdir, path)) 571 | switch { 572 | case os.IsNotExist(err): 573 | return false, fmt.Sprintf("error: item in manifest, missing from pack: %s", path), nil 574 | default: 575 | return false, fmt.Sprintf("error: checking file %s: %s", path, err), nil 576 | case err == nil: 577 | // continue 578 | } 579 | 580 | if st.IsDir() { 581 | return true, "", nil 582 | } 583 | 584 | nd, err := addItem(filepath.Join(workdir, path), st, params) 585 | if err != nil { 586 | return false, "", err 587 | } 588 | 589 | if nd.Cid().String() != hash { 590 | s := fmt.Sprintf("error: checksum mismatch on %s. (%s)", path, nd.Cid().String()) 591 | return false, s, nil 592 | } 593 | return true, "", nil 594 | } 595 | 596 | func addItem(path string, st os.FileInfo, params *h.DagBuilderParams) (node.Node, error) { 597 | if st.Mode()&os.ModeSymlink != 0 { 598 | trgt, err := os.Readlink(path) 599 | if err != nil { 600 | return nil, err 601 | } 602 | 603 | data, err := ft.SymlinkData(trgt) 604 | if err != nil { 605 | return nil, err 606 | } 607 | 608 | nd := new(dag.ProtoNode) 609 | nd.SetData(data) 610 | return nd, nil 611 | } 612 | 613 | fi, err := os.Open(path) 614 | if err != nil { 615 | return nil, err 616 | } 617 | defer fi.Close() 618 | 619 | rf, err := files.NewReaderPathFile(filepath.Base(path), path, fi, st) 620 | if err != nil { 621 | return nil, err 622 | } 623 | 624 | spl := chunk.NewSizeSplitter(rf, chunk.DefaultBlockSize) 625 | dbh := params.New(spl) 626 | 627 | nd, err := balanced.Layout(dbh) 628 | if err != nil { 629 | return nil, err 630 | } 631 | 632 | return nd, nil 633 | } 634 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "whyrusleeping", 3 | "bugs": { 4 | "url": "https://github.com/ipfs/ipfs-pack" 5 | }, 6 | "gx": { 7 | "dvcsimport": "github.com/ipfs/ipfs-pack" 8 | }, 9 | "gxDependencies": [ 10 | { 11 | "hash": "QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn", 12 | "name": "go-ipfs", 13 | "version": "0.4.14" 14 | }, 15 | { 16 | "author": "whyrusleeping", 17 | "hash": "QmVahSzvB3Upf5dAW15dpktF6PXb4z9V5LohmbcUqktyF4", 18 | "name": "cli", 19 | "version": "0.0.0" 20 | } 21 | ], 22 | "gxVersion": "0.10.0", 23 | "language": "go", 24 | "license": "", 25 | "name": "ipfs-pack", 26 | "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", 27 | "version": "0.6.0" 28 | } 29 | 30 | -------------------------------------------------------------------------------- /repo.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | cli "gx/ipfs/QmVahSzvB3Upf5dAW15dpktF6PXb4z9V5LohmbcUqktyF4/cli" 12 | 13 | blockstore "gx/ipfs/QmTVDM4LCSUMFNQzbDLL9zQwp8usE6QHymFdh3h8vL9v6b/go-ipfs-blockstore" 14 | h "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/importer/helpers" 15 | mfs "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/mfs" 16 | pin "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/pin" 17 | gc "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/pin/gc" 18 | fsrepo "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/repo/fsrepo" 19 | ft "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/unixfs" 20 | 21 | cid "gx/ipfs/QmcZfnkapfECQGcLZaf9B79NRg7cRa9EnZh4LSbkCzwNvY/go-cid" 22 | ) 23 | 24 | var repoCommand = cli.Command{ 25 | Name: "repo", 26 | Usage: "manipulate the ipfs repo associated with this pack", 27 | Subcommands: []cli.Command{ 28 | repoRegenCommand, 29 | repoGcCommand, 30 | repoLsCommand, 31 | repoRmCommand, 32 | }, 33 | } 34 | 35 | var repoRegenCommand = cli.Command{ 36 | Name: "regen", 37 | Usage: "regenerate ipfs-pack repo for this pack", 38 | Action: func(c *cli.Context) error { 39 | workdir := cwd 40 | if c.Args().Present() { 41 | argpath, err := filepath.Abs(c.Args().First()) 42 | if err != nil { 43 | return err 44 | } 45 | workdir = argpath 46 | } 47 | 48 | fi, err := openManifestFile(workdir) 49 | if err != nil { 50 | return err 51 | } 52 | 53 | r, err := getRepo(workdir) 54 | if err != nil { 55 | return err 56 | } 57 | 58 | _, dserv := buildDagserv(r.Datastore(), r.FileManager()) 59 | root, err := mfs.NewRoot(context.Background(), dserv, ft.EmptyDirNode(), nil) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | defaultFmts := DefaultImporterSettings.String() 65 | scan := bufio.NewScanner(fi) 66 | for scan.Scan() { 67 | parts := strings.SplitN(scan.Text(), "\t", 3) 68 | hash := parts[0] 69 | fmts := parts[1] 70 | path, err := unescape(parts[2]) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | if fmts != defaultFmts { 76 | return fmt.Errorf("error: unsupported import settings string: %s != %s", fmts, defaultFmts) 77 | } 78 | 79 | params := &h.DagBuilderParams{ 80 | Dagserv: dserv, 81 | NoCopy: true, 82 | RawLeaves: true, 83 | Maxlinks: h.DefaultLinksPerBlock, 84 | } 85 | 86 | st, err := os.Lstat(path) 87 | switch { 88 | case os.IsNotExist(err): 89 | return fmt.Errorf("error: in manifest, missing from pack: %s\n", path) 90 | default: 91 | return fmt.Errorf("error: reading file %s: %s\n", path, err) 92 | case err == nil: 93 | // continue 94 | } 95 | 96 | if st.IsDir() { 97 | // TODO: maybe check that the mfs root records this as being correct? 98 | continue 99 | } 100 | 101 | nd, err := addItem(filepath.Join(workdir, path), st, params) 102 | if err != nil { 103 | return err 104 | } 105 | 106 | if hash != nd.Cid().String() { 107 | return fmt.Errorf("error: checksum fail on %s (exp %s, got %s)", path, hash, nd.Cid()) 108 | } 109 | 110 | err = mfs.Mkdir(root, filepath.Dir(path), mfs.MkdirOpts{Mkparents: true}) 111 | if err != nil { 112 | return fmt.Errorf("error reconstructing tree: %s", err) 113 | } 114 | 115 | err = mfs.PutNode(root, filepath.Clean(path), nd) 116 | if err != nil { 117 | return fmt.Errorf("error adding tree node: %s", err) 118 | } 119 | } 120 | 121 | nd, err := root.GetValue().GetNode() 122 | if err != nil { 123 | return err 124 | } 125 | _ = nd 126 | fmt.Println("ipfs pack repo successfully regenerated.") 127 | 128 | return nil 129 | }, 130 | } 131 | 132 | var repoRmCommand = cli.Command{ 133 | Name: "rm", 134 | Usage: "remove this pack's ipfs repo", 135 | Action: func(c *cli.Context) error { 136 | packpath := filepath.Join(cwd, PackRepo) 137 | if !fsrepo.IsInitialized(packpath) { 138 | return fmt.Errorf("no repo found at ./.ipfs-pack") 139 | } 140 | 141 | return os.RemoveAll(packpath) 142 | }, 143 | } 144 | 145 | var repoGcCommand = cli.Command{ 146 | Name: "gc", 147 | Usage: "garbage collect the pack's ipfs repo", 148 | Action: func(c *cli.Context) error { 149 | packpath := filepath.Join(cwd, PackRepo) 150 | if !fsrepo.IsInitialized(packpath) { 151 | return fmt.Errorf("no repo found at ./.ipfs-pack") 152 | } 153 | 154 | fsr, err := fsrepo.Open(packpath) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | bstore, ds := buildDagserv(fsr.Datastore(), fsr.FileManager()) 160 | gcbs := blockstore.NewGCBlockstore(bstore, blockstore.NewGCLocker()) 161 | pinner := pin.NewPinner(fsr.Datastore(), ds, ds) 162 | 163 | root, err := getManifestRoot(cwd) 164 | if err != nil { 165 | return err 166 | } 167 | 168 | pinner.PinWithMode(root, pin.Recursive) 169 | if err := pinner.Flush(); err != nil { 170 | return err 171 | } 172 | 173 | out := gc.GC(context.Background(), gcbs, fsr.Datastore(), pinner, nil) 174 | if err != nil { 175 | return err 176 | } 177 | 178 | for k := range out { 179 | if k.Error != nil { 180 | fmt.Printf("GC Error: %s\n", k.Error) 181 | } else { 182 | fmt.Printf("removed %s\n", k.KeyRemoved) 183 | } 184 | } 185 | 186 | return nil 187 | }, 188 | } 189 | 190 | var repoLsCommand = cli.Command{ 191 | Name: "ls", 192 | Usage: "list all cids in the pack's ipfs repo", 193 | Action: func(c *cli.Context) error { 194 | packpath := filepath.Join(cwd, PackRepo) 195 | if !fsrepo.IsInitialized(packpath) { 196 | return fmt.Errorf("no repo found at ./.ipfs-pack") 197 | } 198 | 199 | fsr, err := fsrepo.Open(packpath) 200 | if err != nil { 201 | return err 202 | } 203 | 204 | bstore, _ := buildDagserv(fsr.Datastore(), fsr.FileManager()) 205 | keys, err := bstore.AllKeysChan(context.Background()) 206 | if err != nil { 207 | return err 208 | } 209 | 210 | for k := range keys { 211 | fmt.Println(k) 212 | } 213 | return nil 214 | }, 215 | } 216 | 217 | func getManifestRoot(workdir string) (*cid.Cid, error) { 218 | fi, err := os.Open(filepath.Join(workdir, ManifestFilename)) 219 | if err != nil { 220 | switch { 221 | case os.IsNotExist(err): 222 | return nil, fmt.Errorf("error: no %s found in %s", ManifestFilename, workdir) 223 | default: 224 | return nil, fmt.Errorf("error opening %s: %s", ManifestFilename, err) 225 | } 226 | } 227 | 228 | st, err := fi.Stat() 229 | if err != nil { 230 | return nil, err 231 | } 232 | if st.Size() > 1024 { 233 | _, err = fi.Seek(-512, os.SEEK_END) 234 | if err != nil { 235 | return nil, err 236 | } 237 | } 238 | 239 | scan := bufio.NewScanner(fi) 240 | var lastline string 241 | for scan.Scan() { 242 | lastline = scan.Text() 243 | } 244 | 245 | hash := strings.SplitN(lastline, "\t", 2)[0] 246 | return cid.Decode(hash) 247 | } 248 | -------------------------------------------------------------------------------- /spec.md: -------------------------------------------------------------------------------- 1 | # ipfs-pack spec 2 | 3 | ## Background 4 | 5 | Please read the original proposal here: https://github.com/ipfs/notes/issues/205, and the notes describing user stories in this repository: https://github.com/ipfs/archives/issues?utf8=%E2%9C%93&q=label%3Aipfs-pack%20is%3Aissue 6 | 7 | `ipfs-pack` is meant to be a bridge between POSIX and the world of content-addressed hash-linked data (IPFS and IPLD), that increases the safety and integrity of data in POSIX filesystems while avoiding duplicating or moving data into an ipfs repository. `ipfs-pack` establishes the notion of a "pack" of files (like a bundle or bag. We use `pack` to avoid confusing it with a `Bag` from `BagIt`, a similar format that `ipfs-pack` is compatible with). 8 | 9 | ### Status 10 | 11 | `ipfs-pack` is only a spec at present, but the hope is to implement it imminently. 12 | 13 | ## Abstractions 14 | 15 | ### Terms 16 | 17 | - **pack**: a bundle of directories, files, and metadata that represents a single collection of data. A pack is cryptographically secured with hashing (forms a merkle-dag). It is useful to ease working with large collections of data that want to be: secured from tampering and bitrot, certified by authors, and distributed across machines. 18 | - **pack tree**: the filesystem subtree (directories and files) that represents an ipfs-pack. 19 | - **pack root**: the root directory of the **pack tree**. it contains everything. 20 | - **pack contents**: the directories and files that are "contained" in the pack (no pack metadata). 21 | - **PackManifest**: a file that contains a listing of all entries in the pack tree, along with their cryptographic hash. It is a lightweight way to ensure integrity of the whole pack, and to verify what items may have been modified. 22 | - **pack repo**: an ipfs object repository for a pack. It contains and can retrieve all ipld objects within the pack. It can be considered a cache: as long as the **pack tree** itself remains intact, the **pack repo** can be destroyed safely and reconstructed at any point. The **pack repo** helps run ipfs commands on the pack, for example serve and distribute the pack to the rest of the world. A **pack repo** SHOULD be using **filestore**, which means the raw data of a pack will be referenced and not copied. 23 | - **filestore**: an ipfs repo storage strategy that only keeps references to raw file data, instead of copying all the data into another storage engine within the ipfs repo. This is useful when working with massive archives to avoid duplicating the data when importing it into ipfs. 24 | 25 | ### Diagram 26 | 27 | ``` 28 | +---- unixy filesystem -----------------------------------------------------+ 29 | | | 30 | | /a/b/c/d/pack-root-dir | 31 | | | 32 | | +---- ipfs pack +---------------------------------------------------+ | 33 | | | | | 34 | | | +-------------------------------------------------------------+ | | 35 | | | | | | | 36 | | | | | | | 37 | | | | contents - files and dirs contained in the pack | | | 38 | | | | +----> <----+ | | | 39 | | | | | | | | | 40 | | | +-------------------------------------------------------------+ | | 41 | | | | | | | 42 | | | | | | | 43 | | | | | | | 44 | | | +--+-------------+ +--------------------+ +--------------+--+ | | 45 | | | | | | | | | | | 46 | | | | PackManifest | | PackAuth | | .ipfs-pack | | | 47 | | | | (secure index) | | (authentication) | | (object repo) | | | 48 | | | | (MUST) | | (MAY) | | (MAY) | | | 49 | | | +------------+---+ +---+----------------+ +-----------------+ | | 50 | | | ^ | | | 51 | | | | | | | 52 | | | +----------+ | | 53 | | +-------------------------------------------------------------------+ | 54 | | | 55 | +---------------------------------------------------------------------------+ 56 | ``` 57 | 58 | ### Listing 59 | 60 | This is an example listing of a pack 61 | 62 | ``` 63 | > tree /a/b/c/d/pack-root-dir 64 | ├── .ipfs <--- ipfs object repo (optional, cache) 65 | ├── PackAuth <--- cryptographic signatures (optional) 66 | ├── PackManifest <--- cryptographic hash manifest (required) 67 | └── foo <--- files and dirs contained in the pack 68 | ├── bar 69 | │   ├── 1 70 | │   └── 2 71 | └── baz 72 | ├── 2 73 | └── 3 74 | ``` 75 | 76 | 77 | ## Commands 78 | 79 | ``` 80 | > ipfs-pack -h 81 | USAGE 82 | ipfs-pack 83 | 84 | SUBCOMMANDS 85 | make makes the package, overwriting the PackManifest file. 86 | verify verifies the ipfs-pack manifest file is correct. 87 | status shows the status of changes in the pack 88 | repo creates (or updates) a temporary ipfs object repo at `.ipfs-pack` 89 | serve starts an ipfs node serving the pack's contents (to IPFS and/or HTTP). 90 | bag create BagIt spec-compliant bag from a pack. 91 | car create a `.car` certified archive from a pack. 92 | ``` 93 | 94 | ### Usage Example 95 | 96 | ``` 97 | > pwd 98 | /home/jbenet/myPack 99 | 100 | > ls 101 | someJSON.json 102 | someXML.xml 103 | moreData/ 104 | 105 | > ipfs-pack make 106 | > ipfs-pack make -v 107 | wrote PackManifest 108 | 109 | > ls 110 | someJSON.json 111 | someXML.xml 112 | moreData/ 113 | PackManifest 114 | 115 | > cat PackManifest 116 | QmVP2aaAWFe21QjUujMw5hwYRKD1eGx3yYWEBbMtuxpqXs moreData/0 117 | QmV7eDE2WXuwQnvccsoXSzK5CQGXdFfay1LSadZCwyfbDV moreData/1 118 | QmaMY7h9pmTcA5w9S2dsQT5eGLEQ1CwYQ32HwMTXAev5gQ moreData/2 119 | QmQjYU5PscpCHadDbL1fDvTK4P9eXirSwD8hzJbAyrd5mf moreData/3 120 | QmRErwActoLmffucXq7HPtefBC19MjWUcj1DdBoaAnMm6p moreData/4 121 | QmeWvL929Tdhzw27CS5ZVHD73NQ9TT1xvLvCaXCgi7a9YB moreData/5 122 | QmXbzZeh44jJEUueWjFxEiLcfAfzoaKYEy1fMHygkSD3hm moreData/6 123 | QmYL17nYZrZsAhJut5v7ooD9hmz2rBotC1tqC9ZPxzCfer moreData/7 124 | QmPKkidoUYX12PyCuKzehQuhEJofUJ9PPaX2Gc2iYd4GRs moreData/8 125 | QmQAubXA3Gji5v5oaJhMbvmbGbiuwDf1u9sYsN125mcqrn moreData/9 126 | QmYbYduoHMZAUMB5mjHoJHgJ9WndrdWkTCzuQ6yHkbgqkU someJSON.json 127 | QmeWiZD5cdyiJoS3b7h87Cs9G21uQ1sLmeKrunTae9h5qG someXML.xml 128 | QmVizQ5fUceForgWogbb2m2v5RRrE8xEm8uSkbkyNB4Rdm moreData 129 | QmZ7iEGqahTHdUWGGZMUxYRXPwSM3UjBouneLcCmj9e6q6 . 130 | 131 | > ipfs-pack repo make 132 | > ipfs-pack repo make -v 133 | created repo at .ipfs-pack 134 | 135 | > ls -a 136 | ./ 137 | ../ 138 | .ipfs-pack/ 139 | someJSON.json 140 | someXML.xml 141 | moreData/ 142 | PackManifest 143 | ``` 144 | 145 | ### `ipfs-pack make` create (or update) a pack manifest 146 | 147 | This command creates (or updates) the pack's manifest file. It writes (overwrites) the `PackManifest` file. 148 | 149 | ``` 150 | ipfs-pack make 151 | # wrote PackManifest 152 | ``` 153 | 154 | ### `ipfs-pack verify` checks whether a pack matches its manifest 155 | 156 | This command checks whether a pack matches its `PackManifest`. 157 | 158 | ``` 159 | # errors when there is no manifest 160 | > random-files foo 161 | > cd foo 162 | > ipfs-pack verify 163 | error: no PackManifest found 164 | 165 | # succeeds when manifest and pack match 166 | > ipfs-pack make 167 | > ipfs-pack verify 168 | 169 | # errors when manifest and pack do not match 170 | > echo "QmVizQ5fUceForgWogbb2m2v5RRrE8xEm8uSkbkyNB4Rdm non-existent-file1" >>PackManifest 171 | > echo "QmVizQ5fUceForgWogbb2m2v5RRrE8xEm8uSkbkyNB4Rdm non-existent-file2" >>PackManifest 172 | > touch non-manifest-file3 173 | > ipfs-pack verify 174 | error: in manifest, missing from pack: non-existent-file1 175 | error: in manifest, missing from pack: non-existent-file2 176 | error: in pack, missing from manifest: non-manifest-file3 177 | ``` 178 | 179 | ### `ipfs-pack repo` creates (or updates) a temporary ipfs object repo 180 | 181 | This command creates (or updates) a temporary ipfs object repo (eg at `.ipfs-pack`). This repo contains some IPLD objects and positonal metadata in the pack files for the rest (filestore). This way the only IPLD objects stored are intermediate nodes that are not the raw data leaves. The leaves' data is kept in the original files, and referenced according to the filestore tooling. 182 | 183 | Notes: 184 | 185 | - This repo can be considered a cache, to be destroyed and recreated as needed. 186 | - `` is the file position metadata necessary to reconstruct an entire IPLD object from data in the pack. 187 | - Intermediate ipld objects (eg intermediate objects in a file, which are not raw data nodes) may need to be stored in the db. 188 | 189 | ``` 190 | > ipfs-pack repo --help 191 | USAGE 192 | ipfs-pack repo 193 | 194 | SUBCOMMANDS 195 | regen regenerate ipfs-pack repo for this pack 196 | ls lists all cids in the pack repo 197 | gc garbage collect the pack repo (pinset = PackManifest) 198 | rm removes the pack's ipfs repo 199 | ``` 200 | 201 | 202 | ### `ipfs-pack serve` starts an ipfs node serving the pack's contents (to IPFS and/or HTTP). 203 | 204 | This command starts an ipfs node serving the pack's contents (to IPFS and/or HTTP). This command MAY require a full go-ipfs installation to exist. It MAY be a standalone binary (`ipfs-pack-serve`). It MUST use an ephemeral node or a one-off node whose id would be stored locally, in the pack, at `/.ipfs-pack/repo` 205 | 206 | ``` 207 | > ipfs-pack serve --http 208 | Serving pack at /ip4/0.0.0.0/tcp/1234/http - http://127.0.0.1:1234 209 | 210 | > ipfs-pack serve --ipfs 211 | Serving pack at /ip4/0.0.0.0/tcp/1234/ipfs/QmPVUA4rJgckcf1ifrZF5KvwV1Uib5SGjJ7Z5BskEpTaSE 212 | ``` 213 | 214 | ### `ipfs-pack bag` convert to and from BagIt (spec-compliant) bags. 215 | 216 | This command converts between BagIt (spec-compliant) bags, a commonly used [archiving format](https://tools.ietf.org/html/draft-kunze-bagit-06#section-2.1.3) very similar to `ipfs-pack`. It works like this: 217 | 218 | 219 | ``` 220 | > ipfs-pack bag --help 221 | USAGE 222 | ipfs-pack-bag 223 | ipfs-pack-bag 224 | 225 | # convert from pack to bag 226 | > ipfs-pack bag path/to/mypack path/to/mybag 227 | 228 | # convert from bag to pack 229 | > ipfs-pack bag path/to/mybag path/to/mypack 230 | ``` 231 | 232 | ### `ipfs-pack car` convert to and from a car (certified archive). 233 | 234 | This command converts between packs and cars (certified archives). It works like this: 235 | 236 | 237 | ``` 238 | > ipfs-pack car --help 239 | USAGE 240 | ipfs-pack-car 241 | ipfs-pack-car 242 | 243 | # convert from pack to car 244 | > ipfs-pack car path/to/mypack path/to/mycar.car 245 | 246 | # convert from car to pack 247 | > ipfs-pack car path/to/mycar.car path/to/mypack 248 | ``` 249 | 250 | ## filestore in the repo 251 | 252 | Part of the point of ipfs-pack is to avoid copying data, and therefore it benefits tremendously from the filestore repo datastore. 253 | 254 | Maybe the `ipfs repo filestore` abstractions can leverage `ipfs-packs` to understand what is being tracked in a given directory, particularly if those packs have up-to-date local dbs of all their objects. 255 | 256 | 257 | ## Test Cases 258 | 259 | WIP 260 | 261 | ## References 262 | 263 | - [BagIt] The BagIt File Packaging Format, October 2016, 264 | -------------------------------------------------------------------------------- /test/pack-basic.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test_pack() { 4 | exp="$1" 5 | data="$2" 6 | rm -rf packtest 7 | mkdir -p packtest 8 | mv "$2" packtest/ 9 | cd packtest 10 | ipfs-pack make 11 | 12 | if ! grep "$exp" PackManifest; then 13 | echo "pack hash not as expected" 14 | echo "got " $(tail -n1 PackManifest) 15 | exit 1 16 | fi 17 | 18 | if ! ipfs-pack verify > verify_out; then 19 | echo "pack verify failed" 20 | cat verify_out 21 | exit 1 22 | fi 23 | 24 | echo "pack verify succeeded" 25 | } 26 | 27 | random-files -seed=42 -files=6 -depth=3 -dirs=5 stuff 28 | test_pack QmNeY1rrxVCBm1XHBeByHZrBEyfk4eqC1ckY9c6uVaLN8r stuff 29 | 30 | echo "foo" > afile 31 | test_pack QmaSFTnDx8iQJpih2tsJCwYmhnNpAggt2oSc49UZongcPS afile 32 | -------------------------------------------------------------------------------- /test/pack-serve.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | rm -rf packtest 4 | mkdir -p packtest 5 | cd packtest 6 | random-files -seed=42 -files=6 -depth=3 -dirs=5 stuff 7 | ipfs-pack make 8 | 9 | ipfs-pack serve > serve_out & 10 | echo $! > serve-pid 11 | sleep 1 12 | 13 | grep -A1 "Peer ID" serve_out >addrinfo 14 | PEERID=$(cat addrinfo | head -n1 | awk '{ print $3 }') 15 | ADDR=$(cat addrinfo | tail -n1 | awk '{ print $1 }') 16 | echo peerid $PEERID 17 | echo addr $ADDR 18 | 19 | rm -rf ../ipfs 20 | export IPFS_PATH=../ipfs 21 | ipfs init 22 | ipfs bootstrap rm --all 23 | ipfs config --json Discovery.MDNS.Enabled false 24 | ipfs daemon & 25 | echo $! > ipfs-pid 26 | sleep 1 27 | ipfs swarm connect $ADDR/ipfs/$PEERID 28 | 29 | HASH=$(tail -n1 PackManifest | awk '{ print $1 }') 30 | 31 | ipfs refs --timeout=10s -r $HASH 32 | 33 | kill $(cat serve-pid) 34 | kill $(cat ipfs-pid) 35 | -------------------------------------------------------------------------------- /test/sharness/.gitignore: -------------------------------------------------------------------------------- 1 | lib/sharness/ 2 | test-results/ 3 | trash directory.*.sh/ 4 | bin/ 5 | -------------------------------------------------------------------------------- /test/sharness/Makefile: -------------------------------------------------------------------------------- 1 | # Run sharness tests 2 | # 3 | # Copyright (c) 2014 Christian Couder 4 | # MIT Licensed; see the LICENSE file in this repository. 5 | # 6 | # NOTE: run with TEST_VERBOSE=1 for verbose sharness tests. 7 | 8 | T = $(sort $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)) 9 | LIBDIR = lib 10 | SHARNESSDIR = sharness 11 | AGGREGATE = $(LIBDIR)/$(SHARNESSDIR)/aggregate-results.sh 12 | 13 | find_go_files = $(shell find $(1) -name "*.go") 14 | 15 | BINS = bin/ipfs-pack 16 | 17 | # User might want to override those on the command line 18 | GOFLAGS = 19 | 20 | all: aggregate 21 | 22 | clean: clean-test-results 23 | @echo "*** $@ ***" 24 | -rm -rf $(BINS) 25 | 26 | clean-test-results: 27 | @echo "*** $@ ***" 28 | -rm -rf test-results 29 | 30 | $(T): clean-test-results deps 31 | @echo "*** $@ ***" 32 | ./$@ 33 | 34 | aggregate: clean-test-results $(T) 35 | @echo "*** $@ ***" 36 | ls test-results/t*-*.sh.*.counts | $(AGGREGATE) 37 | 38 | deps: sharness $(BINS) 39 | 40 | sharness: 41 | @echo "*** checking $@ ***" 42 | lib/install-sharness.sh 43 | 44 | bin/ipfs-pack: $(call find_go_files, ../../) 45 | mkdir -p bin 46 | go build -o bin/ipfs-pack $(GOFLAGS) github.com/ipfs/ipfs-pack 47 | 48 | .PHONY: all clean clean-test-results $(T) aggregate deps sharness FORCE 49 | 50 | -------------------------------------------------------------------------------- /test/sharness/lib/install-sharness.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # install-sharness.sh 3 | # 4 | # Copyright (c) 2014 Juan Batiz-Benet 5 | # Copyright (c) 2015 Christian Couder 6 | # MIT Licensed; see the LICENSE file in this repository. 7 | # 8 | # This script checks that Sharness is installed in: 9 | # 10 | # $(pwd)/$clonedir/$sharnessdir/ 11 | # 12 | # where $clonedir and $sharnessdir are configured below. 13 | # 14 | # If Sharness is not installed, this script will clone it 15 | # from $urlprefix (defined below). 16 | # 17 | # If Sharness is not uptodate with $version (defined below), 18 | # this script will fetch and will update the installed 19 | # version to $version. 20 | # 21 | 22 | # settings 23 | version=ecba410b0b58400dd6517cfa6594fdac243d9056 24 | urlprefix=https://github.com/chriscool/sharness.git 25 | clonedir=lib 26 | sharnessdir=sharness 27 | 28 | if test -f "$clonedir/$sharnessdir/SHARNESS_VERSION_$version" 29 | then 30 | # There is the right version file. Great, we are done! 31 | exit 0 32 | fi 33 | 34 | die() { 35 | echo >&2 "$@" 36 | exit 1 37 | } 38 | 39 | checkout_version() { 40 | git checkout "$version" || die "Could not checkout '$version'" 41 | rm -f SHARNESS_VERSION_* || die "Could not remove 'SHARNESS_VERSION_*'" 42 | touch "SHARNESS_VERSION_$version" || die "Could not create 'SHARNESS_VERSION_$version'" 43 | echo "Sharness version $version is checked out!" 44 | } 45 | 46 | if test -d "$clonedir/$sharnessdir/.git" 47 | then 48 | # We need to update sharness! 49 | cd "$clonedir/$sharnessdir" || die "Could not cd into '$clonedir/$sharnessdir' directory" 50 | git fetch || die "Could not fetch to update sharness" 51 | else 52 | # We need to clone sharness! 53 | mkdir -p "$clonedir" || die "Could not create '$clonedir' directory" 54 | cd "$clonedir" || die "Could not cd into '$clonedir' directory" 55 | 56 | git clone "$urlprefix" || die "Could not clone '$urlprefix'" 57 | cd "$sharnessdir" || die "Could not cd into '$sharnessdir' directory" 58 | fi 59 | 60 | checkout_version 61 | -------------------------------------------------------------------------------- /test/sharness/lib/test-lib.sh: -------------------------------------------------------------------------------- 1 | # Test framework for ipfs-pack 2 | # 3 | # Copyright (c) 2017 Christian Couder 4 | # MIT Licensed; see the LICENSE file in this repository. 5 | # 6 | # We are using sharness (https://github.com/chriscool/sharness) 7 | # which was extracted from the Git test framework. 8 | 9 | # Add current directory to path, for ipfs-pack. 10 | PATH=$(pwd)/bin:${PATH} 11 | 12 | # Set sharness verbosity. we set the env var directly as 13 | # it's too late to pass in --verbose, and --verbose is harder 14 | # to pass through in some cases. 15 | test "$TEST_VERBOSE" = 1 && verbose=t 16 | 17 | # assert the `ipfs-pack` we're using is the right one. 18 | if test `which ipfs-pack` != $(pwd)/bin/ipfs-pack; then 19 | echo >&2 "Cannot find the tests' local ipfs-pack tool." 20 | echo >&2 "Please check ipfs-pack installation." 21 | exit 1 22 | fi 23 | 24 | SHARNESS_LIB="lib/sharness/sharness.sh" 25 | 26 | . "$SHARNESS_LIB" || { 27 | echo >&2 "Cannot source: $SHARNESS_LIB" 28 | echo >&2 "Please check Sharness installation." 29 | exit 1 30 | } 31 | 32 | # Please put ipfs-pack specific shell functions below 33 | 34 | if test "$TEST_VERBOSE" = 1; then 35 | echo '# TEST_VERBOSE='"$TEST_VERBOSE" 36 | fi 37 | 38 | # Quote arguments for sh eval 39 | shellquote() { 40 | _space='' 41 | for _arg 42 | do 43 | # On Mac OS, sed adds a newline character. 44 | # With a printf wrapper the extra newline is removed. 45 | printf "$_space'%s'" "$(printf "%s" "$_arg" | sed -e "s/'/'\\\\''/g;")" 46 | _space=' ' 47 | done 48 | printf '\n' 49 | } 50 | 51 | # Echo the args, run the cmd, and then also fail, 52 | # making sure a test case fails. 53 | test_fsh() { 54 | echo "> $@" 55 | eval $(shellquote "$@") 56 | echo "" 57 | false 58 | } 59 | 60 | # Same as sharness' test_cmp but using test_fsh (to see the output). 61 | # We have to do it twice, so the first diff output doesn't show unless it's 62 | # broken. 63 | test_cmp() { 64 | diff -q "$@" >/dev/null || test_fsh diff -u "$@" 65 | } 66 | 67 | # Same as test_cmp above, but we sort files before comparing them. 68 | test_sort_cmp() { 69 | sort "$1" >"$1_sorted" && 70 | sort "$2" >"$2_sorted" && 71 | test_cmp "$1_sorted" "$2_sorted" 72 | } 73 | 74 | -------------------------------------------------------------------------------- /test/sharness/t0000-sharness.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | test_description="Show basic features of Sharness" 4 | 5 | . ./lib/test-lib.sh 6 | 7 | test_expect_success "Success is reported like this" " 8 | echo hello world | grep hello 9 | " 10 | 11 | test_expect_success "Commands are chained this way" " 12 | test x = 'x' && 13 | test 2 -gt 1 && 14 | echo success 15 | " 16 | 17 | return_42() { 18 | echo "Will return soon" 19 | return 42 20 | } 21 | 22 | test_expect_success "You can test for a specific exit code" " 23 | test_expect_code 42 return_42 24 | " 25 | 26 | test_expect_failure "We expect this to fail" " 27 | test 1 = 2 28 | " 29 | 30 | test_done 31 | 32 | # vi: set ft=sh : 33 | -------------------------------------------------------------------------------- /test/sharness/t0010-basic-commands.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright (c) 2017 Christian Couder 4 | # MIT Licensed; see the LICENSE file in this repository. 5 | # 6 | 7 | test_description="Test some basic commands" 8 | 9 | . lib/test-lib.sh 10 | 11 | test_expect_success "current dir is writable" ' 12 | echo "It works!" >test.txt 13 | ' 14 | 15 | test_expect_success "'ipfs-pack --version' succeeds" ' 16 | ipfs-pack --version >version.txt 17 | ' 18 | 19 | test_expect_success "'ipfs-pack --version' output looks good" ' 20 | egrep "^ipfs-pack version v[0-9]+\.[0-9]+\.[0-9]" version.txt >/dev/null || 21 | test_fsh cat version.txt 22 | ' 23 | 24 | test_expect_success "'ipfs-pack --help' and 'ipfs-pack help' succeed" ' 25 | ipfs-pack --help >help1.txt && 26 | ipfs-pack help >help2.txt 27 | ' 28 | 29 | test_expect_success "'ipfs-pack --help' and 'ipfs-pack help' output look good" ' 30 | egrep -i "A filesystem packing tool" help1.txt >/dev/null && 31 | egrep -i "A filesystem packing tool" help2.txt >/dev/null && 32 | egrep "ipfs-pack" help1.txt >/dev/null || 33 | test_fsh cat help1.txt && 34 | egrep "ipfs-pack" help2.txt >/dev/null || 35 | test_fsh cat help2.txt 36 | ' 37 | 38 | test_expect_success "'ipfs-pack help' output contain commands" ' 39 | grep -A4 COMMANDS help2.txt | tail -4 | cut -d" " -f6 >commands.txt && 40 | for cmd in $(cat commands.txt) 41 | do 42 | ipfs-pack help "$cmd" >"$cmd.txt" || break 43 | grep "ipfs-pack $cmd" "$cmd.txt" || break 44 | done 45 | ' 46 | 47 | test_done 48 | -------------------------------------------------------------------------------- /test/sharness/t0020-make-command.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright (c) 2017 Christian Couder 4 | # MIT Licensed; see the LICENSE file in this repository. 5 | # 6 | 7 | test_description="Test 'make' command" 8 | 9 | . lib/test-lib.sh 10 | 11 | # We should work in a subdir, so that we can freely 12 | # create files in the parent dir. 13 | 14 | test_expect_success "Create sample data set" ' 15 | mkdir workdir && 16 | ( 17 | cd workdir && 18 | echo Hello >hello.txt && 19 | mkdir subdir && 20 | echo "Hello in subdir" >subdir/hello_in_subdir.txt 21 | echo backslash > "back\slash" 22 | echo ouch > `printf "nasty\tfile\n!"` 23 | ) 24 | ' 25 | 26 | test_expect_success "'ipfs-pack make' succeeds" ' 27 | ( 28 | cd workdir && 29 | ipfs-pack make >../actual 30 | ) 31 | ' 32 | 33 | test_expect_success "'ipfs-pack make' output looks good" ' 34 | grep "Building IPFS Pack" actual && 35 | grep "wrote PackManifest" actual 36 | ' 37 | 38 | test_expect_success "PackManifest looks good" ' 39 | grep "subdir/hello_in_subdir.txt" workdir/PackManifest && 40 | grep "hello.txt" workdir/PackManifest && 41 | grep -F "nasty\tfile\n!" workdir/PackManifest && 42 | grep -F "back\\\\slash" workdir/PackManifest 43 | ' 44 | 45 | test_expect_success "'ipfs-pack verify' succeeds" ' 46 | ( 47 | cd workdir && 48 | ipfs-pack verify >../actual 49 | ) 50 | ' 51 | 52 | test_expect_success "'ipfs-pack verify' output looks good" ' 53 | echo "pack verification succeeded" >expected && 54 | test_cmp expected actual 55 | ' 56 | 57 | test_done 58 | -------------------------------------------------------------------------------- /ui.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | 7 | human "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize" 8 | ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" 9 | net "gx/ipfs/QmXfkENeeBvh3zYA51MaSdGUdBjhQ99cP5WQe8zgr6wchG/go-libp2p-net" 10 | ) 11 | 12 | const ( 13 | QClrLine = "\033[K" 14 | QReset = "\033[2J" 15 | ) 16 | 17 | /* 18 | Move the cursor up N lines: 19 | \033[A 20 | - Move the cursor down N lines: 21 | \033[B 22 | - Move the cursor forward N columns: 23 | \033[C 24 | - Move the cursor backward N columns: 25 | \033[D 26 | */ 27 | 28 | const ( 29 | Clear = 0 30 | ) 31 | 32 | const ( 33 | Black = 30 + iota 34 | Red 35 | Green 36 | Yellow 37 | Blue 38 | Magenta 39 | Cyan 40 | LightGray 41 | ) 42 | 43 | const ( 44 | LightBlue = 94 45 | ) 46 | 47 | func color(color int, s string) string { 48 | return fmt.Sprintf("\x1b[%dm%s\x1b[0m", color, s) 49 | } 50 | 51 | func padPrint(line int, label, value string) { 52 | putMessage(line, fmt.Sprintf("%s%s%s", label, strings.Repeat(" ", 20-len(label)), value)) 53 | } 54 | 55 | func putMessage(line int, mes string) { 56 | fmt.Printf("\033[%d;0H%s%s", line, QClrLine, mes) 57 | } 58 | 59 | func printDataSharedLine(line int, bup uint64, totup int64, rateup float64) { 60 | pad := " " 61 | a := fmt.Sprintf("%d ", bup)[:12] 62 | b := (human.Bytes(uint64(totup)) + pad)[:12] 63 | c := (human.Bytes(uint64(rateup)) + "/s" + pad)[:12] 64 | 65 | padPrint(line, "", a+b+c) 66 | } 67 | 68 | type Log struct { 69 | Size int 70 | StartLine int 71 | Messages []string 72 | beg int 73 | end int 74 | } 75 | 76 | func NewLog(line, size int) *Log { 77 | return &Log{ 78 | Size: size, 79 | StartLine: line, 80 | Messages: make([]string, size), 81 | end: -1, 82 | } 83 | } 84 | 85 | func (l *Log) Add(m string) { 86 | l.end = (l.end + 1) % l.Size 87 | if l.Messages[l.end] != "" { 88 | l.beg++ 89 | } 90 | l.Messages[l.end] = m 91 | } 92 | 93 | func (l *Log) Print() { 94 | for i := 0; i < l.Size; i++ { 95 | putMessage(l.StartLine+i, l.Messages[(l.beg+i)%l.Size]) 96 | } 97 | } 98 | 99 | type LogNotifee struct { 100 | addMes chan<- string 101 | } 102 | 103 | func (ln *LogNotifee) Listen(net.Network, ma.Multiaddr) {} 104 | func (ln *LogNotifee) ListenClose(net.Network, ma.Multiaddr) {} 105 | func (ln *LogNotifee) Connected(_ net.Network, c net.Conn) { 106 | ln.addMes <- fmt.Sprintf("New connection from %s", c.RemotePeer().Pretty()) 107 | } 108 | 109 | func (ln *LogNotifee) Disconnected(_ net.Network, c net.Conn) { 110 | ln.addMes <- fmt.Sprintf("Lost connection to %s", c.RemotePeer().Pretty()) 111 | } 112 | 113 | func (ln *LogNotifee) OpenedStream(net.Network, net.Stream) {} 114 | func (ln *LogNotifee) ClosedStream(net.Network, net.Stream) {} 115 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | blockstore "gx/ipfs/QmTVDM4LCSUMFNQzbDLL9zQwp8usE6QHymFdh3h8vL9v6b/go-ipfs-blockstore" 12 | blockservice "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/blockservice" 13 | cu "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/core/coreunix" 14 | offline "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/exchange/offline" 15 | filestore "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/filestore" 16 | dag "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/merkledag" 17 | repo "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/repo" 18 | config "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/repo/config" 19 | fsrepo "gx/ipfs/QmatUACvrFK3xYg1nd2iLAKfz7Yy5YB56tnzBYHpqiUuhn/go-ipfs/repo/fsrepo" 20 | files "gx/ipfs/QmceUdzxkimdYsgtX733uNgzf1DLHyBKN6ehGSp85ayppM/go-ipfs-cmdkit/files" 21 | ipld "gx/ipfs/Qme5bWv7wtjUNGsK2BNGVUFPKiuxWrsqrtvYwCLRw8YFES/go-ipld-format" 22 | 23 | ds "gx/ipfs/QmPpegoMqhAEqjncrzArm7KVWAkCm78rqL2DPuNjhPrshg/go-datastore" 24 | ) 25 | 26 | func openManifestFile(workdir string) (*os.File, error) { 27 | fi, err := os.Open(filepath.Join(workdir, ManifestFilename)) 28 | if err != nil { 29 | switch { 30 | case os.IsNotExist(err): 31 | return nil, fmt.Errorf("error: no %s found in %s", ManifestFilename, workdir) 32 | default: 33 | return nil, fmt.Errorf("error opening %s: %s", ManifestFilename, err) 34 | } 35 | } 36 | return fi, nil 37 | } 38 | 39 | func getRepo(workdir string) (repo.Repo, error) { 40 | packpath := filepath.Join(workdir, ".ipfs-pack") 41 | if !fsrepo.IsInitialized(packpath) { 42 | cfg, err := config.Init(ioutil.Discard, 1024) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | cfg.Addresses.API = "" 48 | cfg.Addresses.Gateway = "/ip4/127.0.0.1/tcp/0" 49 | cfg.Addresses.Swarm = []string{"/ip4/0.0.0.0/tcp/0"} 50 | cfg.Datastore.NoSync = true 51 | cfg.Experimental.FilestoreEnabled = true 52 | cfg.Reprovider.Interval = "0" 53 | 54 | err = fsrepo.Init(packpath, cfg) 55 | if err != nil { 56 | return nil, err 57 | } 58 | } 59 | return fsrepo.Open(packpath) 60 | } 61 | 62 | func buildDagserv(dstore ds.Batching, fm *filestore.FileManager) (blockstore.Blockstore, ipld.DAGService) { 63 | var bs blockstore.Blockstore = blockstore.NewBlockstore(dstore) 64 | if fm != nil { 65 | bs = filestore.NewFilestore(bs, fm) 66 | } 67 | bserv := blockservice.New(bs, offline.Exchange(bs)) 68 | return bs, dag.NewDAGService(bserv) 69 | } 70 | 71 | func getAdder(dstore ds.Batching, fm *filestore.FileManager) (*cu.Adder, error) { 72 | bstore, dserv := buildDagserv(dstore, fm) 73 | 74 | gcbs := blockstore.NewGCBlockstore(bstore, blockstore.NewGCLocker()) 75 | adder, err := cu.NewAdder(context.Background(), nil, gcbs, dserv) 76 | if err != nil { 77 | return nil, err 78 | } 79 | adder.NoCopy = true 80 | adder.RawLeaves = true 81 | return adder, nil 82 | } 83 | 84 | func getFilteredDirFile(workdir string) (files.File, error) { 85 | contents, err := ioutil.ReadDir(workdir) 86 | if err != nil { 87 | return nil, err 88 | } 89 | dirname := filepath.Base(workdir) 90 | 91 | var farr []files.File 92 | for _, ent := range contents { 93 | if ent.Name() == ManifestFilename || ent.Name() == ManifestFilename+".tmp" { 94 | continue 95 | } 96 | if strings.HasPrefix(ent.Name(), ".") { 97 | continue 98 | } 99 | f, err := files.NewSerialFile(filepath.Join(dirname, ent.Name()), filepath.Join(workdir, ent.Name()), false, ent) 100 | if err != nil { 101 | return nil, err 102 | } 103 | farr = append(farr, f) 104 | } 105 | 106 | return files.NewSliceFile(dirname, workdir, farr), nil 107 | } 108 | --------------------------------------------------------------------------------