├── .dockerignore ├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── Dockerfile ├── HACKING.md ├── LICENSE ├── README.md ├── cmd ├── noms │ ├── .gitignore │ ├── README.md │ ├── commit_iterator.go │ ├── noms.go │ ├── noms_blob.go │ ├── noms_blob_get.go │ ├── noms_blob_get_test.go │ ├── noms_blob_put.go │ ├── noms_commit.go │ ├── noms_commit_test.go │ ├── noms_config.go │ ├── noms_diff.go │ ├── noms_diff_test.go │ ├── noms_ds.go │ ├── noms_ds_test.go │ ├── noms_json.go │ ├── noms_list.go │ ├── noms_log.go │ ├── noms_log_test.go │ ├── noms_map.go │ ├── noms_merge.go │ ├── noms_merge_test.go │ ├── noms_root.go │ ├── noms_root_test.go │ ├── noms_serve.go │ ├── noms_set.go │ ├── noms_show.go │ ├── noms_show_test.go │ ├── noms_stats.go │ ├── noms_struct.go │ ├── noms_sync.go │ ├── noms_sync_test.go │ ├── noms_version.go │ ├── noms_version_test.go │ └── splore │ │ ├── .babelrc │ │ ├── .eslintrc.js │ │ ├── .flowconfig │ │ ├── .gitignore │ │ ├── js2go.py │ │ ├── noms_splore.go │ │ ├── noms_splore_test.go │ │ ├── out.js.go │ │ ├── package.json │ │ ├── src │ │ ├── assert.js │ │ ├── buchheim.js │ │ ├── layout.js │ │ ├── main.js │ │ ├── node.js │ │ └── types.js │ │ ├── webpack.config.js │ │ └── yarn.lock └── util │ └── kingpin_command.go ├── codecov.yml ├── doc ├── cli-screencast.png ├── cli-tour.md ├── data-version-control.png ├── decent │ ├── about.md │ ├── architectures.md │ ├── demo-ipfs-chat.md │ ├── demo-p2p-chat.md │ ├── dist-arch.png │ ├── p2p-arch.png │ └── quickstart.md ├── faq.md ├── go-tour.md ├── intro.md ├── nommy_cropped_smaller.png ├── prolly-tree-construction.png ├── prolly-tree-mutation.png ├── prolly-tree-structure.png ├── spelling.md ├── splore.png └── versioned-database.png ├── go.mod ├── go.sum ├── go ├── chunks │ ├── chunk.go │ ├── chunk_serializer.go │ ├── chunk_serializer_test.go │ ├── chunk_store.go │ ├── chunk_store_common_test.go │ ├── chunk_test.go │ ├── memory_store.go │ ├── memory_store_test.go │ ├── put_cache.go │ ├── remote_requests.go │ ├── remote_requests_test.go │ └── test_utils.go ├── config │ ├── config.go │ ├── config_test.go │ ├── resolver.go │ └── resolver_test.go ├── constants │ ├── http.go │ └── version.go ├── d │ ├── check_error.go │ ├── try.go │ └── try_test.go ├── datas │ ├── commit.go │ ├── commit_options.go │ ├── commit_test.go │ ├── database.go │ ├── database_common.go │ ├── database_server.go │ ├── database_test.go │ ├── dataset.go │ ├── dataset_test.go │ ├── http_chunk_store.go │ ├── http_chunk_store_test.go │ ├── pull.go │ ├── pull_test.go │ ├── pulling.md │ ├── remote_database_handlers.go │ ├── remote_database_handlers_test.go │ ├── serialize_hashes.go │ └── serialize_hashes_test.go ├── diff │ ├── apply_patch.go │ ├── apply_patch_test.go │ ├── diff.go │ ├── diff_test.go │ ├── patch.go │ ├── patch_test.go │ ├── print_diff.go │ └── summary.go ├── hash │ ├── base32.go │ ├── base32_test.go │ ├── hash.go │ ├── hash_slice.go │ ├── hash_slice_test.go │ └── hash_test.go ├── marshal │ ├── decode.go │ ├── decode_test.go │ ├── encode.go │ ├── encode_test.go │ ├── encode_type.go │ └── encode_type_test.go ├── merge │ ├── candidate.go │ ├── three_way.go │ ├── three_way_keyval_test.go │ ├── three_way_list.go │ ├── three_way_list_test.go │ ├── three_way_ordered_sequence.go │ ├── three_way_set_test.go │ └── three_way_test.go ├── metrics │ ├── histogram.go │ └── histogram_test.go ├── nbs │ ├── NBS-on-AWS.md │ ├── README.md │ ├── aws_chunk_source.go │ ├── aws_chunk_source_test.go │ ├── aws_table_persister.go │ ├── aws_table_persister_test.go │ ├── benchmarks │ │ ├── .gitignore │ │ ├── block_store_benchmarks.go │ │ ├── cachedrop │ │ │ └── drop_cache.go │ │ ├── chunker │ │ │ ├── .gitignore │ │ │ └── main.go │ │ ├── data_source.go │ │ ├── drop_cache_linux.go │ │ ├── drop_cache_other.go │ │ ├── file_block_store.go │ │ ├── gen │ │ │ ├── gen.go │ │ │ └── rolling_value_hasher.go │ │ ├── main.go │ │ └── null_block_store.go │ ├── block_store_test.go │ ├── cache.go │ ├── conjoiner.go │ ├── conjoiner_test.go │ ├── dynamo_fake_test.go │ ├── dynamo_manifest.go │ ├── dynamo_manifest_test.go │ ├── dynamo_table_reader.go │ ├── dynamo_table_reader_test.go │ ├── factory.go │ ├── factory_test.go │ ├── fd_cache.go │ ├── fd_cache_test.go │ ├── file_manifest.go │ ├── file_manifest_test.go │ ├── file_table_persister.go │ ├── file_table_persister_test.go │ ├── frag │ │ ├── .gitignore │ │ └── main.go │ ├── fs_table_cache.go │ ├── fs_table_cache_test.go │ ├── manifest.go │ ├── manifest_cache.go │ ├── manifest_cache_test.go │ ├── mem_table.go │ ├── mem_table_test.go │ ├── mmap_table_reader.go │ ├── mmap_table_reader_test.go │ ├── persisting_chunk_source.go │ ├── persisting_chunk_source_test.go │ ├── root_tracker_test.go │ ├── s3_fake_test.go │ ├── s3_table_reader.go │ ├── s3_table_reader_test.go │ ├── stats.go │ ├── stats_test.go │ ├── store.go │ ├── table.go │ ├── table_persister.go │ ├── table_persister_test.go │ ├── table_reader.go │ ├── table_set.go │ ├── table_set_test.go │ ├── table_test.go │ ├── table_writer.go │ └── test │ │ └── manifest_clobber.go ├── ngql │ ├── README.md │ ├── query.go │ ├── query_test.go │ └── types.go ├── nomdl │ ├── lexer.go │ ├── parser.go │ └── parser_test.go ├── perf │ ├── codec-perf-rig │ │ ├── .gitignore │ │ ├── README.md │ │ └── main.go │ ├── hash-perf-rig │ │ ├── README.md │ │ └── main.go │ └── suite │ │ ├── suite.go │ │ └── suite_test.go ├── sloppy │ ├── sloppy.go │ └── sloppy_test.go ├── spec │ ├── absolute_path.go │ ├── absolute_path_test.go │ ├── commit_meta.go │ ├── commit_meta_test.go │ ├── spec.go │ ├── spec_test.go │ └── util.go ├── types │ ├── blob.go │ ├── blob_editor.go │ ├── blob_editor_test.go │ ├── blob_leaf_sequence.go │ ├── blob_test.go │ ├── bool.go │ ├── codec.go │ ├── codec_test.go │ ├── collection.go │ ├── collection_test.go │ ├── common_supertype.go │ ├── common_supertype_test.go │ ├── compare_test.go │ ├── edit_distance.go │ ├── edit_distance_test.go │ ├── encode_human_readable.go │ ├── encode_human_readable_test.go │ ├── encoding_test.go │ ├── equals_test.go │ ├── get_hash.go │ ├── graph_builder.go │ ├── graph_builder_test.go │ ├── incremental_test.go │ ├── indexed_sequence_diff.go │ ├── indexed_sequences.go │ ├── leaf_sequence.go │ ├── less.go │ ├── list.go │ ├── list_editor.go │ ├── list_editor_test.go │ ├── list_iterator.go │ ├── list_iterator_test.go │ ├── list_leaf_sequence.go │ ├── list_test.go │ ├── make_type.go │ ├── map.go │ ├── map_editor.go │ ├── map_iterator.go │ ├── map_iterator_test.go │ ├── map_leaf_sequence.go │ ├── map_test.go │ ├── meta_sequence.go │ ├── noms_kind.go │ ├── number.go │ ├── number_util.go │ ├── opcache.go │ ├── opcache_compare.go │ ├── opcache_test.go │ ├── ordered_sequences.go │ ├── ordered_sequences_diff.go │ ├── ordered_sequences_diff_test.go │ ├── path.go │ ├── path_test.go │ ├── perf │ │ ├── dummy.go │ │ └── perf_test.go │ ├── primitives_test.go │ ├── ref.go │ ├── ref_heap.go │ ├── ref_heap_test.go │ ├── ref_test.go │ ├── rolling_value_hasher.go │ ├── rungen.go │ ├── sequence.go │ ├── sequence_chunker.go │ ├── sequence_concat.go │ ├── sequence_cursor.go │ ├── sequence_cursor_test.go │ ├── set.go │ ├── set_editor.go │ ├── set_iterator.go │ ├── set_iterator_test.go │ ├── set_leaf_sequence.go │ ├── set_test.go │ ├── simplify.go │ ├── simplify_test.go │ ├── string.go │ ├── string_test.go │ ├── struct.go │ ├── struct_test.go │ ├── subtype.go │ ├── subtype_test.go │ ├── type.go │ ├── type_desc.go │ ├── type_test.go │ ├── util_test.go │ ├── validate_type.go │ ├── validating_decoder.go │ ├── validating_decoder_test.go │ ├── value.go │ ├── value_decoder.go │ ├── value_stats.go │ ├── value_store.go │ ├── value_store_test.go │ ├── walk.go │ ├── walk_refs.go │ ├── walk_refs_test.go │ └── walk_test.go └── util │ ├── clienttest │ └── client_test_suite.go │ ├── datetime │ ├── date_time.go │ └── date_time_test.go │ ├── exit │ └── exit.go │ ├── functions │ ├── all.go │ └── all_test.go │ ├── json │ ├── from_json.go │ ├── from_json_test.go │ ├── to_json.go │ └── to_json_test.go │ ├── math │ └── minmax.go │ ├── outputpager │ └── page_output.go │ ├── profile │ └── profile.go │ ├── progressreader │ └── reader.go │ ├── random │ ├── id.go │ └── id_test.go │ ├── sizecache │ ├── size_cache.go │ └── size_cache_test.go │ ├── status │ └── status.go │ ├── test │ └── equals_ignore_hashes.go │ ├── verbose │ └── verbose.go │ └── writers │ ├── max_line_writer.go │ ├── prefix_writer.go │ └── writers_test.go ├── release.sh ├── samples ├── cli │ └── nomsconfig │ │ ├── .nomsconfig │ │ └── README.md └── go │ ├── counter │ ├── .gitignore │ ├── counter.go │ └── counter_test.go │ ├── csv │ ├── README.md │ ├── common.go │ ├── csv-analyze │ │ ├── .gitignore │ │ └── analyze.go │ ├── csv-export │ │ ├── .gitignore │ │ ├── exporter.go │ │ └── exporter_test.go │ ├── csv-import │ │ ├── .gitignore │ │ ├── importer.go │ │ ├── importer_test.go │ │ └── perf_test.go │ ├── csv-invert │ │ ├── .gitignore │ │ └── main.go │ ├── csv_reader.go │ ├── csv_reader_test.go │ ├── kind_slice.go │ ├── kind_slice_test.go │ ├── read.go │ ├── read_test.go │ ├── schema.go │ ├── schema_test.go │ ├── write.go │ └── write_test.go │ ├── decent │ ├── README.md │ ├── data │ │ ├── godfather.html │ │ ├── godfather2.html │ │ └── godfather3.html │ ├── dbg │ │ └── debug.go │ ├── ipfs-chat │ │ └── main.go │ ├── lib │ │ ├── datapager.go │ │ ├── event.go │ │ ├── importer.go │ │ ├── logger.go │ │ ├── model.go │ │ ├── model_test.go │ │ ├── pubsub.go │ │ ├── term_index.go │ │ ├── term_index_test.go │ │ └── termui.go │ └── p2p-chat │ │ ├── README.md │ │ └── main.go │ ├── hr │ ├── .gitignore │ ├── README.md │ ├── build_test_data.sh │ ├── main.go │ ├── main_test.go │ ├── test-data.bak │ │ └── 998se5i5mf15fld7f318818i6ie0c8rr │ └── test-data │ │ ├── 998se5i5mf15fld7f318818i6ie0c8rr │ │ ├── LOCK │ │ ├── bsvid54jt8pjto211lcdl14tbfd39jmn │ │ └── manifest │ ├── nomdex │ ├── Readme.md │ ├── expr.go │ ├── nomdex.go │ ├── nomdex_find.go │ ├── nomdex_test.go │ ├── nomdex_update.go │ ├── parser.go │ ├── parser_test.go │ ├── query_range.go │ └── query_range_test.go │ ├── nomsfs │ ├── .gitignore │ ├── README.md │ ├── basics_test.go │ ├── dir_test.go │ ├── nomsfs.go │ └── rw_test.go │ └── xml-import │ ├── .gitignore │ └── xml_importer.go └── tools ├── file ├── file.go └── file_test.go ├── licensify.py ├── loadtest └── loadtest.go ├── noms ├── README.md ├── __init__.py ├── copy.py ├── copy_test.py ├── pushd.py ├── staging.py ├── staging_test.py ├── symlink.py └── symlink_test.py └── runner ├── serial.go └── serial_test.go /.dockerignore: -------------------------------------------------------------------------------- 1 | .git 2 | doc 3 | codecov.yml 4 | CONTRIBUTING.md 5 | LICENSE 6 | README.md 7 | samples -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | noms.iml 2 | *.pyc 3 | *.swp 4 | .vscode 5 | .idea 6 | .noms 7 | .nomsconfig 8 | .DS_Store 9 | node_modules 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.x 4 | os: 5 | - linux 6 | env: 7 | - GO111MODULE=on 8 | before_install: 9 | # gox simplifies building for multiple architectures 10 | - go get github.com/mitchellh/gox 11 | script: 12 | - go build ./... 13 | - go test ./... 14 | - go vet ./... 15 | - mkdir linux 16 | - mkdir darwin 17 | - gox -os="linux darwin" -arch="amd64" -output="{{.OS}}/{{.Dir}}" ./cmd/noms 18 | - gox -os="linux darwin" -arch="amd64" -output="{{.OS}}/{{.Dir}}" ./samples/go/csv/csv-import 19 | - gox -os="linux darwin" -arch="amd64" -output="{{.OS}}/{{.Dir}}" ./samples/go/csv/csv-export 20 | - gox -os="linux darwin" -arch="amd64" -output="{{.OS}}/{{.Dir}}" ./samples/go/xml-import 21 | - mv darwin osx 22 | - zip -r linux linux 23 | - zip -r osx osx 24 | deploy: 25 | overwrite: true 26 | provider: releases 27 | skip_cleanup: true 28 | api_key: 29 | secure: "N2LCdQDlPquU31TK8WZwlYPRT7SSyfsGPBpNPSp5gpJPtF5hlqLf96Fd1R7SYn/LfTcri8baFMxgPVK4FowAzIsTxwkG57vCnJR24atOFVLkaKzVPdQZ30zXDHq2WO1zYw7KzAZq49YWdzwKSShzT7+SpiNZWEE2UiB5ZSQcd7/fii1TUkphzWPeHCB+d9wf1qUyJmm6HQ3PKe9yYRQHczGin6INUV5o+nzlRws2+5Kj7eg519htLgRY0oloncY0fdwTEwbSTTkkja3eoAWQrdPMJH7mDMwpbdgPl3jW8wDdTPHO5mQHRF4GvJHrY18qMJ9Kf8iQ3bdRtIS5XM8kvo8+Le22XQbYH7Q7Ryj/bJN+71KpVLwqWQhOr3fWRrL7r8DDPAG/myw0SK1uMaXCzT3KiYckJv7Q3el9MkHNblvFNxWC4tIrwE0LtP4hbSiIlZ/MV58yJxU8WXVej9AoFnKHLA7hgJUhHy0EIlfeETalDBrqNrh40iNP0maUrhpNJxLGtpOgAUhrdQ3gd//6pWwejkfvMTQ2b+1Qq11wWsSmRI/U1WGbcO/wzjKgVv2PT2sYPgx7TPwPWje5uFTZ4/sehwGG/LDcvuZ5uBXLRcpFIz9oh31nIFzsxhdatSKaaK4zlMzkxec+xqBGe0SVKeL/rW0MeQUbeSqyAf0wtBQ=" 30 | file: 31 | - linux.zip 32 | - osx.zip 33 | on: 34 | repo: attic-labs/noms 35 | tags: true 36 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:latest AS build 2 | 3 | ENV NOMS_SRC=$GOPATH/src/github.com/attic-labs/noms 4 | ENV CGO_ENABLED=1 5 | ENV GOOS=linux 6 | ENV DOCKER=1 7 | 8 | RUN mkdir -pv $NOMS_SRC 9 | COPY . ${NOMS_SRC} 10 | RUN go test github.com/attic-labs/noms/... 11 | RUN go install -v github.com/attic-labs/noms/cmd/noms 12 | RUN cp $GOPATH/bin/noms /bin/noms 13 | 14 | FROM alpine:latest 15 | 16 | COPY --from=build /bin/noms /bin/noms 17 | 18 | VOLUME /data 19 | EXPOSE 8000 20 | 21 | ENTRYPOINT [ "noms" ] 22 | 23 | CMD ["serve", "/data"] 24 | -------------------------------------------------------------------------------- /HACKING.md: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | 3 | * [Go 1.13 or later](https://golang.org/dl/) 4 | * Mac or Linux (Noms isn't currently supported on Windows) 5 | 6 | # Get 7 | 8 | ``` 9 | git clone https://github.com/attic-labs/noms 10 | ``` 11 | 12 | # Build 13 | 14 | ``` 15 | cd noms 16 | go build ./cmd/noms 17 | ``` 18 | 19 | # Test 20 | 21 | ``` 22 | cd noms 23 | go test ./go/... 24 | go test ./cmd/... 25 | ``` 26 | 27 | # Release 28 | 29 | Travis automatically creates releases for tagged versions, so the following should do it: 30 | 31 | ``` 32 | git tag latest -f 33 | git push origin latest 34 | ``` 35 | -------------------------------------------------------------------------------- /cmd/noms/.gitignore: -------------------------------------------------------------------------------- 1 | noms -------------------------------------------------------------------------------- /cmd/noms/README.md: -------------------------------------------------------------------------------- 1 | ## Example 2 | 3 | ```shell 4 | cd $GOPATH/src/github.com/attic-labs/noms/samples/go/counter 5 | go build 6 | ./counter /tmp/nomsdb::counter 7 | ./counter /tmp/nomsdb::counter 8 | ./counter /tmp/nomsdb::counter 9 | 10 | noms serve /tmp/nomsdb 11 | ``` 12 | 13 | Then, in a separate shell: 14 | 15 | ```shell 16 | # This starts where the previous count left off because we're serving the same database 17 | ./counter http://localhost:8000::counter 18 | 19 | # Display the datasets at this server 20 | noms ds http://localhost:8000 21 | 22 | # Print the history of the counter dataset 23 | noms log http://localhost:8000::counter 24 | ``` 25 | -------------------------------------------------------------------------------- /cmd/noms/noms.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "math/rand" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "github.com/attic-labs/kingpin" 15 | 16 | "github.com/attic-labs/noms/cmd/noms/splore" 17 | "github.com/attic-labs/noms/cmd/util" 18 | "github.com/attic-labs/noms/go/util/profile" 19 | "github.com/attic-labs/noms/go/util/verbose" 20 | ) 21 | 22 | var kingpinCommands = []util.KingpinCommand{ 23 | nomsBlob, 24 | nomsCommit, 25 | nomsConfig, 26 | nomsDiff, 27 | nomsDs, 28 | nomsList, 29 | nomsLog, 30 | nomsMerge, 31 | nomsJSON, 32 | nomsMap, 33 | nomsRoot, 34 | nomsServe, 35 | nomsSet, 36 | nomsShow, 37 | nomsStats, 38 | nomsStruct, 39 | nomsSync, 40 | splore.Cmd, 41 | nomsVersion, 42 | } 43 | 44 | var actions = []string{ 45 | "interacting with", 46 | "poking at", 47 | "goofing with", 48 | "dancing with", 49 | "playing with", 50 | "contemplation of", 51 | "showing off", 52 | "jiggerypokery of", 53 | "singing to", 54 | "nomming on", 55 | } 56 | 57 | func usageString() string { 58 | i := rand.New(rand.NewSource(time.Now().UnixNano())).Intn(len(actions)) 59 | return fmt.Sprintf(`Noms is a tool for %s Noms data.`, actions[i]) 60 | } 61 | 62 | func main() { 63 | // allow short (-h) help 64 | kingpin.EnableFileExpansion = false 65 | kingpin.CommandLine.HelpFlag.Short('h') 66 | noms := kingpin.New("noms", usageString()) 67 | 68 | // global flags 69 | profile.RegisterProfileFlags(noms) 70 | verbose.RegisterVerboseFlags(noms) 71 | 72 | handlers := map[string]util.KingpinHandler{} 73 | 74 | // install kingpin handlers 75 | for _, cmdFunction := range kingpinCommands { 76 | command, handler := cmdFunction(noms) 77 | handlers[command.FullCommand()] = handler 78 | } 79 | 80 | input := kingpin.MustParse(noms.Parse(os.Args[1:])) 81 | 82 | if handler := handlers[strings.Split(input, " ")[0]]; handler != nil { 83 | handler(input) 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /cmd/noms/noms_blob.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "runtime" 9 | "strconv" 10 | 11 | "github.com/attic-labs/kingpin" 12 | 13 | "github.com/attic-labs/noms/cmd/util" 14 | "github.com/attic-labs/noms/go/d" 15 | ) 16 | 17 | func nomsBlob(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 18 | blob := noms.Command("blob", "Interact with blobs.") 19 | 20 | blobPut := blob.Command("put", "imports a blob to a dataset") 21 | concurrency := blobPut.Flag("concurrency", "number of concurrent HTTP calls to retrieve remote resources").Default(strconv.Itoa(runtime.NumCPU())).Int() 22 | putFile := blobPut.Arg("file", "a file to import").Required().String() 23 | putDB := blobPut.Arg("dbSpec", "the database to import into").String() 24 | 25 | blobGet := blob.Command("export", "exports a blob from a dataset") 26 | getDs := blobGet.Arg("dataset", "the dataset to export").Required().String() 27 | getPath := blobGet.Arg("file", "an optional file to save the blob to").String() 28 | 29 | return blob, func(input string) int { 30 | switch input { 31 | case blobPut.FullCommand(): 32 | return nomsBlobPut(*putFile, *putDB, *concurrency) 33 | case blobGet.FullCommand(): 34 | return nomsBlobGet(*getDs, *getPath) 35 | } 36 | d.Panic("notreached") 37 | return 1 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /cmd/noms/noms_blob_get.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "os" 11 | "time" 12 | 13 | "github.com/attic-labs/noms/go/config" 14 | "github.com/attic-labs/noms/go/d" 15 | "github.com/attic-labs/noms/go/types" 16 | "github.com/attic-labs/noms/go/util/profile" 17 | "github.com/attic-labs/noms/go/util/progressreader" 18 | "github.com/attic-labs/noms/go/util/status" 19 | humanize "github.com/dustin/go-humanize" 20 | ) 21 | 22 | func nomsBlobGet(ds string, filePath string) int { 23 | cfg := config.NewResolver() 24 | var blob types.Blob 25 | if db, val, err := cfg.GetPath(ds); err != nil { 26 | d.CheckErrorNoUsage(err) 27 | } else if val == nil { 28 | d.CheckErrorNoUsage(fmt.Errorf("No value at %s", ds)) 29 | } else if b, ok := val.(types.Blob); !ok { 30 | d.CheckErrorNoUsage(fmt.Errorf("Value at %s is not a blob", ds)) 31 | } else { 32 | defer db.Close() 33 | blob = b 34 | } 35 | 36 | defer profile.MaybeStartProfile().Stop() 37 | 38 | if filePath == "" { 39 | blob.Copy(os.Stdout) 40 | return 0 41 | } 42 | 43 | // Note: overwrites any existing file. 44 | file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0644) 45 | d.CheckErrorNoUsage(err) 46 | defer file.Close() 47 | 48 | start := time.Now() 49 | expected := humanize.Bytes(blob.Len()) 50 | 51 | // Create a pipe so that we can connect a progress reader 52 | preader, pwriter := io.Pipe() 53 | 54 | go func() { 55 | blob.Copy(pwriter) 56 | pwriter.Close() 57 | }() 58 | 59 | blobReader := progressreader.New(preader, func(seen uint64) { 60 | elapsed := time.Since(start).Seconds() 61 | rate := uint64(float64(seen) / elapsed) 62 | status.Printf("%s of %s written in %ds (%s/s)...", humanize.Bytes(seen), expected, int(elapsed), humanize.Bytes(rate)) 63 | }) 64 | 65 | io.Copy(file, blobReader) 66 | status.Done() 67 | return 0 68 | } 69 | -------------------------------------------------------------------------------- /cmd/noms/noms_blob_get_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "io/ioutil" 11 | "path/filepath" 12 | "testing" 13 | 14 | "github.com/attic-labs/noms/go/spec" 15 | "github.com/attic-labs/noms/go/types" 16 | "github.com/attic-labs/noms/go/util/clienttest" 17 | "github.com/stretchr/testify/suite" 18 | ) 19 | 20 | func TestNomsBlobGet(t *testing.T) { 21 | suite.Run(t, &nbeSuite{}) 22 | } 23 | 24 | type nbeSuite struct { 25 | clienttest.ClientTestSuite 26 | } 27 | 28 | func (s *nbeSuite) TestNomsBlobGet() { 29 | sp, err := spec.ForDatabase(s.TempDir) 30 | s.NoError(err) 31 | defer sp.Close() 32 | db := sp.GetDatabase() 33 | 34 | blobBytes := []byte("hello") 35 | blob := types.NewBlob(db, bytes.NewBuffer(blobBytes)) 36 | 37 | ref := db.WriteValue(blob) 38 | _, err = db.CommitValue(db.GetDataset("datasetID"), ref) 39 | s.NoError(err) 40 | 41 | hashSpec := fmt.Sprintf("%s::#%s", s.TempDir, ref.TargetHash().String()) 42 | filePath := filepath.Join(s.TempDir, "out") 43 | s.MustRun(main, []string{"blob", "export", hashSpec, filePath}) 44 | 45 | fileBytes, err := ioutil.ReadFile(filePath) 46 | s.NoError(err) 47 | s.Equal(blobBytes, fileBytes) 48 | 49 | stdout, _ := s.MustRun(main, []string{"blob", "export", hashSpec}) 50 | fmt.Println("stdout:", stdout) 51 | s.Equal(blobBytes, []byte(stdout)) 52 | } 53 | -------------------------------------------------------------------------------- /cmd/noms/noms_blob_put.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "io" 11 | "os" 12 | 13 | "github.com/attic-labs/noms/go/config" 14 | "github.com/attic-labs/noms/go/d" 15 | "github.com/attic-labs/noms/go/types" 16 | "github.com/attic-labs/noms/go/util/profile" 17 | ) 18 | 19 | func nomsBlobPut(filePath string, dbPath string, concurrency int) int { 20 | info, err := os.Stat(filePath) 21 | if err != nil { 22 | d.CheckError(errors.New("couldn't stat file")) 23 | } 24 | 25 | defer profile.MaybeStartProfile().Stop() 26 | 27 | fileSize := info.Size() 28 | chunkSize := fileSize / int64(concurrency) 29 | if chunkSize < (1 << 20) { 30 | chunkSize = 1 << 20 31 | } 32 | 33 | readers := make([]io.Reader, fileSize/chunkSize) 34 | for i := 0; i < len(readers); i++ { 35 | r, err := os.Open(filePath) 36 | d.CheckErrorNoUsage(err) 37 | defer r.Close() 38 | r.Seek(int64(i)*chunkSize, 0) 39 | limit := chunkSize 40 | if i == len(readers)-1 { 41 | limit += fileSize % chunkSize // adjust size of last slice to include the final bytes. 42 | } 43 | lr := io.LimitReader(r, limit) 44 | readers[i] = lr 45 | } 46 | 47 | cfg := config.NewResolver() 48 | db, err := cfg.GetDatabase(dbPath) 49 | if err != nil { 50 | fmt.Fprintf(os.Stderr, "Could not open database: %s\n", err) 51 | return 1 52 | } 53 | defer db.Close() 54 | 55 | blob := types.NewBlob(db, readers...) 56 | ref := db.WriteValue(blob) 57 | db.Flush() 58 | fmt.Printf("#%s\n", ref.TargetHash()) 59 | return 0 60 | } 61 | -------------------------------------------------------------------------------- /cmd/noms/noms_commit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "errors" 9 | "fmt" 10 | "os" 11 | 12 | "github.com/attic-labs/kingpin" 13 | "github.com/attic-labs/noms/cmd/util" 14 | "github.com/attic-labs/noms/go/config" 15 | "github.com/attic-labs/noms/go/d" 16 | "github.com/attic-labs/noms/go/datas" 17 | "github.com/attic-labs/noms/go/spec" 18 | ) 19 | 20 | func nomsCommit(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 21 | commit := noms.Command("commit", "Commits a value to a dataset.") 22 | allowDupe := commit.Flag("allow-dupe", "creates a new commit, even if it would be identical (modulo metadata and parents) to the existing HEAD").Bool() 23 | message := commit.Flag("message", "commit message").String() 24 | date := commit.Flag("date", "commit date formatted as 2019-08-08T21:52:46Z - defaults to current date").String() 25 | path := commit.Arg("absolute-path", "absolute path to value to commit - see See Spelling Objects at https://github.com/attic-labs/noms/blob/master/doc/spelling.md").Required().String() 26 | ds := commit.Arg("dataset", "dataset spec to commit to - see Spelling Datasets at https://github.com/attic-labs/noms/blob/master/doc/spelling.md").Required().String() 27 | 28 | return commit, func(input string) int { 29 | cfg := config.NewResolver() 30 | db, ds, err := cfg.GetDataset(*ds) 31 | d.CheckError(err) 32 | defer db.Close() 33 | 34 | absPath, err := spec.NewAbsolutePath(*path) 35 | d.CheckError(err) 36 | 37 | value := absPath.Resolve(db) 38 | if value == nil { 39 | d.CheckErrorNoUsage(errors.New(fmt.Sprintf("Error resolving value: %s", *path))) 40 | } 41 | 42 | oldCommitRef, oldCommitExists := ds.MaybeHeadRef() 43 | if oldCommitExists { 44 | head := ds.HeadValue() 45 | if head.Hash() == value.Hash() && !*allowDupe { 46 | fmt.Fprintf(os.Stdout, "Commit aborted - allow-dupe is set to off and this commit would create a duplicate\n") 47 | return 0 48 | } 49 | } 50 | 51 | meta, err := spec.CreateCommitMetaStruct(db, *date, *message, nil, nil) 52 | d.CheckErrorNoUsage(err) 53 | 54 | ds, err = db.Commit(ds, value, datas.CommitOptions{Meta: meta}) 55 | d.CheckErrorNoUsage(err) 56 | 57 | if oldCommitExists { 58 | fmt.Fprintf(os.Stdout, "New head #%v (was #%v)\n", ds.HeadRef().TargetHash().String(), oldCommitRef.TargetHash().String()) 59 | } else { 60 | fmt.Fprintf(os.Stdout, "New head #%v\n", ds.HeadRef().TargetHash().String()) 61 | } 62 | return 0 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /cmd/noms/noms_config.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | 11 | "github.com/attic-labs/kingpin" 12 | "github.com/attic-labs/noms/cmd/util" 13 | "github.com/attic-labs/noms/go/config" 14 | "github.com/attic-labs/noms/go/d" 15 | ) 16 | 17 | func nomsConfig(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 18 | cfg := noms.Command("config", "Display noms config info.") 19 | 20 | return cfg, func(input string) int { 21 | c, err := config.FindNomsConfig() 22 | if err == config.NoConfig { 23 | fmt.Fprintf(os.Stdout, "no config active\n") 24 | } else { 25 | d.CheckError(err) 26 | fmt.Fprintf(os.Stdout, "%s\n", c.String()) 27 | } 28 | return 0 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /cmd/noms/noms_diff.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/attic-labs/kingpin" 11 | "github.com/attic-labs/noms/cmd/util" 12 | "github.com/attic-labs/noms/go/config" 13 | "github.com/attic-labs/noms/go/d" 14 | "github.com/attic-labs/noms/go/diff" 15 | "github.com/attic-labs/noms/go/util/outputpager" 16 | ) 17 | 18 | func nomsDiff(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 19 | cmd := noms.Command("diff", "Shows the difference between two values.") 20 | stat := cmd.Flag("stat", "writes a summary of the changes instead").Bool() 21 | o1 := cmd.Arg("val1", "first value - see Spelling Values at https://github.com/attic-labs/noms/blob/master/doc/spelling.md").Required().String() 22 | o2 := cmd.Arg("val2", "second value - see Spelling Values at https://github.com/attic-labs/noms/blob/master/doc/spelling.md").Required().String() 23 | outputpager.RegisterOutputpagerFlags(cmd) 24 | 25 | return cmd, func(input string) int { 26 | cfg := config.NewResolver() 27 | db1, value1, err := cfg.GetPath(*o1) 28 | d.CheckErrorNoUsage(err) 29 | if value1 == nil { 30 | d.CheckErrorNoUsage(fmt.Errorf("Value not found: %s", *o1)) 31 | } 32 | defer db1.Close() 33 | 34 | db2, value2, err := cfg.GetPath(*o2) 35 | d.CheckErrorNoUsage(err) 36 | if value2 == nil { 37 | d.CheckErrorNoUsage(fmt.Errorf("Value not found: %s", *o2)) 38 | } 39 | defer db2.Close() 40 | 41 | if *stat { 42 | diff.Summary(value1, value2) 43 | return 0 44 | } 45 | 46 | pgr := outputpager.Start() 47 | defer pgr.Stop() 48 | 49 | diff.PrintDiff(pgr.Writer, value1, value2, false) 50 | return 0 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /cmd/noms/noms_ds.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/attic-labs/kingpin" 11 | "github.com/attic-labs/noms/cmd/util" 12 | "github.com/attic-labs/noms/go/config" 13 | "github.com/attic-labs/noms/go/d" 14 | "github.com/attic-labs/noms/go/types" 15 | ) 16 | 17 | func nomsDs(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 18 | cmd := noms.Command("ds", "Dataset management.") 19 | del := cmd.Flag("delete", "delete a dataset").Short('d').Bool() 20 | name := cmd.Arg("name", "name of the database to list or dataset to delete - see Spelling Objects at https://github.com/attic-labs/noms/blob/master/doc/spelling.md").String() 21 | 22 | return cmd, func(input string) int { 23 | cfg := config.NewResolver() 24 | if *del { 25 | db, set, err := cfg.GetDataset(*name) 26 | d.CheckError(err) 27 | defer db.Close() 28 | 29 | oldCommitRef, errBool := set.MaybeHeadRef() 30 | if !errBool { 31 | d.CheckError(fmt.Errorf("Dataset %v not found", set.ID())) 32 | } 33 | 34 | _, err = set.Database().Delete(set) 35 | d.CheckError(err) 36 | 37 | fmt.Printf("Deleted %v (was #%v)\n", *name, oldCommitRef.TargetHash().String()) 38 | } else { 39 | store, err := cfg.GetDatabase(*name) 40 | d.CheckError(err) 41 | defer store.Close() 42 | 43 | store.Datasets().IterAll(func(k, v types.Value) { 44 | fmt.Println(k) 45 | }) 46 | } 47 | return 0 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /cmd/noms/noms_ds_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/attic-labs/noms/go/datas" 11 | "github.com/attic-labs/noms/go/nbs" 12 | "github.com/attic-labs/noms/go/spec" 13 | "github.com/attic-labs/noms/go/types" 14 | "github.com/attic-labs/noms/go/util/clienttest" 15 | "github.com/stretchr/testify/suite" 16 | ) 17 | 18 | func TestDs(t *testing.T) { 19 | suite.Run(t, &nomsDsTestSuite{}) 20 | } 21 | 22 | type nomsDsTestSuite struct { 23 | clienttest.ClientTestSuite 24 | } 25 | 26 | func (s *nomsDsTestSuite) TestEmptyNomsDs() { 27 | dir := s.DBDir 28 | 29 | cs := nbs.NewLocalStore(dir, clienttest.DefaultMemTableSize) 30 | ds := datas.NewDatabase(cs) 31 | 32 | ds.Close() 33 | 34 | dbSpec := spec.CreateDatabaseSpecString("nbs", dir) 35 | rtnVal, _ := s.MustRun(main, []string{"ds", dbSpec}) 36 | s.Equal("", rtnVal) 37 | } 38 | 39 | func (s *nomsDsTestSuite) TestNomsDs() { 40 | dir := s.DBDir 41 | 42 | cs := nbs.NewLocalStore(dir, clienttest.DefaultMemTableSize) 43 | db := datas.NewDatabase(cs) 44 | 45 | id := "testdataset" 46 | set := db.GetDataset(id) 47 | set, err := db.CommitValue(set, types.String("Commit Value")) 48 | s.NoError(err) 49 | 50 | id2 := "testdataset2" 51 | set2 := db.GetDataset(id2) 52 | set2, err = db.CommitValue(set2, types.String("Commit Value2")) 53 | s.NoError(err) 54 | 55 | err = db.Close() 56 | s.NoError(err) 57 | 58 | dbSpec := spec.CreateDatabaseSpecString("nbs", dir) 59 | datasetName := spec.CreateValueSpecString("nbs", dir, id) 60 | dataset2Name := spec.CreateValueSpecString("nbs", dir, id2) 61 | 62 | // both datasets show up 63 | rtnVal, _ := s.MustRun(main, []string{"ds", dbSpec}) 64 | s.Equal(id+"\n"+id2+"\n", rtnVal) 65 | 66 | // both datasets again, to make sure printing doesn't change them 67 | rtnVal, _ = s.MustRun(main, []string{"ds", dbSpec}) 68 | s.Equal(id+"\n"+id2+"\n", rtnVal) 69 | 70 | // delete one dataset, print message at delete 71 | rtnVal, _ = s.MustRun(main, []string{"ds", "-d", datasetName}) 72 | s.Equal("Deleted "+datasetName+" (was #ld4fuj44sd4gu0pepn7h5hga72282v81)\n", rtnVal) 73 | 74 | // print datasets, just one left 75 | rtnVal, _ = s.MustRun(main, []string{"ds", dbSpec}) 76 | s.Equal(id2+"\n", rtnVal) 77 | 78 | // delete the second dataset 79 | rtnVal, _ = s.MustRun(main, []string{"ds", "-d", dataset2Name}) 80 | s.Equal("Deleted "+dataset2Name+" (was #43qqlvkiainn1jf53g705622nndu1bje)\n", rtnVal) 81 | 82 | // print datasets, none left 83 | rtnVal, _ = s.MustRun(main, []string{"ds", dbSpec}) 84 | s.Equal("", rtnVal) 85 | } 86 | -------------------------------------------------------------------------------- /cmd/noms/noms_root_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/attic-labs/noms/go/spec" 11 | "github.com/attic-labs/noms/go/types" 12 | "github.com/attic-labs/noms/go/util/clienttest" 13 | "github.com/stretchr/testify/suite" 14 | ) 15 | 16 | func TestNomsRoot(t *testing.T) { 17 | suite.Run(t, &nomsRootTestSuite{}) 18 | } 19 | 20 | type nomsRootTestSuite struct { 21 | clienttest.ClientTestSuite 22 | } 23 | 24 | func (s *nomsRootTestSuite) TestBasic() { 25 | datasetName := "root-get" 26 | dsSpec := spec.CreateValueSpecString("nbs", s.DBDir, datasetName) 27 | sp, err := spec.ForDataset(dsSpec) 28 | s.NoError(err) 29 | defer sp.Close() 30 | 31 | ds := sp.GetDataset() 32 | dbSpecStr := spec.CreateDatabaseSpecString("nbs", s.DBDir) 33 | ds, _ = ds.Database().CommitValue(ds, types.String("hello!")) 34 | c1, _ := s.MustRun(main, []string{"root", dbSpecStr}) 35 | s.Equal("5te45oue1g918rpcvmc3d2emqkse4fhq\n", c1) 36 | 37 | ds, _ = ds.Database().CommitValue(ds, types.String("goodbye")) 38 | c2, _ := s.MustRun(main, []string{"root", dbSpecStr}) 39 | s.Equal("nm81pr21t66nec3v8jts5e37njg5ab1g\n", c2) 40 | 41 | // TODO: Would be good to test successful --update too, but requires changes to MustRun to allow 42 | // input because of prompt :(. 43 | } 44 | -------------------------------------------------------------------------------- /cmd/noms/noms_serve.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | 12 | "github.com/attic-labs/kingpin" 13 | "github.com/attic-labs/noms/cmd/util" 14 | "github.com/attic-labs/noms/go/config" 15 | "github.com/attic-labs/noms/go/d" 16 | "github.com/attic-labs/noms/go/datas" 17 | "github.com/attic-labs/noms/go/util/profile" 18 | ) 19 | 20 | func nomsServe(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 21 | cmd := noms.Command("serve", "Serves a Noms database over HTTP.") 22 | address := cmd.Flag("address", "address to listen on").Default("0.0.0.0").String() 23 | port := cmd.Flag("port", "port to listen on").Default("8080").Int() 24 | db := cmd.Arg("db", "database to work with - see Spelling Databases at https://github.com/attic-labs/noms/blob/master/doc/spelling.md").Required().String() 25 | 26 | return cmd, func(_ string) int { 27 | cfg := config.NewResolver() 28 | cs, err := cfg.GetChunkStore(*db) 29 | d.CheckError(err) 30 | server := datas.NewRemoteDatabaseServer(cs, *address, *port) 31 | 32 | // Shutdown server gracefully so that profile may be written 33 | c := make(chan os.Signal, 1) 34 | signal.Notify(c, os.Interrupt) 35 | signal.Notify(c, syscall.SIGTERM) 36 | go func() { 37 | <-c 38 | server.Stop() 39 | }() 40 | 41 | d.Try(func() { 42 | defer profile.MaybeStartProfile().Stop() 43 | server.Run() 44 | }) 45 | return 0 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /cmd/noms/noms_show.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "bytes" 9 | "fmt" 10 | "io" 11 | "os" 12 | 13 | "github.com/attic-labs/kingpin" 14 | "github.com/attic-labs/noms/cmd/util" 15 | "github.com/attic-labs/noms/go/config" 16 | "github.com/attic-labs/noms/go/d" 17 | "github.com/attic-labs/noms/go/types" 18 | "github.com/attic-labs/noms/go/util/datetime" 19 | "github.com/attic-labs/noms/go/util/outputpager" 20 | ) 21 | 22 | func nomsShow(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 23 | cmd := noms.Command("show", "Print Noms values.") 24 | showRaw := cmd.Flag("raw", "dump the value in binary format").Bool() 25 | showStats := cmd.Flag("stats", "report statics related to the value").Bool() 26 | tzName := cmd.Flag("tz", "display formatted date comments in specified timezone, must be: local or utc").Default("local").String() 27 | path := cmd.Arg("path", "value to display - see Spelling Values at https://github.com/attic-labs/noms/blob/master/doc/spelling.md").Required().String() 28 | 29 | return cmd, func(_ string) int { 30 | cfg := config.NewResolver() 31 | database, value, err := cfg.GetPath(*path) 32 | d.CheckErrorNoUsage(err) 33 | defer database.Close() 34 | 35 | if value == nil { 36 | fmt.Fprintf(os.Stderr, "Value not found: %s\n", *path) 37 | return 0 38 | } 39 | 40 | if *showRaw && *showStats { 41 | fmt.Fprintln(os.Stderr, "--raw and --stats are mutually exclusive") 42 | return 0 43 | } 44 | 45 | if *showRaw { 46 | ch := types.EncodeValue(value) 47 | buf := bytes.NewBuffer(ch.Data()) 48 | _, err = io.Copy(os.Stdout, buf) 49 | d.CheckError(err) 50 | return 0 51 | } 52 | 53 | if *showStats { 54 | types.WriteValueStats(os.Stdout, value, database) 55 | return 0 56 | } 57 | 58 | tz, _ := locationFromTimezoneArg(*tzName, nil) 59 | datetime.RegisterHRSCommenter(tz) 60 | 61 | pgr := outputpager.Start() 62 | defer pgr.Stop() 63 | 64 | types.WriteEncodedValue(pgr.Writer, value) 65 | fmt.Fprintln(pgr.Writer) 66 | return 0 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /cmd/noms/noms_stats.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | 10 | "github.com/attic-labs/kingpin" 11 | 12 | "github.com/attic-labs/noms/cmd/util" 13 | "github.com/attic-labs/noms/go/config" 14 | "github.com/attic-labs/noms/go/d" 15 | ) 16 | 17 | func nomsStats(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 18 | stats := noms.Command("stats", "Shows stats summary for a Noms Database.") 19 | database := stats.Arg("database", "See Spelling Objects at https://github.com/attic-labs/noms/blob/master/doc/spelling.md for details on the database argument.").Required().String() 20 | 21 | return stats, func(input string) int { 22 | cfg := config.NewResolver() 23 | store, err := cfg.GetDatabase(*database) 24 | d.CheckError(err) 25 | defer store.Close() 26 | 27 | fmt.Println(store.StatsSummary()) 28 | return 0 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /cmd/noms/noms_version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | 11 | "github.com/attic-labs/kingpin" 12 | "github.com/attic-labs/noms/cmd/util" 13 | "github.com/attic-labs/noms/go/constants" 14 | ) 15 | 16 | func nomsVersion(noms *kingpin.Application) (*kingpin.CmdClause, util.KingpinHandler) { 17 | cmd := noms.Command("version", "Displays the Noms version understood by this command.") 18 | 19 | return cmd, func(_ string) int { 20 | fmt.Fprintf(os.Stdout, "format version: %v\n", constants.NomsVersion) 21 | fmt.Fprintf(os.Stdout, "built from %v\n", constants.NomsGitSHA) 22 | return 0 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /cmd/noms/noms_version_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/attic-labs/noms/go/constants" 12 | "github.com/attic-labs/noms/go/util/clienttest" 13 | "github.com/stretchr/testify/suite" 14 | ) 15 | 16 | func TestVersion(t *testing.T) { 17 | suite.Run(t, &nomsVersionTestSuite{}) 18 | } 19 | 20 | type nomsVersionTestSuite struct { 21 | clienttest.ClientTestSuite 22 | } 23 | 24 | func (s *nomsVersionTestSuite) TestVersion() { 25 | val, _ := s.MustRun(main, []string{"version"}) 26 | expectedVal := fmt.Sprintf("format version: %v\nbuilt from %v\n", constants.NomsVersion, constants.NomsGitSHA) 27 | s.Equal(val, expectedVal) 28 | } 29 | -------------------------------------------------------------------------------- /cmd/noms/splore/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["stage-3", "react"] 3 | } 4 | -------------------------------------------------------------------------------- /cmd/noms/splore/.eslintrc.js: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | module.exports = { 6 | parser: 'babel-eslint', 7 | rules: { 8 | 'arrow-body-style': ['error', 'as-needed'], 9 | camelcase: 'error', 10 | eqeqeq: 'error', 11 | 'flowtype/boolean-style': 'error', 12 | 'flowtype/define-flow-type': 1, 13 | 'flowtype/use-flow-type': 1, 14 | 'no-fallthrough': 'error', 15 | 'no-new-wrappers': 'error', 16 | 'no-throw-literal': 'error', 17 | 'no-unused-vars': ['error', {argsIgnorePattern: '^_$', varsIgnorePattern: '^_$'}], 18 | 'no-var': 'error', 19 | 'prefer-arrow-callback': 'error', 20 | 'prefer-const': 'error', 21 | 'require-yield': 'error', 22 | radix: 'error', 23 | 'react/jsx-no-duplicate-props': 'error', 24 | 'react/jsx-no-undef': 'error', 25 | 'react/jsx-uses-react': 'error', 26 | 'react/jsx-uses-vars': 'error', 27 | }, 28 | env: { 29 | es6: true, 30 | jest: true, 31 | }, 32 | extends: 'eslint:recommended', 33 | ecmaFeatures: { 34 | jsx: true, 35 | experimentalObjectRestSpread: true, 36 | }, 37 | globals: { 38 | 'alert': true, 39 | 'console': true, 40 | 'document': true, 41 | 'fetch': true, 42 | 'window': true, 43 | }, 44 | plugins: ['flowtype', 'react'], 45 | }; 46 | -------------------------------------------------------------------------------- /cmd/noms/splore/.flowconfig: -------------------------------------------------------------------------------- 1 | [ignore] 2 | ; We fork some components by platform 3 | .*/*[.]android.js 4 | 5 | ; Ignore "BUCK" generated dirs 6 | /\.buckd/ 7 | 8 | ; Ignore unexpected extra "@providesModule" 9 | .*/node_modules/.*/node_modules/fbjs/.* 10 | 11 | ; Ignore duplicate module providers 12 | ; For RN Apps installed via npm, "Libraries" folder is inside 13 | ; "node_modules/react-native" but in the source repo it is in the root 14 | .*/Libraries/react-native/React.js 15 | .*/Libraries/react-native/ReactNative.js 16 | 17 | [include] 18 | 19 | [libs] 20 | node_modules/react-native/Libraries/react-native/react-native-interface.js 21 | node_modules/react-native/flow 22 | flow/ 23 | flow-typed/ 24 | 25 | [options] 26 | emoji=true 27 | 28 | module.file_ext=.ios.js 29 | module.file_ext=.android.js 30 | module.file_ext=.js 31 | module.file_ext=.json 32 | 33 | module.system=haste 34 | 35 | esproposal.class_static_fields=enable 36 | esproposal.class_instance_fields=enable 37 | 38 | experimental.strict_type_args=true 39 | 40 | munge_underscores=true 41 | 42 | module.name_mapper='^[./a-zA-Z0-9$_-]+\.\(bmp\|gif\|jpg\|jpeg\|png\|psd\|svg\|webp\|m4v\|mov\|mp4\|mpeg\|mpg\|webm\|aac\|aiff\|caf\|m4a\|mp3\|wav\|html\|pdf\)$' -> 'RelativeImageStub' 43 | 44 | suppress_type=$FlowIssue 45 | suppress_type=$FlowFixMe 46 | suppress_type=$FixMe 47 | 48 | suppress_comment=\\(.\\|\n\\)*\\$FlowIssue: .+ 49 | 50 | suppress_comment=\\(.\\|\n\\)*\\$FlowFixMe\\($\\|[^(]\\|(\\(>=0\\.\\(4[0-5]\\|[1-3][0-9]\\|[0-9]\\).[0-9]\\)? *\\(site=[a-z,_]*react_native[a-z,_]*\\)?)\\) 51 | suppress_comment=\\(.\\|\n\\)*\\$FlowIssue\\((\\(>=0\\.\\(4[0-5]\\|[1-3][0-9]\\|[0-9]\\).[0-9]\\)? *\\(site=[a-z,_]*react_native[a-z,_]*\\)?)\\)?:? #[0-9]+ 52 | suppress_comment=\\(.\\|\n\\)*\\$FlowFixedInNextDeploy 53 | suppress_comment=\\(.\\|\n\\)*\\$FlowExpectedError 54 | 55 | unsafe.enable_getters_and_setters=true 56 | 57 | [version] 58 | ^0.45.0 59 | -------------------------------------------------------------------------------- /cmd/noms/splore/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | out.js 3 | -------------------------------------------------------------------------------- /cmd/noms/splore/js2go.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | def main(): 6 | if len(sys.argv) != 4: 7 | print('usage: %s ' % (sys.argv[0])) 8 | return 9 | 10 | jsf, pkg, varname = sys.argv[1:] 11 | 12 | sys.stdout.write('package %s\n\n' % (pkg,)) 13 | sys.stdout.write('const %s = "' % (varname,)) 14 | with open(jsf, 'r') as js: 15 | sys.stdout.write( 16 | js.read() 17 | .replace('\\', '\\\\') 18 | .replace('"', '\\"') 19 | .replace('\n', '" +\n"')) 20 | sys.stdout.write('"') 21 | 22 | if __name__ == '__main__': 23 | main() 24 | -------------------------------------------------------------------------------- /cmd/noms/splore/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "noms-splore", 3 | "license": "Apache-2.0", 4 | "scripts": { 5 | "build": "webpack src/main.js out.js -d -p", 6 | "buildgo": "npm run build && python ./js2go.py out.js splore outJs > out.js.go && rm out.js", 7 | "prettier": "prettier --single-quote --trailing-comma all --bracket-spacing false --print-width 100 --write src/*.js src/**/*.js", 8 | "start": "webpack src/main.js out.js -d -w", 9 | "test": "eslint src/ && flow src/" 10 | }, 11 | "dependencies": { 12 | "babel-cli": "^6.24.1", 13 | "babel-core": "^6.25.0", 14 | "babel-eslint": "^7.2.3", 15 | "babel-loader": "^7.1.1", 16 | "babel-preset-stage-3": "^6.24.1", 17 | "eslint": "^3.5.0", 18 | "flow-bin": "^0.45.0", 19 | "humanize": "0.0.9", 20 | "prettier": "^1.5.3", 21 | "react": "^15.6.1", 22 | "react-dom": "^15.6.1", 23 | "webpack": "^3.5.3" 24 | }, 25 | "devDependencies": { 26 | "babel-preset-es2015": "^6.24.1", 27 | "babel-preset-react": "^6.24.1", 28 | "eslint-plugin-flowtype": "^2.32.1", 29 | "eslint-plugin-react": "^6.10.3" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /cmd/noms/splore/src/assert.js: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // @flow 6 | 7 | export function notNull(v: ?T): T { 8 | if (v !== null && v !== undefined) { 9 | return v; 10 | } 11 | throw new Error('Non-null assertion failed'); 12 | } 13 | -------------------------------------------------------------------------------- /cmd/noms/splore/src/node.js: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // @flow 6 | 7 | import React, {Component, Element} from 'react'; 8 | 9 | type Props = { 10 | hasChildren: boolean, 11 | isOpen: boolean, 12 | text: string, 13 | fromX: number, 14 | fromY: number, 15 | x: number, 16 | y: number, 17 | spaceX: number, 18 | onClick: (e: MouseEvent, s: string) => any, 19 | }; 20 | 21 | type State = { 22 | x: number, 23 | y: number, 24 | }; 25 | 26 | export default class Node extends Component { 27 | state: State; 28 | 29 | constructor(props: Props) { 30 | super(props); 31 | 32 | this.state = { 33 | x: this.props.fromX, 34 | y: this.props.fromY, 35 | }; 36 | } 37 | 38 | render(): Element { 39 | const {hasChildren, isOpen, onClick, text, x, y} = this.props; 40 | 41 | if (this.state.x !== x || this.state.y !== y) { 42 | window.requestAnimationFrame(() => this.setState({x, y})); 43 | } 44 | 45 | const gStyle = { 46 | transition: 'transform 200ms', 47 | transform: `translate3d(${this.state.x}px, ${this.state.y}px, 0)`, 48 | }; 49 | 50 | const circleStyle = { 51 | cursor: hasChildren ? (isOpen ? 'zoom-out' : 'zoom-in') : 'default', 52 | fill: hasChildren && !isOpen ? 'rgb(176, 196, 222)' : 'white', 53 | stroke: 'steelblue', 54 | strokeWidth: '1.5px', 55 | }; 56 | 57 | const foreignObjStyle = { 58 | overflow: 'visible', // Firefox like 59 | }; 60 | 61 | const paraStyle = { 62 | fontFamily: '"Menlo", monospace', 63 | fontSize: '11px', 64 | overflow: 'hidden', 65 | textAlign: 'right', 66 | textOverflow: 'ellipsis', 67 | whiteSpace: 'nowrap', 68 | }; 69 | 70 | const spanStyle = { 71 | backgroundColor: 'rgba(255, 255, 255, 0.7)', 72 | }; 73 | 74 | return ( 75 | 76 | 77 | 84 |
85 | 86 | {text} 87 | 88 |
89 | 90 | 91 | ); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /cmd/noms/splore/src/types.js: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // @flow 6 | 7 | export type NodeGraph = { 8 | keyLinks: {[key: string]: string[]}, 9 | links: {[key: string]: string[]}, 10 | nodes: {[key: string]: SploreNode}, 11 | open: {[key: string]: boolean}, 12 | }; 13 | 14 | // See node and nodeInfo in noms_splore.go 15 | // The types are unioned here for simplicity. 16 | export type SploreNode = { 17 | children?: SploreNodeChild[], // only in node, not nodeInfo 18 | hasChildren: boolean, 19 | id: string, 20 | name: string, 21 | }; 22 | 23 | // See nodeChild in noms_splore.go 24 | export type SploreNodeChild = { 25 | key: SploreNode, 26 | label: string, 27 | value: SploreNode, 28 | }; 29 | -------------------------------------------------------------------------------- /cmd/noms/splore/webpack.config.js: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | 'use strict'; 6 | 7 | const webpack = require('webpack'); 8 | 9 | const devMode = process.env.NODE_ENV !== 'production'; 10 | 11 | module.exports = { 12 | module: { 13 | loaders: [{ 14 | test: /\.js$/, 15 | loader: 'babel-loader', 16 | exclude: /node_modules/, 17 | }], 18 | }, 19 | devtool: devMode ? '#inline-source-map' : '', 20 | }; 21 | -------------------------------------------------------------------------------- /cmd/util/kingpin_command.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package util 6 | 7 | import "github.com/attic-labs/kingpin" 8 | 9 | type KingpinHandler func(input string) (exitCode int) 10 | type KingpinCommand func(*kingpin.Application) (*kingpin.CmdClause, KingpinHandler) 11 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | codecov: 2 | branch: master 3 | bot: "mikegray" 4 | ci: 5 | - "jenkins3.noms.io" 6 | 7 | coverage: 8 | precision: 2 # how many decimal places to display in the UI: 0 <= value <= 4 9 | round: down # how coverage is rounded: down/up/nearest 10 | range: 70...100 # custom range of coverage colors from red -> yellow -> green 11 | 12 | notify: 13 | slack: 14 | default: 15 | url: "secret:n+BYhIXTXsaCiMKB3vOf6yP68ytdKd3WpXtJFWPEUsEWXDiGnU5dTB5DO2yv8tR0COdxvs7K31hVpEfHEXdoXOaQhUw3FKf3fh8KZDLN7CGTbeDhw1uNGGyBr2d2TWnopzYtcXomdwMmuckARtiWQx0YXJiZY9YyCrIoDK9HIJQ=" 16 | branches: null 17 | threshold: 5.0 18 | attachments: "tree, diff" 19 | 20 | status: 21 | project: 22 | default: 23 | enabled: yes 24 | target: auto 25 | branches: null 26 | threshold: null 27 | if_no_uploads: error 28 | if_not_found: success 29 | if_ci_failed: error 30 | 31 | patch: 32 | default: 33 | enabled: yes 34 | target: auto 35 | branches: null 36 | threshold: null 37 | if_no_uploads: error 38 | if_not_found: success 39 | if_ci_failed: error 40 | 41 | changes: 42 | default: 43 | enabled: yes 44 | branches: null 45 | if_no_uploads: error 46 | if_not_found: success 47 | if_ci_failed: error 48 | 49 | comment: 50 | layout: "tree" 51 | branches: null 52 | behavior: default 53 | -------------------------------------------------------------------------------- /doc/cli-screencast.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/cli-screencast.png -------------------------------------------------------------------------------- /doc/data-version-control.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/data-version-control.png -------------------------------------------------------------------------------- /doc/decent/demo-ipfs-chat.md: -------------------------------------------------------------------------------- 1 | [Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** » 2 | 3 | [About](about.md)  |  [Quickstart](quickstart.md)  |  [Architectures](architectures.md)  |  [P2P Chat Demo](demo-p2p-chat.md)  |  **IPFS Chat Demo** 4 |

5 | # Demo App: IPFS-based Decentralized Chat 6 | 7 | This sample app demonstrates backing a P2P noms app by a decentralized blockstore (in this case, IPFS). Data is pulled off the network dynamically as needed - each client doesn't need a complete copy. 8 | 9 | # Build and Run 10 | 11 | Demo app code is in the 12 | [ipfs-chat](https://github.com/attic-labs/noms/tree/master/samples/go/decent/ipfs-chat/) 13 | directory. To get it up and running take the following steps: 14 | 15 | * Use git to clone the noms repository onto your computer: 16 | 17 | ```shell 18 | go get github.com/attic-labs/noms/samples/go/decent/ipfs-chat 19 | ``` 20 | 21 | * From the noms/samples/go/decent/ipfs-chat directory, build the program with the following command: 22 | 23 | ```shell 24 | go build 25 | ``` 26 | 27 | * Run the ipfs-chat client with the following command: 28 | 29 | ```shell 30 | ./ipfs-chat client --username --node-idx=1 ipfs:/tmp/ipfs1::chat >& /tmp/err1 31 | ``` 32 | 33 | * Run a second ipfs-chat client with the following command: 34 | 35 | ```shell 36 | ./ipfs-chat client --username --node-idx=2 ipfs:/tmp/ipfs2::chat >& /tmp/err2 37 | ``` 38 | 39 | If desired, ipfs-chat can be run as a daemon which will replicate all 40 | chat content in a local store which will enable clients to go offline 41 | without causing data to become unavailable to other clients: 42 | 43 | ```shell 44 | ./ipfs-chat daemon --node-idx=3 ipfs:/tmp/ipfs3::chat 45 | ``` 46 | 47 | Note: the 'node-idx' argument ensures that each IPFS-based program 48 | uses a distinct set of ports. This is useful when running multiple 49 | IPFS-based programs on the same machine. 50 | -------------------------------------------------------------------------------- /doc/decent/demo-p2p-chat.md: -------------------------------------------------------------------------------- 1 | [Home](../../README.md) » [Use Cases](../../README.md#use-cases) » **Decentralized** » 2 | 3 | [About](about.md)  |  [Quickstart](quickstart.md)  |  [Architectures](architectures.md)  |  **P2P Chat Demo**  |  [IPFS Chat Demo](demo-ipfs-chat.md) 4 |

5 | # Demo App: P2P Decentralized Chat 6 | 7 | This sample demonstrates the simplest possible case of building a p2p app on top of Noms. Each node stores a complete copy of the data it is interested in, and peers find each other using [IPFS pubsub](https://ipfs.io/blog/25-pubsub/). 8 | 9 | Currently, nodes have to have a publicly routable IP, but it should be possible to use [libP2P](https://github.com/libp2p) or similar to connect to most nodes. 10 | 11 | # Build and Run 12 | 13 | Demo app code is in the 14 | [p2p](https://github.com/attic-labs/noms/tree/master/samples/go/decent/p2p-chat) 15 | directory. To get it up and running take the following steps: 16 | 17 | * Use git to clone the noms repository onto your computer: 18 | 19 | ```shell 20 | go get github.com/attic-labs/noms/samples/go/decent/p2p-chat 21 | ``` 22 | 23 | * From the noms/samples/go/decent/p2p-chat directory, build the program with the following command: 24 | 25 | ```shell 26 | go build 27 | ``` 28 | 29 | * Run the p2p client with the following command: 30 | 31 | ```shell 32 | mkdir /tmp/noms1 33 | ./p2p-chat client --username= --node-idx=1 /tmp/noms1 >& /tmp/err1 34 | ``` 35 | 36 | * Run a second p2p client with the following command: 37 | 38 | ```shell 39 | mkdir /tmp/noms2 40 | ./p2p-chat client --username= --node-idx=2 /tmp/noms2 >& /tmp/err2 41 | ``` 42 | 43 | Note: the p2p client relies on IPFS for it's pub/sub implementation. The 44 | 'node-idx' argument ensures that each IPFS-based node uses a distinct set 45 | of ports. This is useful when running multiple IPFS-based programs on 46 | the same machine. 47 | -------------------------------------------------------------------------------- /doc/decent/dist-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/decent/dist-arch.png -------------------------------------------------------------------------------- /doc/decent/p2p-arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/decent/p2p-arch.png -------------------------------------------------------------------------------- /doc/nommy_cropped_smaller.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/nommy_cropped_smaller.png -------------------------------------------------------------------------------- /doc/prolly-tree-construction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/prolly-tree-construction.png -------------------------------------------------------------------------------- /doc/prolly-tree-mutation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/prolly-tree-mutation.png -------------------------------------------------------------------------------- /doc/prolly-tree-structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/prolly-tree-structure.png -------------------------------------------------------------------------------- /doc/splore.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/splore.png -------------------------------------------------------------------------------- /doc/versioned-database.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/doc/versioned-database.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/attic-labs/noms 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/BurntSushi/toml v0.3.1 7 | github.com/aboodman/noms-gx v0.0.0-20180714061401-d6cb97cb040b 8 | github.com/alecthomas/kingpin v2.2.6+incompatible // indirect 9 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect 10 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect 11 | github.com/attic-labs/graphql v0.0.0-20190507195614-b6552d20145f 12 | github.com/attic-labs/kingpin v2.2.7-0.20180312050558-442efcfac769+incompatible 13 | github.com/aws/aws-sdk-go v1.19.26 14 | github.com/clbanning/mxj v1.8.4 15 | github.com/codahale/blake2 v0.0.0-20150924215134-8d10d0420cbf 16 | github.com/dustin/go-humanize v1.0.0 17 | github.com/golang/snappy v0.0.1 18 | github.com/hanwen/go-fuse v1.0.0 19 | github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 20 | github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d 21 | github.com/julienschmidt/httprouter v1.2.0 22 | github.com/kch42/buzhash v0.0.0-20160816060738-9bdec3dec7c6 23 | github.com/mattn/go-colorable v0.1.1 // indirect 24 | github.com/mattn/go-isatty v0.0.7 25 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b 26 | github.com/shirou/gopsutil v2.18.12+incompatible 27 | github.com/skratchdot/open-golang v0.0.0-20190402232053-79abb63cd66e 28 | github.com/stretchr/testify v1.3.0 29 | github.com/syndtr/goleveldb v1.0.0 30 | golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c 31 | golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a 32 | golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 33 | gopkg.in/alecthomas/kingpin.v2 v2.2.6 34 | ) 35 | -------------------------------------------------------------------------------- /go/chunks/chunk.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // Package chunks provides facilities for representing, storing, and fetching content-addressed chunks of Noms data. 6 | package chunks 7 | 8 | import ( 9 | "bytes" 10 | 11 | "github.com/attic-labs/noms/go/d" 12 | "github.com/attic-labs/noms/go/hash" 13 | ) 14 | 15 | // Chunk is a unit of stored data in noms 16 | type Chunk struct { 17 | r hash.Hash 18 | data []byte 19 | } 20 | 21 | var EmptyChunk = NewChunk([]byte{}) 22 | 23 | func (c Chunk) Hash() hash.Hash { 24 | return c.r 25 | } 26 | 27 | func (c Chunk) Data() []byte { 28 | return c.data 29 | } 30 | 31 | func (c Chunk) IsEmpty() bool { 32 | return len(c.data) == 0 33 | } 34 | 35 | // NewChunk creates a new Chunk backed by data. This means that the returned Chunk has ownership of this slice of memory. 36 | func NewChunk(data []byte) Chunk { 37 | r := hash.Of(data) 38 | return Chunk{r, data} 39 | } 40 | 41 | // NewChunkWithHash creates a new chunk with a known hash. The hash is not re-calculated or verified. This should obviously only be used in cases where the caller already knows the specified hash is correct. 42 | func NewChunkWithHash(r hash.Hash, data []byte) Chunk { 43 | return Chunk{r, data} 44 | } 45 | 46 | // ChunkWriter wraps an io.WriteCloser, additionally providing the ability to grab the resulting Chunk for all data written through the interface. Calling Chunk() or Close() on an instance disallows further writing. 47 | type ChunkWriter struct { 48 | buffer *bytes.Buffer 49 | c Chunk 50 | } 51 | 52 | func NewChunkWriter() *ChunkWriter { 53 | b := &bytes.Buffer{} 54 | return &ChunkWriter{ 55 | buffer: b, 56 | } 57 | } 58 | 59 | func (w *ChunkWriter) Write(data []byte) (int, error) { 60 | if w.buffer == nil { 61 | d.Panic("Write() cannot be called after Hash() or Close().") 62 | } 63 | size, err := w.buffer.Write(data) 64 | d.Chk.NoError(err) 65 | return size, nil 66 | } 67 | 68 | // Chunk() closes the writer and returns the resulting Chunk. 69 | func (w *ChunkWriter) Chunk() Chunk { 70 | d.Chk.NoError(w.Close()) 71 | return w.c 72 | } 73 | 74 | // Close() closes computes the hash and Puts it into the ChunkSink Note: The Write() method never returns an error. Instead, like other noms interfaces, errors are reported via panic. 75 | func (w *ChunkWriter) Close() error { 76 | if w.buffer == nil { 77 | return nil 78 | } 79 | 80 | w.c = NewChunk(w.buffer.Bytes()) 81 | w.buffer = nil 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /go/chunks/chunk_serializer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package chunks 6 | 7 | import ( 8 | "bytes" 9 | "encoding/binary" 10 | "io" 11 | 12 | "github.com/attic-labs/noms/go/d" 13 | "github.com/attic-labs/noms/go/hash" 14 | ) 15 | 16 | /* 17 | Chunk Serialization: 18 | Chunk 0 19 | Chunk 1 20 | .. 21 | Chunk N 22 | 23 | Chunk: 24 | Hash // 20-byte hash 25 | Len // 4-byte int 26 | Data // len(Data) == Len 27 | */ 28 | 29 | // Serialize a single Chunk to writer. 30 | func Serialize(chunk Chunk, writer io.Writer) { 31 | d.PanicIfFalse(chunk.data != nil) 32 | 33 | h := chunk.Hash() 34 | n, err := io.Copy(writer, bytes.NewReader(h[:])) 35 | d.Chk.NoError(err) 36 | d.PanicIfFalse(int64(hash.ByteLen) == n) 37 | 38 | // Because of chunking at higher levels, no chunk should never be more than 4GB 39 | chunkSize := uint32(len(chunk.Data())) 40 | err = binary.Write(writer, binary.BigEndian, chunkSize) 41 | d.Chk.NoError(err) 42 | 43 | n, err = io.Copy(writer, bytes.NewReader(chunk.Data())) 44 | d.Chk.NoError(err) 45 | d.PanicIfFalse(uint32(n) == chunkSize) 46 | } 47 | 48 | // Deserialize reads off of |reader| until EOF, sending chunks to 49 | // chunkChan in the order they are read. Objects sent over chunkChan are 50 | // *Chunk. 51 | func Deserialize(reader io.Reader, chunkChan chan<- *Chunk) (err error) { 52 | for { 53 | var c Chunk 54 | c, err = deserializeChunk(reader) 55 | if err != nil { 56 | break 57 | } 58 | d.Chk.NotEqual(EmptyChunk.Hash(), c.Hash()) 59 | chunkChan <- &c 60 | } 61 | if err == io.EOF { 62 | err = nil 63 | } 64 | return 65 | } 66 | 67 | func deserializeChunk(reader io.Reader) (Chunk, error) { 68 | h := hash.Hash{} 69 | n, err := io.ReadFull(reader, h[:]) 70 | if err != nil { 71 | return EmptyChunk, err 72 | } 73 | d.PanicIfFalse(int(hash.ByteLen) == n) 74 | 75 | chunkSize := uint32(0) 76 | if err = binary.Read(reader, binary.BigEndian, &chunkSize); err != nil { 77 | return EmptyChunk, err 78 | } 79 | 80 | data := make([]byte, int(chunkSize)) 81 | if n, err = io.ReadFull(reader, data); err != nil { 82 | return EmptyChunk, err 83 | } 84 | d.PanicIfFalse(int(chunkSize) == n) 85 | c := NewChunk(data) 86 | if h != c.Hash() { 87 | d.Panic("%s != %s", h, c.Hash().String()) 88 | } 89 | return c, nil 90 | } 91 | -------------------------------------------------------------------------------- /go/chunks/chunk_serializer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package chunks 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestSerializeRoundTrip(t *testing.T) { 15 | assert := assert.New(t) 16 | inputs := [][]byte{[]byte("abc"), []byte("def")} 17 | chnx := make([]Chunk, len(inputs)) 18 | for i, data := range inputs { 19 | chnx[i] = NewChunk(data) 20 | } 21 | 22 | buf := &bytes.Buffer{} 23 | Serialize(chnx[0], buf) 24 | Serialize(chnx[1], buf) 25 | 26 | chunkChan := make(chan *Chunk) 27 | go func() { 28 | defer close(chunkChan) 29 | err := Deserialize(bytes.NewReader(buf.Bytes()), chunkChan) 30 | assert.NoError(err) 31 | }() 32 | 33 | for c := range chunkChan { 34 | assert.Equal(chnx[0].Hash(), c.Hash()) 35 | chnx = chnx[1:] 36 | } 37 | assert.Len(chnx, 0) 38 | } 39 | 40 | func TestBadSerialization(t *testing.T) { 41 | bad := []byte{0, 1} // Not enough bytes to read first length 42 | ch := make(chan *Chunk) 43 | defer close(ch) 44 | assert.Error(t, Deserialize(bytes.NewReader(bad), ch)) 45 | } 46 | -------------------------------------------------------------------------------- /go/chunks/chunk_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package chunks 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestChunk(t *testing.T) { 14 | c := NewChunk([]byte("abc")) 15 | h := c.Hash() 16 | // See http://www.di-mgt.com.au/sha_testvectors.html 17 | assert.Equal(t, "rmnjb8cjc5tblj21ed4qs821649eduie", h.String()) 18 | } 19 | 20 | func TestChunkWriteAfterCloseFails(t *testing.T) { 21 | assert := assert.New(t) 22 | input := "abc" 23 | w := NewChunkWriter() 24 | _, err := w.Write([]byte(input)) 25 | assert.NoError(err) 26 | 27 | assert.NoError(w.Close()) 28 | assert.Panics(func() { w.Write([]byte(input)) }, "Write() after Close() should barf!") 29 | } 30 | 31 | func TestChunkWriteAfterChunkFails(t *testing.T) { 32 | assert := assert.New(t) 33 | input := "abc" 34 | w := NewChunkWriter() 35 | _, err := w.Write([]byte(input)) 36 | assert.NoError(err) 37 | 38 | _ = w.Chunk() 39 | assert.Panics(func() { w.Write([]byte(input)) }, "Write() after Chunk() should barf!") 40 | } 41 | 42 | func TestChunkChunkCloses(t *testing.T) { 43 | assert := assert.New(t) 44 | input := "abc" 45 | w := NewChunkWriter() 46 | _, err := w.Write([]byte(input)) 47 | assert.NoError(err) 48 | 49 | w.Chunk() 50 | assert.Panics(func() { w.Write([]byte(input)) }, "Write() after Close() should barf!") 51 | } 52 | -------------------------------------------------------------------------------- /go/chunks/memory_store_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package chunks 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/suite" 11 | ) 12 | 13 | func TestMemoryStoreTestSuite(t *testing.T) { 14 | suite.Run(t, &MemoryStoreTestSuite{}) 15 | } 16 | 17 | type MemoryStoreTestSuite struct { 18 | ChunkStoreTestSuite 19 | } 20 | 21 | func (suite *MemoryStoreTestSuite) SetupTest() { 22 | suite.Factory = NewMemoryStoreFactory() 23 | } 24 | 25 | func (suite *MemoryStoreTestSuite) TearDownTest() { 26 | suite.Factory.Shutter() 27 | } 28 | -------------------------------------------------------------------------------- /go/chunks/put_cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package chunks 6 | 7 | import ( 8 | "sync" 9 | 10 | "github.com/attic-labs/noms/go/hash" 11 | ) 12 | 13 | func newUnwrittenPutCache() *unwrittenPutCache { 14 | return &unwrittenPutCache{map[hash.Hash]Chunk{}, &sync.Mutex{}} 15 | } 16 | 17 | type unwrittenPutCache struct { 18 | unwrittenPuts map[hash.Hash]Chunk 19 | mu *sync.Mutex 20 | } 21 | 22 | func (p *unwrittenPutCache) Add(c Chunk) bool { 23 | p.mu.Lock() 24 | defer p.mu.Unlock() 25 | if _, ok := p.unwrittenPuts[c.Hash()]; !ok { 26 | p.unwrittenPuts[c.Hash()] = c 27 | return true 28 | } 29 | 30 | return false 31 | } 32 | 33 | func (p *unwrittenPutCache) Has(c Chunk) (has bool) { 34 | p.mu.Lock() 35 | defer p.mu.Unlock() 36 | _, has = p.unwrittenPuts[c.Hash()] 37 | return 38 | } 39 | 40 | func (p *unwrittenPutCache) Get(r hash.Hash) Chunk { 41 | p.mu.Lock() 42 | defer p.mu.Unlock() 43 | if c, ok := p.unwrittenPuts[r]; ok { 44 | return c 45 | } 46 | return EmptyChunk 47 | } 48 | 49 | func (p *unwrittenPutCache) Clear(chunks []Chunk) { 50 | p.mu.Lock() 51 | defer p.mu.Unlock() 52 | for _, c := range chunks { 53 | delete(p.unwrittenPuts, c.Hash()) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /go/chunks/test_utils.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package chunks 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/d" 9 | "github.com/attic-labs/noms/go/hash" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func assertInputInStore(input string, h hash.Hash, s ChunkStore, assert *assert.Assertions) { 14 | chunk := s.Get(h) 15 | assert.False(chunk.IsEmpty(), "Shouldn't get empty chunk for %s", h.String()) 16 | assert.Equal(input, string(chunk.Data())) 17 | } 18 | 19 | func assertInputNotInStore(input string, h hash.Hash, s ChunkStore, assert *assert.Assertions) { 20 | chunk := s.Get(h) 21 | assert.True(chunk.IsEmpty(), "Shouldn't get non-empty chunk for %s: %v", h.String(), chunk) 22 | } 23 | 24 | type TestStorage struct { 25 | MemoryStorage 26 | } 27 | 28 | func (t *TestStorage) NewView() *TestStoreView { 29 | return &TestStoreView{ChunkStore: t.MemoryStorage.NewView()} 30 | } 31 | 32 | type TestStoreView struct { 33 | ChunkStore 34 | Reads int 35 | Hases int 36 | Writes int 37 | } 38 | 39 | func (s *TestStoreView) Get(h hash.Hash) Chunk { 40 | s.Reads++ 41 | return s.ChunkStore.Get(h) 42 | } 43 | 44 | func (s *TestStoreView) GetMany(hashes hash.HashSet, foundChunks chan *Chunk) { 45 | s.Reads += len(hashes) 46 | s.ChunkStore.GetMany(hashes, foundChunks) 47 | } 48 | 49 | func (s *TestStoreView) Has(h hash.Hash) bool { 50 | s.Hases++ 51 | return s.ChunkStore.Has(h) 52 | } 53 | 54 | func (s *TestStoreView) HasMany(hashes hash.HashSet) hash.HashSet { 55 | s.Hases += len(hashes) 56 | return s.ChunkStore.HasMany(hashes) 57 | } 58 | 59 | func (s *TestStoreView) Put(c Chunk) { 60 | s.Writes++ 61 | s.ChunkStore.Put(c) 62 | } 63 | 64 | type TestStoreFactory struct { 65 | stores map[string]*TestStorage 66 | } 67 | 68 | func NewTestStoreFactory() *TestStoreFactory { 69 | return &TestStoreFactory{map[string]*TestStorage{}} 70 | } 71 | 72 | func (f *TestStoreFactory) CreateStore(ns string) ChunkStore { 73 | if f.stores == nil { 74 | d.Panic("Cannot use TestStoreFactory after Shutter().") 75 | } 76 | if ts, present := f.stores[ns]; present { 77 | return ts.NewView() 78 | } 79 | f.stores[ns] = &TestStorage{} 80 | return f.stores[ns].NewView() 81 | } 82 | 83 | func (f *TestStoreFactory) Shutter() { 84 | f.stores = nil 85 | } 86 | -------------------------------------------------------------------------------- /go/constants/http.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package constants 6 | 7 | const ( 8 | RootPath = "/root/" 9 | GetRefsPath = "/getRefs/" 10 | GetBlobPath = "/getBlob/" 11 | HasRefsPath = "/hasRefs/" 12 | WriteValuePath = "/writeValue/" 13 | BasePath = "/" 14 | 15 | GraphQLPath = "/graphql/" 16 | StatsPath = "/stats/" 17 | ) 18 | -------------------------------------------------------------------------------- /go/constants/version.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // Package constants collects common constants used in Noms, such as the Noms data format version. 6 | package constants 7 | 8 | const NomsVersion = "7.18" 9 | 10 | var NomsGitSHA = "" 11 | -------------------------------------------------------------------------------- /go/d/check_error.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package d 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | 11 | "github.com/attic-labs/kingpin" 12 | "github.com/attic-labs/noms/go/util/exit" 13 | ) 14 | 15 | func CheckError(err error) { 16 | if err != nil { 17 | fmt.Fprintf(os.Stderr, "error: %s\n", err) 18 | kingpin.Usage() 19 | exit.Fail() 20 | } 21 | } 22 | 23 | func CheckErrorNoUsage(err error) { 24 | if err != nil { 25 | fmt.Fprintf(os.Stderr, "error: %s\n", err) 26 | exit.Fail() 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /go/datas/commit_options.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package datas 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/merge" 9 | "github.com/attic-labs/noms/go/types" 10 | ) 11 | 12 | // CommitOptions is used to pass options into Commit. 13 | type CommitOptions struct { 14 | // Parents, if provided is the parent commits of the commit we are 15 | // creating. 16 | Parents types.Set 17 | 18 | // Meta is a Struct that describes arbitrary metadata about this Commit, 19 | // e.g. a timestamp or descriptive text. 20 | Meta types.Struct 21 | 22 | // Policy will be called to attempt to merge this Commit with the current 23 | // Head, if this is not a fast-forward. If Policy is nil, no merging will 24 | // be attempted. Note that because Commit() retries in some cases, Policy 25 | // might also be called multiple times with different values. 26 | Policy merge.Policy 27 | } 28 | -------------------------------------------------------------------------------- /go/datas/serialize_hashes.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package datas 6 | 7 | import ( 8 | "encoding/binary" 9 | "io" 10 | 11 | "github.com/attic-labs/noms/go/chunks" 12 | "github.com/attic-labs/noms/go/d" 13 | "github.com/attic-labs/noms/go/hash" 14 | ) 15 | 16 | func serializedLength(batch chunks.ReadBatch) uint32 { 17 | return uint32(len(batch)*hash.ByteLen + binary.Size(uint32(0))) 18 | } 19 | 20 | func serializeHashes(w io.Writer, batch chunks.ReadBatch) { 21 | err := binary.Write(w, binary.BigEndian, uint32(len(batch))) // 4 billion hashes is probably absurd. Maybe this should be smaller? 22 | d.PanicIfError(err) 23 | for h := range batch { 24 | serializeHash(w, h) 25 | } 26 | } 27 | 28 | func serializeHash(w io.Writer, h hash.Hash) { 29 | _, err := w.Write(h[:]) 30 | d.PanicIfError(err) 31 | } 32 | 33 | func deserializeHashes(reader io.Reader) hash.HashSlice { 34 | count := uint32(0) 35 | err := binary.Read(reader, binary.BigEndian, &count) 36 | d.PanicIfError(err) 37 | 38 | hashes := make(hash.HashSlice, count) 39 | for i := range hashes { 40 | hashes[i] = deserializeHash(reader) 41 | } 42 | return hashes 43 | } 44 | 45 | func deserializeHash(reader io.Reader) hash.Hash { 46 | h := hash.Hash{} 47 | n, err := io.ReadFull(reader, h[:]) 48 | d.PanicIfError(err) 49 | d.PanicIfFalse(int(hash.ByteLen) == n) 50 | return h 51 | } 52 | -------------------------------------------------------------------------------- /go/datas/serialize_hashes_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package datas 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | 11 | "github.com/attic-labs/noms/go/chunks" 12 | "github.com/attic-labs/noms/go/hash" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestHashRoundTrip(t *testing.T) { 17 | b := &bytes.Buffer{} 18 | input := chunks.ReadBatch{ 19 | hash.Parse("00000000000000000000000000000000"): nil, 20 | hash.Parse("00000000000000000000000000000001"): nil, 21 | hash.Parse("00000000000000000000000000000002"): nil, 22 | hash.Parse("00000000000000000000000000000003"): nil, 23 | } 24 | defer input.Close() 25 | 26 | serializeHashes(b, input) 27 | serializedLen := b.Len() 28 | output := deserializeHashes(b) 29 | assert.Len(t, output, len(input), "Output has different number of elements than input: %v, %v", output, input) 30 | for _, h := range output { 31 | _, present := input[h] 32 | assert.True(t, present, "%s is in output but not in input", h) 33 | } 34 | 35 | assert.Equal(t, uint32(serializedLen), serializedLength(input)) 36 | } 37 | -------------------------------------------------------------------------------- /go/hash/base32.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package hash 6 | 7 | import "encoding/base32" 8 | 9 | var encoding = base32.NewEncoding("0123456789abcdefghijklmnopqrstuv") 10 | 11 | func encode(data []byte) string { 12 | return encoding.EncodeToString(data) 13 | } 14 | 15 | func decode(s string) []byte { 16 | slice, _ := encoding.DecodeString(s) 17 | return slice 18 | } 19 | -------------------------------------------------------------------------------- /go/hash/base32_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package hash 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestBase32Encode(t *testing.T) { 14 | assert := assert.New(t) 15 | 16 | d := make([]byte, 20, 20) 17 | assert.Equal("00000000000000000000000000000000", encode(d)) 18 | d[19] = 1 19 | assert.Equal("00000000000000000000000000000001", encode(d)) 20 | d[19] = 10 21 | assert.Equal("0000000000000000000000000000000a", encode(d)) 22 | d[19] = 20 23 | assert.Equal("0000000000000000000000000000000k", encode(d)) 24 | d[19] = 31 25 | assert.Equal("0000000000000000000000000000000v", encode(d)) 26 | d[19] = 32 27 | assert.Equal("00000000000000000000000000000010", encode(d)) 28 | d[19] = 63 29 | assert.Equal("0000000000000000000000000000001v", encode(d)) 30 | d[19] = 64 31 | assert.Equal("00000000000000000000000000000020", encode(d)) 32 | 33 | // Largest! 34 | for i := 0; i < 20; i++ { 35 | d[i] = 0xff 36 | } 37 | assert.Equal("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv", encode(d)) 38 | } 39 | 40 | func TestBase32Decode(t *testing.T) { 41 | assert := assert.New(t) 42 | 43 | d := make([]byte, 20, 20) 44 | assert.Equal(d, decode("00000000000000000000000000000000")) 45 | 46 | d[19] = 1 47 | assert.Equal(d, decode("00000000000000000000000000000001")) 48 | d[19] = 10 49 | assert.Equal(d, decode("0000000000000000000000000000000a")) 50 | d[19] = 20 51 | assert.Equal(d, decode("0000000000000000000000000000000k")) 52 | d[19] = 31 53 | assert.Equal(d, decode("0000000000000000000000000000000v")) 54 | d[19] = 32 55 | assert.Equal(d, decode("00000000000000000000000000000010")) 56 | d[19] = 63 57 | assert.Equal(d, decode("0000000000000000000000000000001v")) 58 | d[19] = 64 59 | assert.Equal(d, decode("00000000000000000000000000000020")) 60 | 61 | // Largest! 62 | for i := 0; i < 20; i++ { 63 | d[i] = 0xff 64 | } 65 | assert.Equal(d, decode("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv")) 66 | } 67 | -------------------------------------------------------------------------------- /go/hash/hash_slice.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package hash 6 | 7 | type HashSlice []Hash 8 | 9 | func (rs HashSlice) Len() int { 10 | return len(rs) 11 | } 12 | 13 | func (rs HashSlice) Less(i, j int) bool { 14 | return rs[i].Less(rs[j]) 15 | } 16 | 17 | func (rs HashSlice) Swap(i, j int) { 18 | rs[i], rs[j] = rs[j], rs[i] 19 | } 20 | 21 | func (rs HashSlice) Equals(other HashSlice) bool { 22 | if len(rs) != len(other) { 23 | return false 24 | } 25 | for i := 0; i < len(rs); i++ { 26 | if rs[i] != other[i] { 27 | return false 28 | } 29 | } 30 | return true 31 | } 32 | 33 | func (rs HashSlice) HashSet() HashSet { 34 | hs := make(HashSet, len(rs)) 35 | for _, h := range rs { 36 | hs[h] = struct{}{} 37 | } 38 | 39 | return hs 40 | } 41 | -------------------------------------------------------------------------------- /go/hash/hash_slice_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package hash 6 | 7 | import ( 8 | "sort" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestHashSliceSort(t *testing.T) { 15 | assert := assert.New(t) 16 | 17 | rs := HashSlice{} 18 | for i := 1; i <= 3; i++ { 19 | for j := 1; j <= 3; j++ { 20 | h := Hash{} 21 | for k := 1; k <= j; k++ { 22 | h[k-1] = byte(i) 23 | } 24 | rs = append(rs, h) 25 | } 26 | } 27 | 28 | rs2 := HashSlice(make([]Hash, len(rs))) 29 | copy(rs2, rs) 30 | sort.Sort(sort.Reverse(rs2)) 31 | assert.False(rs.Equals(rs2)) 32 | 33 | sort.Sort(rs2) 34 | assert.True(rs.Equals(rs2)) 35 | } 36 | -------------------------------------------------------------------------------- /go/merge/three_way_set_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package merge 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/attic-labs/noms/go/types" 11 | "github.com/stretchr/testify/suite" 12 | ) 13 | 14 | func TestThreeWaySetMerge(t *testing.T) { 15 | suite.Run(t, &ThreeWaySetMergeSuite{}) 16 | } 17 | 18 | type items []interface{} 19 | 20 | func (kv items) items() []interface{} { 21 | return kv 22 | } 23 | 24 | type ThreeWaySetMergeSuite struct { 25 | ThreeWayMergeSuite 26 | } 27 | 28 | func (s *ThreeWaySetMergeSuite) SetupSuite() { 29 | s.create = func(i seq) (val types.Value) { 30 | if i != nil { 31 | keyValues := valsToTypesValues(s.create, i.items()...) 32 | val = types.NewSet(s.vs, keyValues...) 33 | } 34 | return 35 | } 36 | s.typeStr = "Set" 37 | } 38 | 39 | var ( 40 | flat = items{"a1", "a2", "a3", "a4"} 41 | flatA = items{"a1", "a2", "a5", "a6"} 42 | flatB = items{"a1", "a4", "a7", "a5"} 43 | flatM = items{"a1", "a5", "a6", "a7"} 44 | 45 | ss1 = items{} 46 | ss1a = items{"k1", flatA, items{"a", 0}} 47 | ss1b = items{"k1", items{"a", 0}, flatB} 48 | ss1Merged = items{"k1", items{"a", 0}, flatA, flatB} 49 | ) 50 | 51 | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_DoNothing() { 52 | s.tryThreeWayMerge(nil, nil, flat, flat) 53 | } 54 | 55 | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_Primitives() { 56 | s.tryThreeWayMerge(flatA, flatB, flat, flatM) 57 | s.tryThreeWayMerge(flatB, flatA, flat, flatM) 58 | } 59 | 60 | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_HandleEmpty() { 61 | s.tryThreeWayMerge(ss1a, ss1b, ss1, ss1Merged) 62 | s.tryThreeWayMerge(ss1b, ss1a, ss1, ss1Merged) 63 | } 64 | 65 | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_HandleNil() { 66 | s.tryThreeWayMerge(ss1a, ss1b, nil, ss1Merged) 67 | s.tryThreeWayMerge(ss1b, ss1a, nil, ss1Merged) 68 | } 69 | 70 | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_Refs() { 71 | strRef := s.vs.WriteValue(types.NewStruct("Foo", types.StructData{"life": types.Number(42)})) 72 | 73 | m := items{s.vs.WriteValue(s.create(flatA)), s.vs.WriteValue(s.create(flatB))} 74 | ma := items{"r1", s.vs.WriteValue(s.create(flatA))} 75 | mb := items{"r1", strRef, s.vs.WriteValue(s.create(flatA))} 76 | mMerged := items{"r1", strRef, s.vs.WriteValue(s.create(flatA))} 77 | 78 | s.tryThreeWayMerge(ma, mb, m, mMerged) 79 | s.tryThreeWayMerge(mb, ma, m, mMerged) 80 | } 81 | 82 | func (s *ThreeWaySetMergeSuite) TestThreeWayMerge_ImmediateConflict() { 83 | s.tryThreeWayConflict(types.NewMap(s.vs), s.create(ss1b), s.create(ss1), "Cannot merge Map<> with "+s.typeStr) 84 | s.tryThreeWayConflict(s.create(ss1b), types.NewMap(s.vs), s.create(ss1), "Cannot merge "+s.typeStr) 85 | } 86 | -------------------------------------------------------------------------------- /go/merge/three_way_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package merge 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/attic-labs/noms/go/chunks" 11 | "github.com/attic-labs/noms/go/types" 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/suite" 14 | ) 15 | 16 | type seq interface { 17 | items() []interface{} 18 | } 19 | 20 | type ThreeWayMergeSuite struct { 21 | suite.Suite 22 | vs *types.ValueStore 23 | create func(seq) types.Value 24 | typeStr string 25 | } 26 | 27 | func (s *ThreeWayMergeSuite) SetupTest() { 28 | storage := &chunks.MemoryStorage{} 29 | s.vs = types.NewValueStore(storage.NewView()) 30 | } 31 | 32 | func (s *ThreeWayMergeSuite) TearDownTest() { 33 | s.vs.Close() 34 | } 35 | 36 | func (s *ThreeWayMergeSuite) tryThreeWayMerge(a, b, p, exp seq) { 37 | merged, err := ThreeWay(s.create(a), s.create(b), s.create(p), s.vs, nil, nil) 38 | if s.NoError(err) { 39 | expected := s.create(exp) 40 | s.True(expected.Equals(merged), "%s != %s", types.EncodedValue(expected), types.EncodedValue(merged)) 41 | } 42 | } 43 | 44 | func (s *ThreeWayMergeSuite) tryThreeWayConflict(a, b, p types.Value, contained string) { 45 | m, err := ThreeWay(a, b, p, s.vs, nil, nil) 46 | if s.Error(err) { 47 | s.Contains(err.Error(), contained) 48 | return 49 | } 50 | s.Fail("Expected error!", "Got successful merge: %s", types.EncodedValue(m)) 51 | } 52 | 53 | func valsToTypesValues(f func(seq) types.Value, items ...interface{}) []types.Value { 54 | keyValues := []types.Value{} 55 | for _, e := range items { 56 | v := valToTypesValue(f, e) 57 | keyValues = append(keyValues, v) 58 | } 59 | return keyValues 60 | } 61 | 62 | func valToTypesValue(f func(seq) types.Value, v interface{}) types.Value { 63 | var v1 types.Value 64 | switch t := v.(type) { 65 | case string: 66 | v1 = types.String(t) 67 | case int: 68 | v1 = types.Number(t) 69 | case seq: 70 | v1 = f(t) 71 | case types.Value: 72 | v1 = t 73 | } 74 | return v1 75 | } 76 | 77 | func TestThreeWayMerge_PrimitiveConflict(t *testing.T) { 78 | threeWayConflict := func(a, b, p types.Value, contained string) { 79 | mrgr := &merger{} 80 | m, err := mrgr.threeWay(a, b, p, nil) 81 | if assert.Error(t, err) { 82 | assert.Contains(t, err.Error(), contained) 83 | return 84 | } 85 | assert.Fail(t, "Expected error!", "Got successful merge: %s", types.EncodedValue(m)) 86 | } 87 | 88 | a, b, p := types.Number(7), types.String("nope"), types.String("parent") 89 | 90 | threeWayConflict(a, b, p, "Number and String on top of") 91 | threeWayConflict(b, a, p, "String and Number on top of") 92 | } 93 | -------------------------------------------------------------------------------- /go/nbs/README.md: -------------------------------------------------------------------------------- 1 | # Noms Block Store 2 | 3 | A horizontally-scalable storage backend for Noms. 4 | 5 | ## Overview 6 | 7 | NBS is a storage layer optimized for the needs of the [Noms](https://github.com/attic-labs/noms) database. 8 | 9 | NBS can run in two configurations: either backed by local disk, or [backed by Amazon AWS](https://github.com/attic-labs/noms/blob/master/go/nbs/NBS-on-AWS.md). 10 | 11 | When backed by local disk, NBS is significantly faster than LevelDB for our workloads and supports full multiprocess concurrency. 12 | 13 | When backed by AWS, NBS stores its data mainly in S3, along with a single DynamoDB item. This configuration makes Noms "[effectively CA](https://research.google.com/pubs/pub45855.html)", in the sense that Noms is always consistent, and Noms+NBS is as available as DynamoDB and S3 are. This configuration also gives Noms the cost profile of S3 with power closer to that of a traditional database. 14 | 15 | ## Details 16 | 17 | * NBS provides storage for a content-addressed DAG of nodes (with exactly one root), where each node is encoded as a sequence of bytes and addressed by a 20-byte hash of the byte-sequence. 18 | * There is no `update` or `delete` -- only `insert`, `update root` and `garbage collect`. 19 | * Insertion of any novel byte-sequence is durable only upon updating the root. 20 | * File-level multiprocess concurrency is supported, with optimistic locking for multiple writers. 21 | * Writers need not worry about re-writing duplicate chunks. NBS will efficiently detect and drop (most) duplicates. 22 | 23 | ## Perf 24 | 25 | For the file back-end, perf is substantially better than LevelDB mainly because LDB spends substantial IO with the goal of keeping KV pairs in key-order which doesn't benenfit Noms at all. NBS locates related chunks together and thus reading data from a NBS store can be done quite alot faster. As an example, storing & retrieving a 1.1GB MP4 video file on a MBP i5 2.9Ghz: 26 | 27 | * LDB 28 | * Initial import: 44 MB/s, size on disk: 1.1 GB. 29 | * Import exact same bytes: 35 MB/s, size on disk: 1.4 GB. 30 | * Export: 60 MB/s 31 | * NBS 32 | * Initial import: 72 MB/s, size on disk: 1.1 GB. 33 | * Import exact same bytes: 92 MB/s, size on disk: 1.1GB. 34 | * Export: 300 MB/s 35 | 36 | ## Status 37 | 38 | NBS is more-or-less "beta". There's still [work we want to do](https://github.com/attic-labs/noms/issues?q=is%3Aopen+is%3Aissue+label%3ANBS), but it now works better than LevelDB for our purposes and so we have made it the default local backend for Noms: 39 | 40 | ```shell 41 | # This uses nbs locally: 42 | ./csv-import foo.csv /Users/bob/csv-store::data 43 | ``` 44 | 45 | The AWS backend is available via the `aws:` scheme: 46 | 47 | ```shell 48 | ./csv-import foo.csv aws:table/bucket/database::data 49 | ``` 50 | -------------------------------------------------------------------------------- /go/nbs/aws_chunk_source_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package nbs 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestAWSChunkSource(t *testing.T) { 14 | chunks := [][]byte{ 15 | []byte("hello2"), 16 | []byte("goodbye2"), 17 | []byte("badbye2"), 18 | } 19 | tableData, h := buildTable(chunks) 20 | 21 | s3 := makeFakeS3(t) 22 | ddb := makeFakeDDB(t) 23 | 24 | s3or := &s3ObjectReader{s3, "bucket", nil, nil} 25 | dts := &ddbTableStore{ddb, "table", nil, nil} 26 | 27 | makeSrc := func(chunkMax int, ic *indexCache) chunkSource { 28 | return newAWSChunkSource( 29 | dts, 30 | s3or, 31 | awsLimits{itemMax: maxDynamoItemSize, chunkMax: uint32(chunkMax)}, 32 | h, 33 | uint32(len(chunks)), 34 | ic, 35 | &Stats{}, 36 | ) 37 | } 38 | 39 | t.Run("Dynamo", func(t *testing.T) { 40 | ddb.putData(fmtTableName(h), tableData) 41 | 42 | t.Run("NoIndexCache", func(t *testing.T) { 43 | src := makeSrc(len(chunks)+1, nil) 44 | assertChunksInReader(chunks, src, assert.New(t)) 45 | }) 46 | 47 | t.Run("WithIndexCache", func(t *testing.T) { 48 | assert := assert.New(t) 49 | index := parseTableIndex(tableData) 50 | cache := newIndexCache(1024) 51 | cache.put(h, index) 52 | 53 | baseline := ddb.numGets 54 | src := makeSrc(len(chunks)+1, cache) 55 | 56 | // constructing the table reader shouldn't have resulted in any reads 57 | assert.Zero(ddb.numGets - baseline) 58 | assertChunksInReader(chunks, src, assert) 59 | }) 60 | }) 61 | 62 | t.Run("S3", func(t *testing.T) { 63 | s3.data[h.String()] = tableData 64 | 65 | t.Run("NoIndexCache", func(t *testing.T) { 66 | src := makeSrc(len(chunks)-1, nil) 67 | assertChunksInReader(chunks, src, assert.New(t)) 68 | }) 69 | 70 | t.Run("WithIndexCache", func(t *testing.T) { 71 | assert := assert.New(t) 72 | index := parseTableIndex(tableData) 73 | cache := newIndexCache(1024) 74 | cache.put(h, index) 75 | 76 | baseline := s3.getCount 77 | src := makeSrc(len(chunks)-1, cache) 78 | 79 | // constructing the table reader shouldn't have resulted in any reads 80 | assert.Zero(s3.getCount - baseline) 81 | assertChunksInReader(chunks, src, assert) 82 | }) 83 | }) 84 | } 85 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/.gitignore: -------------------------------------------------------------------------------- 1 | benchmarks 2 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/cachedrop/drop_cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "log" 9 | "os" 10 | ) 11 | 12 | const dropCaches = "/proc/sys/vm/drop_caches" 13 | 14 | func main() { 15 | f, err := os.OpenFile(dropCaches, os.O_WRONLY, 0666) 16 | if err != nil { 17 | log.Fatalln(err) 18 | } 19 | 20 | if _, err := f.WriteString("1"); err != nil { 21 | log.Fatalln(err) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/chunker/.gitignore: -------------------------------------------------------------------------------- 1 | chunker 2 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/chunker/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "os" 9 | 10 | "github.com/attic-labs/kingpin" 11 | "github.com/dustin/go-humanize" 12 | 13 | "github.com/attic-labs/noms/go/d" 14 | "github.com/attic-labs/noms/go/nbs/benchmarks/gen" 15 | ) 16 | 17 | const ( 18 | KB = uint64(1 << 10) 19 | MB = uint64(1 << 20) 20 | averageChunkSize = 4 * KB 21 | ) 22 | 23 | var ( 24 | genSize = kingpin.Flag("gen", "MiB of data to generate and chunk").Default("1024").Uint64() 25 | chunkInput = kingpin.Flag("chunk", "Treat arg as data file to chunk").Bool() 26 | fileName = kingpin.Arg("file", "filename").String() 27 | ) 28 | 29 | func main() { 30 | kingpin.Parse() 31 | 32 | var fd *os.File 33 | var err error 34 | if *chunkInput { 35 | fd, err = os.Open(*fileName) 36 | d.Chk.NoError(err) 37 | defer fd.Close() 38 | } else { 39 | fd, err = gen.OpenOrGenerateDataFile(*fileName, (*genSize)*humanize.MiByte) 40 | d.Chk.NoError(err) 41 | defer fd.Close() 42 | } 43 | 44 | cm := gen.OpenOrBuildChunkMap(*fileName+".chunks", fd) 45 | defer cm.Close() 46 | 47 | return 48 | } 49 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/drop_cache_linux.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // +build linux 6 | 7 | package main 8 | 9 | import "os/exec" 10 | 11 | func dropCache() error { 12 | return exec.Command("./drop_cache").Run() 13 | } 14 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/drop_cache_other.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // +build !linux 6 | 7 | package main 8 | 9 | func dropCache() error { 10 | return nil 11 | } 12 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/file_block_store.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "bufio" 9 | "bytes" 10 | "io" 11 | 12 | "github.com/attic-labs/noms/go/chunks" 13 | "github.com/attic-labs/noms/go/hash" 14 | "github.com/dustin/go-humanize" 15 | ) 16 | 17 | type fileBlockStore struct { 18 | bw *bufio.Writer 19 | w io.WriteCloser 20 | } 21 | 22 | func newFileBlockStore(w io.WriteCloser) chunks.ChunkStore { 23 | return fileBlockStore{bufio.NewWriterSize(w, humanize.MiByte), w} 24 | } 25 | 26 | func (fb fileBlockStore) Get(h hash.Hash) chunks.Chunk { 27 | panic("not impl") 28 | } 29 | 30 | func (fb fileBlockStore) GetMany(hashes hash.HashSet, foundChunks chan *chunks.Chunk) { 31 | panic("not impl") 32 | } 33 | 34 | func (fb fileBlockStore) Has(h hash.Hash) bool { 35 | panic("not impl") 36 | } 37 | 38 | func (fb fileBlockStore) HasMany(hashes hash.HashSet) (present hash.HashSet) { 39 | panic("not impl") 40 | } 41 | 42 | func (fb fileBlockStore) Put(c chunks.Chunk) { 43 | io.Copy(fb.bw, bytes.NewReader(c.Data())) 44 | } 45 | 46 | func (fb fileBlockStore) Version() string { 47 | panic("not impl") 48 | } 49 | 50 | func (fb fileBlockStore) Close() error { 51 | fb.w.Close() 52 | return nil 53 | } 54 | 55 | func (fb fileBlockStore) Rebase() {} 56 | 57 | func (fb fileBlockStore) Stats() interface{} { 58 | return nil 59 | } 60 | 61 | func (fb fileBlockStore) StatsSummary() string { 62 | return "Unsupported" 63 | } 64 | 65 | func (fb fileBlockStore) Root() hash.Hash { 66 | return hash.Hash{} 67 | } 68 | 69 | func (fb fileBlockStore) Commit(current, last hash.Hash) bool { 70 | fb.bw.Flush() 71 | return true 72 | } 73 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/gen/rolling_value_hasher.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package gen 6 | 7 | import "github.com/kch42/buzhash" 8 | 9 | const ( 10 | chunkPattern = uint32(1<<12 - 1) // Avg Chunk Size of 4k 11 | 12 | // The window size to use for computing the rolling hash. This is way more than necessary assuming random data (two bytes would be sufficient with a target chunk size of 4k). The benefit of a larger window is it allows for better distribution on input with lower entropy. At a target chunk size of 4k, any given byte changing has roughly a 1.5% chance of affecting an existing boundary, which seems like an acceptable trade-off. 13 | chunkWindow = uint32(64) 14 | ) 15 | 16 | type rollingValueHasher struct { 17 | bz *buzhash.BuzHash 18 | } 19 | 20 | func newRollingValueHasher() *rollingValueHasher { 21 | return &rollingValueHasher{buzhash.NewBuzHash(chunkWindow)} 22 | } 23 | 24 | func (rv *rollingValueHasher) HashByte(b byte) bool { 25 | rv.bz.HashByte(b) 26 | return rv.bz.Sum32()&chunkPattern == chunkPattern 27 | } 28 | -------------------------------------------------------------------------------- /go/nbs/benchmarks/null_block_store.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/chunks" 9 | "github.com/attic-labs/noms/go/hash" 10 | ) 11 | 12 | type nullBlockStore struct { 13 | bogus int32 14 | } 15 | 16 | func newNullBlockStore() chunks.ChunkStore { 17 | return nullBlockStore{} 18 | } 19 | 20 | func (nb nullBlockStore) Get(h hash.Hash) chunks.Chunk { 21 | panic("not impl") 22 | } 23 | 24 | func (nb nullBlockStore) GetMany(hashes hash.HashSet, foundChunks chan *chunks.Chunk) { 25 | panic("not impl") 26 | } 27 | 28 | func (nb nullBlockStore) Has(h hash.Hash) bool { 29 | panic("not impl") 30 | } 31 | 32 | func (nb nullBlockStore) HasMany(hashes hash.HashSet) (present hash.HashSet) { 33 | panic("not impl") 34 | } 35 | 36 | func (nb nullBlockStore) Put(c chunks.Chunk) {} 37 | 38 | func (nb nullBlockStore) Version() string { 39 | panic("not impl") 40 | } 41 | 42 | func (nb nullBlockStore) Close() error { 43 | return nil 44 | } 45 | 46 | func (nb nullBlockStore) Rebase() {} 47 | 48 | func (nb nullBlockStore) Stats() interface{} { 49 | return nil 50 | } 51 | 52 | func (nb nullBlockStore) StatsSummary() string { 53 | return "Unsupported" 54 | } 55 | 56 | func (nb nullBlockStore) Root() hash.Hash { 57 | return hash.Hash{} 58 | } 59 | 60 | func (nb nullBlockStore) Commit(current, last hash.Hash) bool { 61 | return true 62 | } 63 | -------------------------------------------------------------------------------- /go/nbs/cache.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package nbs 6 | 7 | import ( 8 | "io/ioutil" 9 | "os" 10 | 11 | "github.com/attic-labs/noms/go/chunks" 12 | "github.com/attic-labs/noms/go/d" 13 | "github.com/attic-labs/noms/go/hash" 14 | ) 15 | 16 | const ( 17 | defaultCacheMemTableSize uint64 = 1 << 27 // 128MiB 18 | ) 19 | 20 | func NewCache() *NomsBlockCache { 21 | dir, err := ioutil.TempDir("", "") 22 | d.PanicIfError(err) 23 | store := NewLocalStore(dir, defaultCacheMemTableSize) 24 | d.Chk.NoError(err, "opening put cache in %s", dir) 25 | return &NomsBlockCache{store, dir} 26 | } 27 | 28 | // NomsBlockCache holds Chunks, allowing them to be retrieved by hash or enumerated in hash order. 29 | type NomsBlockCache struct { 30 | chunks *NomsBlockStore 31 | dbDir string 32 | } 33 | 34 | // Insert stores c in the cache. 35 | func (nbc *NomsBlockCache) Insert(c chunks.Chunk) { 36 | d.PanicIfFalse(nbc.chunks.addChunk(addr(c.Hash()), c.Data())) 37 | } 38 | 39 | // Has checks if the chunk referenced by hash is in the cache. 40 | func (nbc *NomsBlockCache) Has(hash hash.Hash) bool { 41 | return nbc.chunks.Has(hash) 42 | } 43 | 44 | // HasMany returns a set containing the members of hashes present in the 45 | // cache. 46 | func (nbc *NomsBlockCache) HasMany(hashes hash.HashSet) hash.HashSet { 47 | return nbc.chunks.HasMany(hashes) 48 | } 49 | 50 | // Get retrieves the chunk referenced by hash. If the chunk is not present, 51 | // Get returns the empty Chunk. 52 | func (nbc *NomsBlockCache) Get(hash hash.Hash) chunks.Chunk { 53 | return nbc.chunks.Get(hash) 54 | } 55 | 56 | // GetMany gets the Chunks with |hashes| from the store. On return, 57 | // |foundChunks| will have been fully sent all chunks which have been 58 | // found. Any non-present chunks will silently be ignored. 59 | func (nbc *NomsBlockCache) GetMany(hashes hash.HashSet, foundChunks chan *chunks.Chunk) { 60 | nbc.chunks.GetMany(hashes, foundChunks) 61 | } 62 | 63 | // ExtractChunks writes the entire contents of the cache to chunkChan. The 64 | // chunks are extracted in insertion order. 65 | func (nbc *NomsBlockCache) ExtractChunks(chunkChan chan *chunks.Chunk) { 66 | nbc.chunks.extractChunks(chunkChan) 67 | } 68 | 69 | // Count returns the number of items in the cache. 70 | func (nbc *NomsBlockCache) Count() uint32 { 71 | return nbc.chunks.Count() 72 | } 73 | 74 | // Destroy drops the cache and deletes any backing storage. 75 | func (nbc *NomsBlockCache) Destroy() error { 76 | d.Chk.NoError(nbc.chunks.Close()) 77 | return os.RemoveAll(nbc.dbDir) 78 | } 79 | -------------------------------------------------------------------------------- /go/nbs/factory_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package nbs 6 | 7 | import ( 8 | "os" 9 | "path/filepath" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/attic-labs/noms/go/chunks" 14 | "github.com/attic-labs/noms/go/constants" 15 | "github.com/attic-labs/noms/go/hash" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func TestLocalStoreFactory(t *testing.T) { 20 | assert := assert.New(t) 21 | dir := makeTempDir(t) 22 | defer os.RemoveAll(dir) 23 | 24 | f := NewLocalStoreFactory(dir, 0, 8) 25 | stats := &Stats{} 26 | 27 | dbName := "db" 28 | store := f.CreateStore(dbName) 29 | 30 | c := chunks.NewChunk([]byte{0xff}) 31 | store.Put(c) 32 | assert.True(store.Commit(c.Hash(), hash.Hash{})) 33 | 34 | dbDir := filepath.Join(dir, dbName) 35 | exists, contents := fileManifest{dbDir}.ParseIfExists(stats, nil) 36 | assert.True(exists) 37 | assert.Len(contents.specs, 1) 38 | 39 | _, err := os.Stat(filepath.Join(dbDir, contents.specs[0].name.String())) 40 | assert.NoError(err) 41 | 42 | // Simulate another process writing a manifest. 43 | lock := computeAddr([]byte("locker")) 44 | newRoot := hash.Of([]byte("new root")) 45 | err = clobberManifest(dbDir, strings.Join([]string{StorageVersion, constants.NomsVersion, lock.String(), newRoot.String(), contents.specs[0].name.String(), "1"}, ":")) 46 | assert.NoError(err) 47 | 48 | cached := f.CreateStoreFromCache(dbName) 49 | assert.Equal(c.Hash(), cached.Root()) 50 | } 51 | -------------------------------------------------------------------------------- /go/nbs/frag/.gitignore: -------------------------------------------------------------------------------- 1 | frag 2 | -------------------------------------------------------------------------------- /go/nbs/mmap_table_reader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package nbs 6 | 7 | import ( 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestMmapTableReader(t *testing.T) { 17 | assert := assert.New(t) 18 | dir, err := ioutil.TempDir("", "") 19 | assert.NoError(err) 20 | defer os.RemoveAll(dir) 21 | 22 | fc := newFDCache(1) 23 | defer fc.Drop() 24 | 25 | chunks := [][]byte{ 26 | []byte("hello2"), 27 | []byte("goodbye2"), 28 | []byte("badbye2"), 29 | } 30 | 31 | tableData, h := buildTable(chunks) 32 | err = ioutil.WriteFile(filepath.Join(dir, h.String()), tableData, 0666) 33 | assert.NoError(err) 34 | 35 | trc := newMmapTableReader(dir, h, uint32(len(chunks)), nil, fc) 36 | assertChunksInReader(chunks, trc, assert) 37 | } 38 | -------------------------------------------------------------------------------- /go/nbs/persisting_chunk_source_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package nbs 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestPersistingChunkStoreEmpty(t *testing.T) { 14 | mt := newMemTable(testMemTableSize) 15 | ccs := newPersistingChunkSource(mt, nil, newFakeTablePersister(), make(chan struct{}, 1), &Stats{}) 16 | assert.Equal(t, addr{}, ccs.hash()) 17 | assert.Zero(t, ccs.count()) 18 | } 19 | 20 | type pausingFakeTablePersister struct { 21 | tablePersister 22 | trigger <-chan struct{} 23 | } 24 | 25 | func (ftp pausingFakeTablePersister) Persist(mt *memTable, haver chunkReader, stats *Stats) chunkSource { 26 | <-ftp.trigger 27 | return ftp.tablePersister.Persist(mt, haver, stats) 28 | } 29 | 30 | func TestPersistingChunkStore(t *testing.T) { 31 | assert := assert.New(t) 32 | mt := newMemTable(testMemTableSize) 33 | 34 | for _, c := range testChunks { 35 | assert.True(mt.addChunk(computeAddr(c), c)) 36 | } 37 | 38 | trigger := make(chan struct{}) 39 | ccs := newPersistingChunkSource(mt, nil, pausingFakeTablePersister{newFakeTablePersister(), trigger}, make(chan struct{}, 1), &Stats{}) 40 | 41 | assertChunksInReader(testChunks, ccs, assert) 42 | assert.EqualValues(mt.count(), ccs.getReader().count()) 43 | close(trigger) 44 | 45 | assert.NotEqual(addr{}, ccs.hash()) 46 | assert.EqualValues(len(testChunks), ccs.count()) 47 | assertChunksInReader(testChunks, ccs, assert) 48 | 49 | assert.Nil(ccs.mt) 50 | 51 | newChunk := []byte("additional") 52 | mt.addChunk(computeAddr(newChunk), newChunk) 53 | assert.NotEqual(mt.count(), ccs.count()) 54 | assert.False(ccs.has(computeAddr(newChunk))) 55 | } 56 | -------------------------------------------------------------------------------- /go/nbs/s3_table_reader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package nbs 6 | 7 | import ( 8 | "bytes" 9 | "io/ioutil" 10 | "net" 11 | "os" 12 | "testing" 13 | 14 | "golang.org/x/sys/unix" 15 | 16 | "github.com/aws/aws-sdk-go/service/s3" 17 | "github.com/stretchr/testify/assert" 18 | ) 19 | 20 | func TestS3TableReaderAt(t *testing.T) { 21 | s3 := makeFakeS3(t) 22 | 23 | chunks := [][]byte{ 24 | []byte("hello2"), 25 | []byte("goodbye2"), 26 | []byte("badbye2"), 27 | } 28 | 29 | tableData, h := buildTable(chunks) 30 | s3.data[h.String()] = tableData 31 | 32 | t.Run("TolerateFailingReads", func(t *testing.T) { 33 | assert := assert.New(t) 34 | 35 | baseline := s3.getCount 36 | tra := &s3TableReaderAt{&s3ObjectReader{makeFlakyS3(s3), "bucket", nil, nil}, h} 37 | scratch := make([]byte, len(tableData)) 38 | _, err := tra.ReadAtWithStats(scratch, 0, &Stats{}) 39 | assert.NoError(err) 40 | // constructing the table reader should have resulted in 2 reads 41 | assert.Equal(2, s3.getCount-baseline) 42 | assert.Equal(tableData, scratch) 43 | }) 44 | 45 | t.Run("WithTableCache", func(t *testing.T) { 46 | assert := assert.New(t) 47 | dir := makeTempDir(t) 48 | defer os.RemoveAll(dir) 49 | stats := &Stats{} 50 | 51 | tc := newFSTableCache(dir, uint64(2*len(tableData)), 4) 52 | tra := &s3TableReaderAt{&s3ObjectReader{s3, "bucket", nil, tc}, h} 53 | 54 | // First, read when table is not yet cached 55 | scratch := make([]byte, len(tableData)) 56 | baseline := s3.getCount 57 | _, err := tra.ReadAtWithStats(scratch, 0, stats) 58 | assert.NoError(err) 59 | assert.True(s3.getCount > baseline) 60 | 61 | // Cache the table and read again 62 | tc.store(h, bytes.NewReader(tableData), uint64(len(tableData))) 63 | baseline = s3.getCount 64 | _, err = tra.ReadAtWithStats(scratch, 0, stats) 65 | assert.NoError(err) 66 | assert.Zero(s3.getCount - baseline) 67 | }) 68 | } 69 | 70 | type flakyS3 struct { 71 | s3svc 72 | alreadyFailed map[string]struct{} 73 | } 74 | 75 | func makeFlakyS3(svc s3svc) *flakyS3 { 76 | return &flakyS3{svc, map[string]struct{}{}} 77 | } 78 | 79 | func (fs3 *flakyS3) GetObject(input *s3.GetObjectInput) (output *s3.GetObjectOutput, err error) { 80 | output, err = fs3.s3svc.GetObject(input) 81 | if _, ok := fs3.alreadyFailed[*input.Key]; !ok { 82 | fs3.alreadyFailed[*input.Key] = struct{}{} 83 | output.Body = ioutil.NopCloser(resettingReader{}) 84 | } 85 | return 86 | } 87 | 88 | type resettingReader struct{} 89 | 90 | func (rr resettingReader) Read(p []byte) (n int, err error) { 91 | return 0, &net.OpError{Op: "read", Net: "tcp", Err: &os.SyscallError{Syscall: "read", Err: unix.ECONNRESET}} 92 | } 93 | -------------------------------------------------------------------------------- /go/nbs/table_persister_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package nbs 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestPlanCompaction(t *testing.T) { 14 | assert := assert.New(t) 15 | tableContents := [][][]byte{ 16 | {[]byte("hello2"), []byte("goodbye2"), []byte("badbye2")}, 17 | {[]byte("red"), []byte("blue")}, 18 | {[]byte("solo")}, 19 | } 20 | 21 | var sources chunkSources 22 | var dataLens []uint64 23 | var totalUnc uint64 24 | for _, content := range tableContents { 25 | for _, chnk := range content { 26 | totalUnc += uint64(len(chnk)) 27 | } 28 | data, name := buildTable(content) 29 | src := chunkSourceAdapter{newTableReader(parseTableIndex(data), tableReaderAtFromBytes(data), fileBlockSize), name} 30 | dataLens = append(dataLens, uint64(len(data))-indexSize(src.count())-footerSize) 31 | sources = append(sources, src) 32 | } 33 | 34 | plan := planConjoin(sources, &Stats{}) 35 | 36 | var totalChunks uint32 37 | for i, src := range sources { 38 | assert.Equal(dataLens[i], plan.sources[i].dataLen) 39 | totalChunks += src.count() 40 | } 41 | 42 | idx := parseTableIndex(plan.mergedIndex) 43 | 44 | assert.Equal(totalChunks, idx.chunkCount) 45 | assert.Equal(totalUnc, idx.totalUncompressedData) 46 | 47 | tr := newTableReader(idx, tableReaderAtFromBytes(nil), fileBlockSize) 48 | for _, content := range tableContents { 49 | assertChunksInReader(content, tr, assert) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /go/nbs/test/manifest_clobber.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "flag" 9 | "log" 10 | "os" 11 | 12 | "golang.org/x/sys/unix" 13 | ) 14 | 15 | func main() { 16 | flag.Parse() 17 | 18 | if flag.NArg() < 3 { 19 | log.Fatalln("Not enough arguments") 20 | } 21 | 22 | l, err := os.Create(flag.Arg(0)) 23 | if err != nil { 24 | log.Fatalln(err) 25 | } 26 | defer l.Close() 27 | // lock released by closing l. 28 | err = unix.Flock(int(l.Fd()), unix.LOCK_EX|unix.LOCK_NB) 29 | if err == unix.EWOULDBLOCK { 30 | return 31 | } 32 | if err != nil { 33 | log.Fatalln(err) 34 | } 35 | 36 | // Clobber manifest file at flag.Arg(1) with contents at flag.Arg(2) 37 | m, err := os.Create(flag.Arg(1)) 38 | if err != nil { 39 | log.Fatalln(err) 40 | } 41 | defer m.Close() 42 | if _, err = m.WriteString(flag.Arg(2)); err != nil { 43 | log.Fatalln(err) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /go/nomdl/lexer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package nomdl 6 | 7 | import ( 8 | "fmt" 9 | "text/scanner" 10 | ) 11 | 12 | type lexer struct { 13 | scanner *scanner.Scanner 14 | peekToken rune 15 | } 16 | 17 | func (lex *lexer) next() rune { 18 | if lex.peekToken != 0 { 19 | tok := lex.peekToken 20 | lex.peekToken = 0 21 | return tok 22 | } 23 | 24 | return lex.scanner.Scan() 25 | } 26 | 27 | func (lex *lexer) peek() rune { 28 | if lex.peekToken != 0 { 29 | return lex.peekToken 30 | } 31 | tok := lex.scanner.Scan() 32 | lex.peekToken = tok 33 | return tok 34 | } 35 | 36 | func (lex *lexer) pos() scanner.Position { 37 | if lex.peekToken != 0 { 38 | panic("Cannot use pos after peek") 39 | } 40 | return lex.scanner.Pos() 41 | } 42 | 43 | func (lex *lexer) tokenText() string { 44 | if lex.peekToken != 0 { 45 | panic("Cannot use tokenText after peek") 46 | } 47 | return lex.scanner.TokenText() 48 | } 49 | 50 | func (lex *lexer) eat(expected rune) rune { 51 | tok := lex.next() 52 | lex.check(expected, tok) 53 | return tok 54 | } 55 | 56 | func (lex *lexer) eatIf(expected rune) bool { 57 | tok := lex.peek() 58 | if tok == expected { 59 | lex.next() 60 | return true 61 | } 62 | return false 63 | } 64 | 65 | func (lex *lexer) check(expected, actual rune) { 66 | if actual != expected { 67 | lex.tokenMismatch(expected, actual) 68 | } 69 | } 70 | 71 | func (lex *lexer) tokenMismatch(expected, actual rune) { 72 | raiseSyntaxError(fmt.Sprintf("Unexpected token %s, expected %s", scanner.TokenString(actual), scanner.TokenString(expected)), lex.pos()) 73 | } 74 | 75 | func (lex *lexer) unexpectedToken(actual rune) { 76 | raiseSyntaxError(fmt.Sprintf("Unexpected token %s", scanner.TokenString(actual)), lex.pos()) 77 | } 78 | 79 | func raiseSyntaxError(msg string, pos scanner.Position) { 80 | panic(syntaxError{ 81 | msg: msg, 82 | pos: pos, 83 | }) 84 | } 85 | 86 | type syntaxError struct { 87 | msg string 88 | pos scanner.Position 89 | } 90 | 91 | func (e syntaxError) Error() string { 92 | return fmt.Sprintf("%s, %s", e.msg, e.pos) 93 | } 94 | 95 | func catchSyntaxError(f func()) (errRes error) { 96 | defer func() { 97 | if err := recover(); err != nil { 98 | if err, ok := err.(syntaxError); ok { 99 | errRes = err 100 | return 101 | } 102 | panic(err) 103 | } 104 | }() 105 | 106 | f() 107 | return 108 | } 109 | -------------------------------------------------------------------------------- /go/perf/codec-perf-rig/.gitignore: -------------------------------------------------------------------------------- 1 | codec-perf-rig 2 | -------------------------------------------------------------------------------- /go/perf/codec-perf-rig/README.md: -------------------------------------------------------------------------------- 1 | This is a Go performance test rig which measures noms encoding/decoding along various axes. 2 | 3 | As of July 7, these are the numbers I get on my Macbook Pro 2.7 i5. 4 | 5 | Testing List: build 100000 scan 100000 insert 2000 6 | numbers (8 B) 451 ms (1.77 MB/s) 26 ms (30.26 MB/s) 275 ms (0.06 MB/s) 7 | strings (32 B) 444 ms (7.20 MB/s) 35 ms (90.04 MB/s) 321 ms (0.20 MB/s) 8 | structs (64 B) 682 ms (9.38 MB/s) 115 ms (55.34 MB/s) 239 ms (0.53 MB/s) 9 | 10 | Testing Set: build 100000 scan 100000 insert 2000 11 | numbers (8 B) 461 ms (1.73 MB/s) 23 ms (33.62 MB/s) 90 ms (0.18 MB/s) 12 | strings (32 B) 480 ms (6.67 MB/s) 28 ms (112.76 MB/s) 207 ms (0.31 MB/s) 13 | structs (64 B) 1608 ms (3.98 MB/s) 113 ms (56.29 MB/s) 306 ms (0.42 MB/s) 14 | 15 | Testing Map: build 100000 scan 100000 insert 2000 16 | numbers (8 B) 564 ms (1.42 MB/s) 44 ms (18.01 MB/s) 121 ms (0.13 MB/s) 17 | strings (32 B) 586 ms (5.46 MB/s) 61 ms (51.74 MB/s) 521 ms (0.12 MB/s) 18 | structs (64 B) 2005 ms (3.19 MB/s) 260 ms (24.61 MB/s) 530 ms (0.24 MB/s) 19 | 20 | Testing Blob: build 33 MB scan 33 MB 21 | 553 ms (60.60 MB/s) 429 ms (78.21 MB/s) 22 | 23 | 24 | -------------------------------------------------------------------------------- /go/perf/hash-perf-rig/README.md: -------------------------------------------------------------------------------- 1 | This is a performance test rig for the two main types of hashing we do in NOMS - buzhash and sha1. There's also support for sha256, sha512, and blake2b hash functions for comparison. 2 | 3 | As of May 9, these are the numbers I get on a macbook pro 3.1 GHz Intel Core i7. 4 | 5 | - no hashing : 3500 MB/s 6 | - sha1 only : 470 MB/s 7 | - sha256 only : 185 MB/s 8 | - sha512 only : 299 MB/s 9 | - blake2b only : 604 MB/s 10 | - bh only : 139 MB/s 11 | - sha1 and bh : 110 MB/s 12 | - sha256 and bh : 80 MB/s 13 | - sha512 and bh : 96 MB/s 14 | - blake2b and bh: 115 MB/s 15 | 16 | I think that in the no hashing case there is some compiler optimization going 17 | on because I note that if all I do is add a loop that reads out bytes one by 18 | one from the slice, it drops to 1000MB/s. 19 | 20 | One outcome of this is that there's no sense going to sha256 - we should just 21 | jump straight to sha512. -------------------------------------------------------------------------------- /go/perf/hash-perf-rig/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "crypto/sha1" 9 | "crypto/sha256" 10 | "crypto/sha512" 11 | "fmt" 12 | "hash" 13 | "io" 14 | "os" 15 | "time" 16 | 17 | "github.com/attic-labs/kingpin" 18 | "github.com/codahale/blake2" 19 | humanize "github.com/dustin/go-humanize" 20 | "github.com/kch42/buzhash" 21 | ) 22 | 23 | func main() { 24 | useSHA := kingpin.Flag("use-sha", "=no hashing, 1=sha1, 256=sha256, 512=sha512, blake=blake2b").String() 25 | useBH := kingpin.Flag("use-bh", "whether we buzhash the bytes").Bool() 26 | bigFile := kingpin.Arg("bigfile", "input file to chunk").Required().String() 27 | 28 | kingpin.Parse() 29 | 30 | bh := buzhash.NewBuzHash(64 * 8) 31 | f, _ := os.Open(*bigFile) 32 | defer f.Close() 33 | t0 := time.Now() 34 | buf := make([]byte, 4*1024) 35 | l := uint64(0) 36 | 37 | var h hash.Hash 38 | if *useSHA == "1" { 39 | h = sha1.New() 40 | } else if *useSHA == "256" { 41 | h = sha256.New() 42 | } else if *useSHA == "512" { 43 | h = sha512.New() 44 | } else if *useSHA == "blake" { 45 | h = blake2.NewBlake2B() 46 | } 47 | 48 | for { 49 | n, err := f.Read(buf) 50 | l += uint64(n) 51 | if err == io.EOF { 52 | break 53 | } 54 | s := buf[:n] 55 | if h != nil { 56 | h.Write(s) 57 | } 58 | if *useBH { 59 | bh.Write(s) 60 | } 61 | } 62 | 63 | t1 := time.Now() 64 | d := t1.Sub(t0) 65 | fmt.Printf("Read %s in %s (%s/s)\n", humanize.Bytes(l), d, humanize.Bytes(uint64(float64(l)/d.Seconds()))) 66 | digest := []byte{} 67 | if h != nil { 68 | fmt.Printf("%x\n", h.Sum(digest)) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /go/spec/util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package spec 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/d" 9 | "github.com/attic-labs/noms/go/hash" 10 | ) 11 | 12 | func CreateDatabaseSpecString(protocol, db string) string { 13 | return Spec{Protocol: protocol, DatabaseName: db}.String() 14 | } 15 | 16 | func CreateValueSpecString(protocol, db, path string) string { 17 | p, err := NewAbsolutePath(path) 18 | d.Chk.NoError(err) 19 | return Spec{Protocol: protocol, DatabaseName: db, Path: p}.String() 20 | } 21 | 22 | func CreateHashSpecString(protocol, db string, h hash.Hash) string { 23 | return Spec{Protocol: protocol, DatabaseName: db, Path: AbsolutePath{Hash: h}}.String() 24 | } 25 | -------------------------------------------------------------------------------- /go/types/blob_editor_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "math/rand" 9 | "testing" 10 | 11 | "io/ioutil" 12 | 13 | "bytes" 14 | 15 | "io" 16 | 17 | "github.com/attic-labs/noms/go/chunks" 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | func TestBlobReadWriteFuzzer(t *testing.T) { 22 | rounds := 1024 23 | operations := 512 24 | flushEvery := 16 25 | maxInsertCount := uint64(64) 26 | 27 | ts := &chunks.TestStorage{} 28 | cs := ts.NewView() 29 | vs := newValueStoreWithCacheAndPending(cs, 0, 0) 30 | 31 | r := rand.New(rand.NewSource(0)) 32 | nextRandInt := func(from, to uint64) uint64 { 33 | return from + uint64(float64(to-from)*r.Float64()) 34 | } 35 | 36 | for i := 0; i < rounds; i++ { 37 | b := NewBlob(vs) 38 | 39 | f, _ := ioutil.TempFile("", "buff") 40 | be := b.Edit() 41 | 42 | for j := 0; j < operations; j++ { 43 | if j%2 == 1 { 44 | // random read 45 | idx := nextRandInt(0, be.Len()) 46 | l := nextRandInt(0, be.Len()-idx) 47 | f.Seek(int64(idx), 0) 48 | be.Seek(int64(idx), 0) 49 | 50 | ex := make([]byte, l) 51 | ac := make([]byte, l) 52 | 53 | f.Read(ex) 54 | be.Read(ac) 55 | assert.True(t, bytes.Compare(ex, ac) == 0) 56 | } else { 57 | // randon write 58 | idx := nextRandInt(0, be.Len()) 59 | f.Seek(int64(idx), 0) 60 | be.Seek(int64(idx), 0) 61 | 62 | l := nextRandInt(0, maxInsertCount) 63 | data, err := ioutil.ReadAll(&io.LimitedReader{R: r, N: int64(l)}) 64 | assert.NoError(t, err) 65 | f.Write(data) 66 | be.Write(data) 67 | } 68 | if j%flushEvery == 0 { 69 | // Flush 70 | b = be.Blob() 71 | be = b.Edit() 72 | } 73 | } 74 | 75 | f.Sync() 76 | b = be.Blob() 77 | 78 | f.Seek(0, 0) 79 | info, err := f.Stat() 80 | assert.NoError(t, err) 81 | assert.True(t, uint64(info.Size()) == b.Len()) 82 | expect, err := ioutil.ReadAll(f) 83 | assert.NoError(t, err) 84 | 85 | actual := make([]byte, b.Len()) 86 | b.ReadAt(actual, 0) 87 | 88 | assert.True(t, bytes.Compare(expect, actual) == 0) 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /go/types/blob_leaf_sequence.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import "github.com/attic-labs/noms/go/d" 8 | 9 | type blobLeafSequence struct { 10 | leafSequence 11 | } 12 | 13 | func newBlobLeafSequence(vrw ValueReadWriter, data []byte) sequence { 14 | d.PanicIfTrue(vrw == nil) 15 | offsets := make([]uint32, sequencePartValues+1) 16 | w := newBinaryNomsWriter() 17 | offsets[sequencePartKind] = w.offset 18 | BlobKind.writeTo(&w) 19 | offsets[sequencePartLevel] = w.offset 20 | w.writeCount(0) // level 21 | offsets[sequencePartCount] = w.offset 22 | count := uint64(len(data)) 23 | w.writeCount(count) 24 | offsets[sequencePartValues] = w.offset 25 | w.writeBytes(data) 26 | return blobLeafSequence{newLeafSequence(vrw, w.data(), offsets, count)} 27 | } 28 | 29 | func (bl blobLeafSequence) writeTo(w nomsWriter) { 30 | w.writeRaw(bl.buff) 31 | } 32 | 33 | // sequence interface 34 | 35 | func (bl blobLeafSequence) data() []byte { 36 | offset := bl.offsets[sequencePartValues] - bl.offsets[sequencePartKind] 37 | return bl.buff[offset:] 38 | } 39 | 40 | func (bl blobLeafSequence) getCompareFn(other sequence) compareFn { 41 | offsetStart := int(bl.offsets[sequencePartValues] - bl.offsets[sequencePartKind]) 42 | obl := other.(blobLeafSequence) 43 | otherOffsetStart := int(obl.offsets[sequencePartValues] - obl.offsets[sequencePartKind]) 44 | return func(idx, otherIdx int) bool { 45 | return bl.buff[offsetStart+idx] == obl.buff[otherOffsetStart+otherIdx] 46 | } 47 | } 48 | 49 | func (bl blobLeafSequence) getItem(idx int) sequenceItem { 50 | offset := bl.offsets[sequencePartValues] - bl.offsets[sequencePartKind] + uint32(idx) 51 | return bl.buff[offset] 52 | } 53 | 54 | func (bl blobLeafSequence) typeOf() *Type { 55 | return BlobType 56 | } 57 | -------------------------------------------------------------------------------- /go/types/bool.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/hash" 9 | ) 10 | 11 | // Bool is a Noms Value wrapper around the primitive bool type. 12 | type Bool bool 13 | 14 | // Value interface 15 | func (b Bool) Value() Value { 16 | return b 17 | } 18 | 19 | func (b Bool) Equals(other Value) bool { 20 | return b == other 21 | } 22 | 23 | func (b Bool) Less(other Value) bool { 24 | if b2, ok := other.(Bool); ok { 25 | return !bool(b) && bool(b2) 26 | } 27 | return true 28 | } 29 | 30 | func (b Bool) Hash() hash.Hash { 31 | return getHash(b) 32 | } 33 | 34 | func (b Bool) WalkValues(cb ValueCallback) { 35 | } 36 | 37 | func (b Bool) WalkRefs(cb RefCallback) { 38 | } 39 | 40 | func (b Bool) typeOf() *Type { 41 | return BoolType 42 | } 43 | 44 | func (b Bool) Kind() NomsKind { 45 | return BoolKind 46 | } 47 | 48 | func (b Bool) valueReadWriter() ValueReadWriter { 49 | return nil 50 | } 51 | 52 | func (b Bool) writeTo(w nomsWriter) { 53 | BoolKind.writeTo(w) 54 | w.writeBool(bool(b)) 55 | } 56 | 57 | func (b Bool) valueBytes() []byte { 58 | if bool(b) { 59 | return []byte{byte(BoolKind), 1} 60 | } 61 | return []byte{byte(BoolKind), 0} 62 | } 63 | -------------------------------------------------------------------------------- /go/types/codec_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestCodecWriteNumber(t *testing.T) { 14 | test := func(f float64, exp []byte) { 15 | w := newBinaryNomsWriter() 16 | w.writeNumber(Number(f)) 17 | assert.Equal(t, exp, w.data()) 18 | } 19 | 20 | // We use zigzag encoding for the signed bit. For positive n we do 2*n and for negative we do 2*-n - 1 21 | test(0, []byte{0, 0}) // 0 * 2 ** 0 22 | 23 | test(1, []byte{1 * 2, 0}) // 1 * 2 ** 0 24 | test(2, []byte{1 * 2, 1 * 2}) // 1 * 2 ** 1 25 | test(-2, []byte{(1 * 2) - 1, 1 * 2}) // -1 * 2 ** 1 26 | test(.5, []byte{1 * 2, 1*2 - 1}) // 1 * 2 ** -1 27 | test(-.5, []byte{1*2 - 1, 1*2 - 1}) // -1 * 2 ** -1 28 | test(.25, []byte{1 * 2, 2*2 - 1}) // 1 * 2 ** -2 29 | test(3, []byte{3 * 2, 0}) // 0b11 * 2 ** 0 30 | 31 | test(15, []byte{15 * 2, 0}) // 0b1111 * 2**0 32 | test(256, []byte{1 * 2, 8 * 2}) // 1 * 2*8 33 | test(-15, []byte{15*2 - 1, 0}) // -15 * 2*0 34 | } 35 | 36 | func TestCodecReadNumber(t *testing.T) { 37 | test := func(data []byte, exp float64) { 38 | r := binaryNomsReader{buff: data} 39 | n := r.readNumber() 40 | assert.Equal(t, exp, float64(n)) 41 | assert.Equal(t, len(data), int(r.offset)) 42 | } 43 | 44 | test([]byte{0, 0}, 0) // 0 * 2 ** 0 45 | 46 | test([]byte{1 * 2, 0}, 1) // 1 * 2 ** 0 47 | test([]byte{1 * 2, 1 * 2}, 2) // 1 * 2 ** 1 48 | test([]byte{1*2 - 1, 1 + 1}, -2) // -1 * 2 ** 1 49 | test([]byte{1 * 2, 1*2 - 1}, .5) // 1 * 2 ** -1 50 | test([]byte{1*2 - 1, 1*2 - 1}, -.5) // -1 * 2 ** -1 51 | test([]byte{1 * 2, 2*2 - 1}, .25) // 1 * 2 ** -2 52 | test([]byte{3 * 2, 0}, 3) // 0b11 * 2 ** 0 53 | 54 | test([]byte{15 * 2, 0}, 15) // 0b1111 * 2**0 55 | test([]byte{1 * 2, 8 * 2}, 256) // 1 * 2*8 56 | test([]byte{15*2 - 1, 0}, -15) // -15 * 2*0 57 | } 58 | -------------------------------------------------------------------------------- /go/types/collection.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | type Collection interface { 8 | Value 9 | Empty() bool 10 | Len() uint64 11 | asSequence() sequence 12 | } 13 | -------------------------------------------------------------------------------- /go/types/collection_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import "github.com/stretchr/testify/suite" 8 | 9 | type collectionTestSuite struct { 10 | suite.Suite 11 | col Collection 12 | expectType *Type 13 | expectLen uint64 14 | expectChunkCount int 15 | expectPrependChunkDiff int 16 | expectAppendChunkDiff int 17 | validate validateFn 18 | prependOne deltaFn 19 | appendOne deltaFn 20 | } 21 | 22 | type validateFn func(v2 Collection) bool 23 | type deltaFn func() Collection 24 | 25 | func (suite *collectionTestSuite) TestType() { 26 | suite.True(suite.expectType.Equals(TypeOf(suite.col))) 27 | } 28 | 29 | func (suite *collectionTestSuite) TestLen() { 30 | suite.Equal(suite.expectLen, suite.col.Len()) 31 | suite.Equal(suite.col.Empty(), suite.expectLen == 0) 32 | } 33 | 34 | func (suite *collectionTestSuite) TestEquals() { 35 | v2 := suite.col 36 | suite.True(suite.col.Equals(v2)) 37 | suite.True(v2.Equals(suite.col)) 38 | } 39 | 40 | func (suite *collectionTestSuite) TestChunkCountAndType() { 41 | suite.Equal(suite.expectChunkCount, leafCount(suite.col), "chunk count") 42 | refType := MakeRefType(suite.expectType) 43 | suite.col.WalkRefs(func(r Ref) { 44 | suite.True(refType.Equals(TypeOf(r))) 45 | }) 46 | } 47 | 48 | func (suite *collectionTestSuite) TestRoundTripAndValidate() { 49 | suite.True(suite.validate(suite.col)) 50 | } 51 | 52 | func (suite *collectionTestSuite) TestPrependChunkDiff() { 53 | v2 := suite.prependOne() 54 | suite.Equal(suite.expectPrependChunkDiff, leafDiffCount(suite.col, v2), "prepend count") 55 | } 56 | 57 | func (suite *collectionTestSuite) TestAppendChunkDiff() { 58 | v2 := suite.appendOne() 59 | suite.Equal(suite.expectAppendChunkDiff, leafDiffCount(suite.col, v2), "append count") 60 | } 61 | 62 | func deriveCollectionHeight(c Collection) uint64 { 63 | return c.asSequence().treeLevel() 64 | } 65 | 66 | func getRefHeightOfCollection(c Collection) uint64 { 67 | return c.asSequence().getItem(0).(metaTuple).ref().Height() 68 | } 69 | -------------------------------------------------------------------------------- /go/types/equals_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "bytes" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestValueEquals(t *testing.T) { 15 | assert := assert.New(t) 16 | vrw := newTestValueStore() 17 | 18 | values := []func() Value{ 19 | func() Value { return Bool(false) }, 20 | func() Value { return Bool(true) }, 21 | func() Value { return Number(0) }, 22 | func() Value { return Number(-1) }, 23 | func() Value { return Number(1) }, 24 | func() Value { return String("") }, 25 | func() Value { return String("hi") }, 26 | func() Value { return String("bye") }, 27 | func() Value { 28 | return NewBlob(vrw, &bytes.Buffer{}) 29 | }, 30 | func() Value { 31 | return NewBlob(vrw, bytes.NewBufferString("hi")) 32 | }, 33 | func() Value { 34 | return NewBlob(vrw, bytes.NewBufferString("bye")) 35 | }, 36 | func() Value { 37 | b1 := NewBlob(vrw, bytes.NewBufferString("hi")) 38 | b2 := NewBlob(vrw, bytes.NewBufferString("bye")) 39 | return newBlob(newBlobMetaSequence(1, []metaTuple{ 40 | newMetaTuple(NewRef(b1), orderedKeyFromInt(2), 2), 41 | newMetaTuple(NewRef(b2), orderedKeyFromInt(5), 5), 42 | }, nil)) 43 | }, 44 | func() Value { return NewList(vrw) }, 45 | func() Value { return NewList(vrw, String("foo")) }, 46 | func() Value { return NewList(vrw, String("bar")) }, 47 | func() Value { return NewMap(vrw) }, 48 | func() Value { return NewMap(vrw, String("a"), String("a")) }, 49 | func() Value { return NewSet(vrw) }, 50 | func() Value { return NewSet(vrw, String("hi")) }, 51 | 52 | func() Value { return BoolType }, 53 | func() Value { return StringType }, 54 | func() Value { return MakeStructType("a") }, 55 | func() Value { return MakeStructType("b") }, 56 | func() Value { return MakeListType(BoolType) }, 57 | func() Value { return MakeListType(NumberType) }, 58 | func() Value { return MakeSetType(BoolType) }, 59 | func() Value { return MakeSetType(NumberType) }, 60 | func() Value { return MakeRefType(BoolType) }, 61 | func() Value { return MakeRefType(NumberType) }, 62 | func() Value { 63 | return MakeMapType(BoolType, ValueType) 64 | }, 65 | func() Value { 66 | return MakeMapType(NumberType, ValueType) 67 | }, 68 | } 69 | 70 | for i, f1 := range values { 71 | for j, f2 := range values { 72 | if i == j { 73 | assert.True(f1().Equals(f2())) 74 | } else { 75 | assert.False(f1().Equals(f2())) 76 | } 77 | } 78 | v := f1() 79 | if v != nil { 80 | r := NewRef(v) 81 | assert.False(r.Equals(v)) 82 | assert.False(v.Equals(r)) 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /go/types/get_hash.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import "github.com/attic-labs/noms/go/hash" 8 | 9 | var getHashOverride func(v Value) hash.Hash 10 | 11 | func getHash(v Value) hash.Hash { 12 | if getHashOverride != nil { 13 | return getHashOverride(v) 14 | } 15 | return getHashNoOverride(v) 16 | } 17 | 18 | func getHashNoOverride(v Value) hash.Hash { 19 | return EncodeValue(v).Hash() 20 | } 21 | 22 | func EnsureHash(h *hash.Hash, v Value) hash.Hash { 23 | if h.IsEmpty() { 24 | *h = getHash(v) 25 | } 26 | return *h 27 | } 28 | -------------------------------------------------------------------------------- /go/types/less.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/hash" 9 | ) 10 | 11 | type kindAndHash interface { 12 | Kind() NomsKind 13 | Hash() hash.Hash 14 | } 15 | 16 | func valueLess(v1, v2 kindAndHash) bool { 17 | switch v2.Kind() { 18 | case BoolKind, NumberKind, StringKind: 19 | return false 20 | default: 21 | return v1.Hash().Less(v2.Hash()) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /go/types/list_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/d" 9 | ) 10 | 11 | // ListIterator can be used to efficiently iterate through a Noms List. 12 | type ListIterator struct { 13 | cursor *sequenceCursor 14 | } 15 | 16 | // Next returns subsequent Values from a List, starting with the index at which the iterator was 17 | // created. If there are no more Values, Next() returns nil. 18 | func (li ListIterator) Next() (out Value) { 19 | if li.cursor == nil { 20 | d.Panic("Cannot use a nil ListIterator") 21 | } 22 | if li.cursor.valid() { 23 | out = li.cursor.current().(Value) 24 | li.cursor.advance() 25 | } 26 | return 27 | } 28 | -------------------------------------------------------------------------------- /go/types/list_iterator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestListIterator(t *testing.T) { 14 | assert := assert.New(t) 15 | vrw := newTestValueStore() 16 | 17 | numbers := append(generateNumbersAsValues(10), Number(20), Number(25)) 18 | l := NewList(vrw, numbers...) 19 | i := l.Iterator() 20 | vs := iterToSlice(i) 21 | assert.True(vs.Equals(numbers), "Expected: %v != actual: %v", numbers, vs) 22 | 23 | i = l.IteratorAt(3) 24 | vs = iterToSlice(i) 25 | assert.True(vs.Equals(numbers[3:]), "Expected: %v != actual: %v", numbers, vs) 26 | 27 | i = l.IteratorAt(l.Len()) 28 | assert.Nil(i.Next()) 29 | } 30 | -------------------------------------------------------------------------------- /go/types/list_leaf_sequence.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | type listLeafSequence struct { 8 | leafSequence 9 | } 10 | 11 | func newListLeafSequence(vrw ValueReadWriter, vs ...Value) sequence { 12 | return listLeafSequence{newLeafSequenceFromValues(ListKind, vrw, vs...)} 13 | } 14 | 15 | // sequence interface 16 | 17 | func (ll listLeafSequence) getCompareFn(other sequence) compareFn { 18 | return ll.getCompareFnHelper(other.(listLeafSequence).leafSequence) 19 | } 20 | -------------------------------------------------------------------------------- /go/types/map_iterator.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | // MapIterator can efficiently iterate through a Noms Map. 8 | type MapIterator struct { 9 | cursor *sequenceCursor 10 | currentKey Value 11 | currentValue Value 12 | } 13 | 14 | func (mi *MapIterator) Valid() bool { 15 | return mi.cursor.valid() 16 | } 17 | 18 | func (mi *MapIterator) Entry() (k Value, v Value) { 19 | return mi.Key(), mi.Value() 20 | } 21 | 22 | func (mi *MapIterator) Key() Value { 23 | if !mi.cursor.valid() { 24 | return nil 25 | } 26 | return mi.cursor.current().(mapEntry).key 27 | } 28 | 29 | func (mi *MapIterator) Value() Value { 30 | if !mi.cursor.valid() { 31 | return nil 32 | } 33 | return mi.cursor.current().(mapEntry).value 34 | } 35 | 36 | func (mi *MapIterator) Position() uint64 { 37 | if !mi.cursor.valid() { 38 | return 0 39 | } 40 | return uint64(mi.cursor.idx) 41 | } 42 | 43 | // Prev returns the previous entry from the Map. If there is no previous entry, Prev() returns nils. 44 | func (mi *MapIterator) Prev() bool { 45 | if !mi.cursor.valid() { 46 | return false 47 | } 48 | return mi.cursor.retreat() 49 | } 50 | 51 | // Next returns the subsequent entries from the Map, starting with the entry at which the iterator 52 | // was created. If there are no more entries, Next() returns nils. 53 | func (mi *MapIterator) Next() bool { 54 | if !mi.cursor.valid() { 55 | return false 56 | } 57 | return mi.cursor.advance() 58 | } 59 | -------------------------------------------------------------------------------- /go/types/map_iterator_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "fmt" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestMapIterator(t *testing.T) { 15 | assert := assert.New(t) 16 | 17 | vrw := newTestValueStore() 18 | 19 | me := NewMap(vrw).Edit() 20 | for i := 0; i < 5; i++ { 21 | me.Set(String(string(byte(65+i))), Number(i)) 22 | } 23 | 24 | m := me.Map() 25 | 26 | tc := []struct { 27 | reverse bool 28 | iter bool 29 | iterAt uint64 30 | iterFrom string 31 | expected []string 32 | }{ 33 | {false, true, 0, "", []string{"A", "B", "C", "D", "E"}}, 34 | {false, false, 0, "", []string{"A", "B", "C", "D", "E"}}, 35 | {false, false, 2, "", []string{"C", "D", "E"}}, 36 | {false, false, 4, "", []string{"E"}}, 37 | {false, false, 5, "", []string{}}, 38 | {false, false, 0, "A", []string{"A", "B", "C", "D", "E"}}, 39 | {false, false, 0, "C", []string{"C", "D", "E"}}, 40 | {false, false, 4, "E", []string{"E"}}, 41 | {false, false, 0, "AA", []string{"B", "C", "D", "E"}}, 42 | {false, false, 0, "F", []string{}}, 43 | {true, false, 0, "", []string{}}, 44 | {true, true, 0, "", []string{}}, 45 | {true, false, 2, "", []string{"C", "B", "A"}}, 46 | {true, false, 4, "", []string{"E", "D", "C", "B", "A"}}, 47 | {true, false, 5, "", []string{}}, 48 | {true, false, 0, "A", []string{"A"}}, 49 | {true, false, 0, "C", []string{"C", "B", "A"}}, 50 | {true, false, 0, "E", []string{"E", "D", "C", "B", "A"}}, 51 | {true, false, 0, "AA", []string{"B", "A"}}, 52 | {true, false, 0, "F", []string{}}, 53 | } 54 | 55 | for i, t := range tc { 56 | lbl := fmt.Sprintf("test case %d", i) 57 | var it *MapIterator 58 | if t.iter { 59 | it = m.Iterator() 60 | } else if t.iterFrom != "" { 61 | it = m.IteratorFrom(String(t.iterFrom)) 62 | } else { 63 | it = m.IteratorAt(t.iterAt) 64 | } 65 | for i, e := range t.expected { 66 | lbl := fmt.Sprintf("%s: iteration %d", lbl, i) 67 | assert.True(it.Valid(), lbl) 68 | 69 | assert.Equal(e, string(it.Key().(String)), lbl) 70 | assert.True(m.Get(it.Key()).Equals(it.Value()), lbl) 71 | 72 | k, v := it.Entry() 73 | assert.Equal(e, string(k.(String)), lbl) 74 | assert.True(m.Get(it.Key()).Equals(v), lbl) 75 | 76 | assert.True(m.Get(it.Key()).Equals(Number(it.Position())), lbl) 77 | 78 | var last bool 79 | if t.reverse { 80 | last = it.Prev() 81 | } else { 82 | last = it.Next() 83 | } 84 | assert.Equal(i < len(t.expected)-1, last, lbl) 85 | assert.Equal(i < len(t.expected)-1, it.Valid(), lbl) 86 | } 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /go/types/noms_kind.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | // NomsKind allows a TypeDesc to indicate what kind of type is described. 8 | type NomsKind uint8 9 | 10 | // All supported kinds of Noms types are enumerated here. 11 | // The ordering of these (especially Bool, Number and String) is important for ordering of values. 12 | const ( 13 | BoolKind NomsKind = iota 14 | NumberKind 15 | StringKind 16 | BlobKind 17 | ValueKind 18 | ListKind 19 | MapKind 20 | RefKind 21 | SetKind 22 | 23 | // Keep StructKind and CycleKind together. 24 | StructKind 25 | CycleKind 26 | 27 | TypeKind 28 | UnionKind 29 | 30 | // Internal to decoder 31 | hashKind 32 | ) 33 | 34 | var KindToString = map[NomsKind]string{ 35 | BlobKind: "Blob", 36 | BoolKind: "Bool", 37 | CycleKind: "Cycle", 38 | ListKind: "List", 39 | MapKind: "Map", 40 | NumberKind: "Number", 41 | RefKind: "Ref", 42 | SetKind: "Set", 43 | StructKind: "Struct", 44 | StringKind: "String", 45 | TypeKind: "Type", 46 | UnionKind: "Union", 47 | ValueKind: "Value", 48 | } 49 | 50 | // String returns the name of the kind. 51 | func (k NomsKind) String() string { 52 | return KindToString[k] 53 | } 54 | 55 | // IsPrimitiveKind returns true if k represents a Noms primitive type, which excludes collections (List, Map, Set), Refs, Structs, Symbolic and Unresolved types. 56 | func IsPrimitiveKind(k NomsKind) bool { 57 | switch k { 58 | case BoolKind, NumberKind, StringKind, BlobKind, ValueKind, TypeKind: 59 | return true 60 | default: 61 | return false 62 | } 63 | } 64 | 65 | // isKindOrderedByValue determines if a value is ordered by its value instead of its hash. 66 | func isKindOrderedByValue(k NomsKind) bool { 67 | return k <= StringKind 68 | } 69 | 70 | func (k NomsKind) writeTo(w nomsWriter) { 71 | w.writeUint8(uint8(k)) 72 | } 73 | -------------------------------------------------------------------------------- /go/types/number.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "encoding/binary" 9 | "math" 10 | 11 | "github.com/attic-labs/noms/go/d" 12 | "github.com/attic-labs/noms/go/hash" 13 | ) 14 | 15 | // Number is a Noms Value wrapper around the primitive float64 type. 16 | type Number float64 17 | 18 | // Value interface 19 | func (v Number) Value() Value { 20 | return v 21 | } 22 | 23 | func (v Number) Equals(other Value) bool { 24 | return v == other 25 | } 26 | 27 | func (v Number) Less(other Value) bool { 28 | if v2, ok := other.(Number); ok { 29 | return v < v2 30 | } 31 | return NumberKind < other.Kind() 32 | } 33 | 34 | func (v Number) Hash() hash.Hash { 35 | return getHash(v) 36 | } 37 | 38 | func (v Number) WalkValues(cb ValueCallback) { 39 | } 40 | 41 | func (v Number) WalkRefs(cb RefCallback) { 42 | } 43 | 44 | func (v Number) typeOf() *Type { 45 | return NumberType 46 | } 47 | 48 | func (v Number) Kind() NomsKind { 49 | return NumberKind 50 | } 51 | 52 | func (v Number) valueReadWriter() ValueReadWriter { 53 | return nil 54 | } 55 | 56 | func (v Number) writeTo(w nomsWriter) { 57 | NumberKind.writeTo(w) 58 | f := float64(v) 59 | if math.IsNaN(f) || math.IsInf(f, 0) { 60 | d.Panic("%f is not a supported number", f) 61 | } 62 | w.writeNumber(v) 63 | } 64 | 65 | func (v Number) valueBytes() []byte { 66 | // We know the size of the buffer here so allocate it once. 67 | // NumberKind, int (Varint), exp (Varint) 68 | buff := make([]byte, 1+2*binary.MaxVarintLen64) 69 | w := binaryNomsWriter{buff, 0} 70 | v.writeTo(&w) 71 | return buff[:w.offset] 72 | } 73 | -------------------------------------------------------------------------------- /go/types/number_util.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import "math" 8 | 9 | func float64IsInt(f float64) bool { 10 | return math.Trunc(f) == f 11 | } 12 | 13 | // convert float64 to int64 where f == i * 2^exp 14 | func float64ToIntExp(f float64) (int64, int) { 15 | if f == 0 { 16 | return 0, 0 17 | } 18 | 19 | isNegative := math.Signbit(f) 20 | f = math.Abs(f) 21 | 22 | frac, exp := math.Frexp(f) 23 | // frac is [.5, 1) 24 | // Move frac up until it is an integer. 25 | for !float64IsInt(frac) { 26 | frac *= 2 27 | exp-- 28 | } 29 | 30 | if isNegative { 31 | frac *= -1 32 | } 33 | 34 | return int64(frac), exp 35 | } 36 | 37 | // fracExpToFloat returns frac * 2 ** exp 38 | func fracExpToFloat(frac int64, exp int) float64 { 39 | return float64(frac) * math.Pow(2, float64(exp)) 40 | } 41 | -------------------------------------------------------------------------------- /go/types/perf/dummy.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package perf 6 | 7 | // go build fails if there are _test.go but no other go files in a directory. 8 | -------------------------------------------------------------------------------- /go/types/primitives_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestPrimitives(t *testing.T) { 14 | data := []Value{ 15 | Bool(true), Bool(false), 16 | Number(0), Number(-1), 17 | Number(-0.1), Number(0.1), 18 | } 19 | 20 | for i := range data { 21 | for j := range data { 22 | if i == j { 23 | assert.True(t, data[i].Equals(data[j]), "Expected value to equal self at index %d", i) 24 | } else { 25 | assert.False(t, data[i].Equals(data[j]), "Expected values at indices %d and %d to not equal", i, j) 26 | } 27 | } 28 | } 29 | } 30 | 31 | func TestPrimitivesType(t *testing.T) { 32 | data := []struct { 33 | v Value 34 | k NomsKind 35 | }{ 36 | {Bool(false), BoolKind}, 37 | {Number(0), NumberKind}, 38 | } 39 | 40 | for _, d := range data { 41 | assert.True(t, TypeOf(d.v).Equals(MakePrimitiveType(d.k))) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /go/types/ref_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestRefInList(t *testing.T) { 14 | assert := assert.New(t) 15 | 16 | vs := newTestValueStore() 17 | 18 | l := NewList(vs) 19 | r := NewRef(l) 20 | l = l.Edit().Append(r).List() 21 | r2 := l.Get(0) 22 | assert.True(r.Equals(r2)) 23 | } 24 | 25 | func TestRefInSet(t *testing.T) { 26 | assert := assert.New(t) 27 | 28 | vs := newTestValueStore() 29 | 30 | s := NewSet(vs) 31 | r := NewRef(s) 32 | s = s.Edit().Insert(r).Set() 33 | r2 := s.First() 34 | assert.True(r.Equals(r2)) 35 | } 36 | 37 | func TestRefInMap(t *testing.T) { 38 | assert := assert.New(t) 39 | 40 | vs := newTestValueStore() 41 | 42 | m := NewMap(vs) 43 | r := NewRef(m) 44 | m = m.Edit().Set(Number(0), r).Set(r, Number(1)).Map() 45 | r2 := m.Get(Number(0)) 46 | assert.True(r.Equals(r2)) 47 | 48 | i := m.Get(r) 49 | assert.Equal(int32(1), int32(i.(Number))) 50 | } 51 | 52 | func TestRefChunks(t *testing.T) { 53 | assert := assert.New(t) 54 | 55 | vs := newTestValueStore() 56 | 57 | l := NewList(vs) 58 | r := NewRef(l) 59 | assert.Len(getChunks(r), 1) 60 | assert.Equal(r, getChunks(r)[0]) 61 | } 62 | -------------------------------------------------------------------------------- /go/types/rungen.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | //go:generate go run gen/main.go 8 | -------------------------------------------------------------------------------- /go/types/sequence.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/d" 9 | "github.com/attic-labs/noms/go/hash" 10 | ) 11 | 12 | type sequenceItem interface{} 13 | 14 | type compareFn func(x int, y int) bool 15 | 16 | type sequence interface { 17 | asValueImpl() valueImpl 18 | cumulativeNumberOfLeaves(idx int) uint64 19 | Empty() bool 20 | Equals(other Value) bool 21 | getChildSequence(idx int) sequence 22 | getCompareFn(other sequence) compareFn 23 | getCompositeChildSequence(start uint64, length uint64) sequence 24 | getItem(idx int) sequenceItem 25 | Hash() hash.Hash 26 | isLeaf() bool 27 | Kind() NomsKind 28 | Len() uint64 29 | Less(other Value) bool 30 | numLeaves() uint64 31 | seqLen() int 32 | treeLevel() uint64 33 | typeOf() *Type 34 | valueBytes() []byte 35 | valueReadWriter() ValueReadWriter 36 | valuesSlice(from, to uint64) []Value 37 | WalkRefs(cb RefCallback) 38 | writeTo(nomsWriter) 39 | } 40 | 41 | const ( 42 | sequencePartKind = 0 43 | sequencePartLevel = 1 44 | sequencePartCount = 2 45 | sequencePartValues = 3 46 | ) 47 | 48 | type sequenceImpl struct { 49 | valueImpl 50 | len uint64 51 | } 52 | 53 | func newSequenceImpl(vrw ValueReadWriter, buff []byte, offsets []uint32, len uint64) sequenceImpl { 54 | return sequenceImpl{valueImpl{vrw, buff, offsets}, len} 55 | } 56 | 57 | func (seq sequenceImpl) decoderSkipToValues() (valueDecoder, uint64) { 58 | dec := seq.decoderAtPart(sequencePartCount) 59 | count := dec.readCount() 60 | return dec, count 61 | } 62 | 63 | func (seq sequenceImpl) decoderAtPart(part uint32) valueDecoder { 64 | offset := seq.offsets[part] - seq.offsets[sequencePartKind] 65 | return newValueDecoder(seq.buff[offset:], seq.vrw) 66 | } 67 | 68 | func (seq sequenceImpl) Empty() bool { 69 | return seq.Len() == 0 70 | } 71 | 72 | func (seq sequenceImpl) Len() uint64 { 73 | return seq.len 74 | } 75 | 76 | func (seq sequenceImpl) seqLen() int { 77 | _, count := seq.decoderSkipToValues() 78 | return int(count) 79 | } 80 | 81 | func (seq sequenceImpl) getItemOffset(idx int) int { 82 | // kind, level, count, elements... 83 | // 0 1 2 3 n+1 84 | d.PanicIfTrue(idx+sequencePartValues+1 > len(seq.offsets)) 85 | return int(seq.offsets[idx+sequencePartValues] - seq.offsets[sequencePartKind]) 86 | } 87 | 88 | func (seq sequenceImpl) decoderSkipToIndex(idx int) valueDecoder { 89 | offset := seq.getItemOffset(idx) 90 | return seq.decoderAtOffset(offset) 91 | } 92 | -------------------------------------------------------------------------------- /go/types/sequence_concat.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | type newSequenceChunkerFn func(cur *sequenceCursor, vrw ValueReadWriter) *sequenceChunker 8 | 9 | func concat(fst, snd sequence, newSequenceChunker newSequenceChunkerFn) sequence { 10 | if fst.numLeaves() == 0 { 11 | return snd 12 | } 13 | if snd.numLeaves() == 0 { 14 | return fst 15 | } 16 | 17 | // concat works by tricking the sequenceChunker into resuming chunking at a 18 | // cursor to the end of fst, then finalizing chunking to the start of snd - by 19 | // swapping fst cursors for snd cursors in the middle of chunking. 20 | vrw := fst.valueReadWriter() 21 | chunker := newSequenceChunker(newCursorAtIndex(fst, fst.numLeaves()), vrw) 22 | 23 | for cur, ch := newCursorAtIndex(snd, 0), chunker; ch != nil; ch = ch.parent { 24 | // Note that if snd is shallower than fst, then higher chunkers will have 25 | // their cursors set to nil. This has the effect of "dropping" the final 26 | // item in each of those sequences. 27 | ch.cur = cur 28 | if cur != nil { 29 | cur = cur.parent 30 | if cur != nil && ch.parent == nil { 31 | // If fst is shallower than snd, its cur will have a parent whereas the 32 | // chunker to snd won't. In that case, create a parent for fst. 33 | ch.createParent() 34 | } 35 | } 36 | } 37 | 38 | return chunker.Done() 39 | } 40 | -------------------------------------------------------------------------------- /go/types/set_leaf_sequence.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import "sort" 8 | 9 | type setLeafSequence struct { 10 | leafSequence 11 | } 12 | 13 | func newSetLeafSequence(vrw ValueReadWriter, vs ...Value) orderedSequence { 14 | return setLeafSequence{newLeafSequenceFromValues(SetKind, vrw, vs...)} 15 | } 16 | 17 | func (sl setLeafSequence) getCompareFn(other sequence) compareFn { 18 | return sl.getCompareFnHelper(other.(setLeafSequence).leafSequence) 19 | } 20 | 21 | // orderedSequence interface 22 | 23 | func (sl setLeafSequence) getKey(idx int) orderedKey { 24 | return newOrderedKey(sl.getItem(idx).(Value)) 25 | } 26 | 27 | func (sl setLeafSequence) search(key orderedKey) int { 28 | return sort.Search(int(sl.Len()), func(i int) bool { 29 | return !sl.getKey(i).Less(key) 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /go/types/string.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "encoding/binary" 9 | 10 | "github.com/attic-labs/noms/go/hash" 11 | ) 12 | 13 | // String is a Noms Value wrapper around the primitive string type. 14 | type String string 15 | 16 | // Value interface 17 | func (s String) Value() Value { 18 | return s 19 | } 20 | 21 | func (s String) Equals(other Value) bool { 22 | return s == other 23 | } 24 | 25 | func (s String) Less(other Value) bool { 26 | if s2, ok := other.(String); ok { 27 | return s < s2 28 | } 29 | return StringKind < other.Kind() 30 | } 31 | 32 | func (s String) Hash() hash.Hash { 33 | return getHash(s) 34 | } 35 | 36 | func (s String) WalkValues(cb ValueCallback) { 37 | } 38 | 39 | func (s String) WalkRefs(cb RefCallback) { 40 | } 41 | 42 | func (s String) typeOf() *Type { 43 | return StringType 44 | } 45 | 46 | func (s String) Kind() NomsKind { 47 | return StringKind 48 | } 49 | 50 | func (s String) valueReadWriter() ValueReadWriter { 51 | return nil 52 | } 53 | 54 | func (s String) writeTo(w nomsWriter) { 55 | StringKind.writeTo(w) 56 | w.writeString(string(s)) 57 | } 58 | 59 | func (s String) valueBytes() []byte { 60 | // We know the size of the buffer here so allocate it once. 61 | // StringKind, Length (UVarint), UTF-8 encoded string 62 | buff := make([]byte, 1+binary.MaxVarintLen64+len(s)) 63 | w := binaryNomsWriter{buff, 0} 64 | s.writeTo(&w) 65 | return buff[:w.offset] 66 | } 67 | -------------------------------------------------------------------------------- /go/types/string_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestStringEquals(t *testing.T) { 14 | assert := assert.New(t) 15 | s1 := String("foo") 16 | s2 := String("foo") 17 | s3 := s2 18 | s4 := String("bar") 19 | assert.True(s1.Equals(s2)) 20 | assert.True(s2.Equals(s1)) 21 | assert.True(s1.Equals(s3)) 22 | assert.True(s3.Equals(s1)) 23 | assert.False(s1.Equals(s4)) 24 | assert.False(s4.Equals(s1)) 25 | } 26 | 27 | func TestStringString(t *testing.T) { 28 | assert := assert.New(t) 29 | s1 := String("") 30 | s2 := String("foo") 31 | assert.Equal("", string(s1)) 32 | assert.Equal("foo", string(s2)) 33 | } 34 | 35 | func TestStringType(t *testing.T) { 36 | assert.True(t, TypeOf(String("hi")).Equals(StringType)) 37 | } 38 | -------------------------------------------------------------------------------- /go/types/validate_type.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | func validateType(t *Type) { 8 | validateTypeImpl(t, map[string]struct{}{}) 9 | } 10 | 11 | func validateTypeImpl(t *Type, seenStructs map[string]struct{}) { 12 | switch desc := t.Desc.(type) { 13 | case CompoundDesc: 14 | if desc.Kind() == UnionKind { 15 | if len(desc.ElemTypes) == 1 { 16 | panic("Invalid union type") 17 | } 18 | for i := 1; i < len(desc.ElemTypes); i++ { 19 | if !unionLess(desc.ElemTypes[i-1], desc.ElemTypes[i]) { 20 | panic("Invalid union order") 21 | } 22 | } 23 | } 24 | 25 | for _, et := range desc.ElemTypes { 26 | validateTypeImpl(et, seenStructs) 27 | } 28 | case StructDesc: 29 | if desc.Name != "" { 30 | if _, ok := seenStructs[desc.Name]; ok { 31 | return 32 | } 33 | seenStructs[desc.Name] = struct{}{} 34 | } 35 | verifyStructName(desc.Name) 36 | verifyFields(desc.fields) 37 | for _, f := range desc.fields { 38 | validateTypeImpl(f.Type, seenStructs) 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /go/types/validating_decoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "github.com/attic-labs/noms/go/chunks" 9 | "github.com/attic-labs/noms/go/d" 10 | ) 11 | 12 | type ValidatingDecoder struct { 13 | vs *ValueStore 14 | } 15 | 16 | func NewValidatingDecoder(cs chunks.ChunkStore) *ValidatingDecoder { 17 | return &ValidatingDecoder{NewValueStore(cs)} 18 | } 19 | 20 | // DecodedChunk holds a pointer to a Chunk and the Value that results from 21 | // calling DecodeFromBytes(c.Data()). 22 | type DecodedChunk struct { 23 | Chunk *chunks.Chunk 24 | Value *Value 25 | } 26 | 27 | // Decode decodes c and checks that the hash of the resulting value 28 | // matches c.Hash(). It returns a DecodedChunk holding both c and a pointer to 29 | // the decoded Value. 30 | func (vbs *ValidatingDecoder) Decode(c *chunks.Chunk) DecodedChunk { 31 | h := c.Hash() 32 | v := decodeFromBytesWithValidation(c.Data(), vbs.vs) 33 | 34 | if getHash(v) != h { 35 | d.Panic("Invalid hash found") 36 | } 37 | return DecodedChunk{c, &v} 38 | } 39 | -------------------------------------------------------------------------------- /go/types/validating_decoder_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/attic-labs/noms/go/chunks" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestValidatingBatchingSinkDecode(t *testing.T) { 15 | v := Number(42) 16 | c := EncodeValue(v) 17 | storage := &chunks.TestStorage{} 18 | vdc := NewValidatingDecoder(storage.NewView()) 19 | 20 | dc := vdc.Decode(&c) 21 | assert.True(t, v.Equals(*dc.Value)) 22 | } 23 | 24 | func assertPanicsOnInvalidChunk(t *testing.T, data []interface{}) { 25 | storage := &chunks.TestStorage{} 26 | vs := NewValueStore(storage.NewView()) 27 | dataAsByteSlice := toBinaryNomsReaderData(data) 28 | dec := newValueDecoder(dataAsByteSlice, vs) 29 | v := dec.readValue() 30 | 31 | c := EncodeValue(v) 32 | vdc := NewValidatingDecoder(storage.NewView()) 33 | 34 | assert.Panics(t, func() { 35 | vdc.Decode(&c) 36 | }) 37 | } 38 | 39 | func TestValidatingBatchingSinkDecodeInvalidUnion(t *testing.T) { 40 | data := []interface{}{ 41 | uint8(TypeKind), 42 | uint8(UnionKind), uint64(2) /* len */, uint8(NumberKind), uint8(BoolKind), 43 | } 44 | assertPanicsOnInvalidChunk(t, data) 45 | } 46 | 47 | func TestValidatingBatchingSinkDecodeInvalidStructFieldOrder(t *testing.T) { 48 | data := []interface{}{ 49 | uint8(TypeKind), 50 | uint8(StructKind), "S", uint64(2), /* len */ 51 | "b", "a", 52 | uint8(NumberKind), uint8(NumberKind), 53 | false, false, 54 | } 55 | assertPanicsOnInvalidChunk(t, data) 56 | } 57 | 58 | func TestValidatingBatchingSinkDecodeInvalidStructName(t *testing.T) { 59 | data := []interface{}{ 60 | uint8(TypeKind), 61 | uint8(StructKind), "S ", uint64(0), /* len */ 62 | } 63 | assertPanicsOnInvalidChunk(t, data) 64 | } 65 | 66 | func TestValidatingBatchingSinkDecodeInvalidStructFieldName(t *testing.T) { 67 | data := []interface{}{ 68 | uint8(TypeKind), 69 | uint8(StructKind), "S", uint64(1), /* len */ 70 | "b ", uint8(NumberKind), false, 71 | } 72 | assertPanicsOnInvalidChunk(t, data) 73 | } 74 | -------------------------------------------------------------------------------- /go/types/walk.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package types 6 | 7 | import "github.com/attic-labs/noms/go/hash" 8 | 9 | type SkipValueCallback func(v Value) bool 10 | 11 | // WalkValues loads prolly trees progressively by walking down the tree. We don't wants to invoke 12 | // the value callback on internal sub-trees (which are valid values) because they are not logical 13 | // values in the graph 14 | type valueRec struct { 15 | v Value 16 | cb bool 17 | } 18 | 19 | const maxRefCount = 1 << 12 // ~16MB of data 20 | 21 | // WalkValues recursively walks over all types.Values reachable from r and calls cb on them. 22 | func WalkValues(target Value, vr ValueReader, cb SkipValueCallback) { 23 | visited := hash.HashSet{} 24 | refs := map[hash.Hash]bool{} 25 | values := []valueRec{{target, true}} 26 | 27 | for len(values) > 0 || len(refs) > 0 { 28 | for len(values) > 0 { 29 | rec := values[len(values)-1] 30 | values = values[:len(values)-1] 31 | 32 | v := rec.v 33 | if rec.cb && cb(v) { 34 | continue 35 | } 36 | 37 | if _, ok := v.(Blob); ok { 38 | continue // don't traverse into blob ptrees 39 | } 40 | 41 | if r, ok := v.(Ref); ok { 42 | refs[r.TargetHash()] = true 43 | continue 44 | } 45 | 46 | if col, ok := v.(Collection); ok && !col.asSequence().isLeaf() { 47 | col.WalkRefs(func(r Ref) { 48 | refs[r.TargetHash()] = false 49 | }) 50 | continue 51 | } 52 | 53 | v.WalkValues(func(sv Value) { 54 | values = append(values, valueRec{sv, true}) 55 | }) 56 | } 57 | 58 | if len(refs) == 0 { 59 | continue 60 | } 61 | 62 | hs := make(hash.HashSlice, 0, len(refs)) 63 | oldRefs := refs 64 | refs = map[hash.Hash]bool{} 65 | for h := range oldRefs { 66 | if _, ok := visited[h]; ok { 67 | continue 68 | } 69 | 70 | if len(hs) >= maxRefCount { 71 | refs[h] = oldRefs[h] 72 | continue 73 | } 74 | 75 | hs = append(hs, h) 76 | visited.Insert(h) 77 | } 78 | 79 | if len(hs) > 0 { 80 | readValues := vr.ReadManyValues(hs) 81 | for i, sv := range readValues { 82 | values = append(values, valueRec{sv, oldRefs[hs[i]]}) 83 | } 84 | } 85 | } 86 | } 87 | 88 | func mightContainStructs(t *Type) (mightHaveStructs bool) { 89 | if t.TargetKind() == StructKind || t.TargetKind() == ValueKind { 90 | mightHaveStructs = true 91 | return 92 | } 93 | 94 | t.WalkValues(func(v Value) { 95 | mightHaveStructs = mightHaveStructs || mightContainStructs(v.(*Type)) 96 | }) 97 | 98 | return 99 | } 100 | -------------------------------------------------------------------------------- /go/util/exit/exit.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // Package exit provides a mockable implementation of os.Exit. 6 | // That's all! 7 | package exit 8 | 9 | import ( 10 | "os" 11 | ) 12 | 13 | var def = func(code int) { 14 | os.Exit(code) 15 | } 16 | 17 | var Exit = def 18 | 19 | // Reset sets the implementation of Exit() to the default. 20 | func Reset() { 21 | Exit = def 22 | } 23 | 24 | // Fail exits with a failure status. 25 | func Fail() { 26 | Exit(1) 27 | } 28 | 29 | // Success exits with a success status. 30 | func Success() { 31 | Exit(0) 32 | } 33 | -------------------------------------------------------------------------------- /go/util/functions/all.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package functions 6 | 7 | import "sync" 8 | 9 | // All runs all functions in |fs| in parallel, and returns when all functions have returned. 10 | func All(fs ...func()) { 11 | wg := &sync.WaitGroup{} 12 | wg.Add(len(fs)) 13 | for _, f_ := range fs { 14 | f := f_ 15 | go func() { 16 | f() 17 | wg.Done() 18 | }() 19 | } 20 | wg.Wait() 21 | } 22 | -------------------------------------------------------------------------------- /go/util/functions/all_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package functions 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestAll(t *testing.T) { 14 | assert := assert.New(t) 15 | 16 | // Set |res| via |ch| to test it's running in parallel - if not, they'll deadlock. 17 | var res int 18 | ch := make(chan int) 19 | All(func() { ch <- 42 }, func() { res = <-ch }) 20 | 21 | assert.Equal(42, res) 22 | } 23 | -------------------------------------------------------------------------------- /go/util/math/minmax.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package math 6 | 7 | // MaxInt returns the larger of x or y. 8 | func MaxInt(x, y int) int { 9 | if x > y { 10 | return x 11 | } 12 | return y 13 | } 14 | 15 | // MinInt returns the smaller of x or y. 16 | func MinInt(x, y int) int { 17 | if x < y { 18 | return x 19 | } 20 | return y 21 | } 22 | -------------------------------------------------------------------------------- /go/util/outputpager/page_output.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package outputpager 6 | 7 | import ( 8 | "io" 9 | "os" 10 | "os/exec" 11 | "sync" 12 | 13 | "github.com/attic-labs/kingpin" 14 | goisatty "github.com/mattn/go-isatty" 15 | 16 | "github.com/attic-labs/noms/go/d" 17 | ) 18 | 19 | var ( 20 | noPager bool 21 | ) 22 | 23 | type Pager struct { 24 | Writer io.Writer 25 | stdin, stdout *os.File 26 | mtx *sync.Mutex 27 | doneCh chan struct{} 28 | } 29 | 30 | func Start() *Pager { 31 | if noPager || !IsStdoutTty() { 32 | return &Pager{os.Stdout, nil, nil, nil, nil} 33 | } 34 | 35 | lessPath, err := exec.LookPath("less") 36 | d.Chk.NoError(err) 37 | 38 | // -F ... Quit if entire file fits on first screen. 39 | // -S ... Chop (truncate) long lines rather than wrapping. 40 | // -R ... Output "raw" control characters. 41 | // -X ... Don't use termcap init/deinit strings. 42 | cmd := exec.Command(lessPath, "-FSRX") 43 | 44 | stdin, stdout, err := os.Pipe() 45 | d.Chk.NoError(err) 46 | cmd.Stdout = os.Stdout 47 | cmd.Stderr = os.Stderr 48 | cmd.Stdin = stdin 49 | cmd.Start() 50 | 51 | p := &Pager{stdout, stdin, stdout, &sync.Mutex{}, make(chan struct{})} 52 | go func() { 53 | err := cmd.Wait() 54 | d.Chk.NoError(err) 55 | p.closePipe() 56 | p.doneCh <- struct{}{} 57 | }() 58 | return p 59 | } 60 | 61 | func (p *Pager) Stop() { 62 | if p.Writer != os.Stdout { 63 | p.closePipe() 64 | // Wait until less has fully exited, otherwise it might not have printed the terminal restore characters. 65 | <-p.doneCh 66 | } 67 | } 68 | 69 | func (p *Pager) closePipe() { 70 | p.mtx.Lock() 71 | defer p.mtx.Unlock() 72 | if p.stdin != nil { 73 | // Closing the pipe will cause any outstanding writes to stdout fail, and fail from now on. 74 | p.stdin.Close() 75 | p.stdout.Close() 76 | p.stdin, p.stdout = nil, nil 77 | } 78 | } 79 | 80 | func RegisterOutputpagerFlags(cmd *kingpin.CmdClause) { 81 | cmd.Flag("no-pager", "suppress paging functionality").BoolVar(&noPager) 82 | } 83 | 84 | func IsStdoutTty() bool { 85 | return goisatty.IsTerminal(os.Stdout.Fd()) 86 | } 87 | -------------------------------------------------------------------------------- /go/util/profile/profile.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package profile 6 | 7 | import ( 8 | "io" 9 | "os" 10 | "runtime" 11 | "runtime/pprof" 12 | 13 | "github.com/attic-labs/kingpin" 14 | 15 | "github.com/attic-labs/noms/go/d" 16 | ) 17 | 18 | var ( 19 | cpuProfile string 20 | memProfile string 21 | blockProfile string 22 | ) 23 | 24 | func RegisterProfileFlags(app *kingpin.Application) { 25 | // Must reset globals because under test this can get called multiple times. 26 | cpuProfile = "" 27 | memProfile = "" 28 | blockProfile = "" 29 | app.Flag("cpuprofile", "write cpu profile to file").StringVar(&cpuProfile) 30 | app.Flag("memprofile", "write memory profile to file").StringVar(&memProfile) 31 | app.Flag("blockprofile", "write block profile to file").StringVar(&blockProfile) 32 | } 33 | 34 | // MaybeStartProfile checks the -blockProfile, -cpuProfile, and -memProfile flag and, for each that is set, attempts to start gathering profiling data into the appropriate files. It returns an object with one method, Stop(), that must be called in order to flush profile data to disk before the process terminates. 35 | func MaybeStartProfile() interface { 36 | Stop() 37 | } { 38 | p := &prof{} 39 | if blockProfile != "" { 40 | f, err := os.Create(blockProfile) 41 | d.PanicIfError(err) 42 | runtime.SetBlockProfileRate(1) 43 | p.bp = f 44 | } 45 | if cpuProfile != "" { 46 | f, err := os.Create(cpuProfile) 47 | d.PanicIfError(err) 48 | pprof.StartCPUProfile(f) 49 | p.cpu = f 50 | } 51 | if memProfile != "" { 52 | f, err := os.Create(memProfile) 53 | d.PanicIfError(err) 54 | p.mem = f 55 | } 56 | return p 57 | } 58 | 59 | type prof struct { 60 | bp io.WriteCloser 61 | cpu io.Closer 62 | mem io.WriteCloser 63 | } 64 | 65 | func (p *prof) Stop() { 66 | if p.bp != nil { 67 | pprof.Lookup("block").WriteTo(p.bp, 0) 68 | p.bp.Close() 69 | runtime.SetBlockProfileRate(0) 70 | } 71 | if p.cpu != nil { 72 | pprof.StopCPUProfile() 73 | p.cpu.Close() 74 | } 75 | if p.mem != nil { 76 | pprof.WriteHeapProfile(p.mem) 77 | p.mem.Close() 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /go/util/progressreader/reader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // Package progressreader provides an io.Reader that reports progress to a callback 6 | package progressreader 7 | 8 | import ( 9 | "io" 10 | "time" 11 | 12 | "github.com/attic-labs/noms/go/util/status" 13 | ) 14 | 15 | type Callback func(seen uint64) 16 | 17 | func New(inner io.Reader, cb Callback) io.Reader { 18 | return &reader{inner, uint64(0), time.Time{}, cb} 19 | } 20 | 21 | type reader struct { 22 | inner io.Reader 23 | seen uint64 24 | lastTime time.Time 25 | cb Callback 26 | } 27 | 28 | func (r *reader) Read(p []byte) (n int, err error) { 29 | n, err = r.inner.Read(p) 30 | r.seen += uint64(n) 31 | 32 | if now := time.Now(); now.Sub(r.lastTime) >= status.Rate || err == io.EOF { 33 | r.cb(r.seen) 34 | r.lastTime = now 35 | } 36 | return 37 | } 38 | -------------------------------------------------------------------------------- /go/util/random/id.go: -------------------------------------------------------------------------------- 1 | package random 2 | 3 | import ( 4 | "crypto/rand" 5 | "encoding/hex" 6 | 7 | "github.com/attic-labs/noms/go/d" 8 | ) 9 | 10 | var ( 11 | reader = rand.Reader 12 | ) 13 | 14 | // Id creates a unique ID which is a random 16 byte hex string 15 | func Id() string { 16 | data := make([]byte, 16) 17 | _, err := reader.Read(data) 18 | d.Chk.NoError(err) 19 | return hex.EncodeToString(data) 20 | } 21 | -------------------------------------------------------------------------------- /go/util/random/id_test.go: -------------------------------------------------------------------------------- 1 | package random 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | type testReader byte 10 | 11 | func (r *testReader) Read(dest []byte) (int, error) { 12 | for i := 0; i < len(dest); i++ { 13 | dest[i] = byte(*r) 14 | } 15 | return len(dest), nil 16 | } 17 | 18 | func TestBasic(t *testing.T) { 19 | assert := assert.New(t) 20 | 21 | func() { 22 | var r testReader 23 | oldReader := reader 24 | reader = &r 25 | defer func() { 26 | reader = oldReader 27 | }() 28 | 29 | r = testReader(byte(0x00)) 30 | assert.Equal("00000000000000000000000000000000", Id()) 31 | r = testReader(byte(0x01)) 32 | assert.Equal("01010101010101010101010101010101", Id()) 33 | r = testReader(byte(0xFF)) 34 | assert.Equal("ffffffffffffffffffffffffffffffff", Id()) 35 | }() 36 | 37 | one := Id() 38 | two := Id() 39 | assert.NotEqual(one, two) 40 | } 41 | -------------------------------------------------------------------------------- /go/util/status/status.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | // Package status prints status messages to a console, overwriting previous values. 6 | package status 7 | 8 | import ( 9 | "fmt" 10 | "time" 11 | ) 12 | 13 | const ( 14 | clearLine = "\x1b[2K\r" 15 | Rate = 100 * time.Millisecond 16 | ) 17 | 18 | var ( 19 | lastTime time.Time 20 | lastFormat string 21 | lastArgs []interface{} 22 | ) 23 | 24 | func Clear() { 25 | fmt.Print(clearLine) 26 | reset(time.Time{}) 27 | } 28 | 29 | func WillPrint() bool { 30 | return time.Now().Sub(lastTime) >= Rate 31 | } 32 | 33 | func Printf(format string, args ...interface{}) { 34 | now := time.Now() 35 | if now.Sub(lastTime) < Rate { 36 | lastFormat, lastArgs = format, args 37 | } else { 38 | fmt.Printf(clearLine+format, args...) 39 | reset(now) 40 | } 41 | } 42 | 43 | func Done() { 44 | if lastArgs != nil { 45 | fmt.Printf(clearLine+lastFormat, lastArgs...) 46 | } 47 | fmt.Println() 48 | reset(time.Time{}) 49 | } 50 | 51 | func reset(time time.Time) { 52 | lastTime = time 53 | lastFormat, lastArgs = "", nil 54 | } 55 | -------------------------------------------------------------------------------- /go/util/test/equals_ignore_hashes.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package test 6 | 7 | import ( 8 | "regexp" 9 | "strconv" 10 | "strings" 11 | "testing" 12 | 13 | "github.com/attic-labs/noms/go/hash" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | var pattern = regexp.MustCompile("([0-9a-v]{" + strconv.Itoa(hash.StringLen) + "})") 18 | 19 | // EqualsIgnoreHashes compares two strings, ignoring hashes in them. 20 | func EqualsIgnoreHashes(tt *testing.T, expected, actual string) { 21 | if RemoveHashes(expected) != RemoveHashes(actual) { 22 | assert.Equal(tt, expected, actual) 23 | } 24 | } 25 | 26 | func RemoveHashes(str string) string { 27 | return pattern.ReplaceAllString(str, strings.Repeat("*", hash.StringLen)) 28 | } 29 | -------------------------------------------------------------------------------- /go/util/verbose/verbose.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package verbose 6 | 7 | import ( 8 | "log" 9 | 10 | "github.com/attic-labs/kingpin" 11 | ) 12 | 13 | var ( 14 | verbose bool 15 | quiet bool 16 | ) 17 | 18 | // RegisterVerboseFlags registers -v|--verbose flags for general usage 19 | func RegisterVerboseFlags(app *kingpin.Application) { 20 | // Must reset globals because under test this can get called multiple times. 21 | verbose = false 22 | quiet = false 23 | app.Flag("verbose", "show more").Short('v').BoolVar(&verbose) 24 | app.Flag("quite", "show less").Short('q').BoolVar(&quiet) 25 | } 26 | 27 | // Verbose returns True if the verbose flag was set 28 | func Verbose() bool { 29 | return verbose 30 | } 31 | 32 | func SetVerbose(v bool) { 33 | verbose = v 34 | } 35 | 36 | // Quiet returns True if the verbose flag was set 37 | func Quiet() bool { 38 | return quiet 39 | } 40 | 41 | func SetQuiet(q bool) { 42 | quiet = q 43 | } 44 | 45 | // Log calls Printf(format, args...) iff Verbose() returns true. 46 | func Log(format string, args ...interface{}) { 47 | if Verbose() { 48 | if len(args) > 0 { 49 | log.Printf(format+"\n", args...) 50 | } else { 51 | log.Println(format) 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /go/util/writers/max_line_writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package writers 6 | 7 | import "io" 8 | 9 | var ( 10 | // MaxLinesErr is an instance of MaxLinesError that gets returned by 11 | // Write() whenever the number of lines written has exceeded the number 12 | // in |MaxLineWriter.MaxLines|. 13 | MaxLinesErr = MaxLinesError{"Maximum number of lines written"} 14 | ) 15 | 16 | // MaxLinesError is the type of error returned by Write() whenever the number 17 | // of lines written has exceeded the number in |MaxLineWriter.MaxLines|. 18 | type MaxLinesError struct { 19 | msg string 20 | } 21 | 22 | func (e MaxLinesError) Error() string { return e.msg } 23 | 24 | // MaxLineWriter provides an io.Writer interface that counts the number of lines 25 | // that have been written. It will stop writing and returns an error if the 26 | // number of lines written exceeds the number specified in MaxLineWriter.NumLines. 27 | type MaxLineWriter struct { 28 | Dest io.Writer 29 | MaxLines uint32 30 | NumLines uint32 31 | } 32 | 33 | // Write() stops writing and returns an error if an attempt is made to write 34 | // any byte after |MaxLines| newLines have been written. For example, if MaxLines 35 | // is 1, all bytes will be written up to and including the 1st newline. If there 36 | // are any bytes in |data| after the 1st newline, an error will be returned. 37 | // 38 | // Callers can change the value of |w.MaxLines| before any call to Write(). 39 | // Setting MaxLines to 0 will allow any number of newLines. 40 | func (w *MaxLineWriter) Write(data []byte) (int, error) { 41 | if len(data) == 0 { 42 | return 0, nil 43 | } 44 | 45 | checkMax := w.MaxLines > 0 46 | 47 | if checkMax && w.NumLines >= w.MaxLines { 48 | return 0, MaxLinesErr 49 | } 50 | 51 | var err error 52 | byteCnt := 0 53 | 54 | for i, b := range data { 55 | if b == byte('\n') { 56 | w.NumLines++ 57 | if checkMax && w.NumLines > w.MaxLines { 58 | err = MaxLinesErr 59 | break 60 | } 61 | } else if checkMax && w.NumLines >= w.MaxLines { 62 | err = MaxLinesErr 63 | break 64 | } 65 | byteCnt = i 66 | } 67 | 68 | cnt, err1 := w.Dest.Write(data[:byteCnt+1]) 69 | if err1 != nil { 70 | return cnt, err1 71 | } 72 | return cnt, err 73 | } 74 | -------------------------------------------------------------------------------- /go/util/writers/prefix_writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package writers 6 | 7 | import "io" 8 | 9 | // PrefixWriter makes it easy to prefix lines with a custom prefix. Each time 10 | // it writes a byte after a newline('\n') character it calls PrefixFunc() to get 11 | // the byte slice that should be written. |NeedsPrefix| can be set to true to 12 | // cause a prefix to be written immediately. This is useful for causing a prefix 13 | // to get written on the first line. 14 | type PrefixWriter struct { 15 | Dest io.Writer 16 | PrefixFunc func(w *PrefixWriter) []byte 17 | NeedsPrefix bool 18 | NumLines uint32 19 | } 20 | 21 | // Write() will add a prefix to the beginning of each line. It obtains the 22 | // prefix by call |PrefixFunc(w *PrefixWriter)| before printing out any character 23 | // following a newLine. Callers can force a prefix to be printed out before the 24 | // first character in |data| by setting NeedsPrefix to true. Conversely, callers 25 | // can suppress prefixes from being printed by setting NeedsPrefix to false. 26 | 27 | func (w *PrefixWriter) Write(data []byte) (int, error) { 28 | writtenCnt := 0 29 | for i, b := range data { 30 | if w.NeedsPrefix { 31 | w.NeedsPrefix = false 32 | d1 := w.PrefixFunc(w) 33 | cnt, err := w.Dest.Write(d1) 34 | writtenCnt += cnt 35 | if err != nil { 36 | return writtenCnt, err 37 | } 38 | } 39 | if b == byte('\n') { 40 | w.NumLines++ 41 | w.NeedsPrefix = true 42 | } 43 | cnt, err := w.Dest.Write(data[i : i+1]) 44 | writtenCnt += cnt 45 | if err != nil { 46 | return writtenCnt, err 47 | } 48 | } 49 | return writtenCnt, nil 50 | } 51 | -------------------------------------------------------------------------------- /release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | git tag -f latest HEAD 3 | git push -f origin latest 4 | -------------------------------------------------------------------------------- /samples/cli/nomsconfig/.nomsconfig: -------------------------------------------------------------------------------- 1 | # Default database URL to be used whenever a database is not explictly provided 2 | [db.default] 3 | url = "ldb:.noms/tour" # This path is relative to the location of .nomsconfig 4 | 5 | # DB alias named `origin` that refers to the remote cli-tour db 6 | [db.origin] 7 | url = "http://demo.noms.io/cli-tour" 8 | 9 | # DB alias named `temp` that refers to a noms db stored under /tmp 10 | [db.temp] 11 | url = "ldb:/tmp/noms/shared" 12 | 13 | # DB alias named `http` that refers to the local http db 14 | [db.http] 15 | url = "http://localhost:8000" 16 | -------------------------------------------------------------------------------- /samples/go/counter/.gitignore: -------------------------------------------------------------------------------- 1 | counter 2 | -------------------------------------------------------------------------------- /samples/go/counter/counter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | 11 | "github.com/attic-labs/kingpin" 12 | 13 | "github.com/attic-labs/noms/go/config" 14 | "github.com/attic-labs/noms/go/types" 15 | "github.com/attic-labs/noms/go/util/verbose" 16 | ) 17 | 18 | func main() { 19 | app := kingpin.New("counter", "") 20 | dsStr := app.Arg("ds", "dataset to count in").Required().String() 21 | verbose.RegisterVerboseFlags(app) 22 | kingpin.MustParse(app.Parse(os.Args[1:])) 23 | 24 | cfg := config.NewResolver() 25 | db, ds, err := cfg.GetDataset(*dsStr) 26 | if err != nil { 27 | fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err) 28 | return 29 | } 30 | defer db.Close() 31 | 32 | newVal := uint64(1) 33 | if lastVal, ok := ds.MaybeHeadValue(); ok { 34 | newVal = uint64(lastVal.(types.Number)) + 1 35 | } 36 | 37 | _, err = db.CommitValue(ds, types.Number(newVal)) 38 | if err != nil { 39 | fmt.Fprintf(os.Stderr, "Error committing: %s\n", err) 40 | return 41 | } 42 | 43 | fmt.Println(newVal) 44 | } 45 | -------------------------------------------------------------------------------- /samples/go/counter/counter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/attic-labs/noms/go/spec" 11 | "github.com/attic-labs/noms/go/util/clienttest" 12 | "github.com/stretchr/testify/suite" 13 | ) 14 | 15 | func TestCounter(t *testing.T) { 16 | suite.Run(t, &counterTestSuite{}) 17 | } 18 | 19 | type counterTestSuite struct { 20 | clienttest.ClientTestSuite 21 | } 22 | 23 | func (s *counterTestSuite) TestCounter() { 24 | spec := spec.CreateValueSpecString("nbs", s.DBDir, "counter") 25 | args := []string{spec} 26 | stdout, stderr := s.MustRun(main, args) 27 | s.Equal("1\n", stdout) 28 | s.Equal("", stderr) 29 | stdout, stderr = s.MustRun(main, args) 30 | s.Equal("2\n", stdout) 31 | s.Equal("", stderr) 32 | stdout, stderr = s.MustRun(main, args) 33 | s.Equal("3\n", stdout) 34 | s.Equal("", stderr) 35 | } 36 | -------------------------------------------------------------------------------- /samples/go/csv/README.md: -------------------------------------------------------------------------------- 1 | # CSV Importer 2 | 3 | Imports a CSV file as `List` where `T` is a struct with fields corresponding to the CSV's column headers. The struct spec can also be set manually with the `-header` flag. 4 | 5 | ## Usage 6 | 7 | ```shell 8 | $ cd csv-import 9 | $ go build 10 | $ ./csv-import http://localhost:8000::foo 11 | ``` 12 | 13 | ## Some places for CSV files 14 | 15 | - https://data.cityofnewyork.us/api/views/kku6-nxdu/rows.csv?accessType=DOWNLOAD 16 | - http://www.opendatacache.com/ 17 | 18 | # CSV Exporter 19 | 20 | Export a dataset in CSV format to stdout with column headers. 21 | 22 | ## Usage 23 | 24 | ```shell 25 | $ cd csv-export 26 | $ go build 27 | $ ./csv-export http://localhost:8000:foo 28 | ``` 29 | -------------------------------------------------------------------------------- /samples/go/csv/common.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package csv 6 | 7 | import ( 8 | "fmt" 9 | "unicode/utf8" 10 | ) 11 | 12 | // StringToRune returns the rune contained in delimiter or an error. 13 | func StringToRune(delimiter string) (rune, error) { 14 | dlimLen := len(delimiter) 15 | if dlimLen == 0 { 16 | return 0, fmt.Errorf("delimiter flag must contain exactly one character (rune), not an empty string") 17 | } 18 | 19 | d, runeSize := utf8.DecodeRuneInString(delimiter) 20 | if d == utf8.RuneError { 21 | return 0, fmt.Errorf("Invalid utf8 string in delimiter flag: %s", delimiter) 22 | } 23 | if dlimLen != runeSize { 24 | return 0, fmt.Errorf("delimiter flag is too long. It must contain exactly one character (rune), but instead it is: %s", delimiter) 25 | } 26 | return d, nil 27 | } 28 | -------------------------------------------------------------------------------- /samples/go/csv/csv-analyze/.gitignore: -------------------------------------------------------------------------------- 1 | csv-analyze 2 | -------------------------------------------------------------------------------- /samples/go/csv/csv-analyze/analyze.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | "strings" 11 | 12 | "github.com/attic-labs/kingpin" 13 | 14 | "github.com/attic-labs/noms/go/d" 15 | "github.com/attic-labs/noms/go/types" 16 | "github.com/attic-labs/noms/go/util/profile" 17 | "github.com/attic-labs/noms/samples/go/csv" 18 | ) 19 | 20 | func main() { 21 | app := kingpin.New("csv-analyze", "") 22 | // Actually the delimiter uses runes, which can be multiple characters long. 23 | // https://blog.golang.org/strings 24 | delimiter := app.Flag("delimiter", "field delimiter for csv file, must be exactly one character long.").String() 25 | header := app.Flag("header", "header row. If empty, we'll use the first row of the file").String() 26 | skipRecords := app.Flag("skip-records", "number of records to skip at beginning of file").Uint() 27 | detectColumnTypes := app.Flag("detect-column-types", "detect column types by analyzing a portion of csv file").Bool() 28 | detectPrimaryKeys := app.Flag("detect-pk", "detect primary key candidates by analyzing a portion of csv file").Bool() 29 | numSamples := app.Flag("num-samples", "number of records to use for samples").Default("1000000").Int() 30 | numFieldsInPK := app.Flag("num-fields-pk", "maximum number of columns to consider when detecting PKs").Default("3").Int() 31 | r := app.Arg("filepath", "csv file to analyze").Required().File() 32 | 33 | profile.RegisterProfileFlags(app) 34 | 35 | kingpin.MustParse(app.Parse(os.Args[1:])) 36 | 37 | defer profile.MaybeStartProfile().Stop() 38 | defer (*r).Close() 39 | 40 | comma, err := csv.StringToRune(*delimiter) 41 | d.CheckError(err) 42 | 43 | cr := csv.NewCSVReader(*r, comma) 44 | csv.SkipRecords(cr, *skipRecords) 45 | 46 | var headers []string 47 | if *header == "" { 48 | headers, err = cr.Read() 49 | d.PanicIfError(err) 50 | } else { 51 | headers = strings.Split(*header, string(comma)) 52 | } 53 | 54 | kinds := []types.NomsKind{} 55 | if *detectColumnTypes { 56 | kinds = csv.GetSchema(cr, *numSamples, len(headers)) 57 | fmt.Fprintf(os.Stdout, "%s\n", strings.Join(csv.KindsToStrings(kinds), ",")) 58 | } 59 | 60 | if *detectPrimaryKeys { 61 | pks := csv.FindPrimaryKeys(cr, *numSamples, *numFieldsInPK, len(headers)) 62 | for _, pk := range pks { 63 | fmt.Fprintf(os.Stdout, "%s\n", strings.Join(csv.GetFieldNamesFromIndices(headers, pk), ",")) 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /samples/go/csv/csv-export/.gitignore: -------------------------------------------------------------------------------- 1 | csv-export 2 | -------------------------------------------------------------------------------- /samples/go/csv/csv-export/exporter.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | 11 | "github.com/attic-labs/kingpin" 12 | 13 | "github.com/attic-labs/noms/go/config" 14 | "github.com/attic-labs/noms/go/d" 15 | "github.com/attic-labs/noms/go/types" 16 | "github.com/attic-labs/noms/go/util/profile" 17 | "github.com/attic-labs/noms/go/util/verbose" 18 | "github.com/attic-labs/noms/samples/go/csv" 19 | ) 20 | 21 | func main() { 22 | app := kingpin.New("exporter", "") 23 | 24 | // Actually the delimiter uses runes, which can be multiple characters long. 25 | // https://blog.golang.org/strings 26 | delimiter := app.Flag("delimiter", "field delimiter for csv file, must be exactly one character long.").Default(",").String() 27 | dataset := app.Arg("dataset", "dataset to export").Required().String() 28 | 29 | verbose.RegisterVerboseFlags(app) 30 | profile.RegisterProfileFlags(app) 31 | 32 | kingpin.MustParse(app.Parse(os.Args[1:])) 33 | 34 | cfg := config.NewResolver() 35 | db, ds, err := cfg.GetDataset(*dataset) 36 | d.CheckError(err) 37 | 38 | defer db.Close() 39 | 40 | comma, err := csv.StringToRune(*delimiter) 41 | d.CheckError(err) 42 | 43 | err = d.Try(func() { 44 | defer profile.MaybeStartProfile().Stop() 45 | 46 | hv := ds.HeadValue() 47 | if l, ok := hv.(types.List); ok { 48 | structDesc := csv.GetListElemDesc(l, db) 49 | csv.WriteList(l, structDesc, comma, os.Stdout) 50 | } else if m, ok := hv.(types.Map); ok { 51 | structDesc := csv.GetMapElemDesc(m, db) 52 | csv.WriteMap(m, structDesc, comma, os.Stdout) 53 | } else { 54 | panic(fmt.Sprintf("Expected ListKind or MapKind, found %s", hv.Kind())) 55 | } 56 | }) 57 | if err != nil { 58 | fmt.Println("Failed to export dataset as CSV:") 59 | fmt.Println(err) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /samples/go/csv/csv-import/.gitignore: -------------------------------------------------------------------------------- 1 | csv-import 2 | -------------------------------------------------------------------------------- /samples/go/csv/csv-invert/.gitignore: -------------------------------------------------------------------------------- 1 | csv-invert -------------------------------------------------------------------------------- /samples/go/csv/csv_reader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package csv 6 | 7 | import ( 8 | "bufio" 9 | "encoding/csv" 10 | "io" 11 | ) 12 | 13 | var ( 14 | rByte byte = 13 // the byte that corresponds to the '\r' rune. 15 | nByte byte = 10 // the byte that corresponds to the '\n' rune. 16 | ) 17 | 18 | type reader struct { 19 | r *bufio.Reader 20 | } 21 | 22 | // Read replaces CR line endings in the source reader with LF line endings if the CR is not followed by a LF. 23 | func (r reader) Read(p []byte) (n int, err error) { 24 | n, err = r.r.Read(p) 25 | bn, err := r.r.Peek(1) 26 | for i, b := range p { 27 | // if the current byte is a CR and the next byte is NOT a LF then replace the current byte with a LF 28 | if j := i + 1; b == rByte && ((j < len(p) && p[j] != nByte) || (len(bn) > 0 && bn[0] != nByte)) { 29 | p[i] = nByte 30 | } 31 | } 32 | return 33 | } 34 | 35 | func SkipRecords(r *csv.Reader, n uint) error { 36 | var err error 37 | for i := uint(0); i < n; i++ { 38 | _, err = r.Read() 39 | if err != nil { 40 | return err 41 | } 42 | } 43 | return err 44 | } 45 | 46 | // NewCSVReader returns a new csv.Reader that splits on comma 47 | func NewCSVReader(res io.Reader, comma rune) *csv.Reader { 48 | bufRes := bufio.NewReader(res) 49 | r := csv.NewReader(reader{r: bufRes}) 50 | r.Comma = comma 51 | r.FieldsPerRecord = -1 // Don't enforce number of fields. 52 | return r 53 | } 54 | -------------------------------------------------------------------------------- /samples/go/csv/csv_reader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package csv 6 | 7 | import ( 8 | "bytes" 9 | "strings" 10 | "testing" 11 | 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestCR(t *testing.T) { 16 | testFile := []byte("a,b,c\r1,2,3\r") 17 | delimiter, err := StringToRune(",") 18 | 19 | r := NewCSVReader(bytes.NewReader(testFile), delimiter) 20 | lines, err := r.ReadAll() 21 | 22 | assert.NoError(t, err, "An error occurred while reading the data: %v", err) 23 | if len(lines) != 2 { 24 | t.Errorf("Wrong number of lines. Expected 2, got %d", len(lines)) 25 | } 26 | } 27 | 28 | func TestLF(t *testing.T) { 29 | testFile := []byte("a,b,c\n1,2,3\n") 30 | delimiter, err := StringToRune(",") 31 | 32 | r := NewCSVReader(bytes.NewReader(testFile), delimiter) 33 | lines, err := r.ReadAll() 34 | 35 | assert.NoError(t, err, "An error occurred while reading the data: %v", err) 36 | if len(lines) != 2 { 37 | t.Errorf("Wrong number of lines. Expected 2, got %d", len(lines)) 38 | } 39 | } 40 | 41 | func TestCRLF(t *testing.T) { 42 | testFile := []byte("a,b,c\r\n1,2,3\r\n") 43 | delimiter, err := StringToRune(",") 44 | 45 | r := NewCSVReader(bytes.NewReader(testFile), delimiter) 46 | lines, err := r.ReadAll() 47 | 48 | assert.NoError(t, err, "An error occurred while reading the data: %v", err) 49 | if len(lines) != 2 { 50 | t.Errorf("Wrong number of lines. Expected 2, got %d", len(lines)) 51 | } 52 | } 53 | 54 | func TestCRInQuote(t *testing.T) { 55 | testFile := []byte("a,\"foo,\rbar\",c\r1,\"2\r\n2\",3\r") 56 | delimiter, err := StringToRune(",") 57 | 58 | r := NewCSVReader(bytes.NewReader(testFile), delimiter) 59 | lines, err := r.ReadAll() 60 | 61 | assert.NoError(t, err, "An error occurred while reading the data: %v", err) 62 | if len(lines) != 2 { 63 | t.Errorf("Wrong number of lines. Expected 2, got %d", len(lines)) 64 | } 65 | if strings.Contains(lines[1][1], "\n\n") { 66 | t.Error("The CRLF was converted to a LFLF") 67 | } 68 | } 69 | 70 | func TestCRLFEndOfBufferLength(t *testing.T) { 71 | testFile := make([]byte, 4096*2, 4096*2) 72 | testFile[4095] = 13 // \r byte 73 | testFile[4096] = 10 // \n byte 74 | delimiter, err := StringToRune(",") 75 | 76 | r := NewCSVReader(bytes.NewReader(testFile), delimiter) 77 | lines, err := r.ReadAll() 78 | 79 | assert.NoError(t, err, "An error occurred while reading the data: %v", err) 80 | if len(lines) != 2 { 81 | t.Errorf("Wrong number of lines. Expected 2, got %d", len(lines)) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /samples/go/csv/kind_slice.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package csv 6 | 7 | import ( 8 | "fmt" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/attic-labs/noms/go/types" 13 | ) 14 | 15 | // KindSlice is an alias for []types.NomsKind. It's needed because types.NomsKind are really just 8 bit unsigned ints, which are what Go uses to represent 'byte', and this confuses the Go JSON marshal/unmarshal code -- it treats them as byte arrays and base64 encodes them! 16 | type KindSlice []types.NomsKind 17 | 18 | func (ks KindSlice) MarshalJSON() ([]byte, error) { 19 | elems := make([]string, len(ks)) 20 | for i, k := range ks { 21 | elems[i] = fmt.Sprintf("%d", k) 22 | } 23 | return []byte("[" + strings.Join(elems, ",") + "]"), nil 24 | } 25 | 26 | func (ks *KindSlice) UnmarshalJSON(value []byte) error { 27 | elems := strings.Split(string(value[1:len(value)-1]), ",") 28 | *ks = make(KindSlice, len(elems)) 29 | for i, e := range elems { 30 | ival, err := strconv.ParseUint(e, 10, 8) 31 | if err != nil { 32 | return err 33 | } 34 | (*ks)[i] = types.NomsKind(ival) 35 | } 36 | return nil 37 | } 38 | -------------------------------------------------------------------------------- /samples/go/csv/kind_slice_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package csv 6 | 7 | import ( 8 | "encoding/json" 9 | "fmt" 10 | "testing" 11 | 12 | "github.com/attic-labs/noms/go/types" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestKindSliceJSON(t *testing.T) { 17 | assert := assert.New(t) 18 | 19 | ks := KindSlice{types.NumberKind, types.StringKind, types.BoolKind} 20 | b, err := json.Marshal(&ks) 21 | assert.NoError(err) 22 | 23 | assert.Equal(fmt.Sprintf("[%d,%d,%d]", ks[0], ks[1], ks[2]), string(b)) 24 | 25 | var uks KindSlice 26 | err = json.Unmarshal(b, &uks) 27 | assert.NoError(err, "error with json.Unmarshal") 28 | assert.Equal(ks, uks) 29 | } 30 | -------------------------------------------------------------------------------- /samples/go/decent/README.md: -------------------------------------------------------------------------------- 1 | # About 2 | 3 | This directory contains two sample applications that demonstrate using Noms in a decentralized environment. 4 | 5 | Both applications implement multiuser chat, using different strategies. 6 | 7 | `p2p-chat` is the simplest possible example: a fully local noms replica is run on each node, and all nodes synchronize continuously with each other over HTTP. 8 | 9 | `ipfs-chat` backs Noms with the [IPFS](https://ipfs.io/) network, so that nodes don't have to keep a full local replica of all data. However, because [Filecoin](http://filecoin.io/) doesn't yet exist, *some node* does have to keep a full replica, so ipfs-chat has a `daemon` mode so that you can run a persistent node somewhere to be the replica of last resort. 10 | -------------------------------------------------------------------------------- /samples/go/decent/dbg/debug.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package dbg 6 | 7 | import ( 8 | "fmt" 9 | "github.com/attic-labs/noms/go/d" 10 | "log" 11 | "os" 12 | "strconv" 13 | ) 14 | 15 | var ( 16 | Filepath = "/tmp/noms-dbg.log" 17 | lg = NewLogger(Filepath) 18 | ) 19 | 20 | func NewLogger(fp string) *log.Logger { 21 | f, err := os.OpenFile(fp, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) 22 | d.PanicIfError(err) 23 | pid := strconv.FormatInt(int64(os.Getpid()), 10) 24 | return log.New(f, pid+": ", 0644) 25 | } 26 | 27 | func GetLogger() *log.Logger { 28 | return lg 29 | } 30 | 31 | func SetLogger(newLg *log.Logger) { 32 | lg = newLg 33 | } 34 | 35 | func Debug(s string, args ...interface{}) { 36 | s1 := fmt.Sprintf(s, args...) 37 | lg.Println(s1) 38 | } 39 | 40 | func BoxF(s string, args ...interface{}) func() { 41 | s1 := fmt.Sprintf(s, args...) 42 | Debug("starting %s", s1) 43 | f := func() { 44 | Debug("finished %s", s1) 45 | } 46 | return f 47 | } 48 | -------------------------------------------------------------------------------- /samples/go/decent/lib/datapager.go: -------------------------------------------------------------------------------- 1 | // See: https://github.com/attic-labs/noms/issues/3808 2 | // +build ignore 3 | 4 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 5 | // Licensed under the Apache License, version 2.0: 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | package lib 9 | 10 | import ( 11 | "fmt" 12 | "strings" 13 | 14 | "github.com/attic-labs/noms/go/datas" 15 | "github.com/attic-labs/noms/go/marshal" 16 | "github.com/attic-labs/noms/go/types" 17 | ) 18 | 19 | type dataPager struct { 20 | dataset datas.Dataset 21 | msgKeyChan chan types.String 22 | doneChan chan struct{} 23 | msgMap types.Map 24 | terms []string 25 | } 26 | 27 | func NewDataPager(ds datas.Dataset, mkChan chan types.String, doneChan chan struct{}, msgs types.Map, terms []string) *dataPager { 28 | return &dataPager{ 29 | dataset: ds, 30 | msgKeyChan: mkChan, 31 | doneChan: doneChan, 32 | msgMap: msgs, 33 | terms: terms, 34 | } 35 | } 36 | 37 | func (dp *dataPager) Close() { 38 | dp.doneChan <- struct{}{} 39 | } 40 | 41 | func (dp *dataPager) Next() (string, bool) { 42 | msgKey := <-dp.msgKeyChan 43 | if msgKey == "" { 44 | return "", false 45 | } 46 | nm := dp.msgMap.Get(msgKey) 47 | 48 | var m Message 49 | err := marshal.Unmarshal(nm, &m) 50 | if err != nil { 51 | return fmt.Sprintf("ERROR: %s", err.Error()), true 52 | } 53 | 54 | s1 := fmt.Sprintf("%s: %s", m.Author, m.Body) 55 | s2 := highlightTerms(s1, dp.terms) 56 | return s2, true 57 | } 58 | 59 | func (dp *dataPager) Prepend(lines []string, target int) ([]string, bool) { 60 | new := []string{} 61 | m, ok := dp.Next() 62 | if !ok { 63 | return lines, false 64 | } 65 | for ; ok && len(new) < target; m, ok = dp.Next() { 66 | new1 := strings.Split(m, "\n") 67 | new = append(new1, new...) 68 | } 69 | return append(new, lines...), true 70 | } 71 | -------------------------------------------------------------------------------- /samples/go/decent/lib/logger.go: -------------------------------------------------------------------------------- 1 | // See: https://github.com/attic-labs/noms/issues/3808 2 | // +build ignore 3 | 4 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 5 | // Licensed under the Apache License, version 2.0: 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | package lib 9 | 10 | import ( 11 | "fmt" 12 | "log" 13 | "os" 14 | 15 | "github.com/attic-labs/noms/go/d" 16 | "github.com/attic-labs/noms/samples/go/decent/dbg" 17 | ) 18 | 19 | func NewLogger(username string) *log.Logger { 20 | f, err := os.OpenFile(dbg.Filepath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644) 21 | d.PanicIfError(err) 22 | prefix := fmt.Sprintf("%d-%s: ", os.Getpid(), username) 23 | return log.New(f, prefix, 0644) 24 | } 25 | -------------------------------------------------------------------------------- /samples/go/decent/lib/model_test.go: -------------------------------------------------------------------------------- 1 | // See: https://github.com/attic-labs/noms/issues/3808 2 | // +build ignore 3 | 4 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 5 | // Licensed under the Apache License, version 2.0: 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | package lib 9 | 10 | import ( 11 | "testing" 12 | "time" 13 | 14 | "github.com/attic-labs/noms/go/chunks" 15 | "github.com/attic-labs/noms/go/datas" 16 | "github.com/attic-labs/noms/go/marshal" 17 | "github.com/attic-labs/noms/go/util/datetime" 18 | "github.com/stretchr/testify/assert" 19 | ) 20 | 21 | func TestBasics(t *testing.T) { 22 | a := assert.New(t) 23 | db := datas.NewDatabase(chunks.NewMemoryStoreFactory().CreateStore("")) 24 | ds := db.GetDataset("foo") 25 | ml, err := getAllMessages(ds) 26 | a.NoError(err) 27 | a.Equal(0, len(ml)) 28 | 29 | ds, err = AddMessage("body1", "aa", time.Unix(0, 0), ds) 30 | a.NoError(err) 31 | ml, err = getAllMessages(ds) 32 | a.NoError(err) 33 | expected := []Message{ 34 | Message{ 35 | Author: "aa", 36 | Body: "body1", 37 | ClientTime: datetime.DateTime{time.Unix(0, 0)}, 38 | Ordinal: 0, 39 | }, 40 | } 41 | a.Equal(expected, ml) 42 | 43 | ds, err = AddMessage("body2", "bob", time.Unix(1, 0), ds) 44 | a.NoError(err) 45 | ml, err = getAllMessages(ds) 46 | expected = append( 47 | []Message{ 48 | Message{ 49 | Author: "bob", 50 | Body: "body2", 51 | ClientTime: datetime.DateTime{time.Unix(1, 0)}, 52 | Ordinal: 1, 53 | }, 54 | }, 55 | expected..., 56 | ) 57 | a.NoError(err) 58 | a.Equal(expected, ml) 59 | } 60 | 61 | func getAllMessages(ds datas.Dataset) (r []Message, err error) { 62 | doneChan := make(chan struct{}) 63 | mm, keys, _ := ListMessages(ds, nil, doneChan) 64 | for k := range keys { 65 | mv := mm.Get(k) 66 | var m Message 67 | marshal.MustUnmarshal(mv, &m) 68 | r = append(r, m) 69 | } 70 | doneChan <- struct{}{} 71 | return r, nil 72 | } 73 | -------------------------------------------------------------------------------- /samples/go/decent/lib/term_index_test.go: -------------------------------------------------------------------------------- 1 | // See: https://github.com/attic-labs/noms/issues/3808 2 | // +build ignore 3 | 4 | // Copyright 2017 Attic Labs, Inc. All rights reserved. 5 | // Licensed under the Apache License, version 2.0: 6 | // http://www.apache.org/licenses/LICENSE-2.0 7 | 8 | package lib 9 | 10 | import ( 11 | "strings" 12 | "testing" 13 | 14 | "github.com/attic-labs/noms/go/chunks" 15 | "github.com/attic-labs/noms/go/types" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | func TestRun(t *testing.T) { 20 | a := assert.New(t) 21 | 22 | storage := &chunks.MemoryStorage{} 23 | vs := types.NewValueStore(storage.NewView()) 24 | defer vs.Close() 25 | 26 | docs := []struct { 27 | terms string 28 | id int 29 | }{ 30 | {"foo bar baz", 1}, 31 | {"foo baz", 2}, 32 | {"baz bat boo", 3}, 33 | } 34 | 35 | indexEditor := NewTermIndex(vs, types.NewMap(vs)).Edit() 36 | for _, doc := range docs { 37 | indexEditor.InsertAll(strings.Split(doc.terms, " "), types.Number(doc.id)) 38 | } 39 | 40 | index := indexEditor.Value() 41 | 42 | getMap := func(keys ...int) types.Map { 43 | m := types.NewMap(vs).Edit() 44 | for _, k := range keys { 45 | m.Set(types.Number(k), types.Bool(true)) 46 | } 47 | return m.Map() 48 | } 49 | 50 | test := func(search string, expect types.Map) { 51 | actual := index.Search(strings.Split(search, " ")) 52 | a.True(expect.Equals(actual)) 53 | } 54 | 55 | test("foo", getMap(1, 2)) 56 | test("baz", getMap(1, 2, 3)) 57 | test("bar baz", getMap(1)) 58 | test("boo", getMap(3)) 59 | test("blarg", getMap()) 60 | } 61 | -------------------------------------------------------------------------------- /samples/go/decent/p2p-chat/README.md: -------------------------------------------------------------------------------- 1 | This demo application is the simplest p2p chat app you could build using Noms. 2 | 3 | Basic idea: 4 | 5 | - Every node runs a Noms HTTP server (port controlled by --port) flag 6 | - Every node broadcasts its current commit and IP/port continuously 7 | - Every node continuously sync/merges with every other node 8 | (note that due to content addressing, most of these syncs will immediately exit) 9 | 10 | 11 | -------------------------------------------------------------------------------- /samples/go/hr/.gitignore: -------------------------------------------------------------------------------- 1 | hr 2 | -------------------------------------------------------------------------------- /samples/go/hr/README.md: -------------------------------------------------------------------------------- 1 | # HR 2 | 3 | This is a small command line application that manages a very simple hypothetical hr database. 4 | 5 | ## Usage 6 | 7 | ```shell 8 | go build 9 | ./hr --ds /tmp/my-noms::hr add-person 42 Abigail Architect 10 | ./hr --ds /tmp/my-noms::hr add-person 43 Samuel "Chief Laser Operator" 11 | ./hr --ds /tmp/my-noms::hr list-persons 12 | ``` 13 | -------------------------------------------------------------------------------- /samples/go/hr/build_test_data.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -d test-data ]; then 4 | mv test-data test-data.bak 5 | fi 6 | 7 | ./hr --ds test-data::hr add-person 7 "Aaron Boodman" "Chief Evangelism Officer" 8 | ./hr --ds test-data::hr add-person 13 "Samuel Boodman" "VP, Culture" 9 | -------------------------------------------------------------------------------- /samples/go/hr/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | 11 | "github.com/attic-labs/kingpin" 12 | 13 | "github.com/attic-labs/noms/go/config" 14 | "github.com/attic-labs/noms/go/datas" 15 | "github.com/attic-labs/noms/go/marshal" 16 | "github.com/attic-labs/noms/go/types" 17 | "github.com/attic-labs/noms/go/util/verbose" 18 | ) 19 | 20 | func main() { 21 | app := kingpin.New("hr", "") 22 | var dsStr = app.Flag("ds", "noms dataset to read/write from").Required().String() 23 | 24 | addCmd := app.Command("add-person", "Add a new person") 25 | id := addCmd.Arg("id", "unique person id").Required().Uint64() 26 | name := addCmd.Arg("name", "person name").Required().String() 27 | title := addCmd.Arg("title", "person title").Required().String() 28 | 29 | app.Command("list-persons", "list current persons") 30 | 31 | verbose.RegisterVerboseFlags(app) 32 | cmd := kingpin.MustParse(app.Parse(os.Args[1:])) 33 | cfg := config.NewResolver() 34 | db, ds, err := cfg.GetDataset(*dsStr) 35 | if err != nil { 36 | fmt.Fprintf(os.Stderr, "Could not create dataset: %s\n", err) 37 | return 38 | } 39 | defer db.Close() 40 | 41 | switch cmd { 42 | case "add-person": 43 | addPerson(db, ds, *id, *name, *title) 44 | case "list-persons": 45 | listPersons(ds) 46 | } 47 | } 48 | 49 | type Person struct { 50 | Name, Title string 51 | Id uint64 52 | } 53 | 54 | func addPerson(db datas.Database, ds datas.Dataset, id uint64, name, title string) { 55 | np, err := marshal.Marshal(db, Person{name, title, id}) 56 | if err != nil { 57 | fmt.Fprintln(os.Stderr, err) 58 | return 59 | } 60 | 61 | _, err = db.CommitValue(ds, getPersons(ds).Edit().Set(types.Number(id), np).Map()) 62 | if err != nil { 63 | fmt.Fprintf(os.Stderr, "Error committing: %s\n", err) 64 | return 65 | } 66 | } 67 | 68 | func listPersons(ds datas.Dataset) { 69 | d := getPersons(ds) 70 | if d.Empty() { 71 | fmt.Println("No people found") 72 | return 73 | } 74 | 75 | d.IterAll(func(k, v types.Value) { 76 | var p Person 77 | err := marshal.Unmarshal(v, &p) 78 | if err != nil { 79 | fmt.Fprintln(os.Stderr, err) 80 | return 81 | } 82 | fmt.Printf("%s (id: %d, title: %s)\n", p.Name, p.Id, p.Title) 83 | }) 84 | } 85 | 86 | func getPersons(ds datas.Dataset) types.Map { 87 | hv, ok := ds.MaybeHeadValue() 88 | if ok { 89 | return hv.(types.Map) 90 | } 91 | return types.NewMap(ds.Database()) 92 | } 93 | -------------------------------------------------------------------------------- /samples/go/hr/main_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "path" 9 | "runtime" 10 | "testing" 11 | 12 | "github.com/attic-labs/noms/go/spec" 13 | "github.com/attic-labs/noms/go/util/clienttest" 14 | "github.com/stretchr/testify/suite" 15 | ) 16 | 17 | func TestBasics(t *testing.T) { 18 | suite.Run(t, &testSuite{}) 19 | } 20 | 21 | type testSuite struct { 22 | clienttest.ClientTestSuite 23 | } 24 | 25 | func (s *testSuite) TestRoundTrip() { 26 | spec := spec.CreateValueSpecString("nbs", s.DBDir, "hr") 27 | stdout, stderr := s.MustRun(main, []string{"--ds", spec, "list-persons"}) 28 | s.Equal("No people found\n", stdout) 29 | s.Equal("", stderr) 30 | 31 | stdout, stderr = s.MustRun(main, []string{"--ds", spec, "add-person", "42", "Benjamin Kalman", "Programmer, Barista"}) 32 | s.Equal("", stdout) 33 | s.Equal("", stderr) 34 | 35 | stdout, stderr = s.MustRun(main, []string{"--ds", spec, "add-person", "43", "Abigail Boodman", "Chief Architect"}) 36 | s.Equal("", stdout) 37 | s.Equal("", stderr) 38 | 39 | stdout, stderr = s.MustRun(main, []string{"--ds", spec, "list-persons"}) 40 | s.Equal(`Benjamin Kalman (id: 42, title: Programmer, Barista) 41 | Abigail Boodman (id: 43, title: Chief Architect) 42 | `, stdout) 43 | s.Equal("", stderr) 44 | 45 | } 46 | 47 | func (s *testSuite) TestReadCanned() { 48 | _, p, _, _ := runtime.Caller(0) 49 | p = path.Join(path.Dir(p), "test-data") 50 | 51 | stdout, stderr := s.MustRun(main, []string{"--ds", spec.CreateValueSpecString("nbs", p, "hr"), "list-persons"}) 52 | s.Equal(`Aaron Boodman (id: 7, title: Chief Evangelism Officer) 53 | Samuel Boodman (id: 13, title: VP, Culture) 54 | `, stdout) 55 | s.Equal("", stderr) 56 | } 57 | 58 | func (s *testSuite) TestInvalidDatasetSpec() { 59 | // Should not crash 60 | _, _ = s.MustRun(main, []string{"--ds", "invalid-dataset", "list-persons"}) 61 | } 62 | -------------------------------------------------------------------------------- /samples/go/hr/test-data.bak/998se5i5mf15fld7f318818i6ie0c8rr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/samples/go/hr/test-data.bak/998se5i5mf15fld7f318818i6ie0c8rr -------------------------------------------------------------------------------- /samples/go/hr/test-data/998se5i5mf15fld7f318818i6ie0c8rr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/samples/go/hr/test-data/998se5i5mf15fld7f318818i6ie0c8rr -------------------------------------------------------------------------------- /samples/go/hr/test-data/LOCK: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/samples/go/hr/test-data/LOCK -------------------------------------------------------------------------------- /samples/go/hr/test-data/bsvid54jt8pjto211lcdl14tbfd39jmn: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/attic-labs/noms/e5fa29d95e8be72f1add640567f776d1e613d6bc/samples/go/hr/test-data/bsvid54jt8pjto211lcdl14tbfd39jmn -------------------------------------------------------------------------------- /samples/go/hr/test-data/manifest: -------------------------------------------------------------------------------- 1 | 4:7.18:8s92pdafhd4hkhav6r4748u1rjlosh1k:5b1e9knhol2orv0a8ej6tvelc46jp92l:bsvid54jt8pjto211lcdl14tbfd39jmn:2:998se5i5mf15fld7f318818i6ie0c8rr:2 -------------------------------------------------------------------------------- /samples/go/nomdex/nomdex.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "os" 10 | 11 | "github.com/attic-labs/kingpin" 12 | 13 | "github.com/attic-labs/noms/go/d" 14 | "github.com/attic-labs/noms/go/util/profile" 15 | "github.com/attic-labs/noms/go/util/verbose" 16 | ) 17 | 18 | func main() { 19 | registerUpdate() 20 | registerFind() 21 | verbose.RegisterVerboseFlags(kingpin.CommandLine) 22 | profile.RegisterProfileFlags(kingpin.CommandLine) 23 | 24 | switch kingpin.Parse() { 25 | case "up": 26 | runUpdate() 27 | case "find": 28 | runFind() 29 | } 30 | } 31 | 32 | func printError(err error, msgAndArgs ...interface{}) bool { 33 | if err != nil { 34 | err := d.Unwrap(err) 35 | switch len(msgAndArgs) { 36 | case 0: 37 | fmt.Fprintf(os.Stderr, "error: %s\n", err) 38 | case 1: 39 | fmt.Fprintf(os.Stderr, "%s%s\n", msgAndArgs[0], err) 40 | default: 41 | format, ok := msgAndArgs[0].(string) 42 | if ok { 43 | s1 := fmt.Sprintf(format, msgAndArgs[1:]...) 44 | fmt.Fprintf(os.Stderr, "%s%s\n", s1, err) 45 | } else { 46 | fmt.Fprintf(os.Stderr, "error: %s\n", err) 47 | } 48 | } 49 | } 50 | return err != nil 51 | } 52 | -------------------------------------------------------------------------------- /samples/go/nomsfs/.gitignore: -------------------------------------------------------------------------------- 1 | nomsfs 2 | -------------------------------------------------------------------------------- /samples/go/xml-import/.gitignore: -------------------------------------------------------------------------------- 1 | xml-import 2 | -------------------------------------------------------------------------------- /tools/file/file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package file 6 | 7 | import ( 8 | "fmt" 9 | "io" 10 | "os" 11 | "path/filepath" 12 | "runtime" 13 | 14 | "github.com/attic-labs/noms/go/d" 15 | ) 16 | 17 | // DumbCopy copies the contents of a regular file at srcPath (following symlinks) to a new regular file at dstPath. New file is created with same mode. 18 | func DumbCopy(srcPath, dstPath string) { 19 | chkClose := func(c io.Closer) { d.PanicIfError(c.Close()) } 20 | 21 | info, err := os.Stat(srcPath) 22 | d.PanicIfError(err) 23 | 24 | if info.IsDir() { 25 | d.PanicIfError(ErrNoCopyDir) 26 | } 27 | 28 | src, err := os.Open(srcPath) 29 | d.PanicIfError(err) 30 | defer chkClose(src) 31 | 32 | dst, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, info.Mode()) 33 | d.PanicIfError(err) 34 | defer chkClose(dst) 35 | _, err = io.Copy(dst, src) 36 | d.PanicIfError(err) 37 | } 38 | 39 | // MyDir returns the directory in which the file containing the calling source code resides. 40 | func MyDir() string { 41 | _, path, _, ok := runtime.Caller(1) 42 | if !ok { 43 | d.Panic("Should have been able to get Caller.") 44 | } 45 | return filepath.Dir(path) 46 | } 47 | 48 | var ErrNoCopyDir = fmt.Errorf("attempted to copy a directory") 49 | -------------------------------------------------------------------------------- /tools/file/file_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | // Licensed under the Apache License, version 2.0: 3 | // http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | package file 6 | 7 | import ( 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | "testing" 12 | 13 | "github.com/attic-labs/noms/go/d" 14 | "github.com/stretchr/testify/suite" 15 | ) 16 | 17 | const ( 18 | contents = "hey" 19 | ) 20 | 21 | func TestSerialRunnerTestSuite(t *testing.T) { 22 | suite.Run(t, &FileTestSuite{}) 23 | } 24 | 25 | type FileTestSuite struct { 26 | suite.Suite 27 | dir, src, exc string 28 | } 29 | 30 | func (suite *FileTestSuite) SetupTest() { 31 | var err error 32 | suite.dir, err = ioutil.TempDir(os.TempDir(), "") 33 | suite.NoError(err) 34 | suite.src = filepath.Join(suite.dir, "srcfile") 35 | suite.exc = filepath.Join(suite.dir, "excfile") 36 | suite.NoError(ioutil.WriteFile(suite.src, []byte(contents), 0644)) 37 | suite.NoError(ioutil.WriteFile(suite.exc, []byte(contents), 0755)) 38 | } 39 | 40 | func (suite *FileTestSuite) TearDownTest() { 41 | os.Remove(suite.dir) 42 | } 43 | 44 | func (suite *FileTestSuite) TestCopyFile() { 45 | dst := filepath.Join(suite.dir, "dstfile") 46 | 47 | test := func(src string, mode int) { 48 | DumbCopy(src, dst) 49 | 50 | info, err := os.Stat(src) 51 | suite.NoError(err) 52 | suite.Equal(mode, int(info.Mode())) 53 | 54 | out, err := ioutil.ReadFile(dst) 55 | suite.NoError(err) 56 | suite.Equal(contents, string(out)) 57 | } 58 | 59 | test(suite.src, 0644) 60 | test(suite.exc, 0755) 61 | } 62 | 63 | func (suite *FileTestSuite) TestCopyLink() { 64 | link := filepath.Join(suite.dir, "link") 65 | suite.NoError(os.Symlink(suite.src, link)) 66 | 67 | dst := filepath.Join(suite.dir, "dstfile") 68 | DumbCopy(link, dst) 69 | 70 | info, err := os.Lstat(dst) 71 | suite.NoError(err) 72 | suite.True(info.Mode().IsRegular()) 73 | 74 | out, err := ioutil.ReadFile(dst) 75 | suite.NoError(err) 76 | suite.Equal(contents, string(out)) 77 | } 78 | 79 | func (suite *FileTestSuite) TestNoCopyDir() { 80 | dir, err := ioutil.TempDir(suite.dir, "") 81 | suite.NoError(err) 82 | 83 | dst := filepath.Join(suite.dir, "dst") 84 | suite.Error(d.Try(func() { DumbCopy(dir, dst) })) 85 | } 86 | -------------------------------------------------------------------------------- /tools/noms/README.md: -------------------------------------------------------------------------------- 1 | # Noms build script helpers 2 | 3 | These are helper functions for writing your Noms app build and staging scripts. 4 | 5 | ## Writing your scripts 6 | 7 | ### Build scripts 8 | Your build script must be named *build.py*. It will be discovered by the system and executed in the directory in which it's found. It must require no arguments, though environment variables will propagate in. 9 | 10 | ### Staging scripts 11 | After your build script gets run, we'll run your staging script -- the purpose of which is to take your build products and put them in a directory that's ready to be packaged and deployed somewhere. This script must be called *stage.py* and take as its sole argument the path to a directory where all project code is being staged. 12 | 13 | ### Libraries 14 | We have provided a library to make writing your staging scripts easier. Example usage: 15 | ```python 16 | #!/usr/bin/python 17 | 18 | import noms.staging as staging 19 | 20 | if __name__ == '__main__': 21 | staging.Main('nerdosphere', staging.GlobCopier('index.html', 'styles.css', '*.js')) 22 | ``` 23 | Importing and using `noms.staging` handles determining where you should stage your code and creating the necessary directories for you. You just pass it the name of your project and a function that knows how to stage your build artifacts, given a path under which to put everything. 24 | 25 | 26 | ## Develop 27 | 28 | * To run unittests: `python -m unittest discover -p "*_test.py" -s $GOPATH/src/github.com/attic-labs/noms/tools` 29 | -------------------------------------------------------------------------------- /tools/noms/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 Attic Labs, Inc. All rights reserved. 2 | # Licensed under the Apache License, version 2.0: 3 | # http://www.apache.org/licenses/LICENSE-2.0 4 | 5 | -------------------------------------------------------------------------------- /tools/noms/copy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright 2016 Attic Labs, Inc. All rights reserved. 4 | # Licensed under the Apache License, version 2.0: 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | import os.path, shutil 8 | 9 | def Peers(me, dstDir): 10 | """Peers copies the peers of me into dstDir. 11 | 12 | Peers looks for files, directories and symlinks next to me 13 | and copies them (with the same basenames) to dstDir, which is 14 | presumed to exist. 15 | """ 16 | myDir = os.path.dirname(os.path.abspath(me)) 17 | names = os.listdir(myDir) 18 | for basename in names: 19 | src = os.path.join(myDir, basename) 20 | dst = os.path.join(dstDir, basename) 21 | if os.path.samefile(me, src): 22 | continue 23 | 24 | if os.path.islink(src): 25 | linkto = os.readlink(src) 26 | os.symlink(linkto, dst) 27 | elif os.path.isfile(src): 28 | shutil.copy2(src, dst) 29 | elif os.path.isdir(src): 30 | shutil.copytree(src, dst) 31 | else: 32 | raise Exception("Unknown file type at " + src) 33 | -------------------------------------------------------------------------------- /tools/noms/copy_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright 2016 Attic Labs, Inc. All rights reserved. 4 | # Licensed under the Apache License, version 2.0: 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | import os, os.path, shutil, tempfile, unittest 8 | import copy 9 | 10 | class TestCopy(unittest.TestCase): 11 | def setUp(self): 12 | self.tempdir = os.path.realpath(tempfile.mkdtemp()) 13 | 14 | 15 | def tearDown(self): 16 | shutil.rmtree(self.tempdir, ignore_errors=True) 17 | 18 | 19 | def test_CopyPeers(self): 20 | nested = tempfile.mkdtemp(dir=self.tempdir) 21 | otherNested = tempfile.mkdtemp(dir=self.tempdir) 22 | 23 | def mkfile(): 24 | with tempfile.NamedTemporaryFile(dir=nested, delete=False) as f: 25 | return f.name 26 | 27 | me = mkfile() 28 | peerFile = os.path.basename(mkfile()) 29 | peerDir = os.path.basename(tempfile.mkdtemp(dir=nested)) 30 | peerLink = 'link' 31 | peerLinkAbs = os.path.join(nested, 'link') 32 | os.symlink(peerFile, peerLinkAbs) 33 | 34 | copy.Peers(me, otherNested) 35 | self.assertTrue(os.path.islink(os.path.join(otherNested, peerLink))) 36 | self.assertTrue(os.path.isfile(os.path.join(otherNested, peerFile))) 37 | self.assertTrue(os.path.isdir(os.path.join(otherNested, peerDir))) 38 | self.assertFalse(os.path.lexists(os.path.join(otherNested, os.path.basename(me)))) 39 | 40 | 41 | if __name__ == '__main__': 42 | unittest.main() 43 | -------------------------------------------------------------------------------- /tools/noms/pushd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright 2016 Attic Labs, Inc. All rights reserved. 4 | # Licensed under the Apache License, version 2.0: 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | import os 8 | from contextlib import contextmanager 9 | 10 | @contextmanager 11 | def pushd(path): 12 | currentDir = os.getcwd() 13 | os.chdir(path) 14 | yield 15 | os.chdir(currentDir) 16 | -------------------------------------------------------------------------------- /tools/noms/symlink.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright 2016 Attic Labs, Inc. All rights reserved. 4 | # Licensed under the Apache License, version 2.0: 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | import os 8 | 9 | class LinkError(Exception): 10 | """Raised when forcing a symlink fails for a non-OS reason.""" 11 | pass 12 | 13 | def Force(source, linkName): 14 | """ 15 | Force forces linkName to be a symlink to source, as long as its not a dir. 16 | Creates a symlink from linkName to source, clobbering linkName as long as its not a directory. 17 | """ 18 | if not os.path.lexists(linkName): 19 | os.symlink(source, linkName) 20 | return 21 | 22 | if os.path.islink(linkName) or os.path.isfile(linkName): 23 | os.remove(linkName) 24 | os.symlink(source, linkName) 25 | return 26 | 27 | raise LinkError("Refusing to clobber " + linkName) 28 | -------------------------------------------------------------------------------- /tools/noms/symlink_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | # Copyright 2016 Attic Labs, Inc. All rights reserved. 4 | # Licensed under the Apache License, version 2.0: 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | import os, os.path, shutil, tempfile, unittest 8 | import symlink 9 | 10 | class TestForceSymlink(unittest.TestCase): 11 | CONTENTS = 'test file contents' 12 | 13 | def setUp(self): 14 | self.tempdir = tempfile.mkdtemp() 15 | self.source = tempfile.NamedTemporaryFile(dir=self.tempdir, delete=False) 16 | with self.source.file as f: 17 | f.write(self.CONTENTS) 18 | 19 | 20 | def tearDown(self): 21 | shutil.rmtree(self.tempdir, ignore_errors=True) 22 | 23 | 24 | def verifySymlink(self, linkName): 25 | with open(linkName, 'r') as f: 26 | actual = f.read() 27 | self.assertEqual(self.CONTENTS, actual) 28 | 29 | 30 | def test_ClobberFile(self): 31 | linkName = os.path.join(self.tempdir, 'link') 32 | with open(linkName, 'w') as f: 33 | f.write('foo') 34 | 35 | symlink.Force(self.source.name, linkName) 36 | self.verifySymlink(linkName) 37 | 38 | 39 | def test_ClobberSymlink(self): 40 | linkName = os.path.join(self.tempdir, 'link') 41 | os.symlink('nowhere', linkName) 42 | 43 | symlink.Force(self.source.name, linkName) 44 | self.verifySymlink(linkName) 45 | 46 | 47 | def test_NoClobberDir(self): 48 | linkName = os.path.join(self.tempdir, 'link') 49 | os.mkdir(linkName, 0777) 50 | 51 | try: 52 | symlink.Force(self.source.name, linkName) 53 | except symlink.LinkError: 54 | pass 55 | 56 | 57 | if __name__ == '__main__': 58 | unittest.main() 59 | --------------------------------------------------------------------------------