├── .gitignore ├── cmd └── triplestore │ ├── .gitignore │ └── main.go ├── .travis.yml ├── testdata ├── ntriples │ └── w3c_suite │ │ ├── positives │ │ ├── nt-syntax-file-01.nt │ │ ├── nt-syntax-file-03.nt.expected │ │ ├── nt-syntax-file-02.nt │ │ ├── nt-syntax-file-03.nt │ │ ├── literal.nt │ │ ├── nt-syntax-bnode-01.nt │ │ ├── nt-syntax-str-esc-01.nt │ │ ├── nt-syntax-string-01.nt │ │ ├── langtagged_string.nt │ │ ├── literal_with_2_squotes.nt │ │ ├── literal_with_BACKSPACE.nt │ │ ├── literal_with_FORM_FEED.nt │ │ ├── literal_with_LINE_FEED.nt │ │ ├── literal_with_dquote.nt │ │ ├── literal_with_squote.nt │ │ ├── nt-syntax-bnode-01.nt.expected │ │ ├── nt-syntax-str-esc-02.nt │ │ ├── nt-syntax-string-02.nt │ │ ├── literal_with_2_dquotes.nt │ │ ├── literal_with_CARRIAGE_RETURN.nt │ │ ├── literal_with_REVERSE_SOLIDUS.nt │ │ ├── nt-syntax-str-esc-03.nt │ │ ├── nt-syntax-string-03.nt │ │ ├── nt-syntax-uri-01.nt │ │ ├── literal_with_CHARACTER_TABULATION.nt │ │ ├── literal_with_numeric_escape4.nt │ │ ├── lantag_with_subtag.nt │ │ ├── literal_ascii_boundaries.nt.TODO │ │ ├── literal_with_numeric_escape8.nt │ │ ├── literal_all_punctuation.nt │ │ ├── literal_with_REVERSE_SOLIDUS2.nt │ │ ├── literal_with_UTF8_boundaries.nt │ │ ├── nt-syntax-uri-02.nt │ │ ├── literal_true.nt │ │ ├── nt-syntax-uri-03.nt │ │ ├── literal_false.nt │ │ ├── nt-syntax-bnode-02.nt │ │ ├── nt-syntax-bnode-03.nt │ │ ├── nt-syntax-datatypes-01.nt │ │ ├── nt-syntax-datatypes-02.nt │ │ ├── nt-syntax-bnode-02.nt.expected │ │ ├── nt-syntax-bnode-03.nt.expected │ │ ├── nt-syntax-uri-04.nt │ │ ├── literal_all_controls.nt │ │ ├── minimal_whitespace.nt │ │ ├── minimal_whitespace.nt.expected │ │ ├── comment_following_triple.nt │ │ ├── nt-syntax-subm-01.nt.expected │ │ └── nt-syntax-subm-01.nt │ │ └── negatives │ │ ├── nt-syntax-bad-base-01.nt │ │ ├── nt-syntax-bad-prefix-01.nt │ │ ├── nt-syntax-bad-num-01.nt │ │ ├── nt-syntax-bad-num-02.nt │ │ ├── nt-syntax-bad-num-03.nt │ │ ├── nt-syntax-bad-string-02.nt │ │ ├── nt-syntax-bad-string-06.nt │ │ ├── nt-syntax-bad-string-07.nt │ │ ├── nt-syntax-bad-string-01.nt │ │ ├── nt-syntax-bad-string-03.nt │ │ ├── nt-syntax-bad-string-04.nt │ │ ├── nt-syntax-bad-string-05.nt.LENIENT │ │ ├── nt-syntax-bad-esc-01.nt.LENIENT │ │ ├── nt-syntax-bad-esc-02.nt.LENIENT │ │ ├── nt-syntax-bad-lang-01.nt.LENIENT │ │ ├── nt-syntax-bad-esc-03.nt.LENIENT │ │ ├── nt-syntax-bad-uri-06.nt.LENIENT │ │ ├── nt-syntax-bad-uri-07.nt.LENIENT │ │ ├── nt-syntax-bad-uri-08.nt.LENIENT │ │ ├── nt-syntax-bad-struct-01.nt.LENIENT │ │ ├── nt-syntax-bad-uri-01.nt.LENIENT │ │ ├── nt-syntax-bad-uri-09.nt.LENIENT │ │ ├── nt-syntax-bad-uri-02.nt.LENIENT │ │ ├── nt-syntax-bad-uri-03.nt.LENIENT │ │ ├── nt-syntax-bad-struct-02.nt.LENIENT │ │ ├── nt-syntax-bad-uri-04.nt.LENIENT │ │ └── nt-syntax-bad-uri-05.nt.LENIENT ├── sample.nt └── bench │ └── decode_1.bin ├── fuzz ├── binary │ ├── main.go │ └── corpus │ │ └── samples.bin ├── ntriples │ ├── main.go │ └── corpus │ │ └── samples.nt └── README.md ├── util_test.go ├── types.go ├── last-benches.txt ├── ntparser_w3c_test.go ├── struct.go ├── rdf.go ├── tree.go ├── tree_test.go ├── rdf_test.go ├── codecbench_test.go ├── source.go ├── struct_test.go ├── source_test.go ├── streamcodec_test.go ├── ntparser.go ├── decode.go ├── encode.go ├── README.md ├── dsl_test.go ├── ntparser_test.go ├── codec_test.go ├── LICENSE └── dsl.go /.gitignore: -------------------------------------------------------------------------------- 1 | *zip 2 | .idea 3 | -------------------------------------------------------------------------------- /cmd/triplestore/.gitignore: -------------------------------------------------------------------------------- 1 | triplestore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.8 5 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-file-01.nt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-file-03.nt.expected: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-file-02.nt: -------------------------------------------------------------------------------- 1 | #Empty file. 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-base-01.nt: -------------------------------------------------------------------------------- 1 | @base . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-prefix-01.nt: -------------------------------------------------------------------------------- 1 | @prefix : . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-file-03.nt: -------------------------------------------------------------------------------- 1 | #One comment, one empty line. 2 | 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal.nt: -------------------------------------------------------------------------------- 1 | "x" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-num-01.nt: -------------------------------------------------------------------------------- 1 | 1 . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-bnode-01.nt: -------------------------------------------------------------------------------- 1 | _:a . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-num-02.nt: -------------------------------------------------------------------------------- 1 | 1.0 . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-num-03.nt: -------------------------------------------------------------------------------- 1 | 1.0e0 . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-string-02.nt: -------------------------------------------------------------------------------- 1 | 1.0 . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-string-06.nt: -------------------------------------------------------------------------------- 1 | "abc . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-string-07.nt: -------------------------------------------------------------------------------- 1 | abc" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-str-esc-01.nt: -------------------------------------------------------------------------------- 1 | "a\n" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-string-01.nt: -------------------------------------------------------------------------------- 1 | "string" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-string-01.nt: -------------------------------------------------------------------------------- 1 | "abc' . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-string-03.nt: -------------------------------------------------------------------------------- 1 | 1.0e1 . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-string-04.nt: -------------------------------------------------------------------------------- 1 | '''abc''' . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/langtagged_string.nt: -------------------------------------------------------------------------------- 1 | "chat"@en . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_2_squotes.nt: -------------------------------------------------------------------------------- 1 | "x''y" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_BACKSPACE.nt: -------------------------------------------------------------------------------- 1 | "\b" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_FORM_FEED.nt: -------------------------------------------------------------------------------- 1 | "\f" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_LINE_FEED.nt: -------------------------------------------------------------------------------- 1 | "\n" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_dquote.nt: -------------------------------------------------------------------------------- 1 | "x\"y" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_squote.nt: -------------------------------------------------------------------------------- 1 | "x'y" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-bnode-01.nt.expected: -------------------------------------------------------------------------------- 1 | _:a . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-str-esc-02.nt: -------------------------------------------------------------------------------- 1 | "a\u0020b" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-string-02.nt: -------------------------------------------------------------------------------- 1 | "string"@en . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_2_dquotes.nt: -------------------------------------------------------------------------------- 1 | "x\"\"y" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_CARRIAGE_RETURN.nt: -------------------------------------------------------------------------------- 1 | "\r" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_REVERSE_SOLIDUS.nt: -------------------------------------------------------------------------------- 1 | "\\" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-str-esc-03.nt: -------------------------------------------------------------------------------- 1 | "a\U00000020b" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-string-03.nt: -------------------------------------------------------------------------------- 1 | "string"@en-uk . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-uri-01.nt: -------------------------------------------------------------------------------- 1 | . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-string-05.nt.LENIENT: -------------------------------------------------------------------------------- 1 | """abc""" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_CHARACTER_TABULATION.nt: -------------------------------------------------------------------------------- 1 | "\t" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_numeric_escape4.nt: -------------------------------------------------------------------------------- 1 | "\u006F" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/lantag_with_subtag.nt: -------------------------------------------------------------------------------- 1 | "Cheers"@en-UK . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_ascii_boundaries.nt.TODO: -------------------------------------------------------------------------------- 1 | " &([]" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_numeric_escape8.nt: -------------------------------------------------------------------------------- 1 | "\U0000006F" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_all_punctuation.nt: -------------------------------------------------------------------------------- 1 | " !\"#$%&():;<=>?@[]^_`{|}~" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_REVERSE_SOLIDUS2.nt: -------------------------------------------------------------------------------- 1 | "test-\\" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_with_UTF8_boundaries.nt: -------------------------------------------------------------------------------- 1 | "€߿ࠀ࿿က쿿퀀퟿�𐀀𿿽񀀀󿿽􀀀􏿽" . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-esc-01.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad string escape 2 | "a\zb" . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-esc-02.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad string escape 2 | "\uWXYZ" . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-lang-01.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad lang tag 2 | "string"@1 . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-uri-02.nt: -------------------------------------------------------------------------------- 1 | # x53 is capital S 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-esc-03.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad string escape 2 | "\U0000WXYZ" . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-06.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # No relative IRIs in N-Triples 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-07.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # No relative IRIs in N-Triples 2 |

. 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-08.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # No relative IRIs in N-Triples 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_true.nt: -------------------------------------------------------------------------------- 1 | "true"^^ . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-uri-03.nt: -------------------------------------------------------------------------------- 1 | # x53 is capital S 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-struct-01.nt.LENIENT: -------------------------------------------------------------------------------- 1 | , . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_false.nt: -------------------------------------------------------------------------------- 1 | "false"^^ . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-bnode-02.nt: -------------------------------------------------------------------------------- 1 | _:a . 2 | _:a . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-bnode-03.nt: -------------------------------------------------------------------------------- 1 | _:1a . 2 | _:1a . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-datatypes-01.nt: -------------------------------------------------------------------------------- 1 | "123"^^ . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-datatypes-02.nt: -------------------------------------------------------------------------------- 1 | "123"^^ . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-01.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad IRI : space. 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-09.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # No relative IRIs in N-Triples 2 | "foo"^^

. 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-02.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad IRI : bad escape 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-03.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad IRI : bad escape 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-bnode-02.nt.expected: -------------------------------------------------------------------------------- 1 | _:a . 2 | _:a . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-bnode-03.nt.expected: -------------------------------------------------------------------------------- 1 | _:1a . 2 | _:1a . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-struct-02.nt.LENIENT: -------------------------------------------------------------------------------- 1 | ; , . 2 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-04.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad IRI : character escapes not allowed. 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/negatives/nt-syntax-bad-uri-05.nt.LENIENT: -------------------------------------------------------------------------------- 1 | # Bad IRI : character escapes not allowed. 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-uri-04.nt: -------------------------------------------------------------------------------- 1 | # IRI with all chars in it. 2 | . 3 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/literal_all_controls.nt: -------------------------------------------------------------------------------- 1 | "\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\t\u000B\u000C\u000E\u000F\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001A\u001B\u001C\u001D\u001E\u001F" . 2 | -------------------------------------------------------------------------------- /fuzz/binary/main.go: -------------------------------------------------------------------------------- 1 | package binary 2 | 3 | import "github.com/wallix/triplestore" 4 | import "bytes" 5 | 6 | func Fuzz(data []byte) int { 7 | dec := triplestore.NewBinaryDecoder(bytes.NewReader(data)) 8 | if _, err := dec.Decode(); err != nil { 9 | return 0 10 | } 11 | return 1 12 | } 13 | -------------------------------------------------------------------------------- /fuzz/ntriples/main.go: -------------------------------------------------------------------------------- 1 | package ntriples 2 | 3 | import "github.com/wallix/triplestore" 4 | import "bytes" 5 | 6 | func Fuzz(data []byte) int { 7 | dec := triplestore.NewLenientNTDecoder(bytes.NewReader(data)) 8 | if _, err := dec.Decode(); err != nil { 9 | return 0 10 | } 11 | return 1 12 | } 13 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/minimal_whitespace.nt: -------------------------------------------------------------------------------- 1 | . 2 | "Alice". 3 | _:o. 4 | _:s. 5 | _:s"Alice". 6 | _:s_:bnode1. 7 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/minimal_whitespace.nt.expected: -------------------------------------------------------------------------------- 1 | . 2 | "Alice" . 3 | _:o . 4 | _:s . 5 | _:s "Alice" . 6 | _:s _:bnode1 . 7 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/comment_following_triple.nt: -------------------------------------------------------------------------------- 1 | . # comment 2 | _:o . # comment 3 | "o" . # comment 4 | "o"^^ . # comment 5 | "o"@en . # comment -------------------------------------------------------------------------------- /fuzz/binary/corpus/samples.bin: -------------------------------------------------------------------------------- 1 | apredbapredbapredbanonpredbapreda> . 2 | apredc a apredc a capredc a\n\ncapredcchatapred 4 | xsd:stringchat -------------------------------------------------------------------------------- /fuzz/ntriples/corpus/samples.nt: -------------------------------------------------------------------------------- 1 | . 2 | . 3 | .#comment 4 | # comment 5 | _:anon . 6 | _:anon . 7 | . 8 | "c" . 9 | ""^^ . 10 | " "^^ . 11 | "x"^^ . 12 | "\""^^ . 13 | ""^^ . 14 | "a "^^ . 15 | "a c"^^ . 16 | "a\n\nc"^^ . 17 | "chat"^^ . 18 | "chat"@fr . 19 | -------------------------------------------------------------------------------- /fuzz/README.md: -------------------------------------------------------------------------------- 1 | ## Fuzzing decoders 2 | 3 | For context have a look at [go-fuzz](https://github.com/dvyukov/go-fuzz) 4 | 5 | Corpus sample data are in directories `fuzz/{ntriples,binary}/corpus/samples.*` 6 | 7 | For instance, to fuzz the ntriples decoding for instance do the following steps: 8 | 9 | 1. Build with 10 | 11 | ```sh 12 | go-fuzz-build github.com/wallix/triplestore/fuzz/ntriples 13 | ``` 14 | 15 | 2. Then 16 | 17 | ```sh 18 | go-fuzz -bin=ntriples-fuzz.zip -workdir=fuzz/ntriples/corpus 19 | ``` 20 | 21 | 3. Stop (with Ctr+C) when enough. Look at the results. Fix the bugs and clean up the generated unneeded data (`rm -rf fuzz/ntriples/corpus/{corpus,crashers,suppressions}`) 22 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "regexp" 7 | ) 8 | 9 | func contains(arr [][]byte, s []byte) bool { 10 | for _, a := range arr { 11 | if bytes.Equal(s, a) { 12 | return true 13 | } 14 | } 15 | return false 16 | } 17 | 18 | var ( 19 | endOfLineComments = regexp.MustCompile(`(.*\.)\s+(#.*)`) 20 | ) 21 | 22 | func cleanupNTriplesForComparison(b []byte) []byte { 23 | scn := bufio.NewScanner(bytes.NewReader(b)) 24 | var cleaned bytes.Buffer 25 | for scn.Scan() { 26 | line := scn.Text() 27 | if empty, _ := regexp.MatchString(`^\s*$`, line); empty { 28 | continue 29 | } 30 | if comment, _ := regexp.MatchString(`^\s*#`, line); comment { 31 | continue 32 | } 33 | l := endOfLineComments.ReplaceAll([]byte(line), []byte("$1")) 34 | cleaned.Write(l) 35 | cleaned.WriteByte('\n') 36 | } 37 | 38 | return cleaned.Bytes() 39 | } 40 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type XsdType string 9 | 10 | var ( 11 | XsdString = XsdType("xsd:string") 12 | XsdBoolean = XsdType("xsd:boolean") 13 | XsdDateTime = XsdType("xsd:dateTime") 14 | 15 | // 64-bit floating point numbers 16 | XsdDouble = XsdType("xsd:double") 17 | // 32-bit floating point numbers 18 | XsdFloat = XsdType("xsd:float") 19 | 20 | // signed 32 or 64 bit 21 | XsdInteger = XsdType("xsd:integer") 22 | // signed (8 bit) 23 | XsdByte = XsdType("xsd:byte") 24 | // signed (16 bit) 25 | XsdShort = XsdType("xsd:short") 26 | 27 | // unsigned 32 or 64 bit 28 | XsdUinteger = XsdType("xsd:unsignedInt") 29 | // unsigned 8 bit 30 | XsdUnsignedByte = XsdType("xsd:unsignedByte") 31 | // unsigned 16 bit 32 | XsdUnsignedShort = XsdType("xsd:unsignedShort") 33 | ) 34 | 35 | const XMLSchemaNamespace = "http://www.w3.org/2001/XMLSchema" 36 | 37 | func (x XsdType) NTriplesNamespaced() string { 38 | splits := strings.Split(string(x), ":") 39 | if len(splits) != 2 { 40 | return string(x) 41 | } 42 | 43 | return fmt.Sprintf("%s#%s", XMLSchemaNamespace, splits[1]) 44 | } 45 | -------------------------------------------------------------------------------- /last-benches.txt: -------------------------------------------------------------------------------- 1 | goos: linux 2 | goarch: amd64 3 | pkg: github.com/wallix/triplestore 4 | BenchmarkEncodingMemallocation-4 20000 63748 ns/op 32288 B/op 1209 allocs/op 5 | BenchmarkAllEncoding/binary-4 2000 636999 ns/op 303264 B/op 12012 allocs/op 6 | BenchmarkAllEncoding/binary_streaming-4 2000 1149133 ns/op 303268 B/op 12012 allocs/op 7 | BenchmarkAllEncoding/ntriples-4 3000 495783 ns/op 529136 B/op 4014 allocs/op 8 | BenchmarkAllEncoding/ntriples_streaming-4 2000 940655 ns/op 529172 B/op 4015 allocs/op 9 | BenchmarkAllEncoding/ntriples_with_context-4 1000 1212262 ns/op 764839 B/op 8015 allocs/op 10 | BenchmarkAllDecoding/binary-4 3000000 475 ns/op 80 B/op 3 allocs/op 11 | BenchmarkAllDecoding/binary_streaming-4 300000 4129 ns/op 176 B/op 4 allocs/op 12 | BenchmarkAllDecoding/ntriples-4 1000000 1200 ns/op 4112 B/op 2 allocs/op 13 | BenchmarkAllDecoding/ntriples_streaming-4 1000000 2114 ns/op 4212 B/op 3 allocs/op 14 | BenchmarkSnapshotSource-4 1 6622021728 ns/op 1482407016 B/op 22231365 allocs/op 15 | PASS 16 | ok github.com/wallix/triplestore 24.139s 17 | -------------------------------------------------------------------------------- /ntparser_w3c_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | ) 10 | 11 | func TestNTriplesW3CTestSuite(t *testing.T) { 12 | t.Run("positives", func(t *testing.T) { 13 | path := filepath.Join("testdata", "ntriples", "w3c_suite", "positives", "*.nt") 14 | filenames, _ := filepath.Glob(path) 15 | 16 | for _, filename := range filenames { 17 | b, err := ioutil.ReadFile(filename) 18 | if err != nil { 19 | t.Fatalf("cannot read file %s", filename) 20 | } 21 | 22 | tris, err := NewLenientNTDecoder(bytes.NewReader(b)).Decode() 23 | if err != nil { 24 | t.Fatalf("file %s: %s", filename, err) 25 | } 26 | 27 | var buf bytes.Buffer 28 | if err := NewLenientNTEncoder(&buf).Encode(tris...); err != nil { 29 | t.Fatalf("file %s: re-encoding error: %s", filename, err) 30 | } 31 | 32 | expected := cleanupNTriplesForComparison(b) 33 | expectedFilepath := filename + ".expected" 34 | if _, err := os.Stat(expectedFilepath); !os.IsNotExist(err) { 35 | expected, err = ioutil.ReadFile(expectedFilepath) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | } 40 | 41 | if got, want := cleanupNTriplesForComparison(buf.Bytes()), expected; !bytes.Equal(got, want) { 42 | t.Fatalf("file %s: re-encoding mismatch\n\ngot\n%s\n\nwant\n%s\n", filename, got, want) 43 | } 44 | } 45 | }) 46 | 47 | t.Run("negatives", func(t *testing.T) { 48 | path := filepath.Join("testdata", "ntriples", "w3c_suite", "negatives", "*.nt") 49 | filenames, _ := filepath.Glob(path) 50 | 51 | for _, filename := range filenames { 52 | b, err := ioutil.ReadFile(filename) 53 | if err != nil { 54 | t.Fatalf("cannot read file %s", filename) 55 | } 56 | 57 | if _, err := NewLenientNTDecoder(bytes.NewReader(b)).Decode(); err == nil { 58 | t.Fatalf("filename '%s': expected err, got none", filename) 59 | } 60 | } 61 | }) 62 | } 63 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-subm-01.nt.expected: -------------------------------------------------------------------------------- 1 | . 2 | _:anon . 3 | _:anon . 4 | . 5 | . 6 | . 7 | . 8 | "simple literal" . 9 | "backslash:\\" . 10 | "dquote:\"" . 11 | "newline:\n" . 12 | "return\r" . 13 | "tab:\t" . 14 | . 15 | "x" . 16 | _:anon . 17 | "\u00E9" . 18 | "\u20AC" . 19 | ""^^ . 20 | " "^^ . 21 | "x"^^ . 22 | "\""^^ . 23 | ""^^ . 24 | "a "^^ . 25 | "a c"^^ . 26 | "a\n\nc"^^ . 27 | "chat"^^ . 28 | "chat"@fr . 29 | "chat"@en . 30 | "abc"^^ . 31 | -------------------------------------------------------------------------------- /struct.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "reflect" 7 | "time" 8 | ) 9 | 10 | const ( 11 | predTag = "predicate" 12 | bnodeTag = "bnode" 13 | ) 14 | 15 | func init() { 16 | rand.Seed(time.Now().UnixNano()) 17 | } 18 | 19 | // Convert a Struct or ptr to Struct into triples 20 | // using field tags. 21 | // For each struct's field a triple is created: 22 | // - Subject: function first argument 23 | // - Predicate: tag value 24 | // - Literal: actual field value according to field's type 25 | // Unsupported types are ignored 26 | func TriplesFromStruct(sub string, i interface{}, bnodes ...bool) (out []Triple) { 27 | var isBnode bool 28 | if len(bnodes) > 0 { 29 | isBnode = bnodes[0] 30 | } 31 | val := reflect.ValueOf(i) 32 | 33 | var ok bool 34 | val, ok = getStructOrPtrToStruct(val) 35 | if !ok { 36 | return 37 | } 38 | 39 | st := val.Type() 40 | 41 | for i := 0; i < st.NumField(); i++ { 42 | field, fVal := st.Field(i), val.Field(i) 43 | if !fVal.CanInterface() { 44 | continue 45 | } 46 | 47 | intValue := reflect.ValueOf(fVal.Interface()) 48 | if intValue.Kind() == reflect.Ptr && intValue.IsNil() { 49 | continue 50 | } 51 | 52 | pred := field.Tag.Get(predTag) 53 | if tri, ok := buildTripleFromVal(sub, pred, fVal, isBnode); ok { 54 | out = append(out, tri) 55 | } 56 | 57 | bnode, embedded := field.Tag.Lookup(bnodeTag) 58 | fVal, ok := getStructOrPtrToStruct(fVal) 59 | if embedded && ok { 60 | if bnode == "" { 61 | bnode = fmt.Sprintf("%x", rand.Uint32()) 62 | } 63 | tris := TriplesFromStruct(bnode, fVal.Interface(), true) 64 | out = append(out, tris...) 65 | if embedPred, hasPred := field.Tag.Lookup(predTag); hasPred { 66 | out = append(out, SubjPred(sub, embedPred).Bnode(bnode)) 67 | } 68 | continue 69 | } 70 | 71 | switch fVal.Kind() { 72 | case reflect.Slice: 73 | length := fVal.Len() 74 | for i := 0; i < length; i++ { 75 | sliceVal := fVal.Index(i) 76 | if tri, ok := buildTripleFromVal(sub, pred, sliceVal, isBnode); ok { 77 | out = append(out, tri) 78 | } 79 | } 80 | } 81 | 82 | } 83 | 84 | return 85 | } 86 | 87 | func buildTripleFromVal(sub, pred string, v reflect.Value, bnode bool) (Triple, bool) { 88 | if !v.CanInterface() { 89 | return nil, false 90 | } 91 | if pred == "" { 92 | return nil, false 93 | } 94 | objLit, err := ObjectLiteral(v.Interface()) 95 | if err != nil { 96 | return nil, false 97 | } 98 | 99 | if bnode { 100 | return BnodePred(sub, pred).Object(objLit), true 101 | } 102 | return SubjPred(sub, pred).Object(objLit), true 103 | } 104 | 105 | func getStructOrPtrToStruct(v reflect.Value) (reflect.Value, bool) { 106 | switch v.Kind() { 107 | case reflect.Struct: 108 | return v, true 109 | case reflect.Ptr: 110 | if v.Elem().Kind() == reflect.Struct { 111 | return v.Elem(), true 112 | } 113 | } 114 | 115 | return v, false 116 | } 117 | -------------------------------------------------------------------------------- /rdf.go: -------------------------------------------------------------------------------- 1 | // Package triplestore provides APIs to manage, store and query triples, sources and RDFGraphs 2 | package triplestore 3 | 4 | // Triple consists of a subject, a predicate and a object 5 | type Triple interface { 6 | Subject() string 7 | Predicate() string 8 | Object() Object 9 | Equal(Triple) bool 10 | } 11 | 12 | // Object is a resource (i.e. IRI), a literal or a blank node. 13 | type Object interface { 14 | Literal() (Literal, bool) 15 | Resource() (string, bool) 16 | Bnode() (string, bool) 17 | Equal(Object) bool 18 | } 19 | 20 | // Literal is a unicode string associated with a datatype (ex: string, integer, ...). 21 | type Literal interface { 22 | Type() XsdType 23 | Value() string 24 | Lang() string 25 | } 26 | 27 | type triple struct { 28 | sub, pred string 29 | isSubBnode bool 30 | obj object 31 | triKey string 32 | } 33 | 34 | func (t *triple) Object() Object { 35 | return t.obj 36 | } 37 | 38 | func (t *triple) Subject() string { 39 | return t.sub 40 | } 41 | 42 | func (t *triple) Predicate() string { 43 | return t.pred 44 | } 45 | 46 | func (t *triple) key() string { 47 | if t.triKey == "" { 48 | var sub string 49 | if t.isSubBnode { 50 | sub = "_:" + t.sub 51 | } else { 52 | sub = "<" + t.sub + ">" 53 | } 54 | t.triKey = sub + "<" + t.pred + ">" + t.obj.key() 55 | return t.triKey 56 | } 57 | return t.triKey 58 | } 59 | 60 | func (t *triple) clone() *triple { 61 | return &triple{ 62 | sub: t.sub, 63 | pred: t.pred, 64 | obj: t.obj, 65 | triKey: t.triKey, 66 | } 67 | } 68 | 69 | func (t *triple) Equal(other Triple) bool { 70 | switch { 71 | case t == nil: 72 | return other == nil 73 | case other == nil: 74 | return false 75 | default: 76 | otherT, ok := other.(*triple) 77 | if !ok { 78 | return false 79 | } 80 | return t.key() == otherT.key() 81 | } 82 | } 83 | 84 | type object struct { 85 | isLit, isBnode bool 86 | resource, bnode string 87 | lit literal 88 | } 89 | 90 | func (o object) Literal() (Literal, bool) { 91 | return o.lit, o.isLit 92 | } 93 | 94 | func (o object) Resource() (string, bool) { 95 | return o.resource, !o.isLit 96 | } 97 | 98 | func (o object) Bnode() (string, bool) { 99 | return o.bnode, o.isBnode 100 | } 101 | 102 | func (o object) key() string { 103 | if o.isLit { 104 | if o.lit.langtag != "" { 105 | return "\"" + o.lit.val + "\"@" + o.lit.langtag 106 | } 107 | return "\"" + o.lit.val + "\"^^<" + string(o.lit.typ) + ">" 108 | } 109 | if o.isBnode { 110 | return "_:" + o.bnode 111 | } 112 | return "<" + o.resource + ">" 113 | } 114 | 115 | func (o object) Equal(other Object) bool { 116 | lit, ok := o.Literal() 117 | otherLit, otherOk := other.Literal() 118 | if ok != otherOk { 119 | return false 120 | } 121 | if ok { 122 | return lit.Type() == otherLit.Type() && lit.Value() == otherLit.Value() 123 | } 124 | res, ok := o.Resource() 125 | otherRes, otherOk := other.Resource() 126 | if ok != otherOk { 127 | return false 128 | } 129 | if ok { 130 | return res == otherRes 131 | } 132 | return true 133 | } 134 | 135 | type literal struct { 136 | typ XsdType 137 | val, langtag string 138 | } 139 | 140 | func (l literal) Type() XsdType { 141 | return l.typ 142 | } 143 | 144 | func (l literal) Value() string { 145 | return l.val 146 | } 147 | 148 | func (l literal) Lang() string { 149 | return l.langtag 150 | } 151 | -------------------------------------------------------------------------------- /tree.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "fmt" 5 | "sort" 6 | ) 7 | 8 | // A tree is defined from a RDF Graph 9 | // when given a specific predicate as an edge and 10 | // considering triples pointing to RDF resource Object 11 | // 12 | // The tree defined by the graph/predicate should have no cycles 13 | // and node should have at most one parent 14 | type Tree struct { 15 | g RDFGraph 16 | predicate string 17 | } 18 | 19 | func NewTree(g RDFGraph, pred string) *Tree { 20 | if g == nil { 21 | panic("given RDF graph is nil") 22 | } 23 | return &Tree{g: g, predicate: pred} 24 | } 25 | 26 | // Traverse the tree in pre-order depth first search 27 | func (t *Tree) TraverseDFS(node string, each func(RDFGraph, string, int) error, depths ...int) error { 28 | var depth int 29 | if len(depths) > 0 { 30 | depth = depths[0] 31 | } 32 | 33 | if err := each(t.g, node, depth); err != nil { 34 | return err 35 | } 36 | 37 | triples := t.g.WithSubjPred(node, t.predicate) 38 | 39 | var childs []string 40 | for _, tri := range triples { 41 | n, ok := tri.Object().Resource() 42 | if !ok { 43 | return fmt.Errorf("object is not a resource identifier") 44 | } 45 | childs = append(childs, n) 46 | } 47 | 48 | sort.Strings(childs) 49 | 50 | for _, child := range childs { 51 | t.TraverseDFS(child, each, depth+1) 52 | } 53 | 54 | return nil 55 | } 56 | 57 | // Traverse all ancestors from the given node 58 | func (t *Tree) TraverseAncestors(node string, each func(RDFGraph, string, int) error, depths ...int) error { 59 | var depth int 60 | if len(depths) > 0 { 61 | depth = depths[0] 62 | } 63 | 64 | if err := each(t.g, node, depth); err != nil { 65 | return err 66 | } 67 | 68 | triples := t.g.WithPredObj(t.predicate, Resource(node)) 69 | 70 | var parents []string 71 | for _, tri := range triples { 72 | parents = append(parents, tri.Subject()) 73 | } 74 | 75 | sort.Strings(parents) 76 | 77 | for _, parent := range parents { 78 | t.TraverseAncestors(parent, each, depth+1) 79 | } 80 | 81 | return nil 82 | } 83 | 84 | // Traverse siblings of given node. Passed function allow to output the sibling criteria 85 | func (t *Tree) TraverseSiblings(node string, siblingCriteriaFunc func(RDFGraph, string) (string, error), each func(RDFGraph, string, int) error) error { 86 | triples := t.g.WithPredObj(t.predicate, Resource(node)) 87 | 88 | if len(triples) == 0 { 89 | return each(t.g, node, 0) 90 | } 91 | 92 | if len(triples) != 1 { 93 | return fmt.Errorf("tree[%s]: node %s with more than 1 parent: %v", t.predicate, node, triples) 94 | } 95 | 96 | otherChildTriples := t.g.WithSubjPred(triples[0].Subject(), t.predicate) 97 | 98 | var childs []string 99 | for _, c := range otherChildTriples { 100 | child, ok := c.Object().Resource() 101 | if !ok { 102 | return fmt.Errorf("object is not a resource identifier") 103 | } 104 | childs = append(childs, child) 105 | } 106 | 107 | sort.Strings(childs) 108 | 109 | nodeCriteria, err := siblingCriteriaFunc(t.g, node) 110 | if err != nil { 111 | return err 112 | } 113 | 114 | for _, child := range childs { 115 | childCriteria, err := siblingCriteriaFunc(t.g, child) 116 | if err != nil { 117 | return err 118 | } 119 | if nodeCriteria == childCriteria { 120 | if err := each(t.g, child, 0); err != nil { 121 | return err 122 | } 123 | } 124 | } 125 | 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /tree_test.go: -------------------------------------------------------------------------------- 1 | package triplestore_test 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | 8 | tstore "github.com/wallix/triplestore" 9 | ) 10 | 11 | func TestGraphTraverse(t *testing.T) { 12 | s := tstore.NewSource() 13 | s.Add( 14 | tstore.SubjPred("1", "->").Resource("2"), 15 | tstore.SubjPred("2", "->").Resource("3"), 16 | tstore.SubjPred("2", "->").Resource("4"), 17 | tstore.SubjPred("3", "->").Resource("5"), 18 | tstore.SubjPred("3", "->").Resource("6"), 19 | tstore.SubjPred("3", "->").Resource("7"), 20 | tstore.SubjPred("4", "->").Resource("8"), 21 | ) 22 | g := s.Snapshot() 23 | 24 | var result bytes.Buffer 25 | each := func(gph tstore.RDFGraph, subj string, depth int) error { 26 | result.WriteString(fmt.Sprintf("(%d)%s ", depth, subj)) 27 | return nil 28 | } 29 | 30 | tree := tstore.NewTree(g, "->") 31 | 32 | t.Run("depth forst search", func(t *testing.T) { 33 | tree.TraverseDFS("1", each) 34 | if got, want := result.String(), "(0)1 (1)2 (2)3 (3)5 (3)6 (3)7 (2)4 (3)8 "; got != want { 35 | t.Fatalf("got %s, want %s", got, want) 36 | } 37 | 38 | result.Reset() 39 | tree.TraverseDFS("8", each) 40 | if got, want := result.String(), "(0)8 "; got != want { 41 | t.Fatalf("got %s, want %s", got, want) 42 | } 43 | 44 | result.Reset() 45 | tree.TraverseDFS("4", each) 46 | if got, want := result.String(), "(0)4 (1)8 "; got != want { 47 | t.Fatalf("got %s, want %s", got, want) 48 | } 49 | result.Reset() 50 | tree.TraverseDFS("none", each) 51 | if got, want := result.String(), "(0)none "; got != want { 52 | t.Fatalf("got %s, want %s", got, want) 53 | } 54 | }) 55 | 56 | t.Run("ancestors", func(t *testing.T) { 57 | result.Reset() 58 | tree.TraverseAncestors("6", each) 59 | if got, want := result.String(), "(0)6 (1)3 (2)2 (3)1 "; got != want { 60 | t.Fatalf("got %s, want %s", got, want) 61 | } 62 | 63 | result.Reset() 64 | tree.TraverseAncestors("1", each) 65 | if got, want := result.String(), "(0)1 "; got != want { 66 | t.Fatalf("got %s, want %s", got, want) 67 | } 68 | 69 | result.Reset() 70 | tree.TraverseAncestors("none", each) 71 | if got, want := result.String(), "(0)none "; got != want { 72 | t.Fatalf("got %s, want %s", got, want) 73 | } 74 | }) 75 | } 76 | 77 | func TestTraverseSiblings(t *testing.T) { 78 | s := tstore.NewSource() 79 | s.Add( 80 | tstore.SubjPred("1", "->").Resource("2"), 81 | tstore.SubjPred("1", "->").Resource("3"), 82 | tstore.SubjPred("3", "->").Resource("4"), 83 | tstore.SubjPred("3", "->").Resource("5"), 84 | tstore.SubjPred("3", "->").Resource("6"), 85 | tstore.SubjPred("3", "->").Resource("7"), 86 | tstore.SubjPred("3", "->").Resource("8"), 87 | tstore.SubjPred("3", "->").Resource("9"), 88 | tstore.SubjPred("5", "type").StringLiteral("donkey"), 89 | tstore.SubjPred("7", "type").StringLiteral("donkey"), 90 | tstore.SubjPred("9", "type").StringLiteral("donkey"), 91 | ) 92 | g := s.Snapshot() 93 | 94 | var result bytes.Buffer 95 | each := func(gph tstore.RDFGraph, subj string, depth int) error { 96 | result.WriteString(fmt.Sprintf("(%d)%s ", depth, subj)) 97 | return nil 98 | } 99 | 100 | tree := tstore.NewTree(g, "->") 101 | 102 | siblingCriteria := func(g tstore.RDFGraph, node string) (string, error) { 103 | tris := g.WithSubjPred(node, "type") 104 | if len(tris) > 0 { 105 | return tstore.ParseString(tris[0].Object()) 106 | } 107 | return "", nil 108 | } 109 | 110 | tree.TraverseSiblings("5", siblingCriteria, each) 111 | if got, want := result.String(), "(0)5 (0)7 (0)9 "; got != want { 112 | t.Fatalf("got %s, want %s", got, want) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /cmd/triplestore/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "strings" 10 | 11 | tstore "github.com/wallix/triplestore" 12 | ) 13 | 14 | var ( 15 | outFormatFlag, inFormatFlag string 16 | baseFlag string 17 | dotPredicateFlag string 18 | filesFlag arrayFlags 19 | prefixesFlag arrayFlags 20 | useRdfPrefixesFlag bool 21 | ) 22 | 23 | func init() { 24 | flag.StringVar(&outFormatFlag, "out", "ntriples", "output format (ntriples, bin)") 25 | flag.StringVar(&inFormatFlag, "in", "bin", "input format (ntriples, bin)") 26 | flag.Var(&filesFlag, "files", "input file paths") 27 | flag.BoolVar(&useRdfPrefixesFlag, "rdf-prefixes", false, "use default RDF prefixes (rdf, rdfs, xsd)") 28 | flag.Var(&prefixesFlag, "prefix", "RDF custom prefixes (format: \"prefix:http://my.uri\"") 29 | flag.StringVar(&baseFlag, "base", "", "RDF custom base prefix") 30 | flag.StringVar(&dotPredicateFlag, "predicate", "", "Predicate on which to build a dot graph file") 31 | } 32 | 33 | func main() { 34 | flag.Parse() 35 | if len(filesFlag) == 0 { 36 | log.Fatal("need at list an argument `-files INPUT_FILE`") 37 | } 38 | context, err := buildContext(useRdfPrefixesFlag, prefixesFlag, baseFlag) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | if err := convert(filesFlag, outFormatFlag, context); err != nil { 43 | log.Fatal(err) 44 | } 45 | } 46 | 47 | func buildContext(useRdfPrefixes bool, prefixes []string, base string) (*tstore.Context, error) { 48 | var context *tstore.Context 49 | if useRdfPrefixes { 50 | context = tstore.RDFContext 51 | } else { 52 | context = tstore.NewContext() 53 | } 54 | for _, prefix := range prefixes { 55 | splits := strings.SplitN(prefix, ":", 2) 56 | if splits[0] == "" || splits[1] == "" { 57 | return context, fmt.Errorf("invalid prefix format: '%s'. expected \"prefix:http://my.uri\"", prefix) 58 | } 59 | context.Prefixes[splits[0]] = splits[1] 60 | } 61 | context.Base = base 62 | return context, nil 63 | } 64 | 65 | func convert(inFilePaths []string, outFormatFlag string, context *tstore.Context) error { 66 | var inFiles []io.Reader 67 | for _, inFilePath := range inFilePaths { 68 | in, err := os.Open(inFilePath) 69 | if err != nil { 70 | return fmt.Errorf("open input file '%s': %s", inFilePath, err) 71 | } 72 | inFiles = append(inFiles, in) 73 | } 74 | 75 | var inDecoder func(io.Reader) tstore.Decoder 76 | switch inFormatFlag { 77 | case "bin": 78 | inDecoder = tstore.NewBinaryDecoder 79 | case "ntriples": 80 | inDecoder = tstore.NewLenientNTDecoder 81 | default: 82 | return fmt.Errorf("unknown in flag '%s': expect 'ntriples' or 'bin'", outFormatFlag) 83 | } 84 | 85 | triples, err := tstore.NewDatasetDecoder(inDecoder, inFiles...).Decode() 86 | if err != nil { 87 | return err 88 | } 89 | 90 | var encoder tstore.Encoder 91 | switch outFormatFlag { 92 | case "ntriples": 93 | encoder = tstore.NewLenientNTEncoderWithContext(os.Stdout, context) 94 | case "bin": 95 | encoder = tstore.NewBinaryEncoder(os.Stdout) 96 | case "dot": 97 | if dotPredicateFlag == "" { 98 | return fmt.Errorf("missing -predicate param to output to dot format") 99 | } 100 | encoder = tstore.NewDotGraphEncoder(os.Stdout, dotPredicateFlag) 101 | default: 102 | return fmt.Errorf("unknown out flag '%s': expect 'ntriples, 'dot' or 'bin'", outFormatFlag) 103 | } 104 | 105 | if err := encoder.Encode(triples...); err != nil { 106 | return err 107 | } 108 | 109 | return nil 110 | } 111 | 112 | type arrayFlags []string 113 | 114 | func (i *arrayFlags) String() string { 115 | return strings.Join(*i, ",") 116 | } 117 | 118 | func (i *arrayFlags) Set(value string) error { 119 | *i = append(*i, value) 120 | return nil 121 | } 122 | -------------------------------------------------------------------------------- /rdf_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestEquality(t *testing.T) { 8 | emptyTriple := new(triple) 9 | tcases := []struct { 10 | one, other Triple 11 | exp bool 12 | }{ 13 | {one: SubjPred("", "").Resource(""), other: SubjPred("", "").Resource(""), exp: true}, 14 | {one: SubjPred("sub", "pred").Resource("Bonobo"), other: SubjPred("sub", "pred").Resource("Bonobo"), exp: true}, 15 | {one: SubjPred("sub", "pred").Resource("Bonobo"), other: SubjPred("sub", "pred").Resource("Banaba"), exp: false}, 16 | {one: SubjPred("sub", "pred").Resource("Bonobo"), other: SubjPred("sub", "newpred").Resource("Bonobo"), exp: false}, 17 | {one: SubjPred("sub", "pred").Resource("Bonobo"), other: SubjPred("newsub", "pred").Resource("Bonobo"), exp: false}, 18 | 19 | {one: SubjPred("sub", "pred").StringLiteral("Bonobo"), other: SubjPred("sub", "pred").StringLiteral("Bonobo"), exp: true}, 20 | {one: SubjPred("sub", "pred").BooleanLiteral(true), other: SubjPred("sub", "pred").BooleanLiteral(true), exp: true}, 21 | {one: SubjPred("sub", "pred").IntegerLiteral(42), other: SubjPred("sub", "pred").IntegerLiteral(42), exp: true}, 22 | 23 | {one: SubjPred("", "").StringLiteral(""), other: SubjPred("", "").StringLiteral(""), exp: true}, 24 | 25 | {one: SubjPred("sub", "pred").Resource("Bonobo"), other: SubjPred("sub", "pred").StringLiteral("Bonobo"), exp: false}, 26 | {one: SubjPred("sub", "pred").StringLiteral("true"), other: SubjPred("sub", "pred").BooleanLiteral(true), exp: false}, 27 | {one: SubjPred("sub", "pred").StringLiteral("2"), other: SubjPred("sub", "pred").IntegerLiteral(2), exp: false}, 28 | 29 | // langtag 30 | {one: SubjPred("sub", "pred").StringLiteralWithLang("obj", "en"), other: SubjPred("sub", "pred").StringLiteralWithLang("obj", "fr"), exp: false}, 31 | {one: SubjPred("sub", "pred").StringLiteralWithLang("obj", "en"), other: SubjPred("sub", "pred").StringLiteralWithLang("obj", "en"), exp: true}, 32 | 33 | {one: SubjPred("sub", "pred").Resource("Bonobo"), other: emptyTriple, exp: false}, 34 | {one: emptyTriple, other: emptyTriple, exp: true}, 35 | } 36 | for i, tcase := range tcases { 37 | if got, want := tcase.one.Equal(tcase.other), tcase.exp; got != want { 38 | t.Errorf("%d: got %t, want %t", i+1, got, want) 39 | } 40 | if got, want := tcase.other.Equal(tcase.one), tcase.exp; got != want { 41 | t.Errorf("%d: got %t, want %t", i, got, want) 42 | } 43 | } 44 | } 45 | 46 | func TestTripleKey(t *testing.T) { 47 | tcases := []struct { 48 | one *triple 49 | exp string 50 | }{ 51 | {one: SubjPred("", "").Resource(""), exp: "<><><>"}, 52 | {one: SubjPred("", "").StringLiteral(""), exp: "<><>\"\"^^"}, 53 | {one: SubjPred("sub", "pred").Resource("Bonobo"), exp: ""}, 54 | {one: SubjPred("sued").Resource("Bonobo"), exp: "ed>"}, 55 | {one: SubjPred("sub", "pred").StringLiteral("Bonobo"), exp: "\"Bonobo\"^^"}, 56 | {one: SubjPred("sub", "pred").BooleanLiteral(true), exp: "\"true\"^^"}, 57 | {one: SubjPred("sub", "pred").StringLiteral("true"), exp: "\"true\"^^"}, 58 | {one: SubjPred("sub", "pred").IntegerLiteral(42), exp: "\"42\"^^"}, 59 | {one: SubjPred("sub", "pred").StringLiteral("42"), exp: "\"42\"^^"}, 60 | 61 | // bnodes 62 | {one: BnodePred("", "").Resource(""), exp: "_:<><>"}, 63 | {one: BnodePred("", "").StringLiteral(""), exp: "_:<>\"\"^^"}, 64 | {one: BnodePred("sub", "pred").Resource("Bonobo"), exp: "_:sub"}, 65 | 66 | {one: SubjPred("", "").Bnode(""), exp: "<><>_:"}, 67 | {one: SubjPred("", "").Bnode("any"), exp: "<><>_:any"}, 68 | 69 | // langtag 70 | {one: SubjPred("sub", "pred").StringLiteralWithLang("obj", "en"), exp: "\"obj\"@en"}, 71 | } 72 | for i, tcase := range tcases { 73 | if got, want := tcase.one.key(), tcase.exp; got != want { 74 | t.Errorf("%d: got %s, want %s", i+1, got, want) 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /testdata/ntriples/w3c_suite/positives/nt-syntax-subm-01.nt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright World Wide Web Consortium, (Massachusetts Institute of 3 | # Technology, Institut National de Recherche en Informatique et en 4 | # Automatique, Keio University). 5 | # 6 | # All Rights Reserved. 7 | # 8 | # Please see the full Copyright clause at 9 | # 10 | # 11 | # Test file with a variety of legal N-Triples 12 | # 13 | # Dave Beckett - http://purl.org/net/dajobe/ 14 | # 15 | # $Id: test.nt,v 1.7 2003/10/06 15:52:19 dbeckett2 Exp $ 16 | # 17 | ##################################################################### 18 | 19 | # comment lines 20 | # comment line after whitespace 21 | # empty blank line, then one with spaces and tabs 22 | 23 | 24 | . 25 | _:anon . 26 | _:anon . 27 | # spaces and tabs throughout: 28 | . 29 | 30 | # line ending with CR NL (ASCII 13, ASCII 10) 31 | . 32 | 33 | # 2 statement lines separated by single CR (ASCII 10) 34 | . 35 | . 36 | 37 | 38 | # All literal escapes 39 | "simple literal" . 40 | "backslash:\\" . 41 | "dquote:\"" . 42 | "newline:\n" . 43 | "return\r" . 44 | "tab:\t" . 45 | 46 | # Space is optional before final . 47 | . 48 | "x". 49 | _:anon. 50 | 51 | # \u and \U escapes 52 | # latin small letter e with acute symbol \u00E9 - 3 UTF-8 bytes #xC3 #A9 53 | "\u00E9" . 54 | # Euro symbol \u20ac - 3 UTF-8 bytes #xE2 #x82 #xAC 55 | "\u20AC" . 56 | # resource18 test removed 57 | # resource19 test removed 58 | # resource20 test removed 59 | 60 | # XML Literals as Datatyped Literals 61 | ""^^ . 62 | " "^^ . 63 | "x"^^ . 64 | "\""^^ . 65 | ""^^ . 66 | "a "^^ . 67 | "a c"^^ . 68 | "a\n\nc"^^ . 69 | "chat"^^ . 70 | # resource28 test removed 2003-08-03 71 | # resource29 test removed 2003-08-03 72 | 73 | # Plain literals with languages 74 | "chat"@fr . 75 | "chat"@en . 76 | 77 | # Typed Literals 78 | "abc"^^ . 79 | # resource33 test removed 2003-08-03 80 | -------------------------------------------------------------------------------- /codecbench_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | ) 11 | 12 | // BenchmarkEncodingMemallocation-4 20000 71052 ns/op 27488 B/op 1209 allocs/op 13 | func BenchmarkEncodingMemallocation(b *testing.B) { 14 | var triples []Triple 15 | 16 | for i := 0; i < 100; i++ { 17 | triples = append(triples, SubjPred(fmt.Sprint(i), "digit").IntegerLiteral(i)) 18 | } 19 | 20 | b.ResetTimer() 21 | for i := 0; i < b.N; i++ { 22 | var buff bytes.Buffer 23 | err := NewBinaryEncoder(&buff).Encode(triples...) 24 | if err != nil { 25 | b.Fatal(err) 26 | } 27 | } 28 | 29 | } 30 | 31 | //BenchmarkAllEncoding/binary-4 2000 609710 ns/op 295264 B/op 11012 allocs/op 32 | //BenchmarkAllEncoding/binary_streaming-4 2000 1046498 ns/op 295269 B/op 11012 allocs/op 33 | //BenchmarkAllEncoding/ntriples-4 3000 530518 ns/op 529136 B/op 4014 allocs/op 34 | //BenchmarkAllEncoding/ntriples_streaming-4 2000 988511 ns/op 529170 B/op 4015 allocs/op 35 | //BenchmarkAllEncoding/ntriples_with_context-4 1000 1272959 ns/op 764839 B/op 8015 allocs/op 36 | func BenchmarkAllEncoding(b *testing.B) { 37 | var triples []Triple 38 | 39 | for i := 0; i < 1000; i++ { 40 | triples = append(triples, SubjPred(fmt.Sprint(i), "digit").IntegerLiteral(i)) 41 | } 42 | 43 | b.ResetTimer() 44 | 45 | b.Run("binary", func(b *testing.B) { 46 | for i := 0; i < b.N; i++ { 47 | var buff bytes.Buffer 48 | if err := NewBinaryEncoder(&buff).Encode(triples...); err != nil { 49 | b.Fatal(err) 50 | } 51 | } 52 | }) 53 | 54 | b.Run("binary streaming", func(b *testing.B) { 55 | for i := 0; i < b.N; i++ { 56 | b.StopTimer() 57 | triC := make(chan Triple) 58 | go tripleChan(triples, triC) 59 | b.StartTimer() 60 | var buff bytes.Buffer 61 | if err := NewBinaryStreamEncoder(&buff).StreamEncode(context.Background(), triC); err != nil { 62 | b.Fatal(err) 63 | } 64 | } 65 | }) 66 | 67 | b.Run("ntriples", func(b *testing.B) { 68 | for i := 0; i < b.N; i++ { 69 | var buff bytes.Buffer 70 | if err := NewLenientNTEncoder(&buff).Encode(triples...); err != nil { 71 | b.Fatal(err) 72 | } 73 | } 74 | }) 75 | 76 | b.Run("ntriples streaming", func(b *testing.B) { 77 | for i := 0; i < b.N; i++ { 78 | b.StopTimer() 79 | triC := make(chan Triple) 80 | go tripleChan(triples, triC) 81 | b.StartTimer() 82 | var buff bytes.Buffer 83 | if err := NewLenientNTStreamEncoder(&buff).StreamEncode(context.Background(), triC); err != nil { 84 | b.Fatal(err) 85 | } 86 | } 87 | }) 88 | 89 | b.Run("ntriples with context", func(b *testing.B) { 90 | for i := 0; i < b.N; i++ { 91 | var buff bytes.Buffer 92 | if err := NewLenientNTEncoderWithContext(&buff, RDFContext).Encode(triples...); err != nil { 93 | b.Fatal(err) 94 | } 95 | } 96 | }) 97 | } 98 | 99 | //BenchmarkAllDecoding/binary-4 3000000 493 ns/op 72 B/op 3 allocs/op 100 | //BenchmarkAllDecoding/binary_streaming-4 300000 4519 ns/op 168 B/op 4 allocs/op 101 | //BenchmarkAllDecoding/ntriples-4 1000000 1079 ns/op 4112 B/op 2 allocs/op 102 | //BenchmarkAllDecoding/ntriples_streaming-4 1000000 1928 ns/op 4212 B/op 3 allocs/op 103 | func BenchmarkAllDecoding(b *testing.B) { 104 | binaryFile, err := os.Open(filepath.Join("testdata", "bench", "decode_1.bin")) 105 | if err != nil { 106 | b.Fatal(err) 107 | } 108 | defer binaryFile.Close() 109 | 110 | b.ResetTimer() 111 | 112 | b.Run("binary", func(b *testing.B) { 113 | for i := 0; i < b.N; i++ { 114 | if _, err := NewBinaryDecoder(binaryFile).Decode(); err != nil { 115 | b.Fatal(err) 116 | } 117 | } 118 | }) 119 | 120 | b.Run("binary streaming", func(b *testing.B) { 121 | for i := 0; i < b.N; i++ { 122 | results := NewBinaryStreamDecoder(binaryFile).StreamDecode(context.Background()) 123 | for r := range results { 124 | if r.Err != nil { 125 | b.Fatal(r.Err) 126 | } 127 | } 128 | } 129 | }) 130 | 131 | b.Run("ntriples", func(b *testing.B) { 132 | ntFile, err := os.Open(filepath.Join("testdata", "bench", "decode_1.nt")) 133 | if err != nil { 134 | b.Fatal(err) 135 | } 136 | defer ntFile.Close() 137 | b.ResetTimer() 138 | 139 | for i := 0; i < b.N; i++ { 140 | if _, err := NewLenientNTDecoder(ntFile).Decode(); err != nil { 141 | b.Fatal(err) 142 | } 143 | } 144 | }) 145 | 146 | b.Run("ntriples streaming", func(b *testing.B) { 147 | ntFile, err := os.Open(filepath.Join("testdata", "bench", "decode_1.nt")) 148 | if err != nil { 149 | b.Fatal(err) 150 | } 151 | defer ntFile.Close() 152 | b.ResetTimer() 153 | 154 | for i := 0; i < b.N; i++ { 155 | results := NewLenientNTStreamDecoder(ntFile).StreamDecode(context.Background()) 156 | for r := range results { 157 | if r.Err != nil { 158 | b.Fatal(r.Err) 159 | } 160 | } 161 | } 162 | }) 163 | } 164 | 165 | func tripleChan(triples []Triple, triC chan<- Triple) { 166 | for _, t := range triples { 167 | triC <- t 168 | } 169 | close(triC) 170 | } 171 | -------------------------------------------------------------------------------- /source.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "sort" 7 | "strings" 8 | "sync" 9 | "sync/atomic" 10 | ) 11 | 12 | // A source is a persistent yet mutable source or container of triples. 13 | type Source interface { 14 | Add(...Triple) 15 | Remove(...Triple) 16 | Snapshot() RDFGraph 17 | CopyTriples() []Triple 18 | } 19 | 20 | // A RDFGraph is an immutable set of triples. It is a snapshot of a source and it is queryable. 21 | type RDFGraph interface { 22 | Contains(Triple) bool 23 | Triples() []Triple 24 | Count() int 25 | WithSubject(s string) []Triple 26 | WithPredicate(p string) []Triple 27 | WithObject(o Object) []Triple 28 | WithSubjObj(s string, o Object) []Triple 29 | WithSubjPred(s, p string) []Triple 30 | WithPredObj(p string, o Object) []Triple 31 | } 32 | 33 | type Triples []Triple 34 | 35 | func (ts Triples) Equal(others Triples) bool { 36 | if len(ts) != len(others) { 37 | return false 38 | } 39 | 40 | this := make(map[string]struct{}) 41 | for _, tri := range ts { 42 | this[tri.(*triple).key()] = struct{}{} 43 | } 44 | 45 | other := make(map[string]struct{}) 46 | for _, tri := range others { 47 | other[tri.(*triple).key()] = struct{}{} 48 | } 49 | 50 | return reflect.DeepEqual(this, other) 51 | } 52 | 53 | func (ts Triples) Sort() { 54 | sort.Slice(ts, func(i, j int) bool { return ts[i].(*triple).key() > ts[j].(*triple).key() }) 55 | } 56 | 57 | func (ts Triples) Map(fn func(Triple) string) (out []string) { 58 | for _, t := range ts { 59 | out = append(out, fn(t)) 60 | } 61 | return 62 | } 63 | 64 | func (ts Triples) String() string { 65 | joined := strings.Join(ts.Map( 66 | func(t Triple) string { return fmt.Sprint(t) }, 67 | ), "\n") 68 | return fmt.Sprintf("[%s]", joined) 69 | } 70 | 71 | type source struct { 72 | latestSnap atomic.Value 73 | updated uint32 // atomic 74 | mu sync.RWMutex 75 | triples map[string]Triple 76 | } 77 | 78 | // A source is a persistent yet mutable source or container of triples 79 | func NewSource() Source { 80 | s := &source{ 81 | triples: make(map[string]Triple), 82 | } 83 | s.latestSnap.Store(newGraph(0)) 84 | return s 85 | } 86 | 87 | func (s *source) isUpdated() bool { 88 | return atomic.LoadUint32(&s.updated) > 0 89 | } 90 | 91 | func (s *source) update() { 92 | atomic.StoreUint32(&s.updated, uint32(1)) 93 | } 94 | 95 | func (s *source) reset() { 96 | atomic.StoreUint32(&s.updated, uint32(0)) 97 | } 98 | 99 | func (s *source) Add(ts ...Triple) { 100 | s.mu.Lock() 101 | defer s.mu.Unlock() 102 | defer s.update() 103 | 104 | for _, t := range ts { 105 | tr := t.(*triple) 106 | s.triples[tr.key()] = t 107 | } 108 | } 109 | 110 | func (s *source) Remove(ts ...Triple) { 111 | s.mu.Lock() 112 | defer s.mu.Unlock() 113 | defer s.update() 114 | 115 | for _, t := range ts { 116 | tr := t.(*triple) 117 | delete(s.triples, tr.key()) 118 | } 119 | } 120 | 121 | func (s *source) CopyTriples() (out []Triple) { 122 | s.mu.RLock() 123 | defer s.mu.RUnlock() 124 | for _, t := range s.triples { 125 | out = append(out, t.(*triple).clone()) 126 | } 127 | return 128 | } 129 | 130 | func (s *source) Snapshot() RDFGraph { 131 | if !s.isUpdated() { 132 | return s.latestSnap.Load().(RDFGraph) 133 | } 134 | 135 | s.mu.RLock() 136 | defer s.mu.RUnlock() 137 | 138 | gph := newGraph(len(s.triples)) 139 | 140 | for k, t := range s.triples { 141 | objKey := t.Object().(object).key() 142 | sub, pred := t.Subject(), t.Predicate() 143 | 144 | gph.s[sub] = append(gph.s[sub], t) 145 | gph.p[pred] = append(gph.p[pred], t) 146 | gph.o[objKey] = append(gph.o[objKey], t) 147 | 148 | sp := sub + pred 149 | gph.sp[sp] = append(gph.sp[sp], t) 150 | 151 | so := sub + objKey 152 | gph.so[so] = append(gph.so[so], t) 153 | 154 | po := pred + objKey 155 | gph.po[po] = append(gph.po[po], t) 156 | 157 | gph.spo[k] = t 158 | } 159 | 160 | s.latestSnap.Store(gph) 161 | s.reset() 162 | 163 | return gph 164 | } 165 | 166 | type graph struct { 167 | once sync.Once 168 | unique []Triple 169 | s, p, o map[string][]Triple 170 | sp, so, po map[string][]Triple 171 | spo map[string]Triple 172 | } 173 | 174 | func newGraph(cap int) *graph { 175 | return &graph{ 176 | s: make(map[string][]Triple, cap), 177 | p: make(map[string][]Triple, cap), 178 | o: make(map[string][]Triple, cap), 179 | sp: make(map[string][]Triple, cap), 180 | so: make(map[string][]Triple, cap), 181 | po: make(map[string][]Triple, cap), 182 | spo: make(map[string]Triple, cap), 183 | } 184 | } 185 | 186 | func (g *graph) Contains(t Triple) bool { 187 | _, ok := g.spo[t.(*triple).key()] 188 | return ok 189 | } 190 | func (g *graph) Triples() []Triple { 191 | g.once.Do(func() { 192 | for _, t := range g.spo { 193 | g.unique = append(g.unique, t) 194 | } 195 | }) 196 | return g.unique 197 | } 198 | 199 | func (g *graph) Count() int { 200 | return len(g.spo) 201 | } 202 | 203 | func (g *graph) WithSubject(s string) []Triple { 204 | return g.s[s] 205 | } 206 | func (g *graph) WithPredicate(p string) []Triple { 207 | return g.p[p] 208 | } 209 | func (g *graph) WithObject(o Object) []Triple { 210 | return g.o[o.(object).key()] 211 | } 212 | func (g *graph) WithSubjObj(s string, o Object) []Triple { 213 | return g.so[s+o.(object).key()] 214 | } 215 | func (g *graph) WithSubjPred(s, p string) []Triple { 216 | return g.sp[s+p] 217 | } 218 | func (g *graph) WithPredObj(p string, o Object) []Triple { 219 | return g.po[p+o.(object).key()] 220 | } 221 | -------------------------------------------------------------------------------- /struct_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "net" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | type TestStruct struct { 10 | Name string `predicate:"name"` 11 | Age int `predicate:"age"` 12 | Size int64 `predicate:"size"` 13 | Male bool `predicate:"male"` 14 | Birth time.Time `predicate:"birth"` 15 | Surnames []string `predicate:"surnames"` 16 | Counts []int `predicate:"counts"` 17 | 18 | // special cases that should be ignored 19 | NoTag string 20 | Unsupported complex64 `predicate:"complex"` 21 | Pointer *string `predicate:"ptr"` 22 | Slice []complex64 `predicate:"complexes"` 23 | PtrSlice []*string `predicate:"strptr"` 24 | unexported string 25 | } 26 | 27 | type MainStruct struct { 28 | Name string `predicate:"name"` 29 | Age int `predicate:"age"` 30 | E Embedded `predicate:"embedded" bnode:""` 31 | } 32 | 33 | type OtherStruct struct { 34 | Name string `predicate:"name"` 35 | Age int `predicate:"age"` 36 | E Embedded `predicate:"embedded" bnode:"dimension"` 37 | } 38 | 39 | type Embedded struct { 40 | Size int64 `predicate:"size"` 41 | Male bool `predicate:"male"` 42 | } 43 | 44 | func TestEmbeddedStructToTriple(t *testing.T) { 45 | t.Run("name bnode", func(t *testing.T) { 46 | e := Embedded{Size: 186, Male: true} 47 | s := OtherStruct{Name: "donald", Age: 32, E: e} 48 | 49 | tris := TriplesFromStruct("me", s) 50 | src := NewSource() 51 | src.Add(tris...) 52 | snap := src.Snapshot() 53 | 54 | if got, want := snap.Count(), 5; got != want { 55 | t.Fatalf("got %d, want %d", got, want) 56 | } 57 | 58 | all := snap.WithSubjPred("me", "embedded") 59 | if got, want := len(all), 1; got != want { 60 | t.Fatalf("got %d, want %d", got, want) 61 | } 62 | all = snap.WithPredObj("size", IntegerLiteral(186)) 63 | if got, want := len(all), 1; got != want { 64 | t.Fatalf("got %d, want %d", got, want) 65 | } 66 | all = snap.WithPredObj("male", BooleanLiteral(true)) 67 | if got, want := len(all), 1; got != want { 68 | t.Fatalf("got %d, want %d", got, want) 69 | } 70 | if tri := SubjPred("me", "embedded").Bnode("dimension"); !snap.Contains(tri) { 71 | t.Fatalf("snap should contains %v", tri) 72 | } 73 | if tri := BnodePred("dimension", "male").BooleanLiteral(true); !snap.Contains(tri) { 74 | t.Fatalf("snap should contains %v", tri) 75 | } 76 | if tri := BnodePred("dimension", "size").IntegerLiteral(186); !snap.Contains(tri) { 77 | t.Fatalf("snap should contains %v", tri) 78 | } 79 | }) 80 | 81 | t.Run("random bnode", func(t *testing.T) { 82 | e := Embedded{Size: 186, Male: true} 83 | s := MainStruct{Name: "donald", Age: 32, E: e} 84 | 85 | tris := TriplesFromStruct("me", s) 86 | src := NewSource() 87 | src.Add(tris...) 88 | snap := src.Snapshot() 89 | 90 | if got, want := snap.Count(), 5; got != want { 91 | t.Fatalf("got %d, want %d", got, want) 92 | } 93 | 94 | all := snap.WithSubjPred("me", "embedded") 95 | if got, want := len(all), 1; got != want { 96 | t.Fatalf("got %d, want %d", got, want) 97 | } 98 | all = snap.WithPredObj("size", IntegerLiteral(186)) 99 | if got, want := len(all), 1; got != want { 100 | t.Fatalf("got %d, want %d", got, want) 101 | } 102 | all = snap.WithPredObj("male", BooleanLiteral(true)) 103 | if got, want := len(all), 1; got != want { 104 | t.Fatalf("got %d, want %d", got, want) 105 | } 106 | }) 107 | } 108 | 109 | func TestSimpleStructToTriple(t *testing.T) { 110 | now := time.Now() 111 | s := TestStruct{ 112 | Name: "donald", Age: 32, Size: 186, 113 | Male: true, Birth: now, 114 | Surnames: []string{"one", "two", "three"}, 115 | Counts: []int{1, 2, 3}, 116 | } 117 | 118 | exp := []Triple{ 119 | SubjPred("me", "name").StringLiteral("donald"), 120 | SubjPred("me", "age").IntegerLiteral(32), 121 | SubjPred("me", "size").IntegerLiteral(186), 122 | SubjPred("me", "male").BooleanLiteral(true), 123 | SubjPred("me", "birth").DateTimeLiteral(now), 124 | SubjPred("me", "surnames").StringLiteral("one"), 125 | SubjPred("me", "surnames").StringLiteral("two"), 126 | SubjPred("me", "surnames").StringLiteral("three"), 127 | SubjPred("me", "counts").IntegerLiteral(1), 128 | SubjPred("me", "counts").IntegerLiteral(2), 129 | SubjPred("me", "counts").IntegerLiteral(3), 130 | } 131 | 132 | tris := TriplesFromStruct("me", s) 133 | if got, want := Triples(tris), Triples(exp); !got.Equal(want) { 134 | t.Fatalf("got %s\n\n want %s", got, want) 135 | } 136 | 137 | tris = TriplesFromStruct("me", &s) 138 | if got, want := Triples(tris), Triples(exp); !got.Equal(want) { 139 | t.Fatalf("got %s\n\n want %s", got, want) 140 | } 141 | } 142 | 143 | func TestReturnEmptyTriplesOnNonStructElem(t *testing.T) { 144 | var ptr *string 145 | var strPtr *stringer 146 | var ipnet *net.IPNet 147 | tcases := []struct { 148 | Val interface{} `predicate:"anything"` 149 | }{ 150 | {true}, {"any"}, {ptr}, {strPtr}, {ipnet}, 151 | } 152 | 153 | for i, tc := range tcases { 154 | tris := TriplesFromStruct("", tc.Val) 155 | if len(tris) != 0 { 156 | t.Fatalf("case %d: expected no triples", i+1) 157 | } 158 | } 159 | } 160 | 161 | func TestReturnEmptyTriplesOnVoidPointers(t *testing.T) { 162 | type anyStruct struct { 163 | Val interface{} `predicate:"anything"` 164 | } 165 | var ptr *string 166 | var strPtr *stringer 167 | var ipnet *net.IPNet 168 | tcases := []struct { 169 | st anyStruct 170 | }{ 171 | {anyStruct{ptr}}, {anyStruct{strPtr}}, {anyStruct{ipnet}}, 172 | } 173 | 174 | for i, tc := range tcases { 175 | tris := TriplesFromStruct("", tc.st) 176 | if len(tris) != 0 { 177 | t.Fatalf("case %d: expected no triples", i+1) 178 | } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /source_test.go: -------------------------------------------------------------------------------- 1 | package triplestore_test 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "testing" 7 | 8 | tstore "github.com/wallix/triplestore" 9 | ) 10 | 11 | func TestCopyAndCloneTriples(t *testing.T) { 12 | s := tstore.NewSource() 13 | all := []tstore.Triple{ 14 | tstore.SubjPred("one", "two").StringLiteral("three"), 15 | tstore.SubjPred("four", "two").IntegerLiteral(42), 16 | tstore.SubjPred("one", "two").Resource("four"), 17 | } 18 | s.Add(all...) 19 | 20 | copied := tstore.Triples(s.CopyTriples()) 21 | if got, want := len(copied), 3; got != want { 22 | t.Fatalf("got %d, want %d", got, want) 23 | } 24 | 25 | copied.Sort() 26 | 27 | // Full verification of first copy 28 | if got, want := copied[1].Subject(), "one"; got != want { 29 | t.Fatalf("got %s, want %s", got, want) 30 | } 31 | if got, want := copied[1].Predicate(), "two"; got != want { 32 | t.Fatalf("got %s, want %s", got, want) 33 | } 34 | if got, want := copied[1].Object(), tstore.StringLiteral("three"); got != want { 35 | t.Fatalf("got %v, want %v", got, want) 36 | } 37 | 38 | snap := s.Snapshot() 39 | for _, c := range copied { 40 | if !snap.Contains(c) { 41 | t.Fatalf("should contains triple %v", c) 42 | } 43 | } 44 | } 45 | 46 | func TestQueries(t *testing.T) { 47 | all := []tstore.Triple{ 48 | tstore.SubjPred("one", "two").StringLiteral("three"), 49 | tstore.SubjPred("four", "two").IntegerLiteral(42), 50 | tstore.SubjPred("one", "two").Resource("four"), 51 | } 52 | 53 | s := tstore.NewSource() 54 | s.Add(all...) 55 | 56 | g := s.Snapshot() 57 | 58 | if got, want := g.Count(), len(all); got != want { 59 | t.Fatalf("got %d, want %d", got, want) 60 | } 61 | if got, want := tstore.Triples(g.Triples()), tstore.Triples(all); !got.Equal(want) { 62 | t.Fatalf("got %v, want %v", got, want) 63 | } 64 | 65 | exp := tstore.Triples{all[0], all[2]} 66 | if got, want := tstore.Triples(g.WithSubject("one")), exp; !got.Equal(want) { 67 | t.Fatalf("got %v, want %v", got, want) 68 | } 69 | 70 | exp = tstore.Triples{all[0], all[1], all[2]} 71 | if got, want := tstore.Triples(g.WithPredicate("two")), exp; !got.Equal(want) { 72 | t.Fatalf("got %v, want %v", got, want) 73 | } 74 | 75 | exp = tstore.Triples{all[1]} 76 | if got, want := tstore.Triples(g.WithObject(tstore.IntegerLiteral(42))), exp; !got.Equal(want) { 77 | t.Fatalf("got %v, want %v", got, want) 78 | } 79 | 80 | exp = tstore.Triples{all[2]} 81 | if got, want := tstore.Triples(g.WithSubjObj("one", tstore.Resource("four"))), exp; !got.Equal(want) { 82 | t.Fatalf("got %v, want %v", got, want) 83 | } 84 | 85 | exp = tstore.Triples{all[0], all[2]} 86 | if got, want := tstore.Triples(g.WithSubjPred("one", "two")), exp; !got.Equal(want) { 87 | t.Fatalf("got %v, want %v", got, want) 88 | } 89 | 90 | exp = tstore.Triples{all[0]} 91 | if got, want := tstore.Triples(g.WithPredObj("two", tstore.StringLiteral("three"))), exp; !got.Equal(want) { 92 | t.Fatalf("got %v, want %v", got, want) 93 | } 94 | } 95 | 96 | func TestSource(t *testing.T) { 97 | s := tstore.NewSource() 98 | s.Add( 99 | tstore.SubjPred("one", "two").StringLiteral("three"), 100 | tstore.SubjPred("one", "two").Resource("four"), 101 | tstore.SubjPred("four", "two").IntegerLiteral(42), 102 | tstore.SubjPred("one", "two").Resource("four"), 103 | ) 104 | g := s.Snapshot() 105 | expected := []tstore.Triple{ 106 | tstore.SubjPred("one", "two").StringLiteral("three"), 107 | tstore.SubjPred("one", "two").Resource("four"), 108 | tstore.SubjPred("four", "two").IntegerLiteral(42), 109 | } 110 | if got, want := g.Count(), len(expected); got != want { 111 | t.Fatalf("got %d, want %d", got, want) 112 | } 113 | for _, tr := range expected { 114 | if got, want := g.Contains(tr), true; got != want { 115 | t.Fatalf("%v: got %t, want %t", tr, got, want) 116 | } 117 | } 118 | s.Remove(tstore.SubjPred("one", "two").Resource("four")) 119 | newG := s.Snapshot() 120 | 121 | t.Run("old snapshot unmodified", func(t *testing.T) { 122 | if got, want := g.Count(), len(expected); got != want { 123 | t.Fatalf("got %d, want %d", got, want) 124 | } 125 | for _, tr := range expected { 126 | if got, want := g.Contains(tr), true; got != want { 127 | t.Fatalf("%v: got %t, want %t", tr, got, want) 128 | } 129 | } 130 | }) 131 | 132 | t.Run("triple 1 removed in new snapshot", func(t *testing.T) { 133 | if got, want := newG.Count(), 2; got != want { 134 | t.Fatalf("got %d, want %d", got, want) 135 | } 136 | if got, want := newG.Contains(expected[0]), true; got != want { 137 | t.Fatalf("%v: got %t, want %t", expected[0], got, want) 138 | } 139 | if got, want := newG.Contains(expected[1]), false; got != want { 140 | t.Fatalf("%v: got %t, want %t", expected[1], got, want) 141 | } 142 | if got, want := newG.Contains(expected[2]), true; got != want { 143 | t.Fatalf("%v: got %t, want %t", expected[2], got, want) 144 | } 145 | }) 146 | 147 | } 148 | 149 | func TestStoreConcurrentAccess(t *testing.T) { 150 | s := tstore.NewSource() 151 | any := tstore.SubjPred("any", "any").StringLiteral("any") 152 | 153 | var wg sync.WaitGroup 154 | wg.Add(1) 155 | go func() { 156 | defer wg.Done() 157 | for i := 0; i < 50; i++ { 158 | s.Add(any) 159 | } 160 | }() 161 | 162 | wg.Add(1) 163 | go func() { 164 | defer wg.Done() 165 | for i := 0; i < 50; i++ { 166 | s.Add(any) 167 | s.Snapshot() 168 | } 169 | }() 170 | 171 | wg.Add(1) 172 | go func() { 173 | defer wg.Done() 174 | for i := 0; i < 50; i++ { 175 | s.Snapshot() 176 | } 177 | }() 178 | 179 | wg.Wait() 180 | } 181 | 182 | // BenchmarkSnapshotSource-4 1 7462513791 ns/op 183 | func BenchmarkSnapshotSource(b *testing.B) { 184 | s := tstore.NewSource() 185 | for i := 0; i < 100000; i++ { 186 | num := fmt.Sprint(i) 187 | tri := tstore.SubjPred(num, num).IntegerLiteral(i) 188 | s.Add(tri) 189 | } 190 | 191 | b.ResetTimer() 192 | 193 | for i := 0; i < b.N; i++ { 194 | for i := 0; i < 10; i++ { 195 | num := fmt.Sprint(i) 196 | tri := tstore.SubjPred(num, num).IntegerLiteral(i) 197 | s.Add(tri) 198 | s.Snapshot() 199 | s.Snapshot() 200 | s.Snapshot() 201 | s.Remove(tri) 202 | s.Snapshot() 203 | s.Snapshot() 204 | s.Snapshot() 205 | } 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /testdata/sample.nt: -------------------------------------------------------------------------------- 1 | . 2 | "0000000114443708" . 3 | . 4 | . 5 | "2013-07-31" . 6 | "Psychologue. - Professeur, Institute of child development, University of Minnesota, Minneapolis, Minn. (en 2002)" . 7 | . 8 | . 9 | "1999-03-03" . 10 | . 11 | . 12 | "Conflict in child and adolescent development / ed. by Carolyn Uhlinger Shantz, Willard W. Hartup, 1992. - . - LC Authorities. - http://authorities.loc.gov. - 2008-10-06"@fr . 13 | . 14 | . 15 | "Hartup" . 16 | . 17 | . 18 | . 19 | . 20 | . 21 | . 22 | "male" . 23 | . 24 | "13491375"^^ . 25 | . 26 | "Willard W. Hartup"@fr . 27 | . 28 | . 29 | . 30 | . 31 | . 32 | "Psychologue. - Professeur, Institute of child development, University of Minnesota, Minneapolis, Minn. (en 2002)"@fr . 33 | . 34 | . 35 | . 36 | "Willard W." . 37 | . 38 | "1927"^^ . 39 | . 40 | . 41 | . 42 | . 43 | "Willard W. Hartup" . 44 | . 45 | 46 | -------------------------------------------------------------------------------- /streamcodec_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "io/ioutil" 8 | "sync" 9 | "testing" 10 | ) 11 | 12 | func TestStreamBinaryDecoding(t *testing.T) { 13 | var tris []Triple 14 | for i := 0; i < 10; i++ { 15 | tris = append(tris, SubjPred(fmt.Sprint(i), "digit").IntegerLiteral(i)) 16 | } 17 | 18 | t.Run("handles done signal", func(t *testing.T) { 19 | var buf bytes.Buffer 20 | ctx, cancel := context.WithCancel(context.Background()) // will stop the decoding 21 | dec := NewBinaryStreamDecoder(ioutil.NopCloser(&buf)) 22 | results := dec.StreamDecode(ctx) 23 | 24 | var wg sync.WaitGroup 25 | wg.Add(1) 26 | go func() { 27 | defer wg.Done() 28 | for r := range results { 29 | if r.Err != nil { 30 | t.Fatal(r.Err) 31 | } 32 | } 33 | }() 34 | cancel() 35 | wg.Wait() 36 | }) 37 | 38 | t.Run("handles normal stream", func(t *testing.T) { 39 | var buf bytes.Buffer 40 | if err := NewBinaryEncoder(&buf).Encode(tris...); err != nil { 41 | t.Fatal(err) 42 | } 43 | 44 | dec := NewBinaryStreamDecoder(ioutil.NopCloser(&buf)) 45 | results := dec.StreamDecode(context.Background()) 46 | 47 | var all []Triple 48 | 49 | for r := range results { 50 | if r.Err != nil { 51 | t.Fatal(r.Err) 52 | } 53 | all = append(all, r.Tri) 54 | } 55 | 56 | if got, want := len(all), 10; got != want { 57 | t.Fatalf("got %d, want %d", got, want) 58 | } 59 | s := NewSource() 60 | s.Add(all...) 61 | snap := s.Snapshot() 62 | 63 | for _, tri := range tris { 64 | if !snap.Contains(tri) { 65 | t.Fatalf("end result should contains triple %v", tri) 66 | } 67 | } 68 | }) 69 | } 70 | 71 | func TestStreamNTriplesDecoding(t *testing.T) { 72 | var tris []Triple 73 | for i := 0; i < 10; i++ { 74 | tris = append(tris, SubjPred(fmt.Sprint(i), "digit").IntegerLiteral(i)) 75 | } 76 | 77 | t.Run("handles done signal", func(t *testing.T) { 78 | var buf bytes.Buffer 79 | ctx, cancel := context.WithCancel(context.Background()) // will stop the decoding 80 | dec := NewLenientNTStreamDecoder(ioutil.NopCloser(&buf)) 81 | results := dec.StreamDecode(ctx) 82 | 83 | var wg sync.WaitGroup 84 | wg.Add(1) 85 | go func() { 86 | defer wg.Done() 87 | for r := range results { 88 | if r.Err != nil { 89 | t.Fatal(r.Err) 90 | } 91 | } 92 | }() 93 | cancel() 94 | wg.Wait() 95 | }) 96 | 97 | t.Run("handles normal stream", func(t *testing.T) { 98 | var buf bytes.Buffer 99 | if err := NewLenientNTEncoder(&buf).Encode(tris...); err != nil { 100 | t.Fatal(err) 101 | } 102 | 103 | dec := NewLenientNTStreamDecoder(ioutil.NopCloser(&buf)) 104 | results := dec.StreamDecode(context.Background()) 105 | 106 | var all []Triple 107 | 108 | for r := range results { 109 | if r.Err != nil { 110 | t.Fatal(r.Err) 111 | } 112 | all = append(all, r.Tri) 113 | } 114 | 115 | if got, want := len(all), 10; got != want { 116 | t.Fatalf("got %d, want %d", got, want) 117 | } 118 | s := NewSource() 119 | s.Add(all...) 120 | snap := s.Snapshot() 121 | 122 | for _, tri := range tris { 123 | if !snap.Contains(tri) { 124 | t.Fatalf("end result should contains triple %v", tri) 125 | } 126 | } 127 | }) 128 | } 129 | 130 | func TestStreamBinaryEncoding(t *testing.T) { 131 | t.Run("handles nil stream", func(t *testing.T) { 132 | enc := NewBinaryStreamEncoder(bytes.NewBuffer(nil)) 133 | if err := enc.StreamEncode(context.Background(), nil); err != nil { 134 | t.Fatal(err) 135 | } 136 | }) 137 | 138 | t.Run("handles done stream", func(t *testing.T) { 139 | c := make(chan Triple) // will make encoder block 140 | ctx, cancel := context.WithCancel(context.Background()) // will propagate encoding as done 141 | enc := NewBinaryStreamEncoder(bytes.NewBuffer(nil)) 142 | 143 | var wg sync.WaitGroup 144 | wg.Add(1) 145 | go func() { 146 | defer wg.Done() 147 | if err := enc.StreamEncode(ctx, c); err != nil { 148 | t.Fatal(err) 149 | } 150 | }() 151 | cancel() 152 | wg.Wait() 153 | }) 154 | 155 | var tris []Triple 156 | for i := 0; i < 10; i++ { 157 | tris = append(tris, SubjPred(fmt.Sprint(i), "digit").IntegerLiteral(i)) 158 | } 159 | 160 | t.Run("handles normal stream", func(t *testing.T) { 161 | triC := make(chan Triple) 162 | go func() { 163 | for _, tri := range tris { 164 | triC <- tri 165 | } 166 | close(triC) 167 | }() 168 | 169 | var buf bytes.Buffer 170 | 171 | err := NewBinaryStreamEncoder(&buf).StreamEncode(context.Background(), triC) 172 | if err != nil { 173 | t.Fatal(err) 174 | } 175 | 176 | out, err := NewBinaryDecoder(&buf).Decode() 177 | if err != nil { 178 | t.Fatal(err) 179 | } 180 | if got, want := len(out), 10; got != want { 181 | t.Fatalf("got %d, want %d", got, want) 182 | } 183 | s := NewSource() 184 | s.Add(out...) 185 | snap := s.Snapshot() 186 | 187 | for _, tri := range tris { 188 | if !snap.Contains(tri) { 189 | t.Fatalf("end result should contains triple %v", tri) 190 | } 191 | } 192 | }) 193 | } 194 | 195 | func TestStreamNTriplesEncoding(t *testing.T) { 196 | t.Run("handles nil stream", func(t *testing.T) { 197 | enc := NewLenientNTStreamEncoder(bytes.NewBuffer(nil)) 198 | if err := enc.StreamEncode(context.Background(), nil); err != nil { 199 | t.Fatal(err) 200 | } 201 | }) 202 | 203 | var tris []Triple 204 | for i := 0; i < 10; i++ { 205 | tris = append(tris, SubjPred(fmt.Sprint(i), "digit").IntegerLiteral(i)) 206 | } 207 | 208 | t.Run("handles normal stream", func(t *testing.T) { 209 | triC := make(chan Triple) 210 | go func() { 211 | for _, tri := range tris { 212 | triC <- tri 213 | } 214 | close(triC) 215 | }() 216 | 217 | var buf bytes.Buffer 218 | 219 | err := NewLenientNTStreamEncoder(&buf).StreamEncode(context.Background(), triC) 220 | if err != nil { 221 | t.Fatal(err) 222 | } 223 | 224 | out, err := NewLenientNTDecoder(&buf).Decode() 225 | if err != nil { 226 | t.Fatal(err) 227 | } 228 | if got, want := len(out), 10; got != want { 229 | t.Fatalf("got %d, want %d", got, want) 230 | } 231 | s := NewSource() 232 | s.Add(out...) 233 | snap := s.Snapshot() 234 | 235 | for _, tri := range tris { 236 | if !snap.Contains(tri) { 237 | t.Fatalf("end result should contains triple %v", tri) 238 | } 239 | } 240 | }) 241 | } 242 | -------------------------------------------------------------------------------- /ntparser.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "unicode/utf8" 10 | ) 11 | 12 | type lenientNTParser struct { 13 | r io.Reader 14 | } 15 | 16 | func newLenientNTParser(r io.Reader) *lenientNTParser { 17 | return &lenientNTParser{r: r} 18 | } 19 | 20 | func (p *lenientNTParser) Parse() (out []Triple, err error) { 21 | var count int 22 | scanner := bufio.NewScanner(p.r) 23 | for scanner.Scan() { 24 | count++ 25 | line := bytes.TrimLeft(scanner.Bytes(), " \t") 26 | if len(line) < 1 { 27 | continue 28 | } 29 | if line[0] == '#' { 30 | continue 31 | } 32 | t, terr := parseTriple(line) 33 | if terr != nil { 34 | return out, fmt.Errorf("lenient parsing: line %d: %s", count, terr) 35 | } 36 | out = append(out, t) 37 | } 38 | 39 | err = scanner.Err() 40 | return 41 | } 42 | 43 | func parseTriple(b []byte) (Triple, error) { 44 | tBuilder := new(tripleBuilder) 45 | var err error 46 | if bytes.HasPrefix(b, []byte("_:")) { 47 | if tBuilder.sub, b, err = parseBNodeSubject(b[2:]); err != nil { 48 | return nil, err 49 | } 50 | tBuilder.isSubBnode = true 51 | } else if bytes.HasPrefix(b, []byte("<")) { 52 | if tBuilder.sub, b, err = parseIRISubject(b[1:]); err != nil { 53 | return nil, err 54 | } 55 | } else { 56 | return nil, fmt.Errorf("invalid subject in %s", b) 57 | } 58 | 59 | if bytes.HasPrefix(b, []byte{'<'}) { 60 | if tBuilder.pred, b, err = parsePredicate(b[1:]); err != nil { 61 | return nil, err 62 | } 63 | } else { 64 | return nil, fmt.Errorf("invalid predicate in %s", b) 65 | } 66 | 67 | if bytes.HasPrefix(b, []byte{'<'}) { 68 | obj, _, err := parseIRIObject(b[1:]) 69 | return tBuilder.Resource(obj), err 70 | } else if bytes.HasPrefix(b, []byte("_:")) { 71 | obj, _, err := parseBNodeObject(b[2:]) 72 | return tBuilder.Bnode(obj), err 73 | } else if bytes.HasPrefix(b, []byte{'"'}) { 74 | lit, b, err := parseLiteralObject(b[1:]) 75 | if err != nil { 76 | return nil, err 77 | } 78 | if bytes.HasPrefix(b, []byte("^^<")) { 79 | dtype, _, err := parseIRIObject(b[3:]) 80 | obj := object{ 81 | isLit: true, 82 | lit: literal{ 83 | typ: XsdType(dtype), 84 | val: lit, 85 | }, 86 | } 87 | return tBuilder.Object(obj), err 88 | } else if bytes.HasPrefix(b, []byte{'@'}) { 89 | lang, _, err := parseLangtag(b[1:]) 90 | return tBuilder.StringLiteralWithLang(unescapeStringLiteral(lit), lang), err 91 | } else { 92 | return tBuilder.StringLiteral(unescapeStringLiteral(lit)), err 93 | } 94 | } else { 95 | return nil, errors.New("invalid object") 96 | } 97 | } 98 | 99 | func parseLangtag(b []byte) (string, []byte, error) { 100 | var index int 101 | for { 102 | r, size, err, eol := decode(b[index:]) 103 | if err != nil { 104 | return "", nil, err 105 | } 106 | if eol { 107 | return "", nil, errors.New("invalid language tag") 108 | } 109 | index += size 110 | 111 | if r == '.' { 112 | if found, advance := peekNext(b[index:]); found == '#' || found == 0 { 113 | return string(b[:index-1]), b[index-1+advance:], nil 114 | } 115 | } 116 | 117 | if r == ' ' { 118 | if found, advance := peekNext(b[index:]); found == '.' { 119 | return string(b[:index-1]), b[index+advance:], nil 120 | } 121 | } 122 | } 123 | } 124 | 125 | func parseLiteralObject(b []byte) (string, []byte, error) { 126 | var index int 127 | for { 128 | r, size, err, eol := decode(b[index:]) 129 | if err != nil { 130 | return "", nil, err 131 | } 132 | if eol { 133 | return "", nil, errors.New("invalid literal object") 134 | } 135 | index += size 136 | 137 | if r == '"' { 138 | if found, advance, other := doublePeekNext(b[index:]); (found == '.' && other == '#') || (found == '.' && other == 0) || (found == '^' && other == '^') || found == '@' { 139 | return string(b[:index-1]), b[index+advance:], nil 140 | } 141 | } 142 | } 143 | } 144 | 145 | func parsePredicate(b []byte) (string, []byte, error) { 146 | var index int 147 | for { 148 | r, size, err, eol := decode(b[index:]) 149 | if err != nil { 150 | return "", nil, err 151 | } 152 | if eol { 153 | return "", nil, errors.New("invalid predicate") 154 | } 155 | index += size 156 | 157 | if r == '>' { 158 | if found, advance := peekNext(b[index:]); found == '<' || found == '"' || found == '_' { 159 | return string(b[:index-1]), b[index+advance:], nil 160 | } 161 | } 162 | } 163 | } 164 | 165 | func parseIRIObject(b []byte) (string, []byte, error) { 166 | var index int 167 | for { 168 | r, size, err, eol := decode(b[index:]) 169 | if err != nil { 170 | return "", nil, err 171 | } 172 | if eol { 173 | return "", nil, errors.New("invalid IRI object") 174 | } 175 | index += size 176 | 177 | if r == '>' { 178 | if found, advance := peekNext(b[index:]); found == '.' { 179 | return string(b[:index-1]), b[index+advance:], nil 180 | } 181 | } 182 | } 183 | } 184 | 185 | func parseIRISubject(b []byte) (string, []byte, error) { 186 | var index int 187 | for { 188 | r, size, err, eol := decode(b[index:]) 189 | if err != nil { 190 | return "", nil, err 191 | } 192 | if eol { 193 | return "", nil, errors.New("invalid IRI subject") 194 | } 195 | index += size 196 | 197 | if r == '>' { 198 | if found, advance := peekNext(b[index:]); found == '<' { 199 | return string(b[:index-1]), b[index+advance:], nil 200 | } 201 | } 202 | } 203 | } 204 | 205 | func parseBNodeObject(b []byte) (string, []byte, error) { 206 | var index int 207 | for { 208 | r, size, err, eol := decode(b[index:]) 209 | if err != nil { 210 | return "", nil, err 211 | } 212 | if eol { 213 | return "", nil, errors.New("invalid bnode object") 214 | } 215 | index += size 216 | 217 | if r == '.' { 218 | if found, advance := peekNext(b[index:]); found == '#' || found == 0 { 219 | return string(b[:index-1]), b[index-1+advance:], nil 220 | } 221 | } 222 | 223 | if r == ' ' || r == '\t' { 224 | if found, advance := peekNext(b[index:]); found == '.' { 225 | return string(b[:index-1]), b[index+advance:], nil 226 | } 227 | } 228 | } 229 | } 230 | 231 | func parseBNodeSubject(b []byte) (string, []byte, error) { 232 | var index int 233 | for { 234 | r, size, err, eol := decode(b[index:]) 235 | if err != nil { 236 | return "", nil, err 237 | } 238 | if eol { 239 | return "", nil, errors.New("invalid bnode subject") 240 | } 241 | index += size 242 | 243 | if r == '<' { 244 | return string(b[:index-1]), b[index-1:], nil 245 | } 246 | if r == ' ' || r == '\t' { 247 | if found, advance := peekNext(b[index:]); found == '<' { 248 | return string(b[:index-1]), b[index+advance:], nil 249 | } 250 | } 251 | } 252 | } 253 | 254 | func decode(b []byte) (rune, int, error, bool) { 255 | r, size := utf8.DecodeRune(b) 256 | if r == utf8.RuneError && size == 1 { 257 | return r, 0, errors.New("invalid utf8 encoding"), false 258 | } 259 | if r == utf8.RuneError && size == 0 { 260 | return 0, 0, nil, true 261 | } 262 | return r, size, nil, false 263 | } 264 | 265 | func peekNext(b []byte) (found rune, advance int) { 266 | for { 267 | r, size := utf8.DecodeRune(b[advance:]) 268 | if r == utf8.RuneError { 269 | return 0, 0 270 | } 271 | 272 | if r != ' ' && r != '\t' { 273 | found = r 274 | return 275 | } 276 | advance += size 277 | } 278 | } 279 | 280 | func doublePeekNext(b []byte) (first rune, advance int, second rune) { 281 | first, advance = peekNext(b) 282 | if n := advance + utf8.RuneLen(first); len(b) >= n { 283 | second, _ = peekNext(b[n:]) 284 | } 285 | return first, advance, second 286 | } 287 | -------------------------------------------------------------------------------- /decode.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "encoding/binary" 8 | "fmt" 9 | "io" 10 | "os" 11 | "strings" 12 | "sync" 13 | ) 14 | 15 | type Decoder interface { 16 | Decode() ([]Triple, error) 17 | } 18 | 19 | type DecodeResult struct { 20 | Tri Triple 21 | Err error 22 | } 23 | 24 | type StreamDecoder interface { 25 | StreamDecode(context.Context) <-chan DecodeResult 26 | } 27 | 28 | // Use for retro compatibilty when changing file format on existing stores 29 | func NewAutoDecoder(r io.Reader) Decoder { 30 | ok, newR := IsNTFormat(r) 31 | if ok { 32 | return NewLenientNTDecoder(newR) 33 | } 34 | return NewBinaryDecoder(newR) 35 | } 36 | 37 | // Loosely detect if a ntriples format contrary to a binary format 38 | // Used for retro compatibilty when changing file format on existing stores 39 | // Detecttion work with ntriples format flushed by this library (i.e. no comment, no spaces, ...) 40 | func IsNTFormat(r io.Reader) (bool, io.Reader) { 41 | firstChar := make([]byte, 1) 42 | multi := io.MultiReader(bytes.NewReader(firstChar), r) 43 | if _, err := r.Read(firstChar); err != nil { 44 | return false, multi 45 | } 46 | return bytes.Equal(firstChar, []byte{'<'}), multi 47 | } 48 | 49 | func NewLenientNTDecoder(r io.Reader) Decoder { 50 | return &ntDecoder{r: r} 51 | } 52 | 53 | func NewLenientNTStreamDecoder(r io.Reader) StreamDecoder { 54 | return &ntDecoder{r: r} 55 | } 56 | 57 | type ntDecoder struct { 58 | r io.Reader 59 | } 60 | 61 | func (d *ntDecoder) Decode() ([]Triple, error) { 62 | return newLenientNTParser(d.r).Parse() 63 | } 64 | 65 | func (d *ntDecoder) StreamDecode(ctx context.Context) <-chan DecodeResult { 66 | decC := make(chan DecodeResult) 67 | 68 | go func() { 69 | defer close(decC) 70 | 71 | scanner := bufio.NewScanner(d.r) 72 | for { 73 | select { 74 | case <-ctx.Done(): 75 | return 76 | default: 77 | if scanner.Scan() { 78 | tris, err := newLenientNTParser(strings.NewReader(scanner.Text())).Parse() 79 | if err != nil { 80 | decC <- DecodeResult{Err: err} 81 | } else if len(tris) == 1 { 82 | decC <- DecodeResult{Tri: tris[0]} 83 | } 84 | } else { 85 | if err := scanner.Err(); err != nil { 86 | decC <- DecodeResult{Err: err} 87 | } 88 | return 89 | } 90 | } 91 | } 92 | }() 93 | 94 | return decC 95 | } 96 | 97 | type binaryDecoder struct { 98 | r io.Reader 99 | rc io.ReadCloser // for stream decoding 100 | triples []Triple 101 | } 102 | 103 | func NewBinaryStreamDecoder(r io.ReadCloser) StreamDecoder { 104 | return &binaryDecoder{rc: r} 105 | } 106 | 107 | func (dec *binaryDecoder) StreamDecode(ctx context.Context) <-chan DecodeResult { 108 | decC := make(chan DecodeResult) 109 | 110 | go func() { 111 | defer close(decC) 112 | for { 113 | select { 114 | case <-ctx.Done(): 115 | return 116 | default: 117 | tri, done, err := decodeTriple(dec.rc) 118 | if done { 119 | return 120 | } 121 | decC <- DecodeResult{Tri: tri, Err: err} 122 | } 123 | } 124 | }() 125 | 126 | return decC 127 | } 128 | 129 | func NewBinaryDecoder(r io.Reader) Decoder { 130 | return &binaryDecoder{r: r} 131 | } 132 | 133 | func (dec *binaryDecoder) Decode() ([]Triple, error) { 134 | var out []Triple 135 | for { 136 | tri, done, err := decodeTriple(dec.r) 137 | if tri != nil { 138 | out = append(out, tri) 139 | } 140 | if done { 141 | break 142 | } else if err != nil { 143 | return out, err 144 | } 145 | } 146 | 147 | return out, nil 148 | } 149 | 150 | func decodeTriple(r io.Reader) (Triple, bool, error) { 151 | var isSubBNode bool 152 | err := binary.Read(r, binary.BigEndian, &isSubBNode) 153 | if err == io.EOF { 154 | return nil, true, nil 155 | } else if err != nil { 156 | return nil, false, fmt.Errorf("is subject bnode: %s", err) 157 | } 158 | 159 | sub, err := readWord(r) 160 | if err != nil { 161 | return nil, false, fmt.Errorf("subject: %s", err) 162 | } 163 | 164 | pred, err := readWord(r) 165 | if err != nil { 166 | return nil, false, fmt.Errorf("predicate: %s", err) 167 | } 168 | 169 | var objType uint8 170 | if err := binary.Read(r, binary.BigEndian, &objType); err != nil { 171 | return nil, false, fmt.Errorf("object type: %s", err) 172 | } 173 | 174 | var decodedObj object 175 | if objType == resourceTypeEncoding { 176 | resource, err := readWord(r) 177 | if err != nil { 178 | return nil, false, fmt.Errorf("resource: %s", err) 179 | } 180 | decodedObj.resource = string(resource) 181 | } else if objType == bnodeTypeEncoding { 182 | bnode, err := readWord(r) 183 | if err != nil { 184 | return nil, false, fmt.Errorf("bnode object: %s", err) 185 | } 186 | decodedObj.bnode = string(bnode) 187 | decodedObj.isBnode = true 188 | } else { 189 | decodedObj.isLit = true 190 | var decodedLiteral literal 191 | 192 | if objType == literalWithLangEncoding { 193 | lang, err := readWord(r) 194 | if err != nil { 195 | return nil, false, fmt.Errorf("lang: %s", err) 196 | } 197 | decodedLiteral.langtag = string(lang) 198 | } else { 199 | litType, err := readWord(r) 200 | if err != nil { 201 | return nil, false, fmt.Errorf("literate type: %s", err) 202 | } 203 | decodedLiteral.typ = XsdType(litType) 204 | } 205 | 206 | val, err := readWord(r) 207 | if err != nil { 208 | return nil, false, fmt.Errorf("literate: %s", err) 209 | } 210 | if decodedLiteral.typ == XsdString || objType == literalWithLangEncoding { 211 | decodedLiteral.val = unescapeStringLiteral(string(val)) 212 | } else { 213 | decodedLiteral.val = string(val) 214 | } 215 | 216 | decodedObj.lit = decodedLiteral 217 | } 218 | 219 | return &triple{ 220 | isSubBnode: isSubBNode, 221 | sub: string(sub), 222 | pred: string(pred), 223 | obj: decodedObj, 224 | }, false, nil 225 | } 226 | 227 | func readWord(r io.Reader) ([]byte, error) { 228 | var len wordLength 229 | if err := binary.Read(r, binary.BigEndian, &len); err != nil { 230 | return nil, err 231 | } 232 | 233 | word := make([]byte, len) 234 | if _, err := io.ReadFull(r, word); err != nil { 235 | return nil, fmt.Errorf("triplestore: binary: cannot decode word of length %d bytes: %s", len, err) 236 | } 237 | 238 | return word, nil 239 | } 240 | 241 | type datasetDecoder struct { 242 | newDecoderFunc func(io.Reader) Decoder 243 | rs []io.Reader 244 | } 245 | 246 | // NewDatasetDecoder - a dataset is a basically a collection of RDFGraph. 247 | func NewDatasetDecoder(fn func(io.Reader) Decoder, readers ...io.Reader) Decoder { 248 | return &datasetDecoder{newDecoderFunc: fn, rs: readers} 249 | } 250 | 251 | func (dec *datasetDecoder) Decode() ([]Triple, error) { 252 | type result struct { 253 | err error 254 | tris []Triple 255 | reader io.Reader 256 | } 257 | 258 | results := make(chan *result, len(dec.rs)) 259 | done := make(chan struct{}) 260 | defer close(done) 261 | 262 | var wg sync.WaitGroup 263 | for _, reader := range dec.rs { 264 | wg.Add(1) 265 | go func(r io.Reader) { 266 | defer wg.Done() 267 | tris, err := dec.newDecoderFunc(r).Decode() 268 | select { 269 | case results <- &result{tris: tris, err: err, reader: r}: 270 | case <-done: 271 | return 272 | } 273 | }(reader) 274 | } 275 | 276 | go func() { 277 | wg.Wait() 278 | close(results) 279 | }() 280 | 281 | var all []Triple 282 | for r := range results { 283 | if r.err != nil { 284 | switch rr := r.reader.(type) { 285 | case *os.File: 286 | return all, fmt.Errorf("file '%s': %s", rr.Name(), r.err) 287 | default: 288 | return all, r.err 289 | } 290 | } 291 | all = append(all, r.tris...) 292 | } 293 | 294 | return all, nil 295 | } 296 | 297 | var unescaper = strings.NewReplacer("\\n", "\n", "\\r", "\r") 298 | 299 | func unescapeStringLiteral(s string) string { 300 | return unescaper.Replace(s) 301 | } 302 | -------------------------------------------------------------------------------- /encode.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | "net/url" 10 | "strings" 11 | ) 12 | 13 | type Encoder interface { 14 | Encode(tris ...Triple) error 15 | } 16 | 17 | type StreamEncoder interface { 18 | StreamEncode(context.Context, <-chan Triple) error 19 | } 20 | 21 | func NewContext() *Context { 22 | return &Context{Prefixes: make(map[string]string)} 23 | } 24 | 25 | type Context struct { 26 | Base string 27 | Prefixes map[string]string 28 | } 29 | 30 | var RDFContext = &Context{ 31 | Prefixes: map[string]string{ 32 | "xsd": "http://www.w3.org/2001/XMLSchema#", 33 | "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 34 | "rdfs": "http://www.w3.org/2000/01/rdf-schema#", 35 | }, 36 | } 37 | 38 | type wordLength uint32 39 | 40 | const ( 41 | resourceTypeEncoding = uint8(0) 42 | literalTypeEncoding = uint8(1) 43 | bnodeTypeEncoding = uint8(2) 44 | literalWithLangEncoding = uint8(3) 45 | ) 46 | 47 | type binaryEncoder struct { 48 | w io.Writer 49 | } 50 | 51 | func NewBinaryStreamEncoder(w io.Writer) StreamEncoder { 52 | return &binaryEncoder{w} 53 | } 54 | 55 | func NewBinaryEncoder(w io.Writer) Encoder { 56 | return &binaryEncoder{w} 57 | } 58 | 59 | func (enc *binaryEncoder) StreamEncode(ctx context.Context, triples <-chan Triple) error { 60 | if triples == nil { 61 | return nil 62 | } 63 | var buf bytes.Buffer 64 | for { 65 | select { 66 | case tri, ok := <-triples: 67 | if !ok { 68 | return nil 69 | } 70 | if err := enc.writeTriple(tri, &buf); err != nil { 71 | return err 72 | } 73 | case <-ctx.Done(): 74 | return nil 75 | } 76 | } 77 | } 78 | 79 | func (enc *binaryEncoder) Encode(tris ...Triple) error { 80 | var buf bytes.Buffer 81 | for _, t := range tris { 82 | if err := enc.writeTriple(t, &buf); err != nil { 83 | return err 84 | } 85 | } 86 | return nil 87 | } 88 | 89 | func (enc *binaryEncoder) writeTriple(t Triple, buf *bytes.Buffer) error { 90 | if err := encodeBinTriple(t, buf); err != nil { 91 | return err 92 | } 93 | if _, err := enc.w.Write(buf.Bytes()); err != nil { 94 | return err 95 | } 96 | buf.Reset() 97 | return nil 98 | } 99 | 100 | func encodeBinTriple(t Triple, buff *bytes.Buffer) error { 101 | sub, pred := t.Subject(), t.Predicate() 102 | 103 | binary.Write(buff, binary.BigEndian, t.(*triple).isSubBnode) 104 | 105 | binary.Write(buff, binary.BigEndian, wordLength(len(sub))) 106 | buff.WriteString(sub) 107 | 108 | binary.Write(buff, binary.BigEndian, wordLength(len(pred))) 109 | buff.WriteString(pred) 110 | 111 | obj := t.Object() 112 | if lit, isLit := obj.Literal(); isLit { 113 | if lang := lit.Lang(); len(lang) > 0 { 114 | binary.Write(buff, binary.BigEndian, literalWithLangEncoding) 115 | binary.Write(buff, binary.BigEndian, wordLength(len(lang))) 116 | buff.WriteString(string(lang)) 117 | } else { 118 | binary.Write(buff, binary.BigEndian, literalTypeEncoding) 119 | typ := lit.Type() 120 | binary.Write(buff, binary.BigEndian, wordLength(len(typ))) 121 | buff.WriteString(string(typ)) 122 | } 123 | 124 | litVal := lit.Value() 125 | if lit.Type() == XsdString { 126 | litVal = escapeStringLiteral(litVal) 127 | } 128 | binary.Write(buff, binary.BigEndian, wordLength(len(litVal))) 129 | buff.WriteString(litVal) 130 | } else if bnode, isBnode := obj.Bnode(); isBnode { 131 | binary.Write(buff, binary.BigEndian, bnodeTypeEncoding) 132 | binary.Write(buff, binary.BigEndian, wordLength(len(bnode))) 133 | buff.WriteString(bnode) 134 | } else { 135 | binary.Write(buff, binary.BigEndian, resourceTypeEncoding) 136 | res, _ := obj.Resource() 137 | binary.Write(buff, binary.BigEndian, wordLength(len(res))) 138 | buff.WriteString(res) 139 | } 140 | 141 | return nil 142 | } 143 | 144 | type ntriplesEncoder struct { 145 | w io.Writer 146 | c *Context 147 | } 148 | 149 | func NewLenientNTStreamEncoder(w io.Writer) StreamEncoder { 150 | return &ntriplesEncoder{w: w} 151 | } 152 | 153 | func NewLenientNTEncoder(w io.Writer) Encoder { 154 | return &ntriplesEncoder{w: w} 155 | } 156 | 157 | func NewLenientNTEncoderWithContext(w io.Writer, c *Context) Encoder { 158 | return &ntriplesEncoder{w: w, c: c} 159 | } 160 | 161 | func (enc *ntriplesEncoder) StreamEncode(ctx context.Context, triples <-chan Triple) error { 162 | if triples == nil { 163 | return nil 164 | } 165 | var buf bytes.Buffer 166 | finalWrite := func() error { 167 | _, err := enc.w.Write(buf.Bytes()) 168 | return err 169 | } 170 | for { 171 | select { 172 | case tri, ok := <-triples: 173 | if !ok { 174 | return finalWrite() 175 | } 176 | encodeNTriple(tri, enc.c, &buf) 177 | case <-ctx.Done(): 178 | return finalWrite() 179 | } 180 | } 181 | } 182 | 183 | func (enc *ntriplesEncoder) Encode(tris ...Triple) error { 184 | var buff bytes.Buffer 185 | 186 | for _, t := range tris { 187 | encodeNTriple(t, enc.c, &buff) 188 | } 189 | _, err := enc.w.Write(buff.Bytes()) 190 | return err 191 | } 192 | 193 | func encodeNTriple(t Triple, ctx *Context, buff *bytes.Buffer) { 194 | var sub string 195 | if tt := t.(*triple); tt.isSubBnode { 196 | sub = "_:" + buildIRI(ctx, t.Subject()) 197 | } else { 198 | sub = "<" + buildIRI(ctx, t.Subject()) + ">" 199 | } 200 | buff.WriteString(sub + " <" + buildIRI(ctx, t.Predicate()) + "> ") 201 | 202 | if bnode, isBnode := t.Object().Bnode(); isBnode { 203 | buff.WriteString("_:" + bnode) 204 | } else { 205 | if rid, ok := t.Object().Resource(); ok { 206 | buff.WriteString("<" + buildIRI(ctx, rid) + ">") 207 | } else if lit, ok := t.Object().Literal(); ok { 208 | if lit.Lang() != "" { 209 | buff.WriteString("\"" + escapeStringLiteral(lit.Value()) + "\"@" + lit.Lang()) 210 | } else { 211 | switch lit.Type() { 212 | case XsdString: 213 | // namespace empty as per spec 214 | buff.WriteString("\"" + escapeStringLiteral(lit.Value()) + "\"") 215 | default: 216 | if ctx != nil { 217 | if _, ok := ctx.Prefixes["xsd"]; ok { 218 | buff.WriteString("\"" + lit.Value() + "\"^^<" + lit.Type().NTriplesNamespaced() + ">") 219 | } 220 | } else { 221 | buff.WriteString("\"" + lit.Value() + "\"^^<" + string(lit.Type()) + ">") 222 | } 223 | } 224 | } 225 | } 226 | } 227 | buff.Write([]byte(" .\n")) 228 | } 229 | 230 | func buildIRI(ctx *Context, id string) string { 231 | if ctx != nil { 232 | if ctx.Prefixes != nil { 233 | for k, uri := range ctx.Prefixes { 234 | prefix := k + ":" 235 | if strings.HasPrefix(id, prefix) { 236 | id = uri + url.QueryEscape(strings.TrimPrefix(id, prefix)) 237 | continue 238 | } 239 | } 240 | } 241 | if !strings.HasPrefix(id, "http") && ctx.Base != "" { 242 | id = ctx.Base + url.QueryEscape(id) 243 | } 244 | } 245 | return id 246 | } 247 | 248 | type dotGraphEncoder struct { 249 | pred string 250 | w io.Writer 251 | } 252 | 253 | func NewDotGraphEncoder(w io.Writer, predicate string) Encoder { 254 | return &dotGraphEncoder{w: w, pred: predicate} 255 | } 256 | 257 | func (dg *dotGraphEncoder) Encode(tris ...Triple) error { 258 | src := NewSource() 259 | src.Add(tris...) 260 | 261 | snap := src.Snapshot() 262 | all := snap.WithPredicate(dg.pred) 263 | 264 | queryDone := make(map[string][]string) 265 | 266 | getTypes := func(ref string) ([]string, bool) { 267 | if all, ok := queryDone[ref]; ok { 268 | return all, true 269 | } else { 270 | fresh := snap.WithSubjPred(ref, "rdf:type") 271 | for _, typ := range fresh { 272 | val, _ := typ.Object().Resource() 273 | queryDone[ref] = append(queryDone[ref], val) 274 | } 275 | return queryDone[ref], false 276 | } 277 | } 278 | 279 | fmt.Fprintf(dg.w, "digraph \"%s\" {\n", dg.pred) 280 | for _, tri := range all { 281 | sub := tri.Subject() 282 | res, ok := tri.Object().Resource() 283 | if ok { 284 | fmt.Fprintf(dg.w, "\"%s\" -> \"%s\";\n", sub, res) 285 | 286 | subTypes, done := getTypes(sub) 287 | if !done { 288 | for _, typ := range subTypes { 289 | fmt.Fprintf(dg.w, "\"%s\" [label=\"%s<%s>\"];\n", sub, sub, typ) 290 | } 291 | } 292 | 293 | resTypes, done := getTypes(res) 294 | if !done { 295 | for _, typ := range resTypes { 296 | fmt.Fprintf(dg.w, "\"%s\" [label=\"%s<%s>\"];\n", res, res, typ) 297 | } 298 | } 299 | } 300 | } 301 | 302 | fmt.Fprintf(dg.w, "}") 303 | 304 | return nil 305 | } 306 | 307 | var escaper = strings.NewReplacer("\n", "\\n", "\r", "\\r") 308 | 309 | func escapeStringLiteral(s string) string { 310 | return escaper.Replace(s) 311 | } 312 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://api.travis-ci.org/wallix/triplestore.svg?branch=master)](https://travis-ci.org/wallix/triplestore) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/wallix/triplestore)](https://goreportcard.com/report/github.com/wallix/triplestore) 3 | [![GoDoc](https://godoc.org/github.com/wallix/triplestore?status.svg)](https://godoc.org/github.com/wallix/triplestore) 4 | 5 | # Triple Store 6 | 7 | Triple Store is a library to manipulate RDF triples in a fast and fluent fashion. 8 | 9 | RDF triples allow to represent any data and its relations to other data. It is a very versatile concept and is used in [Linked Data](https://en.wikipedia.org/wiki/Linked_data), graphs traversal and storage, etc.... 10 | 11 | Here the RDF triples implementation follows along the [W3C RDF concepts](https://www.w3.org/TR/rdf11-concepts/). (**Note that reification is not implemented**.). More digestible info on [RDF Wikipedia](https://en.wikipedia.org/wiki/Resource_Description_Framework) 12 | 13 | ## Features overview 14 | 15 | - Create and manage triples through a convenient DSL 16 | - Snapshot and query RDFGraphs 17 | - **Binary** encoding/decoding 18 | - **Lenient NTriples** encoding/decoding (see W3C Test suite in _testdata/ntriples/w3c_suite/_) 19 | - [DOT](https://en.wikipedia.org/wiki/DOT_(graph_description_language)) encoding 20 | - Stream encoding/decoding (for binary & NTriples format) for memory conscious program 21 | - CLI (Command line interface) utility to read and convert triples files. 22 | 23 | ## Library 24 | 25 | This library is written using the [Golang](https://golang.org) language. You need to [install Golang](https://golang.org/doc/install) before using it. 26 | 27 | Get it: 28 | 29 | ```sh 30 | go get -u github.com/wallix/triplestore 31 | ``` 32 | 33 | Test it: 34 | 35 | ``` 36 | go test -v -cover -race github.com/wallix/triplestore 37 | ``` 38 | 39 | Bench it: 40 | 41 | ``` 42 | go test -run=none -bench=. -benchmem 43 | ``` 44 | 45 | Import it in your source code: 46 | 47 | ```go 48 | import ( 49 | "github.com/wallix/triplestore" 50 | // tstore "github.com/wallix/triplestore" for less verbosity 51 | ) 52 | ``` 53 | 54 | Get the CLI with: 55 | 56 | ``` 57 | go get -u github.com/wallix/triplestore/cmd/triplestore 58 | ``` 59 | 60 | ## Concepts 61 | 62 | A triple is made of 3 components: 63 | 64 | subject -> predicate -> object 65 | 66 | ... or you can also view that as: 67 | 68 | entity -> attribute -> value 69 | 70 | So 71 | 72 | - A **triple** consists of a *subject*, a *predicate* and a *object*. 73 | - A **subject** is a unicode string. 74 | - A **predicate** is a unicode string. 75 | - An **object** is a *resource* (or IRI) or a *literal* (blank node are not supported). 76 | - A **literal** is a unicode string associated with a datatype (ex: string, integer, ...). 77 | - A **resource**, a.k.a IRI, is a unicode string which point to another resource. 78 | 79 | And 80 | 81 | - A **source** is a persistent yet mutable source or container of triples. 82 | - A **RDFGraph** is an **immutable set of triples**. It is a snapshot of a source and queryable . 83 | - A **dataset** is a basically a collection of *RDFGraph*. 84 | 85 | You can also view the library through the [godoc](https://godoc.org/github.com/wallix/triplestore) 86 | 87 | ## Usage 88 | 89 | #### Create triples 90 | 91 | Although you can build triples the way you want to model any data, they are usually built from known RDF vocabularies & namespace. Ex: [foaf](http://xmlns.com/foaf/spec/), ... 92 | 93 | ```go 94 | triples = append(triples, 95 | SubjPred("me", "name").StringLiteral("jsmith"), 96 | SubjPred("me", "age").IntegerLiteral(26), 97 | SubjPred("me", "male").BooleanLiteral(true), 98 | SubjPred("me", "born").DateTimeLiteral(time.Now()), 99 | SubjPred("me", "mother").Resource("mum#121287"), 100 | ) 101 | ``` 102 | 103 | or dynamically and even shorter with 104 | 105 | ```go 106 | triples = append(triples, 107 | SubjPredLit("me", "age", "jsmith"), // String literal object 108 | SubjPredLit("me", "age", 26), // Integer literal object 109 | SubjPredLit("me", "male", true), // Boolean literal object 110 | SubjPredLit("me", "born", time.now()) // Datetime literal object 111 | SubjPredRes("me", "mother", "mum#121287"), // Resource object 112 | ) 113 | ``` 114 | 115 | or with blank nodes and language tag in literal 116 | 117 | ```go 118 | triples = append(triples, 119 | SubjPred("me", "name").Bnode("jsmith"), 120 | BnodePred("me", "name").StringLiteral("jsmith"), 121 | SubjPred("me", "name").StringLiteralWithLang("jsmith", "en"), 122 | ) 123 | ``` 124 | 125 | #### Create triples from a struct 126 | 127 | As a convenience you can create triples from a singular struct, where you control embedding through bnode. 128 | 129 | Here is an example. 130 | 131 | ```go 132 | type Address struct { 133 | Street string `predicate:"street"` 134 | City string `predicate:"city"` 135 | } 136 | 137 | type Person struct { 138 | Name string `predicate:"name"` 139 | Age int `predicate:"age"` 140 | Size int64 `predicate:"size"` 141 | Male bool `predicate:"male"` 142 | Birth time.Time `predicate:"birth"` 143 | Surnames []string `predicate:"surnames"` 144 | Addr Address `predicate:"address" bnode:"myaddress"` // empty bnode value will make bnode value random 145 | } 146 | 147 | addr := &Address{...} 148 | person := &Person{Addr: addr, ....} 149 | 150 | tris := TriplesFromStruct("jsmith", person) 151 | 152 | src := NewSource() 153 | src.Add(tris) 154 | snap := src.Snapshot() 155 | 156 | snap.Contains(SubjPredLit("jsmith", "name", "...")) 157 | snap.Contains(SubjPredLit("jsmith", "size", 186)) 158 | snap.Contains(SubjPredLit("jsmith", "surnames", "...")) 159 | snap.Contains(SubjPredLit("jsmith", "surnames", "...")) 160 | snap.Contains(SubjPred("me", "address").Bnode("myaddress")) 161 | snap.Contains(BnodePred("myaddress", "street").StringLiteral("5th avenue")) 162 | snap.Contains(BnodePred("myaddress", "city").StringLiteral("New York")) 163 | ``` 164 | 165 | #### Equality 166 | 167 | ```go 168 | me := SubjPred("me", "name").StringLiteral("jsmith") 169 | you := SubjPred("me", "name").StringLiteral("fdupond") 170 | 171 | if me.Equal(you) { 172 | ... 173 | } 174 | ) 175 | ``` 176 | 177 | ### Triple Source 178 | 179 | A source is a persistent yet mutable source or container of triples 180 | 181 | ```go 182 | src := tstore.NewSource() 183 | 184 | src.Add( 185 | SubjPredLit("me", "age", "jsmith"), 186 | SubjPredLit("me", "born", time.now()), 187 | ) 188 | src.Remove(SubjPredLit("me", "age", "jsmith")) 189 | ``` 190 | 191 | ### RDFGraph 192 | 193 | A RDFGraph is an immutable set of triples you can query. You get a RDFGraph by snapshotting a source: 194 | 195 | ```go 196 | graph := src.Snapshot() 197 | 198 | tris := graph.WithSubject("me") 199 | for _, tri := range tris { 200 | ... 201 | } 202 | ``` 203 | 204 | ### Codec 205 | 206 | Triples can be encoded & decoded using either a simple binary format or more standard text format like NTriples, ... 207 | 208 | Triples can therefore be persisted to disk, serialized or sent over the network. 209 | 210 | For example 211 | 212 | ```go 213 | enc := NewBinaryEncoder(myWriter) 214 | err := enc.Encode(triples) 215 | ... 216 | 217 | dec := NewBinaryDecoder(myReader) 218 | triples, err := dec.Decode() 219 | ``` 220 | 221 | Create a file of triples under the lenient NTriples format: 222 | 223 | ```go 224 | f, err := os.Create("./triples.nt") 225 | if err != nil { 226 | return err 227 | } 228 | defer f.Close() 229 | 230 | enc := NewLenientNTEncoder(f) 231 | err := enc.Encode(triples) 232 | 233 | ``` 234 | 235 | Encode to a DOT graph 236 | ```go 237 | tris := []Triple{ 238 | SubjPredRes("me", "rel", "you"), 239 | SubjPredRes("me", "rdf:type", "person"), 240 | SubjPredRes("you", "rel", "other"), 241 | SubjPredRes("you", "rdf:type", "child"), 242 | SubjPredRes("other", "any", "john"), 243 | } 244 | 245 | err := NewDotGraphEncoder(file, "rel").Encode(tris...) 246 | ... 247 | 248 | // output 249 | // digraph "rel" { 250 | // "me" -> "you"; 251 | // "me" [label="me"]; 252 | // "you" -> "other"; 253 | // "you" [label="you"]; 254 | //} 255 | ``` 256 | 257 | Load a binary dataset (i.e. multiple RDFGraph) concurrently from given files: 258 | 259 | ```go 260 | path := filepath.Join(fmt.Sprintf("*%s", fileExt)) 261 | files, _ := filepath.Glob(path) 262 | 263 | var readers []io.Reader 264 | for _, f := range files { 265 | reader, err := os.Open(f) 266 | if err != nil { 267 | return g, fmt.Errorf("loading '%s': %s", f, err) 268 | } 269 | readers = append(readers, reader) 270 | } 271 | 272 | dec := tstore.NewDatasetDecoder(tstore.NewBinaryDecoder, readers...) 273 | tris, err := dec.Decode() 274 | if err != nil { 275 | return err 276 | } 277 | ... 278 | ``` 279 | 280 | ### triplestore CLI 281 | 282 | This CLI is mainly ised for triples files conversion and inspection. Install it with `go get github.com/wallix/triplestore/cmd/triplestore`. Then `triplestore -h` for help. 283 | 284 | Example of usage: 285 | 286 | ```sh 287 | triplestore -in ntriples -out bin -files fuzz/ntriples/corpus/samples.nt 288 | triplestore -in ntriples -out bin -files fuzz/ntriples/corpus/samples.nt 289 | triplestore -in bin -files fuzz/binary/corpus/samples.bin 290 | ``` 291 | 292 | ### RDFGraph as a Tree 293 | 294 | A tree is defined from a RDFGraph given: 295 | 296 | * a specific predicate as an edge 297 | * and considering triples pointing to RDF resource Object 298 | 299 | You can then navigate the tree using the existing API calls 300 | 301 | tree := tstore.NewTree(myGraph, myPredicate) 302 | tree.TraverseDFS(...) 303 | tree.TraverseAncestors(...) 304 | tree.TraverseSiblings(...) 305 | 306 | Have a look at the [godoc](https://godoc.org/github.com/wallix/triplestore) fro more info 307 | 308 | Note that at the moment, constructing a new tree from a graph does not verify if the tree is valid namely no cycle and each child at most one parent. 309 | -------------------------------------------------------------------------------- /dsl_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestBuildTripleWithLangtag(t *testing.T) { 9 | tri := SubjPred("bnode", "pred").StringLiteralWithLang("any", "estonian") 10 | if got, want := tri.Subject(), "bnode"; got != want { 11 | t.Fatalf("got %s, want %s", got, want) 12 | } 13 | if got, want := tri.Predicate(), "pred"; got != want { 14 | t.Fatalf("got %s, want %s", got, want) 15 | } 16 | lit, _ := tri.Object().Literal() 17 | if got, want := lit.Lang(), "estonian"; got != want { 18 | t.Fatalf("got %s, want %s", got, want) 19 | } 20 | if got, want := lit.Value(), "any"; got != want { 21 | t.Fatalf("got %s, want %s", got, want) 22 | } 23 | 24 | lit, _ = StringLiteralWithLang("any", "estonian").Literal() 25 | if got, want := lit.Lang(), "estonian"; got != want { 26 | t.Fatalf("got %s, want %s", got, want) 27 | } 28 | if got, want := lit.Value(), "any"; got != want { 29 | t.Fatalf("got %s, want %s", got, want) 30 | } 31 | } 32 | 33 | func TestBuildTripleWithBnode(t *testing.T) { 34 | t.Run("subject as bnode", func(t *testing.T) { 35 | tri := BnodePred("bnode", "pred").StringLiteral("any") 36 | if got, want := tri.Subject(), "bnode"; got != want { 37 | t.Fatalf("got %s, want %s", got, want) 38 | } 39 | if tri.isSubBnode != true { 40 | t.Fatal("expecting subject to be bnode") 41 | } 42 | if got, want := tri.Predicate(), "pred"; got != want { 43 | t.Fatalf("got %s, want %s", got, want) 44 | } 45 | 46 | tri = BnodePredRes("bnode", "pred", "any") 47 | if got, want := tri.Subject(), "bnode"; got != want { 48 | t.Fatalf("got %s, want %s", got, want) 49 | } 50 | if tri.isSubBnode != true { 51 | t.Fatal("expecting subject to be bnode") 52 | } 53 | if got, want := tri.Predicate(), "pred"; got != want { 54 | t.Fatalf("got %s, want %s", got, want) 55 | } 56 | res, _ := tri.Object().Resource() 57 | if got, want := res, "any"; got != want { 58 | t.Fatalf("got %s, want %s", got, want) 59 | } 60 | }) 61 | 62 | t.Run("object as bnode", func(t *testing.T) { 63 | tri := SubjPred("sub", "pred").Bnode("any") 64 | if got, want := tri.Subject(), "sub"; got != want { 65 | t.Fatalf("got %s, want %s", got, want) 66 | } 67 | if got, want := tri.Predicate(), "pred"; got != want { 68 | t.Fatalf("got %s, want %s", got, want) 69 | } 70 | if obj := tri.obj; obj.isBnode != true { 71 | t.Fatal("expecting obj to be bnode") 72 | } 73 | 74 | tri = SubjPredBnode("sub", "pred", "any") 75 | if got, want := tri.Subject(), "sub"; got != want { 76 | t.Fatalf("got %s, want %s", got, want) 77 | } 78 | if got, want := tri.Predicate(), "pred"; got != want { 79 | t.Fatalf("got %s, want %s", got, want) 80 | } 81 | if obj := tri.obj; obj.isBnode != true { 82 | t.Fatal("expecting obj to be bnode") 83 | } 84 | }) 85 | } 86 | 87 | func TestBuildTriple(t *testing.T) { 88 | tri := SubjPred("subject", "predicate").StringLiteral("any") 89 | if got, want := tri.Subject(), "subject"; got != want { 90 | t.Fatalf("got %s, want %s", got, want) 91 | } 92 | if got, want := tri.Predicate(), "predicate"; got != want { 93 | t.Fatalf("got %s, want %s", got, want) 94 | } 95 | 96 | tri = SubjPredRes("subject", "predicate", "resource") 97 | if got, want := tri.Subject(), "subject"; got != want { 98 | t.Fatalf("got %s, want %s", got, want) 99 | } 100 | if got, want := tri.Predicate(), "predicate"; got != want { 101 | t.Fatalf("got %s, want %s", got, want) 102 | } 103 | res, _ := tri.Object().Resource() 104 | if got, want := res, "resource"; got != want { 105 | t.Fatalf("got %s, want %s", got, want) 106 | } 107 | 108 | tri, _ = SubjPredLit("subject", "predicate", 3) 109 | if got, want := tri.Subject(), "subject"; got != want { 110 | t.Fatalf("got %s, want %s", got, want) 111 | } 112 | if got, want := tri.Predicate(), "predicate"; got != want { 113 | t.Fatalf("got %s, want %s", got, want) 114 | } 115 | lit, _ := ParseInteger(tri.Object()) 116 | if got, want := lit, 3; got != want { 117 | t.Fatalf("got %d, want %d", got, want) 118 | } 119 | } 120 | 121 | func TestBuildObjectFromInterface(t *testing.T) { 122 | obj, _ := ObjectLiteral(true) 123 | if got, want := obj, BooleanLiteral(true); got != want { 124 | t.Fatalf("got %v, want %v", got, want) 125 | } 126 | obj, _ = ObjectLiteral(5) 127 | if got, want := obj, IntegerLiteral(5); got != want { 128 | t.Fatalf("got %v, want %v", got, want) 129 | 130 | } 131 | obj, _ = ObjectLiteral(int64(5)) 132 | if got, want := obj, IntegerLiteral(5); got != want { 133 | t.Fatalf("got %v, want %v", got, want) 134 | } 135 | obj, _ = ObjectLiteral("any") 136 | if got, want := obj, StringLiteral("any"); got != want { 137 | t.Fatalf("got %v, want %v", got, want) 138 | } 139 | now := time.Now() 140 | obj, _ = ObjectLiteral(now) 141 | if got, want := obj, DateTimeLiteral(now); got != want { 142 | t.Fatalf("got %v, want %v", got, want) 143 | } 144 | obj, _ = ObjectLiteral(&now) 145 | if got, want := obj, DateTimeLiteral(now); got != want { 146 | t.Fatalf("got %v, want %v", got, want) 147 | } 148 | } 149 | 150 | type stringer struct { 151 | s string 152 | } 153 | 154 | func (s stringer) String() string { 155 | return s.s 156 | } 157 | 158 | func TestBuildAndParseObjectLiteralFromDifferentTypes(t *testing.T) { 159 | 160 | tcases := []struct { 161 | in interface{} 162 | out Object 163 | exp interface{} 164 | }{ 165 | {stringer{"stuff"}, StringLiteral("stuff"), "stuff"}, 166 | 167 | {float64(2.0), Float64Literal(2.0), float64(2.0)}, 168 | {float32(2.0), Float32Literal(2.0), float32(2.0)}, 169 | 170 | {int8(-2), Int8Literal(-2), int8(-2)}, 171 | {int16(-2), Int16Literal(-2), int16(-2)}, 172 | {int32(-2), IntegerLiteral(-2), int(-2)}, 173 | {int64(-2), IntegerLiteral(-2), int(-2)}, 174 | {int(-2), IntegerLiteral(-2), int(-2)}, 175 | 176 | {uint8(2), Uint8Literal(2), uint8(2)}, 177 | {uint16(2), Uint16Literal(2), uint16(2)}, 178 | {uint32(2), UintegerLiteral(2), uint(2)}, 179 | {uint64(2), UintegerLiteral(2), uint(2)}, 180 | {uint(2), UintegerLiteral(2), uint(2)}, 181 | } 182 | 183 | for _, tcase := range tcases { 184 | obj, err := ObjectLiteral(tcase.in) 185 | if err != nil { 186 | t.Fatal(err) 187 | } 188 | if got, want := obj, tcase.out; !got.Equal(want) { 189 | t.Fatalf("got %v, want %v", got, want) 190 | } 191 | 192 | lit, err := ParseLiteral(tcase.out) 193 | if err != nil { 194 | t.Fatal(err) 195 | } 196 | if got, want := lit, tcase.exp; got != want { 197 | t.Fatalf("got %v (%T), want %v (%T)", got, got, want, want) 198 | } 199 | } 200 | } 201 | 202 | func TestUnsupportedLiteralTypesErr(t *testing.T) { 203 | type any struct{} 204 | 205 | _, err := ObjectLiteral(&any{}) 206 | if err == nil { 207 | t.Fatal("expected error") 208 | } 209 | if _, ok := err.(UnsupportedLiteralTypeError); !ok { 210 | t.Fatal("expected error of known type") 211 | } 212 | } 213 | 214 | func TestParseObject(t *testing.T) { 215 | tri := SubjPred("subject", "predicate").IntegerLiteral(123) 216 | num, err := ParseInteger(tri.Object()) 217 | if err != nil { 218 | t.Fatal(err) 219 | } 220 | if got, want := num, 123; got != want { 221 | t.Fatalf("got %d, want %d", got, want) 222 | } 223 | numInt, err := ParseLiteral(tri.Object()) 224 | if err != nil { 225 | t.Fatal(err) 226 | } 227 | if got, want := numInt, 123; got != want { 228 | t.Fatalf("got %d, want %d", got, want) 229 | } 230 | 231 | tri = SubjPred("subject", "predicate").BooleanLiteral(true) 232 | b, err := ParseBoolean(tri.Object()) 233 | if err != nil { 234 | t.Fatal(err) 235 | } 236 | if got, want := b, true; got != want { 237 | t.Fatalf("got %t, want %t", got, want) 238 | } 239 | 240 | tri = SubjPred("subject", "predicate").BooleanLiteral(true) 241 | bInt, err := ParseLiteral(tri.Object()) 242 | if err != nil { 243 | t.Fatal(err) 244 | } 245 | if got, want := bInt, true; got != want { 246 | t.Fatalf("got %t, want %t", got, want) 247 | } 248 | 249 | now := time.Now() 250 | tri = SubjPred("subject", "predicate").DateTimeLiteral(now) 251 | date, err := ParseDateTime(tri.Object()) 252 | if err != nil { 253 | t.Fatal(err) 254 | } 255 | if got, want := date, now.UTC(); got != want { 256 | t.Fatalf("got %s, want %s", got, want) 257 | } 258 | 259 | tri = SubjPred("subject", "predicate").DateTimeLiteral(now) 260 | dateInt, err := ParseLiteral(tri.Object()) 261 | if err != nil { 262 | t.Fatal(err) 263 | } 264 | if got, want := dateInt, now.UTC(); got != want { 265 | t.Fatalf("got %s, want %s", got, want) 266 | } 267 | 268 | tri = SubjPred("subject", "predicate").StringLiteral("rdf") 269 | s, err := ParseString(tri.Object()) 270 | if err != nil { 271 | t.Fatal(err) 272 | } 273 | if got, want := s, "rdf"; got != want { 274 | t.Fatalf("got %s, want %s", got, want) 275 | } 276 | tri = SubjPred("subject", "predicate").StringLiteral("rdf") 277 | sInt, err := ParseLiteral(tri.Object()) 278 | if err != nil { 279 | t.Fatal(err) 280 | } 281 | if got, want := sInt, "rdf"; got != want { 282 | t.Fatalf("got %s, want %s", got, want) 283 | } 284 | 285 | lit, ok := tri.Object().Literal() 286 | if got, want := ok, true; got != want { 287 | t.Fatalf("got %t, want %t", got, want) 288 | } 289 | if got, want := lit.Value(), "rdf"; got != want { 290 | t.Fatalf("got %s, want %s", got, want) 291 | } 292 | if got, want := lit.Type(), XsdString; got != want { 293 | t.Fatalf("got %s, want %s", got, want) 294 | } 295 | 296 | _, ok = tri.Object().Resource() 297 | if got, want := ok, false; got != want { 298 | t.Fatalf("got %t, want %t", got, want) 299 | } 300 | } 301 | 302 | func TestObjectHasResource(t *testing.T) { 303 | tri := SubjPred("subject", "predicate").Resource("dbpedia:Bonobo") 304 | 305 | rid, ok := tri.Object().Resource() 306 | if got, want := ok, true; got != want { 307 | t.Fatalf("got %t, want %t", got, want) 308 | } 309 | if got, want := rid, "dbpedia:Bonobo"; got != want { 310 | t.Fatalf("got %s, want %s", got, want) 311 | } 312 | _, ok = tri.Object().Literal() 313 | if got, want := ok, false; got != want { 314 | t.Fatalf("got %t, want %t", got, want) 315 | } 316 | } 317 | -------------------------------------------------------------------------------- /ntparser_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | func TestMultilineEmptyAndCommentLine(t *testing.T) { 11 | p := newLenientNTParser(strings.NewReader(` # my triples 12 | 13 | # starting 14 | "obj"@en . 15 | 16 | # ending 17 | 18 | `)) 19 | tris, err := p.Parse() 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | src := NewSource() 24 | src.Add(tris...) 25 | snap := src.Snapshot() 26 | if got, want := snap.Count(), 1; got != want { 27 | t.Fatalf("got %d, want %d", got, want) 28 | } 29 | if !snap.Contains(SubjPred("sub", "pred").StringLiteralWithLang("obj", "en")) { 30 | t.Fatal("expected to contain triple") 31 | } 32 | } 33 | 34 | func TestParsing(t *testing.T) { 35 | tcases := []struct { 36 | input string 37 | expected []Triple 38 | }{ 39 | { 40 | input: ` "quoting "anything".".`, 41 | expected: []Triple{ 42 | SubjPred("sub", "pred").StringLiteral(`quoting "anything".`), 43 | }, 44 | }, 45 | { 46 | input: ` "quoting 'anything'.".`, 47 | expected: []Triple{ 48 | SubjPred("sub", "pred").StringLiteral("quoting 'anything'."), 49 | }, 50 | }, 51 | { 52 | input: " .\n \"lol2\" .", 53 | expected: []Triple{ 54 | SubjPred("sub", "pred").Resource("lol"), 55 | mustTriple("sub2", "pred2", "lol2"), 56 | }, 57 | }, 58 | { 59 | input: " \"2\"^^ .\n .", 60 | expected: []Triple{ 61 | SubjPred("sub", "pred").Object(object{isLit: true, lit: literal{typ: "myinteger", val: "2"}}), 62 | SubjPred("sub2", "pred2").Resource("lol2"), 63 | }, 64 | }, 65 | { 66 | input: " \"2\"^^ .\n \"lol2\"@en.", 67 | expected: []Triple{ 68 | SubjPred("sub", "pred").Object(object{isLit: true, lit: literal{typ: "myinteger", val: "2"}}), 69 | SubjPred("sub2", "pred2").StringLiteralWithLang("lol2", "en"), 70 | }, 71 | }, 72 | { 73 | input: "_:sub. # comment", 74 | expected: []Triple{BnodePred("sub", "pred").Resource("obj")}, 75 | }, 76 | { 77 | input: "_:sub . # comment", 78 | expected: []Triple{BnodePred("sub", "pred").Resource("obj")}, 79 | }, 80 | { 81 | input: " \"dquote:\"\" .\n", 82 | expected: []Triple{SubjPred("sub", "pred").StringLiteral(`dquote:"`)}, 83 | }, 84 | { 85 | input: ".\n", 86 | expected: []Triple{SubjPred("sub", "pred").Resource("obj")}, 87 | }, 88 | { 89 | input: " _:anon.\n", 90 | expected: []Triple{SubjPred("sub", "pred").Bnode("anon")}, 91 | }, 92 | { 93 | input: "_:anon.\n", 94 | expected: []Triple{SubjPred("sub", "pred").Bnode("anon")}, 95 | }, 96 | { 97 | input: ` _:anon.`, 98 | expected: []Triple{SubjPred("sub", "pred").Bnode("anon")}, 99 | }, 100 | { 101 | input: " \"\u00E9\".\n", 102 | expected: []Triple{SubjPred("sub", "pred").StringLiteral("é")}, 103 | }, 104 | { 105 | input: " \"\u00E9\".", 106 | expected: []Triple{SubjPred("sub", "pred").StringLiteral("é")}, 107 | }, 108 | { 109 | input: " \"\032\".", 110 | expected: []Triple{SubjPred("sub", "pred").StringLiteral("\032")}, 111 | }, 112 | { 113 | input: " \"\x1A\".", 114 | expected: []Triple{SubjPred("sub", "pred").StringLiteral("\x1A")}, 115 | }, 116 | } 117 | 118 | for j, tcase := range tcases { 119 | p := newLenientNTParser(strings.NewReader(tcase.input)) 120 | tris, err := p.Parse() 121 | if err != nil { 122 | t.Fatalf("input=[%s]: %s", tcase.input, err) 123 | } 124 | if got, want := len(tris), len(tcase.expected); got != want { 125 | t.Fatalf("triples size (case %d): got %d, want %d", j+1, got, want) 126 | } 127 | for i, tri := range tris { 128 | if got, want := tri, tcase.expected[i]; !got.Equal(want) { 129 | t.Fatalf("case %d: input [%s]: triple (%d)\ngot %#v\n\nwant %#v", j+1, tcase.input, i+1, got, want) 130 | } 131 | } 132 | } 133 | } 134 | 135 | func TestParsingComponents(t *testing.T) { 136 | t.Run("literal object", func(t *testing.T) { 137 | tcases := []struct { 138 | input, left, exp string 139 | }{ 140 | {input: `stuff"`}, 141 | {input: `"`}, 142 | {input: `stuff" .`, exp: "stuff", left: "."}, 143 | {input: `stuff" . `, exp: "stuff", left: ". "}, 144 | {input: `stuff" .# comment`, exp: "stuff", left: ".# comment"}, 145 | {input: `stuff" . # comment`, exp: "stuff", left: ". # comment"}, 146 | {input: ` " .`, exp: " ", left: "."}, 147 | {input: `" .`, exp: "", left: "."}, 148 | {input: `stuff"^`}, 149 | {input: `stuff"^^`, exp: "stuff", left: "^^"}, 150 | {input: `stuff" ^^ `, exp: "stuff", left: "^^ "}, 151 | {input: `stuff"@`, exp: "stuff", left: "@"}, 152 | {input: `stuff" @ `, exp: "stuff", left: "@ "}, 153 | } 154 | for _, tcase := range tcases { 155 | s, left, _ := parseLiteralObject([]byte(tcase.input)) 156 | if got, want := s, tcase.exp; got != want { 157 | t.Fatalf("case [%s]: got '%s', want '%s'", tcase.input, got, want) 158 | } 159 | if got, want := left, []byte(tcase.left); !bytes.Equal(got, want) { 160 | t.Fatalf("case [%s]: left: got '%s', want '%s'", tcase.input, got, want) 161 | } 162 | } 163 | }) 164 | 165 | t.Run("bnode object", func(t *testing.T) { 166 | tcases := []struct { 167 | input, left, exp string 168 | }{ 169 | {input: "stuff"}, 170 | {input: "stuff.", exp: "stuff", left: "."}, 171 | {input: "stuff .", exp: "stuff", left: "."}, 172 | {input: "stuff . ", exp: "stuff", left: ". "}, 173 | {input: "stuff .# comment", exp: "stuff", left: ".# comment"}, 174 | {input: "stuff . # comment", exp: "stuff", left: ". # comment"}, 175 | {input: " .", exp: "", left: "."}, 176 | } 177 | for _, tcase := range tcases { 178 | s, left, _ := parseBNodeObject([]byte(tcase.input)) 179 | if got, want := s, tcase.exp; got != want { 180 | t.Fatalf("case [%s]: got '%s', want '%s'", tcase.input, got, want) 181 | } 182 | if got, want := left, []byte(tcase.left); !bytes.Equal(got, want) { 183 | t.Fatalf("case [%s]: left: got '%s', want '%s'", tcase.input, got, want) 184 | } 185 | } 186 | }) 187 | 188 | t.Run("object iri", func(t *testing.T) { 189 | tcases := []struct { 190 | input, left, exp string 191 | }{ 192 | {input: "stuff>"}, 193 | {input: "stuff>.", exp: "stuff", left: "."}, 194 | {input: "stuff> .", exp: "stuff", left: "."}, 195 | {input: "stuff> . ", exp: "stuff", left: ". "}, 196 | {input: "stuff> .# comment", exp: "stuff", left: ".# comment"}, 197 | {input: "stuff> . # comment", exp: "stuff", left: ". # comment"}, 198 | {input: ">.", left: "."}, 199 | {input: "> .", left: "."}, 200 | } 201 | for _, tcase := range tcases { 202 | s, left, _ := parseIRIObject([]byte(tcase.input)) 203 | if got, want := s, tcase.exp; got != want { 204 | t.Fatalf("case [%s]: got '%s', want '%s'", tcase.input, got, want) 205 | } 206 | if got, want := left, []byte(tcase.left); !bytes.Equal(got, want) { 207 | t.Fatalf("case [%s]: left: got '%s', want '%s'", tcase.input, got, want) 208 | } 209 | } 210 | }) 211 | 212 | t.Run("predicate iri", func(t *testing.T) { 213 | tcases := []struct { 214 | input, left, exp string 215 | }{ 216 | {input: "stuff>"}, 217 | {input: "stuff><", exp: "stuff", left: "<"}, 218 | {input: "stuff> <", exp: "stuff", left: "<"}, 219 | {input: "stuff> "`, exp: "stuff", left: "\""}, 221 | {input: `stuff> "`, exp: "stuff", left: "\""}, 222 | {input: `stuff>_`, exp: "stuff", left: "_"}, 223 | {input: `stuff> _`, exp: "stuff", left: "_"}, 224 | {input: "><", left: "<"}, 225 | } 226 | for _, tcase := range tcases { 227 | s, left, _ := parsePredicate([]byte(tcase.input)) 228 | if got, want := s, tcase.exp; got != want { 229 | t.Fatalf("case [%s]: got '%s', want '%s'", tcase.input, got, want) 230 | } 231 | if got, want := left, []byte(tcase.left); !bytes.Equal(got, want) { 232 | t.Fatalf("case [%s]: left: got '%s', want '%s'", tcase.input, got, want) 233 | } 234 | } 235 | }) 236 | 237 | t.Run("subject iri", func(t *testing.T) { 238 | tcases := []struct { 239 | input, left, exp string 240 | }{ 241 | {input: "stuff>"}, 242 | {input: "stuff><", exp: "stuff", left: "<"}, 243 | {input: "stuff> <", exp: "stuff", left: "<"}, 244 | {input: "stuff> <", left: "<"}, 246 | } 247 | for _, tcase := range tcases { 248 | s, left, _ := parseIRISubject([]byte(tcase.input)) 249 | 250 | if got, want := s, tcase.exp; got != want { 251 | t.Fatalf("case [%s]: got '%s', want '%s'", tcase.input, got, want) 252 | } 253 | if got, want := left, []byte(tcase.left); !bytes.Equal(got, want) { 254 | t.Fatalf("case [%s]: left: got '%s', want '%s'", tcase.input, got, want) 255 | } 256 | } 257 | }) 258 | 259 | t.Run("subject bnode", func(t *testing.T) { 260 | tcases := []struct { 261 | input, left, exp string 262 | }{ 263 | {input: "stuff"}, 264 | {input: "stuff <", exp: "stuff", left: "<"}, 265 | {input: "stuff < ", exp: "stuff", left: "< "}, 266 | } 267 | for _, tcase := range tcases { 268 | s, left, _ := parseBNodeSubject([]byte(tcase.input)) 269 | if got, want := s, tcase.exp; got != want { 270 | t.Fatalf("got '%s', want '%s'", got, want) 271 | } 272 | if got, want := left, []byte(tcase.left); !bytes.Equal(got, want) { 273 | t.Fatalf("case [%s]: left: got '%s', want '%s'", tcase.input, got, want) 274 | } 275 | 276 | } 277 | }) 278 | } 279 | 280 | func TestParserErrorHandling(t *testing.T) { 281 | tcases := []struct { 282 | input string 283 | errContains string 284 | }{ 285 | {input: " 1 ."}, 286 | //{input: " , ."}, passes 287 | } 288 | 289 | for _, tcase := range tcases { 290 | tris, err := newLenientNTParser(strings.NewReader(tcase.input)).Parse() 291 | if err == nil { 292 | t.Fatalf("expected err, got none. Triples parsed:\n%#v", Triples(tris).Map(func(tr Triple) string { return fmt.Sprint(tr) })) 293 | } 294 | if msg := tcase.errContains; msg != "" { 295 | if !strings.Contains(err.Error(), msg) { 296 | t.Fatalf("expected '%s' to contains '%s'", err.Error(), tcase.errContains) 297 | } 298 | } 299 | } 300 | } 301 | 302 | func mustTriple(s, p string, i interface{}) Triple { 303 | t, err := SubjPredLit(s, p, i) 304 | if err != nil { 305 | panic(err) 306 | } 307 | return t 308 | } 309 | -------------------------------------------------------------------------------- /codec_test.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "math" 9 | "os" 10 | "path/filepath" 11 | "reflect" 12 | "strings" 13 | "testing" 14 | "time" 15 | ) 16 | 17 | func TestLooselyDetectNTFormatReturningFullReader(t *testing.T) { 18 | var buff bytes.Buffer 19 | 20 | ok, _ := IsNTFormat(&buff) 21 | if got, want := ok, false; got != want { 22 | t.Fatalf("got %t, want %t", got, want) 23 | } 24 | 25 | triples := []Triple{ 26 | SubjPred("bingo", "2").Resource("3"), 27 | SubjPred("one", "two").Resource("three"), 28 | } 29 | NewBinaryEncoder(&buff).Encode(triples...) 30 | orig := buff.String() 31 | ok, r := IsNTFormat(&buff) 32 | if got, want := ok, false; got != want { 33 | t.Fatalf("got %t, want %t", got, want) 34 | } 35 | b, _ := ioutil.ReadAll(r) 36 | if got, want := string(b), orig; got != want { 37 | t.Fatalf("reader changed: got %s, want %s", got, want) 38 | } 39 | 40 | buff.Reset() 41 | NewLenientNTEncoder(&buff).Encode(triples...) 42 | orig = buff.String() 43 | ok, r = IsNTFormat(&buff) 44 | if got, want := ok, true; got != want { 45 | t.Fatalf("got %t, want %t", got, want) 46 | } 47 | b, _ = ioutil.ReadAll(r) 48 | if got, want := string(b), orig; got != want { 49 | t.Fatalf("reader changed: got %s, want %s", got, want) 50 | } 51 | } 52 | 53 | func TestEncodeAndDecodeAllTripleTypes(t *testing.T) { 54 | tcases := []struct { 55 | in Triple 56 | }{ 57 | {SubjPred("", "").Resource("")}, 58 | {SubjPred("one", "two").Resource("three")}, 59 | 60 | {SubjPred("", "").StringLiteral("")}, 61 | {SubjPred("one", "two").StringLiteral("three")}, 62 | // String literal object with new lines 63 | {SubjPred("one", "two").StringLiteral(`three 64 | four 65 | five`)}, 66 | {SubjPred("one", "two").StringLiteralWithLang(`three 67 | four 68 | five`, "en")}, 69 | 70 | {SubjPred("one", "two").IntegerLiteral(math.MaxInt64)}, 71 | {SubjPred("one", "two").IntegerLiteral(284765293570)}, 72 | {SubjPred("one", "two").IntegerLiteral(-345293239432)}, 73 | 74 | {SubjPred("one", "two").BooleanLiteral(true)}, 75 | {SubjPred("one", "two").BooleanLiteral(false)}, 76 | 77 | {SubjPred("one", "two").DateTimeLiteral(time.Now())}, 78 | 79 | // Bnodes 80 | {BnodePred("one", "two").StringLiteral("three")}, 81 | {BnodePred("one", "two").Bnode("three")}, 82 | 83 | // Langtag 84 | {BnodePred("one", "two").StringLiteralWithLang("three", "de")}, 85 | 86 | // Large data 87 | {SubjPred(strings.Repeat("s", 65000), "two").Resource("three")}, 88 | {SubjPred("one", strings.Repeat("t", 65000)).Resource("three")}, 89 | {SubjPred("one", "two").Resource(strings.Repeat("t", 65000))}, 90 | {SubjPred("one", "two").StringLiteral(strings.Repeat("t", 65000))}, 91 | } 92 | 93 | type codec struct { 94 | newEnc func(io.Writer) Encoder 95 | newDec func(io.Reader) Decoder 96 | } 97 | 98 | codecs := []codec{ 99 | {newEnc: NewBinaryEncoder, newDec: NewBinaryDecoder}, 100 | {newEnc: NewLenientNTEncoder, newDec: NewLenientNTDecoder}, 101 | } 102 | 103 | for i, codec := range codecs { 104 | for _, tcase := range tcases { 105 | var buff bytes.Buffer 106 | enc := codec.newEnc(&buff) 107 | 108 | if err := enc.Encode(tcase.in); err != nil { 109 | t.Fatal(err) 110 | } 111 | 112 | dec := codec.newDec(&buff) 113 | all, err := dec.Decode() 114 | if err != nil { 115 | t.Fatal(err) 116 | } 117 | 118 | if got, want := len(all), 1; got != want { 119 | t.Fatalf("got %d, want %d", got, want) 120 | } 121 | 122 | if got, want := tcase.in, all[0]; !got.Equal(want) { 123 | t.Fatalf("codec %d: case %v: \n\ngot\n%v\n\nwant\n%v\n", i+1, tcase.in, got, want) 124 | } 125 | } 126 | } 127 | } 128 | 129 | func TestEncodeDecodeOnFile(t *testing.T) { 130 | one := SubjPred("one", "pred1").StringLiteral(`a new 131 | line 132 | `) 133 | two := SubjPred("two", "pred2").StringLiteral(`another 134 | new 135 | line`) 136 | 137 | file, err := ioutil.TempFile("", "") 138 | if err != nil { 139 | t.Fatal(err) 140 | } 141 | defer os.Remove(file.Name()) 142 | 143 | if err = NewBinaryEncoder(file).Encode(one, two); err != nil { 144 | t.Fatal(err) 145 | } 146 | 147 | file.Seek(0, 0) 148 | tris, err := NewBinaryDecoder(file).Decode() 149 | if err != nil { 150 | t.Fatal(err) 151 | } 152 | if got, want := len(tris), 2; got != want { 153 | t.Fatalf("got %d, want %d", got, want) 154 | } 155 | 156 | s := NewSource() 157 | s.Add(tris...) 158 | snap := s.Snapshot() 159 | 160 | if !snap.Contains(one) { 161 | t.Fatalf("decoded file should contains %v", one) 162 | } 163 | if !snap.Contains(two) { 164 | t.Fatalf("decoded file should contains %v", two) 165 | } 166 | } 167 | 168 | func TestDecodeDataset(t *testing.T) { 169 | one := SubjPred("one", "pred1").StringLiteral("lit1") 170 | two := SubjPred("two", "pred2").StringLiteral("lit2") 171 | 172 | firstFile, err := ioutil.TempFile("", "") 173 | if err != nil { 174 | t.Fatal(err) 175 | } 176 | defer os.Remove(firstFile.Name()) 177 | 178 | secondFile, err := ioutil.TempFile("", "") 179 | if err != nil { 180 | t.Fatal(err) 181 | } 182 | defer os.Remove(secondFile.Name()) 183 | 184 | if err = NewBinaryEncoder(firstFile).Encode(one); err != nil { 185 | t.Fatal(err) 186 | } 187 | 188 | if err = NewBinaryEncoder(secondFile).Encode(two); err != nil { 189 | t.Fatal(err) 190 | } 191 | 192 | firstFile.Seek(0, 0) 193 | secondFile.Seek(0, 0) 194 | 195 | dec := NewDatasetDecoder(NewBinaryDecoder, firstFile, secondFile) 196 | tris, err := dec.Decode() 197 | if err != nil { 198 | t.Fatal(err) 199 | } 200 | 201 | s := NewSource() 202 | s.Add(tris...) 203 | snap := s.Snapshot() 204 | if got, want := snap.Count(), 2; got != want { 205 | t.Fatalf("got %d, want %d", got, want) 206 | } 207 | 208 | if !snap.Contains(one) { 209 | t.Fatalf("decoded dataset should contains %v", one) 210 | } 211 | 212 | if !snap.Contains(two) { 213 | t.Fatalf("decoded dataset should contains %v", two) 214 | } 215 | } 216 | func TestEncodeDecodeSomeNTriplesSampleFiles(t *testing.T) { 217 | path := filepath.Join("testdata", "*.nt") 218 | filenames, _ := filepath.Glob(path) 219 | 220 | for _, f := range filenames { 221 | b, err := ioutil.ReadFile(f) 222 | 223 | tris, err := NewLenientNTDecoder(bytes.NewReader(b)).Decode() 224 | if err != nil { 225 | t.Fatal(err) 226 | } 227 | 228 | var buff bytes.Buffer 229 | err = NewLenientNTEncoder(&buff).Encode(tris...) 230 | if err != nil { 231 | t.Fatal(err) 232 | } 233 | 234 | compareMultiline(t, buff.Bytes(), b) 235 | } 236 | } 237 | 238 | func TestDecodeNTriples(t *testing.T) { 239 | t.Run("with new lines in literal string object", func(t *testing.T) { 240 | s := `"three\nfour\n" .` 241 | tris, err := NewLenientNTDecoder(strings.NewReader(s)).Decode() 242 | if err != nil { 243 | t.Fatal(err) 244 | } 245 | 246 | got := tris[0] 247 | want := SubjPred("one", "two").StringLiteral(`three 248 | four 249 | `) 250 | if !got.Equal(want) { 251 | t.Fatalf("got \n%#v\nwant \n%#v\n", got, want) 252 | } 253 | 254 | gotLit, _ := got.Object().Literal() 255 | wantLit, _ := want.Object().Literal() 256 | if !reflect.DeepEqual(gotLit, wantLit) { 257 | t.Fatalf("got \n%#v\nwant \n%#v\n", gotLit, wantLit) 258 | } 259 | }) 260 | } 261 | 262 | func TestEncodeNTriples(t *testing.T) { 263 | t.Run("with new lines in literal string object", func(t *testing.T) { 264 | triples := []Triple{ 265 | SubjPred("one", "two").StringLiteral(`first line 266 | second line 267 | 268 | third line`), 269 | } 270 | 271 | var result bytes.Buffer 272 | if err := NewLenientNTEncoder(&result).Encode(triples...); err != nil { 273 | t.Fatal(err) 274 | } 275 | 276 | expect := " \"first line\\nsecond line\\n\\nthird line\" .\n" 277 | if got, want := result.String(), expect; got != want { 278 | t.Fatalf("got \n%q\nwant \n%q\n", got, want) 279 | } 280 | }) 281 | 282 | t.Run("with namespaces", func(t *testing.T) { 283 | triples := []Triple{ 284 | SubjPred("one", "rdf:type").Resource("onetype"), 285 | SubjPred("one", "prop1").StringLiteral("two"), 286 | SubjPred("http://my-url-to.test/#one", "prop2").IntegerLiteral(284765293570), 287 | SubjPred("one", "prop3").BooleanLiteral(true), 288 | SubjPred("one", "cloud:launched").DateTimeLiteral(time.Unix(1233456789, 0).UTC()), 289 | SubjPred("co").StringLiteral("with\"special . 305 | "two" . 306 | "284765293570"^^ . 307 | "true"^^ . 308 | "2009-02-01T02:53:09Z"^^ . 309 | "with"special . 311 | ` 312 | if got, want := buff.String(), expect; got != want { 313 | t.Fatalf("got \n%s\nwant \n%s\n", got, want) 314 | } 315 | }) 316 | } 317 | 318 | func TestEncodeDotGraph(t *testing.T) { 319 | tris := []Triple{ 320 | SubjPredRes("me", "rel", "you"), 321 | SubjPredRes("me", "rdf:type", "person"), 322 | SubjPredRes("you", "rel", "other"), 323 | SubjPredRes("you", "rdf:type", "child"), 324 | SubjPredRes("other", "any", "john"), 325 | } 326 | 327 | var buff bytes.Buffer 328 | if err := NewDotGraphEncoder(&buff, "rel").Encode(tris...); err != nil { 329 | t.Fatal(err) 330 | } 331 | 332 | exp := strings.Split(`digraph "rel" { 333 | "me" -> "you"; 334 | "me" [label="me"]; 335 | "you" -> "other"; 336 | "you" [label="you"]; 337 | }`, "\n") 338 | 339 | result := buff.String() 340 | if splits := strings.Split(result, "\n"); len(splits) != 6 { 341 | t.Fatalf("expected 6 lines count in \n%s\n", result) 342 | } 343 | 344 | for _, line := range exp { 345 | if got, want := buff.String(), line; !strings.Contains(got, want) { 346 | t.Fatalf("\n%q\n should contain \n%q\n", got, want) 347 | } 348 | } 349 | } 350 | 351 | func compareMultiline(t *testing.T, actual, expected []byte) { 352 | expected = cleanupNTriplesForComparison(expected) 353 | actual = cleanupNTriplesForComparison(actual) 354 | actuals := bytes.Split(actual, []byte("\n")) 355 | expecteds := bytes.Split(expected, []byte("\n")) 356 | 357 | for _, a := range actuals { 358 | if !contains(expecteds, a) { 359 | fmt.Printf("\texpected content\n%q\n", expected) 360 | fmt.Printf("\tactual content\n%q\n", actual) 361 | t.Fatalf("extra line not in expected content\n%s\n", a) 362 | } 363 | } 364 | 365 | for _, e := range expecteds { 366 | if !contains(actuals, e) { 367 | t.Fatalf("expected line is missing from actual content\n'%s'\n", e) 368 | } 369 | } 370 | } 371 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /dsl.go: -------------------------------------------------------------------------------- 1 | package triplestore 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "reflect" 7 | "strconv" 8 | "time" 9 | ) 10 | 11 | func SubjPredRes(s, p, r string) *triple { 12 | return &triple{ 13 | sub: s, pred: p, 14 | obj: Resource(r).(object), 15 | } 16 | } 17 | 18 | func BnodePredRes(s, p, r string) *triple { 19 | return &triple{ 20 | isSubBnode: true, 21 | sub: s, 22 | pred: p, 23 | obj: Resource(r).(object), 24 | } 25 | } 26 | 27 | func SubjPredBnode(s, p, r string) *triple { 28 | return &triple{ 29 | sub: s, 30 | pred: p, 31 | obj: object{bnode: r, isBnode: true}, 32 | } 33 | } 34 | 35 | func SubjPredLit(s, p string, l interface{}) (*triple, error) { 36 | o, err := ObjectLiteral(l) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return &triple{ 41 | sub: s, 42 | pred: p, 43 | obj: o.(object), 44 | }, nil 45 | } 46 | 47 | type tripleBuilder struct { 48 | sub, pred string 49 | isSubBnode bool 50 | langtag string 51 | } 52 | 53 | func SubjPred(s, p string) *tripleBuilder { 54 | return &tripleBuilder{sub: s, pred: p} 55 | } 56 | 57 | func BnodePred(s, p string) *tripleBuilder { 58 | return &tripleBuilder{sub: s, isSubBnode: true, pred: p} 59 | } 60 | 61 | func Resource(s string) Object { 62 | return object{resource: s} 63 | } 64 | 65 | func (b *tripleBuilder) Lang(l string) *tripleBuilder { 66 | b.langtag = l 67 | return b 68 | } 69 | 70 | func (b *tripleBuilder) Resource(s string) *triple { 71 | return &triple{ 72 | isSubBnode: b.isSubBnode, 73 | sub: b.sub, 74 | pred: b.pred, 75 | obj: Resource(s).(object), 76 | } 77 | } 78 | 79 | func (b *tripleBuilder) Object(o Object) *triple { 80 | return &triple{ 81 | isSubBnode: b.isSubBnode, 82 | sub: b.sub, 83 | pred: b.pred, 84 | obj: o.(object), 85 | } 86 | } 87 | 88 | func (b *tripleBuilder) Bnode(s string) *triple { 89 | return &triple{ 90 | isSubBnode: b.isSubBnode, 91 | sub: b.sub, 92 | pred: b.pred, 93 | obj: object{bnode: s, isBnode: true}, 94 | } 95 | } 96 | 97 | type UnsupportedLiteralTypeError struct { 98 | i interface{} 99 | } 100 | 101 | func (e UnsupportedLiteralTypeError) Error() string { 102 | return fmt.Sprintf("unsupported literal type %T", e.i) 103 | } 104 | 105 | func ObjectLiteral(i interface{}) (Object, error) { 106 | switch ii := i.(type) { 107 | case string: 108 | return StringLiteral(ii), nil 109 | case bool: 110 | return BooleanLiteral(ii), nil 111 | case int: 112 | return IntegerLiteral(ii), nil 113 | case int64, int32: 114 | r := reflect.ValueOf(ii) 115 | return IntegerLiteral(int(r.Int())), nil 116 | case int16: 117 | return Int16Literal(ii), nil 118 | case int8: 119 | return Int8Literal(ii), nil 120 | case float32: 121 | return Float32Literal(ii), nil 122 | case float64: 123 | return Float64Literal(ii), nil 124 | case uint: 125 | return UintegerLiteral(uint(ii)), nil 126 | case uint64, uint32: 127 | r := reflect.ValueOf(ii) 128 | return UintegerLiteral(uint(r.Uint())), nil 129 | case uint16: 130 | return Uint16Literal(ii), nil 131 | case uint8: 132 | return Uint8Literal(ii), nil 133 | case time.Time: 134 | return DateTimeLiteral(ii), nil 135 | case *time.Time: 136 | return DateTimeLiteral(*ii), nil 137 | case fmt.Stringer: 138 | return StringLiteral(ii.String()), nil 139 | default: 140 | return nil, UnsupportedLiteralTypeError{i} 141 | } 142 | } 143 | 144 | func ParseLiteral(obj Object) (interface{}, error) { 145 | if lit, ok := obj.Literal(); ok { 146 | switch lit.Type() { 147 | case XsdBoolean: 148 | return ParseBoolean(obj) 149 | case XsdDateTime: 150 | return ParseDateTime(obj) 151 | case XsdInteger: 152 | return ParseInteger(obj) 153 | case XsdByte: 154 | return ParseInt8(obj) 155 | case XsdShort: 156 | return ParseInt16(obj) 157 | case XsdUinteger: 158 | return ParseUinteger(obj) 159 | case XsdUnsignedByte: 160 | return ParseUint8(obj) 161 | case XsdUnsignedShort: 162 | return ParseUint16(obj) 163 | case XsdDouble: 164 | return ParseFloat64(obj) 165 | case XsdFloat: 166 | return ParseFloat32(obj) 167 | case XsdString: 168 | return ParseString(obj) 169 | default: 170 | return nil, fmt.Errorf("unknown literal type: %s", lit.Type()) 171 | } 172 | } 173 | return nil, errors.New("cannot parse literal: object is not literal") 174 | } 175 | 176 | func BooleanLiteral(bl bool) Object { 177 | return object{ 178 | isLit: true, 179 | lit: literal{typ: XsdBoolean, val: fmt.Sprint(bl)}, 180 | } 181 | } 182 | 183 | func (b *tripleBuilder) BooleanLiteral(bl bool) *triple { 184 | return &triple{ 185 | isSubBnode: b.isSubBnode, 186 | sub: b.sub, 187 | pred: b.pred, 188 | obj: BooleanLiteral(bl).(object), 189 | } 190 | } 191 | 192 | func ParseBoolean(obj Object) (bool, error) { 193 | if lit, ok := obj.Literal(); ok { 194 | if lit.Type() != XsdBoolean { 195 | return false, fmt.Errorf("literal is not an %s but %s", XsdBoolean, lit.Type()) 196 | } 197 | 198 | return strconv.ParseBool(lit.Value()) 199 | } 200 | 201 | return false, fmt.Errorf("cannot parse %s: object is not literal", XsdBoolean) 202 | } 203 | 204 | func IntegerLiteral(i int) Object { 205 | return object{ 206 | isLit: true, 207 | lit: literal{typ: XsdInteger, val: fmt.Sprint(i)}, 208 | } 209 | } 210 | 211 | func (b *tripleBuilder) IntegerLiteral(i int) *triple { 212 | return &triple{ 213 | isSubBnode: b.isSubBnode, 214 | sub: b.sub, 215 | pred: b.pred, 216 | obj: IntegerLiteral(i).(object), 217 | } 218 | } 219 | 220 | func ParseInteger(obj Object) (int, error) { 221 | if lit, ok := obj.Literal(); ok { 222 | if lit.Type() != XsdInteger { 223 | return 0, fmt.Errorf("literal is not an %s but %s", XsdInteger, lit.Type()) 224 | } 225 | 226 | return strconv.Atoi(lit.Value()) 227 | } 228 | 229 | return 0, fmt.Errorf("cannot parse %s: object is not literal", XsdInteger) 230 | } 231 | 232 | func Int8Literal(i int8) Object { 233 | return object{ 234 | isLit: true, 235 | lit: literal{typ: XsdByte, val: fmt.Sprint(i)}, 236 | } 237 | } 238 | 239 | func (b *tripleBuilder) Int8Literal(i int8) *triple { 240 | return &triple{ 241 | isSubBnode: b.isSubBnode, 242 | sub: b.sub, 243 | pred: b.pred, 244 | obj: Int8Literal(i).(object), 245 | } 246 | } 247 | 248 | func ParseInt8(obj Object) (int8, error) { 249 | if lit, ok := obj.Literal(); ok { 250 | if lit.Type() != XsdByte { 251 | return 0, fmt.Errorf("literal is not an %s but %s", XsdByte, lit.Type()) 252 | } 253 | 254 | num, err := strconv.ParseInt(lit.Value(), 10, 8) 255 | if err != nil { 256 | return 0, err 257 | } 258 | return int8(num), nil 259 | } 260 | 261 | return 0, fmt.Errorf("cannot parse %s: object is not literal", XsdByte) 262 | } 263 | 264 | func Int16Literal(i int16) Object { 265 | return object{ 266 | isLit: true, 267 | lit: literal{typ: XsdShort, val: fmt.Sprint(i)}, 268 | } 269 | } 270 | 271 | func (b *tripleBuilder) Int16Literal(i int16) *triple { 272 | return &triple{ 273 | isSubBnode: b.isSubBnode, 274 | sub: b.sub, 275 | pred: b.pred, 276 | obj: Int16Literal(i).(object), 277 | } 278 | } 279 | 280 | func ParseInt16(obj Object) (int16, error) { 281 | if lit, ok := obj.Literal(); ok { 282 | if lit.Type() != XsdShort { 283 | return 0, fmt.Errorf("literal is not an %s but %s", XsdShort, lit.Type()) 284 | } 285 | 286 | num, err := strconv.ParseInt(lit.Value(), 10, 16) 287 | if err != nil { 288 | return 0, err 289 | } 290 | return int16(num), nil 291 | } 292 | 293 | return 0, fmt.Errorf("cannot parse %s: object is not literal", XsdShort) 294 | } 295 | 296 | func UintegerLiteral(i uint) Object { 297 | return object{ 298 | isLit: true, 299 | lit: literal{typ: XsdUinteger, val: fmt.Sprint(i)}, 300 | } 301 | } 302 | 303 | func (b *tripleBuilder) UintegerLiteral(i uint) *triple { 304 | return &triple{ 305 | isSubBnode: b.isSubBnode, 306 | sub: b.sub, 307 | pred: b.pred, 308 | obj: UintegerLiteral(i).(object), 309 | } 310 | } 311 | 312 | func ParseUinteger(obj Object) (uint, error) { 313 | if lit, ok := obj.Literal(); ok { 314 | if lit.Type() != XsdUinteger { 315 | return 0, fmt.Errorf("literal is not an %s but %s", XsdUinteger, lit.Type()) 316 | } 317 | 318 | num, err := strconv.ParseUint(lit.Value(), 10, 64) 319 | if err != nil { 320 | return 0, err 321 | } 322 | return uint(num), nil 323 | } 324 | 325 | return 0, fmt.Errorf("cannot parse %s: object is not literal", XsdUinteger) 326 | } 327 | 328 | func Uint8Literal(i uint8) Object { 329 | return object{ 330 | isLit: true, 331 | lit: literal{typ: XsdUnsignedByte, val: fmt.Sprint(i)}, 332 | } 333 | } 334 | 335 | func (b *tripleBuilder) Uint8(i uint8) *triple { 336 | return &triple{ 337 | isSubBnode: b.isSubBnode, 338 | sub: b.sub, 339 | pred: b.pred, 340 | obj: Uint8Literal(i).(object), 341 | } 342 | } 343 | 344 | func ParseUint8(obj Object) (uint8, error) { 345 | if lit, ok := obj.Literal(); ok { 346 | if lit.Type() != XsdUnsignedByte { 347 | return 0, fmt.Errorf("literal is not an %s but %s", XsdUnsignedByte, lit.Type()) 348 | } 349 | 350 | num, err := strconv.ParseUint(lit.Value(), 10, 8) 351 | if err != nil { 352 | return 0, err 353 | } 354 | return uint8(num), nil 355 | } 356 | 357 | return 0, fmt.Errorf("cannot parse %s: object is not literal", XsdUnsignedByte) 358 | } 359 | 360 | func Uint16Literal(i uint16) Object { 361 | return object{ 362 | isLit: true, 363 | lit: literal{typ: XsdUnsignedShort, val: fmt.Sprint(i)}, 364 | } 365 | } 366 | 367 | func (b *tripleBuilder) Uint16(i uint16) *triple { 368 | return &triple{ 369 | isSubBnode: b.isSubBnode, 370 | sub: b.sub, 371 | pred: b.pred, 372 | obj: Uint16Literal(i).(object), 373 | } 374 | } 375 | 376 | func ParseUint16(obj Object) (uint16, error) { 377 | if lit, ok := obj.Literal(); ok { 378 | if lit.Type() != XsdUnsignedShort { 379 | return 0, fmt.Errorf("literal is not an %s but %s", XsdUnsignedShort, lit.Type()) 380 | } 381 | 382 | num, err := strconv.ParseUint(lit.Value(), 10, 16) 383 | if err != nil { 384 | return 0, err 385 | } 386 | return uint16(num), nil 387 | } 388 | 389 | return 0, fmt.Errorf("cannot parse %s: object is not literal", XsdUnsignedShort) 390 | } 391 | 392 | func Float64Literal(i float64) Object { 393 | return object{ 394 | isLit: true, 395 | lit: literal{typ: XsdDouble, val: fmt.Sprint(i)}, 396 | } 397 | } 398 | 399 | func (b *tripleBuilder) Float64Literal(i float64) *triple { 400 | return &triple{ 401 | isSubBnode: b.isSubBnode, 402 | sub: b.sub, 403 | pred: b.pred, 404 | obj: Float64Literal(i).(object), 405 | } 406 | } 407 | 408 | func ParseFloat64(obj Object) (float64, error) { 409 | if lit, ok := obj.Literal(); ok { 410 | if lit.Type() != XsdDouble { 411 | return 0, fmt.Errorf("literal is not an %s but %s", XsdDouble, lit.Type()) 412 | } 413 | 414 | return strconv.ParseFloat(lit.Value(), 64) 415 | } 416 | 417 | return 0, fmt.Errorf("cannot parse %s: object is not literal", XsdDouble) 418 | } 419 | 420 | func Float32Literal(i float32) Object { 421 | return object{ 422 | isLit: true, 423 | lit: literal{typ: XsdFloat, val: fmt.Sprint(i)}, 424 | } 425 | } 426 | 427 | func (b *tripleBuilder) Float32Literal(i float32) *triple { 428 | return &triple{ 429 | isSubBnode: b.isSubBnode, 430 | sub: b.sub, 431 | pred: b.pred, 432 | obj: Float32Literal(i).(object), 433 | } 434 | } 435 | 436 | func ParseFloat32(obj Object) (float32, error) { 437 | if lit, ok := obj.Literal(); ok { 438 | if lit.Type() != XsdFloat { 439 | return 0, fmt.Errorf("literal is not an %s but %s", XsdFloat, lit.Type()) 440 | } 441 | 442 | conv, err := strconv.ParseFloat(lit.Value(), 32) 443 | if err != nil { 444 | return 0, err 445 | } 446 | return float32(conv), nil 447 | } 448 | 449 | return 0, fmt.Errorf("cannot parse %s: object is not literal", XsdFloat) 450 | } 451 | 452 | func StringLiteral(s string) Object { 453 | return object{ 454 | isLit: true, 455 | lit: literal{typ: XsdString, val: s}, 456 | } 457 | } 458 | 459 | func StringLiteralWithLang(s, l string) Object { 460 | return object{ 461 | isLit: true, 462 | lit: literal{typ: XsdString, val: s, langtag: l}, 463 | } 464 | } 465 | 466 | func (b *tripleBuilder) StringLiteral(s string) *triple { 467 | return &triple{ 468 | isSubBnode: b.isSubBnode, 469 | sub: b.sub, 470 | pred: b.pred, 471 | obj: StringLiteral(s).(object), 472 | } 473 | } 474 | 475 | func (b *tripleBuilder) StringLiteralWithLang(s, l string) *triple { 476 | return &triple{ 477 | isSubBnode: b.isSubBnode, 478 | sub: b.sub, 479 | pred: b.pred, 480 | obj: StringLiteralWithLang(s, l).(object), 481 | } 482 | } 483 | 484 | func ParseString(obj Object) (string, error) { 485 | if lit, ok := obj.Literal(); ok { 486 | if lit.Type() != XsdString { 487 | return "", fmt.Errorf("literal is not a %s but %s", XsdString, lit.Type()) 488 | } 489 | 490 | return lit.Value(), nil 491 | } 492 | 493 | return "", fmt.Errorf("cannot parse %s: object is not literal", XsdString) 494 | } 495 | 496 | func DateTimeLiteral(tm time.Time) Object { 497 | text, err := tm.UTC().MarshalText() 498 | if err != nil { 499 | panic(fmt.Errorf("date time literal: %s", err)) 500 | } 501 | 502 | return object{ 503 | isLit: true, 504 | lit: literal{typ: XsdDateTime, val: string(text)}, 505 | } 506 | } 507 | 508 | func (b *tripleBuilder) DateTimeLiteral(tm time.Time) *triple { 509 | return &triple{ 510 | isSubBnode: b.isSubBnode, 511 | sub: b.sub, 512 | pred: b.pred, 513 | obj: DateTimeLiteral(tm).(object), 514 | } 515 | } 516 | 517 | func ParseDateTime(obj Object) (time.Time, error) { 518 | var t time.Time 519 | if lit, ok := obj.Literal(); ok { 520 | if lit.Type() != XsdDateTime { 521 | return t, fmt.Errorf("literal is not an %s but %s", XsdDateTime, lit.Type()) 522 | } 523 | 524 | err := t.UnmarshalText([]byte(lit.Value())) 525 | if err != nil { 526 | return t, err 527 | } 528 | return t, nil 529 | } 530 | 531 | return t, fmt.Errorf("cannot parse %s: object is not literal", XsdDateTime) 532 | } 533 | -------------------------------------------------------------------------------- /testdata/bench/decode_1.bin: -------------------------------------------------------------------------------- 1 | 0digit xsd:integer01digit xsd:integer12digit xsd:integer23digit xsd:integer34digit xsd:integer45digit xsd:integer56digit xsd:integer67digit xsd:integer78digit xsd:integer89digit xsd:integer910digit xsd:integer1011digit xsd:integer1112digit xsd:integer1213digit xsd:integer1314digit xsd:integer1415digit xsd:integer1516digit xsd:integer1617digit xsd:integer1718digit xsd:integer1819digit xsd:integer1920digit xsd:integer2021digit xsd:integer2122digit xsd:integer2223digit xsd:integer2324digit xsd:integer2425digit xsd:integer2526digit xsd:integer2627digit xsd:integer2728digit xsd:integer2829digit xsd:integer2930digit xsd:integer3031digit xsd:integer3132digit xsd:integer3233digit xsd:integer3334digit xsd:integer3435digit xsd:integer3536digit xsd:integer3637digit xsd:integer3738digit xsd:integer3839digit xsd:integer3940digit xsd:integer4041digit xsd:integer4142digit xsd:integer4243digit xsd:integer4344digit xsd:integer4445digit xsd:integer4546digit xsd:integer4647digit xsd:integer4748digit xsd:integer4849digit xsd:integer4950digit xsd:integer5051digit xsd:integer5152digit xsd:integer5253digit xsd:integer5354digit xsd:integer5455digit xsd:integer5556digit xsd:integer5657digit xsd:integer5758digit xsd:integer5859digit xsd:integer5960digit xsd:integer6061digit xsd:integer6162digit xsd:integer6263digit xsd:integer6364digit xsd:integer6465digit xsd:integer6566digit xsd:integer6667digit xsd:integer6768digit xsd:integer6869digit xsd:integer6970digit xsd:integer7071digit xsd:integer7172digit xsd:integer7273digit xsd:integer7374digit xsd:integer7475digit xsd:integer7576digit xsd:integer7677digit xsd:integer7778digit xsd:integer7879digit xsd:integer7980digit xsd:integer8081digit xsd:integer8182digit xsd:integer8283digit xsd:integer8384digit xsd:integer8485digit xsd:integer8586digit xsd:integer8687digit xsd:integer8788digit xsd:integer8889digit xsd:integer8990digit xsd:integer9091digit xsd:integer9192digit xsd:integer9293digit xsd:integer9394digit xsd:integer9495digit xsd:integer9596digit xsd:integer9697digit xsd:integer9798digit xsd:integer9899digit xsd:integer99100digit xsd:integer100101digit xsd:integer101102digit xsd:integer102103digit xsd:integer103104digit xsd:integer104105digit xsd:integer105106digit xsd:integer106107digit xsd:integer107108digit xsd:integer108109digit xsd:integer109110digit xsd:integer110111digit xsd:integer111112digit xsd:integer112113digit xsd:integer113114digit xsd:integer114115digit xsd:integer115116digit xsd:integer116117digit xsd:integer117118digit xsd:integer118119digit xsd:integer119120digit xsd:integer120121digit xsd:integer121122digit xsd:integer122123digit xsd:integer123124digit xsd:integer124125digit xsd:integer125126digit xsd:integer126127digit xsd:integer127128digit xsd:integer128129digit xsd:integer129130digit xsd:integer130131digit xsd:integer131132digit xsd:integer132133digit xsd:integer133134digit xsd:integer134135digit xsd:integer135136digit xsd:integer136137digit xsd:integer137138digit xsd:integer138139digit xsd:integer139140digit xsd:integer140141digit xsd:integer141142digit xsd:integer142143digit xsd:integer143144digit xsd:integer144145digit xsd:integer145146digit xsd:integer146147digit xsd:integer147148digit xsd:integer148149digit xsd:integer149150digit xsd:integer150151digit xsd:integer151152digit xsd:integer152153digit xsd:integer153154digit xsd:integer154155digit xsd:integer155156digit xsd:integer156157digit xsd:integer157158digit xsd:integer158159digit xsd:integer159160digit xsd:integer160161digit xsd:integer161162digit xsd:integer162163digit xsd:integer163164digit xsd:integer164165digit xsd:integer165166digit xsd:integer166167digit xsd:integer167168digit xsd:integer168169digit xsd:integer169170digit xsd:integer170171digit xsd:integer171172digit xsd:integer172173digit xsd:integer173174digit xsd:integer174175digit xsd:integer175176digit xsd:integer176177digit xsd:integer177178digit xsd:integer178179digit xsd:integer179180digit xsd:integer180181digit xsd:integer181182digit xsd:integer182183digit xsd:integer183184digit xsd:integer184185digit xsd:integer185186digit xsd:integer186187digit xsd:integer187188digit xsd:integer188189digit xsd:integer189190digit xsd:integer190191digit xsd:integer191192digit xsd:integer192193digit xsd:integer193194digit xsd:integer194195digit xsd:integer195196digit xsd:integer196197digit xsd:integer197198digit xsd:integer198199digit xsd:integer199200digit xsd:integer200201digit xsd:integer201202digit xsd:integer202203digit xsd:integer203204digit xsd:integer204205digit xsd:integer205206digit xsd:integer206207digit xsd:integer207208digit xsd:integer208209digit xsd:integer209210digit xsd:integer210211digit xsd:integer211212digit xsd:integer212213digit xsd:integer213214digit xsd:integer214215digit xsd:integer215216digit xsd:integer216217digit xsd:integer217218digit xsd:integer218219digit xsd:integer219220digit xsd:integer220221digit xsd:integer221222digit xsd:integer222223digit xsd:integer223224digit xsd:integer224225digit xsd:integer225226digit xsd:integer226227digit xsd:integer227228digit xsd:integer228229digit xsd:integer229230digit xsd:integer230231digit xsd:integer231232digit xsd:integer232233digit xsd:integer233234digit xsd:integer234235digit xsd:integer235236digit xsd:integer236237digit xsd:integer237238digit xsd:integer238239digit xsd:integer239240digit xsd:integer240241digit xsd:integer241242digit xsd:integer242243digit xsd:integer243244digit xsd:integer244245digit xsd:integer245246digit xsd:integer246247digit xsd:integer247248digit xsd:integer248249digit xsd:integer249250digit xsd:integer250251digit xsd:integer251252digit xsd:integer252253digit xsd:integer253254digit xsd:integer254255digit xsd:integer255256digit xsd:integer256257digit xsd:integer257258digit xsd:integer258259digit xsd:integer259260digit xsd:integer260261digit xsd:integer261262digit xsd:integer262263digit xsd:integer263264digit xsd:integer264265digit xsd:integer265266digit xsd:integer266267digit xsd:integer267268digit xsd:integer268269digit xsd:integer269270digit xsd:integer270271digit xsd:integer271272digit xsd:integer272273digit xsd:integer273274digit xsd:integer274275digit xsd:integer275276digit xsd:integer276277digit xsd:integer277278digit xsd:integer278279digit xsd:integer279280digit xsd:integer280281digit xsd:integer281282digit xsd:integer282283digit xsd:integer283284digit xsd:integer284285digit xsd:integer285286digit xsd:integer286287digit xsd:integer287288digit xsd:integer288289digit xsd:integer289290digit xsd:integer290291digit xsd:integer291292digit xsd:integer292293digit xsd:integer293294digit xsd:integer294295digit xsd:integer295296digit xsd:integer296297digit xsd:integer297298digit xsd:integer298299digit xsd:integer299300digit xsd:integer300301digit xsd:integer301302digit xsd:integer302303digit xsd:integer303304digit xsd:integer304305digit xsd:integer305306digit xsd:integer306307digit xsd:integer307308digit xsd:integer308309digit xsd:integer309310digit xsd:integer310311digit xsd:integer311312digit xsd:integer312313digit xsd:integer313314digit xsd:integer314315digit xsd:integer315316digit xsd:integer316317digit xsd:integer317318digit xsd:integer318319digit xsd:integer319320digit xsd:integer320321digit xsd:integer321322digit xsd:integer322323digit xsd:integer323324digit xsd:integer324325digit xsd:integer325326digit xsd:integer326327digit xsd:integer327328digit xsd:integer328329digit xsd:integer329330digit xsd:integer330331digit xsd:integer331332digit xsd:integer332333digit xsd:integer333334digit xsd:integer334335digit xsd:integer335336digit xsd:integer336337digit xsd:integer337338digit xsd:integer338339digit xsd:integer339340digit xsd:integer340341digit xsd:integer341342digit xsd:integer342343digit xsd:integer343344digit xsd:integer344345digit xsd:integer345346digit xsd:integer346347digit xsd:integer347348digit xsd:integer348349digit xsd:integer349350digit xsd:integer350351digit xsd:integer351352digit xsd:integer352353digit xsd:integer353354digit xsd:integer354355digit xsd:integer355356digit xsd:integer356357digit xsd:integer357358digit xsd:integer358359digit xsd:integer359360digit xsd:integer360361digit xsd:integer361362digit xsd:integer362363digit xsd:integer363364digit xsd:integer364365digit xsd:integer365366digit xsd:integer366367digit xsd:integer367368digit xsd:integer368369digit xsd:integer369370digit xsd:integer370371digit xsd:integer371372digit xsd:integer372373digit xsd:integer373374digit xsd:integer374375digit xsd:integer375376digit xsd:integer376377digit xsd:integer377378digit xsd:integer378379digit xsd:integer379380digit xsd:integer380381digit xsd:integer381382digit xsd:integer382383digit xsd:integer383384digit xsd:integer384385digit xsd:integer385386digit xsd:integer386387digit xsd:integer387388digit xsd:integer388389digit xsd:integer389390digit xsd:integer390391digit xsd:integer391392digit xsd:integer392393digit xsd:integer393394digit xsd:integer394395digit xsd:integer395396digit xsd:integer396397digit xsd:integer397398digit xsd:integer398399digit xsd:integer399400digit xsd:integer400401digit xsd:integer401402digit xsd:integer402403digit xsd:integer403404digit xsd:integer404405digit xsd:integer405406digit xsd:integer406407digit xsd:integer407408digit xsd:integer408409digit xsd:integer409410digit xsd:integer410411digit xsd:integer411412digit xsd:integer412413digit xsd:integer413414digit xsd:integer414415digit xsd:integer415416digit xsd:integer416417digit xsd:integer417418digit xsd:integer418419digit xsd:integer419420digit xsd:integer420421digit xsd:integer421422digit xsd:integer422423digit xsd:integer423424digit xsd:integer424425digit xsd:integer425426digit xsd:integer426427digit xsd:integer427428digit xsd:integer428429digit xsd:integer429430digit xsd:integer430431digit xsd:integer431432digit xsd:integer432433digit xsd:integer433434digit xsd:integer434435digit xsd:integer435436digit xsd:integer436437digit xsd:integer437438digit xsd:integer438439digit xsd:integer439440digit xsd:integer440441digit xsd:integer441442digit xsd:integer442443digit xsd:integer443444digit xsd:integer444445digit xsd:integer445446digit xsd:integer446447digit xsd:integer447448digit xsd:integer448449digit xsd:integer449450digit xsd:integer450451digit xsd:integer451452digit xsd:integer452453digit xsd:integer453454digit xsd:integer454455digit xsd:integer455456digit xsd:integer456457digit xsd:integer457458digit xsd:integer458459digit xsd:integer459460digit xsd:integer460461digit xsd:integer461462digit xsd:integer462463digit xsd:integer463464digit xsd:integer464465digit xsd:integer465466digit xsd:integer466467digit xsd:integer467468digit xsd:integer468469digit xsd:integer469470digit xsd:integer470471digit xsd:integer471472digit xsd:integer472473digit xsd:integer473474digit xsd:integer474475digit xsd:integer475476digit xsd:integer476477digit xsd:integer477478digit xsd:integer478479digit xsd:integer479480digit xsd:integer480481digit xsd:integer481482digit xsd:integer482483digit xsd:integer483484digit xsd:integer484485digit xsd:integer485486digit xsd:integer486487digit xsd:integer487488digit xsd:integer488489digit xsd:integer489490digit xsd:integer490491digit xsd:integer491492digit xsd:integer492493digit xsd:integer493494digit xsd:integer494495digit xsd:integer495496digit xsd:integer496497digit xsd:integer497498digit xsd:integer498499digit xsd:integer499500digit xsd:integer500501digit xsd:integer501502digit xsd:integer502503digit xsd:integer503504digit xsd:integer504505digit xsd:integer505506digit xsd:integer506507digit xsd:integer507508digit xsd:integer508509digit xsd:integer509510digit xsd:integer510511digit xsd:integer511512digit xsd:integer512513digit xsd:integer513514digit xsd:integer514515digit xsd:integer515516digit xsd:integer516517digit xsd:integer517518digit xsd:integer518519digit xsd:integer519520digit xsd:integer520521digit xsd:integer521522digit xsd:integer522523digit xsd:integer523524digit xsd:integer524525digit xsd:integer525526digit xsd:integer526527digit xsd:integer527528digit xsd:integer528529digit xsd:integer529530digit xsd:integer530531digit xsd:integer531532digit xsd:integer532533digit xsd:integer533534digit xsd:integer534535digit xsd:integer535536digit xsd:integer536537digit xsd:integer537538digit xsd:integer538539digit xsd:integer539540digit xsd:integer540541digit xsd:integer541542digit xsd:integer542543digit xsd:integer543544digit xsd:integer544545digit xsd:integer545546digit xsd:integer546547digit xsd:integer547548digit xsd:integer548549digit xsd:integer549550digit xsd:integer550551digit xsd:integer551552digit xsd:integer552553digit xsd:integer553554digit xsd:integer554555digit xsd:integer555556digit xsd:integer556557digit xsd:integer557558digit xsd:integer558559digit xsd:integer559560digit xsd:integer560561digit xsd:integer561562digit xsd:integer562563digit xsd:integer563564digit xsd:integer564565digit xsd:integer565566digit xsd:integer566567digit xsd:integer567568digit xsd:integer568569digit xsd:integer569570digit xsd:integer570571digit xsd:integer571572digit xsd:integer572573digit xsd:integer573574digit xsd:integer574575digit xsd:integer575576digit xsd:integer576577digit xsd:integer577578digit xsd:integer578579digit xsd:integer579580digit xsd:integer580581digit xsd:integer581582digit xsd:integer582583digit xsd:integer583584digit xsd:integer584585digit xsd:integer585586digit xsd:integer586587digit xsd:integer587588digit xsd:integer588589digit xsd:integer589590digit xsd:integer590591digit xsd:integer591592digit xsd:integer592593digit xsd:integer593594digit xsd:integer594595digit xsd:integer595596digit xsd:integer596597digit xsd:integer597598digit xsd:integer598599digit xsd:integer599600digit xsd:integer600601digit xsd:integer601602digit xsd:integer602603digit xsd:integer603604digit xsd:integer604605digit xsd:integer605606digit xsd:integer606607digit xsd:integer607608digit xsd:integer608609digit xsd:integer609610digit xsd:integer610611digit xsd:integer611612digit xsd:integer612613digit xsd:integer613614digit xsd:integer614615digit xsd:integer615616digit xsd:integer616617digit xsd:integer617618digit xsd:integer618619digit xsd:integer619620digit xsd:integer620621digit xsd:integer621622digit xsd:integer622623digit xsd:integer623624digit xsd:integer624625digit xsd:integer625626digit xsd:integer626627digit xsd:integer627628digit xsd:integer628629digit xsd:integer629630digit xsd:integer630631digit xsd:integer631632digit xsd:integer632633digit xsd:integer633634digit xsd:integer634635digit xsd:integer635636digit xsd:integer636637digit xsd:integer637638digit xsd:integer638639digit xsd:integer639640digit xsd:integer640641digit xsd:integer641642digit xsd:integer642643digit xsd:integer643644digit xsd:integer644645digit xsd:integer645646digit xsd:integer646647digit xsd:integer647648digit xsd:integer648649digit xsd:integer649650digit xsd:integer650651digit xsd:integer651652digit xsd:integer652653digit xsd:integer653654digit xsd:integer654655digit xsd:integer655656digit xsd:integer656657digit xsd:integer657658digit xsd:integer658659digit xsd:integer659660digit xsd:integer660661digit xsd:integer661662digit xsd:integer662663digit xsd:integer663664digit xsd:integer664665digit xsd:integer665666digit xsd:integer666667digit xsd:integer667668digit xsd:integer668669digit xsd:integer669670digit xsd:integer670671digit xsd:integer671672digit xsd:integer672673digit xsd:integer673674digit xsd:integer674675digit xsd:integer675676digit xsd:integer676677digit xsd:integer677678digit xsd:integer678679digit xsd:integer679680digit xsd:integer680681digit xsd:integer681682digit xsd:integer682683digit xsd:integer683684digit xsd:integer684685digit xsd:integer685686digit xsd:integer686687digit xsd:integer687688digit xsd:integer688689digit xsd:integer689690digit xsd:integer690691digit xsd:integer691692digit xsd:integer692693digit xsd:integer693694digit xsd:integer694695digit xsd:integer695696digit xsd:integer696697digit xsd:integer697698digit xsd:integer698699digit xsd:integer699700digit xsd:integer700701digit xsd:integer701702digit xsd:integer702703digit xsd:integer703704digit xsd:integer704705digit xsd:integer705706digit xsd:integer706707digit xsd:integer707708digit xsd:integer708709digit xsd:integer709710digit xsd:integer710711digit xsd:integer711712digit xsd:integer712713digit xsd:integer713714digit xsd:integer714715digit xsd:integer715716digit xsd:integer716717digit xsd:integer717718digit xsd:integer718719digit xsd:integer719720digit xsd:integer720721digit xsd:integer721722digit xsd:integer722723digit xsd:integer723724digit xsd:integer724725digit xsd:integer725726digit xsd:integer726727digit xsd:integer727728digit xsd:integer728729digit xsd:integer729730digit xsd:integer730731digit xsd:integer731732digit xsd:integer732733digit xsd:integer733734digit xsd:integer734735digit xsd:integer735736digit xsd:integer736737digit xsd:integer737738digit xsd:integer738739digit xsd:integer739740digit xsd:integer740741digit xsd:integer741742digit xsd:integer742743digit xsd:integer743744digit xsd:integer744745digit xsd:integer745746digit xsd:integer746747digit xsd:integer747748digit xsd:integer748749digit xsd:integer749750digit xsd:integer750751digit xsd:integer751752digit xsd:integer752753digit xsd:integer753754digit xsd:integer754755digit xsd:integer755756digit xsd:integer756757digit xsd:integer757758digit xsd:integer758759digit xsd:integer759760digit xsd:integer760761digit xsd:integer761762digit xsd:integer762763digit xsd:integer763764digit xsd:integer764765digit xsd:integer765766digit xsd:integer766767digit xsd:integer767768digit xsd:integer768769digit xsd:integer769770digit xsd:integer770771digit xsd:integer771772digit xsd:integer772773digit xsd:integer773774digit xsd:integer774775digit xsd:integer775776digit xsd:integer776777digit xsd:integer777778digit xsd:integer778779digit xsd:integer779780digit xsd:integer780781digit xsd:integer781782digit xsd:integer782783digit xsd:integer783784digit xsd:integer784785digit xsd:integer785786digit xsd:integer786787digit xsd:integer787788digit xsd:integer788789digit xsd:integer789790digit xsd:integer790791digit xsd:integer791792digit xsd:integer792793digit xsd:integer793794digit xsd:integer794795digit xsd:integer795796digit xsd:integer796797digit xsd:integer797798digit xsd:integer798799digit xsd:integer799800digit xsd:integer800801digit xsd:integer801802digit xsd:integer802803digit xsd:integer803804digit xsd:integer804805digit xsd:integer805806digit xsd:integer806807digit xsd:integer807808digit xsd:integer808809digit xsd:integer809810digit xsd:integer810811digit xsd:integer811812digit xsd:integer812813digit xsd:integer813814digit xsd:integer814815digit xsd:integer815816digit xsd:integer816817digit xsd:integer817818digit xsd:integer818819digit xsd:integer819820digit xsd:integer820821digit xsd:integer821822digit xsd:integer822823digit xsd:integer823824digit xsd:integer824825digit xsd:integer825826digit xsd:integer826827digit xsd:integer827828digit xsd:integer828829digit xsd:integer829830digit xsd:integer830831digit xsd:integer831832digit xsd:integer832833digit xsd:integer833834digit xsd:integer834835digit xsd:integer835836digit xsd:integer836837digit xsd:integer837838digit xsd:integer838839digit xsd:integer839840digit xsd:integer840841digit xsd:integer841842digit xsd:integer842843digit xsd:integer843844digit xsd:integer844845digit xsd:integer845846digit xsd:integer846847digit xsd:integer847848digit xsd:integer848849digit xsd:integer849850digit xsd:integer850851digit xsd:integer851852digit xsd:integer852853digit xsd:integer853854digit xsd:integer854855digit xsd:integer855856digit xsd:integer856857digit xsd:integer857858digit xsd:integer858859digit xsd:integer859860digit xsd:integer860861digit xsd:integer861862digit xsd:integer862863digit xsd:integer863864digit xsd:integer864865digit xsd:integer865866digit xsd:integer866867digit xsd:integer867868digit xsd:integer868869digit xsd:integer869870digit xsd:integer870871digit xsd:integer871872digit xsd:integer872873digit xsd:integer873874digit xsd:integer874875digit xsd:integer875876digit xsd:integer876877digit xsd:integer877878digit xsd:integer878879digit xsd:integer879880digit xsd:integer880881digit xsd:integer881882digit xsd:integer882883digit xsd:integer883884digit xsd:integer884885digit xsd:integer885886digit xsd:integer886887digit xsd:integer887888digit xsd:integer888889digit xsd:integer889890digit xsd:integer890891digit xsd:integer891892digit xsd:integer892893digit xsd:integer893894digit xsd:integer894895digit xsd:integer895896digit xsd:integer896897digit xsd:integer897898digit xsd:integer898899digit xsd:integer899900digit xsd:integer900901digit xsd:integer901902digit xsd:integer902903digit xsd:integer903904digit xsd:integer904905digit xsd:integer905906digit xsd:integer906907digit xsd:integer907908digit xsd:integer908909digit xsd:integer909910digit xsd:integer910911digit xsd:integer911912digit xsd:integer912913digit xsd:integer913914digit xsd:integer914915digit xsd:integer915916digit xsd:integer916917digit xsd:integer917918digit xsd:integer918919digit xsd:integer919920digit xsd:integer920921digit xsd:integer921922digit xsd:integer922923digit xsd:integer923924digit xsd:integer924925digit xsd:integer925926digit xsd:integer926927digit xsd:integer927928digit xsd:integer928929digit xsd:integer929930digit xsd:integer930931digit xsd:integer931932digit xsd:integer932933digit xsd:integer933934digit xsd:integer934935digit xsd:integer935936digit xsd:integer936937digit xsd:integer937938digit xsd:integer938939digit xsd:integer939940digit xsd:integer940941digit xsd:integer941942digit xsd:integer942943digit xsd:integer943944digit xsd:integer944945digit xsd:integer945946digit xsd:integer946947digit xsd:integer947948digit xsd:integer948949digit xsd:integer949950digit xsd:integer950951digit xsd:integer951952digit xsd:integer952953digit xsd:integer953954digit xsd:integer954955digit xsd:integer955956digit xsd:integer956957digit xsd:integer957958digit xsd:integer958959digit xsd:integer959960digit xsd:integer960961digit xsd:integer961962digit xsd:integer962963digit xsd:integer963964digit xsd:integer964965digit xsd:integer965966digit xsd:integer966967digit xsd:integer967968digit xsd:integer968969digit xsd:integer969970digit xsd:integer970971digit xsd:integer971972digit xsd:integer972973digit xsd:integer973974digit xsd:integer974975digit xsd:integer975976digit xsd:integer976977digit xsd:integer977978digit xsd:integer978979digit xsd:integer979980digit xsd:integer980981digit xsd:integer981982digit xsd:integer982983digit xsd:integer983984digit xsd:integer984985digit xsd:integer985986digit xsd:integer986987digit xsd:integer987988digit xsd:integer988989digit xsd:integer989990digit xsd:integer990991digit xsd:integer991992digit xsd:integer992993digit xsd:integer993994digit xsd:integer994995digit xsd:integer995996digit xsd:integer996997digit xsd:integer997998digit xsd:integer998999digit xsd:integer999 --------------------------------------------------------------------------------