├── .github └── workflows │ └── tests.yml ├── .gitignore ├── CODEOWNERS ├── LICENSE ├── README.md ├── dot ├── dot.go └── dot_test.go ├── formats.go ├── gml ├── gml.go └── gml_test.go ├── go.mod ├── go.sum ├── graphml ├── graphml.go └── graphml_test.go ├── json ├── json.go └── json_test.go ├── jsonld ├── jsonld.go └── jsonld_test.go ├── nquad_tests.tar.gz ├── nquads ├── nquads.go ├── nquads.rl ├── raw.go ├── raw.rl ├── raw_test.go ├── typed.go ├── typed.rl └── typed_test.go ├── ntriple_tests.tar.gz ├── pquads ├── pio │ ├── io.go │ ├── io_test.go │ └── varint.go ├── pquads.go ├── pquads_test.go ├── quads.go ├── quads.pb.go └── quads.proto ├── quad.go ├── rw.go ├── value.go ├── value_test.go └── voc ├── core └── all.go ├── owl └── owl.go ├── rdf └── rdf.go ├── rdfs └── rdfs.go ├── schema └── schema.go ├── voc.go ├── voc_test.go └── xsd └── xsd.go /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: [ "master" ] 6 | pull_request: 7 | branches: [ "master" ] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Set up Go 17 | uses: actions/setup-go@v5 18 | with: 19 | go-version: '1.21' 20 | 21 | - name: Test 22 | run: go test -v ./... 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | main 3 | *.test 4 | *.peg.go 5 | .DS_Store 6 | 7 | vendor/ 8 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @dennwc -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Quad formats for Go 2 | 3 | ![Tests](https://github.com/cayleygraph/quad/actions/workflows/tests.yml/badge.svg) 4 | 5 | This library provides encoding and decoding support for NQuad/NTriple-compatible formats. 6 | 7 | ## Supported formats 8 | 9 | | ID | Name | Read | Write | Ext | 10 | |---------------|--------------|------|-------|---------------| 11 | | `nquads` | NQuads | + | + | `.nq`, `.nt` | 12 | | `jsonld` | JSON-LD | + | + | `.jsonld` | 13 | | `graphviz` | DOT/Graphviz | - | + | `.gv`, `.dot` | 14 | | `gml` | GML | - | + | `.gml` | 15 | | `graphml` | GraphML | - | + | `.graphml` | 16 | | `pquads` | ProtoQuads | + | + | `.pq` | 17 | | `json` | JSON | + | + | `.json` | 18 | | `json-stream` | JSON Stream | + | + | - | 19 | 20 | ## Community 21 | 22 | * Slack: [cayleygraph.slack.com](https://cayleygraph.slack.com) -- Invite [here](https://cayley-slackin.herokuapp.com/) 23 | -------------------------------------------------------------------------------- /dot/dot.go: -------------------------------------------------------------------------------- 1 | // Package dot provides an encoder for DOT format (graphviz). 2 | package dot 3 | 4 | import ( 5 | "fmt" 6 | "io" 7 | "strings" 8 | 9 | "github.com/cayleygraph/quad" 10 | ) 11 | 12 | func init() { 13 | quad.RegisterFormat(quad.Format{ 14 | Name: "graphviz", 15 | Ext: []string{".gv", ".dot"}, 16 | Writer: func(w io.Writer) quad.WriteCloser { return NewWriter(w) }, 17 | }) 18 | } 19 | 20 | func NewWriter(w io.Writer) *Writer { 21 | return &Writer{w: w} 22 | } 23 | 24 | type Writer struct { 25 | w io.Writer 26 | written bool 27 | err error 28 | } 29 | 30 | var escaper = strings.NewReplacer( 31 | `\`, `\\`, 32 | `"`, `\"`, 33 | ) 34 | 35 | func escape(s string) string { 36 | return `"` + escaper.Replace(s) + `"` 37 | } 38 | 39 | func (w *Writer) writeString(s string) { 40 | if w.err != nil { 41 | return 42 | } 43 | _, w.err = w.w.Write([]byte(s)) 44 | } 45 | 46 | func (w *Writer) WriteQuad(q quad.Quad) error { 47 | if w.err != nil { 48 | return w.err 49 | } else if !q.IsValid() { 50 | return quad.ErrInvalid 51 | } 52 | if !w.written { 53 | if _, err := w.w.Write([]byte(header)); err != nil { 54 | return err 55 | } 56 | w.written = true 57 | } 58 | // TODO: use label 59 | w.writeString("\t") 60 | w.writeString(escape(q.Subject.String())) 61 | w.writeString(" -> ") 62 | w.writeString(escape(q.Object.String())) 63 | w.writeString(" [ label = ") 64 | w.writeString(escape(q.Predicate.String())) 65 | w.writeString(" ];\n") 66 | return w.err 67 | } 68 | 69 | func (w *Writer) WriteQuads(buf []quad.Quad) (int, error) { 70 | for i, q := range buf { 71 | if err := w.WriteQuad(q); err != nil { 72 | return i, err 73 | } 74 | } 75 | return len(buf), nil 76 | } 77 | 78 | func (w *Writer) Close() error { 79 | if w.err != nil { 80 | return w.err 81 | } 82 | if !w.written { 83 | if _, w.err = w.w.Write([]byte(header)); w.err != nil { 84 | return w.err 85 | } 86 | } 87 | if _, w.err = w.w.Write([]byte(footer)); w.err != nil { 88 | return w.err 89 | } 90 | w.err = fmt.Errorf("closed") 91 | return nil 92 | } 93 | 94 | const header = `digraph cayley_graph { 95 | ` 96 | const footer = "}\n" 97 | -------------------------------------------------------------------------------- /dot/dot_test.go: -------------------------------------------------------------------------------- 1 | package dot_test 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/cayleygraph/quad" 8 | "github.com/cayleygraph/quad/dot" 9 | ) 10 | 11 | var testData = []struct { 12 | quads []quad.Quad 13 | data string 14 | }{ 15 | { 16 | []quad.Quad{ 17 | { 18 | Subject: quad.BNode("subject1"), 19 | Predicate: quad.IRI("/film/performance/character"), 20 | Object: quad.String("Tomas de Torquemada"), 21 | Label: nil, 22 | }, 23 | { 24 | Subject: quad.BNode("subject1"), 25 | Predicate: quad.IRI("http://an.example/predicate1"), 26 | Object: quad.String("object1"), 27 | Label: nil, 28 | }, 29 | { 30 | Subject: quad.IRI("http://example.org/bob#me"), 31 | Predicate: quad.IRI("http://schema.org/birthDate"), 32 | Object: quad.TypedString{ 33 | Value: "1990-07-04", 34 | Type: "http://www.w3.org/2001/XMLSchema#date", 35 | }, 36 | Label: nil, 37 | }, 38 | { 39 | Subject: quad.BNode("subject1"), 40 | Predicate: quad.IRI("text"), 41 | Object: quad.String(`some "text"`), 42 | Label: nil, 43 | }, 44 | }, 45 | `digraph cayley_graph { 46 | "_:subject1" -> "\"Tomas de Torquemada\"" [ label = "" ]; 47 | "_:subject1" -> "\"object1\"" [ label = "" ]; 48 | "" -> "\"1990-07-04\"^^" [ label = "" ]; 49 | "_:subject1" -> "\"some \\\"text\\\"\"" [ label = "" ]; 50 | } 51 | `, 52 | }, 53 | } 54 | 55 | func TestWriter(t *testing.T) { 56 | buf := bytes.NewBuffer(nil) 57 | for _, c := range testData { 58 | buf.Reset() 59 | w := dot.NewWriter(buf) 60 | n, err := quad.Copy(w, quad.NewReader(c.quads)) 61 | if err != nil { 62 | t.Fatalf("write failed after %d quads: %v", n, err) 63 | } 64 | if err = w.Close(); err != nil { 65 | t.Fatal("error on close:", err) 66 | } 67 | if c.data != buf.String() { 68 | t.Fatalf("wrong output:\n%s\n\nvs\n\n%s", buf.String(), c.data) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /formats.go: -------------------------------------------------------------------------------- 1 | package quad 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | ) 7 | 8 | // Format is a description for quad-file formats. 9 | type Format struct { 10 | // Name is a short format name used as identifier for RegisterFormat. 11 | Name string 12 | // Ext is a list of file extensions, allowed for file format. Can be used to detect file format, given a path. 13 | Ext []string 14 | // Mime is a list of MIME (content) types, allowed for file format. Can be used in HTTP request/responses. 15 | Mime []string 16 | // Reader is a function for creating format reader, that reads serialized data from io.Reader. 17 | Reader func(io.Reader) ReadCloser 18 | // Writer is a function for creating format writer, that streams serialized data to io.Writer. 19 | Writer func(io.Writer) WriteCloser 20 | // Binary is set to true if format is not human-readable. 21 | Binary bool 22 | // MarshalValue encodes one value in specific a format. 23 | MarshalValue func(v Value) ([]byte, error) 24 | // UnmarshalValue decodes a value from specific format. 25 | UnmarshalValue func(b []byte) (Value, error) 26 | } 27 | 28 | var ( 29 | formatsByName = make(map[string]*Format) 30 | formatsByExt = make(map[string]*Format) 31 | formatsByMime = make(map[string]*Format) 32 | ) 33 | 34 | // RegisterFormat registers a new quad-file format. 35 | func RegisterFormat(f Format) { 36 | if _, ok := formatsByName[f.Name]; ok { 37 | panic(fmt.Errorf("format %s is allready registered", f.Name)) 38 | } 39 | formatsByName[f.Name] = &f 40 | for _, m := range f.Ext { 41 | if sf, ok := formatsByExt[m]; ok { 42 | panic(fmt.Errorf("format %s is allready registered with MIME %s", sf.Name, m)) 43 | } 44 | formatsByExt[m] = &f 45 | } 46 | for _, m := range f.Mime { 47 | if sf, ok := formatsByMime[m]; ok { 48 | panic(fmt.Errorf("format %s is allready registered with MIME %s", sf.Name, m)) 49 | } 50 | formatsByMime[m] = &f 51 | } 52 | } 53 | 54 | // FormatByName returns a registered format by its name. 55 | // Will return nil if format is not found. 56 | func FormatByName(name string) *Format { 57 | return formatsByName[name] 58 | } 59 | 60 | // FormatByExt returns a registered format by its file extension. 61 | // Will return nil if format is not found. 62 | func FormatByExt(name string) *Format { 63 | return formatsByExt[name] 64 | } 65 | 66 | // FormatByMime returns a registered format by its MIME type. 67 | // Will return nil if format is not found. 68 | func FormatByMime(name string) *Format { 69 | return formatsByMime[name] 70 | } 71 | 72 | // Formats returns a list of all supported quad formats. 73 | func Formats() []Format { 74 | list := make([]Format, 0, len(formatsByName)) 75 | for _, f := range formatsByName { 76 | list = append(list, *f) 77 | } 78 | return list 79 | } 80 | -------------------------------------------------------------------------------- /gml/gml.go: -------------------------------------------------------------------------------- 1 | // Package gml provides an encoder for Graph Modeling Format 2 | package gml 3 | 4 | import ( 5 | "fmt" 6 | "io" 7 | "strings" 8 | 9 | "github.com/cayleygraph/quad" 10 | ) 11 | 12 | func init() { 13 | quad.RegisterFormat(quad.Format{ 14 | Name: "gml", 15 | Ext: []string{".gml"}, 16 | Writer: func(w io.Writer) quad.WriteCloser { return NewWriter(w) }, 17 | }) 18 | } 19 | 20 | func NewWriter(w io.Writer) *Writer { 21 | return &Writer{w: w} 22 | } 23 | 24 | type Writer struct { 25 | w io.Writer 26 | written bool 27 | err error 28 | 29 | nodes map[string]int 30 | cur int 31 | } 32 | 33 | func (w *Writer) writeNode(s string) int { 34 | if w.err != nil { 35 | return -1 36 | } 37 | i, ok := w.nodes[s] 38 | if ok { 39 | return i 40 | } 41 | i = w.cur 42 | w.cur++ 43 | w.nodes[s] = i 44 | _, w.err = fmt.Fprintf(w.w, "\tnode [ id %d label %s ]\n", i, escape(s)) 45 | if w.err != nil { 46 | return -1 47 | } 48 | return i 49 | } 50 | 51 | var escaper = strings.NewReplacer( // TODO: ISO 8859-1? 52 | `&`, `&`, 53 | `"`, `"`, 54 | 55 | // `<`,`<`, 56 | // `>`, `>`, 57 | ) 58 | 59 | func escape(s string) string { 60 | return `"` + escaper.Replace(s) + `"` 61 | } 62 | 63 | func (w *Writer) WriteQuad(q quad.Quad) error { 64 | if w.err != nil { 65 | return w.err 66 | } else if !q.IsValid() { 67 | return quad.ErrInvalid 68 | } 69 | if !w.written { 70 | if _, err := w.w.Write([]byte(header)); err != nil { 71 | return err 72 | } 73 | w.written = true 74 | w.nodes = make(map[string]int) 75 | } 76 | s := w.writeNode(q.Subject.String()) 77 | o := w.writeNode(q.Object.String()) 78 | if w.err != nil { 79 | return w.err 80 | } 81 | _, w.err = fmt.Fprintf(w.w, "\tedge [ source %d target %d label %s ]\n", 82 | s, o, escape(q.Predicate.String())) 83 | return w.err 84 | } 85 | 86 | func (w *Writer) WriteQuads(buf []quad.Quad) (int, error) { 87 | for i, q := range buf { 88 | if err := w.WriteQuad(q); err != nil { 89 | return i, err 90 | } 91 | } 92 | return len(buf), nil 93 | } 94 | 95 | func (w *Writer) Close() error { 96 | if w.err != nil { 97 | return w.err 98 | } 99 | if !w.written { 100 | if _, w.err = w.w.Write([]byte(header)); w.err != nil { 101 | return w.err 102 | } 103 | } 104 | if _, w.err = w.w.Write([]byte(footer)); w.err != nil { 105 | return w.err 106 | } 107 | w.err = fmt.Errorf("closed") 108 | return nil 109 | } 110 | 111 | const header = "Creator \"Cayley\"\ngraph [ directed 1\n" 112 | const footer = "]\n" 113 | -------------------------------------------------------------------------------- /gml/gml_test.go: -------------------------------------------------------------------------------- 1 | package gml_test 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/cayleygraph/quad" 8 | "github.com/cayleygraph/quad/gml" 9 | ) 10 | 11 | var testData = []struct { 12 | quads []quad.Quad 13 | data string 14 | }{ 15 | { 16 | []quad.Quad{ 17 | { 18 | Subject: quad.BNode("subject1"), 19 | Predicate: quad.IRI("/film/performance/character"), 20 | Object: quad.String("Tomas de Torquemada"), 21 | Label: nil, 22 | }, 23 | { 24 | Subject: quad.BNode("subject1"), 25 | Predicate: quad.IRI("http://an.example/predicate1"), 26 | Object: quad.String("object1"), 27 | Label: nil, 28 | }, 29 | { 30 | Subject: quad.IRI("http://example.org/bob#me"), 31 | Predicate: quad.IRI("http://schema.org/birthDate"), 32 | Object: quad.TypedString{ 33 | Value: "1990-07-04", 34 | Type: "http://www.w3.org/2001/XMLSchema#date", 35 | }, 36 | Label: nil, 37 | }, 38 | }, 39 | `Creator "Cayley" 40 | graph [ directed 1 41 | node [ id 0 label "_:subject1" ] 42 | node [ id 1 label ""Tomas de Torquemada"" ] 43 | edge [ source 0 target 1 label "" ] 44 | node [ id 2 label ""object1"" ] 45 | edge [ source 0 target 2 label "" ] 46 | node [ id 3 label "" ] 47 | node [ id 4 label ""1990-07-04"^^" ] 48 | edge [ source 3 target 4 label "" ] 49 | ] 50 | `, 51 | }, 52 | } 53 | 54 | func TestWriter(t *testing.T) { 55 | buf := bytes.NewBuffer(nil) 56 | for _, c := range testData { 57 | buf.Reset() 58 | w := gml.NewWriter(buf) 59 | n, err := quad.Copy(w, quad.NewReader(c.quads)) 60 | if err != nil { 61 | t.Fatalf("write failed after %d quads: %v", n, err) 62 | } 63 | if err = w.Close(); err != nil { 64 | t.Fatal("error on close:", err) 65 | } 66 | if c.data != buf.String() { 67 | t.Fatalf("wrong output:\n%s\n\nvs\n\n%s", buf.String(), c.data) 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/cayleygraph/quad 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/piprate/json-gold v0.5.0 7 | github.com/stretchr/testify v1.9.0 8 | google.golang.org/protobuf v1.34.2 9 | ) 10 | 11 | require ( 12 | github.com/davecgh/go-spew v1.1.1 // indirect 13 | github.com/pmezard/go-difflib v1.0.0 // indirect 14 | github.com/pquerna/cachecontrol v0.2.0 // indirect 15 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 16 | gopkg.in/yaml.v3 v3.0.1 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 5 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 6 | github.com/piprate/json-gold v0.5.0 h1:RmGh1PYboCFcchVFuh2pbSWAZy4XJaqTMU4KQYsApbM= 7 | github.com/piprate/json-gold v0.5.0/go.mod h1:WZ501QQMbZZ+3pXFPhQKzNwS1+jls0oqov3uQ2WasLs= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/pquerna/cachecontrol v0.2.0 h1:vBXSNuE5MYP9IJ5kjsdo8uq+w41jSPgvba2DEnkRx9k= 11 | github.com/pquerna/cachecontrol v0.2.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI= 12 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 13 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 14 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 15 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 16 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= 17 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= 18 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 19 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 20 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 21 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 22 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 23 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 24 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 25 | -------------------------------------------------------------------------------- /graphml/graphml.go: -------------------------------------------------------------------------------- 1 | // Package graphml provides an encoder for GraphML format 2 | package graphml 3 | 4 | import ( 5 | "encoding/xml" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/cayleygraph/quad" 10 | ) 11 | 12 | func init() { 13 | quad.RegisterFormat(quad.Format{ 14 | Name: "graphml", 15 | Ext: []string{".graphml"}, 16 | Mime: []string{"application/xml"}, 17 | Writer: func(w io.Writer) quad.WriteCloser { return NewWriter(w) }, 18 | }) 19 | } 20 | 21 | func NewWriter(w io.Writer) *Writer { 22 | return &Writer{w: w} 23 | } 24 | 25 | type Writer struct { 26 | w io.Writer 27 | written bool 28 | err error 29 | 30 | nodes map[string]int 31 | cur int 32 | } 33 | 34 | func (w *Writer) writeNode(s string) int { 35 | if w.err != nil { 36 | return -1 37 | } 38 | i, ok := w.nodes[s] 39 | if ok { 40 | return i 41 | } 42 | i = w.cur 43 | w.cur++ 44 | w.nodes[s] = i 45 | _, w.err = fmt.Fprintf(w.w, "\t\t", i) 46 | if w.err != nil { 47 | return -1 48 | } 49 | if w.err = xml.EscapeText(w.w, []byte(s)); w.err != nil { 50 | return -1 51 | } 52 | if _, w.err = w.w.Write([]byte("\n")); w.err != nil { 53 | return -1 54 | } 55 | return i 56 | } 57 | 58 | func (w *Writer) WriteQuad(q quad.Quad) error { 59 | if w.err != nil { 60 | return w.err 61 | } else if !q.IsValid() { 62 | return quad.ErrInvalid 63 | } 64 | if !w.written { 65 | if _, err := w.w.Write([]byte(header)); err != nil { 66 | return err 67 | } 68 | w.written = true 69 | w.nodes = make(map[string]int) 70 | } 71 | s := w.writeNode(q.Subject.String()) 72 | o := w.writeNode(q.Object.String()) 73 | if w.err != nil { 74 | return w.err 75 | } 76 | _, w.err = fmt.Fprintf(w.w, "\t\t", s, o) 77 | if w.err != nil { 78 | return w.err 79 | } 80 | if w.err = xml.EscapeText(w.w, []byte(q.Predicate.String())); w.err != nil { 81 | return w.err 82 | } 83 | _, w.err = w.w.Write([]byte("\n")) 84 | return w.err 85 | } 86 | 87 | func (w *Writer) WriteQuads(buf []quad.Quad) (int, error) { 88 | for i, q := range buf { 89 | if err := w.WriteQuad(q); err != nil { 90 | return i, err 91 | } 92 | } 93 | return len(buf), nil 94 | } 95 | 96 | func (w *Writer) Close() error { 97 | if w.err != nil { 98 | return w.err 99 | } 100 | if !w.written { 101 | if _, w.err = w.w.Write([]byte(header)); w.err != nil { 102 | return w.err 103 | } 104 | } 105 | if _, w.err = w.w.Write([]byte(footer)); w.err != nil { 106 | return w.err 107 | } 108 | w.err = fmt.Errorf("closed") 109 | return nil 110 | } 111 | 112 | const header = ` 113 | 116 | 117 | 118 | 119 | ` 120 | const footer = "\t\n\n" 121 | -------------------------------------------------------------------------------- /graphml/graphml_test.go: -------------------------------------------------------------------------------- 1 | package graphml_test 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/cayleygraph/quad" 8 | "github.com/cayleygraph/quad/graphml" 9 | ) 10 | 11 | var testData = []struct { 12 | quads []quad.Quad 13 | data string 14 | }{ 15 | { 16 | []quad.Quad{ 17 | { 18 | Subject: quad.BNode("subject1"), 19 | Predicate: quad.IRI("/film/performance/character"), 20 | Object: quad.String("Tomás de Torquemada"), 21 | Label: nil, 22 | }, 23 | { 24 | Subject: quad.BNode("subject1"), 25 | Predicate: quad.IRI("http://an.example/predicate1"), 26 | Object: quad.String("object1"), 27 | Label: nil, 28 | }, 29 | { 30 | Subject: quad.IRI("http://example.org/bob#me"), 31 | Predicate: quad.IRI("http://schema.org/birthDate"), 32 | Object: quad.TypedString{ 33 | Value: "1990-07-04", 34 | Type: "http://www.w3.org/2001/XMLSchema#date", 35 | }, 36 | Label: nil, 37 | }, 38 | }, 39 | ` 40 | 43 | 44 | 45 | 46 | _:subject1 47 | "Tomás de Torquemada" 48 | </film/performance/character> 49 | "object1" 50 | <http://an.example/predicate1> 51 | <http://example.org/bob#me> 52 | "1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date> 53 | <http://schema.org/birthDate> 54 | 55 | 56 | `, 57 | }, 58 | } 59 | 60 | func TestWriter(t *testing.T) { 61 | buf := bytes.NewBuffer(nil) 62 | for _, c := range testData { 63 | buf.Reset() 64 | w := graphml.NewWriter(buf) 65 | n, err := quad.Copy(w, quad.NewReader(c.quads)) 66 | if err != nil { 67 | t.Fatalf("write failed after %d quads: %v", n, err) 68 | } 69 | if err = w.Close(); err != nil { 70 | t.Fatal("error on close:", err) 71 | } 72 | if c.data != buf.String() { 73 | t.Fatalf("wrong output:\n%s\n\nvs\n\n%s", buf.String(), c.data) 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /json/json.go: -------------------------------------------------------------------------------- 1 | // Package json provides an encoder/decoder for JSON quad formats 2 | package json 3 | 4 | import ( 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "io" 9 | 10 | "github.com/cayleygraph/quad" 11 | ) 12 | 13 | func init() { 14 | quad.RegisterFormat(quad.Format{ 15 | Name: "json", 16 | Ext: []string{".json"}, 17 | Mime: []string{"application/json"}, 18 | Writer: func(w io.Writer) quad.WriteCloser { return NewWriter(w) }, 19 | Reader: func(r io.Reader) quad.ReadCloser { return NewReader(r) }, 20 | MarshalValue: func(v quad.Value) ([]byte, error) { 21 | return json.Marshal(quad.ToString(v)) 22 | }, 23 | UnmarshalValue: func(b []byte) (quad.Value, error) { 24 | var s *string 25 | if err := json.Unmarshal(b, &s); err != nil { 26 | return nil, err 27 | } else if s == nil { 28 | return nil, nil 29 | } 30 | return quad.StringToValue(*s), nil 31 | }, 32 | }) 33 | quad.RegisterFormat(quad.Format{ 34 | Name: "json-stream", 35 | Mime: []string{"application/x-json-stream"}, 36 | Writer: func(w io.Writer) quad.WriteCloser { return NewStreamWriter(w) }, 37 | Reader: func(r io.Reader) quad.ReadCloser { return NewStreamReader(r) }, 38 | }) 39 | } 40 | 41 | func NewReader(r io.Reader) *Reader { 42 | var quads []quad.Quad 43 | err := json.NewDecoder(r).Decode(&quads) 44 | return &Reader{ // TODO(dennwc): stream-friendly reader 45 | quads: quads, 46 | err: err, 47 | } 48 | } 49 | 50 | type Reader struct { 51 | quads []quad.Quad 52 | n int 53 | err error 54 | } 55 | 56 | func (r *Reader) ReadQuad() (quad.Quad, error) { 57 | if r.err != nil { 58 | return quad.Quad{}, r.err 59 | } 60 | if r.n >= len(r.quads) { 61 | return quad.Quad{}, io.EOF 62 | } 63 | q := r.quads[r.n] 64 | r.n++ 65 | if !q.IsValid() { 66 | return quad.Quad{}, fmt.Errorf("invalid quad at index %d. %s", r.n-1, q) 67 | } 68 | return q, nil 69 | } 70 | func (r *Reader) Close() error { return nil } 71 | 72 | func NewStreamReader(r io.Reader) *StreamReader { 73 | return &StreamReader{dec: json.NewDecoder(r)} 74 | } 75 | 76 | type StreamReader struct { 77 | dec *json.Decoder 78 | err error 79 | } 80 | 81 | func (r *StreamReader) ReadQuad() (quad.Quad, error) { 82 | if r.err != nil { 83 | return quad.Quad{}, r.err 84 | } 85 | var q quad.Quad 86 | r.err = r.dec.Decode(&q) 87 | return q, r.err 88 | } 89 | func (r *StreamReader) Close() error { return nil } 90 | 91 | func NewWriter(w io.Writer) *Writer { 92 | return &Writer{w: w} 93 | } 94 | 95 | type Writer struct { 96 | w io.Writer 97 | written bool 98 | closed bool 99 | } 100 | 101 | func (w *Writer) WriteQuad(q quad.Quad) error { 102 | if w.closed { 103 | return errors.New("closed") 104 | } else if !q.IsValid() { 105 | return quad.ErrInvalid 106 | } 107 | if !w.written { 108 | if _, err := w.w.Write([]byte("[\n\t")); err != nil { 109 | return err 110 | } 111 | w.written = true 112 | } else { 113 | if _, err := w.w.Write([]byte(",\n\t")); err != nil { 114 | return err 115 | } 116 | } 117 | data, err := json.Marshal(q) 118 | if err != nil { 119 | return err 120 | } 121 | _, err = w.w.Write(data) 122 | return err 123 | } 124 | 125 | func (w *Writer) WriteQuads(buf []quad.Quad) (int, error) { 126 | for i, q := range buf { 127 | if err := w.WriteQuad(q); err != nil { 128 | return i, err 129 | } 130 | } 131 | return len(buf), nil 132 | } 133 | 134 | func (w *Writer) Close() error { 135 | if w.closed { 136 | return nil 137 | } 138 | w.closed = true 139 | if !w.written { 140 | _, err := w.w.Write([]byte("null\n")) 141 | return err 142 | } 143 | _, err := w.w.Write([]byte("\n]\n")) 144 | return err 145 | } 146 | 147 | func NewStreamWriter(w io.Writer) *StreamWriter { 148 | return &StreamWriter{enc: json.NewEncoder(w)} 149 | } 150 | 151 | type StreamWriter struct { 152 | enc *json.Encoder 153 | } 154 | 155 | func (w *StreamWriter) WriteQuad(q quad.Quad) error { 156 | if !q.IsValid() { 157 | return quad.ErrInvalid 158 | } 159 | return w.enc.Encode(q) 160 | } 161 | 162 | func (w *StreamWriter) WriteQuads(buf []quad.Quad) (int, error) { 163 | for i, q := range buf { 164 | if err := w.WriteQuad(q); err != nil { 165 | return i, err 166 | } 167 | } 168 | return len(buf), nil 169 | } 170 | 171 | func (w *StreamWriter) Close() error { return nil } 172 | -------------------------------------------------------------------------------- /json/json_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package json 16 | 17 | import ( 18 | "bytes" 19 | "fmt" 20 | "reflect" 21 | "strings" 22 | "testing" 23 | 24 | "github.com/cayleygraph/quad" 25 | "github.com/stretchr/testify/require" 26 | ) 27 | 28 | var readTests = []struct { 29 | message string 30 | input string 31 | expect []quad.Quad 32 | err error 33 | }{ 34 | { 35 | message: "parse correct JSON", 36 | input: `[ 37 | {"subject": "foo", "predicate": "bar", "object": "baz"}, 38 | {"subject":"_:foo","predicate":"\u003cbar\u003e","object":"\"baz\"@en"}, 39 | {"subject": "foo", "predicate": "bar", "object": "baz", "label": "graph"} 40 | ]`, 41 | expect: []quad.Quad{ 42 | quad.MakeRaw("foo", "bar", "baz", ""), 43 | quad.Make(quad.Raw("_:foo"), quad.Raw(""), quad.Raw(`"baz"@en`), nil), 44 | quad.MakeRaw("foo", "bar", "baz", "graph"), 45 | }, 46 | err: nil, 47 | }, 48 | { 49 | message: "parse correct JSON with extra field", 50 | input: `[ 51 | {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} 52 | ]`, 53 | expect: []quad.Quad{ 54 | quad.MakeRaw("foo", "bar", "foo", ""), 55 | }, 56 | err: nil, 57 | }, 58 | { 59 | message: "reject incorrect JSON", 60 | input: `[ 61 | {"subject": "foo", "predicate": "bar"} 62 | ]`, 63 | expect: nil, 64 | err: fmt.Errorf("invalid quad at index %d. %v", 0, quad.MakeRaw("foo", "bar", "", "")), 65 | }, 66 | { 67 | message: "unescape values", 68 | input: `[ 69 | {"subject": "foo", "predicate": "bar", "object": "\"baz\""} 70 | ]`, 71 | expect: []quad.Quad{ 72 | quad.MakeRaw("foo", "bar", `"baz"`, ""), 73 | }, 74 | err: nil, 75 | }, 76 | } 77 | 78 | func TestReadJSON(t *testing.T) { 79 | for _, test := range readTests { 80 | qr := NewReader(strings.NewReader(test.input)) 81 | got, err := quad.ReadAll(qr) 82 | qr.Close() 83 | if fmt.Sprint(err) != fmt.Sprint(test.err) { 84 | t.Errorf("Failed to %v with unexpected error, got:%v expected %v", test.message, err, test.err) 85 | } 86 | if !reflect.DeepEqual(got, test.expect) { 87 | t.Errorf("Failed to %v, got:\n%v\nexpect:\n%v", test.message, got, test.expect) 88 | } 89 | } 90 | } 91 | 92 | var writeTests = []struct { 93 | message string 94 | input []quad.Quad 95 | expect string 96 | err error 97 | }{ 98 | { 99 | message: "write empty JSON", 100 | input: []quad.Quad{}, 101 | expect: "null\n", 102 | err: nil, 103 | }, 104 | { 105 | message: "write JSON", 106 | input: []quad.Quad{ 107 | quad.MakeRaw("foo", "bar", "baz", ""), 108 | quad.Make(quad.BNode("foo"), quad.IRI("bar"), quad.LangString{"baz", "en"}, nil), 109 | quad.MakeRaw("foo", "bar", "baz", "graph"), 110 | }, 111 | expect: `[ 112 | {"subject":"foo","predicate":"bar","object":"baz"}, 113 | {"subject":"_:foo","predicate":"\u003cbar\u003e","object":"\"baz\"@en"}, 114 | {"subject":"foo","predicate":"bar","object":"baz","label":"graph"} 115 | ] 116 | `, 117 | err: nil, 118 | }, 119 | { 120 | message: "escape values", 121 | input: []quad.Quad{ 122 | quad.MakeRaw("foo", "bar", `"baz"`, ""), 123 | }, 124 | expect: `[ 125 | {"subject":"foo","predicate":"bar","object":"baz"} 126 | ] 127 | `, 128 | err: nil, 129 | }, 130 | } 131 | 132 | func TestWriteJSON(t *testing.T) { 133 | buf := bytes.NewBuffer(nil) 134 | for _, test := range writeTests { 135 | buf.Reset() 136 | qw := NewWriter(buf) 137 | _, err := quad.Copy(qw, quad.NewReader(test.input)) 138 | if err != nil { 139 | t.Errorf("Failed to %v: %v", test.message, err) 140 | continue 141 | } 142 | qw.Close() 143 | if fmt.Sprint(err) != fmt.Sprint(test.err) { 144 | t.Errorf("Failed to %v with unexpected error, got:%v expected %v", test.message, err, test.err) 145 | } 146 | if got := buf.String(); got != test.expect { 147 | t.Errorf("Failed to %v, got:%v expect:%v", test.message, got, test.expect) 148 | } 149 | } 150 | } 151 | 152 | func TestValueEncoding(t *testing.T) { 153 | vals := []quad.Value{ 154 | quad.String("some val"), 155 | quad.IRI("iri"), 156 | quad.BNode("bnode"), 157 | quad.TypedString{Value: "10", Type: "int"}, 158 | quad.LangString{Value: "val", Lang: "en"}, 159 | } 160 | enc := []string{ 161 | `"some val"`, 162 | `"\u003ciri\u003e"`, 163 | `"_:bnode"`, 164 | `"\"10\"^^\u003cint\u003e"`, 165 | `"\"val\"@en"`, 166 | } 167 | f := quad.FormatByName("json") 168 | for i, v := range vals { 169 | data, err := f.MarshalValue(v) 170 | require.NoError(t, err) 171 | require.Equal(t, enc[i], string(data), string(data)) 172 | v2, err := f.UnmarshalValue(data) 173 | require.NoError(t, err) 174 | require.Equal(t, v, v2) 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /jsonld/jsonld.go: -------------------------------------------------------------------------------- 1 | // Package jsonld provides an encoder/decoder for JSON-LD quad format 2 | package jsonld 3 | 4 | import ( 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/cayleygraph/quad" 10 | "github.com/cayleygraph/quad/voc" 11 | "github.com/cayleygraph/quad/voc/xsd" 12 | "github.com/piprate/json-gold/ld" 13 | ) 14 | 15 | // AutoConvertTypedString allows to convert TypedString values to native 16 | // equivalents directly while parsing. It will call ToNative on all TypedString values. 17 | // 18 | // If conversion error occurs, it will preserve original TypedString value. 19 | var AutoConvertTypedString = true 20 | 21 | func init() { 22 | quad.RegisterFormat(quad.Format{ 23 | Name: "jsonld", 24 | Ext: []string{".jsonld"}, 25 | Mime: []string{"application/ld+json"}, 26 | Writer: func(w io.Writer) quad.WriteCloser { return NewWriter(w) }, 27 | Reader: func(r io.Reader) quad.ReadCloser { return NewReader(r) }, 28 | }) 29 | } 30 | 31 | // NewReader returns quad reader for JSON-LD stream. 32 | func NewReader(r io.Reader) *Reader { 33 | var o interface{} 34 | if err := json.NewDecoder(r).Decode(&o); err != nil { 35 | return &Reader{err: err} 36 | } 37 | return NewReaderFromMap(o) 38 | } 39 | 40 | // NewReaderFromMap returns quad reader for JSON-LD map object. 41 | func NewReaderFromMap(o interface{}) *Reader { 42 | opts := ld.NewJsonLdOptions("") 43 | processor := ld.NewJsonLdProcessor() 44 | data, err := processor.ToRDF(o, opts) 45 | if err != nil { 46 | return &Reader{err: err} 47 | } 48 | return &Reader{ 49 | graphs: data.(*ld.RDFDataset).Graphs, 50 | } 51 | } 52 | 53 | var _ quad.Reader = &Reader{} 54 | 55 | // Reader implements the quad.Reader interface 56 | type Reader struct { 57 | err error 58 | name string 59 | n int 60 | graphs map[string][]*ld.Quad 61 | } 62 | 63 | // ReadQuad implements the quad.Reader interface 64 | func (r *Reader) ReadQuad() (quad.Quad, error) { 65 | if r.err != nil { 66 | return quad.Quad{}, r.err 67 | } 68 | next: 69 | if len(r.graphs) == 0 { 70 | return quad.Quad{}, io.EOF 71 | } 72 | if r.name == "" { 73 | for gname := range r.graphs { 74 | r.name = gname 75 | break 76 | } 77 | } 78 | if r.n >= len(r.graphs[r.name]) { 79 | r.n = 0 80 | delete(r.graphs, r.name) 81 | r.name = "" 82 | goto next 83 | } 84 | cur := r.graphs[r.name][r.n] 85 | r.n++ 86 | var graph quad.Value 87 | if r.name != "@default" { 88 | graph = quad.IRI(r.name) 89 | } 90 | return quad.Quad{ 91 | Subject: toValue(cur.Subject), 92 | Predicate: toValue(cur.Predicate), 93 | Object: toValue(cur.Object), 94 | Label: graph, 95 | }, nil 96 | } 97 | 98 | // Close implements quad.Reader 99 | func (r *Reader) Close() error { 100 | r.graphs = nil 101 | return r.err 102 | } 103 | 104 | var _ quad.Writer = &Writer{} 105 | 106 | // Writer implements quad.Writer 107 | type Writer struct { 108 | w io.Writer 109 | ds *ld.RDFDataset 110 | ctx interface{} 111 | } 112 | 113 | // NewWriter constructs a new Writer 114 | func NewWriter(w io.Writer) *Writer { 115 | return &Writer{w: w, ds: ld.NewRDFDataset()} 116 | } 117 | 118 | // SetLdContext defines a context for the emitted JSON-LD data 119 | // See: https://json-ld.org/spec/latest/json-ld/#the-context 120 | func (w *Writer) SetLdContext(ctx interface{}) { 121 | w.ctx = ctx 122 | } 123 | 124 | // WriteQuad implements quad.Writer 125 | func (w *Writer) WriteQuad(q quad.Quad) error { 126 | if !q.IsValid() { 127 | return quad.ErrInvalid 128 | } 129 | var graph string 130 | if q.Label == nil { 131 | graph = "@default" 132 | } else if iri, ok := q.Label.(quad.IRI); ok { 133 | graph = string(iri) 134 | } else { 135 | graph = q.Label.String() 136 | } 137 | g := w.ds.Graphs[graph] 138 | g = append(g, ld.NewQuad( 139 | toTerm(q.Subject), 140 | toTerm(q.Predicate), 141 | toTerm(q.Object), 142 | graph, 143 | )) 144 | w.ds.Graphs[graph] = g 145 | return nil 146 | } 147 | 148 | // WriteQuads implements quad.Writer 149 | func (w *Writer) WriteQuads(buf []quad.Quad) (int, error) { 150 | for i, q := range buf { 151 | if err := w.WriteQuad(q); err != nil { 152 | return i, err 153 | } 154 | } 155 | return len(buf), nil 156 | } 157 | 158 | // Close implements quad.Writer 159 | func (w *Writer) Close() error { 160 | opts := ld.NewJsonLdOptions("") 161 | api := ld.NewJsonLdApi() 162 | processor := ld.NewJsonLdProcessor() 163 | var data interface{} 164 | data, err := api.FromRDF(w.ds, opts) 165 | if err != nil { 166 | return err 167 | } 168 | if w.ctx != nil { 169 | out, err := processor.Compact(data, w.ctx, opts) 170 | if err != nil { 171 | return err 172 | } 173 | data = out 174 | } 175 | return json.NewEncoder(w.w).Encode(data) 176 | } 177 | 178 | func toTerm(v quad.Value) ld.Node { 179 | switch v := v.(type) { 180 | case quad.IRI: 181 | return ld.NewIRI(string(v)) 182 | case quad.BNode: 183 | return ld.NewBlankNode(string(v)) 184 | case quad.String: 185 | return ld.NewLiteral(string(v), "", "") 186 | case quad.TypedString: 187 | return ld.NewLiteral(string(v.Value), string(v.Type), "") 188 | case quad.LangString: 189 | return ld.NewLiteral(string(v.Value), "", string(v.Lang)) 190 | case quad.TypedStringer: 191 | return toTerm(v.TypedString()) 192 | default: 193 | return ld.NewLiteral(v.String(), "", "") 194 | } 195 | } 196 | 197 | // FromValue converts quad value to a JSON-LD compatible object. 198 | func FromValue(v quad.Value) interface{} { 199 | switch v := v.(type) { 200 | case quad.IRI: 201 | return map[string]interface{}{ 202 | "@id": string(v), 203 | } 204 | case quad.BNode: 205 | return map[string]interface{}{ 206 | "@id": v.String(), 207 | } 208 | case quad.String: 209 | return string(v) 210 | case quad.LangString: 211 | return map[string]interface{}{ 212 | "@value": string(v.Value), 213 | "@language": string(v.Lang), 214 | } 215 | case quad.TypedString: 216 | return typedStringToJSON(v) 217 | case quad.TypedStringer: 218 | return typedStringToJSON(v.TypedString()) 219 | default: 220 | return v.String() 221 | } 222 | } 223 | 224 | // ToNode transforms a quad.Value to ld.Node 225 | func ToNode(value quad.Value) (ld.Node, error) { 226 | switch v := value.(type) { 227 | case quad.IRI: 228 | return ld.NewIRI(string(v)), nil 229 | case quad.BNode: 230 | return ld.NewBlankNode(string(v)), nil 231 | case quad.String: 232 | return ld.NewLiteral(string(v), "", ""), nil 233 | case quad.TypedString: 234 | return ld.NewLiteral(string(v.Value), string(v.Type), ""), nil 235 | case quad.LangString: 236 | return ld.NewLiteral(string(v.Value), "", v.Lang), nil 237 | default: 238 | return nil, fmt.Errorf("Can not convert %v to ld.Node", value) 239 | } 240 | } 241 | 242 | func isKnownTimeType(dataType quad.IRI) bool { 243 | for _, iri := range quad.KnownTimeTypes { 244 | if iri == dataType { 245 | return true 246 | } 247 | } 248 | return false 249 | } 250 | 251 | func typedStringToJSON(v quad.TypedString) interface{} { 252 | if AutoConvertTypedString && quad.HasStringConversion(v.Type) && !isKnownTimeType(v.Type) { 253 | return v.Native() 254 | } 255 | return map[string]interface{}{ 256 | "@value": string(v.Value), 257 | "@type": string(v.Type), 258 | } 259 | } 260 | 261 | var stringDataType = voc.FullIRI(xsd.String) 262 | 263 | func toValue(t ld.Node) quad.Value { 264 | switch t := t.(type) { 265 | case *ld.IRI: 266 | return quad.IRI(t.Value) 267 | case *ld.BlankNode: 268 | return quad.BNode(t.Attribute) 269 | case *ld.Literal: 270 | if t.Language != "" { 271 | return quad.LangString{ 272 | Value: quad.String(t.Value), 273 | Lang: t.Language, 274 | } 275 | } else if t.Datatype != "" && t.Datatype != stringDataType { 276 | ts := quad.TypedString{ 277 | Value: quad.String(t.Value), 278 | Type: quad.IRI(t.Datatype), 279 | } 280 | if AutoConvertTypedString { 281 | if v, err := ts.ParseValue(); err == nil { 282 | return v 283 | } 284 | } 285 | return ts 286 | } 287 | return quad.String(t.Value) 288 | default: 289 | panic(fmt.Errorf("unexpected term type: %T", t)) 290 | } 291 | } 292 | -------------------------------------------------------------------------------- /jsonld/jsonld_test.go: -------------------------------------------------------------------------------- 1 | package jsonld 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "reflect" 7 | "sort" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | "github.com/cayleygraph/quad" 13 | "github.com/cayleygraph/quad/voc/xsd" 14 | "github.com/piprate/json-gold/ld" 15 | "github.com/stretchr/testify/require" 16 | ) 17 | 18 | var testReadCases = []struct { 19 | data string 20 | expect []quad.Quad 21 | }{ 22 | { 23 | `{ 24 | "@context": { 25 | "ex": "http://example.org/", 26 | "term1": {"@id": "ex:term1", "@type": "ex:datatype"}, 27 | "term2": {"@id": "ex:term2", "@type": "@id"}, 28 | "term3": {"@id": "ex:term3", "@language": "en"} 29 | }, 30 | "@id": "ex:id1", 31 | "@type": ["ex:Type1", "ex:Type2"], 32 | "term1": "v1", 33 | "term2": "ex:id2", 34 | "term3": "v3" 35 | }`, 36 | []quad.Quad{ 37 | { 38 | Subject: quad.IRI(`http://example.org/id1`), 39 | Predicate: quad.IRI(`http://example.org/term1`), 40 | Object: quad.TypedString{ 41 | Value: "v1", Type: "http://example.org/datatype", 42 | }, 43 | Label: nil, 44 | }, 45 | { 46 | Subject: quad.IRI(`http://example.org/id1`), 47 | Predicate: quad.IRI(`http://example.org/term2`), 48 | Object: quad.IRI(`http://example.org/id2`), 49 | Label: nil, 50 | }, 51 | { 52 | Subject: quad.IRI(`http://example.org/id1`), 53 | Predicate: quad.IRI(`http://example.org/term3`), 54 | Object: quad.LangString{ 55 | Value: "v3", Lang: "en", 56 | }, 57 | Label: nil, 58 | }, 59 | { 60 | Subject: quad.IRI(`http://example.org/id1`), 61 | Predicate: quad.IRI(`http://www.w3.org/1999/02/22-rdf-syntax-ns#type`), 62 | Object: quad.IRI(`http://example.org/Type1`), 63 | Label: nil, 64 | }, 65 | { 66 | Subject: quad.IRI(`http://example.org/id1`), 67 | Predicate: quad.IRI(`http://www.w3.org/1999/02/22-rdf-syntax-ns#type`), 68 | Object: quad.IRI(`http://example.org/Type2`), 69 | Label: nil, 70 | }, 71 | }, 72 | }, 73 | } 74 | 75 | type ByQuad []quad.Quad 76 | 77 | func (a ByQuad) Len() int { return len(a) } 78 | func (a ByQuad) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 79 | func (a ByQuad) Less(i, j int) bool { return a[i].NQuad() < a[j].NQuad() } 80 | 81 | func TestRead(t *testing.T) { 82 | for i, c := range testReadCases { 83 | r := NewReader(strings.NewReader(c.data)) 84 | quads, err := quad.ReadAll(r) 85 | if err != nil { 86 | t.Errorf("case %d failed: %v", i, err) 87 | } 88 | sort.Sort(ByQuad(quads)) 89 | sort.Sort(ByQuad(c.expect)) 90 | if !reflect.DeepEqual(quads, c.expect) { 91 | t.Errorf("case %d failed: wrong quads returned:\n%v\n%v", i, quads, c.expect) 92 | } 93 | r.Close() 94 | } 95 | } 96 | 97 | var testWriteCases = []struct { 98 | data []quad.Quad 99 | ctx interface{} 100 | expect string 101 | }{ 102 | { 103 | []quad.Quad{ 104 | { 105 | Subject: quad.IRI(`http://example.org/id1`), 106 | Predicate: quad.IRI(`http://example.org/term1`), 107 | Object: quad.TypedString{ 108 | Value: "v1", Type: "http://example.org/datatype", 109 | }, 110 | Label: nil, 111 | }, 112 | { 113 | Subject: quad.IRI(`http://example.org/id1`), 114 | Predicate: quad.IRI(`http://example.org/term2`), 115 | Object: quad.IRI(`http://example.org/id2`), 116 | Label: nil, 117 | }, 118 | { 119 | Subject: quad.IRI(`http://example.org/id1`), 120 | Predicate: quad.IRI(`http://example.org/term3`), 121 | Object: quad.LangString{ 122 | Value: "v3", Lang: "en", 123 | }, 124 | Label: nil, 125 | }, 126 | { 127 | Subject: quad.IRI(`http://example.org/id1`), 128 | Predicate: quad.IRI(`http://www.w3.org/1999/02/22-rdf-syntax-ns#type`), 129 | Object: quad.IRI(`http://example.org/Type1`), 130 | Label: nil, 131 | }, 132 | { 133 | Subject: quad.IRI(`http://example.org/id1`), 134 | Predicate: quad.IRI(`http://www.w3.org/1999/02/22-rdf-syntax-ns#type`), 135 | Object: quad.IRI(`http://example.org/Type2`), 136 | Label: nil, 137 | }, 138 | }, 139 | map[string]interface{}{ 140 | "ex": "http://example.org/", 141 | }, 142 | `{ 143 | "@context": { 144 | "ex": "http://example.org/" 145 | }, 146 | "@id": "ex:id1", 147 | "@type": [ 148 | "ex:Type1", 149 | "ex:Type2" 150 | ], 151 | "ex:term1": { 152 | "@type": "ex:datatype", 153 | "@value": "v1" 154 | }, 155 | "ex:term2": { 156 | "@id": "ex:id2" 157 | }, 158 | "ex:term3": { 159 | "@language": "en", 160 | "@value": "v3" 161 | } 162 | } 163 | `, 164 | }, 165 | } 166 | 167 | func TestWrite(t *testing.T) { 168 | buf := bytes.NewBuffer(nil) 169 | for i, c := range testWriteCases { 170 | buf.Reset() 171 | w := NewWriter(buf) 172 | w.SetLdContext(c.ctx) 173 | _, err := quad.Copy(w, quad.NewReader(c.data)) 174 | if err != nil { 175 | t.Errorf("case %d failed: %v", i, err) 176 | } else if err = w.Close(); err != nil { 177 | t.Errorf("case %d failed: %v", i, err) 178 | } 179 | data := make([]byte, buf.Len()) 180 | copy(data, buf.Bytes()) 181 | buf.Reset() 182 | json.Indent(buf, data, "", " ") 183 | if buf.String() != c.expect { 184 | t.Errorf("case %d failed: wrong data returned:\n%v\n%v", i, buf.String(), c.expect) 185 | } 186 | } 187 | } 188 | 189 | var testRoundtripCases = []struct { 190 | data []quad.Quad 191 | }{ 192 | { 193 | []quad.Quad{ 194 | { 195 | Subject: quad.IRI(`http://example.org/id1`), 196 | Predicate: quad.IRI(`http://example.org/term1`), 197 | Object: quad.TypedString{ 198 | Value: "v1", Type: "http://example.org/datatype", 199 | }, 200 | Label: nil, 201 | }, 202 | { 203 | Subject: quad.IRI(`http://example.org/id1`), 204 | Predicate: quad.IRI(`http://example.org/term2`), 205 | Object: quad.IRI(`http://example.org/id2`), 206 | Label: nil, 207 | }, 208 | { 209 | Subject: quad.IRI(`http://example.org/id1`), 210 | Predicate: quad.IRI(`http://example.org/term3`), 211 | Object: quad.LangString{ 212 | Value: "v3", Lang: "en", 213 | }, 214 | Label: nil, 215 | }, 216 | { 217 | Subject: quad.IRI(`http://example.org/id1`), 218 | Predicate: quad.IRI(`http://www.w3.org/1999/02/22-rdf-syntax-ns#type`), 219 | Object: quad.IRI(`http://example.org/Type1`), 220 | Label: nil, 221 | }, 222 | { 223 | Subject: quad.IRI(`http://example.org/id1`), 224 | Predicate: quad.IRI(`http://www.w3.org/1999/02/22-rdf-syntax-ns#type`), 225 | Object: quad.IRI(`http://example.org/Type2`), 226 | Label: nil, 227 | }, 228 | }, 229 | }, 230 | } 231 | 232 | func TestRoundtrip(t *testing.T) { 233 | buf := bytes.NewBuffer(nil) 234 | for i, c := range testRoundtripCases { 235 | buf.Reset() 236 | w := NewWriter(buf) 237 | _, err := quad.Copy(w, quad.NewReader(c.data)) 238 | if err != nil { 239 | t.Errorf("case %d failed: %v", i, err) 240 | } else if err = w.Close(); err != nil { 241 | t.Errorf("case %d failed: %v", i, err) 242 | } 243 | arr, err := quad.ReadAll(NewReader(buf)) 244 | sort.Sort(quad.ByQuadString(arr)) 245 | sort.Sort(quad.ByQuadString(c.data)) 246 | if err != nil { 247 | t.Errorf("case %d failed: %v", i, err) 248 | } else if !reflect.DeepEqual(arr, c.data) { 249 | t.Errorf("case %d failed: wrong data returned:\n%v\n%v", i, arr, c.data) 250 | } 251 | } 252 | } 253 | 254 | var fromValueTestCases = []struct { 255 | name string 256 | value quad.Value 257 | jsonLd interface{} 258 | }{ 259 | { 260 | name: "Simple text", 261 | value: quad.String("Alice"), 262 | jsonLd: "Alice", 263 | }, 264 | { 265 | name: "Localized text", 266 | value: quad.LangString{Value: "Alice", Lang: "en"}, 267 | jsonLd: map[string]interface{}{"@value": "Alice", "@language": "en"}, 268 | }, 269 | { 270 | name: "Known typed string", 271 | value: quad.TypedString{Value: quad.String("Alice"), Type: xsd.String}, 272 | jsonLd: "Alice", 273 | }, 274 | { 275 | name: "Known typed integer", 276 | value: quad.Int(1), 277 | jsonLd: int64(1), 278 | }, 279 | { 280 | name: "Known typed floating-point number", 281 | value: quad.Float(1.0), 282 | jsonLd: 1.0, 283 | }, 284 | { 285 | name: "Known typed boolean", 286 | value: quad.Bool(true), 287 | jsonLd: true, 288 | }, 289 | { 290 | name: "Datetime", 291 | value: quad.Time(time.Time{}), 292 | jsonLd: map[string]interface{}{ 293 | "@value": "0001-01-01T00:00:00Z", 294 | "@type": xsd.DateTime, 295 | }, 296 | }, 297 | } 298 | 299 | func TestFromValue(t *testing.T) { 300 | for _, c := range fromValueTestCases { 301 | t.Run(c.name, func(t *testing.T) { 302 | require.Equal(t, c.jsonLd, FromValue(c.value)) 303 | }) 304 | } 305 | } 306 | 307 | var toValueTestCases = []struct { 308 | name string 309 | jsonLd ld.Node 310 | value quad.Value 311 | }{ 312 | { 313 | name: "Simple text", 314 | jsonLd: ld.NewLiteral("Alice", "", ""), 315 | value: quad.String("Alice"), 316 | }, 317 | { 318 | name: "Localized text", 319 | jsonLd: ld.NewLiteral("Alice", "", "en"), 320 | value: quad.LangString{Value: "Alice", Lang: "en"}, 321 | }, 322 | { 323 | name: "Known typed string", 324 | jsonLd: ld.NewLiteral("Alice", xsd.String, ""), 325 | value: quad.String("Alice"), 326 | }, 327 | { 328 | name: "Known typed integer", 329 | jsonLd: ld.NewLiteral("1", xsd.Integer, ""), 330 | value: quad.Int(1), 331 | }, 332 | { 333 | name: "Known typed floating-point number (xsd:double)", 334 | jsonLd: ld.NewLiteral("1.1", xsd.Double, ""), 335 | value: quad.Float(1.1), 336 | }, 337 | { 338 | name: "Known typed floating-point number (xsd:float)", 339 | jsonLd: ld.NewLiteral("1.1", xsd.Float, ""), 340 | value: quad.Float(1.1), 341 | }, 342 | { 343 | name: "Known typed boolean", 344 | jsonLd: ld.NewLiteral("true", xsd.Boolean, ""), 345 | value: quad.Bool(true), 346 | }, 347 | { 348 | name: "Datetime", 349 | jsonLd: ld.NewLiteral("0001-01-01T00:00:00Z", xsd.DateTime, ""), 350 | value: quad.Time(time.Time{}), 351 | }, 352 | } 353 | 354 | func TestToValue(t *testing.T) { 355 | for _, c := range toValueTestCases { 356 | t.Run(c.name, func(t *testing.T) { 357 | require.Equal(t, c.value, toValue(c.jsonLd)) 358 | }) 359 | } 360 | } 361 | -------------------------------------------------------------------------------- /nquad_tests.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cayleygraph/quad/a9b1aedeecc3a8120c50a0c23bbb423a3ed4c5a2/nquad_tests.tar.gz -------------------------------------------------------------------------------- /nquads/nquads.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package nquads implements parsing the RDF 1.1 N-Quads like line-based syntax 16 | // for RDF datasets. 17 | // 18 | // Typed parsing is performed as based on a simplified grammar derived from 19 | // the N-Quads grammar defined by http://www.w3.org/TR/n-quads/. 20 | // 21 | // Raw parsing is performed as defined by http://www.w3.org/TR/n-quads/ 22 | // with the exception that parser will allow relative IRI values, 23 | // which are prohibited by the N-Quads quad-Quads specifications. 24 | // 25 | // For a complete definition of the grammar, see cquads.rl and nquads.rl. 26 | package nquads 27 | 28 | import ( 29 | "bufio" 30 | "bytes" 31 | "fmt" 32 | "io" 33 | "strconv" 34 | 35 | "github.com/cayleygraph/quad" 36 | ) 37 | 38 | //go:generate ragel -Z -G2 typed.rl 39 | //go:generate ragel -Z -G2 raw.rl 40 | 41 | // AutoConvertTypedString allows to convert TypedString values to native 42 | // equivalents directly while parsing. It will call ToNative on all TypedString values. 43 | // 44 | // If conversion error occurs, it will preserve original TypedString value. 45 | var AutoConvertTypedString = true 46 | 47 | var DecodeRaw = false 48 | 49 | func init() { 50 | quad.RegisterFormat(quad.Format{ 51 | Name: "nquads", 52 | Ext: []string{".nq", ".nt"}, 53 | Mime: []string{"application/n-quads", "application/n-triples"}, 54 | Reader: func(r io.Reader) quad.ReadCloser { 55 | return NewReader(r, DecodeRaw) 56 | }, 57 | Writer: func(w io.Writer) quad.WriteCloser { return NewWriter(w) }, 58 | MarshalValue: func(v quad.Value) ([]byte, error) { 59 | if v == nil { 60 | return nil, nil 61 | } 62 | return []byte(v.String()), nil 63 | }, 64 | UnmarshalValue: func(b []byte) (quad.Value, error) { 65 | // TODO: proper parser for a single value 66 | r := NewReader(bytes.NewReader(bytes.Join([][]byte{ 67 | []byte("

"), 68 | b, 69 | []byte(" .\n"), 70 | }, nil)), false) 71 | q, err := r.ReadQuad() 72 | if err == io.EOF { 73 | return nil, quad.ErrInvalid 74 | } else if err != nil { 75 | return nil, err 76 | } 77 | return q.Object, nil 78 | }, 79 | }) 80 | } 81 | 82 | // Reader implements N-Quad document parsing according to the RDF 83 | // 1.1 N-Quads specification. 84 | type Reader struct { 85 | r *bufio.Reader 86 | line []byte 87 | raw bool 88 | } 89 | 90 | // NewReader returns an N-Quad decoder that takes its input from the 91 | // provided io.Reader. 92 | func NewReader(r io.Reader, raw bool) *Reader { 93 | return &Reader{r: bufio.NewReader(r), raw: raw} 94 | } 95 | 96 | // ReadQuad returns the next valid N-Quad as a quad.Quad, or an error. 97 | func (dec *Reader) ReadQuad() (quad.Quad, error) { 98 | dec.line = dec.line[:0] 99 | var line []byte 100 | for { 101 | for { 102 | l, pre, err := dec.r.ReadLine() 103 | if err != nil { 104 | return quad.Quad{}, err 105 | } 106 | dec.line = append(dec.line, l...) 107 | if !pre { 108 | break 109 | } 110 | } 111 | if line = bytes.TrimSpace(dec.line); len(line) != 0 && line[0] != '#' { 112 | break 113 | } 114 | dec.line = dec.line[:0] 115 | } 116 | var ( 117 | q quad.Quad 118 | err error 119 | ) 120 | if dec.raw { 121 | q, err = ParseRaw(string(line)) 122 | } else { 123 | q, err = Parse(string(line)) 124 | } 125 | if err != nil { 126 | return quad.Quad{}, fmt.Errorf("failed to parse %q: %v", dec.line, err) 127 | } 128 | if !q.IsValid() { 129 | return dec.ReadQuad() 130 | } 131 | return q, nil 132 | } 133 | func (dec *Reader) Close() error { return nil } 134 | 135 | func unEscape(r []rune, spec int, isQuoted, isEscaped bool) quad.Value { 136 | raw := r 137 | var sp []rune 138 | if spec > 0 { 139 | r, sp = r[:spec], r[spec:] 140 | isQuoted = true 141 | } 142 | if isQuoted { 143 | r = r[1 : len(r)-1] 144 | } else { 145 | if len(r) >= 2 && r[0] == '<' && r[len(r)-1] == '>' { 146 | return quad.IRI(r[1 : len(r)-1]) 147 | } 148 | if len(r) >= 2 && r[0] == '_' && r[1] == ':' { 149 | return quad.BNode(string(r[2:])) 150 | } 151 | } 152 | var val string 153 | if isEscaped { 154 | buf := bytes.NewBuffer(make([]byte, 0, len(r))) 155 | 156 | for i := 0; i < len(r); { 157 | switch r[i] { 158 | case '\\': 159 | i++ 160 | var c byte 161 | switch r[i] { 162 | case 't': 163 | c = '\t' 164 | case 'b': 165 | c = '\b' 166 | case 'n': 167 | c = '\n' 168 | case 'r': 169 | c = '\r' 170 | case 'f': 171 | c = '\f' 172 | case '"': 173 | c = '"' 174 | case '\'': 175 | c = '\'' 176 | case '\\': 177 | c = '\\' 178 | case 'u': 179 | rc, err := strconv.ParseInt(string(r[i+1:i+5]), 16, 32) 180 | if err != nil { 181 | panic(fmt.Errorf("internal parser error: %v", err)) 182 | } 183 | buf.WriteRune(rune(rc)) 184 | i += 5 185 | continue 186 | case 'U': 187 | rc, err := strconv.ParseInt(string(r[i+1:i+9]), 16, 32) 188 | if err != nil { 189 | panic(fmt.Errorf("internal parser error: %v", err)) 190 | } 191 | buf.WriteRune(rune(rc)) 192 | i += 9 193 | continue 194 | } 195 | buf.WriteByte(c) 196 | default: 197 | buf.WriteRune(r[i]) 198 | } 199 | i++ 200 | } 201 | val = buf.String() 202 | } else { 203 | val = string(r) 204 | } 205 | if len(sp) == 0 { 206 | if isQuoted { 207 | return quad.String(val) 208 | } 209 | return quad.Raw(string(val)) 210 | } 211 | if sp[0] == '@' { 212 | return quad.LangString{ 213 | Value: quad.String(val), 214 | Lang: string(sp[1:]), 215 | } 216 | } else if len(sp) >= 4 && sp[0] == '^' && sp[1] == '^' && sp[2] == '<' && sp[len(sp)-1] == '>' { 217 | v := quad.TypedString{ 218 | Value: quad.String(val), 219 | Type: quad.IRI(sp[3 : len(sp)-1]), 220 | } 221 | if AutoConvertTypedString { 222 | nv, err := v.ParseValue() 223 | if err == nil { 224 | return nv 225 | } 226 | } 227 | return v 228 | } 229 | return quad.Raw(string(raw)) 230 | } 231 | 232 | func unEscapeRaw(r []rune, isEscaped bool) quad.Value { 233 | if !isEscaped { 234 | return quad.Raw(string(r)) 235 | } 236 | 237 | buf := bytes.NewBuffer(make([]byte, 0, len(r))) 238 | 239 | for i := 0; i < len(r); { 240 | switch r[i] { 241 | case '\\': 242 | i++ 243 | var c byte 244 | switch r[i] { 245 | case 't': 246 | c = '\t' 247 | case 'b': 248 | c = '\b' 249 | case 'n': 250 | c = '\n' 251 | case 'r': 252 | c = '\r' 253 | case 'f': 254 | c = '\f' 255 | case '"': 256 | c = '"' 257 | case '\'': 258 | c = '\'' 259 | case '\\': 260 | c = '\\' 261 | case 'u': 262 | rc, err := strconv.ParseInt(string(r[i+1:i+5]), 16, 32) 263 | if err != nil { 264 | panic(fmt.Errorf("internal parser error: %v", err)) 265 | } 266 | buf.WriteRune(rune(rc)) 267 | i += 5 268 | continue 269 | case 'U': 270 | rc, err := strconv.ParseInt(string(r[i+1:i+9]), 16, 32) 271 | if err != nil { 272 | panic(fmt.Errorf("internal parser error: %v", err)) 273 | } 274 | buf.WriteRune(rune(rc)) 275 | i += 9 276 | continue 277 | } 278 | buf.WriteByte(c) 279 | default: 280 | buf.WriteRune(r[i]) 281 | } 282 | i++ 283 | } 284 | 285 | return quad.Raw(buf.String()) 286 | } 287 | 288 | // NewWriter returns an N-Quad encoder that writes its output to the 289 | // provided io.Writer. 290 | func NewWriter(w io.Writer) *Writer { return &Writer{bw: bufio.NewWriter(w)} } 291 | 292 | // Writer implements N-Quad document generator according to the RDF 293 | // 1.1 N-Quads specification. 294 | type Writer struct { 295 | bw *bufio.Writer 296 | err error 297 | } 298 | 299 | func (enc *Writer) writeValue(v quad.Value, sep string) { 300 | if enc.err != nil { 301 | return 302 | } 303 | _, enc.err = enc.bw.WriteString(v.String()) 304 | if enc.err != nil { 305 | return 306 | } else if sep == "" { 307 | return 308 | } 309 | _, enc.err = enc.bw.WriteString(sep) 310 | if enc.err != nil { 311 | return 312 | } 313 | } 314 | 315 | func (enc *Writer) WriteQuad(q quad.Quad) error { 316 | if !q.IsValid() { 317 | return quad.ErrInvalid 318 | } 319 | enc.writeValue(q.Subject, " ") 320 | enc.writeValue(q.Predicate, " ") 321 | if q.Label == nil { 322 | enc.writeValue(q.Object, " .\n") 323 | } else { 324 | enc.writeValue(q.Object, " ") 325 | enc.writeValue(q.Label, " .\n") 326 | } 327 | return enc.err 328 | } 329 | 330 | func (enc *Writer) WriteQuads(buf []quad.Quad) (int, error) { 331 | for i, q := range buf { 332 | if err := enc.WriteQuad(q); err != nil { 333 | return i, err 334 | } 335 | } 336 | return len(buf), nil 337 | } 338 | 339 | func (enc *Writer) Close() error { 340 | if enc.err == nil { 341 | enc.err = enc.bw.Flush() 342 | } 343 | return enc.err 344 | } 345 | -------------------------------------------------------------------------------- /nquads/nquads.rl: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Ragel gramar definition derived from http://www.w3.org/TR/n-quads/#sec-grammar. 16 | 17 | %%{ 18 | machine nquads; 19 | 20 | alphtype rune; 21 | 22 | PN_CHARS_BASE = [A-Za-z] 23 | | 0x00c0 .. 0x00d6 24 | | 0x00d8 .. 0x00f6 25 | | 0x00f8 .. 0x02ff 26 | | 0x0370 .. 0x037d 27 | | 0x037f .. 0x1fff 28 | | 0x200c .. 0x200d 29 | | 0x2070 .. 0x218f 30 | | 0x2c00 .. 0x2fef 31 | | 0x3001 .. 0xd7ff 32 | | 0xf900 .. 0xfdcf 33 | | 0xfdf0 .. 0xfffd 34 | | 0x10000 .. 0xeffff 35 | ; 36 | 37 | PN_CHARS_U = PN_CHARS_BASE | '_' | ':' ; 38 | 39 | PN_CHARS = PN_CHARS_U 40 | | '-' 41 | | [0-9] 42 | | 0xb7 43 | | 0x0300 .. 0x036f 44 | | 0x203f .. 0x2040 45 | ; 46 | 47 | ECHAR = ('\\' [tbnrf"'\\]) %Escape ; 48 | 49 | UCHAR = ('\\u' xdigit {4} 50 | | '\\U' xdigit {8}) %Escape 51 | ; 52 | 53 | BLANK_NODE_LABEL = '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? ; 54 | 55 | STRING_LITERAL = ( 56 | '!' 57 | | '#' .. '[' 58 | | ']' .. 0x7e 59 | | 0x80 .. 0x10ffff 60 | | ECHAR 61 | | UCHAR)+ - ('_:' | any* '.' | '#' any*) 62 | ; 63 | 64 | STRING_LITERAL_QUOTE = '"' ( 65 | 0x00 .. 0x09 66 | | 0x0b .. 0x0c 67 | | 0x0e .. '!' 68 | | '#' .. '[' 69 | | ']' .. 0x10ffff 70 | | ECHAR 71 | | UCHAR)* 72 | '"' 73 | ; 74 | 75 | IRIREF = '<' ( 76 | '!' 77 | | '#' .. ';' 78 | | '=' 79 | | '?' .. '[' 80 | | ']' 81 | | '_' 82 | | 'a' .. 'z' 83 | | '~' 84 | | 0x80 .. 0x10ffff 85 | | UCHAR)* 86 | '>' 87 | ; 88 | 89 | LANGTAG = '@' [a-zA-Z]+ ('-' [a-zA-Z0-9]+)* ; 90 | 91 | whitespace = [ \t] ; 92 | }%% 93 | -------------------------------------------------------------------------------- /nquads/raw.rl: -------------------------------------------------------------------------------- 1 | // GO SOURCE FILE MACHINE GENERATED BY RAGEL; DO NOT EDIT 2 | 3 | // Copyright 2014 The Cayley Authors. All rights reserved. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package nquads 18 | 19 | import ( 20 | "fmt" 21 | "unicode" 22 | 23 | "github.com/cayleygraph/quad" 24 | ) 25 | 26 | %%{ 27 | machine raw; 28 | 29 | action Escape { 30 | isEscaped = true 31 | } 32 | 33 | action StartSubject { 34 | subject = p 35 | } 36 | 37 | action StartPredicate { 38 | predicate = p 39 | } 40 | 41 | action StartObject { 42 | object = p 43 | } 44 | 45 | action StartLabel { 46 | label = p 47 | } 48 | 49 | action SetSubject { 50 | if subject < 0 { 51 | panic("unexpected parser state: subject start not set") 52 | } 53 | q.Subject = unEscapeRaw(data[subject:p], isEscaped) 54 | isEscaped = false 55 | } 56 | 57 | action SetPredicate { 58 | if predicate < 0 { 59 | panic("unexpected parser state: predicate start not set") 60 | } 61 | q.Predicate = unEscapeRaw(data[predicate:p], isEscaped) 62 | isEscaped = false 63 | } 64 | 65 | action SetObject { 66 | if object < 0 { 67 | panic("unexpected parser state: object start not set") 68 | } 69 | q.Object = unEscapeRaw(data[object:p], isEscaped) 70 | isEscaped = false 71 | } 72 | 73 | action SetLabel { 74 | if label < 0 { 75 | panic("unexpected parser state: label start not set") 76 | } 77 | q.Label = unEscapeRaw(data[label:p], isEscaped) 78 | isEscaped = false 79 | } 80 | 81 | action Return { 82 | return q, nil 83 | } 84 | 85 | action Comment { 86 | } 87 | 88 | action Error { 89 | if p < len(data) { 90 | if r := data[p]; r < unicode.MaxASCII { 91 | return q, fmt.Errorf("%v: unexpected rune %q at %d", quad.ErrInvalid, data[p], p) 92 | } else { 93 | return q, fmt.Errorf("%v: unexpected rune %q (\\u%04x) at %d", quad.ErrInvalid, data[p], data[p], p) 94 | } 95 | } 96 | return q, quad.ErrIncomplete 97 | } 98 | 99 | include nquads "nquads.rl"; 100 | 101 | literal = STRING_LITERAL_QUOTE ('^^' IRIREF | LANGTAG)? ; 102 | 103 | subject = IRIREF | BLANK_NODE_LABEL ; 104 | predicate = IRIREF ; 105 | object = IRIREF | BLANK_NODE_LABEL | literal ; 106 | graphLabel = IRIREF | BLANK_NODE_LABEL ; 107 | 108 | statement := ( 109 | whitespace* subject >StartSubject %SetSubject 110 | whitespace* predicate >StartPredicate %SetPredicate 111 | whitespace* object >StartObject %SetObject 112 | (whitespace* graphLabel >StartLabel %SetLabel)? 113 | whitespace* '.' whitespace* ('#' any*)? >Comment 114 | ) %Return @!Error ; 115 | 116 | write data; 117 | }%% 118 | 119 | // ParseRaw returns a valid quad.Quad or a non-nil error. ParseRaw does 120 | // handle comments except where the comment placement does not prevent 121 | // a complete valid quad.Quad from being defined. 122 | func ParseRaw(statement string) (quad.Quad, error) { 123 | data := []rune(statement) 124 | 125 | var ( 126 | cs, p int 127 | pe = len(data) 128 | eof = pe 129 | 130 | subject = -1 131 | predicate = -1 132 | object = -1 133 | label = -1 134 | 135 | isEscaped bool 136 | 137 | q quad.Quad 138 | ) 139 | 140 | %%write init; 141 | 142 | %%write exec; 143 | 144 | return quad.Quad{}, quad.ErrInvalid 145 | } 146 | -------------------------------------------------------------------------------- /nquads/raw_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package nquads 16 | 17 | import ( 18 | "archive/tar" 19 | "compress/gzip" 20 | "fmt" 21 | "io" 22 | "os" 23 | "path/filepath" 24 | "reflect" 25 | "strings" 26 | "testing" 27 | 28 | "github.com/cayleygraph/quad" 29 | ) 30 | 31 | var testNQuadsRaw = []struct { 32 | message string 33 | input string 34 | expect quad.Quad 35 | err error 36 | }{ 37 | // Tests taken from http://www.w3.org/TR/n-quads/ and http://www.w3.org/TR/n-triples/. 38 | 39 | // _:100000 . # example from 30movies 40 | { 41 | message: "parse triple with commment", 42 | input: `_:100000 . # example from 30movies`, 43 | expect: quad.Quad{ 44 | Subject: quad.Raw("_:100000"), 45 | Predicate: quad.Raw(""), 46 | Object: quad.Raw(""), 47 | Label: nil, 48 | }, 49 | err: nil, 50 | }, 51 | // _:10011 "Tomás de Torquemada" . # example from 30movies with unicode 52 | { 53 | message: "parse triple with commment", 54 | input: `_:10011 "Tomás de Torquemada" . # example from 30movies with unicode`, 55 | expect: quad.Quad{ 56 | Subject: quad.Raw("_:10011"), 57 | Predicate: quad.Raw(""), 58 | Object: quad.Raw(`"Tomás de Torquemada"`), 59 | Label: nil, 60 | }, 61 | err: nil, 62 | }, 63 | 64 | // N-Triples example 1. 65 | { 66 | message: "parse triple with commment", 67 | input: ` . # comments here`, 68 | expect: quad.Quad{ 69 | Subject: quad.Raw(""), 70 | Predicate: quad.Raw(""), 71 | Object: quad.Raw(""), 72 | Label: nil, 73 | }, 74 | err: nil, 75 | }, 76 | { 77 | message: "parse triple with blank subject node, literal object and no comment (1)", 78 | input: `_:subject1 "object1" .`, 79 | expect: quad.Quad{ 80 | Subject: quad.Raw("_:subject1"), 81 | Predicate: quad.Raw(""), 82 | Object: quad.Raw(`"object1"`), 83 | Label: nil, 84 | }, 85 | err: nil, 86 | }, 87 | { 88 | message: "parse triple with blank subject node, literal object and no comment (2)", 89 | input: `_:subject2 "object2" .`, 90 | expect: quad.Quad{ 91 | Subject: quad.Raw("_:subject2"), 92 | Predicate: quad.Raw(""), 93 | Object: quad.Raw(`"object2"`), 94 | Label: nil, 95 | }, 96 | err: nil, 97 | }, 98 | 99 | // N-Triples example 2. 100 | { 101 | message: "parse triple with three IRIREFs", 102 | input: ` .`, 103 | expect: quad.Quad{ 104 | Subject: quad.Raw(""), 105 | Predicate: quad.Raw(""), 106 | Object: quad.Raw(""), 107 | Label: nil, 108 | }, 109 | err: nil, 110 | }, 111 | 112 | // N-Triples example 3. 113 | { 114 | message: "parse triple with blank node labelled subject and object and IRIREF predicate (1)", 115 | input: `_:alice _:bob .`, 116 | expect: quad.Quad{ 117 | Subject: quad.Raw("_:alice"), 118 | Predicate: quad.Raw(""), 119 | Object: quad.Raw("_:bob"), 120 | Label: nil, 121 | }, 122 | err: nil, 123 | }, 124 | { 125 | message: "parse triple with blank node labelled subject and object and IRIREF predicate (2)", 126 | input: `_:bob _:alice .`, 127 | expect: quad.Quad{ 128 | Subject: quad.Raw("_:bob"), 129 | Predicate: quad.Raw(""), 130 | Object: quad.Raw("_:alice"), 131 | Label: nil, 132 | }, 133 | err: nil, 134 | }, 135 | 136 | // N-Quads example 1. 137 | { 138 | message: "parse quad with commment", 139 | input: ` . # comments here`, 140 | expect: quad.Quad{ 141 | Subject: quad.Raw(""), 142 | Predicate: quad.Raw(""), 143 | Object: quad.Raw(""), 144 | Label: quad.Raw(""), 145 | }, 146 | err: nil, 147 | }, 148 | { 149 | message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (1)", 150 | input: `_:subject1 "object1" .`, 151 | expect: quad.Quad{ 152 | Subject: quad.Raw("_:subject1"), 153 | Predicate: quad.Raw(""), 154 | Object: quad.Raw(`"object1"`), 155 | Label: quad.Raw(""), 156 | }, 157 | err: nil, 158 | }, 159 | { 160 | message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (2)", 161 | input: `_:subject2 "object2" .`, 162 | expect: quad.Quad{ 163 | Subject: quad.Raw("_:subject2"), 164 | Predicate: quad.Raw(""), 165 | Object: quad.Raw(`"object2"`), 166 | Label: quad.Raw(""), 167 | }, 168 | err: nil, 169 | }, 170 | 171 | // N-Quads example 2. 172 | { 173 | message: "parse quad with all IRIREF parts", 174 | input: ` .`, 175 | expect: quad.Quad{ 176 | Subject: quad.Raw(""), 177 | Predicate: quad.Raw(""), 178 | Object: quad.Raw(""), 179 | Label: quad.Raw(""), 180 | }, 181 | err: nil, 182 | }, 183 | 184 | // N-Quads example 3. 185 | { 186 | message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (1)", 187 | input: `_:alice _:bob .`, 188 | expect: quad.Quad{ 189 | Subject: quad.Raw("_:alice"), 190 | Predicate: quad.Raw(""), 191 | Object: quad.Raw("_:bob"), 192 | Label: quad.Raw(""), 193 | }, 194 | err: nil, 195 | }, 196 | { 197 | message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (2)", 198 | input: `_:bob _:alice .`, 199 | expect: quad.Quad{ 200 | Subject: quad.Raw("_:bob"), 201 | Predicate: quad.Raw(""), 202 | Object: quad.Raw("_:alice"), 203 | Label: quad.Raw(""), 204 | }, 205 | err: nil, 206 | }, 207 | 208 | // N-Triples tests. 209 | { 210 | message: "parse triple with all IRIREF parts", 211 | input: ` .`, 212 | expect: quad.Quad{ 213 | Subject: quad.Raw(""), 214 | Predicate: quad.Raw(""), 215 | Object: quad.Raw(""), 216 | Label: nil, 217 | }, 218 | err: nil, 219 | }, 220 | { 221 | message: "parse triple with all IRIREF parts", 222 | input: ` .`, 223 | expect: quad.Quad{ 224 | Subject: quad.Raw(""), 225 | Predicate: quad.Raw(""), 226 | Object: quad.Raw(""), 227 | Label: nil, 228 | }, 229 | err: nil, 230 | }, 231 | { 232 | message: "parse triple with IRIREF schema on literal object", 233 | input: ` "1990-07-04"^^ .`, 234 | expect: quad.Quad{ 235 | Subject: quad.Raw(""), 236 | Predicate: quad.Raw(""), 237 | Object: quad.Raw(`"1990-07-04"^^`), 238 | Label: nil, 239 | }, 240 | err: nil, 241 | }, 242 | { 243 | message: "parse commented IRIREF in triple", 244 | input: ` .`, 245 | expect: quad.Quad{ 246 | Subject: quad.Raw(""), 247 | Predicate: quad.Raw(""), 248 | Object: quad.Raw(""), 249 | Label: nil, 250 | }, 251 | err: nil, 252 | }, 253 | { 254 | message: "parse triple with literal subject", 255 | input: ` "Mona Lisa" .`, 256 | expect: quad.Quad{ 257 | Subject: quad.Raw(""), 258 | Predicate: quad.Raw(""), 259 | Object: quad.Raw(`"Mona Lisa"`), 260 | Label: nil, 261 | }, 262 | err: nil, 263 | }, 264 | { 265 | message: "parse triple with all IRIREF parts (1)", 266 | input: ` .`, 267 | expect: quad.Quad{ 268 | Subject: quad.Raw(""), 269 | Predicate: quad.Raw(""), 270 | Object: quad.Raw(""), 271 | Label: nil, 272 | }, 273 | err: nil, 274 | }, 275 | { 276 | message: "parse triple with all IRIREF parts (2)", 277 | input: ` .`, 278 | expect: quad.Quad{ 279 | Subject: quad.Raw(""), 280 | Predicate: quad.Raw(""), 281 | Object: quad.Raw(""), 282 | Label: nil, 283 | }, 284 | err: nil, 285 | }, 286 | 287 | // N-Quads tests. 288 | { 289 | message: "parse commented IRIREF in quad (1)", 290 | input: ` .`, 291 | expect: quad.Quad{ 292 | Subject: quad.Raw(""), 293 | Predicate: quad.Raw(""), 294 | Object: quad.Raw(""), 295 | Label: quad.Raw(""), 296 | }, 297 | err: nil, 298 | }, 299 | { 300 | message: "parse quad with all IRIREF parts", 301 | input: ` .`, 302 | expect: quad.Quad{ 303 | Subject: quad.Raw(""), 304 | Predicate: quad.Raw(""), 305 | Object: quad.Raw(""), 306 | Label: quad.Raw(""), 307 | }, 308 | err: nil, 309 | }, 310 | { 311 | message: "parse quad with IRIREF schema on literal object", 312 | input: ` "1990-07-04"^^ .`, 313 | expect: quad.Quad{ 314 | Subject: quad.Raw(""), 315 | Predicate: quad.Raw(""), 316 | Object: quad.Raw(`"1990-07-04"^^`), 317 | Label: quad.Raw(""), 318 | }, 319 | err: nil, 320 | }, 321 | { 322 | message: "parse commented IRIREF in quad (2)", 323 | input: ` .`, 324 | expect: quad.Quad{ 325 | Subject: quad.Raw(""), 326 | Predicate: quad.Raw(""), 327 | Object: quad.Raw(""), 328 | Label: quad.Raw(""), 329 | }, 330 | err: nil, 331 | }, 332 | { 333 | message: "parse literal object and colon qualified label in quad", 334 | input: ` "Mona Lisa" .`, 335 | expect: quad.Quad{ 336 | Subject: quad.Raw(""), 337 | Predicate: quad.Raw(""), 338 | Object: quad.Raw(`"Mona Lisa"`), 339 | Label: quad.Raw(""), 340 | }, 341 | err: nil, 342 | }, 343 | { 344 | message: "parse all IRIREF parts with colon qualified label in quad (1)", 345 | input: ` .`, 346 | expect: quad.Quad{ 347 | Subject: quad.Raw(""), 348 | Predicate: quad.Raw(""), 349 | Object: quad.Raw(""), 350 | Label: quad.Raw(""), 351 | }, 352 | err: nil, 353 | }, 354 | { 355 | message: "parse all IRIREF parts with colon qualified label in quad (2)", 356 | input: ` .`, 357 | expect: quad.Quad{ 358 | Subject: quad.Raw(""), 359 | Predicate: quad.Raw(""), 360 | Object: quad.Raw(""), 361 | Label: quad.Raw(""), 362 | }, 363 | err: nil, 364 | }, 365 | { 366 | message: "parse all IRIREF parts (quad section - 1)", 367 | input: ` .`, 368 | expect: quad.Quad{ 369 | Subject: quad.Raw(""), 370 | Predicate: quad.Raw(""), 371 | Object: quad.Raw(""), 372 | Label: nil, 373 | }, 374 | err: nil, 375 | }, 376 | { 377 | message: "parse all IRIREF parts (quad section - 2)", 378 | input: ` .`, 379 | expect: quad.Quad{ 380 | Subject: quad.Raw(""), 381 | Predicate: quad.Raw(""), 382 | Object: quad.Raw(""), 383 | Label: nil, 384 | }, 385 | err: nil, 386 | }, 387 | 388 | // Invalid input. 389 | { 390 | message: "parse empty", 391 | input: ``, 392 | expect: quad.Quad{}, 393 | err: quad.ErrIncomplete, 394 | }, 395 | { 396 | message: "parse commented", 397 | input: `# comment`, 398 | expect: quad.Quad{}, 399 | err: fmt.Errorf("%v: unexpected rune '#' at 0", quad.ErrInvalid), 400 | }, 401 | { 402 | message: "parse incomplete quad", 403 | input: ` .`, 404 | expect: quad.Quad{ 405 | Subject: quad.Raw(""), 406 | Predicate: quad.Raw(""), 407 | Object: nil, 408 | Label: nil, 409 | }, 410 | err: fmt.Errorf("%v: unexpected rune '.' at 78", quad.ErrInvalid), 411 | }, 412 | { 413 | message: "parse incomplete quad", 414 | input: ` .`, 415 | expect: quad.Quad{ 416 | Subject: quad.Raw(""), 417 | Predicate: quad.Raw(""), 418 | Object: nil, 419 | Label: nil, 420 | }, 421 | err: fmt.Errorf("%v: unexpected rune '.' at 78", quad.ErrInvalid), 422 | }, 423 | 424 | // Example quad from issue #140. 425 | { 426 | message: "parse incomplete quad", 427 | input: "\t\t.", 428 | expect: quad.Quad{ 429 | Subject: quad.Raw(""), 430 | Predicate: quad.Raw(""), 431 | Object: nil, 432 | Label: nil, 433 | }, 434 | err: fmt.Errorf("%v: unexpected rune '\"' at 99", quad.ErrInvalid), 435 | }, 436 | } 437 | 438 | func TestParseRaw(t *testing.T) { 439 | for _, test := range testNQuadsRaw { 440 | got, err := ParseRaw(test.input) 441 | if err != test.err && (err != nil && err.Error() != test.err.Error()) { 442 | t.Errorf("Unexpected error when %s: got:%v expect:%v", test.message, err, test.err) 443 | } 444 | if !reflect.DeepEqual(got, test.expect) { 445 | t.Errorf("Failed to %s, %q, got:%#v expect:%#v", test.message, test.input, got, test.expect) 446 | } 447 | } 448 | } 449 | 450 | func TestRawDecoder(t *testing.T) { 451 | dec := NewReader(strings.NewReader(document), true) 452 | var n int 453 | for { 454 | q, err := dec.ReadQuad() 455 | if err != nil { 456 | if err != io.EOF { 457 | t.Fatalf("Failed to read documentRaw: %v", err) 458 | } 459 | break 460 | } 461 | if !q.IsValid() { 462 | t.Errorf("Unexpected quad, got:%v", q) 463 | } 464 | n++ 465 | } 466 | if n != 20 { 467 | t.Errorf("Unexpected number of quads read, got:%d expect:20", n) 468 | } 469 | } 470 | 471 | func TestRDFWorkingGroupSuitRaw(t *testing.T) { 472 | // These tests erroneously pass because the parser does not 473 | // perform semantic testing on the URI in the IRIRef as required 474 | // by the specification. So, we skip them. 475 | skip := map[string]bool{ 476 | // N-Triples. 477 | "nt-syntax-bad-uri-06.nt": true, 478 | "nt-syntax-bad-uri-07.nt": true, 479 | "nt-syntax-bad-uri-08.nt": true, 480 | "nt-syntax-bad-uri-09.nt": true, 481 | 482 | // N-Quads. 483 | "nq-syntax-bad-uri-01.nq": true, 484 | "nt-syntax-bad-uri-06.nq": true, 485 | "nt-syntax-bad-uri-07.nq": true, 486 | "nt-syntax-bad-uri-08.nq": true, 487 | "nt-syntax-bad-uri-09.nq": true, 488 | } 489 | 490 | for _, file := range []string{ 491 | filepath.Join("..", "ntriple_tests.tar.gz"), 492 | filepath.Join("..", "nquad_tests.tar.gz"), 493 | } { 494 | suite, err := os.Open(file) 495 | if err != nil { 496 | t.Fatalf("Failed to open test suite in %q: %v", file, err) 497 | } 498 | defer suite.Close() 499 | 500 | r, err := gzip.NewReader(suite) 501 | if err != nil { 502 | t.Fatalf("Failed to uncompress test suite in %q: %v", file, err) 503 | } 504 | 505 | tr := tar.NewReader(r) 506 | for { 507 | h, err := tr.Next() 508 | if err != nil { 509 | if err == io.EOF { 510 | break 511 | } 512 | t.Fatalf("Unexpected error while reading suite archive: %v", err) 513 | } 514 | 515 | h.Name = filepath.Base(h.Name) 516 | if (filepath.Ext(h.Name) != ".nt" && filepath.Ext(h.Name) != ".nq") || skip[h.Name] { 517 | continue 518 | } 519 | 520 | isBad := strings.Contains(h.Name, "bad") 521 | 522 | dec := NewReader(tr, true) 523 | for { 524 | _, err := dec.ReadQuad() 525 | if err == io.EOF { 526 | break 527 | } 528 | got := err == nil 529 | if got == isBad { 530 | t.Errorf("Unexpected error return for test suite item %q, got: %v", h.Name, err) 531 | } 532 | } 533 | } 534 | } 535 | } 536 | 537 | func TestUnescapeRaw(t *testing.T) { 538 | for _, test := range escapeSequenceTests { 539 | got := unEscapeRaw([]rune(test.input), true) 540 | if got == nil || quad.ToString(got) != test.expect { 541 | t.Errorf("Failed to properly unescape %q, got:%q expect:%q", test.input, got, test.expect) 542 | } 543 | } 544 | } 545 | 546 | func BenchmarkParserRaw(b *testing.B) { 547 | for n := 0; n < b.N; n++ { 548 | result, _ = ParseRaw(" \"object of some real\\tlength\"@en . # comment") 549 | } 550 | } 551 | -------------------------------------------------------------------------------- /nquads/typed.rl: -------------------------------------------------------------------------------- 1 | // GO SOURCE FILE MACHINE GENERATED BY RAGEL; DO NOT EDIT 2 | 3 | // Copyright 2014 The Cayley Authors. All rights reserved. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package nquads 18 | 19 | import ( 20 | "fmt" 21 | "unicode" 22 | 23 | "github.com/cayleygraph/quad" 24 | ) 25 | 26 | %%{ 27 | machine typed; 28 | 29 | action Escape { 30 | isEscaped = true 31 | } 32 | 33 | action Quote { 34 | isQuoted = true 35 | } 36 | 37 | action StartSubject { 38 | subject = p 39 | } 40 | 41 | action StartPredicate { 42 | predicate = p 43 | } 44 | 45 | action StartObject { 46 | object = p 47 | } 48 | 49 | action StartLabel { 50 | label = p 51 | } 52 | 53 | action Spec { 54 | spec = p 55 | } 56 | 57 | action SetSubject { 58 | if subject < 0 { 59 | panic("unexpected parser state: subject start not set") 60 | } 61 | q.Subject = unEscape(data[subject:p], spec-subject, isQuoted, isEscaped) 62 | isEscaped = false 63 | isQuoted = false 64 | } 65 | 66 | action SetPredicate { 67 | if predicate < 0 { 68 | panic("unexpected parser state: predicate start not set") 69 | } 70 | q.Predicate = unEscape(data[predicate:p], spec-predicate, isQuoted, isEscaped) 71 | isEscaped = false 72 | isQuoted = false 73 | } 74 | 75 | action SetObject { 76 | if object < 0 { 77 | panic("unexpected parser state: object start not set") 78 | } 79 | q.Object = unEscape(data[object:p], spec-object, isQuoted, isEscaped) 80 | isEscaped = false 81 | isQuoted = false 82 | } 83 | 84 | action SetLabel { 85 | if label < 0 { 86 | panic("unexpected parser state: label start not set") 87 | } 88 | q.Label = unEscape(data[label:p], spec-label, isQuoted, isEscaped) 89 | isEscaped = false 90 | isQuoted = false 91 | } 92 | 93 | action Return { 94 | return q, nil 95 | } 96 | 97 | action Comment { 98 | } 99 | 100 | action Error { 101 | if p < len(data) { 102 | if r := data[p]; r < unicode.MaxASCII { 103 | return q, fmt.Errorf("%v: unexpected rune %q at %d", quad.ErrInvalid, data[p], p) 104 | } else { 105 | return q, fmt.Errorf("%v: unexpected rune %q (\\u%04x) at %d", quad.ErrInvalid, data[p], data[p], p) 106 | } 107 | } 108 | return q, quad.ErrIncomplete 109 | } 110 | 111 | include nquads "nquads.rl"; 112 | 113 | literal = STRING_LITERAL | STRING_LITERAL_QUOTE % Quote | STRING_LITERAL_QUOTE ('^^' >Spec IRIREF | LANGTAG >Spec) ; 114 | 115 | subject = (literal | BLANK_NODE_LABEL) ; 116 | predicate = literal ; 117 | object = (literal | BLANK_NODE_LABEL) ; 118 | graphLabel = (literal | BLANK_NODE_LABEL) ; 119 | 120 | statement := ( 121 | whitespace* subject >StartSubject %SetSubject 122 | whitespace+ predicate >StartPredicate %SetPredicate 123 | whitespace+ object >StartObject %SetObject 124 | (whitespace+ graphLabel >StartLabel %SetLabel)? 125 | whitespace* '.' whitespace* ('#' any*)? >Comment 126 | ) %Return @!Error ; 127 | 128 | write data; 129 | }%% 130 | 131 | // ParseTyped returns a valid quad.Quad or a non-nil error. ParseTyped does 132 | // handle comments except where the comment placement does not prevent 133 | // a complete valid quad.Quad from being defined. 134 | func Parse(statement string) (quad.Quad, error) { 135 | data := []rune(statement) 136 | 137 | var ( 138 | cs, p int 139 | pe = len(data) 140 | eof = pe 141 | 142 | subject = -1 143 | predicate = -1 144 | object = -1 145 | label = -1 146 | 147 | spec = -1 148 | 149 | isEscaped bool 150 | isQuoted bool 151 | 152 | q quad.Quad 153 | ) 154 | 155 | %%write init; 156 | 157 | %%write exec; 158 | 159 | return quad.Quad{}, quad.ErrInvalid 160 | } 161 | -------------------------------------------------------------------------------- /nquads/typed_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package nquads 16 | 17 | import ( 18 | "archive/tar" 19 | "compress/gzip" 20 | "fmt" 21 | "io" 22 | "os" 23 | "path/filepath" 24 | "reflect" 25 | "strings" 26 | "testing" 27 | "time" 28 | 29 | "github.com/cayleygraph/quad" 30 | "github.com/stretchr/testify/require" 31 | ) 32 | 33 | var testNQuads = []struct { 34 | message string 35 | input string 36 | expect quad.Quad 37 | err error 38 | }{ 39 | // Tests from original nquads. 40 | 41 | // NTriple tests. 42 | { 43 | message: "parse simple triples", 44 | input: "this is valid .", 45 | expect: quad.Quad{ 46 | Subject: quad.Raw("this"), 47 | Predicate: quad.Raw("is"), 48 | Object: quad.Raw("valid"), 49 | Label: nil, 50 | }, 51 | }, 52 | { 53 | message: "parse quoted triples", 54 | input: `this is "valid too" .`, 55 | expect: quad.Quad{ 56 | Subject: quad.Raw("this"), 57 | Predicate: quad.Raw("is"), 58 | Object: quad.String("valid too"), 59 | Label: nil, 60 | }, 61 | }, 62 | { 63 | message: "parse escaped quoted triples", 64 | input: `he said "\"That's all folks\"" .`, 65 | expect: quad.Quad{ 66 | Subject: quad.Raw("he"), 67 | Predicate: quad.Raw("said"), 68 | Object: quad.String(`"That's all folks"`), 69 | Label: nil, 70 | }, 71 | }, 72 | { 73 | message: "parse an example real triple", 74 | input: `":/guid/9202a8c04000641f80000000010c843c" "name" "George Morris" .`, 75 | expect: quad.Quad{ 76 | Subject: quad.String(":/guid/9202a8c04000641f80000000010c843c"), 77 | Predicate: quad.String("name"), 78 | Object: quad.String("George Morris"), 79 | Label: nil, 80 | }, 81 | }, 82 | { 83 | message: "parse a pathologically spaced triple", 84 | input: "foo is \"\\tA big tough\\r\\nDeal\\\\\" .", 85 | expect: quad.Quad{ 86 | Subject: quad.Raw("foo"), 87 | Predicate: quad.Raw("is"), 88 | Object: quad.String("\tA big tough\r\nDeal\\"), 89 | Label: nil, 90 | }, 91 | }, 92 | 93 | // NQuad tests. 94 | { 95 | message: "parse a simple quad", 96 | input: "this is valid quad .", 97 | expect: quad.Quad{ 98 | Subject: quad.Raw("this"), 99 | Predicate: quad.Raw("is"), 100 | Object: quad.Raw("valid"), 101 | Label: quad.Raw("quad"), 102 | }, 103 | }, 104 | { 105 | message: "parse a quoted quad", 106 | input: `this is valid "quad thing" .`, 107 | expect: quad.Quad{ 108 | Subject: quad.Raw("this"), 109 | Predicate: quad.Raw("is"), 110 | Object: quad.Raw("valid"), 111 | Label: quad.String("quad thing"), 112 | }, 113 | }, 114 | { 115 | message: "parse crazy escaped quads", 116 | input: `"\"this" "\"is" "\"valid" "\"quad thing".`, 117 | expect: quad.Quad{ 118 | Subject: quad.String(`"this`), 119 | Predicate: quad.String(`"is`), 120 | Object: quad.String(`"valid`), 121 | Label: quad.String(`"quad thing`), 122 | }, 123 | }, 124 | 125 | // NTriple official tests. 126 | { 127 | message: "handle simple case with comments", 128 | input: " . # comment", 129 | expect: quad.Quad{ 130 | Subject: quad.IRI("http://example/s"), 131 | Predicate: quad.IRI("http://example/p"), 132 | Object: quad.IRI("http://example/o"), 133 | Label: nil, 134 | }, 135 | }, 136 | { 137 | message: "handle simple case with comments", 138 | input: " _:o . # comment", 139 | expect: quad.Quad{ 140 | Subject: quad.IRI("http://example/s"), 141 | Predicate: quad.IRI("http://example/p"), 142 | Object: quad.BNode("o"), 143 | Label: nil, 144 | }, 145 | }, 146 | { 147 | message: "handle simple case with comments", 148 | input: " \"o\" . # comment", 149 | expect: quad.Quad{ 150 | Subject: quad.IRI("http://example/s"), 151 | Predicate: quad.IRI("http://example/p"), 152 | Object: quad.String("o"), 153 | Label: nil, 154 | }, 155 | }, 156 | { 157 | message: "handle simple case with comments", 158 | input: " \"o\"^^ . # comment", 159 | expect: quad.Quad{ 160 | Subject: quad.IRI("http://example/s"), 161 | Predicate: quad.IRI("http://example/p"), 162 | Object: quad.TypedString{Value: "o", Type: "http://example/dt"}, 163 | Label: nil, 164 | }, 165 | }, 166 | { 167 | message: "handle simple case with typed string", 168 | input: ` "\U000000b7\n\\\u00b7"^^ . # comment`, 169 | expect: quad.Quad{ 170 | Subject: quad.IRI("http://example/s"), 171 | Predicate: quad.IRI("http://example/p"), 172 | Object: quad.TypedString{Value: "·\n\\·", Type: "http://example/dt"}, 173 | Label: nil, 174 | }, 175 | }, 176 | { 177 | message: "handle simple case with comments", 178 | input: " \"o\"@en . # comment", 179 | expect: quad.Quad{ 180 | Subject: quad.IRI("http://example/s"), 181 | Predicate: quad.IRI("http://example/p"), 182 | Object: quad.LangString{Value: "o", Lang: "en"}, 183 | Label: nil}, 184 | }, 185 | { 186 | message: "handle simple case with lang string", 187 | input: " \"Tomás de Torquemada\"@es . # comment", 188 | expect: quad.Quad{ 189 | Subject: quad.IRI("http://example/s"), 190 | Predicate: quad.IRI("http://example/p"), 191 | Object: quad.LangString{Value: "Tomás de Torquemada", Lang: "es"}, 192 | Label: nil}, 193 | }, 194 | 195 | // Tests taken from http://www.w3.org/TR/n-quads/ and http://www.w3.org/TR/n-triples/. 196 | 197 | // _:100000 . # example from 30movies 198 | { 199 | message: "parse triple with commment", 200 | input: `_:100000 . # example from 30movies`, 201 | expect: quad.Quad{ 202 | Subject: quad.BNode("100000"), 203 | Predicate: quad.IRI("/film/performance/actor"), 204 | Object: quad.IRI("/en/larry_fine_1902"), 205 | Label: nil, 206 | }, 207 | err: nil, 208 | }, 209 | // _:10011 "Tomás de Torquemada" . # example from 30movies with unicode 210 | { 211 | message: "parse triple with commment", 212 | input: `_:10011 "Tomás de Torquemada" . # example from 30movies with unicode`, 213 | expect: quad.Quad{ 214 | Subject: quad.BNode("10011"), 215 | Predicate: quad.IRI("/film/performance/character"), 216 | Object: quad.String("Tomás de Torquemada"), 217 | Label: nil, 218 | }, 219 | err: nil, 220 | }, 221 | 222 | // N-Triples example 1. 223 | { 224 | message: "parse triple with commment", 225 | input: ` . # comments here`, 226 | expect: quad.Quad{ 227 | Subject: quad.IRI("http://one.example/subject1"), 228 | Predicate: quad.IRI("http://one.example/predicate1"), 229 | Object: quad.IRI("http://one.example/object1"), 230 | Label: nil, 231 | }, 232 | err: nil, 233 | }, 234 | { 235 | message: "parse triple with blank subject node, literal object and no comment (1)", 236 | input: `_:subject1 "object1" .`, 237 | expect: quad.Quad{ 238 | Subject: quad.BNode("subject1"), 239 | Predicate: quad.IRI("http://an.example/predicate1"), 240 | Object: quad.String("object1"), 241 | Label: nil, 242 | }, 243 | err: nil, 244 | }, 245 | { 246 | message: "parse triple with blank subject node, literal object and no comment (2)", 247 | input: `_:subject2 "object2" .`, 248 | expect: quad.Quad{ 249 | Subject: quad.BNode("subject2"), 250 | Predicate: quad.IRI("http://an.example/predicate2"), 251 | Object: quad.String("object2"), 252 | Label: nil, 253 | }, 254 | err: nil, 255 | }, 256 | 257 | // N-Triples example 2. 258 | { 259 | message: "parse triple with three IRIREFs", 260 | input: ` .`, 261 | expect: quad.Quad{ 262 | Subject: quad.IRI("http://example.org/#spiderman"), 263 | Predicate: quad.IRI("http://www.perceive.net/schemas/relationship/enemyOf"), 264 | Object: quad.IRI("http://example.org/#green-goblin"), 265 | Label: nil, 266 | }, 267 | err: nil, 268 | }, 269 | 270 | // N-Triples example 3. 271 | { 272 | message: "parse triple with blank node labelled subject and object and IRIREF predicate (1)", 273 | input: `_:alice _:bob .`, 274 | expect: quad.Quad{ 275 | Subject: quad.BNode("alice"), 276 | Predicate: quad.IRI("http://xmlns.com/foaf/0.1/knows"), 277 | Object: quad.BNode("bob"), 278 | Label: nil, 279 | }, 280 | err: nil, 281 | }, 282 | { 283 | message: "parse triple with blank node labelled subject and object and IRIREF predicate (2)", 284 | input: `_:bob _:alice .`, 285 | expect: quad.Quad{ 286 | Subject: quad.BNode("bob"), 287 | Predicate: quad.IRI("http://xmlns.com/foaf/0.1/knows"), 288 | Object: quad.BNode("alice"), 289 | Label: nil, 290 | }, 291 | err: nil, 292 | }, 293 | 294 | // N-Quads example 1. 295 | { 296 | message: "parse quad with commment", 297 | input: ` . # comments here`, 298 | expect: quad.Quad{ 299 | Subject: quad.IRI("http://one.example/subject1"), 300 | Predicate: quad.IRI("http://one.example/predicate1"), 301 | Object: quad.IRI("http://one.example/object1"), 302 | Label: quad.IRI("http://example.org/graph3"), 303 | }, 304 | err: nil, 305 | }, 306 | { 307 | message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (1)", 308 | input: `_:subject1 "object1" .`, 309 | expect: quad.Quad{ 310 | Subject: quad.BNode("subject1"), 311 | Predicate: quad.IRI("http://an.example/predicate1"), 312 | Object: quad.String("object1"), 313 | Label: quad.IRI("http://example.org/graph1"), 314 | }, 315 | err: nil, 316 | }, 317 | { 318 | message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (2)", 319 | input: `_:subject2 "object2" .`, 320 | expect: quad.Quad{ 321 | Subject: quad.BNode("subject2"), 322 | Predicate: quad.IRI("http://an.example/predicate2"), 323 | Object: quad.String("object2"), 324 | Label: quad.IRI("http://example.org/graph5"), 325 | }, 326 | err: nil, 327 | }, 328 | 329 | // N-Quads example 2. 330 | { 331 | message: "parse quad with all IRIREF parts", 332 | input: ` .`, 333 | expect: quad.Quad{ 334 | Subject: quad.IRI("http://example.org/#spiderman"), 335 | Predicate: quad.IRI("http://www.perceive.net/schemas/relationship/enemyOf"), 336 | Object: quad.IRI("http://example.org/#green-goblin"), 337 | Label: quad.IRI("http://example.org/graphs/spiderman"), 338 | }, 339 | err: nil, 340 | }, 341 | 342 | // N-Quads example 3. 343 | { 344 | message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (1)", 345 | input: `_:alice _:bob .`, 346 | expect: quad.Quad{ 347 | Subject: quad.BNode("alice"), 348 | Predicate: quad.IRI("http://xmlns.com/foaf/0.1/knows"), 349 | Object: quad.BNode("bob"), 350 | Label: quad.IRI("http://example.org/graphs/john"), 351 | }, 352 | err: nil, 353 | }, 354 | { 355 | message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (2)", 356 | input: `_:bob _:alice .`, 357 | expect: quad.Quad{ 358 | Subject: quad.BNode("bob"), 359 | Predicate: quad.IRI("http://xmlns.com/foaf/0.1/knows"), 360 | Object: quad.BNode("alice"), 361 | Label: quad.IRI("http://example.org/graphs/james"), 362 | }, 363 | err: nil, 364 | }, 365 | 366 | // N-Triples tests. 367 | { 368 | message: "parse triple with all IRIREF parts", 369 | input: ` .`, 370 | expect: quad.Quad{ 371 | Subject: quad.IRI("http://example.org/bob#me"), 372 | Predicate: quad.IRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), 373 | Object: quad.IRI("http://xmlns.com/foaf/0.1/Person"), 374 | Label: nil, 375 | }, 376 | err: nil, 377 | }, 378 | { 379 | message: "parse triple with all IRIREF parts", 380 | input: ` .`, 381 | expect: quad.Quad{ 382 | Subject: quad.IRI("http://example.org/bob#me"), 383 | Predicate: quad.IRI("http://xmlns.com/foaf/0.1/knows"), 384 | Object: quad.IRI("http://example.org/alice#me"), 385 | Label: nil, 386 | }, 387 | err: nil, 388 | }, 389 | { 390 | message: "parse triple with IRIREF schema on literal object", 391 | input: ` "1990-07-04"^^ .`, 392 | expect: quad.Quad{ 393 | Subject: quad.IRI("http://example.org/bob#me"), 394 | Predicate: quad.IRI("http://schema.org/birthDate"), 395 | Object: quad.TypedString{Value: "1990-07-04", Type: "http://www.w3.org/2001/XMLSchema#date"}, 396 | Label: nil, 397 | }, 398 | err: nil, 399 | }, 400 | { 401 | message: "parse triple with IRIREF schema on literal object and dateTime", 402 | input: ` "1990-07-04T17:25:41Z"^^ .`, 403 | expect: quad.Quad{ 404 | Subject: quad.IRI("http://example.org/bob#me"), 405 | Predicate: quad.IRI("http://schema.org/birthDate"), 406 | Object: func() quad.Time { 407 | t, err := time.Parse(time.RFC3339, "1990-07-04T17:25:41Z") 408 | if err != nil { 409 | panic(err) 410 | } 411 | return quad.Time(t) 412 | }(), 413 | Label: nil, 414 | }, 415 | err: nil, 416 | }, 417 | { 418 | message: "parse commented IRIREF in triple", 419 | input: ` .`, 420 | expect: quad.Quad{ 421 | Subject: quad.IRI("http://example.org/bob#me"), 422 | Predicate: quad.IRI("http://xmlns.com/foaf/0.1/topic_interest"), 423 | Object: quad.IRI("http://www.wikidata.org/entity/Q12418"), 424 | Label: nil, 425 | }, 426 | err: nil, 427 | }, 428 | { 429 | message: "parse triple with literal subject", 430 | input: ` "Mona Lisa" .`, 431 | expect: quad.Quad{ 432 | Subject: quad.IRI("http://www.wikidata.org/entity/Q12418"), 433 | Predicate: quad.IRI("http://purl.org/dc/terms/title"), 434 | Object: quad.String("Mona Lisa"), 435 | Label: nil, 436 | }, 437 | err: nil, 438 | }, 439 | { 440 | message: "parse triple with all IRIREF parts (1)", 441 | input: ` .`, 442 | expect: quad.Quad{ 443 | Subject: quad.IRI("http://www.wikidata.org/entity/Q12418"), 444 | Predicate: quad.IRI("http://purl.org/dc/terms/creator"), 445 | Object: quad.IRI("http://dbpedia.org/resource/Leonardo_da_Vinci"), 446 | Label: nil, 447 | }, 448 | err: nil, 449 | }, 450 | { 451 | message: "parse triple with all IRIREF parts (2)", 452 | input: ` .`, 453 | expect: quad.Quad{ 454 | Subject: quad.IRI("http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619"), 455 | Predicate: quad.IRI("http://purl.org/dc/terms/subject"), 456 | Object: quad.IRI("http://www.wikidata.org/entity/Q12418"), 457 | Label: nil, 458 | }, 459 | err: nil, 460 | }, 461 | 462 | // N-Quads tests. 463 | { 464 | message: "parse commented IRIREF in quad (1)", 465 | input: ` .`, 466 | expect: quad.Quad{ 467 | Subject: quad.IRI("http://example.org/bob#me"), 468 | Predicate: quad.IRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), 469 | Object: quad.IRI("http://xmlns.com/foaf/0.1/Person"), 470 | Label: quad.IRI("http://example.org/bob"), 471 | }, 472 | err: nil, 473 | }, 474 | { 475 | message: "parse quad with all IRIREF parts", 476 | input: ` .`, 477 | expect: quad.Quad{ 478 | Subject: quad.IRI("http://example.org/bob#me"), 479 | Predicate: quad.IRI("http://xmlns.com/foaf/0.1/knows"), 480 | Object: quad.IRI("http://example.org/alice#me"), 481 | Label: quad.IRI("http://example.org/bob"), 482 | }, 483 | err: nil, 484 | }, 485 | { 486 | message: "parse quad with IRIREF schema on literal object", 487 | input: ` "1990-07-04"^^ .`, 488 | expect: quad.Quad{ 489 | Subject: quad.IRI("http://example.org/bob#me"), 490 | Predicate: quad.IRI("http://schema.org/birthDate"), 491 | Object: quad.TypedString{Value: "1990-07-04", Type: "http://www.w3.org/2001/XMLSchema#date"}, 492 | Label: quad.IRI("http://example.org/bob"), 493 | }, 494 | err: nil, 495 | }, 496 | { 497 | message: "parse commented IRIREF in quad (2)", 498 | input: ` .`, 499 | expect: quad.Quad{ 500 | Subject: quad.IRI("http://example.org/bob#me"), 501 | Predicate: quad.IRI("http://xmlns.com/foaf/0.1/topic_interest"), 502 | Object: quad.IRI("http://www.wikidata.org/entity/Q12418"), 503 | Label: quad.IRI("http://example.org/bob"), 504 | }, 505 | err: nil, 506 | }, 507 | { 508 | message: "parse literal object and colon qualified label in quad", 509 | input: ` "Mona Lisa" .`, 510 | expect: quad.Quad{ 511 | Subject: quad.IRI("http://www.wikidata.org/entity/Q12418"), 512 | Predicate: quad.IRI("http://purl.org/dc/terms/title"), 513 | Object: quad.String("Mona Lisa"), 514 | Label: quad.IRI("https://www.wikidata.org/wiki/Special:EntityData/Q12418"), 515 | }, 516 | err: nil, 517 | }, 518 | { 519 | message: "parse all IRIREF parts with colon qualified label in quad (1)", 520 | input: ` .`, 521 | expect: quad.Quad{ 522 | Subject: quad.IRI("http://www.wikidata.org/entity/Q12418"), 523 | Predicate: quad.IRI("http://purl.org/dc/terms/creator"), 524 | Object: quad.IRI("http://dbpedia.org/resource/Leonardo_da_Vinci"), 525 | Label: quad.IRI("https://www.wikidata.org/wiki/Special:EntityData/Q12418"), 526 | }, 527 | err: nil, 528 | }, 529 | { 530 | message: "parse all IRIREF parts with colon qualified label in quad (2)", 531 | input: ` .`, 532 | expect: quad.Quad{ 533 | Subject: quad.IRI("http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619"), 534 | Predicate: quad.IRI("http://purl.org/dc/terms/subject"), 535 | Object: quad.IRI("http://www.wikidata.org/entity/Q12418"), 536 | Label: quad.IRI("https://www.wikidata.org/wiki/Special:EntityData/Q12418"), 537 | }, 538 | err: nil, 539 | }, 540 | { 541 | message: "parse all IRIREF parts (quad section - 1)", 542 | input: ` .`, 543 | expect: quad.Quad{ 544 | Subject: quad.IRI("http://example.org/bob"), 545 | Predicate: quad.IRI("http://purl.org/dc/terms/publisher"), 546 | Object: quad.IRI("http://example.org"), 547 | Label: nil, 548 | }, 549 | err: nil, 550 | }, 551 | { 552 | message: "parse all IRIREF parts (quad section - 2)", 553 | input: ` .`, 554 | expect: quad.Quad{ 555 | Subject: quad.IRI("http://example.org/bob"), 556 | Predicate: quad.IRI("http://purl.org/dc/terms/rights"), 557 | Object: quad.IRI("http://creativecommons.org/licenses/by/3.0/"), 558 | Label: nil, 559 | }, 560 | err: nil, 561 | }, 562 | 563 | // Invalid input. 564 | { 565 | message: "parse empty", 566 | input: ``, 567 | expect: quad.Quad{}, 568 | err: quad.ErrIncomplete, 569 | }, 570 | { 571 | message: "parse commented", 572 | input: `# is a comment`, 573 | expect: quad.Quad{}, 574 | err: fmt.Errorf("%v: unexpected rune '#' at 0", quad.ErrInvalid), 575 | }, 576 | { 577 | message: "parse commented internal (1)", 578 | input: `is # a comment`, 579 | expect: quad.Quad{Subject: quad.Raw("is")}, 580 | err: fmt.Errorf("%v: unexpected rune '#' at 3", quad.ErrInvalid), 581 | }, 582 | { 583 | message: "parse commented internal (2)", 584 | input: `is a # comment`, 585 | expect: quad.Quad{Subject: quad.Raw("is"), Predicate: quad.Raw("a")}, 586 | err: fmt.Errorf("%v: unexpected rune '#' at 5", quad.ErrInvalid), 587 | }, 588 | { 589 | message: "parse incomplete quad (1)", 590 | input: ` .`, 591 | expect: quad.Quad{ 592 | Subject: quad.IRI("http://example.org/bob#me"), 593 | Predicate: quad.IRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), 594 | Object: nil, 595 | Label: nil, 596 | }, 597 | err: quad.ErrIncomplete, 598 | }, 599 | { 600 | message: "parse incomplete quad (2)", 601 | input: ` .`, 602 | expect: quad.Quad{ 603 | Subject: quad.IRI("http://example.org/bob#me"), 604 | Predicate: quad.IRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), 605 | Object: nil, 606 | Label: nil, 607 | }, 608 | err: quad.ErrIncomplete, 609 | }, 610 | 611 | // Example quad from issue #140 in two forms: strict N-Quads and as quoted in issue. 612 | { 613 | message: "parse incomplete quad", 614 | input: "\t\t.", 615 | expect: quad.Quad{ 616 | Subject: quad.IRI("ns:m.0y_chx"), 617 | Predicate: quad.IRI("ns:music.recording.lyrics_website..common.webpage.uri"), 618 | Object: quad.Raw(".", 626 | expect: quad.Quad{ 627 | Subject: quad.Raw("ns:m.0y_chx"), 628 | Predicate: quad.Raw("ns:music.recording.lyrics_website..common.webpage.uri"), 629 | Object: quad.Raw(" . 655 | _:100001 . 656 | _:100002 . 657 | _:100003 . 658 | _:100004 . 659 | _:100005 . 660 | _:100006 . 661 | _:100007 . 662 | _:100008 . 663 | _:100009 . 664 | #last ten lines of 30kmoviedata.nq 665 | "Bill Fishman" . 666 | 667 | . 668 | "Matthew J. Evans" . 669 | . 670 | "Nina Bonherry" . 671 | . 672 | "Bill Roberts" . 673 | . 674 | "Christopher Ashley" . 675 | . 676 | ` 677 | 678 | func TestDecoder(t *testing.T) { 679 | dec := NewReader(strings.NewReader(document), false) 680 | var n int 681 | for { 682 | q, err := dec.ReadQuad() 683 | if err != nil { 684 | if err != io.EOF { 685 | t.Fatalf("Failed to read document: %v", err) 686 | } 687 | break 688 | } 689 | if !q.IsValid() { 690 | t.Errorf("Unexpected quad, got:%v", q) 691 | } 692 | n++ 693 | } 694 | if n != 20 { 695 | t.Errorf("Unexpected number of quads read, got:%d expect:20", n) 696 | } 697 | } 698 | 699 | func TestRDFWorkingGroupSuit(t *testing.T) { 700 | // Tests that are not passable by cquads parsing from the RDF 701 | // Working Group Suite: 702 | // 703 | // [1] Because we don't require literal quoting, we cannot 704 | // distinguish quad terms without separating whitespace. 705 | // 706 | // [2] The cquads grammar accepts these because of its relaxation. 707 | // 708 | // [3] These tests pass because the parser does not perform 709 | // semantic testing on the URI in the IRIRef as required by 710 | // the specification. 711 | skip := map[string]bool{ 712 | // N-Triples. 713 | // [1] 714 | "minimal_whitespace.nt": true, 715 | 716 | // [2] 717 | "nt-syntax-bad-num-01.nt": true, 718 | "nt-syntax-bad-num-02.nt": true, 719 | "nt-syntax-bad-num-03.nt": true, 720 | "nt-syntax-bad-prefix-01.nt": true, 721 | "nt-syntax-bad-string-02.nt": true, 722 | "nt-syntax-bad-string-03.nt": true, 723 | "nt-syntax-bad-string-04.nt": true, 724 | "nt-syntax-bad-struct-01.nt": true, 725 | "nt-syntax-bad-uri-01.nt": true, 726 | "nt-syntax-bad-uri-04.nt": true, 727 | 728 | // [3] 729 | "nt-syntax-bad-uri-06.nt": true, 730 | "nt-syntax-bad-uri-07.nt": true, 731 | "nt-syntax-bad-uri-08.nt": true, 732 | "nt-syntax-bad-uri-09.nt": true, 733 | 734 | // N-Quads. 735 | // [1] 736 | "minimal_whitespace.nq": true, 737 | 738 | // [2] 739 | "nq-syntax-bad-literal-01.nq": true, 740 | "nq-syntax-bad-literal-02.nq": true, 741 | "nq-syntax-bad-literal-03.nq": true, 742 | "nt-syntax-bad-num-01.nq": true, 743 | "nt-syntax-bad-num-02.nq": true, 744 | "nt-syntax-bad-num-03.nq": true, 745 | "nt-syntax-bad-prefix-01.nq": true, 746 | "nt-syntax-bad-string-02.nq": true, 747 | "nt-syntax-bad-string-03.nq": true, 748 | "nt-syntax-bad-string-04.nq": true, 749 | "nt-syntax-bad-struct-01.nq": true, 750 | "nt-syntax-bad-uri-01.nq": true, 751 | "nt-syntax-bad-uri-04.nq": true, 752 | 753 | // [3] 754 | "nq-syntax-bad-uri-01.nq": true, 755 | "nt-syntax-bad-uri-06.nq": true, 756 | "nt-syntax-bad-uri-07.nq": true, 757 | "nt-syntax-bad-uri-08.nq": true, 758 | "nt-syntax-bad-uri-09.nq": true, 759 | } 760 | 761 | for _, file := range []string{ 762 | filepath.Join("..", "ntriple_tests.tar.gz"), 763 | filepath.Join("..", "nquad_tests.tar.gz"), 764 | } { 765 | suite, err := os.Open(file) 766 | if err != nil { 767 | t.Fatalf("Failed to open test suite in %q: %v", file, err) 768 | } 769 | defer suite.Close() 770 | 771 | r, err := gzip.NewReader(suite) 772 | if err != nil { 773 | t.Fatalf("Failed to uncompress test suite in %q: %v", file, err) 774 | } 775 | 776 | tr := tar.NewReader(r) 777 | for { 778 | h, err := tr.Next() 779 | if err != nil { 780 | if err == io.EOF { 781 | break 782 | } 783 | t.Fatalf("Unexpected error while reading suite archive: %v", err) 784 | } 785 | 786 | h.Name = filepath.Base(h.Name) 787 | if (filepath.Ext(h.Name) != ".nt" && filepath.Ext(h.Name) != ".nq") || skip[h.Name] { 788 | continue 789 | } 790 | 791 | isBad := strings.Contains(h.Name, "bad") 792 | 793 | dec := NewReader(tr, false) 794 | for { 795 | _, err := dec.ReadQuad() 796 | if err == io.EOF { 797 | break 798 | } 799 | got := err == nil 800 | if got == isBad { 801 | t.Errorf("Unexpected error return for test suite item %q, got: %v", h.Name, err) 802 | } 803 | } 804 | } 805 | } 806 | } 807 | 808 | var escapeSequenceTests = []struct { 809 | input string 810 | expect string 811 | }{ 812 | {input: `\t`, expect: "\t"}, 813 | {input: `\b`, expect: "\b"}, 814 | {input: `\n`, expect: "\n"}, 815 | {input: `\r`, expect: "\r"}, 816 | {input: `\f`, expect: "\f"}, 817 | {input: `\\`, expect: "\\"}, 818 | {input: `\u00b7`, expect: "·"}, 819 | {input: `\U000000b7`, expect: "·"}, 820 | 821 | {input: `\t\u00b7`, expect: "\t·"}, 822 | {input: `\b\U000000b7`, expect: "\b·"}, 823 | {input: `\u00b7\n`, expect: "·\n"}, 824 | {input: `\U000000b7\r`, expect: "·\r"}, 825 | {input: `\u00b7\f\U000000b7`, expect: "·\f·"}, 826 | {input: `\U000000b7\\\u00b7`, expect: "·\\·"}, 827 | } 828 | 829 | func TestUnescape(t *testing.T) { 830 | for _, test := range escapeSequenceTests { 831 | got := unEscape([]rune(test.input), -1, false, true) 832 | if got == nil || quad.ToString(got) != test.expect { 833 | t.Errorf("Failed to properly unescape %q, got:%q expect:%q", test.input, got, test.expect) 834 | } 835 | } 836 | } 837 | 838 | var result quad.Quad 839 | 840 | func BenchmarkParser(b *testing.B) { 841 | for n := 0; n < b.N; n++ { 842 | result, _ = Parse(" \"object of some real\\tlength\"@en . # comment") 843 | } 844 | } 845 | 846 | func TestValueEncoding(t *testing.T) { 847 | vals := []quad.Value{ 848 | quad.String("some val"), 849 | quad.IRI("iri"), 850 | quad.BNode("bnode"), 851 | quad.TypedString{Value: "10", Type: "int"}, 852 | quad.LangString{Value: "val", Lang: "en"}, 853 | } 854 | enc := []string{ 855 | `"some val"`, 856 | ``, 857 | `_:bnode`, 858 | `"10"^^`, 859 | `"val"@en`, 860 | } 861 | f := quad.FormatByName("nquads") 862 | for i, v := range vals { 863 | data, err := f.MarshalValue(v) 864 | require.NoError(t, err) 865 | require.Equal(t, enc[i], string(data), string(data)) 866 | v2, err := f.UnmarshalValue(data) 867 | require.NoError(t, err) 868 | require.Equal(t, v, v2) 869 | } 870 | } 871 | -------------------------------------------------------------------------------- /ntriple_tests.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cayleygraph/quad/a9b1aedeecc3a8120c50a0c23bbb423a3ed4c5a2/ntriple_tests.tar.gz -------------------------------------------------------------------------------- /pquads/pio/io.go: -------------------------------------------------------------------------------- 1 | // Extensions for Protocol Buffers to create more go like structures. 2 | // 3 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 4 | // http://github.com/gogo/protobuf/gogoproto 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | package pio 30 | 31 | import ( 32 | "google.golang.org/protobuf/proto" 33 | ) 34 | 35 | type Writer interface { 36 | WriteMsg(proto.Message) (int, error) 37 | } 38 | 39 | type Reader interface { 40 | ReadMsg(msg proto.Message) error 41 | SkipMsg() error 42 | } 43 | 44 | type marshaler interface { 45 | MarshalTo(data []byte) (n int, err error) 46 | } 47 | 48 | func getSize(v interface{}) (int, bool) { 49 | if sz, ok := v.(interface { 50 | Size() (n int) 51 | }); ok { 52 | return sz.Size(), true 53 | } else if sz, ok := v.(interface { 54 | ProtoSize() (n int) 55 | }); ok { 56 | return sz.ProtoSize(), true 57 | } else { 58 | return 0, false 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /pquads/pio/io_test.go: -------------------------------------------------------------------------------- 1 | // Extensions for Protocol Buffers to create more go like structures. 2 | // 3 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 4 | // http://github.com/gogo/protobuf/gogoproto 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | package pio_test 30 | 31 | import ( 32 | "bytes" 33 | "errors" 34 | goio "io" 35 | "math/rand" 36 | "strconv" 37 | "testing" 38 | "time" 39 | 40 | "google.golang.org/protobuf/proto" 41 | 42 | "github.com/cayleygraph/quad/pquads" 43 | io "github.com/cayleygraph/quad/pquads/pio" 44 | ) 45 | 46 | func iotest(writer io.Writer, reader io.Reader) error { 47 | size := 1000 48 | msgs := make([]*pquads.Quad, size) 49 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 50 | for i := range msgs { 51 | msgs[i] = &pquads.Quad{Subject: strconv.Itoa(r.Int())} 52 | //issue 31 53 | if i == 5 { 54 | msgs[i] = &pquads.Quad{} 55 | } 56 | //issue 31 57 | if i == 999 { 58 | msgs[i] = &pquads.Quad{} 59 | } 60 | _, err := writer.WriteMsg(msgs[i]) 61 | if err != nil { 62 | return err 63 | } 64 | } 65 | i := 0 66 | for { 67 | msg := &pquads.Quad{} 68 | if err := reader.ReadMsg(msg); err != nil { 69 | if err == goio.EOF { 70 | break 71 | } 72 | return err 73 | } 74 | if !proto.Equal(msg, msgs[i]) { 75 | return errors.New("message not equal") 76 | } 77 | i++ 78 | } 79 | if i != size { 80 | panic("not enough messages read") 81 | } 82 | return nil 83 | } 84 | 85 | func TestVarintNormal(t *testing.T) { 86 | buf := bytes.NewBuffer(nil) 87 | writer := io.NewWriter(buf) 88 | reader := io.NewReader(buf, 1024*1024) 89 | if err := iotest(writer, reader); err != nil { 90 | t.Error(err) 91 | } 92 | } 93 | 94 | func TestVarintNoClose(t *testing.T) { 95 | buf := bytes.NewBuffer(nil) 96 | writer := io.NewWriter(buf) 97 | reader := io.NewReader(buf, 1024*1024) 98 | if err := iotest(writer, reader); err != nil { 99 | t.Error(err) 100 | } 101 | } 102 | 103 | // issue 32 104 | func TestVarintMaxSize(t *testing.T) { 105 | buf := bytes.NewBuffer(nil) 106 | writer := io.NewWriter(buf) 107 | reader := io.NewReader(buf, 20) 108 | if err := iotest(writer, reader); err != goio.ErrShortBuffer { 109 | t.Error(err) 110 | } 111 | } 112 | 113 | func TestVarintError(t *testing.T) { 114 | buf := bytes.NewBuffer(nil) 115 | buf.Write([]byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7f}) 116 | reader := io.NewReader(buf, 1024*1024) 117 | msg := &pquads.Quad{} 118 | err := reader.ReadMsg(msg) 119 | if err == nil { 120 | t.Fatalf("Expected error") 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /pquads/pio/varint.go: -------------------------------------------------------------------------------- 1 | // Extensions for Protocol Buffers to create more go like structures. 2 | // 3 | // Copyright (c) 2013, Vastech SA (PTY) LTD. All rights reserved. 4 | // http://github.com/gogo/protobuf/gogoproto 5 | // 6 | // Redistribution and use in source and binary forms, with or without 7 | // modification, are permitted provided that the following conditions are 8 | // met: 9 | // 10 | // * Redistributions of source code must retain the above copyright 11 | // notice, this list of conditions and the following disclaimer. 12 | // * Redistributions in binary form must reproduce the above 13 | // copyright notice, this list of conditions and the following disclaimer 14 | // in the documentation and/or other materials provided with the 15 | // distribution. 16 | // 17 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | package pio 30 | 31 | import ( 32 | "bufio" 33 | "encoding/binary" 34 | "io" 35 | 36 | "google.golang.org/protobuf/proto" 37 | ) 38 | 39 | func NewWriter(w io.Writer) Writer { 40 | return &varintWriter{w: w, lenBuf: make([]byte, binary.MaxVarintLen64)} 41 | } 42 | 43 | type varintWriter struct { 44 | w io.Writer 45 | lenBuf []byte 46 | buffer []byte 47 | } 48 | 49 | func (w *varintWriter) WriteMsg(msg proto.Message) (_ int, err error) { 50 | var data []byte 51 | if m, ok := msg.(marshaler); ok { 52 | n, ok := getSize(m) 53 | if !ok { 54 | data, err = proto.Marshal(msg) 55 | if err != nil { 56 | return 0, err 57 | } 58 | } 59 | if n >= len(w.buffer) { 60 | w.buffer = make([]byte, n) 61 | } 62 | _, err = m.MarshalTo(w.buffer) 63 | if err != nil { 64 | return 0, err 65 | } 66 | data = w.buffer[:n] 67 | } else { 68 | data, err = proto.Marshal(msg) 69 | if err != nil { 70 | return 0, err 71 | } 72 | } 73 | length := uint64(len(data)) 74 | n := binary.PutUvarint(w.lenBuf, length) 75 | n, err = w.w.Write(w.lenBuf[:n]) 76 | if err != nil { 77 | return n, err 78 | } 79 | nd, err := w.w.Write(data) 80 | return n + nd, err 81 | } 82 | 83 | func NewReader(r io.Reader, maxSize int) Reader { 84 | return &varintReader{r: bufio.NewReader(r), maxSize: maxSize} 85 | } 86 | 87 | type varintReader struct { 88 | r *bufio.Reader 89 | buf []byte 90 | maxSize int 91 | 92 | readLen bool 93 | len int 94 | } 95 | 96 | func (r *varintReader) readLength() error { 97 | if r.readLen { 98 | return nil 99 | } 100 | length64, err := binary.ReadUvarint(r.r) 101 | if err != nil { 102 | return err 103 | } 104 | length := int(length64) 105 | r.readLen, r.len = true, length 106 | return nil 107 | } 108 | 109 | func (r *varintReader) SkipMsg() error { 110 | if err := r.readLength(); err != nil { 111 | return err 112 | } 113 | if r.len < 0 { 114 | return io.ErrShortBuffer 115 | } 116 | r.readLen = false 117 | if _, err := r.r.Discard(r.len); err != nil { 118 | return err 119 | } 120 | return nil 121 | } 122 | 123 | func (r *varintReader) ReadMsg(msg proto.Message) error { 124 | if err := r.readLength(); err != nil { 125 | return err 126 | } 127 | if r.len < 0 || r.len > r.maxSize { 128 | return io.ErrShortBuffer 129 | } 130 | r.readLen = false 131 | if len(r.buf) < r.len { 132 | r.buf = make([]byte, r.len) 133 | } 134 | buf := r.buf[:r.len] 135 | if _, err := io.ReadFull(r.r, buf); err != nil { 136 | return err 137 | } 138 | return proto.Unmarshal(buf, msg) 139 | } 140 | -------------------------------------------------------------------------------- /pquads/pquads.go: -------------------------------------------------------------------------------- 1 | // Package pquads implements Cayley-specific protobuf-based quads format. 2 | package pquads 3 | 4 | import ( 5 | "bytes" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | 10 | "google.golang.org/protobuf/proto" 11 | 12 | "github.com/cayleygraph/quad" 13 | "github.com/cayleygraph/quad/pquads/pio" 14 | ) 15 | 16 | var DefaultMaxSize = 1024 * 1024 17 | 18 | const currentVersion = 1 19 | 20 | var magic = [4]byte{0, 'p', 'q', 0} 21 | 22 | const ContentType = "application/x-protobuf" 23 | 24 | func init() { 25 | quad.RegisterFormat(quad.Format{ 26 | Name: "pquads", Binary: true, 27 | Ext: []string{".pq"}, 28 | Mime: []string{ContentType, "application/octet-stream"}, 29 | Writer: func(w io.Writer) quad.WriteCloser { return NewWriter(w, nil) }, 30 | Reader: func(r io.Reader) quad.ReadCloser { return NewReader(r, DefaultMaxSize) }, 31 | MarshalValue: MarshalValue, 32 | UnmarshalValue: UnmarshalValue, 33 | }) 34 | } 35 | 36 | type Writer struct { 37 | pw pio.Writer 38 | max int 39 | err error 40 | opts Options 41 | s, p, o quad.Value 42 | cl io.Closer 43 | } 44 | 45 | type Options struct { 46 | // Full can be set to disable quad values compaction. 47 | // 48 | // This will increase files size, but skip will work faster by omitting unmarshal entirely. 49 | Full bool 50 | // Strict can be set to only marshal quads allowed by RDF spec. 51 | Strict bool 52 | } 53 | 54 | // NewWriter creates protobuf quads encoder. 55 | func NewWriter(w io.Writer, opts *Options) *Writer { 56 | // Write file magic and version 57 | buf := make([]byte, 8) 58 | copy(buf[:4], magic[:]) 59 | binary.LittleEndian.PutUint32(buf[4:], currentVersion) 60 | if _, err := w.Write(buf); err != nil { 61 | return &Writer{err: err} 62 | } 63 | pw := pio.NewWriter(w) 64 | if opts == nil { 65 | opts = &Options{} 66 | } 67 | // Write options header 68 | _, err := pw.WriteMsg(&Header{ 69 | Full: opts.Full, 70 | NotStrict: !opts.Strict, 71 | }) 72 | return &Writer{pw: pw, err: err, opts: *opts} 73 | } 74 | func (w *Writer) WriteQuad(q quad.Quad) error { 75 | if w.err != nil { 76 | return w.err 77 | } else if !q.IsValid() { 78 | return quad.ErrInvalid 79 | } 80 | if !w.opts.Full { 81 | if q.Subject == w.s { 82 | q.Subject = nil 83 | } else { 84 | w.s = q.Subject 85 | } 86 | if q.Predicate == w.p { 87 | q.Predicate = nil 88 | } else { 89 | w.p = q.Predicate 90 | } 91 | if q.Object == w.o { 92 | q.Object = nil 93 | } else { 94 | w.o = q.Object 95 | } 96 | } 97 | var m proto.Message 98 | if w.opts.Strict { 99 | m, w.err = makeStrictQuad(q) 100 | if w.err != nil { 101 | return w.err 102 | } 103 | } else { 104 | m = makeWireQuad(q) 105 | } 106 | var n int 107 | n, w.err = w.pw.WriteMsg(m) 108 | if n > w.max { 109 | w.max = n 110 | } 111 | return w.err 112 | } 113 | 114 | func (w *Writer) WriteQuads(buf []quad.Quad) (int, error) { 115 | for i, q := range buf { 116 | if err := w.WriteQuad(q); err != nil { 117 | return i, err 118 | } 119 | } 120 | return len(buf), nil 121 | } 122 | 123 | // MaxSize returns a maximal message size written. 124 | func (w *Writer) MaxSize() int { 125 | return w.max 126 | } 127 | func (w *Writer) SetCloser(c io.Closer) { 128 | w.cl = c 129 | } 130 | func (w *Writer) Close() error { 131 | if w.cl != nil { 132 | return w.cl.Close() 133 | } 134 | return nil 135 | } 136 | 137 | type Reader struct { 138 | pr pio.Reader 139 | err error 140 | opts Options 141 | s, p, o quad.Value 142 | cl io.Closer 143 | } 144 | 145 | func (r *Reader) SetCloser(c io.Closer) { 146 | r.cl = c 147 | } 148 | 149 | var _ quad.Skipper = (*Reader)(nil) 150 | 151 | // NewReader creates protobuf quads decoder. 152 | // 153 | // MaxSize argument limits maximal size of the buffer used to read quads. 154 | func NewReader(r io.Reader, maxSize int) *Reader { 155 | if maxSize <= 0 { 156 | maxSize = DefaultMaxSize 157 | } 158 | qr := &Reader{} 159 | buf := make([]byte, 8) 160 | if _, err := io.ReadFull(r, buf); err != nil { 161 | qr.err = err 162 | return qr 163 | } else if bytes.Compare(magic[:], buf[:4]) != 0 { 164 | qr.err = fmt.Errorf("not a pquads file") 165 | return qr 166 | } 167 | vers := binary.LittleEndian.Uint32(buf[4:]) 168 | if vers != currentVersion { 169 | qr.err = fmt.Errorf("unsupported pquads version: %d", vers) 170 | return qr 171 | } 172 | 173 | qr.pr = pio.NewReader(r, maxSize) 174 | var h Header 175 | if err := qr.pr.ReadMsg(&h); err != nil { 176 | qr.err = err 177 | } 178 | qr.opts = Options{ 179 | Full: h.Full, 180 | Strict: !h.NotStrict, 181 | } 182 | return qr 183 | } 184 | func (r *Reader) ReadQuad() (quad.Quad, error) { 185 | if r.err != nil { 186 | return quad.Quad{}, r.err 187 | } 188 | var q quad.Quad 189 | if r.opts.Strict { 190 | var pq StrictQuad 191 | if r.err = r.pr.ReadMsg(&pq); r.err != nil { 192 | return quad.Quad{}, r.err 193 | } 194 | q = pq.ToNative() 195 | } else { 196 | var pq WireQuad 197 | if r.err = r.pr.ReadMsg(&pq); r.err != nil { 198 | return quad.Quad{}, r.err 199 | } 200 | q = pq.ToNative() 201 | } 202 | if q.Subject == nil { 203 | q.Subject = r.s 204 | } else { 205 | r.s = q.Subject 206 | } 207 | if q.Predicate == nil { 208 | q.Predicate = r.p 209 | } else { 210 | r.p = q.Predicate 211 | } 212 | if q.Object == nil { 213 | q.Object = r.o 214 | } else { 215 | r.o = q.Object 216 | } 217 | return q, nil 218 | } 219 | func (r *Reader) SkipQuad() error { 220 | if !r.opts.Full { 221 | // TODO(dennwc): read pb fields as bytes and unmarshal them only if ReadQuad is called 222 | _, err := r.ReadQuad() 223 | return err 224 | } 225 | r.err = r.pr.SkipMsg() 226 | return r.err 227 | } 228 | func (r *Reader) Close() error { 229 | if r.cl != nil { 230 | return r.cl.Close() 231 | } 232 | return nil 233 | } 234 | -------------------------------------------------------------------------------- /pquads/pquads_test.go: -------------------------------------------------------------------------------- 1 | package pquads_test 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "testing" 7 | 8 | "github.com/cayleygraph/quad" 9 | "github.com/cayleygraph/quad/pquads" 10 | ) 11 | 12 | var testData = []struct { 13 | quads []quad.Quad 14 | }{ 15 | { 16 | []quad.Quad{ 17 | { 18 | Subject: quad.BNode("subject1"), 19 | Predicate: quad.IRI("/film/performance/character"), 20 | Object: quad.String("Tomás de Torquemada"), 21 | Label: quad.IRI("subgraph"), 22 | }, 23 | { 24 | Subject: quad.BNode("subject1"), 25 | Predicate: quad.IRI("http://an.example/predicate1"), 26 | Object: quad.String("object1"), 27 | Label: quad.IRI("subgraph"), 28 | }, 29 | { 30 | Subject: quad.BNode("subject2"), 31 | Predicate: quad.IRI("http://an.example/predicate1"), 32 | Object: quad.IRI("object1"), 33 | Label: quad.BNode("subgraph"), 34 | }, 35 | { 36 | Subject: quad.IRI("http://example.org/bob#me"), 37 | Predicate: quad.IRI("http://schema.org/birthDate"), 38 | Object: quad.TypedString{ 39 | Value: "1990-07-04", 40 | Type: "http://www.w3.org/2001/XMLSchema#date", 41 | }, 42 | Label: nil, 43 | }, 44 | { 45 | Subject: quad.IRI("http://example.org/bob#me"), 46 | Predicate: quad.IRI("http://schema.org/name"), 47 | Object: quad.LangString{ 48 | Value: "Bob", 49 | Lang: "en", 50 | }, 51 | Label: nil, 52 | }, 53 | }, 54 | }, 55 | } 56 | 57 | func TestPQuads(t *testing.T) { 58 | buf := bytes.NewBuffer(nil) 59 | for _, opts := range []pquads.Options{ 60 | {Full: false, Strict: false}, 61 | {Full: false, Strict: true}, 62 | {Full: true, Strict: false}, 63 | {Full: true, Strict: true}, 64 | } { 65 | name := "" 66 | if opts.Full { 67 | name += "full" 68 | } else { 69 | name += "compressed" 70 | } 71 | if opts.Strict { 72 | name += " strict" 73 | } 74 | t.Run(name, func(t *testing.T) { 75 | for _, c := range testData { 76 | buf.Reset() 77 | w := pquads.NewWriter(buf, &opts) 78 | n, err := quad.Copy(w, quad.NewReader(c.quads)) 79 | if err != nil { 80 | t.Fatalf("write failed after %d quads: %v", n, err) 81 | } 82 | if err = w.Close(); err != nil { 83 | t.Fatal("error on close:", err) 84 | } 85 | t.Log("size:", buf.Len()) 86 | r := pquads.NewReader(buf, 0) 87 | quads, err := quad.ReadAll(r) 88 | if err != nil { 89 | t.Fatalf("read failed: %v", err) 90 | } 91 | if err = r.Close(); err != nil { 92 | t.Fatal("error on close:", err) 93 | } 94 | if !reflect.DeepEqual(c.quads, quads) { 95 | t.Fatalf("corrupted quads:\n%#v\n%#v", c.quads, quads) 96 | } 97 | } 98 | }) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /pquads/quads.go: -------------------------------------------------------------------------------- 1 | package pquads 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "google.golang.org/protobuf/proto" 8 | 9 | "github.com/cayleygraph/quad" 10 | ) 11 | 12 | //go:generate protoc --go_opt=paths=source_relative --proto_path=. --go_out=. quads.proto 13 | 14 | // MakeValue converts quad.Value to its protobuf representation. 15 | func MakeValue(qv quad.Value) *Value { 16 | if qv == nil { 17 | return nil 18 | } 19 | switch v := qv.(type) { 20 | //case quad.Raw: 21 | // return &Value{&Value_Raw{[]byte(v)}} 22 | case quad.String: 23 | return &Value{Value: &Value_Str{string(v)}} 24 | case quad.IRI: 25 | return &Value{Value: &Value_Iri{string(v)}} 26 | case quad.BNode: 27 | return &Value{Value: &Value_Bnode{string(v)}} 28 | case quad.TypedString: 29 | return &Value{Value: &Value_TypedStr{&Value_TypedString{ 30 | Value: string(v.Value), 31 | Type: string(v.Type), 32 | }}} 33 | case quad.LangString: 34 | return &Value{Value: &Value_LangStr{&Value_LangString{ 35 | Value: string(v.Value), 36 | Lang: v.Lang, 37 | }}} 38 | case quad.Int: 39 | return &Value{Value: &Value_Int{int64(v)}} 40 | case quad.Float: 41 | return &Value{Value: &Value_Float{float64(v)}} 42 | case quad.Bool: 43 | return &Value{Value: &Value_Boolean{bool(v)}} 44 | case quad.Time: 45 | t := time.Time(v) 46 | seconds := t.Unix() 47 | nanos := int32(t.Sub(time.Unix(seconds, 0))) 48 | return &Value{Value: &Value_Time{&Value_Timestamp{ 49 | Seconds: seconds, 50 | Nanos: nanos, 51 | }}} 52 | default: 53 | panic(fmt.Errorf("unsupported type: %T", qv)) 54 | } 55 | } 56 | 57 | // MarshalValue is a helper for serialization of quad.Value. 58 | func MarshalValue(v quad.Value) ([]byte, error) { 59 | if v == nil { 60 | return nil, nil 61 | } 62 | return proto.Marshal(MakeValue(v)) 63 | } 64 | 65 | // UnmarshalValue is a helper for deserialization of quad.Value. 66 | func UnmarshalValue(data []byte) (quad.Value, error) { 67 | if len(data) == 0 { 68 | return nil, nil 69 | } 70 | var v Value 71 | if err := proto.Unmarshal(data, &v); err != nil { 72 | return nil, err 73 | } 74 | return v.ToNative(), nil 75 | } 76 | 77 | // ToNative converts protobuf Value to quad.Value. 78 | func (m *Value) ToNative() (qv quad.Value) { 79 | if m == nil { 80 | return nil 81 | } 82 | switch v := m.Value.(type) { 83 | case *Value_Raw: 84 | return quad.StringToValue(string(v.Raw)) 85 | case *Value_Str: 86 | return quad.String(v.Str) 87 | case *Value_Iri: 88 | return quad.IRI(v.Iri) 89 | case *Value_Bnode: 90 | return quad.BNode(v.Bnode) 91 | case *Value_TypedStr: 92 | return quad.TypedString{ 93 | Value: quad.String(v.TypedStr.Value), 94 | Type: quad.IRI(v.TypedStr.Type), 95 | } 96 | case *Value_LangStr: 97 | return quad.LangString{ 98 | Value: quad.String(v.LangStr.Value), 99 | Lang: v.LangStr.Lang, 100 | } 101 | case *Value_Int: 102 | return quad.Int(v.Int) 103 | case *Value_Float: 104 | return quad.Float(v.Float) 105 | case *Value_Boolean: 106 | return quad.Bool(v.Boolean) 107 | case *Value_Time: 108 | var t time.Time 109 | if v.Time == nil { 110 | t = time.Unix(0, 0).UTC() 111 | } else { 112 | t = time.Unix(v.Time.Seconds, int64(v.Time.Nanos)).UTC() 113 | } 114 | return quad.Time(t) 115 | default: 116 | panic(fmt.Errorf("unsupported type: %T", m.Value)) 117 | } 118 | } 119 | 120 | // ToNative converts protobuf Value to quad.Value. 121 | func (m *StrictQuad_Ref) ToNative() (qv quad.Value) { 122 | if m == nil { 123 | return nil 124 | } 125 | switch v := m.Value.(type) { 126 | case *StrictQuad_Ref_Iri: 127 | return quad.IRI(v.Iri) 128 | case *StrictQuad_Ref_BnodeLabel: 129 | return quad.BNode(v.BnodeLabel) 130 | default: 131 | panic(fmt.Errorf("unsupported type: %T", m.Value)) 132 | } 133 | } 134 | 135 | // MakeQuad converts quad.Quad to its protobuf representation. 136 | func MakeQuad(q quad.Quad) *Quad { 137 | return &Quad{ 138 | SubjectValue: MakeValue(q.Subject), 139 | PredicateValue: MakeValue(q.Predicate), 140 | ObjectValue: MakeValue(q.Object), 141 | LabelValue: MakeValue(q.Label), 142 | } 143 | } 144 | 145 | func makeRef(v quad.Value) (*StrictQuad_Ref, error) { 146 | if v == nil { 147 | return nil, nil 148 | } 149 | var sv isStrictQuad_Ref_Value 150 | switch v := v.(type) { 151 | case quad.BNode: 152 | sv = &StrictQuad_Ref_BnodeLabel{BnodeLabel: string(v)} 153 | case quad.IRI: 154 | sv = &StrictQuad_Ref_Iri{Iri: string(v)} 155 | default: 156 | return nil, fmt.Errorf("unexpected type for ref: %T", v) 157 | } 158 | return &StrictQuad_Ref{Value: sv}, nil 159 | } 160 | 161 | func makeWireQuad(q quad.Quad) *WireQuad { 162 | return &WireQuad{ 163 | Subject: MakeValue(q.Subject), 164 | Predicate: MakeValue(q.Predicate), 165 | Object: MakeValue(q.Object), 166 | Label: MakeValue(q.Label), 167 | } 168 | } 169 | 170 | func makeStrictQuad(q quad.Quad) (sq *StrictQuad, err error) { 171 | sq = new(StrictQuad) 172 | if sq.Subject, err = makeRef(q.Subject); err != nil { 173 | return nil, err 174 | } 175 | if sq.Predicate, err = makeRef(q.Predicate); err != nil { 176 | return nil, err 177 | } 178 | sq.Object = MakeValue(q.Object) 179 | if sq.Label, err = makeRef(q.Label); err != nil { 180 | return nil, err 181 | } 182 | return sq, nil 183 | } 184 | 185 | // ToNative converts protobuf Quad to quad.Quad. 186 | func (m *Quad) ToNative() (q quad.Quad) { 187 | if m == nil { 188 | return 189 | } 190 | if m.SubjectValue != nil { 191 | q.Subject = m.SubjectValue.ToNative() 192 | } else if m.Subject != "" { 193 | q.Subject = quad.StringToValue(m.Subject) 194 | } 195 | if m.PredicateValue != nil { 196 | q.Predicate = m.PredicateValue.ToNative() 197 | } else if m.Predicate != "" { 198 | q.Predicate = quad.StringToValue(m.Predicate) 199 | } 200 | if m.ObjectValue != nil { 201 | q.Object = m.ObjectValue.ToNative() 202 | } else if m.Object != "" { 203 | q.Object = quad.StringToValue(m.Object) 204 | } 205 | if m.LabelValue != nil { 206 | q.Label = m.LabelValue.ToNative() 207 | } else if m.Label != "" { 208 | q.Label = quad.StringToValue(m.Label) 209 | } 210 | return 211 | } 212 | 213 | // ToNative converts protobuf StrictQuad to quad.Quad. 214 | func (m *StrictQuad) ToNative() (q quad.Quad) { 215 | if m == nil { 216 | return 217 | } 218 | if m.Subject != nil { 219 | q.Subject = m.Subject.ToNative() 220 | } 221 | if m.Predicate != nil { 222 | q.Predicate = m.Predicate.ToNative() 223 | } 224 | if m.Object != nil { 225 | q.Object = m.Object.ToNative() 226 | } 227 | if m.Label != nil { 228 | q.Label = m.Label.ToNative() 229 | } 230 | return 231 | } 232 | 233 | // ToNative converts protobuf WireQuad to quad.Quad. 234 | func (m *WireQuad) ToNative() (q quad.Quad) { 235 | if m == nil { 236 | return 237 | } 238 | if m.Subject != nil { 239 | q.Subject = m.Subject.ToNative() 240 | } 241 | if m.Predicate != nil { 242 | q.Predicate = m.Predicate.ToNative() 243 | } 244 | if m.Object != nil { 245 | q.Object = m.Object.ToNative() 246 | } 247 | if m.Label != nil { 248 | q.Label = m.Label.ToNative() 249 | } 250 | return 251 | } 252 | 253 | func (m *Quad) Upgrade() { 254 | if m.SubjectValue == nil { 255 | m.SubjectValue = MakeValue(quad.StringToValue(m.Subject)) 256 | m.Subject = "" 257 | } 258 | if m.PredicateValue == nil { 259 | m.PredicateValue = MakeValue(quad.StringToValue(m.Predicate)) 260 | m.Predicate = "" 261 | } 262 | if m.ObjectValue == nil { 263 | m.ObjectValue = MakeValue(quad.StringToValue(m.Object)) 264 | m.Object = "" 265 | } 266 | if m.LabelValue == nil && m.Label != "" { 267 | m.LabelValue = MakeValue(quad.StringToValue(m.Label)) 268 | m.Label = "" 269 | } 270 | } 271 | -------------------------------------------------------------------------------- /pquads/quads.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2015 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package pquads; 18 | 19 | option go_package = "github.com/cayleygraph/quad/pquads"; 20 | 21 | // Quad is in internal representation of quad used by Cayley. 22 | message Quad { 23 | // TODO(dennwc): get rid of legacy fields with first opportunity and bump version 24 | 25 | string subject = 1; 26 | string predicate = 2; 27 | string object = 3; 28 | string label = 4; 29 | 30 | Value subject_value = 5; 31 | Value predicate_value = 6; 32 | Value object_value = 7; 33 | Value label_value = 8; 34 | } 35 | 36 | // WireQuad is a quad that allows any value for it's directions. 37 | message WireQuad { 38 | Value subject = 1; 39 | Value predicate = 2; 40 | Value object = 3; 41 | Value label = 4; 42 | } 43 | 44 | // WireQuadRaw is the same as WireQuad, but doesn't decode underlying values. 45 | message WireQuadRaw { 46 | bytes subject = 1; 47 | bytes predicate = 2; 48 | bytes object = 3; 49 | bytes label = 4; 50 | } 51 | 52 | // StrictQuad is a quad as described by RDF spec. 53 | message StrictQuad { 54 | message Ref { 55 | reserved 1; // uint64 bnode = 1; 56 | oneof value { 57 | string bnode_label = 2; 58 | string iri = 3; 59 | } 60 | } 61 | Ref subject = 1; 62 | Ref predicate = 2; 63 | Value object = 3; 64 | Ref label = 4; 65 | } 66 | 67 | // StrictQuadRaw is the same as StrictQuad, but doesn't decode underlying values. 68 | message StrictQuadRaw { 69 | bytes subject = 1; 70 | bytes predicate = 2; 71 | bytes object = 3; 72 | bytes label = 4; 73 | } 74 | 75 | message Value { 76 | message TypedString { 77 | string value = 1; 78 | string type = 2; 79 | } 80 | message LangString { 81 | string value = 1; 82 | string lang = 2; 83 | } 84 | // From https://github.com/golang/protobuf/blob/master/ptypes/timestamp/timestamp.proto 85 | message Timestamp { 86 | int64 seconds = 1; 87 | int32 nanos = 2; 88 | } 89 | oneof value { 90 | bytes raw = 1; 91 | string str = 2; 92 | string iri = 3; 93 | string bnode = 4; 94 | TypedString typed_str = 5; 95 | LangString lang_str = 6; 96 | int64 int = 7; 97 | double float = 8; 98 | bool boolean = 9; 99 | Timestamp time = 10; 100 | } 101 | } 102 | 103 | message Header { 104 | // Full is set if encoder always writes every quad directions instead of 105 | // skipping duplicated values on each direction (except label) for subsequent quads. 106 | bool full = 1; 107 | // NotStrict is set if encoder emits WireQuad instead of StrictQuad messages. 108 | bool not_strict = 2; 109 | } -------------------------------------------------------------------------------- /quad.go: -------------------------------------------------------------------------------- 1 | // Copyright 2014 The Cayley Authors. All rights reserved. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // Package quad defines quad and triple handling. 16 | package quad 17 | 18 | // Defines the struct which makes the QuadStore possible -- the quad. 19 | // 20 | // At its heart, it consists of three fields -- Subject, Predicate, and Object. 21 | // Three IDs that relate to each other. That's all there is to it. The quads 22 | // are the links in the graph, and the existence of node IDs is defined by the 23 | // fact that some quad in the graph mentions them. 24 | // 25 | // This means that a complete representation of the graph is equivalent to a 26 | // list of quads. The rest is just indexing for speed. 27 | // 28 | // Adding fields to the quad is not to be taken lightly. You'll see I mention 29 | // label, but don't as yet use it in any backing store. In general, there 30 | // can be features that can be turned on or off for any store, but I haven't 31 | // decided how to allow/disallow them yet. Another such example would be to add 32 | // a forward and reverse index field -- forward being "order the list of 33 | // objects pointed at by this subject with this predicate" such as first and 34 | // second children, top billing, what have you. 35 | // 36 | // There will never be that much in this file except for the definition, but 37 | // the consequences are not to be taken lightly. But do suggest cool features! 38 | 39 | import ( 40 | "encoding/json" 41 | "errors" 42 | "fmt" 43 | ) 44 | 45 | var ( 46 | ErrInvalid = errors.New("invalid N-Quad") 47 | ErrIncomplete = errors.New("incomplete N-Quad") 48 | ) 49 | 50 | // Make creates a quad with provided values. 51 | func Make(subject, predicate, object, label interface{}) (q Quad) { 52 | var ok bool 53 | if q.Subject, ok = AsValue(subject); !ok { 54 | q.Subject = String(fmt.Sprint(subject)) 55 | } 56 | if q.Predicate, ok = AsValue(predicate); !ok { 57 | q.Predicate = String(fmt.Sprint(predicate)) 58 | } 59 | if q.Object, ok = AsValue(object); !ok { 60 | q.Object = String(fmt.Sprint(object)) 61 | } 62 | if q.Label, ok = AsValue(label); !ok { 63 | q.Label = String(fmt.Sprint(label)) 64 | } 65 | return 66 | } 67 | 68 | // MakeRaw creates a quad with provided raw values (nquads-escaped). 69 | // 70 | // Deprecated: use Make pr MakeIRI instead. 71 | func MakeRaw(subject, predicate, object, label string) (q Quad) { 72 | if subject != "" { 73 | q.Subject = Raw(subject) 74 | } 75 | if predicate != "" { 76 | q.Predicate = Raw(predicate) 77 | } 78 | if object != "" { 79 | q.Object = Raw(object) 80 | } 81 | if label != "" { 82 | q.Label = Raw(label) 83 | } 84 | return 85 | } 86 | 87 | // MakeIRI creates a quad with provided IRI values. 88 | func MakeIRI(subject, predicate, object, label string) (q Quad) { 89 | if subject != "" { 90 | q.Subject = IRI(subject) 91 | } 92 | if predicate != "" { 93 | q.Predicate = IRI(predicate) 94 | } 95 | if object != "" { 96 | q.Object = IRI(object) 97 | } 98 | if label != "" { 99 | q.Label = IRI(label) 100 | } 101 | return 102 | } 103 | 104 | var ( 105 | _ json.Marshaler = Quad{} 106 | _ json.Unmarshaler = (*Quad)(nil) 107 | ) 108 | 109 | // Our quad struct, used throughout. 110 | type Quad struct { 111 | Subject Value `json:"subject"` 112 | Predicate Value `json:"predicate"` 113 | Object Value `json:"object"` 114 | Label Value `json:"label,omitempty"` 115 | } 116 | 117 | type rawQuad struct { 118 | Subject string `json:"subject"` 119 | Predicate string `json:"predicate"` 120 | Object string `json:"object"` 121 | Label string `json:"label,omitempty"` 122 | } 123 | 124 | func (q Quad) MarshalJSON() ([]byte, error) { 125 | rq := rawQuad{ 126 | Subject: ToString(q.Subject), 127 | Predicate: ToString(q.Predicate), 128 | Object: ToString(q.Object), 129 | } 130 | if q.Label != nil { 131 | rq.Label = ToString(q.Label) 132 | } 133 | return json.Marshal(rq) 134 | } 135 | func (q *Quad) UnmarshalJSON(data []byte) error { 136 | var rq rawQuad 137 | if err := json.Unmarshal(data, &rq); err != nil { 138 | return err 139 | } 140 | // TODO(dennwc): parse nquads? or use StringToValue hack? 141 | *q = MakeRaw(rq.Subject, rq.Predicate, rq.Object, rq.Label) 142 | return nil 143 | } 144 | 145 | // Direction specifies an edge's type. 146 | type Direction byte 147 | 148 | // List of the valid directions of a quad. 149 | const ( 150 | Any Direction = iota 151 | Subject 152 | Predicate 153 | Object 154 | Label 155 | ) 156 | 157 | var Directions = []Direction{Subject, Predicate, Object, Label} 158 | 159 | func (d Direction) Prefix() byte { 160 | switch d { 161 | case Any: 162 | return 'a' 163 | case Subject: 164 | return 's' 165 | case Predicate: 166 | return 'p' 167 | case Label: 168 | return 'c' 169 | case Object: 170 | return 'o' 171 | default: 172 | return '\x00' 173 | } 174 | } 175 | 176 | func (d Direction) String() string { 177 | switch d { 178 | case Any: 179 | return "any" 180 | case Subject: 181 | return "subject" 182 | case Predicate: 183 | return "predicate" 184 | case Label: 185 | return "label" 186 | case Object: 187 | return "object" 188 | default: 189 | return fmt.Sprint("illegal direction:", byte(d)) 190 | } 191 | } 192 | 193 | func (d Direction) GoString() string { 194 | switch d { 195 | case Any: 196 | return "quad.Any" 197 | case Subject: 198 | return "quad.Subject" 199 | case Predicate: 200 | return "quad.Predicate" 201 | case Label: 202 | return "quad.Label" 203 | case Object: 204 | return "quad.Object" 205 | default: 206 | return fmt.Sprintf("%x", byte(d)) 207 | } 208 | } 209 | 210 | // Per-field accessor for quads. 211 | func (q Quad) Get(d Direction) Value { 212 | switch d { 213 | case Subject: 214 | return q.Subject 215 | case Predicate: 216 | return q.Predicate 217 | case Label: 218 | return q.Label 219 | case Object: 220 | return q.Object 221 | default: 222 | panic(d.String()) 223 | } 224 | } 225 | 226 | func (q *Quad) Set(d Direction, v Value) { 227 | switch d { 228 | case Subject: 229 | q.Subject = v 230 | case Predicate: 231 | q.Predicate = v 232 | case Label: 233 | q.Label = v 234 | case Object: 235 | q.Object = v 236 | default: 237 | panic(d.String()) 238 | } 239 | } 240 | 241 | // Per-field accessor for quads that returns strings instead of values. 242 | func (q Quad) GetString(d Direction) string { 243 | switch d { 244 | case Subject: 245 | return StringOf(q.Subject) 246 | case Predicate: 247 | return StringOf(q.Predicate) 248 | case Object: 249 | return StringOf(q.Object) 250 | case Label: 251 | return StringOf(q.Label) 252 | default: 253 | panic(d.String()) 254 | } 255 | } 256 | 257 | // Pretty-prints a quad. 258 | func (q Quad) String() string { 259 | return fmt.Sprintf("%v -- %v -> %v", q.Subject, q.Predicate, q.Object) 260 | } 261 | 262 | func (q Quad) IsValid() bool { 263 | return IsValidValue(q.Subject) && IsValidValue(q.Predicate) && IsValidValue(q.Object) 264 | } 265 | 266 | // Prints a quad in N-Quad format. 267 | func (q Quad) NQuad() string { 268 | if q.Label == nil || q.Label.String() == "" { 269 | return fmt.Sprintf("%s %s %s .", q.Subject, q.Predicate, q.Object) 270 | } 271 | return fmt.Sprintf("%s %s %s %s .", q.Subject, q.Predicate, q.Object, q.Label) 272 | } 273 | 274 | type ByQuadString []Quad 275 | 276 | func (o ByQuadString) Len() int { return len(o) } 277 | func (o ByQuadString) Less(i, j int) bool { 278 | switch { // TODO: optimize 279 | case StringOf(o[i].Subject) < StringOf(o[j].Subject), 280 | 281 | StringOf(o[i].Subject) == StringOf(o[j].Subject) && 282 | StringOf(o[i].Predicate) < StringOf(o[j].Predicate), 283 | 284 | StringOf(o[i].Subject) == StringOf(o[j].Subject) && 285 | StringOf(o[i].Predicate) == StringOf(o[j].Predicate) && 286 | StringOf(o[i].Object) < StringOf(o[j].Object), 287 | 288 | StringOf(o[i].Subject) == StringOf(o[j].Subject) && 289 | StringOf(o[i].Predicate) == StringOf(o[j].Predicate) && 290 | StringOf(o[i].Object) == StringOf(o[j].Object) && 291 | StringOf(o[i].Label) < StringOf(o[j].Label): 292 | 293 | return true 294 | 295 | default: 296 | return false 297 | } 298 | } 299 | func (o ByQuadString) Swap(i, j int) { o[i], o[j] = o[j], o[i] } 300 | -------------------------------------------------------------------------------- /rw.go: -------------------------------------------------------------------------------- 1 | package quad 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | // Writer is a minimal interface for quad writers. Used for quad serializers and quad stores. 8 | type Writer interface { 9 | // WriteQuad writes a single quad and returns an error, if any. 10 | // 11 | // Deprecated: use WriteQuads instead. 12 | WriteQuad(Quad) error 13 | BatchWriter 14 | } 15 | 16 | type WriteCloser interface { 17 | Writer 18 | io.Closer 19 | } 20 | 21 | // BatchWriter is an interface for writing quads in batches. 22 | type BatchWriter interface { 23 | // WriteQuads returns a number of quads that where written and an error, if any. 24 | WriteQuads(buf []Quad) (int, error) 25 | } 26 | 27 | // Reader is a minimal interface for quad readers. Used for quad deserializers and quad iterators. 28 | // 29 | // ReadQuad reads next valid Quad. It returns io.EOF if no quads are left. 30 | type Reader interface { 31 | ReadQuad() (Quad, error) 32 | } 33 | 34 | // Skipper is an interface for quad reader that can skip quads efficiently without decoding them. 35 | // 36 | // It returns io.EOF if no quads are left. 37 | type Skipper interface { 38 | SkipQuad() error 39 | } 40 | 41 | type ReadCloser interface { 42 | Reader 43 | io.Closer 44 | } 45 | 46 | type ReadSkipCloser interface { 47 | Reader 48 | Skipper 49 | io.Closer 50 | } 51 | 52 | // BatchReader is an interface for reading quads in batches. 53 | // 54 | // ReadQuads reads at most len(buf) quads into buf. It returns number of quads that were read and an error. 55 | // It returns an io.EOF if there is no more quads to read. 56 | type BatchReader interface { 57 | ReadQuads(buf []Quad) (int, error) 58 | } 59 | 60 | type Quads struct { 61 | s []Quad 62 | } 63 | 64 | func (r *Quads) WriteQuad(q Quad) error { 65 | r.s = append(r.s, q) 66 | return nil 67 | } 68 | 69 | func (r *Quads) ReadQuad() (Quad, error) { 70 | if r == nil || len(r.s) == 0 { 71 | return Quad{}, io.EOF 72 | } 73 | q := r.s[0] 74 | r.s = r.s[1:] 75 | if len(r.s) == 0 { 76 | r.s = nil 77 | } 78 | return q, nil 79 | } 80 | 81 | // NewReader creates a quad reader from a quad slice. 82 | func NewReader(quads []Quad) *Quads { 83 | return &Quads{s: quads} 84 | } 85 | 86 | // Copy will copy all quads from src to dst. It returns copied quads count and an error, if it failed. 87 | // 88 | // Copy will try to cast dst to BatchWriter and will switch to CopyBatch implementation in case of success. 89 | func Copy(dst Writer, src Reader) (n int, err error) { 90 | if bw, ok := dst.(BatchWriter); ok { 91 | return CopyBatch(bw, src, 0) 92 | } 93 | var q Quad 94 | for { 95 | q, err = src.ReadQuad() 96 | if err == io.EOF { 97 | err = nil 98 | return 99 | } else if err != nil { 100 | return 101 | } 102 | if err = dst.WriteQuad(q); err != nil { 103 | return 104 | } 105 | n++ 106 | } 107 | } 108 | 109 | type batchReader struct { 110 | Reader 111 | } 112 | 113 | func (r batchReader) ReadQuads(quads []Quad) (n int, err error) { 114 | for ; n < len(quads); n++ { 115 | quads[n], err = r.ReadQuad() 116 | if err != nil { 117 | break 118 | } 119 | } 120 | return 121 | } 122 | 123 | var DefaultBatch = 10000 124 | 125 | // CopyBatch will copy all quads from src to dst in a batches of batchSize. 126 | // It returns copied quads count and an error, if it failed. 127 | // 128 | // If batchSize <= 0 default batch size will be used. 129 | func CopyBatch(dst BatchWriter, src Reader, batchSize int) (cnt int, err error) { 130 | if batchSize <= 0 { 131 | batchSize = DefaultBatch 132 | } 133 | buf := make([]Quad, batchSize) 134 | bsrc, ok := src.(BatchReader) 135 | if !ok { 136 | bsrc = batchReader{src} 137 | } 138 | var n int 139 | for err == nil { 140 | n, err = bsrc.ReadQuads(buf) 141 | if err != nil && err != io.EOF { 142 | return 143 | } 144 | eof := err == io.EOF 145 | n, err = dst.WriteQuads(buf[:n]) 146 | cnt += n 147 | if eof { 148 | break 149 | } 150 | } 151 | return 152 | } 153 | 154 | // ReadAll reads all quads from r until EOF. 155 | // It returns a slice with all quads that were read and an error, if any. 156 | func ReadAll(r Reader) (arr []Quad, err error) { 157 | switch rt := r.(type) { 158 | case *Quads: 159 | arr = make([]Quad, len(rt.s)) 160 | copy(arr, rt.s) 161 | rt.s = nil 162 | return 163 | } 164 | var q Quad 165 | for { 166 | q, err = r.ReadQuad() 167 | if err == io.EOF { 168 | return arr, nil 169 | } else if err != nil { 170 | return nil, err 171 | } 172 | arr = append(arr, q) 173 | } 174 | } 175 | 176 | // IRIOptions is a set of option 177 | type IRIOptions struct { 178 | Format IRIFormat // short vs full IRI format 179 | // Func is executed after all other options and have a chance to replace the value. 180 | // Returning an empty IRI changes the value to nil. 181 | Func func(d Direction, iri IRI) (IRI, error) 182 | } 183 | 184 | // apply transforms the IRI using the specified options. 185 | func (opt IRIOptions) apply(d Direction, v IRI) (IRI, error) { 186 | v = v.Format(opt.Format) 187 | if opt.Func != nil { 188 | var err error 189 | v, err = opt.Func(d, v) 190 | if err != nil { 191 | return "", err 192 | } else if v == "" { 193 | return "", nil 194 | } 195 | } 196 | return v, nil 197 | } 198 | 199 | // IRIWriter is a writer implementation that converts all IRI values in quads 200 | // according to the IRIOptions. 201 | func IRIWriter(w Writer, opt IRIOptions) Writer { 202 | return ValuesWriter(w, func(d Direction, v Value) (Value, error) { 203 | switch v := v.(type) { 204 | case IRI: 205 | var err error 206 | v, err = opt.apply(d, v) 207 | if err != nil { 208 | return nil, err 209 | } else if v == "" { 210 | return nil, nil 211 | } 212 | return v, nil 213 | case TypedString: 214 | var err error 215 | v.Type, err = opt.apply(d, v.Type) 216 | if err != nil { 217 | return nil, err 218 | } 219 | return v, nil 220 | } 221 | return v, nil 222 | }) 223 | } 224 | 225 | // ValuesWriter is a writer implementation that converts all quad values using the callback function. 226 | func ValuesWriter(w Writer, fnc func(d Direction, v Value) (Value, error)) Writer { 227 | return &valueWriter{w: w, fnc: fnc} 228 | } 229 | 230 | type valueWriter struct { 231 | w Writer 232 | buf []Quad 233 | fnc func(d Direction, v Value) (Value, error) 234 | } 235 | 236 | func (w *valueWriter) apply(q *Quad) error { 237 | if v, err := w.fnc(Subject, q.Subject); err != nil { 238 | return err 239 | } else { 240 | q.Subject = v 241 | } 242 | if v, err := w.fnc(Predicate, q.Predicate); err != nil { 243 | return err 244 | } else { 245 | q.Predicate = v 246 | } 247 | if v, err := w.fnc(Object, q.Object); err != nil { 248 | return err 249 | } else { 250 | q.Object = v 251 | } 252 | if v, err := w.fnc(Label, q.Label); err != nil { 253 | return err 254 | } else { 255 | q.Label = v 256 | } 257 | return nil 258 | } 259 | 260 | func (w *valueWriter) WriteQuad(q Quad) error { 261 | if err := w.apply(&q); err != nil { 262 | return err 263 | } 264 | _, err := w.w.WriteQuads([]Quad{q}) 265 | return err 266 | } 267 | 268 | func (w *valueWriter) WriteQuads(buf []Quad) (int, error) { 269 | w.buf = append(w.buf[:0], buf...) 270 | for i := range w.buf { 271 | if err := w.apply(&w.buf[i]); err != nil { 272 | return 0, err 273 | } 274 | } 275 | n, err := w.w.WriteQuads(w.buf) 276 | w.buf = w.buf[:0] 277 | return n, err 278 | } 279 | -------------------------------------------------------------------------------- /value.go: -------------------------------------------------------------------------------- 1 | package quad 2 | 3 | import ( 4 | "crypto/sha1" 5 | "fmt" 6 | "hash" 7 | "math/rand" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | "sync/atomic" 12 | "time" 13 | 14 | "github.com/cayleygraph/quad/voc" 15 | "github.com/cayleygraph/quad/voc/schema" 16 | "github.com/cayleygraph/quad/voc/xsd" 17 | ) 18 | 19 | // IsValidValue checks if the value is valid. It returns false if the value is nil, an empty IRI or an empty BNode. 20 | func IsValidValue(v Value) bool { 21 | if v == nil { 22 | return false 23 | } 24 | switch v := v.(type) { 25 | case IRI: 26 | return v != "" 27 | case BNode: 28 | return v != "" 29 | } 30 | return true 31 | } 32 | 33 | // Value is a type used by all quad directions. 34 | type Value interface { 35 | String() string 36 | // Native converts Value to a closest native Go type. 37 | // 38 | // If type has no analogs in Go, Native return an object itself. 39 | Native() interface{} 40 | } 41 | 42 | var ( 43 | _ Identifier = IRI("") 44 | _ Identifier = BNode("") 45 | ) 46 | 47 | // Identifier is a union of IRI and BNode. 48 | type Identifier interface { 49 | Value 50 | isIdentifier() 51 | } 52 | 53 | type TypedStringer interface { 54 | TypedString() TypedString 55 | } 56 | 57 | // Equaler interface is implemented by values, that needs a special equality check. 58 | type Equaler interface { 59 | Equal(v Value) bool 60 | } 61 | 62 | // HashSize is a size of the slice, returned by HashOf. 63 | const HashSize = sha1.Size 64 | 65 | var hashPool = sync.Pool{ 66 | New: func() interface{} { return sha1.New() }, 67 | } 68 | 69 | // HashOf calculates a hash of value v. 70 | func HashOf(v Value) []byte { 71 | key := make([]byte, HashSize) 72 | HashTo(v, key) 73 | return key 74 | } 75 | 76 | // HashTo calculates a hash of value v, storing it in a slice p. 77 | func HashTo(v Value, p []byte) { 78 | h := hashPool.Get().(hash.Hash) 79 | h.Reset() 80 | defer hashPool.Put(h) 81 | if len(p) < HashSize { 82 | panic("buffer too small to fit the hash") 83 | } 84 | if v != nil { 85 | // TODO(kortschak,dennwc) Remove dependence on String() method. 86 | h.Write([]byte(v.String())) 87 | } 88 | h.Sum(p[:0]) 89 | } 90 | 91 | // StringOf safely call v.String, returning empty string in case of nil Value. 92 | func StringOf(v Value) string { 93 | if v == nil { 94 | return "" 95 | } 96 | return v.String() 97 | } 98 | 99 | // NativeOf safely call v.Native, returning nil in case of nil Value. 100 | func NativeOf(v Value) interface{} { 101 | if v == nil { 102 | return nil 103 | } 104 | return v.Native() 105 | } 106 | 107 | // AsValue converts native type into closest Value representation. 108 | // It returns false if type was not recognized. 109 | func AsValue(v interface{}) (out Value, ok bool) { 110 | if v == nil { 111 | return nil, true 112 | } 113 | switch v := v.(type) { 114 | case Value: 115 | out = v 116 | case string: 117 | out = String(v) 118 | case int: 119 | out = Int(v) 120 | case int8: 121 | out = Int(v) 122 | case int16: 123 | out = Int(v) 124 | case int32: 125 | out = Int(v) 126 | case int64: 127 | out = Int(v) 128 | case uint: 129 | out = Int(v) 130 | case uint8: 131 | out = Int(v) 132 | case uint16: 133 | out = Int(v) 134 | case uint32: 135 | out = Int(v) 136 | case uint64: 137 | out = Int(v) 138 | case float64: 139 | out = Float(v) 140 | case float32: 141 | out = Float(v) 142 | case bool: 143 | out = Bool(v) 144 | case time.Time: 145 | out = Time(v) 146 | default: 147 | return nil, false 148 | } 149 | return out, true 150 | } 151 | 152 | // StringToValue is a function to convert strings to typed 153 | // quad values. 154 | // 155 | // Warning: should not be used directly - will be deprecated. 156 | func StringToValue(v string) Value { 157 | if v == "" { 158 | return nil 159 | } 160 | if len(v) > 2 { 161 | if v[0] == '<' && v[len(v)-1] == '>' { 162 | return IRI(v[1 : len(v)-1]) 163 | } else if v[:2] == "_:" { 164 | return BNode(v[2:]) 165 | } else if i := strings.Index(v, `"^^<`); i > 0 && v[0] == '"' && v[len(v)-1] == '>' { 166 | return TypedString{Value: String(v[1:i]), Type: IRI(v[i+4 : len(v)-1])} 167 | } else if i := strings.Index(v, `"@`); i > 0 && v[0] == '"' && v[len(v)-1] != '"' { 168 | return LangString{Value: String(v[1:i]), Lang: v[i+2:]} 169 | } 170 | } 171 | return String(v) 172 | } 173 | 174 | // ToString casts a values to String or falls back to StringOf. 175 | func ToString(v Value) string { 176 | if s, ok := v.(String); ok { 177 | return string(s) 178 | } 179 | return StringOf(v) 180 | } 181 | 182 | // Raw is a Turtle/NQuads-encoded value. 183 | // 184 | // Deprecated: use IRI or String instead. 185 | func Raw(s string) Value { 186 | if len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"' { 187 | return String(s[1 : len(s)-1]) 188 | } 189 | return StringToValue(s) 190 | } 191 | 192 | // String is an RDF string value (ex: "name"). 193 | type String string 194 | 195 | var escaper = strings.NewReplacer( 196 | "\\", "\\\\", 197 | "\"", "\\\"", 198 | "\n", "\\n", 199 | "\r", "\\r", 200 | "\t", "\\t", 201 | ) 202 | 203 | func (s String) String() string { 204 | //TODO(barakmich): Proper escaping. 205 | return `"` + escaper.Replace(string(s)) + `"` 206 | } 207 | func (s String) GoString() string { 208 | return "quad.String(" + strconv.Quote(string(s)) + ")" 209 | } 210 | func (s String) Native() interface{} { return string(s) } 211 | 212 | // TypedString is an RDF value with type (ex: "name"^^). 213 | type TypedString struct { 214 | Value String 215 | Type IRI 216 | } 217 | 218 | func (s TypedString) String() string { 219 | return s.Value.String() + `^^` + s.Type.String() 220 | } 221 | func (s TypedString) Native() interface{} { 222 | if s.Type == "" { 223 | return s.Value.Native() 224 | } 225 | if v, err := s.ParseValue(); err == nil && v != s { 226 | return v.Native() 227 | } 228 | return s 229 | } 230 | 231 | // ParseValue will try to parse underlying string value using registered functions. 232 | // 233 | // It will return unchanged value if suitable function is not available. 234 | // 235 | // Error will be returned if the type was recognizes, but parsing failed. 236 | func (s TypedString) ParseValue() (Value, error) { 237 | fnc := knownConversions[s.Type.Full()] 238 | if fnc == nil { 239 | return s, nil 240 | } 241 | return fnc(string(s.Value)) 242 | } 243 | 244 | // LangString is an RDF string with language (ex: "name"@lang). 245 | type LangString struct { 246 | Value String 247 | Lang string 248 | } 249 | 250 | func (s LangString) String() string { 251 | return s.Value.String() + `@` + s.Lang 252 | } 253 | func (s LangString) Native() interface{} { return s.Value.Native() } 254 | 255 | // IRIFormat is a format of IRI. 256 | type IRIFormat int 257 | 258 | const ( 259 | // IRIDefault preserves current IRI formatting. 260 | IRIDefault = IRIFormat(iota) 261 | // IRIShort changes IRI to use a short namespace prefix (ex: ). 262 | IRIShort 263 | // IRIFull changes IRI to use full form (ex: ). 264 | IRIFull 265 | ) 266 | 267 | // IRI is an RDF Internationalized Resource Identifier (ex: ). 268 | type IRI string 269 | 270 | // isIdentifier implements Identifier. 271 | func (s IRI) isIdentifier() {} 272 | 273 | // String prints IRI in "" form. 274 | func (s IRI) String() string { return `<` + string(s) + `>` } 275 | 276 | // Format the IRI according to selection. 277 | func (s IRI) Format(format IRIFormat) IRI { 278 | switch format { 279 | case IRIShort: 280 | return s.Short() 281 | case IRIFull: 282 | return s.Full() 283 | } 284 | return s 285 | } 286 | 287 | // GoString overrides IRI's %#v printing behaviour to include the type name. 288 | func (s IRI) GoString() string { 289 | return "quad.IRI(" + strconv.Quote(string(s)) + ")" 290 | } 291 | 292 | // Short uses voc package to convert a full IRI prefix (if any) to a short namespace prefix. 293 | // The prefix must be registered in the voc package. 294 | func (s IRI) Short() IRI { 295 | return IRI(voc.ShortIRI(string(s))) 296 | } 297 | 298 | // Full uses voc package to convert a short namespace prefix (if any) to a full IRI prefix. 299 | // The prefix must be registered in the voc package. 300 | func (s IRI) Full() IRI { 301 | return IRI(voc.FullIRI(string(s))) 302 | } 303 | 304 | // Native returns an IRI value unchanged (to not collide with String values). 305 | func (s IRI) Native() interface{} { 306 | return s 307 | } 308 | 309 | // ShortWith uses the provided namespace to convert a full IRI prefix (if any) to a short namespace prefix. 310 | func (s IRI) ShortWith(n *voc.Namespaces) IRI { 311 | return IRI(n.ShortIRI(string(s))) 312 | } 313 | 314 | // FullWith uses provided namespace to convert a short namespace prefix (if any) to a full IRI prefix. 315 | func (s IRI) FullWith(n *voc.Namespaces) IRI { 316 | return IRI(n.FullIRI(string(s))) 317 | } 318 | 319 | // BNode is an RDF Blank Node (ex: _:name). 320 | type BNode string 321 | 322 | // isIdentifier implements Identifier. 323 | func (s BNode) isIdentifier() {} 324 | 325 | func (s BNode) String() string { return `_:` + string(s) } 326 | func (s BNode) GoString() string { 327 | return "quad.BNode(" + strconv.Quote(string(s)) + ")" 328 | } 329 | func (s BNode) Native() interface{} { return s } 330 | 331 | // Native support for basic types 332 | 333 | // StringConversion is a function to convert string values with a 334 | // specific IRI type to their native equivalents. 335 | type StringConversion func(string) (Value, error) 336 | 337 | // Following the JSON-LD specification: https://w3c.github.io/json-ld-syntax/#conversion-of-native-data-types 338 | const ( 339 | defaultStringType IRI = xsd.String 340 | defaultIntType IRI = xsd.Integer 341 | defaultFloatType IRI = xsd.Double 342 | defaultBoolType IRI = xsd.Boolean 343 | defaultTimeType IRI = xsd.DateTime 344 | ) 345 | 346 | // KnownIntTypes consists of known IRIs of integer types 347 | var KnownIntTypes = []IRI{ 348 | defaultIntType, 349 | xsd.Int, 350 | xsd.Long, 351 | schema.Integer, 352 | } 353 | 354 | // KnownBoolTypes consists of known IRIs of boolean types 355 | var KnownBoolTypes = []IRI{ 356 | defaultBoolType, 357 | schema.Boolean, 358 | } 359 | 360 | // KnownFloatTypes consists of known IRIs of floating-point numbers types 361 | var KnownFloatTypes = []IRI{ 362 | defaultFloatType, 363 | xsd.Float, 364 | schema.Float, 365 | schema.Number, 366 | } 367 | 368 | // KnownTimeTypes consists of known IRIs of datetime types 369 | var KnownTimeTypes = []IRI{ 370 | defaultTimeType, 371 | xsd.DateTime, 372 | schema.DateTime, 373 | } 374 | 375 | func init() { 376 | // string types 377 | RegisterStringConversion(defaultStringType, stringToString) 378 | // int types 379 | RegisterStringConversions(KnownIntTypes, stringToInt) 380 | // bool types 381 | RegisterStringConversions(KnownBoolTypes, stringToBool) 382 | // float types 383 | RegisterStringConversions(KnownFloatTypes, stringToFloat) 384 | // time types 385 | RegisterStringConversions(KnownTimeTypes, stringToTime) 386 | } 387 | 388 | var knownConversions = make(map[IRI]StringConversion) 389 | 390 | // RegisterStringConversion will register an automatic conversion of 391 | // TypedString values with provided type to a native equivalent such as Int, Time, etc. 392 | // 393 | // If fnc is nil, automatic conversion from selected type will be removed. 394 | func RegisterStringConversion(dataType IRI, fnc StringConversion) { 395 | if fnc == nil { 396 | delete(knownConversions, dataType) 397 | } else { 398 | knownConversions[dataType] = fnc 399 | if short := dataType.Short(); short != dataType { 400 | knownConversions[short] = fnc 401 | } 402 | if full := dataType.Full(); full != dataType { 403 | knownConversions[full] = fnc 404 | } 405 | } 406 | } 407 | 408 | // RegisterStringConversions calls RegisterStringConversion with every IRI in dataTypes and fnc 409 | func RegisterStringConversions(dataTypes []IRI, fnc StringConversion) { 410 | for _, iri := range dataTypes { 411 | RegisterStringConversion(iri, fnc) 412 | } 413 | } 414 | 415 | // HasStringConversion returns whether IRI has a string conversion 416 | func HasStringConversion(dataType IRI) bool { 417 | _, ok := knownConversions[dataType] 418 | return ok 419 | } 420 | 421 | func stringToString(s string) (Value, error) { 422 | return String(s), nil 423 | } 424 | 425 | func stringToInt(s string) (Value, error) { 426 | v, err := strconv.ParseInt(s, 10, 64) 427 | if err != nil { 428 | return nil, err 429 | } 430 | return Int(v), nil 431 | } 432 | 433 | func stringToBool(s string) (Value, error) { 434 | v, err := strconv.ParseBool(s) 435 | if err != nil { 436 | return nil, err 437 | } 438 | return Bool(v), nil 439 | } 440 | 441 | func stringToFloat(s string) (Value, error) { 442 | v, err := strconv.ParseFloat(s, 64) 443 | if err != nil { 444 | return nil, err 445 | } 446 | return Float(v), nil 447 | } 448 | 449 | func stringToTime(s string) (Value, error) { 450 | v, err := time.Parse(time.RFC3339, s) 451 | if err != nil { 452 | return nil, err 453 | } 454 | return Time(v), nil 455 | } 456 | 457 | // Int is a native wrapper for int64 type. 458 | // 459 | // It uses NQuad notation similar to TypedString. 460 | type Int int64 461 | 462 | func (s Int) String() string { 463 | return s.TypedString().String() 464 | } 465 | func (s Int) Native() interface{} { return int64(s) } 466 | func (s Int) TypedString() TypedString { 467 | return TypedString{ 468 | Value: String(strconv.Itoa(int(s))), 469 | Type: defaultIntType, 470 | } 471 | } 472 | 473 | // Float is a native wrapper for float64 type. 474 | // 475 | // It uses NQuad notation similar to TypedString. 476 | type Float float64 477 | 478 | func (s Float) String() string { 479 | return s.TypedString().String() 480 | } 481 | func (s Float) Native() interface{} { return float64(s) } 482 | func (s Float) TypedString() TypedString { 483 | return TypedString{ 484 | Value: String(strconv.FormatFloat(float64(s), 'E', -1, 64)), 485 | Type: defaultFloatType, 486 | } 487 | } 488 | 489 | // Bool is a native wrapper for bool type. 490 | // 491 | // It uses NQuad notation similar to TypedString. 492 | type Bool bool 493 | 494 | func (s Bool) String() string { 495 | if bool(s) { 496 | return `"True"^^<` + string(defaultBoolType) + `>` 497 | } 498 | return `"False"^^<` + string(defaultBoolType) + `>` 499 | } 500 | func (s Bool) Native() interface{} { return bool(s) } 501 | func (s Bool) TypedString() TypedString { 502 | v := "False" 503 | if bool(s) { 504 | v = "True" 505 | } 506 | return TypedString{ 507 | Value: String(v), 508 | Type: defaultBoolType, 509 | } 510 | } 511 | 512 | var _ Equaler = Time{} 513 | 514 | // Time is a native wrapper for time.Time type. 515 | // 516 | // It uses NQuad notation similar to TypedString. 517 | type Time time.Time 518 | 519 | func (s Time) String() string { 520 | return s.TypedString().String() 521 | } 522 | func (s Time) Native() interface{} { return time.Time(s) } 523 | func (s Time) Equal(v Value) bool { 524 | t, ok := v.(Time) 525 | if !ok { 526 | return false 527 | } 528 | return time.Time(s).Equal(time.Time(t)) 529 | } 530 | func (s Time) TypedString() TypedString { 531 | return TypedString{ 532 | // TODO(dennwc): this is used to compute hash, thus we might want to include nanos 533 | Value: String(time.Time(s).UTC().Format(time.RFC3339)), 534 | Type: defaultTimeType, 535 | } 536 | } 537 | 538 | type ByValueString []Value 539 | 540 | func (o ByValueString) Len() int { return len(o) } 541 | func (o ByValueString) Less(i, j int) bool { return StringOf(o[i]) < StringOf(o[j]) } 542 | func (o ByValueString) Swap(i, j int) { o[i], o[j] = o[j], o[i] } 543 | 544 | // Sequence is an object to generate a sequence of Blank Nodes. 545 | type Sequence struct { 546 | last uint64 547 | } 548 | 549 | // Next returns a new blank node. It's safe for concurrent use. 550 | func (s *Sequence) Next() BNode { 551 | n := atomic.AddUint64(&s.last, 1) 552 | return BNode(fmt.Sprintf("n%d", n)) 553 | } 554 | 555 | var randSource = rand.New(rand.NewSource(time.Now().UnixNano())) 556 | 557 | // RandomBlankNode returns a randomly generated Blank Node. 558 | func RandomBlankNode() BNode { 559 | return BNode(fmt.Sprintf("n%d", randSource.Int())) 560 | } 561 | -------------------------------------------------------------------------------- /value_test.go: -------------------------------------------------------------------------------- 1 | package quad 2 | 3 | import ( 4 | "encoding/hex" 5 | "testing" 6 | ) 7 | 8 | var hashCases = []struct { 9 | val Value 10 | hash string 11 | }{ 12 | {String(`abc`), "b87f4bf9b7b07f594430548b653b4998e4b40402"}, 13 | {Raw(`"abc"`), "b87f4bf9b7b07f594430548b653b4998e4b40402"}, 14 | {BNode(`abc`), "3603f98d3203a037ffa6b8780b97ef8bc964fd94"}, 15 | {Raw(`_:abc`), "3603f98d3203a037ffa6b8780b97ef8bc964fd94"}, 16 | {IRI(`abc`), "b301db80a006fb0c667f3feffbf8c68a7b38fe7e"}, 17 | {Raw(``), "b301db80a006fb0c667f3feffbf8c68a7b38fe7e"}, 18 | } 19 | 20 | func TestHashOf(t *testing.T) { 21 | for i, c := range hashCases { 22 | h := hex.EncodeToString(HashOf(c.val)) 23 | if h != c.hash { 24 | t.Errorf("unexpected hash for case %d: %v vs %v", i+1, h, c.hash) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /voc/core/all.go: -------------------------------------------------------------------------------- 1 | // Package core imports all well-known RDF vocabularies. 2 | package core 3 | 4 | import ( 5 | _ "github.com/cayleygraph/quad/voc/rdf" 6 | _ "github.com/cayleygraph/quad/voc/rdfs" 7 | _ "github.com/cayleygraph/quad/voc/schema" 8 | ) 9 | -------------------------------------------------------------------------------- /voc/owl/owl.go: -------------------------------------------------------------------------------- 1 | // Package owl contains constants of the Web Ontology Language (OWL) 2 | package owl 3 | 4 | import "github.com/cayleygraph/quad/voc" 5 | 6 | func init() { 7 | voc.RegisterPrefix(Prefix, NS) 8 | } 9 | 10 | const ( 11 | NS = `http://www.w3.org/2002/07/owl#` 12 | Prefix = `owl:` 13 | ) 14 | 15 | const ( 16 | UnionOf = Prefix + "unionOf" 17 | Restriction = Prefix + "Restriction" 18 | OnProperty = Prefix + "onProperty" 19 | Cardinality = Prefix + "cardinality" 20 | MaxCardinality = Prefix + "maxCardinality" 21 | Thing = Prefix + "Thing" 22 | Class = Prefix + "Class" 23 | DatatypeProperty = Prefix + "DatatypeProperty" 24 | ObjectProperty = Prefix + "ObjectProperty" 25 | ) 26 | -------------------------------------------------------------------------------- /voc/rdf/rdf.go: -------------------------------------------------------------------------------- 1 | // Package rdf contains constants of the RDF Concepts Vocabulary (RDF) 2 | package rdf 3 | 4 | import "github.com/cayleygraph/quad/voc" 5 | 6 | func init() { 7 | voc.RegisterPrefix(Prefix, NS) 8 | } 9 | 10 | const ( 11 | NS = `http://www.w3.org/1999/02/22-rdf-syntax-ns#` 12 | Prefix = `rdf:` 13 | ) 14 | 15 | const ( 16 | // Types 17 | 18 | // The datatype of RDF literals storing fragments of HTML content 19 | HTML = Prefix + `HTML` 20 | // The datatype of language-tagged string values 21 | LangString = Prefix + `langString` 22 | // The class of plain (i.e. untyped) literal values, as used in RIF and OWL 2 23 | PlainLiteral = Prefix + `PlainLiteral` 24 | // The class of RDF properties. 25 | Property = Prefix + `Property` 26 | // The class of RDF statements. 27 | Statement = Prefix + `Statement` 28 | 29 | // Properties 30 | 31 | // The subject is an instance of a class. 32 | Type = Prefix + `type` 33 | // Idiomatic property used for structured values. 34 | Value = Prefix + `value` 35 | // The subject of the subject RDF statement. 36 | Subject = Prefix + `subject` 37 | // The predicate of the subject RDF statement. 38 | Predicate = Prefix + `predicate` 39 | // The object of the subject RDF statement. 40 | Object = Prefix + `object` 41 | 42 | // The class of unordered containers. 43 | Bag = Prefix + `Bag` 44 | // The class of ordered containers. 45 | Seq = Prefix + `Seq` 46 | // The class of containers of alternatives. 47 | Alt = Prefix + `Alt` 48 | // The class of RDF Lists. 49 | List = Prefix + `List` 50 | // The empty list, with no items in it. If the rest of a list is nil then the list has no more items in it. 51 | Nil = Prefix + `nil` 52 | // The first item in the subject RDF list. 53 | First = Prefix + `first` 54 | // The rest of the subject RDF list after the first item. 55 | Rest = Prefix + `rest` 56 | // The datatype of XML literal values. 57 | XMLLiteral = Prefix + `XMLLiteral` 58 | ) 59 | -------------------------------------------------------------------------------- /voc/rdfs/rdfs.go: -------------------------------------------------------------------------------- 1 | // Package rdfs contains constants of the RDF Schema vocabulary (RDFS) 2 | package rdfs 3 | 4 | import "github.com/cayleygraph/quad/voc" 5 | 6 | func init() { 7 | voc.RegisterPrefix(Prefix, NS) 8 | } 9 | 10 | const ( 11 | NS = `http://www.w3.org/2000/01/rdf-schema#` 12 | Prefix = `rdfs:` 13 | ) 14 | 15 | const ( 16 | // Classes 17 | 18 | // The class resource, everything. 19 | Resource = Prefix + `Resource` 20 | // The class of classes. 21 | Class = Prefix + `Class` 22 | // The class of literal values, eg. textual strings and integers. 23 | Literal = Prefix + `Literal` 24 | // The class of RDF containers. 25 | Container = Prefix + `Container` 26 | // The class of RDF datatypes. 27 | Datatype = Prefix + `Datatype` 28 | // The class of container membership properties, rdf:_1, rdf:_2, ..., all of which are sub-properties of 'member'. 29 | ContainerMembershipProperty = Prefix + `ContainerMembershipProperty` 30 | 31 | // Properties 32 | 33 | // The subject is a subclass of a class. 34 | SubClassOf = Prefix + `subClassOf` 35 | // The subject is a subproperty of a property. 36 | SubPropertyOf = Prefix + `subPropertyOf` 37 | // A description of the subject resource. 38 | Comment = Prefix + `comment` 39 | // A human-readable name for the subject. 40 | Label = Prefix + `label` 41 | // A domain of the subject property. 42 | Domain = Prefix + `domain` 43 | // A range of the subject property. 44 | Range = Prefix + `range` 45 | // Further information about the subject resource. 46 | SeeAlso = Prefix + `seeAlso` 47 | // The defininition of the subject resource. 48 | IsDefinedBy = Prefix + `isDefinedBy` 49 | // A member of the subject resource. 50 | Member = Prefix + `member` 51 | ) 52 | -------------------------------------------------------------------------------- /voc/schema/schema.go: -------------------------------------------------------------------------------- 1 | // Package schema contains constants of the Schema.org vocabulary. 2 | package schema 3 | 4 | import "github.com/cayleygraph/quad/voc" 5 | 6 | func init() { 7 | voc.RegisterPrefix(Prefix, NS) 8 | } 9 | 10 | const ( 11 | NS = `http://schema.org/` 12 | Prefix = `schema:` 13 | ) 14 | 15 | const ( 16 | // Types 17 | 18 | // The basic data types such as Integers, Strings, etc. 19 | DataType = Prefix + `DataType` 20 | // Boolean: True or False. 21 | Boolean = Prefix + `Boolean` 22 | // The boolean value false. 23 | False = Prefix + `False` 24 | // The boolean value true. 25 | True = Prefix + `True` 26 | // Data type: Text. 27 | Text = Prefix + `Text` 28 | // Data type: URL. 29 | URL = Prefix + `URL` 30 | // Data type: Number. 31 | Number = Prefix + `Number` 32 | // Data type: Floating number. 33 | Float = Prefix + `Float` 34 | // Data type: Integer. 35 | Integer = Prefix + `Integer` 36 | // A date value in ISO 8601 date format. 37 | Date = Prefix + `Date` 38 | // A point in time recurring on multiple days in the form hh:mm:ss[Z|(+|-)hh:mm]. 39 | Time = Prefix + `Time` 40 | // A combination of date and time of day in the form [-]CCYY-MM-DDThh:mm:ss[Z|(+|-)hh:mm] (see Chapter 5.4 of ISO 8601). 41 | DateTime = Prefix + `DateTime` 42 | 43 | // A class, also often called a 'Type'; equivalent to rdfs:Class. 44 | Class = Prefix + "Class" 45 | // A property, used to indicate attributes and relationships of some Thing; equivalent to rdf:Property. 46 | Property = Prefix + "Property" 47 | ) 48 | 49 | const ( 50 | // The name of the item. 51 | Name = Prefix + `name` 52 | UrlProp = Prefix + `url` 53 | ) 54 | -------------------------------------------------------------------------------- /voc/voc.go: -------------------------------------------------------------------------------- 1 | // Package voc implements an RDF namespace (vocabulary) registry. 2 | package voc 3 | 4 | import ( 5 | "strings" 6 | "sync" 7 | ) 8 | 9 | // Namespace is a RDF namespace (vocabulary). 10 | type Namespace struct { 11 | Full string 12 | Prefix string 13 | } 14 | 15 | type ByFullName []Namespace 16 | 17 | func (o ByFullName) Len() int { return len(o) } 18 | func (o ByFullName) Less(i, j int) bool { return o[i].Full < o[j].Full } 19 | func (o ByFullName) Swap(i, j int) { o[i], o[j] = o[j], o[i] } 20 | 21 | // Namespaces is a set of registered namespaces. 22 | type Namespaces struct { 23 | Safe bool // if set, assume no locking is required 24 | mu sync.RWMutex 25 | prefixes map[string]string 26 | } 27 | 28 | // Register adds namespace to registered list. 29 | func (p *Namespaces) Register(ns Namespace) { 30 | if !p.Safe { 31 | p.mu.Lock() 32 | defer p.mu.Unlock() 33 | } 34 | if p.prefixes == nil { 35 | p.prefixes = make(map[string]string) 36 | } 37 | p.prefixes[ns.Prefix] = ns.Full 38 | } 39 | 40 | // ShortIRI replaces a base IRI of a known vocabulary with it's prefix. 41 | // 42 | // ShortIRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type") // returns "rdf:type" 43 | func (p *Namespaces) ShortIRI(iri string) string { 44 | if !p.Safe { 45 | p.mu.RLock() 46 | defer p.mu.RUnlock() 47 | } 48 | for pref, ns := range p.prefixes { 49 | if strings.HasPrefix(iri, ns) { 50 | return pref + iri[len(ns):] 51 | } 52 | } 53 | return iri 54 | } 55 | 56 | // FullIRI replaces known prefix in IRI with it's full vocabulary IRI. 57 | // 58 | // FullIRI("rdf:type") // returns "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" 59 | func (p *Namespaces) FullIRI(iri string) string { 60 | if !p.Safe { 61 | p.mu.RLock() 62 | defer p.mu.RUnlock() 63 | } 64 | for pref, ns := range p.prefixes { 65 | if strings.HasPrefix(iri, pref) { 66 | return ns + iri[len(pref):] 67 | } 68 | } 69 | return iri 70 | } 71 | 72 | // List enumerates all registered namespace pairs. 73 | func (p *Namespaces) List() (out []Namespace) { 74 | if !p.Safe { 75 | p.mu.RLock() 76 | defer p.mu.RUnlock() 77 | } 78 | out = make([]Namespace, 0, len(p.prefixes)) 79 | for pref, ns := range p.prefixes { 80 | out = append(out, Namespace{Prefix: pref, Full: ns}) 81 | } 82 | return 83 | } 84 | 85 | // Clone makes a copy of namespaces list. 86 | func (p *Namespaces) Clone() *Namespaces { 87 | if !p.Safe { 88 | p.mu.RLock() 89 | defer p.mu.RUnlock() 90 | } 91 | p2 := Namespaces{ 92 | prefixes: make(map[string]string, len(p.prefixes)), 93 | } 94 | for pref, ns := range p.prefixes { 95 | p2.prefixes[pref] = ns 96 | } 97 | return &p2 98 | } 99 | 100 | // CloneTo adds registered namespaces to a given list. 101 | func (p *Namespaces) CloneTo(p2 *Namespaces) { 102 | if p == p2 { 103 | return 104 | } 105 | if !p.Safe { 106 | p.mu.RLock() 107 | defer p.mu.RUnlock() 108 | } 109 | if !p2.Safe { 110 | p2.mu.Lock() 111 | defer p2.mu.Unlock() 112 | } 113 | if p2.prefixes == nil { 114 | p2.prefixes = make(map[string]string, len(p.prefixes)) 115 | } 116 | for pref, ns := range p.prefixes { 117 | p2.prefixes[pref] = ns 118 | } 119 | } 120 | 121 | var global Namespaces 122 | 123 | // Register adds namespace to a global registered list. 124 | func Register(ns Namespace) { 125 | global.Register(ns) 126 | } 127 | 128 | // RegisterPrefix globally associates a given prefix with a base vocabulary IRI. 129 | func RegisterPrefix(pref string, ns string) { 130 | Register(Namespace{Prefix: pref, Full: ns}) 131 | } 132 | 133 | // ShortIRI replaces a base IRI of a known vocabulary with it's prefix. 134 | // 135 | // ShortIRI("http://www.w3.org/1999/02/22-rdf-syntax-ns#type") // returns "rdf:type" 136 | func ShortIRI(iri string) string { 137 | return global.ShortIRI(iri) 138 | } 139 | 140 | // FullIRI replaces known prefix in IRI with it's full vocabulary IRI. 141 | // 142 | // FullIRI("rdf:type") // returns "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" 143 | func FullIRI(iri string) string { 144 | return global.FullIRI(iri) 145 | } 146 | 147 | // List enumerates all registered namespace pairs. 148 | func List() []Namespace { 149 | return global.List() 150 | } 151 | 152 | // Clone makes a copy of global namespaces list. 153 | func Clone() *Namespaces { 154 | return global.Clone() 155 | } 156 | 157 | // CloneTo adds all global namespaces to a given list. 158 | func CloneTo(p *Namespaces) { 159 | global.CloneTo(p) 160 | } 161 | -------------------------------------------------------------------------------- /voc/voc_test.go: -------------------------------------------------------------------------------- 1 | package voc 2 | 3 | import "testing" 4 | 5 | var casesShortIRI = []struct { 6 | full string 7 | short string 8 | }{ 9 | {full: "http://example.com/name", short: "ex:name"}, 10 | } 11 | 12 | func TestShortIRI(t *testing.T) { 13 | RegisterPrefix("ex:", "http://example.com/") 14 | for _, c := range casesShortIRI { 15 | if f := FullIRI(c.full); f != c.full { 16 | t.Fatal("unexpected full iri:", f) 17 | } 18 | s := ShortIRI(c.full) 19 | if s != c.short { 20 | t.Fatal("unexpected short iri:", s) 21 | } 22 | if f := FullIRI(s); f != c.full { 23 | t.Fatal("unexpected full iri:", f) 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /voc/xsd/xsd.go: -------------------------------------------------------------------------------- 1 | // Package xsd contains constants of the W3C XML Schema Definition Language https://www.w3.org/TR/xmlschema11-1/ 2 | package xsd 3 | 4 | import "github.com/cayleygraph/quad/voc" 5 | 6 | func init() { 7 | voc.RegisterPrefix(Prefix, NS) 8 | } 9 | 10 | const ( 11 | NS = "http://www.w3.org/2001/XMLSchema#" 12 | Prefix = "xsd:" 13 | ) 14 | 15 | // Base types 16 | const ( 17 | // Boolean represents the values of two-valued logic. 18 | Boolean = Prefix + `boolean` 19 | // String represents character strings 20 | String = Prefix + `string` 21 | // Double datatype is patterned after the IEEE double-precision 64-bit floating point datatype [IEEE 754-2008]. Each floating point datatype has a value space that is a subset of the rational numbers. Floating point numbers are often used to approximate arbitrary real numbers. 22 | Double = Prefix + `double` 23 | // DateTime represents instants of time, optionally marked with a particular time zone offset. Values representing the same instant but having different time zone offsets are equal but not identical. 24 | DateTime = Prefix + `dateTime` 25 | ) 26 | 27 | // Extra numeric types 28 | const ( 29 | // Integer is derived from decimal by fixing the value of fractionDigits to be 0 and disallowing the trailing decimal point. This results in the standard mathematical concept of the integer numbers. 30 | Integer = Prefix + `integer` 31 | // Long is derived from integer by setting the value of maxInclusive to be 9223372036854775807 and minInclusive to be -9223372036854775808. The base type of long is integer. 32 | Long = Prefix + `long` 33 | // Int is derived from long by setting the value of maxInclusive to be 2147483647 and minInclusive to be -2147483648. 34 | Int = Prefix + `int` 35 | // Float datatype is patterned after the IEEE single-precision 32-bit floating point datatype 36 | Float = Prefix + `float` 37 | ) 38 | --------------------------------------------------------------------------------