├── img ├── logo.png └── IMG_1945.jpg ├── go.mod ├── merkle7574_test.go ├── id_test.go ├── y.FIRST.md ├── heap.go ├── go.sum ├── y.L.md ├── iter_test.go ├── y.X.md ├── jdr_test.go ├── zipint_test.go ├── z.md ├── ron.go ├── ron_test.go ├── merkle7574.go ├── rdx_test.go ├── test.go ├── y.E.md ├── md_test.go ├── y.P.md ├── tlv.go ├── itheap_test.go ├── zipint.go ├── id.go ├── DISCONT.md ├── JDR.go.rl ├── itheap.go ├── README.md ├── jdr.go ├── rdx.go └── JDR.rl.go /img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gritzko/go-rdx/HEAD/img/logo.png -------------------------------------------------------------------------------- /img/IMG_1945.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gritzko/go-rdx/HEAD/img/IMG_1945.jpg -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gritzko/go-rdx 2 | 3 | go 1.23 4 | 5 | require ( 6 | github.com/davecgh/go-spew v1.1.1 // indirect 7 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 8 | github.com/pmezard/go-difflib v1.0.0 // indirect 9 | github.com/stretchr/testify v1.10.0 // indirect 10 | gopkg.in/yaml.v3 v3.0.1 // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /merkle7574_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func Test7574(t *testing.T) { 9 | a := Sha256Of([]byte("a")) 10 | assert.Equal(t, "ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb", a.String()) 11 | aa := a.Merkle2(a) 12 | assert.Equal(t, "251a262291b87cb3c93a6ed71865da1f2c090c3d0196661a8f4a705b65836f71", aa.String()) 13 | } 14 | -------------------------------------------------------------------------------- /id_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestLexize(t *testing.T) { 9 | cases := [][]string{ 10 | {"bob-0", "bob-0"}, 11 | {"bob-bebe", "bob-ebeb000000"}, 12 | {"bob-bububebebo", "bob-obebebubub"}, 13 | {"bob-Fbububebebo", "bob-obebebubub"}, 14 | {"bob-boo0", "bob-oob000000"}, 15 | {"bob-boo", "bob-oob0000000"}, 16 | } 17 | for _, c := range cases { 18 | a, _ := ParseID([]byte(c[0])) 19 | aa := Revert64(a.Seq) 20 | id := ID{Seq: aa, Src: a.Src} 21 | assert.Equal(t, c[1], string(id.RonString())) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /y.FIRST.md: -------------------------------------------------------------------------------- 1 | `` ` 2 | # FIRST element merge 3 | 4 | FIRST: Float, Integer, Reference, String, Term. 5 | Merges are last-write-wins, based on: 6 | 7 | 1. ID seq, then 8 | 2. ID src, then 9 | 3. type, then 10 | 4. value (type dependant). 11 | 12 | If 1-4 are equal, elements are identical. 13 | ``` 14 | a, 15 | a, 16 | a, 17 | ~, 18 | a, 19 | 20 | b, 21 | a, 22 | ~, 23 | b, 24 | 25 | a b; 26 | a:b:c; 27 | ~ 28 | a:b:c 29 | 30 | a, 31 | b, 32 | ~, 33 | b, 34 | 35 | 1 @a-1, 36 | 2 @b-2, 37 | 3 @c-3, 38 | ~, 39 | 3@c-3, 40 | 41 | 1.23, 42 | 4.5, 43 | ~, 44 | 4.5, 45 | 46 | 1.23, 47 | 123, 48 | ~, 49 | 123, 50 | 51 | "C" @c-3, 52 | "B" @b-2, 53 | "A" @a-1, 54 | ~, 55 | "C"@c-3, 56 | 57 | ``` 58 | `` ` 59 | 60 | -------------------------------------------------------------------------------- /heap.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import "sort" 4 | 5 | func HeapUp(ih sort.Interface) { 6 | a := ih.Len() - 1 7 | if a < 0 { 8 | return 9 | } 10 | for { 11 | b := (a - 1) / 2 // parent 12 | if b == a || !ih.Less(a, b) { 13 | break 14 | } 15 | ih.Swap(a, b) 16 | a = b 17 | } 18 | } 19 | 20 | func HeapDownN(ih sort.Interface, i0 int) { 21 | n := ih.Len() 22 | i := i0 23 | for { 24 | j1 := 2*i + 1 25 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 26 | break 27 | } 28 | j := j1 // left child 29 | if j2 := j1 + 1; j2 < n && ih.Less(j2, j1) { 30 | j = j2 // = 2*i + 2 // right child 31 | } 32 | if !ih.Less(j, i) { 33 | break 34 | } 35 | ih.Swap(i, j) 36 | i = j 37 | } 38 | } 39 | 40 | func HeapDown(ih sort.Interface) { 41 | HeapDownN(ih, 0) 42 | } 43 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= 4 | github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 5 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 6 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 7 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 8 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 10 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 11 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 12 | -------------------------------------------------------------------------------- /y.L.md: -------------------------------------------------------------------------------- 1 | `` ` 2 | # This is a merge test set for Linear collections. 3 | 4 | 1. Unstamped collections can only be appended to; 5 | items can be replaced. 6 | ``` 7 | [1, 2, 3, 4] 8 | [1, 22@2, 3] 9 | ~ 10 | [1, 22@2, 3, 4] 11 | ``` 12 | 2. Appends also work nicely for stamped collections. 13 | They are also more efficient; no need to cite the entire array. 14 | ``` 15 | [a@10 b@30 c@50] 16 | [c@50 d@70] 17 | ~ 18 | [a@10 b@30 c@50 d@70] 19 | ``` 20 | 3. Inserts are only possible into stamped collections. 21 | ``` 22 | [a@10 b@30 c@50] 23 | [b@30 e@40 c@50 d@70] 24 | ~ 25 | [a@10 b@30 e@40 c@50 d@70] 26 | ``` 27 | 4. With the numbering space depleted we have to use insert trains. 28 | ``` 29 | [a@10 b@30 e@40 c@50 d@70] 30 | [a@10 aa@bob-20 ab@10 ac@210 ad@410 ae@610] 31 | ~ 32 | [a@10 aa@bob-20 ab@10 ac@210 b@30 e@40 ad@410 c@50 ae@610 d@70] 33 | ``` 34 | 5. Although it may take some time for 64 bits to be depleted 35 | ``` 36 | [0.0@20 1.0@30] 37 | [0.4@420] 38 | [0.2@220 0.3@320] 39 | [0.5@520 0.6@620] 40 | ~ 41 | [0.0@20 0.2@220 1.0@30 0.3@320 0.4@420 0.5@520 0.6@620] 42 | ``` 43 | ` `` 44 | -------------------------------------------------------------------------------- /iter_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestIter(t *testing.T) { 9 | rdx := []byte{} 10 | idA := ID{0xA, 0x1} 11 | val := ZipInt64(-123) 12 | rdx = WriteRDX(rdx, LitInteger, idA, val) 13 | rdx = AppendInteger(rdx, 123) 14 | str := make([]byte, 300) 15 | str[222] = 33 16 | rdx = AppendString(rdx, str) 17 | 18 | it := NewIter(rdx) 19 | assert.True(t, it.Read()) 20 | assert.Nil(t, it.Error()) 21 | assert.Equal(t, byte(LitInteger), it.Lit()) 22 | assert.Equal(t, val, it.Value()) 23 | assert.Equal(t, idA, it.ID()) 24 | assert.Equal(t, byte('i'), it.Rest()[0]) 25 | assert.Equal(t, byte('i'), it.Record()[0]) 26 | assert.False(t, it.IsLive()) 27 | 28 | assert.True(t, it.Read()) 29 | assert.Nil(t, it.Error()) 30 | assert.Equal(t, byte(LitInteger), it.Lit()) 31 | assert.Equal(t, ID{}, it.ID()) 32 | assert.Equal(t, ZipInt64(123), it.Value()) 33 | assert.True(t, it.IsLive()) 34 | 35 | assert.True(t, it.Read()) 36 | assert.Nil(t, it.Error()) 37 | assert.Equal(t, byte(LitString), it.Lit()) 38 | assert.Equal(t, ID{}, it.ID()) 39 | assert.Equal(t, 0, len(it.Rest())) 40 | assert.True(t, it.IsLive()) 41 | assert.Equal(t, str, it.Value()) 42 | 43 | assert.False(t, it.Read()) 44 | assert.True(t, it.IsEmpty()) 45 | assert.Equal(t, nil, it.Error()) 46 | } 47 | -------------------------------------------------------------------------------- /y.X.md: -------------------------------------------------------------------------------- 1 | `` ` 2 | # Multiplexed collection merge 3 | 4 | X collections are counters, version vectors and suchlike. 5 | X elements get sorted by the author, elements by the same author get merged. 6 | For two versions of an element, higher revision wins. 7 | For two variants of an element , higher value wins. 8 | Enveloping rules are standard for all PLEX collections . 9 | ``` 10 | <1>,<3>,<4@5>, ~, <4@5>, 11 | 12 | <1>,<3>,<2, 4@b0b-5>, ~, <3, 4@b0b-5>, 13 | 14 | <0@b0b-1, 0@a1ec-4>, 15 | <0@b0b-3, 0@a1ec-2>, 16 | ~, 17 | <0@b0b-3, 0@a1ec-4>, 18 | 19 | <0@b0b-3, 0@a1ec-2>, 20 | <0@b0b-1, 0@a1ec-4>, 21 | ~, 22 | <0@b0b-3, 0@a1ec-4>, 23 | 24 | <1@be-1, -4@a1e-4>, 25 | <3@be-3, -2@a1e-2>, 26 | <2@be-2, -3@a1e-3>, 27 | ~, 28 | <3@be-3, -4@a1e-4>, 29 | 30 | <3@be-3, -4@a1-4>, 31 | <2@be-2, -3@a1-3>, 32 | ~, 33 | <3@be-3, -4@a1-4>, 34 | 35 | <1@be-1, -4@a-4>, 36 | <3@be-3, -3@a-3>, 37 | ~, 38 | <3@be-3, -4@a-4>, 39 | 40 | ``` 41 | Right, higher version wins 42 | ``` 43 | <1@be-1, -4@alex-4>, 44 | <3@be-3, -2@alex-2>, 45 | <2@be-2, -3@alex-3>, 46 | <1@be-1, -4@alex-4>, 47 | <3@be-3, -2@alex-2>, 48 | <2@be-2, -3@alex-3>, 49 | ~, 50 | <3@be-3, -4@alex-4>, 51 | 52 | ``` 53 | Here we delete an element 54 | ``` 55 | <1@b0b-1, 1234@a1ec-0>, 56 | <2@b0b-2, <@a1ec-1 1234>>, 57 | ~, 58 | <2@b0b-2, <@a1ec-1 1234>>, 59 | 60 | -------------------------------------------------------------------------------- /jdr_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | func TestReadWriteJDR(t *testing.T) { 10 | cases := []string{ 11 | "1:2:3", 12 | "1", 13 | "1 2 3", 14 | "1.2e+03", 15 | "12-3", 16 | "\"one\\ttwo three\"", 17 | "one two three", 18 | "():()", 19 | "()", 20 | "(())", 21 | "(1)", 22 | "1:2.3e+04:56-78", 23 | "<1@alice-1,2@bob-2>", 24 | "{\"one\",\"two\",\"three\"}", 25 | } 26 | for _, c := range cases { 27 | state := JDRstate{jdr: []byte(c), stack: []Mark{Mark{}}} 28 | err := JDRlexer(&state) 29 | if err != nil { 30 | _, _ = fmt.Fprintln(os.Stderr, c) 31 | t.Fatal(err) 32 | } 33 | j2, err := WriteAllJDR(nil, state.rdx, 0) 34 | jdr2 := string(j2) 35 | if jdr2 != c { 36 | t.Error("'" + jdr2 + "' != '" + c + "'") 37 | } 38 | } 39 | } 40 | 41 | func TestTuples(t *testing.T) { 42 | cases := []string{ 43 | "1:2:3 4:5:6 ()", 44 | "(1 2 3)(4 5 6)()", 45 | "1 2 3; 4:5:6; ;", 46 | } 47 | correct := "1:2:3 4:5:6 ()" 48 | for _, c := range cases { 49 | state := JDRstate{jdr: []byte(c), stack: []Mark{Mark{}}} 50 | err := JDRlexer(&state) 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | j2, err := WriteAllJDR(nil, state.rdx, 0) 55 | jdr2 := string(j2) 56 | if jdr2 != correct { 57 | t.Error("'" + jdr2 + "' != '" + correct + "' for '" + c + "'") 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /zipint_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestEncodeVarPair(t *testing.T) { 10 | nums := []uint64{ 11 | 0, 12 | 0xca, 13 | 0xbeff, 14 | 0x12345678, 15 | 0x7777777788888888, 16 | } 17 | for i := 0; i < len(nums); i++ { 18 | for j := 0; j < len(nums); j++ { 19 | one := nums[i] 20 | two := nums[j] 21 | bin := ZipUint64Pair(one, two) 22 | einz, twei := UnzipUint64Pair(bin) 23 | assert.Equal(t, one, einz) 24 | assert.Equal(t, two, twei) 25 | } 26 | } 27 | } 28 | 29 | func TestZigZagInt64(t *testing.T) { 30 | test := map[int64]uint64{ 31 | 0: 0, 32 | -14: 27, 33 | -10: 19, 34 | 7: 14, 35 | 20: 40, 36 | } 37 | for i, u := range test { 38 | u2 := ZigZagInt64(int64(i)) 39 | assert.Equal(t, u, u2) 40 | i2 := ZagZigUint64(u2) 41 | assert.Equal(t, i, i2) 42 | } 43 | } 44 | 45 | func TestZipFloat64(t *testing.T) { 46 | test := map[float64]int{ 47 | 0: 0, 48 | 1: 2, 49 | 1234: 3, 50 | 12.25: 3, 51 | } 52 | for f, l := range test { 53 | u := ZipFloat64(f) 54 | assert.Equal(t, l, len(u)) 55 | f2 := UnzipFloat64(u) 56 | assert.Equal(t, f, f2) 57 | } 58 | } 59 | 60 | func TestZipID(t *testing.T) { 61 | var i0 uint64 = 7 62 | var u0 uint64 = 15 63 | zip := ZipUint64Pair(i0, u0) 64 | i, u := UnzipUint64Pair(zip) 65 | assert.Equal(t, i0, i) 66 | assert.Equal(t, u0, u) 67 | } 68 | -------------------------------------------------------------------------------- /z.md: -------------------------------------------------------------------------------- 1 | `` ` 2 | This file tests the value-order of RDX elements. 3 | That is the order Eulerian containers use; it is also used as a fallback in other orders. 4 | ``` 5 | 6 | ``` 7 | An empty tuple is the leastest element. 8 | Other tuples get ordered according to their key element (i.e. the first one). 9 | A tuple of one element is the same thing as the element itself. 10 | ``` 11 | (), 12 | ``` 13 | Floats get ordered numerically. 14 | ``` 15 | -0.123, 16 | 1.23, 17 | 1.24 @a1ec-1, 18 | ``` 19 | Integers are like floats. 20 | ``` 21 | -3, 22 | -2, 23 | 1, 24 | 2, 25 | 3 @b0b-1, 26 | ``` 27 | Again, a tuple is ordered the same as its key. 28 | ``` 29 | 0:5, 30 | 2:3, 31 | 4:1, 32 | ``` 33 | References get ordered in the "Lamport order" (seq, then src). 34 | ``` 35 | b0b-2, 36 | b0b-3 @b0b-4, 37 | b0b-4 @b0b-3, 38 | b0b-5 @a1ec-1, 39 | a1ec-7, 40 | ``` 41 | Strings get ordered lexicographically (strcmp wise). 42 | ``` 43 | "Alice", 44 | "Bob" @b0b-1, 45 | "Bobby", 46 | "Carol", 47 | ``` 48 | Terms: same as strings. 49 | ``` 50 | false, 51 | once:twice, 52 | one:two:three, 53 | true, 54 | ``` 55 | Eulerian collections: by their id. 56 | ``` 57 | { 1, 2, 3, }, 58 | { @e-1 1, 2, 3 }, 59 | { @e-2 }, 60 | ``` 61 | ...that applies to arrays as well... 62 | ``` 63 | [ 1, 2, 3 ], 64 | [ @a-1 2, 3, 4, 5 ], 65 | [ @a-2 3 1,2,3 ], 66 | ``` 67 | ...as well as to multiplexed collections. 68 | ``` 69 | < 1@a1ec-1, 2@b0b-1 >, 70 | < @b0b-0 1@a1ec-1, 2@b0b-1 >, 71 | <@b0b-1 ~@b0b-2, ~@a1ec-3 >, 72 | <@a1ec-1 ~@b0b-3, ~@a1ec-3 >, 73 | -------------------------------------------------------------------------------- /ron.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import "math/bits" 4 | import "errors" 5 | 6 | type Ron60 uint64 7 | 8 | func (r Ron60) IsValid() bool { 9 | return r == (r & Ron60(Mask60bit)) 10 | } 11 | 12 | const Ron60Top = Ron60((uint64(1) << 60) - 1 - (uint64(1) << 54)) 13 | 14 | func NewRon60(x uint64) Ron60 { 15 | x &= Mask60bit 16 | if x == 0 { 17 | return Ron60Top 18 | } 19 | shift := (((bits.LeadingZeros64(x) - 4) / 6) * 6) 20 | x <<= shift 21 | return Ron60(x) 22 | } 23 | 24 | var ErrBadRon60Syntax = errors.New("bad RON 60 syntax") 25 | 26 | func ParseRon60(txt []byte) (r Ron60, err error) { 27 | val, rest := ParseRON64(txt) 28 | if len(rest) > 0 { 29 | return 0, ErrBadRon60Syntax 30 | } 31 | n := NewRon60(val) 32 | return n, nil 33 | } 34 | 35 | func (r Ron60) Uint64() uint64 { 36 | if r == Ron60Top { 37 | return 0 38 | } 39 | x := uint64(r) 40 | shift := (bits.TrailingZeros64(x) / 6) * 6 41 | return x >> shift 42 | } 43 | 44 | func (r Ron60) String() string { 45 | return string(RON64String(r.Uint64())) 46 | } 47 | 48 | const ron60bit = uint64(1) << 54 49 | const Ron60Bottom = Ron60((uint64(63) << 54)) 50 | const Ron60Inc = Ron60(uint64(1) << 42) 51 | 52 | func (a Ron60) Less(b Ron60) bool { 53 | aa := (uint64(a) + ron60bit) & Mask60bit 54 | bb := (uint64(b) + ron60bit) & Mask60bit 55 | return aa < bb 56 | } 57 | 58 | func (a Ron60) Fit(b Ron60) Ron60 { 59 | if b.Less(a + 2) { 60 | return 0 61 | } 62 | c := a ^ b 63 | lz := 60 - (bits.LeadingZeros64(uint64(c)) - 4) 64 | lzrem := lz % 6 65 | if lzrem < 4 { 66 | lz -= lzrem 67 | } else if lz >= 6 { 68 | lz -= lzrem + 6 69 | } else { 70 | lz -= lzrem 71 | } 72 | inc := uint64(1) << lz 73 | return a + Ron60(inc) 74 | } 75 | -------------------------------------------------------------------------------- /ron_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import "testing" 4 | 5 | func TestRon60Parse(t *testing.T) { 6 | var cases = [][2]string{ 7 | {"0", "0"}, 8 | {"10", "1"}, 9 | {"~08", "~08"}, 10 | } 11 | for _, c := range cases { 12 | r, e := ParseRon60([]byte(c[0])) 13 | if e != nil { 14 | t.Error(e) 15 | } 16 | if r.String() != c[1] { 17 | t.Errorf("%s != %s\n", r.String(), c[1]) 18 | } 19 | } 20 | } 21 | 22 | func TestRon60Order(t *testing.T) { 23 | var cases = []string{ 24 | "~000000001", 25 | "~12", 26 | "~123", 27 | "~124", 28 | "~z", 29 | "10", 30 | "12", 31 | "123", 32 | "0", 33 | } 34 | for n, c := range cases { 35 | x, _ := ParseRon60([]byte(c)) 36 | p := Ron60Bottom 37 | if n > 0 { 38 | p, _ = ParseRon60([]byte(cases[n-1])) 39 | } 40 | if !p.Less(x) { 41 | t.Errorf("bad order %s and %s\n", p.String(), x.String()) 42 | } 43 | } 44 | } 45 | 46 | /* 47 | func TestRon60(t *testing.T) { 48 | var cases = [][3]string{ 49 | {"a", "b", "a1"}, 50 | {"1A", "zA", "1B"}, 51 | {"a", "c", "b"}, 52 | {"2000a", "c", "2000b"}, 53 | {"a", "~~~~~~~~~~", "a000000001"}, 54 | {"0", "c", "1"}, 55 | {"0", "C", "1"}, 56 | } 57 | for _, c := range cases { 58 | a, _ := ParseRON64([]byte(c[0])) 59 | b, _ := ParseRON64([]byte(c[1])) 60 | m, _ := ParseRON64([]byte(c[2])) 61 | f := RonLFit(a, b) 62 | if f != m { 63 | t.Errorf("insert between %s and %s,\nwant %s got %s\n", 64 | c[0], c[1], c[2], RON64String(f)) 65 | } 66 | if RonLCompare(a, b) != Less { 67 | t.Errorf("order") 68 | } 69 | if RonLCompare(a, m) != Less { 70 | t.Errorf("order") 71 | } 72 | if RonLCompare(a, f) != Less { 73 | t.Errorf("order") 74 | } 75 | if RonLCompare(b, m) != Grtr { 76 | t.Errorf("order") 77 | } 78 | if RonLCompare(f, b) != Less { 79 | t.Errorf("order") 80 | } 81 | } 82 | }*/ 83 | -------------------------------------------------------------------------------- /merkle7574.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "bytes" 5 | "crypto/sha256" 6 | "encoding/hex" 7 | "errors" 8 | ) 9 | 10 | const Sha256Bytes = 32 11 | 12 | type Sha256 [Sha256Bytes]byte 13 | type Sha256Merkle7574 [64]Sha256 14 | 15 | var Sha256Zero = Sha256{} 16 | 17 | var ErrBadSha256Hex = errors.New("malformed SHA256 hex") 18 | 19 | func (sha *Sha256) Clear() { 20 | copy(sha[:], Sha256Zero[:]) 21 | } 22 | 23 | func (sha Sha256) Equal(b Sha256) bool { 24 | return bytes.Equal(sha[:], b[:]) 25 | } 26 | 27 | func (sha Sha256) Bytes() []byte { 28 | return sha[:] 29 | } 30 | 31 | func (sha Sha256) String() string { 32 | return hex.EncodeToString(sha[:]) 33 | } 34 | 35 | func ParseSha256(str []byte) (sha Sha256, err error) { 36 | if len(str) != Sha256Bytes*2 { 37 | err = ErrBadSha256Hex 38 | return 39 | } 40 | _, err = hex.Decode(sha[:], str) 41 | if err != nil { 42 | err = ErrBadSha256Hex 43 | } 44 | return 45 | } 46 | 47 | func (sha Sha256) IsEmpty() bool { 48 | return bytes.Equal(sha[:], Sha256Zero[:]) 49 | } 50 | 51 | func Sha256Of(data []byte) (ret Sha256) { 52 | hash := sha256.New() 53 | hash.Write(data[:]) 54 | hash.Sum(ret[:0]) 55 | return 56 | } 57 | 58 | func (sha Sha256) Merkle2(b Sha256) (sum Sha256) { 59 | hash := sha256.New() 60 | hash.Write(sha[:]) 61 | hash.Write(b[:]) 62 | hash.Sum(sum[:0]) 63 | return 64 | } 65 | 66 | var ErrOutOfRange = errors.New("out of hash tree range") 67 | 68 | func (line *Sha256Merkle7574) Append(next Sha256) error { 69 | p := next 70 | i := 0 71 | for ; !line[i].IsEmpty() && i < len(line); i++ { 72 | p = line[i].Merkle2(p) 73 | line[i] = Sha256Zero 74 | } 75 | if i == len(line) { 76 | return ErrOutOfRange 77 | } 78 | line[i] = p 79 | return nil 80 | } 81 | 82 | func (line *Sha256Merkle7574) Sum() (sum Sha256) { 83 | hash := sha256.New() 84 | for i := 0; i < len(line); i++ { 85 | hash.Write(line[i][:]) 86 | } 87 | b := sum[:0] 88 | hash.Sum(b) 89 | return 90 | } 91 | -------------------------------------------------------------------------------- /rdx_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | func TestNormalize(t *testing.T) { 10 | cases := [][2]string{ 11 | // {"{1 4 2 2 3 3 3}", "{1 2 3 4}"}, 12 | {"{{{}0{}0\"\"\"\"\"\"1}}", "{{0 1 \"\" {}}}"}, 13 | } 14 | for _, c := range cases { 15 | rdx1, err := ParseJDR([]byte(c[0])) 16 | if err != nil { 17 | return 18 | } 19 | fmt.Println("---") 20 | norm, err := Normalize(rdx1) 21 | fmt.Println("---") 22 | if err != nil { 23 | return 24 | } 25 | jdr := RenderJDR(norm, 0) 26 | if string(jdr) != c[1] { 27 | t.Errorf("Normalize() fails\norigin:|%s|\nnormal:|%x|\nas jdr:|%s|\nparsed:|%x|\n", 28 | c[0], norm, string(jdr), rdx1) 29 | } 30 | } 31 | } 32 | 33 | func FuzzJDR(f *testing.F) { 34 | f.Add("10000000Ae-0") 35 | f.Add("\"\x00\"") 36 | f.Add("(:)") 37 | f.Add("(,:)000") 38 | f.Add("0-0") 39 | f.Add("(]0") 40 | f.Add("{a:1 b:2.0 [c@12 d]}") 41 | f.Add("(") 42 | f.Fuzz(func(t *testing.T, jdr1 string) { 43 | rdx1, err := ParseJDR([]byte(jdr1)) 44 | if err != nil { 45 | return 46 | } 47 | norm, err := Normalize(rdx1) 48 | if err != nil { 49 | return 50 | } 51 | jdr := RenderJDR(norm, 0) 52 | parsed, err2 := ParseJDR(jdr) 53 | if err2 != nil { 54 | t.Error(err2) 55 | } 56 | if !bytes.Equal(norm, parsed) { 57 | t.Errorf("round-trip fails\norigin:|%s|\nnormal:|%x|\nas jdr:|%s|\nparsed:|%x|\n", string(jdr1), norm, string(jdr), parsed) 58 | } 59 | }) 60 | } 61 | 62 | /* 63 | func FuzzRDX(f *testing.F) { 64 | f.Add([]byte(L0(I0(1), S0("abc"), T0("def"), F0(1.23)))) 65 | f.Fuzz(func(t *testing.T, stream []byte) { 66 | norm, err := Normalize(stream) 67 | if err != nil { 68 | return 69 | } 70 | jdr := RenderJDR(norm, 0) 71 | parsed, err2 := ParseJDR(jdr) 72 | if err2 != nil { 73 | t.Error(err2) 74 | } 75 | if !bytes.Equal(norm, parsed) { 76 | t.Errorf("round-trip fails: %s", string(jdr)) 77 | } 78 | }) 79 | } 80 | */ 81 | -------------------------------------------------------------------------------- /test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | var Reset = "\033[0m" 9 | var Red = "\033[31m" 10 | var Green = "\033[32m" 11 | var Yellow = "\033[33m" 12 | var Blue = "\033[34m" 13 | var Magenta = "\033[35m" 14 | var Cyan = "\033[36m" 15 | var Gray = "\033[37m" 16 | var White = "\033[97m" 17 | 18 | type Tester func(rdx []byte) error 19 | 20 | func ProcessTestFile(path string, tester Tester) (err error) { 21 | // FIXME closing quotes 22 | jdr, err := os.ReadFile(path) 23 | if err != nil { 24 | return err 25 | } 26 | rdx, err := ParseJDR(jdr) 27 | if err != nil { 28 | return err 29 | } 30 | rest := rdx 31 | state := 0 32 | start := len(rest) 33 | finish := len(rest) 34 | var cases [][]byte 35 | var legend []byte 36 | var leg []byte 37 | var legends []string 38 | for len(rest) > 0 && err == nil { 39 | var lit byte 40 | var val []byte 41 | pre := len(rest) 42 | lit, _, val, rest, err = ReadRDX(rest) 43 | switch state { 44 | case 0: 45 | if lit == LitString && len(val) == 0 { 46 | state++ 47 | finish = pre 48 | } 49 | case 1: 50 | if lit == LitString { 51 | state++ 52 | leg = val 53 | } else { 54 | state = 0 55 | } 56 | case 2: 57 | if lit == LitString && len(val) == 0 { 58 | if finish < start { 59 | cases = append(cases, rdx[len(rdx)-start:len(rdx)-finish]) 60 | legends = append(legends, string(legend)) 61 | } 62 | start = len(rest) 63 | finish = start 64 | legend = leg 65 | } 66 | state = 0 67 | } 68 | } 69 | finish = 0 70 | if finish < start { 71 | cases = append(cases, rdx[len(rdx)-start:len(rdx)-finish]) 72 | legends = append(legends, string(legend)) 73 | } 74 | 75 | for n, c := range cases { 76 | err = tester(c) 77 | fmt.Print(legends[n]) 78 | if err != nil { 79 | pj, _ := WriteAllJDR(nil, c, 0) 80 | fmt.Printf(Red+"Test case %d fails: %s\n"+Gray+" %s"+Reset+"\n", n, err.Error(), pj) 81 | } 82 | } 83 | 84 | return 85 | } 86 | -------------------------------------------------------------------------------- /y.E.md: -------------------------------------------------------------------------------- 1 | `` ` 2 | # Eulerian containers (sets, maps) merge 3 | 4 | The merge rules for an Eulerian set are rather straightforward. 5 | 6 | Versions of the same set get merged, element by element. 7 | ``` 8 | {2:(5 6)}, 9 | {1:(2 33 4)}, 10 | {1:(2 33)}, 11 | ~, 12 | {1:(2 33 4),2:(5 6)}, 13 | 14 | {@alices-A 1 2 4} 15 | {@alices-A 1 3 5} 16 | ~ 17 | {@alices-A 1 2 3 4 5} 18 | ``` 19 | That applies to no-id sets as well: they count as the same set. 20 | ``` 21 | {1}, 22 | {1}, 23 | ~, 24 | {1}, 25 | 26 | ``` 27 | ...entries merge... 28 | ``` 29 | {1 3}, 30 | {4 5}, 31 | ~, 32 | {1,3,4,5}, 33 | 34 | {4 5}, 35 | {1 3}, 36 | ~, 37 | {1,3,4,5}, 38 | 39 | ``` 40 | ...tuple entries merge... 41 | ``` 42 | {1:2}, 43 | {3:4}, 44 | ~, 45 | {1:2,3:4}, 46 | 47 | {1:2, 3:4}, 48 | {1:1, 3:5, 4:5}, 49 | ~, 50 | {1:2, 3:5, 4:5}, 51 | 52 | ``` 53 | ...nested sets merge... 54 | ``` 55 | {1:2, 3:{4,5}}, 56 | {1:2, 3:6, 7:8}, 57 | ~, 58 | {1:2, 3:{4,5}, 7:8}, 59 | 60 | ``` 61 | The rules apply recursively. When we merge two versions of a set, 62 | versions of its elements also get merged. 63 | ``` 64 | {1:2, 3:{4,5}}, 65 | {1:2, 3:{4:10}, 7:8}, 66 | ~, 67 | {1:2, 3:{4:10,5}, 7:8}, 68 | 69 | ``` 70 | The above rules applied recursively again: 71 | ``` 72 | {}, 73 | {{}}, 74 | {{{}}}, 75 | ~, 76 | {{{}}}, 77 | 78 | {1:(2 3)}, 79 | {1:(2 3 44)}, 80 | {2:(5 6)}, 81 | ~, 82 | {1:(2,3,44),2:(5,6)}, 83 | 84 | {1:(2 3)}, 85 | {1:(2 3 4), 22:(5 6)}, 86 | ~, 87 | {1:(2 3 4),22:(5 6)}, 88 | 89 | {2:(5 6)}, 90 | {1:(2 33 4)}, 91 | {1:(2 33)}, 92 | ~, 93 | {1:(2 33 4),2:(5 6)}, 94 | 95 | {2:(5 6)}, 96 | {1:(2 3)}, 97 | {1:(2 3 444)}, 98 | {1:(2 3 444)}, 99 | {2:(5 6)}, 100 | {1:(2 3)}, 101 | ~, 102 | {1:(2 3 444),2:(5 6)}, 103 | 104 | ``` 105 | Different sets (different ids) never get merged; it is either one or another. 106 | ``` 107 | {@alices-A0 1 2 3} 108 | {@bobs-B0 4 5} 109 | ~ 110 | {@bobs-B0 4 5} 111 | 112 | ``` 113 | In the example above, the bigger id wins. 114 | In case we want to replace one set by another, we change the revision 115 | ``` 116 | ({@bobs-B 4 5}) 117 | (@20 {@alices-A 1 2 3}) 118 | ~ 119 | (@20 {@alices-A 1 2 3}) 120 | 121 | ``` 122 | Revision-envelopes with odd numbers count as tombstones. 123 | So, set deletion looks like: (TODO maybe skip the contents of a deleted E?) 124 | ``` 125 | {@alices-A 1 2 3} 126 | {@alices-B} 127 | ~ 128 | {@alices-B 1 2 3} 129 | 130 | ``` 131 | Correspondingly, here is undeletion: 132 | ``` 133 | (@1 {@alices-A 1 2 3}) 134 | (@2 {@alices-A 1 2 3}) 135 | ~ 136 | (@2 {@alices-A 1 2 3}) 137 | 138 | ``` 139 | Again, enveloping is only relevant when our data model is not 140 | a clean JSON-like tree, but we have to include arbitrary objects 141 | by reference. If you want to keep things simple, use trees. 142 | `` ` 143 | 144 | -------------------------------------------------------------------------------- /md_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func testValueOrder(rdx []byte) error { 13 | i := NewIter(rdx) 14 | n := 0 15 | for i.HasData() { 16 | prev := i 17 | n++ 18 | if !i.Read() { 19 | return i.Error() 20 | } 21 | z := CompareEuler(&prev, &i) 22 | if z > 0 { 23 | pj, _, _ := WriteJDR(nil, prev.Record(), 0) 24 | tj, _, _ := WriteJDR(nil, i.Record(), 0) 25 | fmt.Printf(Red+"Bad order %d and %d"+Reset+"\n\t|%s|\n\t|%s|", n, n+1, pj, tj) 26 | return errors.New("bad order") 27 | } 28 | } 29 | return nil 30 | } 31 | 32 | func TestValueOrder(t *testing.T) { 33 | err := ProcessTestFile("z.md", testValueOrder) 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | } 38 | 39 | var Tilde = []byte{'~'} 40 | 41 | func mergeTest(correct []byte, inputs [][]byte) (err error) { 42 | var merged []byte 43 | merged, err = Merge(nil, inputs) 44 | if err != nil { 45 | return err 46 | } 47 | if !bytes.Equal(correct, merged) { 48 | pj, _, _ := WriteJDR(nil, correct, 0) 49 | tj, _, _ := WriteJDR(nil, merged, 0) 50 | fmt.Printf(Red+"Bad merge\n"+Green+"\t|%s|\n"+Red+"\t|%s|\n"+Reset, pj, tj) 51 | return errors.New("bad merge") 52 | } 53 | return nil 54 | } 55 | 56 | func testMerge(rdx []byte) (err error) { 57 | i := NewIter(rdx) 58 | inputs := make([][]byte, 0, 32) 59 | for i.HasData() && err == nil { 60 | if !i.Read() { 61 | return i.Error() 62 | } 63 | if i.Lit() == LitTerm && bytes.Equal(i.Value(), Tilde) { 64 | if !i.Read() { 65 | err = i.Error() 66 | break 67 | } 68 | err = mergeTest(i.Record(), inputs) 69 | inputs = inputs[:0] 70 | } else { 71 | j := i 72 | inputs = append(inputs, j.Record()) 73 | } 74 | } 75 | return err 76 | } 77 | 78 | func TestOneSpecialCase(t *testing.T) { 79 | inputs := [][]byte{[]byte("(@1 {@Alice-1 \"one\":1})"), 80 | []byte("(@1 {@Alice-1 \"two\":2})")} 81 | rdxins := [][]byte{} 82 | for _, in := range inputs { 83 | rdx, err := ParseJDR(in) 84 | if err != nil { 85 | t.Fatal(err) 86 | } 87 | rdxins = append(rdxins, rdx) 88 | } 89 | out, err := Merge(nil, rdxins) 90 | if err != nil { 91 | t.Fatal(err) 92 | } 93 | jdr, _, err := WriteJDR(nil, out, 0) 94 | assert.Equal(t, "(@1 {@Alice-1 \"one\":1,\"two\":2})", string(jdr)) 95 | } 96 | 97 | func TestFirstMerge(t *testing.T) { 98 | err := ProcessTestFile("y.FIRST.md", testMerge) 99 | if err != nil { 100 | t.Fatal(err) 101 | } 102 | } 103 | 104 | func TestTupleMerge(t *testing.T) { 105 | err := ProcessTestFile("y.P.md", testMerge) 106 | if err != nil { 107 | t.Fatal(err) 108 | } 109 | } 110 | 111 | func TestLinearMerge(t *testing.T) { 112 | err := ProcessTestFile("y.L.md", testMerge) 113 | if err != nil { 114 | t.Fatal(err) 115 | } 116 | } 117 | 118 | func TestEulerMerge(t *testing.T) { 119 | err := ProcessTestFile("y.E.md", testMerge) 120 | if err != nil { 121 | t.Fatal(err) 122 | } 123 | } 124 | 125 | func TestMultixMerge(t *testing.T) { 126 | err := ProcessTestFile("y.X.md", testMerge) 127 | if err != nil { 128 | t.Fatal(err) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /y.P.md: -------------------------------------------------------------------------------- 1 | `` ` 2 | # RDX tuple merge rules 3 | 4 | Tuples are very important in RDX as they are part primitives 5 | part collections and, essentially, glue everything together. 6 | 7 | The merge rules are simple... if you can learn rules of poker, 8 | you can learn these rules too. 9 | 10 | First of all, an empty tuple is the RDX value for "nothing". 11 | As such, it can act as a tombstone or as a "null". 12 | ``` 13 | 1 14 | 2 15 | ~ 16 | 2 17 | 18 | (() ()) 19 | ():() 20 | ~ 21 | (() ()) 22 | 23 | 1:2:3 24 | 1:():3 25 | ~ 26 | 1:2:3, 27 | 28 | ``` 29 | Any element, including a tuple, can have a revision id attached. 30 | Odd revisions count as tombstones (the element is deleted). 31 | No explicit revision means the revision is 0. 32 | 33 | A revision can be a sequence number (one int, preferred) or 34 | a logical stamp consisting of two ints: *src* and *seq*, e.g. 35 | Alice-1b2. Stamps are rendered in RON Base64 so Alice and 36 | Bob are ints like 1 or 2. 37 | 38 | In the colon notation, the tuple revision is zero. 39 | ``` 40 | 1@2 41 | 2:3 42 | ~ 43 | 1@2 44 | 45 | 1:2:4 46 | 1@2:2:3 47 | ~ 48 | 1@2:2:4 49 | 50 | ``` 51 | The *id* an RDX element has can be seen as a revision id but 52 | also as an object id. The distinction here is rather fluid, 53 | very much like the distinction between time and space in 54 | relativistic physics. If we want to, we can always merge 55 | two objects exactly the same way we merge two versions of 56 | an object. That is the reason we address those broadly as 57 | "elements" not as "objects" or "versions". 58 | 59 | For FIRST elements that distinction is moot as we can not 60 | update them piecemeal. For PLEX collections, that is the 61 | difference between inclusion by value and inclusion by 62 | reference. Once a PLEX collection has a two-component id, 63 | it has an *identity* so it can be included by reference. 64 | When two distinct-identity PLEX elements are merged, it 65 | is one or the other, and the higher id wins. 66 | ``` 67 | (@Alice-1st "Tweedledum" rattle:spoiled) 68 | (@Alice-2nd "Tweedledee") 69 | ~ 70 | (@Alice-2nd "Tweedledee") 71 | 72 | ``` 73 | A tuple of one element, algebraically speaking, is the element 74 | itself. Within RDX, a tuple of one is called an *envelope*. 75 | Within collections, it orders exactly like its contained element 76 | and its only purpose is to revision-stamp that element. That is 77 | only *necessary* in the case if that element has an identity. 78 | Then, the revision is put on the envelope, not on the element 79 | itself, as changes to the outer collection do not affect such 80 | an element; it "lives" at a different address. 81 | 82 | ``` 83 | 1:2:3 84 | (@bob-2 1) 85 | ~ 86 | (@bob-2 1) 87 | 88 | 1:2:3 89 | (1) 90 | (@alice-2) 91 | ~ 92 | (@alice-2) 93 | 94 | {@Alice-seq name: "Alice"} 95 | (@sez {@Alice-seq}) 96 | ~ 97 | (@sez {@Alice-seq}) 98 | 99 | {@Alice-1st name: "Tweedledum"} 100 | (@1st {@Alice-1st}) 101 | (@2nd {@Alice-2nd name: "Tweedledee"}) 102 | ~ 103 | (@2nd {@Alice-2nd name: "Tweedledee"}) 104 | 105 | ``` 106 | Revision 0 (no revision, no envelope or no stamp on an envelope) 107 | is the original "intact" element. Odd revision numbers are put 108 | on "tombstones", i.e. placeholders for deleted elements. 109 | Even revision numbers express undeleted elements. 110 | ``` 111 | "string to be deleted" 112 | (@1 "string to be deleted") 113 | ~ 114 | (@1 "string to be deleted") 115 | 116 | (@1 "string to be undeleted") 117 | (@2 "string to be undeleted") 118 | ~ 119 | (@2 "string to be undeleted") 120 | 121 | ``` 122 | The way tuples themselves are merged depends on their id. 123 | Same id: recursive merge of the contents. 124 | Different id: higher id wins unilaterally. 125 | 126 | This rule applies to other PLEX elements as well. 127 | 128 | So, for two distinct tuples, higher id wins: 129 | ``` 130 | (@Alice-1st name:"Tweedledum" rattle:spoiled) 131 | (@Alice-2nd name:"Tweedledee") 132 | ~ 133 | (@Alice-2nd name:"Tweedledee") 134 | 135 | 1:2:3 136 | 3:2:1 137 | ~ 138 | 3:2:3 139 | ``` 140 | For two *variants* of a PLEX element (same id), nested elements 141 | will be merged, recursively. When merging specifically a tuple, 142 | elements in the same position get merged. 143 | ``` 144 | (@1 {@Alice-1 "one":1 } ) 145 | (@1 {@Alice-1 "two":2 } ) 146 | ~ 147 | (@1 {@Alice-1 "one":1 "two":2 } ) 148 | 149 | 1:1:3 150 | 1:2:1 151 | ~ 152 | 1:2:3 153 | 154 | 1:2:3 155 | 1:4:5 156 | ~ 157 | 1:4:5 158 | 159 | ``` 160 | ...now recursively: 161 | ``` 162 | 1:(2,3) 163 | 1:(1,4) 164 | 1:(2,5) 165 | ~ 166 | 1:(2,5) 167 | 168 | ``` 169 | For two different tuples (different id), the contents will not 170 | be merged: newer id just wins. Again, we may understand the id 171 | as an object id or as a version id, that does not matter. 172 | ``` 173 | ( one:1 ) 174 | (@1 two:2 ) 175 | three:3 176 | (@bob-2 four:4) 177 | ~ 178 | (@bob-2 four:4) 179 | 180 | (@1 (@Alice-3 1 2 3)) 181 | (@3 (@carol-4 four five)) 182 | ~ 183 | (@3 (@carol-4 four five)) 184 | 185 | (@carol-1 one two) 186 | (@2 (@Alice-3 1 2 3)) 187 | ~ 188 | (@2 (@Alice-3 1 2 3)) 189 | 190 | 1@Bob-1:2 191 | 1@Bob-1:2:3 192 | 1@Bob-1:4 193 | ~ 194 | 1@Bob-1:4:3 195 | 196 | ``` 197 | The id is attached to the opening bracket. 198 | ``` 199 | 200 | (@Alice-1 1 2) 201 | (@Alice-2 1 2 3) 202 | (@Alice-1 1 4) 203 | ~ 204 | (@Alice-2 1 4 3) 205 | 206 | 1 @1 207 | (@2 2, 2) 208 | ~ 209 | (@2 2, 2) 210 | 211 | (@1 1, 2) 212 | 2@2 213 | ~ 214 | 2@2 215 | 216 | `` 217 | -------------------------------------------------------------------------------- /tlv.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | ) 7 | 8 | var ( 9 | ErrIncomplete = errors.New("incomplete data") 10 | ErrBadRecord = errors.New("bad TLV record format") 11 | ErrBadHeader = errors.New("bad BRIX file header") 12 | ErrBadNesting = errors.New("bad TLV nesting") 13 | ) 14 | 15 | const CaseBit uint8 = 'a' - 'A' 16 | 17 | const MaxRecLen = 0x7fffffff 18 | 19 | func ReadTLV(data []byte) (lit byte, value, rest []byte, err error) { 20 | if len(data) == 0 { 21 | return 0, nil, nil, nil 22 | } 23 | dlit := data[0] 24 | if dlit >= 'a' && dlit <= 'z' { // short 25 | lit = dlit - CaseBit 26 | if len(data) < 2 || len(data) < 2+int(data[1]) { 27 | rest = data 28 | err = ErrIncomplete 29 | return 30 | } 31 | e := int(data[1]) + 2 32 | value = data[2:e] 33 | rest = data[e:] 34 | } else if dlit >= 'A' && dlit <= 'Z' { // long 35 | if len(data) < 5 { 36 | err = ErrIncomplete 37 | return 38 | } 39 | bl := binary.LittleEndian.Uint32(data[1:5]) 40 | if bl > MaxRecLen || int(bl) > len(data)+5 { 41 | if bl > MaxRecLen { 42 | err = ErrBadRecord 43 | } else { 44 | err = ErrIncomplete 45 | } 46 | return 47 | } 48 | lit = dlit 49 | value = data[5 : 5+bl] 50 | rest = data[5+bl:] 51 | } else { 52 | err = ErrBadRecord 53 | } 54 | return 55 | } 56 | 57 | func ReadTLKV(data []byte) (lit byte, key, value, rest []byte, err error) { 58 | lit, value, rest, err = ReadTLV(data) 59 | if err == nil && len(value) > 0 { 60 | if len(value) < int(value[0])+1 { 61 | err = ErrBadRecord 62 | } else { 63 | key = value[1 : 1+value[0]] 64 | value = value[1+value[0]:] 65 | } 66 | } 67 | return 68 | } 69 | 70 | func WriteTLV(data []byte, lit byte, value []byte) []byte { 71 | ret := data 72 | if len(value) > 0xff { 73 | ret = append(ret, lit) 74 | ret = binary.LittleEndian.AppendUint32(ret, uint32(len(value))) 75 | } else { 76 | ret = append(ret, lit|CaseBit) 77 | ret = append(ret, byte(len(value))) 78 | } 79 | ret = append(ret, value...) 80 | return ret 81 | } 82 | 83 | func WriteTLKV(data []byte, lit byte, key, value []byte) []byte { 84 | ret := data 85 | if len(key) > 0xff { // todo 86 | key = key[:0xff] 87 | } 88 | l := len(key) + len(value) + 1 89 | if l > 0xff { 90 | ret = append(ret, lit) 91 | ret = binary.LittleEndian.AppendUint32(ret, uint32(l)) 92 | } else { 93 | ret = append(ret, lit|CaseBit) 94 | ret = append(ret, byte(l)) 95 | } 96 | ret = append(ret, byte(len(key))) 97 | ret = append(ret, key...) 98 | ret = append(ret, value...) 99 | return ret 100 | } 101 | 102 | type Mark struct { 103 | Start int // start pos of the element 104 | LastBreak int // or , ; 105 | LastElement int // start pos of the last complete child element 106 | Lit byte 107 | Pre byte // FIRST PLEX ,;: 0 108 | } 109 | 110 | type Marks []Mark 111 | 112 | func (stack Marks) Len() int { 113 | return len(stack) 114 | } 115 | 116 | func (stack Marks) TopLit() byte { 117 | if len(stack) == 0 { 118 | return 0 119 | } 120 | return stack[len(stack)-1].Lit 121 | } 122 | 123 | func spliceTLV(data []byte, lit byte, pos int) []byte { 124 | l := len(data) - pos + 1 125 | if l <= 0xff { 126 | data = append(data, 0, 0, 0) 127 | copy(data[pos+3:], data[pos:]) 128 | data[pos] = lit | CaseBit 129 | data[pos+1] = byte(l) 130 | data[pos+2] = 0 131 | } else { 132 | data = append(data, 0, 0, 0, 0, 0, 0) 133 | copy(data[pos+6:], data[pos:]) 134 | data[pos] = lit &^ CaseBit 135 | binary.LittleEndian.PutUint32(data[pos+1:pos+5], uint32(l)) 136 | data[pos+5] = 0 137 | } 138 | return data 139 | } 140 | 141 | func OpenTLV(data []byte, lit byte, stack *Marks) []byte { 142 | if lit < 'A' || lit > 'Z' { 143 | panic("bad Lit") 144 | } 145 | *stack = append(*stack, Mark{Start: len(data), Lit: lit}) 146 | data = append(data, lit, 0, 0, 0, 0) 147 | return data 148 | } 149 | 150 | func OpenShortTLV(data []byte, lit byte, stack *Marks) []byte { 151 | if lit < 'A' || lit > 'Z' { 152 | panic("bad Lit") 153 | } 154 | lit |= CaseBit 155 | *stack = append(*stack, Mark{Start: len(data), Lit: lit}) 156 | data = append(data, lit, 0) 157 | return data 158 | } 159 | 160 | func upper(lit byte) byte { 161 | return lit &^ CaseBit 162 | } 163 | 164 | func CancelTLV(data []byte, lit byte, stack *Marks) (ret []byte, err error) { 165 | if len(*stack) == 0 { 166 | return nil, ErrBadNesting 167 | } 168 | nl := len(*stack) - 1 169 | last := (*stack)[nl] 170 | if upper(last.Lit) != lit { 171 | return nil, ErrBadNesting 172 | } 173 | ret = data[:last.Start] 174 | *stack = (*stack)[:nl] 175 | return 176 | } 177 | 178 | func CloseTLV(data []byte, lit byte, stack *Marks) (ret []byte, err error) { 179 | if len(*stack) == 0 { 180 | return nil, ErrBadNesting 181 | } 182 | nl := len(*stack) - 1 183 | last := (*stack)[nl] 184 | *stack = (*stack)[:nl] 185 | if upper(last.Lit) != lit || last.Start+2 > len(data) || data[last.Start]&^CaseBit != lit { 186 | return nil, ErrBadNesting 187 | } 188 | fact := len(data) - last.Start 189 | if 0 == (data[last.Start] & CaseBit) { // A 190 | fact -= 5 191 | if fact < 0 { 192 | return nil, ErrBadNesting 193 | } 194 | if fact < 0x100 { 195 | copy(data[last.Start+2:len(data)-3], data[last.Start+5:len(data)]) 196 | data = data[:len(data)-3] 197 | data[last.Start] |= CaseBit 198 | } 199 | } else { // a 200 | fact -= 2 201 | if fact < 0 { 202 | return nil, ErrBadNesting 203 | } 204 | if fact >= 0x100 { 205 | l := len(data) 206 | data = append(data, 0, 0, 0) 207 | copy(data[last.Start+5:len(data)], data[last.Start+2:l]) 208 | data[last.Start] &= ^CaseBit 209 | } 210 | } 211 | if fact < 0x100 { 212 | data[last.Start+1] = byte(fact) 213 | } else { 214 | binary.LittleEndian.PutUint32(data[last.Start+1:], uint32(fact)) 215 | } 216 | return data, nil 217 | } 218 | -------------------------------------------------------------------------------- /itheap_test.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestIntersectNext(t *testing.T) { 9 | // Helper function to create a multi-element stream 10 | multiStream := func(elements ...[]byte) []byte { 11 | var result []byte 12 | for _, elem := range elements { 13 | result = append(result, elem...) 14 | } 15 | return result 16 | } 17 | 18 | tests := []struct { 19 | name string 20 | inputs [][]byte 21 | totalInputs int 22 | expected [][]byte // Expected sequence of outputs from IntersectNext calls 23 | expectError bool 24 | }{ 25 | { 26 | name: "simple intersection - all have common element", 27 | inputs: [][]byte{ 28 | multiStream(I0(1), I0(2), I0(3)), 29 | multiStream(I0(2), I0(4), I0(5)), 30 | multiStream(I0(2), I0(6), I0(7)), 31 | }, 32 | totalInputs: 3, 33 | expected: [][]byte{I0(2)}, // Only 2 appears in all inputs 34 | expectError: false, 35 | }, 36 | { 37 | name: "no intersection", 38 | inputs: [][]byte{ 39 | multiStream(I0(1), I0(2)), 40 | multiStream(I0(3), I0(4)), 41 | multiStream(I0(5), I0(6)), 42 | }, 43 | totalInputs: 3, 44 | expected: [][]byte{}, // No common elements 45 | expectError: false, 46 | }, 47 | { 48 | name: "multiple intersections", 49 | inputs: [][]byte{ 50 | multiStream(I0(1), I0(2), I0(3), I0(4)), 51 | multiStream(I0(1), I0(3), I0(5), I0(7)), 52 | multiStream(I0(1), I0(3), I0(6), I0(8)), 53 | }, 54 | totalInputs: 3, 55 | expected: [][]byte{I0(1), I0(3)}, // 1 and 3 appear in all inputs 56 | expectError: false, 57 | }, 58 | { 59 | name: "single element intersection", 60 | inputs: [][]byte{ 61 | I0(42), 62 | I0(42), 63 | I0(42), 64 | }, 65 | totalInputs: 3, 66 | expected: [][]byte{I0(42)}, 67 | expectError: false, 68 | }, 69 | { 70 | name: "different types - no intersection", 71 | inputs: [][]byte{ 72 | I0(1), 73 | S0("1"), 74 | F0(1.0), 75 | }, 76 | totalInputs: 3, 77 | expected: [][]byte{}, // Different types, no intersection 78 | expectError: false, 79 | }, 80 | { 81 | name: "string intersection", 82 | inputs: [][]byte{ 83 | multiStream(S0("apple"), S0("banana"), S0("cherry")), 84 | multiStream(S0("banana"), S0("date"), S0("elderberry")), 85 | multiStream(S0("banana"), S0("fig"), S0("grape")), 86 | }, 87 | totalInputs: 3, 88 | expected: [][]byte{S0("banana")}, 89 | expectError: false, 90 | }, 91 | { 92 | name: "empty inputs", 93 | inputs: [][]byte{ 94 | {}, 95 | {}, 96 | {}, 97 | }, 98 | totalInputs: 3, 99 | expected: [][]byte{}, // No elements to intersect 100 | expectError: false, 101 | }, 102 | { 103 | name: "one empty input", 104 | inputs: [][]byte{ 105 | multiStream(I0(1), I0(2)), 106 | {}, // Empty input means no intersection possible 107 | multiStream(I0(1), I0(3)), 108 | }, 109 | totalInputs: 3, 110 | expected: [][]byte{}, // Empty input = no intersection 111 | expectError: false, 112 | }, 113 | { 114 | name: "tuple intersection", 115 | inputs: [][]byte{ 116 | multiStream(P0(S0("key"), I0(1)), P0(S0("key2"), I0(2))), 117 | multiStream(P0(S0("key"), I0(1)), P0(S0("key2"), I0(3))), 118 | multiStream(P0(S0("key"), I0(1)), P0(S0("key2"), I0(4))), 119 | }, 120 | totalInputs: 3, 121 | expected: [][]byte{P0(S0("key"), I0(1)), P0(S0("key2"), I0(2))}, 122 | expectError: false, 123 | }, 124 | { 125 | name: "reference intersection", 126 | inputs: [][]byte{ 127 | multiStream(R0(ID{1, 1}), R0(ID{2, 2}), R0(ID{3, 3})), 128 | multiStream(R0(ID{2, 2}), R0(ID{4, 4})), 129 | multiStream(R0(ID{2, 2}), R0(ID{5, 5})), 130 | }, 131 | totalInputs: 3, 132 | expected: [][]byte{R0(ID{2, 2})}, 133 | expectError: false, 134 | }, 135 | { 136 | name: "complex intersection with mixed types", 137 | inputs: [][]byte{ 138 | multiStream(I0(10), S0("common"), I0(20)), 139 | multiStream(S0("common"), F0(3.14), I0(30)), 140 | multiStream(I0(40), S0("common"), T0("term")), 141 | }, 142 | totalInputs: 3, 143 | expected: [][]byte{S0("common")}, // Only string "common" appears in all 144 | expectError: false, 145 | }, 146 | { 147 | name: "early termination - first input exhausted", 148 | inputs: [][]byte{ 149 | I0(1), // Only one element 150 | multiStream(I0(1), I0(2), I0(3)), 151 | multiStream(I0(1), I0(4), I0(5)), 152 | }, 153 | totalInputs: 3, 154 | expected: [][]byte{I0(1)}, // After first element, first input is exhausted 155 | expectError: false, 156 | }, 157 | } 158 | 159 | for _, tt := range tests { 160 | t.Run(tt.name, func(t *testing.T) { 161 | // Create heap from inputs 162 | heap, err := Heapize(tt.inputs, CompareEuler) 163 | if err != nil { 164 | if !tt.expectError { 165 | t.Fatalf("Heapize failed: %v", err) 166 | } 167 | return 168 | } 169 | 170 | var result Iter 171 | var outputs [][]byte 172 | totalInputs := len(tt.inputs) 173 | 174 | // Keep calling IntersectNext until heap is exhausted or error 175 | for len(heap) > 0 && len(heap) == totalInputs && err == nil { 176 | result, err = heap.IntersectNext(CompareEuler) 177 | 178 | if result.HasData() { 179 | outputs = append(outputs, result.Record()) 180 | } 181 | } 182 | 183 | if tt.expectError && err == nil { 184 | t.Errorf("Expected error but got none") 185 | return 186 | } 187 | if !tt.expectError && err != nil { 188 | t.Errorf("Unexpected error: %v", err) 189 | return 190 | } 191 | 192 | // Check if we got the expected number of outputs 193 | if len(outputs) != len(tt.expected) { 194 | t.Errorf("Expected %d outputs, got %d", len(tt.expected), len(outputs)) 195 | return 196 | } 197 | 198 | // Check each output matches expected 199 | for i, expected := range tt.expected { 200 | if !bytes.Equal(outputs[i], expected) { 201 | t.Errorf("Output %d: expected %v, got %v", i, expected, outputs[i]) 202 | } 203 | } 204 | }) 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /zipint.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "encoding/binary" 5 | "math" 6 | "math/bits" 7 | ) 8 | 9 | const Bytes4 = 0xffffffff 10 | const Bytes2 = 0xffff 11 | const Bytes1 = 0xff 12 | 13 | func byteLen(n uint64) int { 14 | if n <= Bytes1 { 15 | if n == 0 { 16 | return 0 17 | } 18 | return 1 19 | } 20 | if n <= Bytes2 { 21 | return 2 22 | } 23 | if n <= Bytes4 { 24 | return 4 25 | } 26 | return 8 27 | } 28 | 29 | // ZipUint64Pair packs a pair of uint64 into a byte string. 30 | // The smaller the ints, the shorter the string TODO 4+3 etc 31 | func ZipUint64Pair(big, lil uint64) []byte { 32 | var ret = [16]byte{} 33 | pat := (byteLen(big) << 4) | byteLen(lil) 34 | switch pat { 35 | case 0x00: 36 | return ret[0:0] 37 | case 0x10: 38 | ret[0] = byte(big) 39 | return ret[0:1] 40 | case 0x01, 0x11: // 2 41 | ret[0] = byte(big) 42 | ret[1] = byte(lil) 43 | return ret[0:2] 44 | case 0x20, 0x21: 45 | binary.LittleEndian.PutUint16(ret[0:2], uint16(big)) 46 | ret[2] = byte(lil) 47 | return ret[0:3] 48 | case 0x02, 0x12, 0x22: 49 | binary.LittleEndian.PutUint16(ret[0:2], uint16(big)) 50 | binary.LittleEndian.PutUint16(ret[2:4], uint16(lil)) 51 | return ret[0:4] 52 | case 0x40, 0x41: 53 | binary.LittleEndian.PutUint32(ret[0:4], uint32(big)) 54 | ret[4] = byte(lil) 55 | return ret[0:5] 56 | case 0x42: 57 | binary.LittleEndian.PutUint32(ret[0:4], uint32(big)) 58 | binary.LittleEndian.PutUint16(ret[4:6], uint16(lil)) 59 | return ret[0:6] 60 | case 0x04, 0x14, 0x24, 0x44: 61 | binary.LittleEndian.PutUint32(ret[0:4], uint32(big)) 62 | binary.LittleEndian.PutUint32(ret[4:8], uint32(lil)) 63 | return ret[0:8] 64 | case 0x80, 0x81: 65 | binary.LittleEndian.PutUint64(ret[0:8], big) 66 | ret[8] = byte(lil) 67 | return ret[0:9] 68 | case 0x82: 69 | binary.LittleEndian.PutUint64(ret[0:8], big) 70 | binary.LittleEndian.PutUint16(ret[8:10], uint16(lil)) 71 | return ret[0:10] 72 | case 0x84: 73 | binary.LittleEndian.PutUint64(ret[0:8], big) 74 | binary.LittleEndian.PutUint32(ret[8:12], uint32(lil)) 75 | return ret[0:12] 76 | case 0x08, 0x18, 0x28: 77 | binary.LittleEndian.PutUint16(ret[0:2], uint16(big)) 78 | binary.LittleEndian.PutUint64(ret[3:11], lil) 79 | return ret[0:11] 80 | case 0x48: 81 | binary.LittleEndian.PutUint32(ret[0:4], uint32(big)) 82 | binary.LittleEndian.PutUint64(ret[5:13], lil) 83 | return ret[0:13] 84 | case 0x88: 85 | binary.LittleEndian.PutUint64(ret[0:8], big) 86 | binary.LittleEndian.PutUint64(ret[8:16], lil) 87 | return ret[0:16] 88 | } 89 | return ret[:] 90 | } 91 | 92 | func UnzipUint64Pair(buf []byte) (big, lil uint64) { 93 | switch len(buf) { 94 | case 0: 95 | return 0, 0 96 | case 1: 97 | return uint64(buf[0]), 0 98 | case 2: 99 | return uint64(buf[0]), uint64(buf[1]) 100 | case 3: 101 | big = uint64(binary.LittleEndian.Uint16(buf[0:2])) 102 | lil = uint64(buf[2]) 103 | case 4: 104 | big = uint64(binary.LittleEndian.Uint16(buf[0:2])) 105 | lil = uint64(binary.LittleEndian.Uint16(buf[2:4])) 106 | case 5: 107 | big = uint64(binary.LittleEndian.Uint32(buf[0:4])) 108 | lil = uint64(buf[4]) 109 | case 6: 110 | big = uint64(binary.LittleEndian.Uint32(buf[0:4])) 111 | lil = uint64(binary.LittleEndian.Uint16(buf[4:6])) 112 | case 8: 113 | big = uint64(binary.LittleEndian.Uint32(buf[0:4])) 114 | lil = uint64(binary.LittleEndian.Uint32(buf[4:8])) 115 | case 9: 116 | big = uint64(binary.LittleEndian.Uint64(buf[0:8])) 117 | lil = uint64(buf[8]) 118 | case 10: 119 | big = binary.LittleEndian.Uint64(buf[0:8]) 120 | lil = uint64(binary.LittleEndian.Uint16(buf[8:10])) 121 | case 11: 122 | big = uint64(binary.LittleEndian.Uint16(buf[0:2])) 123 | lil = uint64(binary.LittleEndian.Uint64(buf[3:11])) 124 | case 12: 125 | big = binary.LittleEndian.Uint64(buf[0:8]) 126 | lil = uint64(binary.LittleEndian.Uint32(buf[8:12])) 127 | case 13: 128 | big = uint64(binary.LittleEndian.Uint32(buf[0:4])) 129 | lil = uint64(binary.LittleEndian.Uint64(buf[5:13])) 130 | case 16: 131 | big = binary.LittleEndian.Uint64(buf[0:8]) 132 | lil = binary.LittleEndian.Uint64(buf[8:16]) 133 | default: // what about 7 11 13 14 15 ? 134 | // error! 135 | } 136 | return 137 | } 138 | 139 | func UnzipUint32Pair(buf []byte) (big, lil uint32) { 140 | switch len(buf) { 141 | case 0: 142 | return 0, 0 143 | case 1: 144 | return uint32(buf[0]), 0 145 | case 2: 146 | return uint32(buf[0]), uint32(buf[1]) 147 | case 3: 148 | big = uint32(binary.LittleEndian.Uint16(buf[0:2])) 149 | lil = uint32(buf[2]) 150 | case 4: 151 | big = uint32(binary.LittleEndian.Uint16(buf[0:2])) 152 | lil = uint32(binary.LittleEndian.Uint16(buf[2:4])) 153 | case 5: 154 | big = uint32(binary.LittleEndian.Uint32(buf[0:4])) 155 | lil = uint32(buf[4]) 156 | case 6: 157 | big = uint32(binary.LittleEndian.Uint32(buf[0:4])) 158 | lil = uint32(binary.LittleEndian.Uint16(buf[4:6])) 159 | case 8: 160 | big = uint32(binary.LittleEndian.Uint32(buf[0:4])) 161 | lil = uint32(binary.LittleEndian.Uint32(buf[4:8])) 162 | default: 163 | // error! 164 | } 165 | return 166 | } 167 | 168 | func Uint32Pair(a, b uint32) (x uint64) { 169 | return uint64(a) | (uint64(b) << 32) 170 | } 171 | 172 | func Uint32Unpair(x uint64) (a, b uint32) { 173 | return uint32(x), uint32(x >> 32) 174 | } 175 | 176 | // ZipUint64 packs uint64 into a shortest possible byte string 177 | func ZipUint64(v uint64) []byte { 178 | buf := [8]byte{} 179 | i := 0 180 | for v > 0 { 181 | buf[i] = uint8(v) 182 | v >>= 8 183 | i++ 184 | } 185 | return buf[0:i] 186 | } 187 | 188 | func UnzipUint64(zip []byte) (v uint64) { 189 | for i := len(zip) - 1; i >= 0; i-- { 190 | v <<= 8 191 | v |= uint64(zip[i]) 192 | } 193 | return 194 | } 195 | 196 | func ZigZagInt64(i int64) uint64 { 197 | return uint64(i*2) ^ uint64(i>>63) 198 | } 199 | 200 | func ZagZigUint64(u uint64) int64 { 201 | half := u >> 1 202 | mask := -(u & 1) 203 | return int64(half ^ mask) 204 | } 205 | 206 | func ZipZagInt64(i int64) []byte { 207 | return ZipUint64(ZigZagInt64(i)) 208 | } 209 | 210 | func ZipInt64(v int64) []byte { 211 | return ZipUint64(ZigZagInt64(v)) 212 | } 213 | 214 | func UnzipInt64(zip []byte) int64 { 215 | return ZagZigUint64(UnzipUint64(zip)) 216 | } 217 | func ZipFloat64(f float64) []byte { 218 | fb := math.Float64bits(float64(f)) 219 | b := bits.Reverse64(fb) 220 | return ZipUint64(b) 221 | } 222 | 223 | func UnzipFloat64(zip []byte) float64 { 224 | b := UnzipUint64(zip) 225 | return math.Float64frombits(bits.Reverse64(b)) 226 | } 227 | 228 | func ZipInteger(v Integer) []byte { 229 | return ZipInt64(int64(v)) 230 | } 231 | 232 | func UnzipInteger(zip []byte) Integer { 233 | return Integer(UnzipInt64(zip)) 234 | } 235 | 236 | func ZipFloat(f Float) []byte { 237 | return ZipFloat64(float64(f)) 238 | } 239 | 240 | func UnzipFloat(zip []byte) Float { 241 | return Float(UnzipFloat64(zip)) 242 | } 243 | -------------------------------------------------------------------------------- /id.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | "time" 7 | ) 8 | 9 | type ID struct { 10 | Src uint64 11 | Seq uint64 12 | } 13 | 14 | var ZeroID = ID{} 15 | var MaxID = ID{math.MaxUint64, math.MaxUint64} 16 | 17 | func ZipID(id ID) []byte { 18 | return ZipUint64Pair(id.Seq, id.Src) 19 | } 20 | 21 | func UnzipID(b []byte) (id ID) { 22 | id.Seq, id.Src = UnzipUint64Pair(b) 23 | return 24 | } 25 | 26 | func ParseRON64(ron []byte) (val uint64, rest []byte) { 27 | rest = ron 28 | l := 0 29 | for len(rest) > 0 && l <= 10 { 30 | n := RON64REV[rest[0]] 31 | if n == 0xff { 32 | break 33 | } 34 | val = (val << 6) | uint64(n) 35 | rest = rest[1:] 36 | l++ 37 | } 38 | return 39 | } 40 | 41 | func RON64String(u uint64) []byte { 42 | var ret [16]byte 43 | p := 16 44 | for { 45 | p-- 46 | ret[p] = RON64[u&63] 47 | u >>= 6 48 | if u == 0 { 49 | break 50 | } 51 | } 52 | return ret[p:16] 53 | } 54 | 55 | func (id ID) FormalString() []byte { 56 | var _ret [32]byte 57 | ret := _ret[:0] 58 | ret = append(ret, RON64String(id.Src)...) 59 | if (ret[len(ret)-1] | CaseBit) == 'e' { // todo nicer 60 | ret = _ret[:0] 61 | ret = append(ret, '0') 62 | if id.Src == 0xe || id.Src == 41 { 63 | ret = append(ret, '0') 64 | } 65 | ret = append(ret, RON64String(id.Src)...) 66 | } 67 | ret = append(ret, '-') 68 | ret = append(ret, RON64String(id.Seq)...) 69 | return ret 70 | } 71 | 72 | func (id ID) String() string { 73 | return string(id.RonString()) 74 | } 75 | 76 | func (id ID) RonString() []byte { 77 | if id.Src == 0 { 78 | return RON64String(id.Seq) 79 | } 80 | var _ret [32]byte 81 | ret := _ret[:0] 82 | ret = append(ret, RON64String(id.Src)...) 83 | ret = append(ret, '-') 84 | ret = append(ret, RON64String(id.Seq)...) 85 | return ret 86 | } 87 | 88 | const MaskNoRev = ^uint64(63) 89 | 90 | func (id ID) Eq(b ID) bool { 91 | return id.Src == b.Src && id.Seq == b.Seq 92 | } 93 | 94 | func (id ID) IsZero() bool { 95 | return id.Src == 0 && id.Seq == 0 96 | } 97 | 98 | func (id ID) IsBaseZero() bool { 99 | return (id.Seq & MaskNoRev) == 0 100 | } 101 | 102 | func (id ID) Base() ID { 103 | return ID{id.Src, id.Seq &^ 63} 104 | } 105 | 106 | func (id ID) Rev() uint64 { 107 | return id.Seq & 63 108 | } 109 | 110 | func (a ID) Less(b ID) bool { 111 | if a.Seq < b.Seq { 112 | return true 113 | } else if a.Seq > b.Seq { 114 | return false 115 | } else { 116 | return a.Src < b.Src 117 | } 118 | } 119 | 120 | var ErrBadIDSyntax = errors.New("bad id syntax") 121 | 122 | func NewID(txt []byte) (id ID, err error) { 123 | var rest []byte 124 | id.Src, rest = ParseRON64(txt) 125 | if (len(txt)-len(rest) > 11) || id.Src > Mask60bit { 126 | err = ErrBadIDSyntax 127 | } else if len(rest) == 0 { 128 | id.Seq, id.Src = id.Src, 0 129 | } else if rest[0] != '-' || len(rest) > 10+1 { 130 | err = ErrBadIDSyntax 131 | } else if id.Seq, rest = ParseRON64(rest[1:]); len(rest) > 0 || id.Seq > Mask60bit { 132 | err = ErrBadIDSyntax 133 | } 134 | return 135 | } 136 | 137 | func ParseID(txt []byte) (id ID, rest []byte) { 138 | id.Src, rest = ParseRON64(txt) 139 | if len(rest) > 0 && rest[0] == '-' { 140 | rest = rest[1:] 141 | id.Seq, rest = ParseRON64(rest) 142 | } else { 143 | id.Seq = id.Src 144 | id.Src = 0 145 | } 146 | return 147 | } 148 | 149 | func ParseIDString(txt string) (id ID) { 150 | id, _ = ParseID([]byte(txt)) 151 | return 152 | } 153 | 154 | func (a ID) RevCompare(b ID) int { 155 | if a.Seq < b.Seq { 156 | return Less 157 | } else if a.Seq > b.Seq { 158 | return Grtr 159 | } else if a.Src < b.Src { 160 | return Less 161 | } else if a.Src > b.Src { 162 | return Grtr 163 | } else { 164 | return Eq 165 | } 166 | } 167 | 168 | func (a ID) LCompare(b ID) int { 169 | aron := Ron60(a.Seq) 170 | bron := Ron60(b.Seq) 171 | if aron.Less(bron) { 172 | return Less 173 | } else if bron.Less(aron) { 174 | return Grtr 175 | } else if a.Src < b.Src { 176 | return Less 177 | } else if a.Src > b.Src { 178 | return Grtr 179 | } else { 180 | return Eq 181 | } 182 | } 183 | 184 | func (a ID) Compare(b ID) int { 185 | aseq := a.Seq & MaskNoRev 186 | bseq := b.Seq & MaskNoRev 187 | if aseq < bseq { 188 | return Less 189 | } else if aseq > bseq { 190 | return Grtr 191 | } else if a.Src < b.Src { 192 | return Less 193 | } else if a.Src > b.Src { 194 | return Grtr 195 | } else { 196 | return Eq 197 | } 198 | } 199 | 200 | const IdRevBits = 6 201 | 202 | func (a ID) Stem() ID { 203 | return ID{a.Src, a.Seq & MaskNoRev} 204 | } 205 | 206 | func (a ID) Xor() uint64 { 207 | x := a.Src ^ (a.Seq >> IdRevBits) 208 | x ^= x >> 32 209 | x ^= x >> 16 210 | x ^= x >> 8 211 | x ^= x >> 4 212 | return x 213 | } 214 | 215 | func (a ID) IsLive() bool { 216 | return (a.Seq & 1) == 0 217 | } 218 | 219 | func (a ID) Removed() ID { 220 | return ID{a.Src, (a.Seq + 1) | 1} 221 | } 222 | 223 | const MaskNot1 = ^uint64(1) 224 | 225 | func (a ID) Recovered() ID { 226 | return ID{a.Src, (a.Seq + 2) & MaskNot1} 227 | } 228 | 229 | func (a ID) Replaced() ID { 230 | return ID{0, (a.Seq & MaskNoRev) + 64} 231 | } 232 | 233 | // 2588GWn000 234 | func Timestamp() (t uint64) { 235 | now := time.Now() 236 | y := uint64(now.Year() - 2000) 237 | t = t | ((y / 10) << (9 * 6)) 238 | t = t | ((y % 10) << (8 * 6)) 239 | t = t | (uint64(now.Month()) << (7 * 6)) 240 | t = t | (uint64(now.Day()) << (6 * 6)) 241 | t = t | (uint64(now.Hour()) << (5 * 6)) 242 | t = t | (uint64(now.Minute()) << (4 * 6)) 243 | t = t | (uint64(now.Second()) << (3 * 6)) 244 | t = t | (uint64(now.Nanosecond() >> 2)) 245 | return 246 | } 247 | 248 | const Mask60bit = (uint64(1) << 60) - 1 249 | 250 | func Revert64(x uint64) (y uint64) { 251 | x = x & Mask60bit 252 | shift := 60 253 | for x != 0 { 254 | y = (y << 6) | (x & 63) 255 | shift -= 6 256 | x >>= 6 257 | } 258 | y <<= shift 259 | return 260 | } 261 | 262 | const RON64 = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz~" 263 | 264 | var RON64REV = []byte{ 265 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 266 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 267 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 268 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 269 | 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xff, 0xff, 270 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10, 271 | 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 272 | 0x1d, 0x1e, 0x1f, 0x20, 0x21, 0x22, 0x23, 0xff, 0xff, 0xff, 0xff, 0x24, 273 | 0xff, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 274 | 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3a, 0x3b, 275 | 0x3c, 0x3d, 0x3e, 0xff, 0xff, 0xff, 0x3f, 0xff, 0xff, 0xff, 0xff, 0xff, 276 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 277 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 278 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 279 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 280 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 281 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 282 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 283 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 284 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 285 | 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 286 | 0xff, 0xff, 0xff, 0xff, 287 | } 288 | -------------------------------------------------------------------------------- /DISCONT.md: -------------------------------------------------------------------------------- 1 | # Discontinuity CRDT (DISCONT) 2 | 3 | DISCONT is a greatly simplified CRDT algorithm that 4 | fits into the constraints of an [LSM][l] merge operator. 5 | DISCONT replaces CausalTree/RGA for Linear RDX elements. 6 | 7 | The DISCONT algorithm is essentially merge sort using the 8 | order of element IDs. That makes it comparable to Figma's 9 | [ersatz-CRDT][f] in terms of simplicity. To make it an actual 10 | CRDT, DISCONT makes a little twist which is more 11 | like a roundhouse kick, see below. 12 | 13 | Implementation-wise, DISCONT is a [merge sort][m] using a heap of 14 | iterators. The technique is popular in the [LSM database world][c]. 15 | Although, if we would use any other merge sort implementation, 16 | that will not change much. 17 | When merging versions of the state and patches, DISCONT 18 | creates an iterator for each input, puts iterators in a heap 19 | and... iterates like a merge sort should do. Importantly, 20 | copies of elements get merged. Imagine two versions of an array; 21 | those would naturally have lots of the same elements. 22 | 23 | RDX has 128-bit element [IDs][i] consisting of a 64-bit sequence 24 | number `seq` and a 64-bit replica id `src`. Those are essentially 25 | Lamport timestamps. The array type (`L`) additionally subdivides the `seq` 26 | into 6 bit of "revision" and 64-6=58 bit "locator". The locator 27 | (mainly) defines the position of a member element within the 28 | array thus enabling the *insert* operation. The revision part 29 | allows element versioning thus enabling *overwrites* and *deletes*. 30 | 31 | Here is the Figma algorithm as the [authors][f] put it: 32 | 33 | > At a high level, an object’s position in its parent’s array 34 | > of children is represented as a fraction between 0 and 1 35 | > exclusive. The order of an object’s children is determined 36 | > by sorting them by their positions. You can insert an object 37 | > between two other objects by setting its position to the 38 | > average of the positions of the two other objects. 39 | 40 | So, by merge-sorting versions of an array in the order of labels 41 | while merging element copies we produce the new version of the array. 42 | 58 bits of `seq` is enough to run the Figma's algorithm for quite 43 | a long time before we encounter the situation of `IDleft+1 == IDright` 44 | that prevents any further inserts at that point. Broadly, if 45 | `IDright-IDleft < new_elem_count`, we cannot fit the inserted 46 | elements into the data structure. (Here and later, we assume all 47 | arithmetics is applied to locators; what is the value of `src` 48 | is irrelevant at this stage.) 49 | 50 | DISCONT resolves two key shortcomings of Figma's label-sorting 51 | approach: 52 | 53 | 1. the requirement of ID space being dense (between any two 54 | ID values we can find a new one, `IDleft < IDnew < IDright`) 55 | RDX IDs are integers, hence not dense. 56 | 2. the interleaving anomaly when concurrently inserted spans 57 | of elements go interleaved because of ID range overlap. 58 | 59 | In both cases, the solution is to use insertion *trains* instead 60 | of individual insertions. Assume we insert `k+1` elements 61 | `ID0, ID1, ... IDk` between `IDleft` and `IDright`. We call 62 | `ID0` the train *head*, `IDleft < ID0 < IDright`. 63 | `ID1..IDk` we call the train's *tail*. 64 | Very counter-intuitively, we assign tail IDs so that 65 | `ID1 < ID2 < ... < IDk < IDleft`. In other words, 66 | we see an array as a concatenation of several monotonously 67 | sorted chunks and merge it that way. The entire ID sequence 68 | may not be sorted. The key here is the handling of points of 69 | *discontinuity*, namely train heads. From the inner workings of 70 | merge sort we may see: if the head gets selected and moved to 71 | the output array, the tail will follow *immediately* as it 72 | follows the head in the input and its IDs are *lower*! 73 | 74 | For merge sort, that would be a bug: input arrays are not 75 | fully sorted. For DISCONT, that is a feature! 76 | That way, we only need one unclaimed ID to attach the head 77 | to the original array. Also, interleaving can not happen. 78 | Both issues of the Figma scheme are solved. 79 | 80 | In case we do not plan to splice an array in the future, we 81 | may use all-zero IDs for all the elements. In such a case, 82 | we can only do overwrites, deletes and appends. 83 | 84 | Overall, we win complexity wise, because CausalTree/RGA requires a 85 | supplementary data structure and its computational complexity 86 | is `O(NlogN)`. DISCONT only reads the inputs and its time 87 | complexity is pure `O(N)`. 88 | We can also compare DISCONT to Logoot, a true CRDT using 89 | dense ID space (strings). With more complicated editing histories 90 | Logoot IDs become longer and longer. DISCONT uses integer IDs, 91 | which cause much slimmer overheads. 92 | 93 | There is an additional expense though. Specifying DISCONT 94 | patches requires more context than a single attachment point ID. 95 | In fact, we may need to mention a sequence of head IDs to 96 | exactly locate the new element in an array. That feels a lot 97 | like TreeDoc, of course. That is not only acceptable, but 98 | also beneficial if we consider how merge works inside an LSM 99 | engine. Patches may need to be merged to other patches before 100 | being applied to the entire state; CT/RGA implementations 101 | may find that requirement problematic. 102 | 103 | [f]: https://www.figma.com/blog/how-figmas-multiplayer-technology-works/ 104 | [l]: https://en.wikipedia.org/wiki/Log-structured_merge-tree 105 | [m]: https://www.geeksforgeeks.org/dsa/merge-sort/ 106 | [i]: ./id.go 107 | [c]: https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/utils/MergeIterator.java#L101C8-L101C9 108 | 109 | ## Growing and splitting text blocks 110 | 111 | In application to collaborative text editing, changes are 112 | most often tracked on a per-letter basis. For efficiency, 113 | letters with sequential ids get packed into solid blocks. 114 | In case a new letter has to be inserted in the middle of 115 | a block, the block gets split. 116 | 117 | Overall, RDX implies word-based change tracking with the 118 | assumption that sharing less-than-a-word is not worth it. 119 | Still, both per-letter granularity and the block-splitting 120 | technique can be emulated in RDX. 121 | 122 | As RDX Linear supports element overwrites, we may overwrite 123 | a prefix of a word with a longer prefix, till the word is 124 | complete. That gives the real-time per-letter experience. 125 | LSM storage compacts overwrites, so this produces no long 126 | term overhead. 127 | 128 | Similarly, we may agree to ship sentence-long blocks and 129 | split them into words in a deterministic way. That way, if 130 | two editors split a sentence block concurrently, that 131 | would produce exactly the same result. While editor appends 132 | text, we grow a block, on the first insert we split it. 133 | 134 | With these techniques, we effectively achieve the same 135 | effect as CT/RGA sequential-ID blocks, minimizing the 136 | metadata overhead for solid chunks of text. Whether the 137 | gain is worth the complexity is a separate question though. 138 | 139 | ## The pomological proof 140 | 141 | 142 | This picture depends three main cases for DISCONT. 143 | The vertical axis is the ID value, the horizontal axis is 144 | the factual position in the array. So, arrays read left- 145 | to-right. 146 | 147 | - Case A is the normal Figma numbering, no issues. Elements 148 | (green apple, red apple, peach) are sorted in the order of 149 | their IDs, everything is straightforward. 150 | 151 | - Case B is the moment of numbering space saturation. Elements 152 | orange and pear, once inserted, have IDs that differ by 1, 153 | so no new IDs can be inserted between them. 154 | 155 | - Case C is where we use the train-insertion technique to add 156 | two cherries and a grape; their IDs are lower than the orange 157 | and the pear have. 158 | 159 | Now, imagine we merge any number of versions of that array. 160 | All versions fall into one of the classes: A, B, or C. Those that 161 | have the orange are B, those that also have a cherry are C. 162 | Let's think how merge sort would process all these inputs. 163 | (Note that our version of merge sort merges copies of the same element.) 164 | In case we have no C inputs, there is nothing to talk about, 165 | merge sort works normally. 166 | 167 | Now suppose we have B and C inputs. Apples will be merged just fine, 168 | there is no intrigue. Then, copies of the orange will be merged. 169 | Then, the train will immediately follow because IDs of its elements 170 | are lower than the ID of the orange, so if the orange "won" in the 171 | comparison, the cherry will win immediately next. We may have several 172 | C inputs with varying numbers of elements in the train, but the cherry 173 | train will be definitely handled before we get to the pear. 174 | 175 | Now suppose we have A, B and C inputs. In such a case, there is no 176 | intrigue either; as the orange is present in the inputs, its turn 177 | will be after the apples, before the pear, so the cherry train will 178 | be processed in the right order, immediately after the orange. 179 | 180 | This logic applies recursively, we may add another nested train 181 | between the cherries and the grape. Similarly, we may add several 182 | insertion trains at the top level. 183 | 184 | The key trick is: once the head of the train (the orange) is present, 185 | it orders normally in the top-level array, and its train immediately 186 | follows. 187 | 188 | Potentially, we may mess things up in several ways. First of all, 189 | we may execute the algorithm incorrectly and that will lead to 190 | really bad consequences, as train insertion is more fragile than 191 | regular Figma-style element sorting. But this case is on the "data 192 | corruption" side. There are subtler issues, e.g. interleaving. 193 | No matter whether we do it at the top level or in any train, 194 | suppose we concurrently do non-train insertion of many elements into 195 | the same spot. Then their relative order in the resulting sequence 196 | is basically random. It will depend on the choice of IDs by those 197 | who inserted the elements. They may oven overwrite each other in 198 | case we skip the `src` part of the ID (that is allowed in RDX). 199 | 200 | Overall, our choice of train vs plain insertions and the choice of 201 | IDs for the new elements both depend on the semantics of the collection 202 | in question. These things have to be handled carefully at a higher 203 | level where user intention is more clear. For example, we may want 204 | for concurrent insertions to overwrite each other, e.g. in the case 205 | of typo correction "baguete"->"baguette" done concurrently by two 206 | users we do not want "baguettte" as a result (the case of CT/RGA). 207 | DISCONT provides some options here which have to be handled 208 | carefully. 209 | 210 | Finally, RDX does not prescribe granularity of the edits; 211 | it can work per-letter, per-word or per-line, there is no difference. 212 | So, all that nuance is left for higher-level code to sort out. 213 | -------------------------------------------------------------------------------- /JDR.go.rl: -------------------------------------------------------------------------------- 1 | package main 2 | import "errors" 3 | // action indices for the parser 4 | 5 | const (JDRenum = 0 6 | JDRNL = JDRenum+1 7 | JDRUtf8cp1 = JDRenum+10 8 | JDRUtf8cp2 = JDRenum+11 9 | JDRUtf8cp3 = JDRenum+12 10 | JDRUtf8cp4 = JDRenum+13 11 | JDRInt = JDRenum+19 12 | JDRFloat = JDRenum+20 13 | JDRTerm = JDRenum+21 14 | JDRRef = JDRenum+22 15 | JDRString = JDRenum+23 16 | JDRMLString = JDRenum+24 17 | JDRStamp = JDRenum+25 18 | JDRNoStamp = JDRenum+26 19 | JDROpenP = JDRenum+27 20 | JDRCloseP = JDRenum+28 21 | JDROpenL = JDRenum+29 22 | JDRCloseL = JDRenum+30 23 | JDROpenE = JDRenum+31 24 | JDRCloseE = JDRenum+32 25 | JDROpenX = JDRenum+33 26 | JDRCloseX = JDRenum+34 27 | JDRComma = JDRenum+35 28 | JDRColon = JDRenum+36 29 | JDRSemicolon = JDRenum+37 30 | JDROpen = JDRenum+38 31 | JDRClose = JDRenum+39 32 | JDRInter = JDRenum+40 33 | JDRFIRST = JDRenum+42 34 | JDRRoot = JDRenum+43 35 | ) 36 | 37 | // user functions (callbacks) for the parser 38 | // func JDRonNL (tok []byte, state *JDRstate) error 39 | // func JDRonUtf8cp1 (tok []byte, state *JDRstate) error 40 | // func JDRonUtf8cp2 (tok []byte, state *JDRstate) error 41 | // func JDRonUtf8cp3 (tok []byte, state *JDRstate) error 42 | // func JDRonUtf8cp4 (tok []byte, state *JDRstate) error 43 | // func JDRonInt (tok []byte, state *JDRstate) error 44 | // func JDRonFloat (tok []byte, state *JDRstate) error 45 | // func JDRonTerm (tok []byte, state *JDRstate) error 46 | // func JDRonRef (tok []byte, state *JDRstate) error 47 | // func JDRonString (tok []byte, state *JDRstate) error 48 | // func JDRonMLString (tok []byte, state *JDRstate) error 49 | // func JDRonStamp (tok []byte, state *JDRstate) error 50 | // func JDRonNoStamp (tok []byte, state *JDRstate) error 51 | // func JDRonOpenP (tok []byte, state *JDRstate) error 52 | // func JDRonCloseP (tok []byte, state *JDRstate) error 53 | // func JDRonOpenL (tok []byte, state *JDRstate) error 54 | // func JDRonCloseL (tok []byte, state *JDRstate) error 55 | // func JDRonOpenE (tok []byte, state *JDRstate) error 56 | // func JDRonCloseE (tok []byte, state *JDRstate) error 57 | // func JDRonOpenX (tok []byte, state *JDRstate) error 58 | // func JDRonCloseX (tok []byte, state *JDRstate) error 59 | // func JDRonComma (tok []byte, state *JDRstate) error 60 | // func JDRonColon (tok []byte, state *JDRstate) error 61 | // func JDRonSemicolon (tok []byte, state *JDRstate) error 62 | // func JDRonOpen (tok []byte, state *JDRstate) error 63 | // func JDRonClose (tok []byte, state *JDRstate) error 64 | // func JDRonInter (tok []byte, state *JDRstate) error 65 | // func JDRonFIRST (tok []byte, state *JDRstate) error 66 | // func JDRonRoot (tok []byte, state *JDRstate) error 67 | 68 | 69 | 70 | %%{ 71 | 72 | machine JDR; 73 | 74 | alphtype byte; 75 | 76 | # ragel actions 77 | action JDRNL0 { mark0[JDRNL] = p; } 78 | action JDRNL1 { 79 | err = JDRonNL(data[mark0[JDRNL] : p], state); 80 | if err!=nil { 81 | fbreak; 82 | } 83 | } 84 | action JDRUtf8cp10 { mark0[JDRUtf8cp1] = p; } 85 | action JDRUtf8cp11 { 86 | err = JDRonUtf8cp1(data[mark0[JDRUtf8cp1] : p], state); 87 | if err!=nil { 88 | fbreak; 89 | } 90 | } 91 | action JDRUtf8cp20 { mark0[JDRUtf8cp2] = p; } 92 | action JDRUtf8cp21 { 93 | err = JDRonUtf8cp2(data[mark0[JDRUtf8cp2] : p], state); 94 | if err!=nil { 95 | fbreak; 96 | } 97 | } 98 | action JDRUtf8cp30 { mark0[JDRUtf8cp3] = p; } 99 | action JDRUtf8cp31 { 100 | err = JDRonUtf8cp3(data[mark0[JDRUtf8cp3] : p], state); 101 | if err!=nil { 102 | fbreak; 103 | } 104 | } 105 | action JDRUtf8cp40 { mark0[JDRUtf8cp4] = p; } 106 | action JDRUtf8cp41 { 107 | err = JDRonUtf8cp4(data[mark0[JDRUtf8cp4] : p], state); 108 | if err!=nil { 109 | fbreak; 110 | } 111 | } 112 | action JDRInt0 { mark0[JDRInt] = p; } 113 | action JDRInt1 { 114 | err = JDRonInt(data[mark0[JDRInt] : p], state); 115 | if err!=nil { 116 | fbreak; 117 | } 118 | } 119 | action JDRFloat0 { mark0[JDRFloat] = p; } 120 | action JDRFloat1 { 121 | err = JDRonFloat(data[mark0[JDRFloat] : p], state); 122 | if err!=nil { 123 | fbreak; 124 | } 125 | } 126 | action JDRTerm0 { mark0[JDRTerm] = p; } 127 | action JDRTerm1 { 128 | err = JDRonTerm(data[mark0[JDRTerm] : p], state); 129 | if err!=nil { 130 | fbreak; 131 | } 132 | } 133 | action JDRRef0 { mark0[JDRRef] = p; } 134 | action JDRRef1 { 135 | err = JDRonRef(data[mark0[JDRRef] : p], state); 136 | if err!=nil { 137 | fbreak; 138 | } 139 | } 140 | action JDRString0 { mark0[JDRString] = p; } 141 | action JDRString1 { 142 | err = JDRonString(data[mark0[JDRString] : p], state); 143 | if err!=nil { 144 | fbreak; 145 | } 146 | } 147 | action JDRMLString0 { mark0[JDRMLString] = p; } 148 | action JDRMLString1 { 149 | err = JDRonMLString(data[mark0[JDRMLString] : p], state); 150 | if err!=nil { 151 | fbreak; 152 | } 153 | } 154 | action JDRStamp0 { mark0[JDRStamp] = p; } 155 | action JDRStamp1 { 156 | err = JDRonStamp(data[mark0[JDRStamp] : p], state); 157 | if err!=nil { 158 | fbreak; 159 | } 160 | } 161 | action JDRNoStamp0 { mark0[JDRNoStamp] = p; } 162 | action JDRNoStamp1 { 163 | err = JDRonNoStamp(data[mark0[JDRNoStamp] : p], state); 164 | if err!=nil { 165 | fbreak; 166 | } 167 | } 168 | action JDROpenP0 { mark0[JDROpenP] = p; } 169 | action JDROpenP1 { 170 | err = JDRonOpenP(data[mark0[JDROpenP] : p], state); 171 | if err!=nil { 172 | fbreak; 173 | } 174 | } 175 | action JDRCloseP0 { mark0[JDRCloseP] = p; } 176 | action JDRCloseP1 { 177 | err = JDRonCloseP(data[mark0[JDRCloseP] : p], state); 178 | if err!=nil { 179 | fbreak; 180 | } 181 | } 182 | action JDROpenL0 { mark0[JDROpenL] = p; } 183 | action JDROpenL1 { 184 | err = JDRonOpenL(data[mark0[JDROpenL] : p], state); 185 | if err!=nil { 186 | fbreak; 187 | } 188 | } 189 | action JDRCloseL0 { mark0[JDRCloseL] = p; } 190 | action JDRCloseL1 { 191 | err = JDRonCloseL(data[mark0[JDRCloseL] : p], state); 192 | if err!=nil { 193 | fbreak; 194 | } 195 | } 196 | action JDROpenE0 { mark0[JDROpenE] = p; } 197 | action JDROpenE1 { 198 | err = JDRonOpenE(data[mark0[JDROpenE] : p], state); 199 | if err!=nil { 200 | fbreak; 201 | } 202 | } 203 | action JDRCloseE0 { mark0[JDRCloseE] = p; } 204 | action JDRCloseE1 { 205 | err = JDRonCloseE(data[mark0[JDRCloseE] : p], state); 206 | if err!=nil { 207 | fbreak; 208 | } 209 | } 210 | action JDROpenX0 { mark0[JDROpenX] = p; } 211 | action JDROpenX1 { 212 | err = JDRonOpenX(data[mark0[JDROpenX] : p], state); 213 | if err!=nil { 214 | fbreak; 215 | } 216 | } 217 | action JDRCloseX0 { mark0[JDRCloseX] = p; } 218 | action JDRCloseX1 { 219 | err = JDRonCloseX(data[mark0[JDRCloseX] : p], state); 220 | if err!=nil { 221 | fbreak; 222 | } 223 | } 224 | action JDRComma0 { mark0[JDRComma] = p; } 225 | action JDRComma1 { 226 | err = JDRonComma(data[mark0[JDRComma] : p], state); 227 | if err!=nil { 228 | fbreak; 229 | } 230 | } 231 | action JDRColon0 { mark0[JDRColon] = p; } 232 | action JDRColon1 { 233 | err = JDRonColon(data[mark0[JDRColon] : p], state); 234 | if err!=nil { 235 | fbreak; 236 | } 237 | } 238 | action JDRSemicolon0 { mark0[JDRSemicolon] = p; } 239 | action JDRSemicolon1 { 240 | err = JDRonSemicolon(data[mark0[JDRSemicolon] : p], state); 241 | if err!=nil { 242 | fbreak; 243 | } 244 | } 245 | action JDROpen0 { mark0[JDROpen] = p; } 246 | action JDROpen1 { 247 | err = JDRonOpen(data[mark0[JDROpen] : p], state); 248 | if err!=nil { 249 | fbreak; 250 | } 251 | } 252 | action JDRClose0 { mark0[JDRClose] = p; } 253 | action JDRClose1 { 254 | err = JDRonClose(data[mark0[JDRClose] : p], state); 255 | if err!=nil { 256 | fbreak; 257 | } 258 | } 259 | action JDRInter0 { mark0[JDRInter] = p; } 260 | action JDRInter1 { 261 | err = JDRonInter(data[mark0[JDRInter] : p], state); 262 | if err!=nil { 263 | fbreak; 264 | } 265 | } 266 | action JDRFIRST0 { mark0[JDRFIRST] = p; } 267 | action JDRFIRST1 { 268 | err = JDRonFIRST(data[mark0[JDRFIRST] : p], state); 269 | if err!=nil { 270 | fbreak; 271 | } 272 | } 273 | action JDRRoot0 { mark0[JDRRoot] = p; } 274 | action JDRRoot1 { 275 | err = JDRonRoot(data[mark0[JDRRoot] : p], state); 276 | if err!=nil { 277 | fbreak; 278 | } 279 | } 280 | 281 | # ragel grammar rules 282 | JDRNL = ( "\n" ) >JDRNL0 %JDRNL1; 283 | JDRws = ( [\r\t ] | JDRNL ); # no ws callback 284 | JDRhex = ( [0-9a-fA-Z] ); # no hex callback 285 | JDRron64 = ( [0-9A-Za-z_~] ); # no ron64 callback 286 | JDRutf8cont = ( (0x80..0xbf) ); # no utf8cont callback 287 | JDRutf8lead1 = ( (0x00..0x7f) ); # no utf8lead1 callback 288 | JDRutf8lead2 = ( (0xc0..0xdf) ); # no utf8lead2 callback 289 | JDRutf8lead3 = ( (0xe0..0xef) ); # no utf8lead3 callback 290 | JDRutf8lead4 = ( (0xf0..0xf7) ); # no utf8lead4 callback 291 | JDRUtf8cp1 = ( JDRutf8lead1 ) >JDRUtf8cp10 %JDRUtf8cp11; 292 | JDRUtf8cp2 = ( JDRutf8lead2 JDRutf8cont ) >JDRUtf8cp20 %JDRUtf8cp21; 293 | JDRUtf8cp3 = ( JDRutf8lead3 JDRutf8cont JDRutf8cont ) >JDRUtf8cp30 %JDRUtf8cp31; 294 | JDRUtf8cp4 = ( JDRutf8lead4 JDRutf8cont JDRutf8cont JDRutf8cont ) >JDRUtf8cp40 %JDRUtf8cp41; 295 | JDRutf8cp = ( JDRUtf8cp1 | JDRUtf8cp2 | JDRUtf8cp3 | JDRUtf8cp4 ); # no utf8cp callback 296 | JDResc = ( [\\] ["\\/bfnrt] ); # no esc callback 297 | JDRhexEsc = ( "\\u" JDRhex{4} ); # no hexEsc callback 298 | JDRutf8esc = ( (JDRutf8cp - ["\\\r\n]) | JDResc | JDRhexEsc ); # no utf8esc callback 299 | JDRid128 = ( JDRron64+ ("-" JDRron64+)? ); # no id128 callback 300 | JDRInt = ( [\-]? ( [0] | [1-9] [0-9]* ) ) >JDRInt0 %JDRInt1; 301 | JDRFloat = ( ( [\-]? ( [0] | [1-9] [0-9]* ) 302 | ("." [0-9]+)? 303 | ([eE] [\-+]? [0-9]+ )? ) - JDRInt ) >JDRFloat0 %JDRFloat1; 304 | JDRTerm = ( JDRron64+ -JDRInt -JDRFloat ) >JDRTerm0 %JDRTerm1; 305 | JDRRef = ( JDRid128 -JDRFloat -JDRInt -JDRTerm ) >JDRRef0 %JDRRef1; 306 | JDRString = ( ["] JDRutf8esc* ["] ) >JDRString0 %JDRString1; 307 | JDRMLString = ( "`" (JDRutf8cp - [`])* "`" ) >JDRMLString0 %JDRMLString1; 308 | JDRStamp = ( "@" JDRid128 ) >JDRStamp0 %JDRStamp1; 309 | JDRNoStamp = ( "" ) >JDRNoStamp0 %JDRNoStamp1; 310 | JDROpenP = ( "(" ) >JDROpenP0 %JDROpenP1; 311 | JDRCloseP = ( ")" ) >JDRCloseP0 %JDRCloseP1; 312 | JDROpenL = ( "[" ) >JDROpenL0 %JDROpenL1; 313 | JDRCloseL = ( "]" ) >JDRCloseL0 %JDRCloseL1; 314 | JDROpenE = ( "{" ) >JDROpenE0 %JDROpenE1; 315 | JDRCloseE = ( "}" ) >JDRCloseE0 %JDRCloseE1; 316 | JDROpenX = ( "<" ) >JDROpenX0 %JDROpenX1; 317 | JDRCloseX = ( ">" ) >JDRCloseX0 %JDRCloseX1; 318 | JDRComma = ( "," ) >JDRComma0 %JDRComma1; 319 | JDRColon = ( ":" ) >JDRColon0 %JDRColon1; 320 | JDRSemicolon = ( ";" ) >JDRSemicolon0 %JDRSemicolon1; 321 | JDROpen = ( (JDROpenP | JDROpenL | JDROpenE | JDROpenX) JDRws* (JDRStamp JDRws* | JDRNoStamp) ) >JDROpen0 %JDROpen1; 322 | JDRClose = ( (JDRCloseP | JDRCloseL | JDRCloseE | JDRCloseX) JDRws* ) >JDRClose0 %JDRClose1; 323 | JDRInter = ( (JDRComma | JDRColon | JDRSemicolon) JDRws* ) >JDRInter0 %JDRInter1; 324 | JDRdelim = ( JDROpen | JDRClose | JDRInter ); # no delim callback 325 | JDRFIRST = ( ( JDRFloat | JDRInt | JDRRef | JDRString | JDRMLString | JDRTerm ) JDRws* ( JDRStamp JDRws* | JDRNoStamp) ) >JDRFIRST0 %JDRFIRST1; 326 | JDRRoot = ( ( JDRws | JDRFIRST | JDRdelim )** ) >JDRRoot0 %JDRRoot1; 327 | 328 | main := JDRRoot; 329 | 330 | }%% 331 | 332 | %%write data; 333 | 334 | // the public API function 335 | func JDRlexer (state *JDRstate) (err error) { 336 | 337 | data := state.text 338 | var mark0 [64]int 339 | cs, p, pe, eof := 0, 0, len(data), len(data) 340 | 341 | %% write init; 342 | %% write exec; 343 | 344 | if (p!=len(data) || cs < JDR_first_final) { 345 | state.text = state.text[p:]; 346 | return errors.New("JDR bad syntax") 347 | } 348 | return nil; 349 | } 350 | -------------------------------------------------------------------------------- /itheap.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "fmt" 7 | "io" 8 | ) 9 | 10 | type Seeker interface { 11 | // Seek moves to the first equal-or-greater record; 12 | // returns Eq or Grtr; if none found, returns Less 13 | Seek(id ID) int 14 | Error() error 15 | } 16 | 17 | type Reader interface { 18 | // moves to the next record 19 | Read() bool 20 | 21 | Record() Stream 22 | Parsed() (lit byte, id ID, value []byte) 23 | 24 | Error() error 25 | } 26 | 27 | type ReadSeekCloser interface { 28 | Reader 29 | Seeker 30 | io.Closer 31 | } 32 | 33 | type Getter interface { 34 | Get(id ID) (value Stream, err error) 35 | } 36 | 37 | // Iter is an Stream byte stream iterator. Start state: before the 1st element. 38 | // Convention: when passing a whole byte stream, use the start state; when 39 | // passing an element or a group, always position the iterator appropriately. 40 | type Iter struct { 41 | data []byte 42 | vallen int 43 | hdrlen uint8 44 | idlen uint8 45 | lit byte 46 | errndx int8 47 | } 48 | 49 | var ErrIOFail = errors.New("IO failed") 50 | var ErrWrongType = errors.New("wrong record type") 51 | 52 | const ( 53 | ErrOKNdx = iota 54 | ErrIncompleteNdx 55 | ErrBadRecordNdx 56 | ErrIOFailNdx 57 | ErrWrongTypeNdx 58 | ErrEoFNdx 59 | ) 60 | 61 | var iterr = []error{ 62 | nil, 63 | ErrIncomplete, 64 | ErrBadRecord, 65 | ErrIOFail, 66 | ErrEoS, 67 | } 68 | 69 | func NewIter(data []byte) Iter { 70 | return Iter{data: data} 71 | } 72 | 73 | func BadIter(data []byte, err int8) Iter { 74 | return Iter{data: data, errndx: err} 75 | } 76 | 77 | func (it *Iter) Inner() Iter { 78 | if IsPLEX(it.Lit()) { 79 | return NewIter(it.Value()) 80 | } 81 | return Iter{} 82 | } 83 | 84 | func (it *Iter) IsEmpty() bool { 85 | return it.hdrlen == 0 86 | } 87 | 88 | func (it *Iter) HasData() bool { 89 | return len(it.data) != 0 90 | } 91 | 92 | func (it *Iter) IsAtStart() bool { 93 | return it.errndx == 0 && it.hdrlen == 0 94 | } 95 | 96 | func (it *Iter) Rest() []byte { 97 | return it.data[int(it.hdrlen+it.idlen)+it.vallen:] 98 | } 99 | 100 | func (it *Iter) Error() error { 101 | return iterr[it.errndx] 102 | } 103 | 104 | func (it *Iter) ErrNdx() int8 { 105 | return it.errndx 106 | } 107 | 108 | func (it *Iter) HasMore() bool { 109 | return len(it.Rest()) > 0 110 | } 111 | 112 | func (it *Iter) HasFailed() bool { 113 | return it.errndx > 0 114 | } 115 | 116 | func (it *Iter) Into() bool { 117 | if !IsPLEX(it.Lit()) { 118 | return false 119 | } 120 | *it = NewIter(it.Value()) 121 | return true 122 | } 123 | 124 | func (it *Iter) Read() bool { 125 | if len(it.data) == 0 || it.errndx > 0 { 126 | return false 127 | } 128 | it.data = it.data[int(it.hdrlen+it.idlen)+it.vallen:] 129 | if len(it.data) == 0 { 130 | *it = Iter{errndx: it.errndx} 131 | return false 132 | } 133 | it.lit = it.data[0] 134 | if (it.lit & CaseBit) != 0 { 135 | it.lit -= CaseBit 136 | it.hdrlen = 3 137 | if len(it.data) < int(it.hdrlen) { 138 | it.errndx = 1 139 | return false 140 | } 141 | it.vallen = int(it.data[1]) 142 | } else { 143 | it.hdrlen = 6 144 | if len(it.data) < int(it.hdrlen) { 145 | it.errndx = 1 146 | return false 147 | } 148 | de := binary.LittleEndian.Uint32(it.data[1:5]) 149 | if de >= (1 << 30) { 150 | it.errndx = 2 151 | return false 152 | } 153 | it.vallen = int(de) 154 | } 155 | if len(it.data) < int(it.hdrlen)+it.vallen-1 { 156 | it.errndx = 1 157 | return false 158 | } 159 | it.idlen = it.data[it.hdrlen-1] 160 | if int(it.idlen) > it.vallen { 161 | it.errndx = 2 162 | return false 163 | } 164 | it.vallen -= int(it.idlen) + 1 165 | return true 166 | } 167 | 168 | func Peek(rdx []byte) byte { 169 | if len(rdx) == 0 { 170 | return 0 171 | } 172 | return rdx[0] & ^CaseBit 173 | } 174 | 175 | func (it *Iter) Seek(id ID) int { 176 | if !it.HasData() { 177 | return Less 178 | } 179 | if it.IsAtStart() && !it.Read() { 180 | return Less 181 | } 182 | z := it.ID().Compare(id) // FIXME b, e !!! 183 | for z < Eq && it.Read() { 184 | z = it.ID().Compare(id) 185 | } 186 | return z 187 | } 188 | 189 | func (it *Iter) Parsed() (lit byte, id ID, value []byte) { 190 | if len(it.data) == 0 { 191 | return 192 | } 193 | b := int(it.hdrlen + it.idlen) 194 | return it.data[0] & ^CaseBit, 195 | UnzipID(it.data[it.hdrlen:b]), 196 | it.data[b : b+it.vallen] 197 | } 198 | 199 | func (i *Iter) Lit() byte { 200 | if len(i.data) == 0 { 201 | return 0 202 | } 203 | return i.data[0] & ^CaseBit 204 | } 205 | 206 | func (i Iter) Peek() byte { 207 | return Peek(i.Rest()) 208 | } 209 | 210 | func (it *Iter) ID() ID { 211 | return UnzipID(it.data[it.hdrlen : it.hdrlen+it.idlen]) 212 | } 213 | 214 | func (it *Iter) Value() []byte { 215 | b := int(it.hdrlen + it.idlen) 216 | return it.data[b : b+it.vallen] 217 | } 218 | 219 | func (it *Iter) Reference() ID { 220 | if it.Lit() != LitReference { // todo conversions 221 | return ID{} 222 | } 223 | return UnzipID(it.Value()) 224 | } 225 | 226 | func (it *Iter) Integer() Integer { 227 | if it.Lit() != LitInteger { 228 | return 0 229 | } 230 | return Integer(UnzipInt64(it.Value())) 231 | } 232 | 233 | func (it *Iter) Float() Float { 234 | if it.Lit() != LitFloat { 235 | return 0 236 | } 237 | return Float(UnzipFloat64(it.Value())) 238 | } 239 | 240 | func (it *Iter) Record() Stream { 241 | return it.data[:int(it.hdrlen+it.idlen)+it.vallen] 242 | } 243 | 244 | func (it *Iter) IsLive() bool { 245 | return it.idlen == 0 || (it.data[it.hdrlen]&1) == 0 246 | } 247 | 248 | func (i *Iter) NextLive() (ok bool) { 249 | ok = i.Read() 250 | for ok && !i.IsLive() { 251 | ok = i.Read() 252 | } 253 | return 254 | } 255 | 256 | func (it *Iter) String() string { 257 | switch it.Lit() { 258 | case 0: 259 | return "" 260 | case LitFloat: 261 | return fmt.Sprintf("%e", UnzipFloat64(it.Value())) 262 | case LitInteger: 263 | return fmt.Sprintf("%d", UnzipInt64(it.Value())) 264 | case LitReference: 265 | return string(UnzipID(it.Value()).RonString()) 266 | case LitString: 267 | return string(it.Value()) 268 | case LitTerm: 269 | return string(it.Value()) 270 | case LitTuple: 271 | return "()" 272 | case LitLinear: 273 | return "[]" 274 | case LitEuler: 275 | return "{}" 276 | case LitMultix: 277 | return "<>" 278 | default: 279 | return "" 280 | } 281 | } 282 | 283 | func (it *Iter) Close() error { 284 | *it = Iter{} 285 | return nil 286 | } 287 | 288 | type Heap []Iter 289 | 290 | func Heapize(rdx [][]byte, z Compare) (heap Heap, err error) { 291 | heap = make(Heap, 0, len(rdx)) 292 | for _, r := range rdx { 293 | if len(r) == 0 { 294 | continue 295 | } 296 | i := NewIter(r) 297 | if i.Read() { 298 | heap = append(heap, i) 299 | } else if i.Error() != nil { 300 | return nil, i.Error() 301 | } 302 | heap.LastUp(z) 303 | } 304 | return 305 | } 306 | 307 | func (heap *Heap) LastUp(z Compare) { 308 | heap.Up(len(*heap)-1, z) 309 | } 310 | 311 | func Iterize(rdx [][]byte) (heap Heap, err error) { 312 | heap = make(Heap, 0, len(rdx)) 313 | for _, r := range rdx { 314 | if len(r) == 0 { 315 | continue 316 | } 317 | i := NewIter(r) 318 | if i.Read() { 319 | heap = append(heap, i) 320 | } else if i.Error() != nil { 321 | return nil, i.Error() 322 | } 323 | } 324 | return 325 | } 326 | 327 | func (ih Heap) Up(a int, z Compare) { 328 | for { 329 | b := (a - 1) / 2 // parent 330 | if b == a || z(&ih[a], &ih[b]) >= Eq { 331 | break 332 | } 333 | ih[b], ih[a] = ih[a], ih[b] 334 | a = b 335 | } 336 | } 337 | 338 | func (ih Heap) EqUp(z Compare) (eqs int) { 339 | if len(ih) < 2 { 340 | return len(ih) 341 | } 342 | q := make([]int, 0, MaxInputs) 343 | q = append(q, 1, 2) 344 | eqs = 1 345 | for len(q) > 0 && q[0] < len(ih) { 346 | n := q[0] 347 | if Eq == z(&ih[0], &ih[n]) { 348 | j1 := 2*n + 1 349 | q = append(q, j1, j1+1) 350 | ih[eqs], ih[n] = ih[n], ih[eqs] 351 | eqs++ 352 | } 353 | q = q[1:] 354 | } 355 | return 356 | } 357 | 358 | func (heap *Heap) Remove(i int, z Compare) { 359 | ih := *heap 360 | l := len(ih) - 1 361 | ih[l], ih[i] = ih[i], ih[l] 362 | *heap = ih[:l] 363 | if i < len(*heap) { 364 | (*heap).Down(i, z) 365 | } 366 | } 367 | 368 | func (heap *Heap) NextK(k int, z Compare) (err error) { 369 | for i := k - 1; i >= 0; i-- { 370 | if (*heap)[i].Read() { 371 | (*heap).Down(i, z) 372 | } else if (*heap)[i].HasFailed() { 373 | err = (*heap)[i].Error() 374 | break 375 | } else { 376 | heap.Remove(i, z) 377 | } 378 | } 379 | return err 380 | } 381 | 382 | func (ih Heap) Down(i0 int, z Compare) bool { 383 | n := len(ih) 384 | i := i0 385 | for { 386 | j1 := 2*i + 1 387 | if j1 >= n || j1 < 0 { // j1 < 0 after int overflow 388 | break 389 | } 390 | j := j1 // left child 391 | if j2 := j1 + 1; j2 < n && z(&ih[j2], &ih[j1]) < Eq { 392 | j = j2 // = 2*i + 2 // right child 393 | } 394 | if z(&ih[j], &ih[i]) >= Eq { 395 | break 396 | } 397 | ih[i], ih[j] = ih[j], ih[i] 398 | i = j 399 | } 400 | return i > i0 401 | } 402 | 403 | func (heap *Heap) MergeNext(data []byte, Z Compare) ([]byte, error) { 404 | var err error = nil 405 | eqlen := heap.EqUp(Z) 406 | if eqlen == 1 { 407 | data = append(data, (*heap)[0].Record()...) 408 | } else { 409 | eqs := (*heap)[:eqlen] 410 | data, err = MergeSameSpotElements(data, eqs) 411 | } 412 | if err == nil { 413 | err = heap.NextK(eqlen, Z) // FIXME signature 414 | if eqlen > 1 { 415 | for i := eqlen; i < len(*heap); i++ { // FIXME bad 416 | heap.Up(i, Z) 417 | } 418 | } 419 | } 420 | return data, err 421 | } 422 | 423 | func (heap *Heap) IntersectNext(Z Compare) (ret Iter, err error) { 424 | l := len(*heap) 425 | if l == 0 { 426 | return ret, ErrEoS 427 | } 428 | for err == nil && !ret.HasData() { 429 | eqlen := heap.EqUp(Z) 430 | if eqlen == l { 431 | ret = (*heap)[0] 432 | } 433 | err = heap.NextK(eqlen, Z) 434 | if len(*heap) != l { 435 | (*heap) = (*heap)[:0] 436 | break 437 | } 438 | } 439 | return 440 | } 441 | 442 | func HeapMerge(data []byte, inputs [][]byte, Z Compare) (res []byte, err error) { 443 | var heap Heap 444 | heap, err = Heapize(inputs, Z) 445 | res = data 446 | for len(heap) > 0 && err == nil { 447 | res, err = heap.MergeNext(res, Z) 448 | } 449 | return 450 | } 451 | 452 | type LessFn func(a, b Iter) bool 453 | 454 | // note: on format violation, drops the input 455 | type MergeFn func(inputs []Iter, pre Stream) Stream 456 | 457 | type MapFn func(input Iter, pre Stream) Stream 458 | 459 | type Heap2 struct { 460 | inputs []Iter 461 | out Stream 462 | oldlen int 463 | z LessFn 464 | y MergeFn 465 | } 466 | 467 | func (heap *Heap2) Len() int { 468 | return len(heap.inputs) 469 | } 470 | func (heap *Heap2) Less(i, j int) bool { 471 | return heap.z(heap.inputs[i], heap.inputs[j]) 472 | } 473 | func (heap *Heap2) Swap(i, j int) { 474 | heap.inputs[i], heap.inputs[j] = heap.inputs[j], heap.inputs[i] 475 | } 476 | func (heap *Heap2) AddIter(it Iter) { 477 | heap.inputs = append(heap.inputs, it) 478 | HeapUp(heap) 479 | } 480 | func (heap *Heap2) Read() bool { 481 | if len(heap.inputs) == 0 { 482 | return false 483 | } 484 | eqs := make([]int, 0, MaxInputs) 485 | eqs = append(eqs, 0) 486 | for i := 0; i < len(eqs); i++ { 487 | k := eqs[i] 488 | kl := k*2 + 1 489 | if kl < len(heap.inputs) { 490 | if !heap.z(heap.inputs[0], heap.inputs[kl]) { 491 | eqs = append(eqs, kl) 492 | } 493 | kr := kl + 1 494 | if kr < len(heap.inputs) && !heap.z(heap.inputs[0], heap.inputs[kr]) { 495 | eqs = append(eqs, kr) 496 | } 497 | } 498 | } 499 | var merge []Iter 500 | if eqs[len(eqs)-1] == len(eqs)-1 { 501 | merge = heap.inputs[:len(eqs)] 502 | } else { 503 | merge = make([]Iter, 0, len(eqs)) 504 | for i := 0; i < len(eqs); i++ { 505 | merge = append(merge, heap.inputs[eqs[i]]) 506 | } 507 | } 508 | heap.oldlen = len(heap.out) 509 | heap.out = heap.y(merge, heap.out) 510 | for i := len(eqs) - 1; i >= 0; i-- { 511 | heap.inputs[eqs[i]].Read() 512 | } 513 | for i := len(eqs) - 1; i >= 0; i-- { 514 | k := eqs[i] 515 | if !heap.inputs[k].HasData() { 516 | heap.inputs[k] = heap.inputs[len(heap.inputs)-1] 517 | heap.inputs = heap.inputs[:len(heap.inputs)-1] 518 | } 519 | HeapDownN(heap, k) 520 | } 521 | return true 522 | } 523 | func (heap *Heap2) Record() Stream { 524 | return heap.out[heap.oldlen:] 525 | } 526 | func (heap *Heap2) Parsed() (lit byte, id ID, value []byte) { 527 | it := NewIter(heap.Record()) 528 | it.Read() 529 | return it.Parsed() 530 | } 531 | func (heap *Heap2) Error() error { 532 | return nil 533 | } 534 | func (heap *Heap2) ReadAll() (err error) { 535 | return 536 | } 537 | func MakeHeap2(inputs []Iter, y MergeFn, z LessFn) (heap Heap2) { 538 | heap.y = y 539 | heap.z = z 540 | for _, i := range inputs { 541 | heap.AddIter(i) 542 | } 543 | return 544 | } 545 | 546 | type ObjectReader struct { 547 | it Iter 548 | Key string 549 | Value Iter 550 | } 551 | 552 | func NewObjectReader(rdx []byte) (o ObjectReader, err error) { 553 | i := NewIter(rdx) 554 | if !i.Read() || i.Lit() != LitEuler { 555 | return ObjectReader{}, ErrBadRecord 556 | } 557 | o.it = NewIter(i.Value()) 558 | return 559 | } 560 | 561 | func (o *ObjectReader) Read() bool { 562 | for o.it.Read() { 563 | if o.it.Lit() != LitTuple { 564 | continue 565 | } 566 | o.Value = NewIter(o.it.Value()) 567 | if !o.Value.Read() || o.Value.Lit() != LitTerm { 568 | continue 569 | } 570 | o.Key = o.Value.String() 571 | o.Value.Read() 572 | return true 573 | } 574 | return false 575 | } 576 | 577 | func (o *ObjectReader) Record() Stream { 578 | return o.Value.Rest() 579 | } 580 | 581 | func (o *ObjectReader) Parsed() (lit byte, id ID, value []byte) { 582 | return o.Value.Parsed() 583 | } 584 | 585 | func (o *ObjectReader) Error() error { 586 | return o.it.Error() 587 | } 588 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RDX: Replicated Data eXchange format 2 | 3 | 4 | RDX is a bit-precise data replication and synchronization format. 5 | Technically, RDX is a JSON superset with CRDT merge operation defined. 6 | The spec guarantees that any RDX implementations: 7 | 8 | 1. interpret RDX data, be text or binary, in exactly the same way, and 9 | 2. can merge any two documents, all producing exactly the same result. 10 | 11 | Neither JSON nor CBOR provide guarantees of [identical reading][f] 12 | between implementations, not to mention diff/patch/merge operations. 13 | That makes the outcomes implementation-dependent, thus uncertain. 14 | The multitude of "JSON superset" formats do not improve on that front 15 | either, mostly focusing on the syntax and the supported types. 16 | 17 | The formats used by BitTorrent, BitCoin or `git` all guarantee bit-precise 18 | results only at the level of *immutable binary blobs*. 19 | Consider `git` merges: those are human-in-the-loop non-deterministic acts. 20 | RDX provides deterministic bit-precise results for *mutable structured data*. 21 | That affects the reproducibility, portability, interoperability, and 22 | security properties of anything using the format. 23 | 24 | The way RDX makes that guarantee is: 25 | 26 | 1. Limited, but arbitrarily composable primitives; no feature sprawl. 27 | 2. Careful bit-precise specification; formal grammars and defined limits. 28 | 3. Controlled invariants and issue detection/resolution if 1-2 fell short. 29 | 30 | Controlled invariants are the key feature. Any spec has ambiguities, any 31 | code has bugs. We declare key invariants an RDX implementation MUST satisfy. 32 | For example, there must be cross-implementation round-trip guarantee for 33 | binary (RDX) to text (JDR) format conversions. RDX implementations would 34 | be checked and cross-checked by any means available, so any developer can 35 | rely on that equivalence. Similarly, RDX guarantees that the result of 36 | a merge is fully immune to reordering and/or duplication of updates, etc. 37 | 38 | Performance-wise, RDX logic can fit into the constraints of a merge 39 | operator in a commodity LSM database (RocksDB, Pebble, Cassandra, etc). 40 | Essentially, it can turn any database into a CRDT database. 41 | 42 | 43 | ## Element types 44 | 45 | RDX has five "primitive" `FIRST` types, all defined in the least 46 | surprising way: 47 | 48 | - Float (IEEE 754, no `NaN`s), 49 | - Integer (`int64_t`), 50 | - Reference (128 bit Lamport ID), 51 | - String (UTF-8), and 52 | - Term (`true`, `null` etc). 53 | 54 | The text-based JDR format reuses JSON primitives with minor changes. 55 | The binary RDX is little-endian with zipint compression. 56 | On top of that, there are four `PLEX` container types which allow for 57 | arbitrary nesting: 58 | 59 | - tuPles: `("tomates rondes", 1, kg, 3.14)`, 60 | - Linear containers: `[1, 2, 3]`, 61 | - Eulerian sets `{A, B}` and/or maps `{1:one, 2:two}` and 62 | - multipleXed containers: version vectors, counters. 63 | 64 | This minimalistic type system is what one should reasonably expect, 65 | except maybe for multiplexed containers. Those are ordinary in the 66 | context of distributed systems. Also, any RDX element can bear 67 | a Lamport timestamp, e.g. `("tomates rondes", 1, kg, 3.14@bob-25A1)`. 68 | Similarly, that is necessary for versioning in a distributed system. 69 | 70 | The binary RDX format is the most straightforward [type-length-value][t] 71 | with one byte standing for record *type*, 1 or 4 bytes for value *length*. 72 | Further, the first byte of the *value* specifies the record's *timestamp* 73 | length thus separating it from the *payload*. That puts the typical 74 | overhead to 3 bytes per element. Here, RDX prioritizes simplicity over 75 | efficiency. 76 | 77 | The following section contains the detailed breakdown of the types 78 | and their serialization formats. 79 | 80 | ### FIRST 81 | 82 | A last-write-wins register is the simplest CRDT data type to implement. 83 | It is also the most popular type in practical use. 84 | For each LWW element, we only pick its latest "winner" value. 85 | An RDX element of any type has a logical timestamp attached. 86 | So, picking the latest revision is straightforward. 87 | For the RDX timestamp/ID/reference inner structure, see the respective 88 | section below. 89 | In the binary RDX, Float, Integer and Reference payloads use a 90 | variable-length integer serialization scheme named *zip-ints*, see below. 91 | Strings are UTF-8 and Terms are Base64 ASCII. 92 | Overlong encodings are forbidden both for UTF-8 strings and for zip-ints. 93 | There must be only one correct way to serialize a value. 94 | 95 | | Type | JDR | RDX | 96 | |---------------------------------|----------|----------------------------| 97 | | Float: 64 bit IEEE float |`1.23e+2`|`66 0400 027a03` | 98 | | (Float) |`-0.1E-1` |`66 0900 fd215e87e27528de` | 99 | | (Float) |`1.2` |`66 0900 fccfcccccccccccc` | 100 | | Integer: 64 bit two's complement| `0` |`69 0100 ` | 101 | | (Integer) | `-4` |`69 0200 07` | 102 | | (Integer) |`65536` |`69 0400 000002` | 103 | | Reference, 128 bit UUID/timestamp|Alice-123 |`72 0900 83100000e9d9c20a` | 104 | | (Reference) |0-232BKMEDHz|`72 0a00 7ed43816b508830000`| 105 | | (Reference) |0-0 |`72 0100` | 106 | | String, UTF-8 string |"Hello" |`73 0600 48656c6c6f` | 107 | | (String) |"код" |`73 0700 d0bad0bed0b4` | 108 | | Term, Base64 literal | null |`74 0500 6e756c6c` | 109 | | (Term) | true |`74 0500 74727565` | 110 | 111 | ### PLEX 112 | 113 | Collection types allow for arbitrary bundling and nesting of `FIRST` values and other collections. 114 | The JDR syntax for PLEX collections is simply brackets: `()` `[]` `{}` and `<>`, respectively. 115 | Elements inside the brackets can be separated by either a comma or some whitespace or both. 116 | For example, `(1 2)` or `(1,2)` or `( 1, 2 )`. 117 | 118 | The key difference between the collection types is their respective *element order*. 119 | It defines how to arrange elements in the collection, how to search them and, 120 | importantly, which elements contend for the same *spot*. 121 | 122 | *Tuples* are short fixed-order collections: couples, triples, quadruples, and so on. 123 | Those can be as simple as `(1 2)`, a couple of integers or `()`, an empty tuple. 124 | The order within tuples is fixed: what was added as the second element, will always be second. 125 | The ways to edit a tuple is to replace or to append an element. 126 | There is an alternative colon-based syntax for tuples, see below. 127 | 128 | *Linear* collections are essentially arrays, like `["Alice", "Bob", "Carol"]`. 129 | Differently from x-ples, arrays can have elements inserted or removed. 130 | For example, we can edit `[1, 3, 4, 5]` to become `[1, 2, 3, 4]`. 131 | While the relative order of elements is preserved on edit, positions can change. 132 | The order is mainly based on the stamps of the elements, in accordance with the [DISCONT CRDT][n] algorithm. 133 | 134 | *Eulerian* collections are sets, maps and suchlike. 135 | Their elements go in the *value order*, e.g. `{1 2 3}`, `{A B C}` or `{"one", "three", "two"}`. 136 | Note that a map is a set containing key-value couples, e.g. `{(1 "one") (2 "two")}`. 137 | Tuples within sets get ordered according to their *key*, i.e. the first element. 138 | 139 | *Multiplexed* collections are version vectors, counters, etc. 140 | Their elements get ordered according to the *source* part of their timestamp. 141 | In such a collection, each author's contribution is kept separately, and 142 | there is at most one contribution from each author: `<40@a1ec-3, 20@b0b-1>`. 143 | Those can be added or updated. 144 | 145 | For all `PLEX` types, the TLV form is same as `FIRST` elements have, except the 146 | payload is a sequence of contained elements in their TLV form, properly ordered. 147 | 148 | The text-based format can order elements arbitrarily. On parse, those have to 149 | be normalized, i.e. ordered and same-spot elements merged. 150 | 151 | | Container type | JDR | RDX | 152 | |-----------------------------|--------------|-------------------------------------| 153 | | Tuple | (1 2 3) |`70 0d00 69020002 69020004 69020006` | 154 | | Tuple (colons) |"Bob":"Smith";|`70 0f00 730400426f62 730600536d697468`| 155 | | Linear container |`[a b c]` |`6c 0d00 74020061 74020062 74020063`| 156 | | Eulerians: sets and maps |`{1.0 2 three}`|`65 1200 660300fc0f 69020004 7406007468726565` | 157 | | Multiplexed containers |<14@Alice-232BLRh 52@Bob-232kLVg> |`78 1f00 690c0a10eeae5ff50a8300e6bc68 690e0c8a25b25bb5088300e9d9c20a1c`| 158 | 159 | ## Invariants 160 | 161 | Only some aspects of the format can be specified exhaustively, e.g. the formal grammar. 162 | Other aspects get manifested in human-language descriptions, test suites or code. 163 | Invariants are guarantees an implementation of the format provides to the end users. 164 | In case the spec omits an important detail, or maybe contradicts some invariant, then 165 | the spec will be amended to maintain that invariant. 166 | 167 | ### JDR round-trip 168 | 169 | For any valid JDR document `J`, `rdx(J) = rdx(jdr(rdx(J)))`, where `rdx` is the 170 | function parsing any valid text-based JDR into equivalent binary RDX and `jdr` is the 171 | opposite function rendering RDX in JDR. The `=` operator is bitwise equality. 172 | 173 | Note that there is only one `rdx`, but the number of possible `jdr` functions is 174 | infinite, due to possible formatting variations: `[1 2 3]`, `[1,2,3]`, `[1, 2, 3]` 175 | and so on. 176 | 177 | ### RDX round-trip 178 | 179 | For any valid RDX document `R`, `R = rdx(jdr(R))`. 180 | Two round-trip invariants effectively manifest the perfect equivalence of RDX and JDR. 181 | 182 | ### Idempotence 183 | 184 | For any valid RDX document `R`, `R + R = R`. 185 | Here, `+` is the *merge* function. 186 | 187 | For example, when we merge `{1 2 3}` and `{1 2 3}`, the result is `{1 2 3}`. 188 | 189 | ### Commutativity 190 | 191 | For any two valid RDX documents `A` and `B`, `A + B = B + A`. 192 | 193 | For example, `{1 2} + {3} = {3} + {1 2} = {1 2 3}`. 194 | 195 | ### Associativity 196 | 197 | For any two valid RDX documents `A`, `B` and `C`, `(A+B) + C = A + (B+C)`. 198 | 199 | ### Diffing/Patching 200 | 201 | For any two valid RDX documents `A` and `B` there is always a *difference* document 202 | `D = B-A` so that `|A + D| = |B|`. Here, `||` is metadata stripping, 203 | i.e. the results are not strictly identical, but their user-perceived form 204 | is the same (see below). 205 | 206 | In fact, there might be an infinite number of such difference documents. 207 | For example, if `A=B` then `D=0` or `D=A` are both diffs. 208 | So, we specifically denote the minimal possible diff doc which we call a `patch`. 209 | 210 | ## Orders 211 | 212 | There are different ways to order elements in a container or to decide on the 213 | precedence of contending elements or revisions. Every such order is a 214 | less-equal-greater relation. In most cases, we use a stack of orders, so 215 | fallback orders can resolve ties. 216 | 217 | 1. Type order: `PLEX` wins over `FIRST`, otherwise alphabetical. 218 | 2. Source order: according to the *source* half of a timestamp. 219 | 3. Time order: according to the *time* half of a timestamp. 220 | 4. Revision order: according to the *revision* part of a timestamp. 221 | 5. Identity order: according to the timestamp, first the time part 222 | (but ignoring the revision) then the source part, then the type. 223 | 224 | Value orders are only applicable for same-type elements: 225 | 226 | 1. Float: normally, 227 | 2. Integer: normally, 228 | 3. Reference: time order then source order, 229 | 4. String: string comparison, aka `strcmp()`, 230 | 5. Term: `strcmp()`, 231 | 6. PLEX: identity order. 232 | 233 | ## Technical aspects 234 | 235 | This section covers various aspects of RDX/JDR in greater detail. 236 | 237 | ### Merge 238 | 239 | The logic of RDX merge has two main cases: merging elements contending for 240 | the same spot and merging collections containing multiple elements which 241 | potentially contend for a spot. 242 | 243 | Same-spot merges are essentially last-write-wins. 244 | First, we look at the identity order. 245 | In case of a tie, those are different revisions of the same element. 246 | For PLEX elements, we recur into the elements doing collection merge. 247 | For FIRST elements, we use the value order to pick the winner. 248 | In case of a tie, we just have identical elements. 249 | 250 | The merge of PLEX elements is essentially a generalization of [merge sort][m]. 251 | Elements within same-identity containers are sorted according to the same order. 252 | We do a parallel pass of the containers, mutually ordering non-contending 253 | elements, while for contending ones we do same-spot merge. 254 | 255 | ### Zip coding of numbers 256 | 257 | One popular variable-length integer coding is [LEB128][b]. That one relies on 258 | the top bit in each byte as a continue/stop signal. As RDX employs TLV, 259 | lengths are known in advance, so a simpler *zip coding* is used. Namely, 260 | 261 | 1. Integer values are [zig-zag][g] coded, so -11 becomes 21 or `15` in hex, 262 | then the implementation uses the minimum number of bytes necessary 263 | to record the value. 264 | 2. Float values are byte-flipped (byte endianness conversion) to make 265 | "convenient" floats take less bytes. Then, use the minimum number of 266 | bytes necessary to record the resulting value. For example, `0.0` 267 | takes 0 bytes, or 3 bytes with headers included: `66 01 00` 268 | 3. Reference values use two 64-bit unsigned integers. As a TLV record 269 | only has one length field, there is a table specifying time/source 270 | lengths for each record length, e.g. 2=1+1, means `4-5` is recorded 271 | as `04 05` or `72 03 00 04 05` with the headers included. 272 | 273 | ### Stamp/Reference/ID bit layout 274 | 275 | RDX timestamps and references are logical (Lamport) timestamps of the 276 | same 128-bits layout: 64-bit *source* identifier (i.e. the originating 277 | replica) and 64-bit *time* value. Lower 6 bits of the time value are 278 | considered the *revision* number; odd time value means an element is 279 | deleted (a tombstone), even (0 and up) are live. The upper 4 bit of 280 | both 64-bit parts are currently reserved for future use. 281 | 282 | The *identity* part of an ID is all of it except the revision bits. 283 | If the identity bits are equal and the element type is equal, 284 | it is considered the same element. 285 | In such a case, recursive merge of container contents is possible. 286 | In case of different identity, it is either one or another. 287 | Essentially, the revision number allows to edit a FIRST element in 288 | place or to delete/undelete FIRST/PLEX elements. The identity bits 289 | affect the ordering of an element in the parent container, although 290 | the exact effect depends on the parent container type. 291 | The distinction between the revision bits and the rest of time value 292 | is moot in many cases. 293 | 294 | Other than that, RDX does not prescribe any particular logical 295 | clock algorithm: hybrid clocks, sequential numbering or even random 296 | identifiers. 297 | 298 | ### Tuple syntaxes 299 | 300 | Apart from the normal bracket syntax, tuples can be written in 301 | colon-semicolon notation, e.g. `(1 2 3)` or `1:2:3` or `1 2 3;` 302 | or `1:2:3;` all produce the same RDX. That was made for "backward 303 | compatibility" with other notations, primarily JSON. 304 | The bracket notation is safer, but the colon notation is clearer 305 | in many use cases, especially those involving nested tuples 306 | (remember LISP). 307 | 308 | ### Size limits 309 | 310 | Integers and Floats are up to `8` bytes long, IDs are `0` to `16` bytes. 311 | The general rule for the maximum length of other records is to be limited by the bit capacity of the length field. 312 | Short TLV records have payloads of `0` to `0xff` bytes, including the stamp-length byte and the stamp. 313 | Long TLV records have payloads of `0x100` to `0xffffffff` bytes. 314 | String and Term lengths are only limited by the maximum record length. 315 | Same applies to any container element. 316 | In theory, that compromises commutativity of the merge, as a container may exceed the maximum length at some stage. 317 | In practice, it is recommended to have ingress data parcel size limits set well below 4GB. 318 | 319 | ### Metadata stripping and tombstone handling 320 | 321 | The least significant bit of the revision number is used as 322 | a *tombstone* flag. As in many MVCC and distributed systems, 323 | deleted elements just get marked and stay around. 324 | 325 | Metadata stripping is the process of removing any tombstones, 326 | any non-zero timestamps and other parts of a document that have 327 | no effect, e.g. empty tuples in an Eulerian container. 328 | 329 | The result is the "user-perceived" state of the document. 330 | 331 | ### Normalization 332 | 333 | Normalization is the process of bringing a document to its 334 | canonic state: redoing overlong codings, sorting PLEX elements 335 | properly and similar corrections. 336 | 337 | Violations that can not be obviously corrected, e.g. bad UTF-8 338 | or NaNs, must trigger an error/exception. 339 | 340 | [a]: http://google.com/?q=automerge 341 | [b]: https://en.wikipedia.org/wiki/LEB128 342 | [c]: https://github.com/drpcorg/chotki 343 | [d]: https://en.wikipedia.org/wiki/RDX 344 | [g]: https://protobuf.dev/programming-guides/encoding/ 345 | [f]: https://seriot.ch/projects/parsing_json.html 346 | [j]: ./z.md 347 | [l]: https://martin.kleppmann.com/papers/local-first.pdf 348 | [n]: ./DISCONT.md 349 | [m]: https://en.wikipedia.org/wiki/Merge_sort 350 | [p]: https://en.wikipedia.org/wiki/Remote_procedure_call 351 | [r]: https://www.educative.io/answers/how-are-vector-clocks-used-in-dynamo 352 | [t]: https://en.wikipedia.org/wiki/Type%E2%80%93length%E2%80%93value 353 | [v]: https://en.wikipedia.org/wiki/Version_vector 354 | [w]: https://en.wikipedia.org/wiki/CRDT 355 | [x]: https://en.wikipedia.org/wiki/Causal_consistency 356 | [z]: ./zipint.go 357 | -------------------------------------------------------------------------------- /jdr.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "math" 7 | "strconv" 8 | ) 9 | 10 | type JDRstate struct { 11 | jdr []byte 12 | rdx []byte 13 | 14 | stack Marks 15 | val []byte 16 | 17 | line int 18 | col int 19 | } 20 | 21 | var RDXEmptyTuple []byte = []byte{'p', 1, 0} 22 | 23 | var ErrBadJDRSyntax = errors.New("bad JDR syntax") 24 | var ErrBadJDRNesting = errors.New("bad JDR syntax (nesting)") 25 | 26 | const inlineTuple = 'p' 27 | 28 | const ( 29 | StyleStampSpace = Style(256 << iota) 30 | StyleStamps 31 | StyleUseComma 32 | StyleUseSpace 33 | StyleUseLF 34 | StyleUseLFSparingly 35 | StyleTopCommaNL 36 | StyleIndentTab 37 | StyleIndentSpace4 38 | StyleTrailingComma 39 | StyleBracketTuples 40 | StyleShortInlineTuples 41 | StyleSkipComma 42 | StyleYell 43 | ) 44 | 45 | var JDRNormalStyle = NewStyle( 46 | StyleUseComma, 47 | StyleIndentSpace4, 48 | StyleUseLF, 49 | StyleShortInlineTuples, 50 | StyleUseLFSparingly, 51 | ) 52 | 53 | type Style uint64 54 | 55 | func (style Style) Has(bit Style) bool { 56 | return 0 != (style & bit) 57 | } 58 | 59 | func (style Style) Without(bits Style) Style { 60 | return style & ^bits 61 | } 62 | 63 | func (style Style) With(bits Style) Style { 64 | return style | bits 65 | } 66 | 67 | func (style *Style) Add(bits Style) { 68 | *style |= bits 69 | } 70 | 71 | func (style *Style) Del(bits Style) { 72 | *style &= ^bits 73 | } 74 | 75 | func NewStyle(bits ...Style) (ret Style) { 76 | for _, s := range bits { 77 | ret |= s 78 | } 79 | return 80 | } 81 | 82 | const StyleCommaSpacers = StyleUseComma | StyleIndentSpace4 | StyleIndentTab 83 | 84 | func JDRonNL(tok []byte, state *JDRstate) error { 85 | state.line++ 86 | return nil 87 | } 88 | func JDRonUtf8cp1(tok []byte, state *JDRstate) error { return nil } 89 | 90 | func JDRonUtf8cp2(tok []byte, state *JDRstate) error { 91 | var cp uint32 92 | cp = uint32(tok[0]) & 0x1f 93 | cp = (cp << 6) | (uint32(tok[1]) & 0x3f) 94 | if cp >= 0xd800 && cp < 0xe000 { 95 | return ErrBadUtf8 96 | } 97 | return nil 98 | } 99 | 100 | func JDRonUtf8cp3(tok []byte, state *JDRstate) error { 101 | var cp uint32 102 | cp = uint32(tok[0]) & 0x1f 103 | cp = (cp << 6) | uint32(tok[1])&0x3f 104 | cp = (cp << 6) | uint32(tok[2])&0x3f 105 | if cp >= 0xd800 && cp < 0xe000 { 106 | return ErrBadUtf8 107 | } 108 | return nil 109 | } 110 | func JDRonUtf8cp4(tok []byte, state *JDRstate) error { 111 | // TODO codepoint ranges 112 | return nil 113 | } 114 | 115 | func JDRonFIRST0(tok []byte, state *JDRstate, lit byte) (err error) { 116 | if state.Line().Lit == inlineTuple && state.Line().Pre != ':' { 117 | err = closeInlineTuple(state) 118 | } 119 | state.Line().LastElement = len(state.rdx) 120 | state.rdx = OpenTLV(state.rdx, lit, &state.stack) 121 | state.val = tok 122 | return 123 | } 124 | 125 | func JDRonInt(tok []byte, state *JDRstate) error { return JDRonFIRST0(tok, state, LitInteger) } 126 | 127 | func JDRonFloat(tok []byte, state *JDRstate) error { return JDRonFIRST0(tok, state, LitFloat) } 128 | 129 | func JDRonTerm(tok []byte, state *JDRstate) error { return JDRonFIRST0(tok, state, LitTerm) } 130 | 131 | func JDRonRef(tok []byte, state *JDRstate) error { return JDRonFIRST0(tok, state, LitReference) } 132 | 133 | func JDRonString(tok []byte, state *JDRstate) error { return JDRonFIRST0(tok, state, LitString) } 134 | 135 | func JDRonMLString(tok []byte, state *JDRstate) error { return JDRonFIRST0(tok, state, LitString) } 136 | 137 | func JDRonStamp(tok []byte, state *JDRstate) error { 138 | idstr := tok[1:] 139 | id, err := NewID(idstr) 140 | if err != nil { 141 | return err 142 | } 143 | zip := ZipID(id) 144 | state.rdx = append(state.rdx, byte(len(zip))) 145 | state.rdx = append(state.rdx, zip...) 146 | state.Line().LastElement = len(state.rdx) 147 | state.Line().LastBreak = len(state.rdx) 148 | return nil 149 | } 150 | 151 | func JDRonNoStamp(tok []byte, state *JDRstate) error { 152 | state.rdx = append(state.rdx, 0) 153 | state.Line().LastElement = len(state.rdx) 154 | state.Line().LastBreak = len(state.rdx) 155 | return nil 156 | } 157 | 158 | func JDRonFIRST(tok []byte, state *JDRstate) (err error) { 159 | lit := state.Line().Lit 160 | switch lit { 161 | case LitFloat: 162 | f, _ := strconv.ParseFloat(string(state.val), 64) 163 | if math.IsInf(f, 0) { 164 | if math.IsInf(f, 1) { 165 | f = math.MaxFloat64 166 | } else { 167 | f = -math.MaxFloat64 168 | } 169 | } 170 | state.rdx = append(state.rdx, ZipFloat64(f)...) 171 | case LitInteger: 172 | i, _ := strconv.ParseInt(string(state.val), 10, 64) 173 | state.rdx = append(state.rdx, ZipInt64(i)...) 174 | case LitReference: 175 | id, e := NewID(state.val) 176 | if e != nil { 177 | return e 178 | } 179 | state.rdx = append(state.rdx, ZipID(id)...) 180 | case LitString: 181 | state.rdx = appendUnescaped(state.rdx, state.val[1:len(state.val)-1]) 182 | case LitTerm: 183 | state.rdx = append(state.rdx, state.val...) 184 | } 185 | state.rdx, err = CloseTLV(state.rdx, lit, &state.stack) 186 | state.Line().Pre = '1' 187 | return err 188 | } 189 | 190 | // . . . 191 | 192 | // the :; notation creates "it was a tuple" situation 193 | func retroOpenTuple(state *JDRstate, pos int) (err error) { 194 | if len(state.stack) == 0 { 195 | return ErrBadJDRNesting 196 | } 197 | if len(state.rdx) < pos { 198 | return ErrBadState 199 | } 200 | state.rdx = append(state.rdx, 0, 0, 0) 201 | copy(state.rdx[pos+3:len(state.rdx)], state.rdx[pos:len(state.rdx)-3]) 202 | state.rdx[pos] = 'p' 203 | state.rdx[pos+1] = 0 204 | state.rdx[pos+2] = 0 205 | state.stack = append(state.stack, Mark{ 206 | Start: pos, 207 | Lit: 'p', 208 | }) 209 | return 210 | } 211 | 212 | func appendRDXEmptyTuple(state *JDRstate) (err error) { 213 | state.rdx = append(state.rdx, RDXEmptyTuple...) 214 | last := &state.stack[len(state.stack)-1] 215 | last.Pre = 'p' 216 | return nil 217 | } 218 | 219 | func closeInlineTuple(state *JDRstate) (err error) { 220 | if state.Line().Pre == ':' { 221 | state.rdx = append(state.rdx, RDXEmptyTuple...) 222 | } 223 | state.rdx, err = CloseTLV(state.rdx, LitTuple, &state.stack) 224 | if len(state.stack) == 0 { 225 | return ErrBadState 226 | } 227 | state.Line().LastBreak = len(state.rdx) 228 | state.Line().Pre = 'p' 229 | return 230 | } 231 | 232 | func JDRonSemicolon(tok []byte, state *JDRstate) (err error) { 233 | if state.Line().Lit != inlineTuple { 234 | pos := state.Line().LastBreak 235 | if err = retroOpenTuple(state, pos); err != nil { 236 | return 237 | } 238 | } 239 | if err = closeInlineTuple(state); err != nil { 240 | return 241 | } 242 | state.Line().LastBreak = len(state.rdx) 243 | return 244 | } 245 | 246 | func JDRonOpenPLEX(tok []byte, state *JDRstate, plex byte) (err error) { 247 | if state.Line().Lit == inlineTuple && state.Line().Pre != ':' { 248 | if err = closeInlineTuple(state); err != nil { 249 | return 250 | } 251 | } 252 | state.Line().LastElement = len(state.rdx) 253 | state.rdx = OpenTLV(state.rdx, plex, &state.stack) 254 | state.Line().Pre = 0 255 | return 256 | } 257 | 258 | func (state *JDRstate) Line() *Mark { 259 | return &state.stack[len(state.stack)-1] 260 | } 261 | 262 | func JDRonClosePLEX(tok []byte, state *JDRstate, lit byte) (err error) { 263 | if state.Line().Lit == inlineTuple { 264 | err = closeInlineTuple(state) 265 | } else if state.Line().Pre == ',' { 266 | err = appendRDXEmptyTuple(state) 267 | } 268 | if !IsPLEX(lit) || lit != state.Line().Lit { 269 | err = ErrBadJDRNesting 270 | } 271 | if err != nil { 272 | return err 273 | } 274 | state.rdx, err = CloseTLV(state.rdx, lit, &state.stack) 275 | state.Line().Pre = lit 276 | if err == nil && len(state.stack) == 0 { 277 | err = ErrBadState 278 | } 279 | return err 280 | } 281 | 282 | func JDRonOpenP(tok []byte, state *JDRstate) error { 283 | return JDRonOpenPLEX(tok, state, LitTuple) 284 | } 285 | func JDRonCloseP(tok []byte, state *JDRstate) error { 286 | return JDRonClosePLEX(tok, state, LitTuple) 287 | } 288 | func JDRonOpenL(tok []byte, state *JDRstate) error { 289 | return JDRonOpenPLEX(tok, state, LitLinear) 290 | } 291 | func JDRonCloseL(tok []byte, state *JDRstate) error { 292 | return JDRonClosePLEX(tok, state, LitLinear) 293 | } 294 | func JDRonOpenE(tok []byte, state *JDRstate) error { 295 | return JDRonOpenPLEX(tok, state, LitEuler) 296 | } 297 | func JDRonCloseE(tok []byte, state *JDRstate) error { 298 | return JDRonClosePLEX(tok, state, LitEuler) 299 | } 300 | func JDRonOpenX(tok []byte, state *JDRstate) error { 301 | return JDRonOpenPLEX(tok, state, LitMultix) 302 | } 303 | func JDRonCloseX(tok []byte, state *JDRstate) error { 304 | return JDRonClosePLEX(tok, state, LitMultix) 305 | } 306 | func JDRonComma(tok []byte, state *JDRstate) (err error) { 307 | if state.Line().Lit == inlineTuple { 308 | err = closeInlineTuple(state) 309 | } 310 | pre := state.Line().Pre 311 | if pre == 0 || pre == ',' || pre == ';' { 312 | _ = appendRDXEmptyTuple(state) 313 | } 314 | state.Line().LastBreak = len(state.rdx) 315 | state.Line().Pre = ',' 316 | return 317 | } 318 | func JDRonColon(tok []byte, state *JDRstate) (err error) { 319 | if state.Line().Lit != inlineTuple { 320 | pos := state.Line().LastElement 321 | if state.Line().LastBreak > pos { 322 | pos = state.Line().LastBreak 323 | } 324 | err = retroOpenTuple(state, pos) 325 | } else if state.Line().Pre == 0 || state.Line().Pre == ':' { 326 | err = appendRDXEmptyTuple(state) 327 | } 328 | state.Line().Pre = ':' 329 | return err 330 | } 331 | 332 | func JDRonOpen(tok []byte, state *JDRstate) error { return nil } 333 | 334 | func JDRonClose(tok []byte, state *JDRstate) error { return nil } 335 | 336 | func JDRonInter(tok []byte, state *JDRstate) error { return nil } 337 | 338 | func JDRonRoot(tok []byte, state *JDRstate) (err error) { 339 | if state.Line().Lit == inlineTuple { 340 | err = closeInlineTuple(state) 341 | } 342 | return 343 | } 344 | 345 | func appendUnescaped(rdx, jdr []byte) []byte { 346 | for len(jdr) > 0 { 347 | c := jdr[0] 348 | jdr = jdr[1:] 349 | if c != '\\' { 350 | rdx = append(rdx, c) 351 | continue 352 | } 353 | if len(jdr) == 0 { 354 | rdx = append(rdx, c) 355 | continue 356 | } 357 | c = jdr[0] 358 | jdr = jdr[1:] 359 | switch c { 360 | case 't': 361 | rdx = append(rdx, '\t') 362 | case 'r': 363 | rdx = append(rdx, '\r') 364 | case 'n': 365 | rdx = append(rdx, '\n') 366 | case 'b': 367 | rdx = append(rdx, '\b') 368 | case 'f': 369 | rdx = append(rdx, '\f') 370 | case '\\': 371 | rdx = append(rdx, '\\') 372 | case '/': 373 | rdx = append(rdx, '/') 374 | case '"': 375 | rdx = append(rdx, '"') 376 | //case '0': 377 | // rdx = append(rdx, 0) 378 | default: 379 | rdx = append(rdx, c) 380 | } 381 | } 382 | return rdx 383 | } 384 | 385 | func appendEscaped(jdr, val []byte) []byte { 386 | for _, a := range val { 387 | switch a { 388 | case '\t': 389 | jdr = append(jdr, '\\', 't') 390 | case '\r': 391 | jdr = append(jdr, '\\', 'r') 392 | case '\n': 393 | jdr = append(jdr, '\\', 'n') 394 | case '\b': 395 | jdr = append(jdr, '\\', 'b') 396 | case '\f': 397 | jdr = append(jdr, '\\', 'f') 398 | case '\\': 399 | jdr = append(jdr, '\\', '\\') 400 | case '"': 401 | jdr = append(jdr, '\\', '"') 402 | //case 0: 403 | // jdr = append(jdr, '\\', '0') 404 | // TODO \u etc 405 | default: 406 | jdr = append(jdr, a) 407 | } 408 | } 409 | return jdr 410 | } 411 | 412 | func appendJDRStamp(jdr []byte, id ID) []byte { 413 | if id.IsZero() { 414 | return jdr 415 | } 416 | jdr = append(jdr, '@') 417 | jdr = append(jdr, id.RonString()...) 418 | return jdr 419 | } 420 | 421 | func IsAllTerm(rdx []byte) bool { 422 | var err error 423 | for len(rdx) > 0 && err == nil { 424 | var lit byte 425 | lit, _, _, rdx, err = ReadRDX(rdx) 426 | if lit != LitTerm { 427 | return false 428 | } 429 | } 430 | return err == nil 431 | } 432 | 433 | func IsTupleAllFIRST(rdx []byte) bool { 434 | var err error 435 | l := 0 436 | for len(rdx) > 0 && err == nil { 437 | var lit byte 438 | var val []byte 439 | lit, _, val, rdx, err = ReadRDX(rdx) 440 | if IsPLEX(lit) && !(lit == LitTuple && len(val) == 0) { 441 | return false 442 | } 443 | l++ 444 | } 445 | if l < 2 { 446 | return false 447 | } 448 | return true 449 | } 450 | 451 | func appendCRLF(jdr []byte, style Style) []byte { 452 | if !style.Has(StyleUseLF) { 453 | return jdr 454 | } 455 | jdr = append(jdr, '\n') 456 | if style.Has(StyleIndentTab) { 457 | for i := 0; i < int(style&0xff); i++ { 458 | jdr = append(jdr, '\t') 459 | } 460 | } else if style.Has(StyleIndentSpace4) { 461 | for i := 0; i < int(style&0xff); i++ { 462 | jdr = append(jdr, ' ', ' ', ' ', ' ') 463 | } 464 | } 465 | return jdr 466 | } 467 | 468 | func appendJDRList(jdr, rdx []byte, style Style) (res []byte, err error) { 469 | commanl := style.Has(StyleUseComma) || (style.Has(StyleTopCommaNL) && (style&0xff) == 0) 470 | for len(rdx) > 0 && err == nil { 471 | jdr, rdx, err = WriteJDR(jdr, rdx, style) 472 | if len(rdx) > 0 || style.Has(StyleTrailingComma) { 473 | jdr = append(jdr, ',') 474 | if commanl { 475 | jdr = appendCRLF(jdr, style) 476 | } 477 | } 478 | } 479 | if commanl { 480 | jdr = append(jdr, '\n') 481 | } 482 | return jdr, err 483 | } 484 | 485 | func appendInlineTuple(jdr, rdx []byte, style Style) (res []byte, err error) { 486 | res = jdr 487 | for len(rdx) > 0 && err == nil { 488 | res, rdx, err = WriteJDR(res, rdx, (style|StyleBracketTuples)+1) 489 | if len(rdx) > 0 { 490 | res = append(res, ':') 491 | } 492 | } 493 | return res, err 494 | } 495 | 496 | func renderFIRSTElementJDR(pre []byte, rdx Iter) (jdr []byte) { 497 | jdr = pre 498 | switch rdx.Lit() { 499 | case LitFloat: 500 | f := rdx.Float() 501 | jdr = strconv.AppendFloat(jdr, float64(f), 'e', -1, 64) 502 | jdr = appendJDRStamp(jdr, rdx.ID()) 503 | case LitInteger: 504 | i := rdx.Integer() 505 | jdr = strconv.AppendInt(jdr, int64(i), 10) 506 | jdr = appendJDRStamp(jdr, rdx.ID()) 507 | case LitReference: 508 | i := rdx.Reference() 509 | jdr = append(jdr, i.FormalString()...) 510 | jdr = appendJDRStamp(jdr, rdx.ID()) 511 | case LitString: 512 | jdr = append(jdr, '"') 513 | jdr = appendEscaped(jdr, rdx.Value()) 514 | jdr = append(jdr, '"') 515 | jdr = appendJDRStamp(jdr, rdx.ID()) 516 | case LitTerm: 517 | jdr = append(jdr, rdx.Value()...) 518 | jdr = appendJDRStamp(jdr, rdx.ID()) 519 | default: // ? 520 | } 521 | return jdr 522 | } 523 | 524 | func lastByte(buf []byte) byte { 525 | if len(buf) == 0 { 526 | return 0 527 | } 528 | return buf[len(buf)-1] 529 | } 530 | 531 | func renderJDRList(pre []byte, it Iter, style Style) (jdr []byte) { 532 | jdr = pre 533 | if len(it.Rest()) < 32 && style.Has(StyleUseLFSparingly) { 534 | style.Del(StyleUseLF) 535 | } 536 | for it.Read() { 537 | if style.Has(StyleUseLF) { 538 | jdr = appendCRLF(jdr, style) 539 | } 540 | if IsPLEX(it.Lit()) { 541 | if style.Has(StyleYell) && IsYellTuple(it) { 542 | jdr = renderYellTupleJDR(jdr, it, style) 543 | } else if it.Lit() == LitTuple && style.Has(StyleShortInlineTuples) && isShortishTuple(it) { 544 | jdr = renderInlineTupleJDR(jdr, it, style) 545 | } else { 546 | jdr = renderPLEXElementJDR(jdr, it, style) 547 | } 548 | } else { 549 | jdr = renderFIRSTElementJDR(jdr, it) 550 | } 551 | if lastByte(jdr) == ';' { 552 | } else if len(it.Rest()) > 0 { 553 | if style.Has(StyleUseComma) { 554 | jdr = append(jdr, ',') 555 | } 556 | if 0 == (style&StyleUseComma) || style.Has(StyleUseSpace) { 557 | jdr = append(jdr, ' ') 558 | } 559 | } else { 560 | if style.Has(StyleTrailingComma) { 561 | jdr = append(jdr, ',') 562 | } 563 | } 564 | } 565 | if style.Has(StyleUseLF) { 566 | if 0 != (style & 0xff) { 567 | style -= 1 568 | } 569 | jdr = appendCRLF(jdr, style) 570 | } 571 | return 572 | } 573 | 574 | func renderPLEXElementJDR(pre []byte, rdx Iter, style Style) (jdr []byte) { 575 | var oc, cc byte 576 | switch rdx.Lit() { 577 | case LitTuple: 578 | oc, cc = '(', ')' 579 | case LitLinear: 580 | oc, cc = '[', ']' 581 | case LitEuler: 582 | oc, cc = '{', '}' 583 | case LitMultix: 584 | oc, cc = '<', '>' 585 | default: 586 | // ? 587 | } 588 | jdr = pre 589 | jdr = append(jdr, oc) 590 | if !rdx.ID().IsZero() { 591 | jdr = appendJDRStamp(jdr, rdx.ID()) 592 | jdr = append(jdr, ' ') 593 | } 594 | in := rdx.Inner() 595 | jdr = renderJDRList(jdr, in, style+1) 596 | jdr = append(jdr, cc) 597 | return 598 | } 599 | 600 | func IsYellTuple(p Iter) bool { 601 | if p.Lit() != LitTuple { 602 | return false 603 | } 604 | in := p.Inner() 605 | if !in.Read() { 606 | return false 607 | } 608 | switch in.Lit() { 609 | case LitTerm: 610 | return len(in.Value()) <= 10 611 | case LitReference: 612 | return true 613 | default: 614 | return false 615 | } 616 | } 617 | 618 | func isShortishTuple(p Iter) bool { 619 | if p.Lit() != LitTuple || len(p.Value()) > 64 { 620 | return false 621 | } 622 | in := p.Inner() 623 | c := 0 624 | for in.Read() { 625 | if IsPLEX(in.Lit()) && in.Lit() != LitTuple { // todo 626 | return false 627 | } 628 | c++ 629 | } 630 | return c > 1 631 | } 632 | 633 | func renderYellTupleJDR(pre []byte, rdx Iter, style Style) (jdr []byte) { 634 | in := rdx.Inner() 635 | in.Read() 636 | jdr = append(pre, in.String()...) 637 | mask := NewStyle(StyleShortInlineTuples, StyleUseLF, StyleYell) 638 | if in.Peek() != LitTuple { 639 | jdr = append(jdr, ' ') 640 | mask = mask.With(StyleUseComma) 641 | } 642 | style2 := style & ^mask 643 | jdr = renderJDRList(jdr, in, style2) 644 | jdr = append(jdr, ';') 645 | return 646 | } 647 | 648 | func renderInlineTupleJDR(pre []byte, rdx Iter, style Style) (jdr []byte) { 649 | jdr = pre 650 | mask := NewStyle(StyleShortInlineTuples, StyleUseLF) 651 | style2 := style & ^mask 652 | in := rdx.Inner() 653 | if len(in.Rest()) == 0 { 654 | return append(pre, '(', ')') 655 | } 656 | for in.Read() { 657 | if IsPLEX(in.Lit()) { 658 | if !bytes.Equal(in.Record(), RDXEmptyTuple) { 659 | jdr = renderPLEXElementJDR(jdr, in, style2) 660 | } 661 | } else { 662 | jdr = renderFIRSTElementJDR(jdr, in) 663 | } 664 | if len(in.Rest()) > 0 { 665 | jdr = append(jdr, ':') 666 | } 667 | } 668 | return jdr 669 | } 670 | 671 | func RenderJDR(rdx Stream, style Style) (jdr []byte) { 672 | it := NewIter(rdx) 673 | return renderJDRList(nil, it, style) 674 | } 675 | 676 | func WriteJDR(jdr, rdx []byte, style Style) (jdr2, rest []byte, err error) { 677 | var lit byte 678 | var id ID 679 | var val []byte 680 | lit, id, val, rest, err = ReadRDX(rdx) 681 | if err != nil { 682 | return 683 | } 684 | 685 | switch lit { 686 | case LitFloat: 687 | f := UnzipFloat64(val) 688 | jdr = strconv.AppendFloat(jdr, f, 'e', -1, 64) 689 | jdr = appendJDRStamp(jdr, id) 690 | case LitInteger: 691 | i := UnzipInt64(val) 692 | jdr = strconv.AppendInt(jdr, i, 10) 693 | jdr = appendJDRStamp(jdr, id) 694 | case LitReference: 695 | i := UnzipID(val) 696 | jdr = append(jdr, i.RonString()...) 697 | jdr = appendJDRStamp(jdr, id) 698 | case LitString: 699 | jdr = append(jdr, '"') 700 | jdr = appendEscaped(jdr, val) 701 | jdr = append(jdr, '"') 702 | jdr = appendJDRStamp(jdr, id) 703 | case LitTerm: 704 | jdr = append(jdr, val...) 705 | jdr = appendJDRStamp(jdr, id) 706 | case LitTuple: 707 | if style.Has(StyleBracketTuples) || !IsTupleAllFIRST(val) || !id.IsZero() { 708 | jdr = append(jdr, '(') 709 | if !id.IsZero() { 710 | jdr = appendJDRStamp(jdr, id) 711 | jdr = append(jdr, ' ') 712 | } 713 | jdr, err = appendJDRList(jdr, val, style+1) 714 | jdr = append(jdr, ')') 715 | } else { 716 | jdr, err = appendInlineTuple(jdr, val, style) 717 | } 718 | case LitLinear: 719 | jdr = append(jdr, '[') 720 | if !id.IsZero() { 721 | jdr = appendJDRStamp(jdr, id) 722 | jdr = append(jdr, ' ') 723 | } 724 | jdr, err = appendJDRList(jdr, val, style+1) 725 | jdr = append(jdr, ']') 726 | case LitEuler: 727 | jdr = append(jdr, '{') 728 | if !id.IsZero() { 729 | jdr = appendJDRStamp(jdr, id) 730 | jdr = append(jdr, ' ') 731 | } 732 | jdr, err = appendJDRList(jdr, val, style+1) 733 | jdr = append(jdr, '}') 734 | case LitMultix: 735 | jdr = append(jdr, '<') 736 | if !id.IsZero() { 737 | jdr = appendJDRStamp(jdr, id) 738 | jdr = append(jdr, ' ') 739 | } 740 | jdr, err = appendJDRList(jdr, val, style+1) 741 | jdr = append(jdr, '>') 742 | default: 743 | err = ErrBadRecord 744 | } 745 | jdr2 = jdr 746 | return 747 | } 748 | 749 | func WriteAllJDR(jdr, rdx []byte, style Style) (jdr2 []byte, err error) { 750 | jdr2 = jdr 751 | for len(rdx) > 0 && err == nil { 752 | jdr2, rdx, err = WriteJDR(jdr2, rdx, style) 753 | if len(rdx) > 0 { 754 | jdr2 = append(jdr2, ' ') 755 | } 756 | } 757 | return jdr2, err 758 | } 759 | 760 | // ParseJDR parses with no normalization, hence return type is []byte not Stream 761 | func ParseJDR(jdr []byte) (rdx []byte, err error) { 762 | state := JDRstate{ 763 | jdr: jdr, 764 | stack: make(Marks, 0, MaxNesting+1), 765 | } 766 | state.stack = append(state.stack, Mark{Lit: ' '}) 767 | err = JDRlexer(&state) 768 | if err == nil && (len(state.stack) != 1 || state.stack[0].Lit != ' ') { 769 | err = ErrBadJDRNesting 770 | } 771 | rdx = state.rdx 772 | return 773 | } 774 | 775 | func ParseNormalizeJDR(jdr []byte) (normal Stream, err error) { 776 | var parsed Stream 777 | parsed, err = ParseJDR([]byte(jdr)) 778 | if err != nil { 779 | return 780 | } 781 | normal, err = Normalize(parsed) 782 | return 783 | } 784 | -------------------------------------------------------------------------------- /rdx.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "math" 7 | "math/bits" 8 | "os" 9 | "sort" 10 | "unicode/utf8" 11 | ) 12 | 13 | const ( 14 | LitFloat = 'F' 15 | LitInteger = 'I' 16 | LitReference = 'R' 17 | LitString = 'S' 18 | LitTerm = 'T' 19 | 20 | LitTuple = 'P' 21 | LitLinear = 'L' 22 | LitEuler = 'E' 23 | LitMultix = 'X' 24 | ) 25 | 26 | const MaxInputs = 64 27 | const MaxNesting = 255 28 | 29 | type Stream []byte 30 | 31 | type Float float64 32 | type Integer int64 33 | type String string 34 | type Term []byte 35 | 36 | var ( 37 | ErrBadRDXRecord = errors.New("bad Stream record format") 38 | ErrWrongRDXRecordType = errors.New("wrong Stream record type") 39 | ErrBadUtf8 = errors.New("bad UTF8 codepoint") 40 | ErrBadState = errors.New("bad state") 41 | ErrBadOrder = errors.New("bad Stream order") 42 | ErrEoS = errors.New("end of file") 43 | ) 44 | 45 | func IsPLEX(lit byte) bool { 46 | return lit == LitTuple || lit == LitLinear || lit == LitEuler || lit == LitMultix 47 | } 48 | 49 | func IsFIRST(lit byte) bool { 50 | return lit == LitFloat || lit == LitInteger || lit == LitReference || lit == LitString || lit == LitTerm 51 | } 52 | 53 | func ReadRDX(data []byte) (lit byte, id ID, value, rest []byte, err error) { 54 | var pair []byte 55 | lit, pair, value, rest, err = ReadTLKV(data) 56 | if err == nil { 57 | id.Seq, id.Src = UnzipUint64Pair(pair) 58 | } 59 | return 60 | } 61 | 62 | func WriteRDX(data []byte, lit byte, id ID, value []byte) []byte { 63 | pair := ZipID(id) 64 | return WriteTLKV(data, lit, pair, value) 65 | } 66 | 67 | type Merger func(data []byte, bare Heap) ([]byte, error) 68 | 69 | func mergeValuesF(data []byte, bare [][]byte) ([]byte, error) { 70 | var mx float64 71 | var win []byte 72 | for i, b := range bare { 73 | n := UnzipFloat64(b) 74 | if i == 0 || n > mx { 75 | mx = n 76 | win = b 77 | } 78 | } 79 | data = append(data, win...) 80 | return data, nil 81 | } 82 | 83 | func mergeValuesI(data []byte, bare [][]byte) ([]byte, error) { 84 | var max int64 85 | var win []byte 86 | for i, b := range bare { 87 | n := UnzipInt64(b) 88 | if i == 0 || n > max { 89 | max = n 90 | win = b 91 | } 92 | } 93 | data = append(data, win...) 94 | return data, nil 95 | } 96 | 97 | func mergeValuesR(data []byte, bare [][]byte) ([]byte, error) { 98 | var max ID 99 | var win []byte 100 | for i, b := range bare { 101 | n := UnzipID(b) 102 | if i == 0 || max.Compare(n) < 0 { 103 | max = n 104 | win = b 105 | } 106 | } 107 | data = append(data, win...) 108 | return data, nil 109 | } 110 | 111 | func mergeValuesS(data []byte, bare [][]byte) ([]byte, error) { 112 | var win []byte 113 | for i, b := range bare { 114 | if i == 0 || bytes.Compare(win, b) < 0 { 115 | win = b 116 | } 117 | } 118 | data = append(data, win...) 119 | return data, nil 120 | } 121 | 122 | func mergeValuesT(data []byte, bare [][]byte) ([]byte, error) { 123 | return mergeValuesS(data, bare) 124 | } 125 | 126 | func Merge(data []byte, bare [][]byte) (ret []byte, err error) { 127 | return mergeElementsP(data, bare) 128 | } 129 | 130 | func mergeElementsP(data []byte, bare [][]byte) (ret []byte, err error) { 131 | return HeapMerge(data, bare, CompareTuple) 132 | } 133 | 134 | func mergeElementsL(data []byte, bare [][]byte) ([]byte, error) { 135 | return HeapMerge(data, bare, CompareLinear) 136 | } 137 | 138 | func mergeElementsE(data []byte, bare [][]byte) ([]byte, error) { 139 | return HeapMerge(data, bare, CompareEuler) 140 | } 141 | 142 | func mergeElementsX(data []byte, bare [][]byte) ([]byte, error) { 143 | return HeapMerge(data, bare, CompareMultix) 144 | } 145 | 146 | // same element, maybe different revision 147 | func IsSame(a, b *Iter) bool { 148 | return a.Lit() == b.Lit() && a.ID().Base() == b.ID().Base() 149 | } 150 | 151 | func MergeSameSpotElements(data []byte, heap Heap) (ret []byte, err error) { 152 | eq := 1 153 | id := heap[0].ID() 154 | for i := 1; i < len(heap); i++ { 155 | var z int 156 | if IsSame(&heap[0], &heap[i]) && IsPLEX(heap[0].Lit()) { 157 | z = Eq 158 | } else { 159 | z = CompareLWW(&heap[0], &heap[i]) 160 | } 161 | if z < Eq { 162 | heap[0], heap[i] = heap[i], heap[0] 163 | id = heap[0].ID() 164 | eq = 1 165 | } else if z > Eq { 166 | pl := len(heap) - 1 167 | heap[pl], heap[i] = heap[i], heap[pl] 168 | heap = heap[:pl] 169 | i-- 170 | } else { 171 | heap[eq], heap[i] = heap[i], heap[eq] 172 | if id.Less(heap[eq].ID()) { 173 | id = heap[eq].ID() 174 | } 175 | eq++ 176 | } 177 | } 178 | eqs := heap[:eq] 179 | lit := eqs[0].Lit() 180 | vals := make([][]byte, 0, MaxInputs) 181 | stack := make(Marks, 0, 16) 182 | ret = OpenTLV(data, lit, &stack) 183 | key := ZipID(id) 184 | ret = append(ret, byte(len(key))) 185 | ret = append(ret, key...) // TODO 186 | for _, val := range eqs { 187 | vals = append(vals, val.Value()) 188 | } // FIXME 1 189 | switch lit { 190 | case LitFloat: 191 | ret, err = mergeValuesF(ret, vals) 192 | case LitInteger: 193 | ret, err = mergeValuesI(ret, vals) 194 | case LitReference: 195 | ret, err = mergeValuesR(ret, vals) 196 | case LitString: 197 | ret, err = mergeValuesS(ret, vals) 198 | case LitTerm: 199 | ret, err = mergeValuesT(ret, vals) 200 | case LitTuple: 201 | ret, err = mergeElementsP(ret, vals) 202 | case LitLinear: 203 | ret, err = mergeElementsL(ret, vals) 204 | case LitEuler: 205 | ret, err = mergeElementsE(ret, vals) 206 | case LitMultix: 207 | ret, err = mergeElementsX(ret, vals) 208 | default: 209 | ret, err = nil, ErrBadRDXRecord 210 | } 211 | if err == nil { 212 | ret, err = CloseTLV(ret, lit, &stack) 213 | } 214 | return 215 | } 216 | 217 | const ( 218 | Less = -2 219 | LEq = -1 220 | Eq = 0 221 | GrEq = 1 222 | Grtr = 2 223 | ) 224 | 225 | type Compare func(a *Iter, b *Iter) int 226 | 227 | func IsEmptyTuple(a *Iter) bool { 228 | return a.Lit() == LitTuple && len(a.Value()) == 0 229 | } 230 | 231 | func CompareLWW(a *Iter, b *Iter) int { 232 | z := CompareIDRev(a, b) 233 | if z == Eq { 234 | z = CompareType(a, b) 235 | if z == Eq { 236 | z = CompareValue(a, b) 237 | } else if z < Eq { 238 | if IsEmptyTuple(b) { 239 | z = Grtr 240 | } 241 | } else { // z > Eq 242 | if IsEmptyTuple(a) { 243 | z = Less 244 | } 245 | } 246 | } 247 | return z 248 | } 249 | 250 | func CompareFloat(a *Iter, b *Iter) int { 251 | af := UnzipFloat64(a.Value()) 252 | bf := UnzipFloat64(b.Value()) 253 | if af == bf { 254 | return Eq 255 | } else if af < bf { 256 | return Less 257 | } else { 258 | return Grtr 259 | } 260 | } 261 | 262 | func CompareInteger(a *Iter, b *Iter) int { 263 | af := UnzipInt64(a.Value()) 264 | bf := UnzipInt64(b.Value()) 265 | if af == bf { 266 | return Eq 267 | } else if af < bf { 268 | return Less 269 | } else { 270 | return Grtr 271 | } 272 | } 273 | 274 | func CompareReference(a *Iter, b *Iter) int { 275 | aid := UnzipID(a.Value()) 276 | bid := UnzipID(b.Value()) 277 | return aid.Compare(bid) 278 | } 279 | 280 | func CompareString(a *Iter, b *Iter) int { 281 | return bytes.Compare(a.Value(), b.Value()) * 2 282 | } 283 | 284 | func CompareTerm(a *Iter, b *Iter) int { 285 | return CompareString(a, b) 286 | } 287 | 288 | func UnwrapTuple(a *Iter) *Iter { 289 | b := NewIter(a.Value()) 290 | b.Read() 291 | return &b 292 | } 293 | 294 | func CompareTuple(a *Iter, b *Iter) int { 295 | return Eq 296 | } 297 | 298 | func CompareLinear(a *Iter, b *Iter) int { 299 | aa := NewRon60(a.ID().Seq >> 6) 300 | bb := NewRon60(b.ID().Seq >> 6) 301 | if aa.Less(bb) { 302 | return Less 303 | } else if bb.Less(aa) { 304 | return Grtr 305 | } 306 | if a.ID().Src < b.ID().Src { 307 | return Less 308 | } else if a.ID().Src > b.ID().Src { 309 | return Grtr 310 | } else { 311 | return Eq 312 | } 313 | } 314 | 315 | func CompareType(a *Iter, b *Iter) int { 316 | al := a.Lit() 317 | bl := b.Lit() 318 | if al == bl { 319 | return Eq 320 | } 321 | ap := IsPLEX(al) 322 | bp := IsPLEX(bl) 323 | if ap != bp { 324 | if ap { 325 | return Grtr 326 | } else { 327 | return Less 328 | } 329 | } 330 | if al < bl { 331 | return Less 332 | } else { 333 | return Grtr 334 | } 335 | } 336 | 337 | const SeqMask = ^uint64(0x3f) 338 | 339 | func CompareRevID(ai *Iter, bi *Iter) int { 340 | a := ai.ID() 341 | b := bi.ID() 342 | a.Seq = a.Seq & SeqMask 343 | b.Seq = b.Seq & SeqMask 344 | if a.Seq < b.Seq { 345 | return Less 346 | } else if a.Seq > b.Seq { 347 | return Grtr 348 | } else if a.Src < b.Src { 349 | return Less 350 | } else if a.Src > b.Src { 351 | return Grtr 352 | } else { 353 | return Eq 354 | } 355 | } 356 | 357 | func CompareID(ai *Iter, bi *Iter) int { 358 | return ai.ID().Compare(bi.ID()) 359 | } 360 | 361 | func CompareIDRev(ai *Iter, bi *Iter) int { 362 | return ai.ID().RevCompare(bi.ID()) 363 | } 364 | 365 | func CompareValue(a *Iter, b *Iter) int { 366 | if a.IsEmpty() || b.IsEmpty() { 367 | if a.IsEmpty() { 368 | if b.IsEmpty() { 369 | return Eq 370 | } else { 371 | return Less 372 | } 373 | } else { 374 | return Grtr 375 | } 376 | } 377 | al := a.Lit() 378 | bl := b.Lit() 379 | if al != bl { 380 | return CompareType(a, b) 381 | } 382 | switch al { 383 | case LitFloat: 384 | return CompareFloat(a, b) 385 | case LitInteger: 386 | return CompareInteger(a, b) 387 | case LitReference: 388 | return CompareReference(a, b) 389 | case LitString: 390 | return CompareString(a, b) 391 | case LitTerm: 392 | return CompareTerm(a, b) 393 | case LitTuple: 394 | return CompareID(a, b) 395 | case LitLinear: 396 | return CompareID(a, b) 397 | case LitEuler: 398 | return CompareID(a, b) 399 | case LitMultix: 400 | return CompareID(a, b) 401 | default: 402 | return Eq 403 | } 404 | } 405 | 406 | func CompareEuler(a *Iter, b *Iter) int { 407 | if a.Lit() == LitTuple { 408 | aa := a.Inner() 409 | aa.Read() 410 | a = &aa 411 | } 412 | if b.Lit() == LitTuple { 413 | bb := b.Inner() 414 | bb.Read() 415 | b = &bb 416 | } 417 | return CompareValue(a, b) 418 | } 419 | 420 | func CompareMultix(a *Iter, b *Iter) int { 421 | if a.ID().Src < b.ID().Src { 422 | return Less 423 | } else if a.ID().Src > b.ID().Src { 424 | return Grtr 425 | } 426 | return Eq 427 | } 428 | 429 | func ReadTerm(rdx []byte) (val []byte, id ID, rest []byte, err error) { 430 | var lit byte 431 | lit, id, val, rest, err = ReadRDX(rdx) 432 | if err == nil && lit != LitTerm { 433 | err = ErrWrongRDXRecordType 434 | } 435 | return 436 | } 437 | 438 | func ReadString(rdx []byte) (val string, id ID, rest []byte, err error) { 439 | var lit byte 440 | var v []byte 441 | lit, id, v, rest, err = ReadRDX(rdx) 442 | if err == nil && lit != LitString { 443 | err = ErrWrongRDXRecordType 444 | } else { 445 | val = string(v) 446 | } 447 | return 448 | } 449 | 450 | func ReadID(rdx []byte) (val, id ID, rest []byte, err error) { 451 | var v []byte 452 | var lit byte 453 | lit, id, v, rest, err = ReadRDX(rdx) 454 | if err != nil { 455 | return 456 | } 457 | if lit != LitReference || len(v) > 16 { 458 | err = ErrBadRecord 459 | return 460 | } 461 | val = UnzipID(v) 462 | return 463 | } 464 | 465 | func ReadInteger(rdx []byte) (val int64, id ID, rest []byte, err error) { 466 | var v []byte 467 | var lit byte 468 | lit, id, v, rest, err = ReadRDX(rdx) 469 | if err != nil { 470 | return 471 | } 472 | if lit != LitInteger || len(v) > 8 { 473 | err = ErrBadRecord 474 | return 475 | } 476 | val = UnzipInt64(v) 477 | return 478 | } 479 | 480 | func TopBit(v uint64) uint64 { 481 | l := bits.LeadingZeros64(v) 482 | return uint64(1) << (63 - l) 483 | } 484 | 485 | func AppendInteger(data []byte, val Integer) []byte { 486 | b := ZipInteger(val) 487 | return WriteTLKV(data, LitInteger, nil, b) 488 | } 489 | 490 | func AppendFloat(data []byte, val Float) []byte { 491 | b := ZipFloat(val) 492 | return WriteTLKV(data, LitInteger, nil, b) 493 | } 494 | 495 | func MakeString(term string) Stream { 496 | return AppendString(nil, []byte(term)) 497 | } 498 | 499 | func MakeTerm(term string) Stream { 500 | return AppendTerm(nil, []byte(term)) 501 | } 502 | 503 | func MakeTuple(id ID, val Stream) Stream { 504 | return Stream{}.AppendTuple(id, val) 505 | } 506 | 507 | func MakeEuler(id ID, val Stream) Stream { 508 | return Stream{}.AppendEuler(id, val) 509 | } 510 | 511 | func F(id ID, val Float) Stream { 512 | return WriteRDX(nil, LitFloat, id, ZipFloat(val)) 513 | } 514 | func I(id ID, val Integer) Stream { 515 | return WriteRDX(nil, LitInteger, id, ZipInteger(val)) 516 | } 517 | func R(id ID, val ID) Stream { 518 | return WriteRDX(nil, LitReference, id, ZipID(val)) 519 | } 520 | func S(id ID, val string) Stream { 521 | return WriteRDX(nil, LitString, id, []byte(val)) 522 | } 523 | func T(id ID, val string) Stream { 524 | return WriteRDX(nil, LitTerm, id, []byte(val)) 525 | } 526 | 527 | func F0(val Float) Stream { 528 | return WriteRDX(nil, LitFloat, ID0, ZipFloat(val)) 529 | } 530 | func I0(val Integer) Stream { 531 | return WriteRDX(nil, LitInteger, ID0, ZipInteger(val)) 532 | } 533 | func R0(val ID) Stream { 534 | return WriteRDX(nil, LitReference, ID0, ZipID(val)) 535 | } 536 | func S0(val string) Stream { 537 | return WriteRDX(nil, LitString, ID0, []byte(val)) 538 | } 539 | func T0(val string) Stream { 540 | return WriteRDX(nil, LitTerm, ID0, []byte(val)) 541 | } 542 | 543 | func MakePLEXOf(lit byte, id ID, val []Stream, z Compare) Stream { 544 | if z != nil { 545 | sort.Slice(val, func(i, j int) bool { 546 | ii := NewIter(val[i]) 547 | jj := NewIter(val[j]) 548 | return z(&ii, &jj) < Eq 549 | }) 550 | } 551 | l := 0 552 | for _, v := range val { 553 | l += len(v) 554 | } 555 | marks := make(Marks, 0, 1) 556 | ret := make(Stream, 0, l+24) 557 | if l <= 0xff { 558 | ret = OpenShortTLV(ret, lit, &marks) // FIXME we know len!!! 559 | } else { 560 | ret = OpenTLV(ret, lit, &marks) 561 | } 562 | zip := ZipID(id) 563 | ret = append(ret, byte(len(zip))) 564 | ret = append(ret, zip...) 565 | for _, v := range val { 566 | ret = append(ret, v...) 567 | } 568 | ret, _ = CloseTLV(ret, lit, &marks) 569 | return ret 570 | } 571 | 572 | func P0(vals ...Stream) Stream { 573 | return MakePLEXOf(LitTuple, ID0, vals, CompareTuple) 574 | } 575 | func L0(val ...Stream) Stream { 576 | return MakePLEXOf(LitLinear, ID0, val, nil) 577 | } 578 | func E0(val ...Stream) Stream { 579 | return MakePLEXOf(LitEuler, ID0, val, CompareEuler) 580 | } 581 | func X0(val ...Stream) Stream { 582 | return MakePLEXOf(LitMultix, ID0, val, CompareMultix) 583 | } 584 | 585 | func P(id ID, val ...Stream) Stream { 586 | return MakePLEXOf(LitTuple, id, val, CompareTuple) 587 | } 588 | func L(id ID, val ...Stream) Stream { 589 | return MakePLEXOf(LitLinear, id, val, nil) 590 | } 591 | func E(id ID, val ...Stream) Stream { 592 | return MakePLEXOf(LitEuler, id, val, CompareEuler) 593 | } 594 | func X(id ID, val ...Stream) Stream { 595 | return MakePLEXOf(LitMultix, id, val, CompareMultix) 596 | } 597 | 598 | func AppendString(data []byte, val []byte) []byte { 599 | return WriteTLKV(data, LitString, nil, val) 600 | } 601 | 602 | func AppendTerm(data []byte, val []byte) []byte { 603 | return WriteTLKV(data, LitTerm, nil, val) 604 | } 605 | 606 | func AppendReference(data []byte, val ID) []byte { 607 | return WriteTLKV(data, LitReference, nil, ZipID(val)) 608 | } 609 | 610 | func (rdx Stream) AppendReference(val ID) Stream { 611 | return AppendReference(rdx, val) 612 | } 613 | 614 | func (rdx Stream) AppendString(val string) Stream { 615 | return AppendString(rdx, []byte(val)) 616 | } 617 | 618 | func (rdx Stream) AppendTerm(val string) Stream { 619 | return AppendTerm(rdx, []byte(val)) 620 | } 621 | 622 | func (rdx Stream) AppendInteger(val int64) Stream { 623 | return AppendInteger(rdx, Integer(val)) 624 | } 625 | 626 | func (rdx Stream) AppendPLEX(lit byte, id ID, val Stream) (ret Stream) { 627 | marks := make(Marks, 0, 1) 628 | if len(val) <= 0xff { 629 | ret = OpenShortTLV(rdx, lit, &marks) 630 | } else { 631 | ret = OpenTLV(rdx, lit, &marks) 632 | } 633 | zip := ZipID(id) 634 | ret = append(ret, byte(len(zip))) 635 | ret = append(ret, zip...) 636 | ret = append(ret, val...) 637 | ret, _ = CloseTLV(ret, lit, &marks) 638 | return 639 | } 640 | 641 | func (rdx Stream) AppendTuple(id ID, val Stream) (ret Stream) { 642 | return rdx.AppendPLEX(LitTuple, id, val) 643 | } 644 | 645 | func (rdx Stream) AppendLinear(id ID, val Stream) (ret Stream) { 646 | return rdx.AppendPLEX(LitLinear, id, val) 647 | } 648 | 649 | func (rdx Stream) AppendEuler(id ID, val Stream) (ret Stream) { 650 | return rdx.AppendPLEX(LitEuler, id, val) 651 | } 652 | 653 | func (rdx Stream) AppendMultix(id ID, val Stream) (ret Stream) { 654 | return rdx.AppendPLEX(LitMultix, id, val) 655 | } 656 | 657 | var ErrBadFloatRecord = errors.New("bad Float record format") 658 | var ErrBadIntegerRecord = errors.New("bad Integer record format") 659 | var ErrBadReferenceRecord = errors.New("bad Reference record format") 660 | var ErrBadStringRecord = errors.New("bad RonString record format") 661 | var ErrBadTermRecord = errors.New("bad Term record format") 662 | 663 | // Normalizes a raw Stream input (all keys Value order, no duplicates, no overlong 664 | // encoding, etc etc. Inputs that are *certainly* normalized get mentioned as 665 | // `rdx.Stream` while not-necessarily-normalized go as `[]byte`. 666 | func Normalize(rdx []byte) (RDX []byte, err error) { 667 | data := make([]byte, 0, len(rdx)) 668 | stack := Marks{} 669 | return normalize(data, rdx, nil, &stack) 670 | } 671 | 672 | func normalize(data, rdx []byte, z Compare, stack *Marks) (norm Stream, err error) { 673 | norm = data 674 | if len(rdx) == 0 { 675 | return 676 | } 677 | chunks := [][]byte{} 678 | at := NewIter(rdx) 679 | at.Read() 680 | next := at 681 | oc := len(norm) 682 | for at.HasData() && err == nil { 683 | norm, err = appendNorm(norm, at, stack) 684 | next.Read() 685 | if err == nil && next.HasData() && z != nil && z(&at, &next) != Less { 686 | chunks = append(chunks, norm[oc:]) 687 | oc = len(norm) 688 | } 689 | at = next 690 | } 691 | if at.HasFailed() { 692 | err = next.Error() 693 | } 694 | if len(chunks) > 0 && err == nil { 695 | chunks = append(chunks, norm[oc:]) 696 | sorted := make([]byte, 0, len(norm)-len(data)) 697 | sorted, err = HeapMerge(sorted, chunks, z) 698 | norm = append(data, sorted...) 699 | } 700 | return 701 | } 702 | 703 | func appendNorm(to []byte, it Iter, stack *Marks) (norm []byte, err error) { 704 | val := it.Value() 705 | lit := it.Lit() 706 | idbytes := ZipID(it.ID()) 707 | norm = to 708 | switch lit { 709 | case LitFloat: 710 | if len(val) > 8 { 711 | return nil, ErrBadFloatRecord 712 | } 713 | f := UnzipFloat(val) 714 | if math.IsNaN(float64(f)) { 715 | return nil, ErrBadFloatRecord 716 | } 717 | val = ZipFloat(f) 718 | norm = WriteTLKV(norm, lit, idbytes, val) 719 | case LitInteger: 720 | if len(val) > 8 { 721 | return nil, ErrBadIntegerRecord 722 | } 723 | f := UnzipInteger(val) 724 | val = ZipInteger(f) 725 | norm = WriteTLKV(norm, lit, idbytes, val) 726 | case LitReference: 727 | if len(val) > 16 { // todo bad sizes 728 | return nil, ErrBadReferenceRecord 729 | } 730 | i := UnzipID(val) 731 | val = ZipID(i) 732 | norm = WriteTLKV(norm, lit, idbytes, val) 733 | case LitString: 734 | if !utf8.Valid(val) { 735 | return nil, ErrBadStringRecord 736 | } 737 | norm = WriteTLKV(norm, lit, idbytes, val) 738 | case LitTerm: 739 | for _, c := range val { 740 | if RON64REV[c] == 0xff { 741 | return nil, ErrBadTermRecord 742 | } 743 | } 744 | norm = WriteTLKV(norm, lit, idbytes, val) 745 | case LitTuple: 746 | plit := stack.TopLit() 747 | norm = OpenTLV(norm, LitTuple, stack) 748 | norm = append(norm, byte(len(idbytes))) 749 | norm = append(norm, idbytes...) 750 | l := len(norm) 751 | norm, err = normalize(norm, val, nil, stack) 752 | if err != nil { 753 | return nil, err 754 | } 755 | if plit == LitEuler && l == len(norm) { // () 756 | norm, err = CancelTLV(norm, LitTuple, stack) 757 | } else { 758 | norm, err = CloseTLV(norm, LitTuple, stack) 759 | } 760 | case LitLinear: 761 | norm = OpenTLV(norm, LitLinear, stack) 762 | norm = append(norm, byte(len(idbytes))) 763 | norm = append(norm, idbytes...) 764 | norm, err = normalize(norm, val, nil, stack) 765 | if err != nil { 766 | return 767 | } 768 | norm, err = CloseTLV(norm, LitLinear, stack) 769 | case LitEuler: 770 | norm = OpenTLV(norm, LitEuler, stack) 771 | norm = append(norm, byte(len(idbytes))) 772 | norm = append(norm, idbytes...) 773 | norm, err = normalize(norm, val, CompareEuler, stack) 774 | if err != nil { 775 | return 776 | } 777 | norm, err = CloseTLV(norm, LitEuler, stack) 778 | case LitMultix: 779 | norm = OpenTLV(norm, LitMultix, stack) 780 | norm = append(norm, byte(len(idbytes))) 781 | norm = append(norm, idbytes...) 782 | norm, err = normalize(norm, val, CompareMultix, stack) 783 | if err != nil { 784 | return 785 | } 786 | norm, err = CloseTLV(norm, LitMultix, stack) 787 | } 788 | return 789 | } 790 | 791 | func flatten(data, rdx []byte, stack *Marks) (flat []byte, err error) { 792 | flat = data 793 | trim := len(flat) 794 | for len(rdx) > 0 && err == nil { 795 | var lit byte 796 | var idb []byte 797 | var val, rest []byte 798 | lit, idb, val, rest, err = ReadTLKV(rdx) 799 | id := UnzipID(idb) 800 | if err != nil { 801 | break 802 | } 803 | if !id.IsLive() { // TODO math 804 | if len(*stack) > 0 && (*stack)[len(*stack)-1].Lit == LitTuple { 805 | flat = append(flat, LitTuple|CaseBit, 1, 0) 806 | } 807 | } else if len(*stack) > 0 && stack.TopLit() == LitMultix { 808 | id.Seq = 0 809 | idz := ZipID(id) 810 | flat = OpenTLV(flat, lit, stack) 811 | flat = append(flat, byte(len(idz))) 812 | flat = append(flat, idz...) 813 | flat, err = flatten(flat, val, stack) 814 | flat, err = CloseTLV(flat, lit, stack) 815 | trim = len(flat) // todo better 816 | } else if IsFIRST(lit) { 817 | flat = WriteTLKV(flat, lit, nil, val) 818 | trim = len(flat) 819 | } else { 820 | flat = OpenTLV(flat, lit, stack) 821 | flat = append(flat, 0) 822 | flat, err = flatten(flat, val, stack) 823 | if err == nil { 824 | start := (*stack)[len(*stack)-1].Start // todo nicer 825 | etup := lit == LitTuple && start+3 == len(flat) 826 | flat, err = CloseTLV(flat, lit, stack) 827 | if etup { 828 | trim = start 829 | } else { 830 | trim = len(flat) 831 | } 832 | } 833 | } 834 | rdx = rest 835 | } 836 | flat = flat[:trim] 837 | return 838 | } 839 | 840 | func Flatten(data, rdx []byte) (flat []byte, err error) { 841 | stack := make(Marks, 0, 32) 842 | return flatten(data, rdx, &stack) 843 | } 844 | 845 | func delve(data Stream, path Iter, z Compare) (found Stream, err error) { 846 | it := NewIter(data) 847 | i := Less 848 | for i < Eq && it.Read() { 849 | i = z(&it, &path) 850 | } 851 | if !it.HasData() || i > Eq { 852 | err = ErrRecordNotFound 853 | return 854 | } 855 | if !path.Read() { 856 | return it.Record(), nil 857 | } 858 | return scan(it, path) 859 | } 860 | 861 | var ErrBadPath = errors.New("bad path") 862 | 863 | func delveP(data Stream, path Iter) (found Stream, err error) { 864 | if path.Lit() != LitInteger { 865 | err = ErrBadPath 866 | return 867 | } 868 | it := NewIter(data) 869 | n := UnzipInt64(path.Value()) 870 | for n >= 0 && it.Read() { 871 | n-- 872 | } 873 | if !it.HasData() { 874 | err = ErrRecordNotFound 875 | return 876 | } 877 | if !path.Read() { 878 | return it.Record(), nil 879 | } else if !IsPLEX(it.Lit()) { 880 | err = ErrRecordNotFound 881 | return 882 | } else { 883 | return scan(it, path) 884 | } 885 | } 886 | 887 | func scan(it Iter, path Iter) (found Stream, err error) { 888 | switch it.Lit() { 889 | case LitTuple: 890 | found, err = delveP(it.Value(), path) 891 | case LitLinear: 892 | found, err = delve(it.Value(), path, CompareLinear) 893 | case LitEuler: 894 | found, err = delve(it.Value(), path, CompareEuler) 895 | case LitMultix: 896 | found, err = delve(it.Value(), path, CompareMultix) 897 | default: 898 | err = ErrRecordNotFound 899 | } 900 | return 901 | } 902 | 903 | var ErrNoKeyProvided = errors.New("no key provided") 904 | var ErrNotPLEX = errors.New("not a PLEX container element") 905 | 906 | func Pick(key, data Stream) (entry Stream, err error) { 907 | dit := NewIter(data) 908 | kit := NewIter(key) 909 | if !kit.Read() { 910 | return nil, ErrNoKeyProvided 911 | } 912 | if !dit.Read() { 913 | return nil, ErrBadRecord 914 | } 915 | var z Compare 916 | switch dit.Lit() { 917 | case LitTuple: 918 | z = nil // fixme 919 | case LitLinear: 920 | z = CompareLinear 921 | case LitEuler: 922 | z = CompareEuler 923 | case LitMultix: 924 | z = CompareMultix 925 | default: 926 | return nil, ErrNotPLEX 927 | } 928 | it := NewIter(dit.Value()) 929 | i := Less 930 | if z != nil { 931 | for i < Eq && it.Read() { 932 | i = z(&it, &kit) 933 | } 934 | } else { 935 | k := kit.Integer() 936 | for k >= 0 && it.Read() { 937 | k-- 938 | } 939 | if it.HasData() { 940 | i = Eq 941 | } 942 | } 943 | if i == Eq { 944 | entry = it.Record() 945 | } else { 946 | err = ErrRecordNotFound 947 | } 948 | return 949 | } 950 | 951 | func Delve(data, path Stream) (entry Stream, err error) { 952 | pi := NewIter(path) 953 | if !pi.Read() { 954 | return data, nil 955 | } 956 | return delveP(data, pi) 957 | } 958 | 959 | func DebugIter(it Iter) (lit, header, id, value []byte) { 960 | if !it.HasData() { 961 | return 962 | } 963 | return it.data[0:1], 964 | it.data[1:it.hdrlen], 965 | it.data[it.hdrlen : it.hdrlen+it.idlen], 966 | it.data[it.hdrlen+it.idlen : int(it.hdrlen+it.idlen)+it.vallen] 967 | } 968 | 969 | var ErrRecordNotFound = errors.New("no such record") 970 | 971 | var ID0 = ID{} 972 | 973 | func WriteAll(file *os.File, data ...[]byte) (err error) { 974 | for err == nil && len(data) > 0 { 975 | next := data[0] 976 | data = data[1:] 977 | n := 0 978 | for err == nil && len(next) > 0 { 979 | next = next[n:] 980 | n, err = file.Write(next) 981 | } 982 | } 983 | return 984 | } 985 | -------------------------------------------------------------------------------- /JDR.rl.go: -------------------------------------------------------------------------------- 1 | package rdx 2 | 3 | // action indices for the parser 4 | 5 | const ( 6 | JDRenum = 0 7 | JDRNL = JDRenum + 1 8 | JDRUtf8cp1 = JDRenum + 10 9 | JDRUtf8cp2 = JDRenum + 11 10 | JDRUtf8cp3 = JDRenum + 12 11 | JDRUtf8cp4 = JDRenum + 13 12 | JDRInt = JDRenum + 19 13 | JDRFloat = JDRenum + 20 14 | JDRTerm = JDRenum + 21 15 | JDRRef = JDRenum + 22 16 | JDRString = JDRenum + 23 17 | JDRMLString = JDRenum + 24 18 | JDRStamp = JDRenum + 25 19 | JDRNoStamp = JDRenum + 26 20 | JDROpenP = JDRenum + 27 21 | JDRCloseP = JDRenum + 28 22 | JDROpenL = JDRenum + 29 23 | JDRCloseL = JDRenum + 30 24 | JDROpenE = JDRenum + 31 25 | JDRCloseE = JDRenum + 32 26 | JDROpenX = JDRenum + 33 27 | JDRCloseX = JDRenum + 34 28 | JDRComma = JDRenum + 35 29 | JDRColon = JDRenum + 36 30 | JDRSemicolon = JDRenum + 37 31 | JDROpen = JDRenum + 38 32 | JDRClose = JDRenum + 39 33 | JDRInter = JDRenum + 40 34 | JDRFIRST = JDRenum + 42 35 | JDRRoot = JDRenum + 43 36 | ) 37 | 38 | // user functions (callbacks) for the parser 39 | // func JDRonNL (tok []byte, state *JDRstate) error 40 | // func JDRonUtf8cp1 (tok []byte, state *JDRstate) error 41 | // func JDRonUtf8cp2 (tok []byte, state *JDRstate) error 42 | // func JDRonUtf8cp3 (tok []byte, state *JDRstate) error 43 | // func JDRonUtf8cp4 (tok []byte, state *JDRstate) error 44 | // func JDRonInt (tok []byte, state *JDRstate) error 45 | // func JDRonFloat (tok []byte, state *JDRstate) error 46 | // func JDRonTerm (tok []byte, state *JDRstate) error 47 | // func JDRonRef (tok []byte, state *JDRstate) error 48 | // func JDRonString (tok []byte, state *JDRstate) error 49 | // func JDRonMLString (tok []byte, state *JDRstate) error 50 | // func JDRonStamp (tok []byte, state *JDRstate) error 51 | // func JDRonNoStamp (tok []byte, state *JDRstate) error 52 | // func JDRonOpenP (tok []byte, state *JDRstate) error 53 | // func JDRonCloseP (tok []byte, state *JDRstate) error 54 | // func JDRonOpenL (tok []byte, state *JDRstate) error 55 | // func JDRonCloseL (tok []byte, state *JDRstate) error 56 | // func JDRonOpenE (tok []byte, state *JDRstate) error 57 | // func JDRonCloseE (tok []byte, state *JDRstate) error 58 | // func JDRonOpenX (tok []byte, state *JDRstate) error 59 | // func JDRonCloseX (tok []byte, state *JDRstate) error 60 | // func JDRonComma (tok []byte, state *JDRstate) error 61 | // func JDRonColon (tok []byte, state *JDRstate) error 62 | // func JDRonSemicolon (tok []byte, state *JDRstate) error 63 | // func JDRonOpen (tok []byte, state *JDRstate) error 64 | // func JDRonClose (tok []byte, state *JDRstate) error 65 | // func JDRonInter (tok []byte, state *JDRstate) error 66 | // func JDRonFIRST (tok []byte, state *JDRstate) error 67 | // func JDRonRoot (tok []byte, state *JDRstate) error 68 | 69 | var _JDR_actions []byte = []byte{ 70 | 0, 1, 0, 1, 1, 1, 2, 1, 3, 71 | 1, 4, 1, 5, 1, 6, 1, 7, 72 | 1, 8, 1, 9, 1, 11, 1, 13, 73 | 1, 15, 1, 17, 1, 19, 1, 21, 74 | 1, 22, 1, 23, 1, 27, 1, 29, 75 | 1, 31, 1, 33, 1, 35, 1, 37, 76 | 1, 39, 1, 41, 1, 43, 1, 45, 77 | 1, 47, 1, 56, 1, 57, 2, 1, 78 | 0, 2, 1, 22, 2, 1, 57, 2, 79 | 3, 2, 2, 3, 4, 2, 3, 6, 80 | 2, 3, 8, 2, 5, 2, 2, 5, 81 | 4, 2, 5, 6, 2, 5, 8, 2, 82 | 7, 2, 2, 7, 4, 2, 7, 6, 83 | 2, 7, 8, 2, 9, 2, 2, 9, 84 | 4, 2, 9, 6, 2, 9, 8, 2, 85 | 11, 0, 2, 11, 22, 2, 13, 0, 86 | 2, 13, 22, 2, 15, 0, 2, 15, 87 | 22, 2, 17, 0, 2, 17, 22, 2, 88 | 19, 0, 2, 19, 22, 2, 21, 0, 89 | 2, 21, 22, 2, 23, 0, 2, 27, 90 | 0, 2, 27, 22, 2, 29, 0, 2, 91 | 31, 0, 2, 31, 22, 2, 33, 0, 92 | 2, 35, 0, 2, 35, 22, 2, 37, 93 | 0, 2, 39, 0, 2, 39, 22, 2, 94 | 41, 0, 2, 43, 0, 2, 45, 0, 95 | 2, 47, 0, 2, 48, 26, 2, 48, 96 | 30, 2, 48, 34, 2, 48, 38, 2, 97 | 49, 57, 2, 50, 28, 2, 50, 32, 98 | 2, 50, 36, 2, 50, 40, 2, 51, 99 | 57, 2, 52, 42, 2, 52, 44, 2, 100 | 52, 46, 2, 53, 57, 2, 54, 18, 101 | 2, 54, 20, 2, 55, 57, 2, 56, 102 | 0, 2, 56, 57, 3, 1, 48, 26, 103 | 3, 1, 48, 30, 3, 1, 48, 34, 104 | 3, 1, 48, 38, 3, 1, 49, 57, 105 | 3, 1, 50, 28, 3, 1, 50, 32, 106 | 3, 1, 50, 36, 3, 1, 50, 40, 107 | 3, 1, 51, 57, 3, 1, 52, 42, 108 | 3, 1, 52, 44, 3, 1, 52, 46, 109 | 3, 1, 53, 57, 3, 1, 54, 18, 110 | 3, 1, 54, 20, 3, 1, 55, 57, 111 | 3, 23, 49, 57, 3, 23, 55, 57, 112 | 3, 29, 51, 57, 3, 33, 51, 57, 113 | 3, 37, 51, 57, 3, 41, 51, 57, 114 | 3, 43, 53, 57, 3, 45, 53, 57, 115 | 3, 47, 53, 57, 3, 49, 48, 26, 116 | 3, 49, 48, 30, 3, 49, 48, 34, 117 | 3, 49, 48, 38, 3, 49, 50, 28, 118 | 3, 49, 50, 32, 3, 49, 50, 36, 119 | 3, 49, 50, 40, 3, 49, 52, 42, 120 | 3, 49, 52, 44, 3, 49, 52, 46, 121 | 3, 49, 54, 18, 3, 49, 54, 20, 122 | 3, 51, 48, 26, 3, 51, 48, 30, 123 | 3, 51, 48, 34, 3, 51, 48, 38, 124 | 3, 51, 50, 28, 3, 51, 50, 32, 125 | 3, 51, 50, 36, 3, 51, 50, 40, 126 | 3, 51, 52, 42, 3, 51, 52, 44, 127 | 3, 51, 52, 46, 3, 51, 54, 18, 128 | 3, 51, 54, 20, 3, 53, 48, 26, 129 | 3, 53, 48, 30, 3, 53, 48, 34, 130 | 3, 53, 48, 38, 3, 53, 50, 28, 131 | 3, 53, 50, 32, 3, 53, 50, 36, 132 | 3, 53, 50, 40, 3, 53, 52, 42, 133 | 3, 53, 52, 44, 3, 53, 52, 46, 134 | 3, 53, 54, 18, 3, 53, 54, 20, 135 | 3, 54, 12, 10, 3, 54, 16, 14, 136 | 3, 55, 48, 26, 3, 55, 48, 30, 137 | 3, 55, 48, 34, 3, 55, 48, 38, 138 | 3, 55, 50, 28, 3, 55, 50, 32, 139 | 3, 55, 50, 36, 3, 55, 50, 40, 140 | 3, 55, 52, 42, 3, 55, 52, 44, 141 | 3, 55, 52, 46, 3, 55, 54, 18, 142 | 3, 55, 54, 20, 3, 56, 48, 26, 143 | 3, 56, 48, 30, 3, 56, 48, 34, 144 | 3, 56, 48, 38, 3, 56, 50, 28, 145 | 3, 56, 50, 32, 3, 56, 50, 36, 146 | 3, 56, 50, 40, 3, 56, 52, 42, 147 | 3, 56, 52, 44, 3, 56, 52, 46, 148 | 3, 56, 54, 18, 3, 56, 54, 20, 149 | 4, 1, 49, 48, 26, 4, 1, 49, 150 | 48, 30, 4, 1, 49, 48, 34, 4, 151 | 1, 49, 48, 38, 4, 1, 49, 50, 152 | 28, 4, 1, 49, 50, 32, 4, 1, 153 | 49, 50, 36, 4, 1, 49, 50, 40, 154 | 4, 1, 49, 52, 42, 4, 1, 49, 155 | 52, 44, 4, 1, 49, 52, 46, 4, 156 | 1, 49, 54, 18, 4, 1, 49, 54, 157 | 20, 4, 1, 51, 48, 26, 4, 1, 158 | 51, 48, 30, 4, 1, 51, 48, 34, 159 | 4, 1, 51, 48, 38, 4, 1, 51, 160 | 50, 28, 4, 1, 51, 50, 32, 4, 161 | 1, 51, 50, 36, 4, 1, 51, 50, 162 | 40, 4, 1, 51, 52, 42, 4, 1, 163 | 51, 52, 44, 4, 1, 51, 52, 46, 164 | 4, 1, 51, 54, 18, 4, 1, 51, 165 | 54, 20, 4, 1, 53, 48, 26, 4, 166 | 1, 53, 48, 30, 4, 1, 53, 48, 167 | 34, 4, 1, 53, 48, 38, 4, 1, 168 | 53, 50, 28, 4, 1, 53, 50, 32, 169 | 4, 1, 53, 50, 36, 4, 1, 53, 170 | 50, 40, 4, 1, 53, 52, 42, 4, 171 | 1, 53, 52, 44, 4, 1, 53, 52, 172 | 46, 4, 1, 53, 54, 18, 4, 1, 173 | 53, 54, 20, 4, 1, 54, 12, 10, 174 | 4, 1, 54, 16, 14, 4, 1, 55, 175 | 48, 26, 4, 1, 55, 48, 30, 4, 176 | 1, 55, 48, 34, 4, 1, 55, 48, 177 | 38, 4, 1, 55, 50, 28, 4, 1, 178 | 55, 50, 32, 4, 1, 55, 50, 36, 179 | 4, 1, 55, 50, 40, 4, 1, 55, 180 | 52, 42, 4, 1, 55, 52, 44, 4, 181 | 1, 55, 52, 46, 4, 1, 55, 54, 182 | 18, 4, 1, 55, 54, 20, 4, 23, 183 | 49, 48, 26, 4, 23, 49, 48, 30, 184 | 4, 23, 49, 48, 34, 4, 23, 49, 185 | 48, 38, 4, 23, 49, 50, 28, 4, 186 | 23, 49, 50, 32, 4, 23, 49, 50, 187 | 36, 4, 23, 49, 50, 40, 4, 23, 188 | 49, 52, 42, 4, 23, 49, 52, 44, 189 | 4, 23, 49, 52, 46, 4, 23, 49, 190 | 54, 18, 4, 23, 49, 54, 20, 4, 191 | 23, 55, 48, 26, 4, 23, 55, 48, 192 | 30, 4, 23, 55, 48, 34, 4, 23, 193 | 55, 48, 38, 4, 23, 55, 50, 28, 194 | 4, 23, 55, 50, 32, 4, 23, 55, 195 | 50, 36, 4, 23, 55, 50, 40, 4, 196 | 23, 55, 52, 42, 4, 23, 55, 52, 197 | 44, 4, 23, 55, 52, 46, 4, 23, 198 | 55, 54, 18, 4, 23, 55, 54, 20, 199 | 4, 24, 25, 49, 57, 4, 24, 25, 200 | 55, 57, 4, 29, 51, 48, 26, 4, 201 | 29, 51, 48, 30, 4, 29, 51, 48, 202 | 34, 4, 29, 51, 48, 38, 4, 29, 203 | 51, 50, 28, 4, 29, 51, 50, 32, 204 | 4, 29, 51, 50, 36, 4, 29, 51, 205 | 50, 40, 4, 29, 51, 52, 42, 4, 206 | 29, 51, 52, 44, 4, 29, 51, 52, 207 | 46, 4, 29, 51, 54, 18, 4, 29, 208 | 51, 54, 20, 4, 33, 51, 48, 26, 209 | 4, 33, 51, 48, 30, 4, 33, 51, 210 | 48, 34, 4, 33, 51, 48, 38, 4, 211 | 33, 51, 50, 28, 4, 33, 51, 50, 212 | 32, 4, 33, 51, 50, 36, 4, 33, 213 | 51, 50, 40, 4, 33, 51, 52, 42, 214 | 4, 33, 51, 52, 44, 4, 33, 51, 215 | 52, 46, 4, 33, 51, 54, 18, 4, 216 | 33, 51, 54, 20, 4, 37, 51, 48, 217 | 26, 4, 37, 51, 48, 30, 4, 37, 218 | 51, 48, 34, 4, 37, 51, 48, 38, 219 | 4, 37, 51, 50, 28, 4, 37, 51, 220 | 50, 32, 4, 37, 51, 50, 36, 4, 221 | 37, 51, 50, 40, 4, 37, 51, 52, 222 | 42, 4, 37, 51, 52, 44, 4, 37, 223 | 51, 52, 46, 4, 37, 51, 54, 18, 224 | 4, 37, 51, 54, 20, 4, 41, 51, 225 | 48, 26, 4, 41, 51, 48, 30, 4, 226 | 41, 51, 48, 34, 4, 41, 51, 48, 227 | 38, 4, 41, 51, 50, 28, 4, 41, 228 | 51, 50, 32, 4, 41, 51, 50, 36, 229 | 4, 41, 51, 50, 40, 4, 41, 51, 230 | 52, 42, 4, 41, 51, 52, 44, 4, 231 | 41, 51, 52, 46, 4, 41, 51, 54, 232 | 18, 4, 41, 51, 54, 20, 4, 43, 233 | 53, 48, 26, 4, 43, 53, 48, 30, 234 | 4, 43, 53, 48, 34, 4, 43, 53, 235 | 48, 38, 4, 43, 53, 50, 28, 4, 236 | 43, 53, 50, 32, 4, 43, 53, 50, 237 | 36, 4, 43, 53, 50, 40, 4, 43, 238 | 53, 52, 42, 4, 43, 53, 52, 44, 239 | 4, 43, 53, 52, 46, 4, 43, 53, 240 | 54, 18, 4, 43, 53, 54, 20, 4, 241 | 45, 53, 48, 26, 4, 45, 53, 48, 242 | 30, 4, 45, 53, 48, 34, 4, 45, 243 | 53, 48, 38, 4, 45, 53, 50, 28, 244 | 4, 45, 53, 50, 32, 4, 45, 53, 245 | 50, 36, 4, 45, 53, 50, 40, 4, 246 | 45, 53, 52, 42, 4, 45, 53, 52, 247 | 44, 4, 45, 53, 52, 46, 4, 45, 248 | 53, 54, 18, 4, 45, 53, 54, 20, 249 | 4, 47, 53, 48, 26, 4, 47, 53, 250 | 48, 30, 4, 47, 53, 48, 34, 4, 251 | 47, 53, 48, 38, 4, 47, 53, 50, 252 | 28, 4, 47, 53, 50, 32, 4, 47, 253 | 53, 50, 36, 4, 47, 53, 50, 40, 254 | 4, 47, 53, 52, 42, 4, 47, 53, 255 | 52, 44, 4, 47, 53, 52, 46, 4, 256 | 47, 53, 54, 18, 4, 47, 53, 54, 257 | 20, 4, 49, 54, 12, 10, 4, 49, 258 | 54, 16, 14, 4, 51, 54, 12, 10, 259 | 4, 51, 54, 16, 14, 4, 53, 54, 260 | 12, 10, 4, 53, 54, 16, 14, 4, 261 | 55, 54, 12, 10, 4, 55, 54, 16, 262 | 14, 4, 56, 54, 12, 10, 4, 56, 263 | 54, 16, 14, 5, 1, 24, 25, 49, 264 | 57, 5, 1, 24, 25, 55, 57, 5, 265 | 1, 49, 54, 12, 10, 5, 1, 49, 266 | 54, 16, 14, 5, 1, 51, 54, 12, 267 | 10, 5, 1, 51, 54, 16, 14, 5, 268 | 1, 53, 54, 12, 10, 5, 1, 53, 269 | 54, 16, 14, 5, 1, 55, 54, 12, 270 | 10, 5, 1, 55, 54, 16, 14, 5, 271 | 11, 24, 25, 55, 57, 5, 13, 24, 272 | 25, 55, 57, 5, 15, 24, 25, 55, 273 | 57, 5, 17, 24, 25, 55, 57, 5, 274 | 19, 24, 25, 55, 57, 5, 21, 24, 275 | 25, 55, 57, 5, 23, 49, 54, 12, 276 | 10, 5, 23, 55, 54, 12, 10, 5, 277 | 24, 25, 49, 48, 26, 5, 24, 25, 278 | 49, 48, 30, 5, 24, 25, 49, 48, 279 | 34, 5, 24, 25, 49, 48, 38, 5, 280 | 24, 25, 49, 50, 28, 5, 24, 25, 281 | 49, 50, 32, 5, 24, 25, 49, 50, 282 | 36, 5, 24, 25, 49, 50, 40, 5, 283 | 24, 25, 49, 52, 42, 5, 24, 25, 284 | 49, 52, 44, 5, 24, 25, 49, 52, 285 | 46, 5, 24, 25, 49, 54, 18, 5, 286 | 24, 25, 49, 54, 20, 5, 24, 25, 287 | 55, 48, 26, 5, 24, 25, 55, 48, 288 | 30, 5, 24, 25, 55, 48, 34, 5, 289 | 24, 25, 55, 48, 38, 5, 24, 25, 290 | 55, 50, 28, 5, 24, 25, 55, 50, 291 | 32, 5, 24, 25, 55, 50, 36, 5, 292 | 24, 25, 55, 50, 40, 5, 24, 25, 293 | 55, 52, 42, 5, 24, 25, 55, 52, 294 | 44, 5, 24, 25, 55, 52, 46, 5, 295 | 24, 25, 55, 54, 18, 5, 24, 25, 296 | 55, 54, 20, 5, 27, 24, 25, 49, 297 | 57, 5, 29, 51, 54, 12, 10, 5, 298 | 29, 51, 54, 16, 14, 5, 31, 24, 299 | 25, 49, 57, 5, 33, 51, 54, 12, 300 | 10, 5, 33, 51, 54, 16, 14, 5, 301 | 35, 24, 25, 49, 57, 5, 37, 51, 302 | 54, 12, 10, 5, 37, 51, 54, 16, 303 | 14, 5, 39, 24, 25, 49, 57, 5, 304 | 41, 51, 54, 12, 10, 5, 41, 51, 305 | 54, 16, 14, 5, 43, 53, 54, 12, 306 | 10, 5, 43, 53, 54, 16, 14, 5, 307 | 45, 53, 54, 12, 10, 5, 45, 53, 308 | 54, 16, 14, 5, 47, 53, 54, 12, 309 | 10, 5, 47, 53, 54, 16, 14, 5, 310 | 54, 12, 10, 16, 14, 6, 1, 24, 311 | 25, 49, 48, 26, 6, 1, 24, 25, 312 | 49, 48, 30, 6, 1, 24, 25, 49, 313 | 48, 34, 6, 1, 24, 25, 49, 48, 314 | 38, 6, 1, 24, 25, 49, 50, 28, 315 | 6, 1, 24, 25, 49, 50, 32, 6, 316 | 1, 24, 25, 49, 50, 36, 6, 1, 317 | 24, 25, 49, 50, 40, 6, 1, 24, 318 | 25, 49, 52, 42, 6, 1, 24, 25, 319 | 49, 52, 44, 6, 1, 24, 25, 49, 320 | 52, 46, 6, 1, 24, 25, 49, 54, 321 | 18, 6, 1, 24, 25, 49, 54, 20, 322 | 6, 1, 24, 25, 55, 48, 26, 6, 323 | 1, 24, 25, 55, 48, 30, 6, 1, 324 | 24, 25, 55, 48, 34, 6, 1, 24, 325 | 25, 55, 48, 38, 6, 1, 24, 25, 326 | 55, 50, 28, 6, 1, 24, 25, 55, 327 | 50, 32, 6, 1, 24, 25, 55, 50, 328 | 36, 6, 1, 24, 25, 55, 50, 40, 329 | 6, 1, 24, 25, 55, 52, 42, 6, 330 | 1, 24, 25, 55, 52, 44, 6, 1, 331 | 24, 25, 55, 52, 46, 6, 1, 24, 332 | 25, 55, 54, 18, 6, 1, 24, 25, 333 | 55, 54, 20, 6, 1, 54, 12, 10, 334 | 16, 14, 6, 11, 24, 25, 55, 48, 335 | 26, 6, 11, 24, 25, 55, 48, 30, 336 | 6, 11, 24, 25, 55, 48, 34, 6, 337 | 11, 24, 25, 55, 48, 38, 6, 11, 338 | 24, 25, 55, 50, 28, 6, 11, 24, 339 | 25, 55, 50, 32, 6, 11, 24, 25, 340 | 55, 50, 36, 6, 11, 24, 25, 55, 341 | 50, 40, 6, 11, 24, 25, 55, 52, 342 | 42, 6, 11, 24, 25, 55, 52, 44, 343 | 6, 11, 24, 25, 55, 52, 46, 6, 344 | 11, 24, 25, 55, 54, 18, 6, 11, 345 | 24, 25, 55, 54, 20, 6, 13, 24, 346 | 25, 55, 48, 26, 6, 13, 24, 25, 347 | 55, 48, 30, 6, 13, 24, 25, 55, 348 | 48, 34, 6, 13, 24, 25, 55, 48, 349 | 38, 6, 13, 24, 25, 55, 50, 28, 350 | 6, 13, 24, 25, 55, 50, 32, 6, 351 | 13, 24, 25, 55, 50, 36, 6, 13, 352 | 24, 25, 55, 50, 40, 6, 13, 24, 353 | 25, 55, 52, 42, 6, 13, 24, 25, 354 | 55, 52, 44, 6, 13, 24, 25, 55, 355 | 52, 46, 6, 13, 24, 25, 55, 54, 356 | 18, 6, 13, 24, 25, 55, 54, 20, 357 | 6, 15, 24, 25, 55, 48, 26, 6, 358 | 15, 24, 25, 55, 48, 30, 6, 15, 359 | 24, 25, 55, 48, 34, 6, 15, 24, 360 | 25, 55, 48, 38, 6, 15, 24, 25, 361 | 55, 50, 28, 6, 15, 24, 25, 55, 362 | 50, 32, 6, 15, 24, 25, 55, 50, 363 | 36, 6, 15, 24, 25, 55, 50, 40, 364 | 6, 15, 24, 25, 55, 52, 42, 6, 365 | 15, 24, 25, 55, 52, 44, 6, 15, 366 | 24, 25, 55, 52, 46, 6, 15, 24, 367 | 25, 55, 54, 18, 6, 15, 24, 25, 368 | 55, 54, 20, 6, 17, 24, 25, 55, 369 | 48, 26, 6, 17, 24, 25, 55, 48, 370 | 30, 6, 17, 24, 25, 55, 48, 34, 371 | 6, 17, 24, 25, 55, 48, 38, 6, 372 | 17, 24, 25, 55, 50, 28, 6, 17, 373 | 24, 25, 55, 50, 32, 6, 17, 24, 374 | 25, 55, 50, 36, 6, 17, 24, 25, 375 | 55, 50, 40, 6, 17, 24, 25, 55, 376 | 52, 42, 6, 17, 24, 25, 55, 52, 377 | 44, 6, 17, 24, 25, 55, 52, 46, 378 | 6, 17, 24, 25, 55, 54, 18, 6, 379 | 17, 24, 25, 55, 54, 20, 6, 19, 380 | 24, 25, 55, 48, 26, 6, 19, 24, 381 | 25, 55, 48, 30, 6, 19, 24, 25, 382 | 55, 48, 34, 6, 19, 24, 25, 55, 383 | 48, 38, 6, 19, 24, 25, 55, 50, 384 | 28, 6, 19, 24, 25, 55, 50, 32, 385 | 6, 19, 24, 25, 55, 50, 36, 6, 386 | 19, 24, 25, 55, 50, 40, 6, 19, 387 | 24, 25, 55, 52, 42, 6, 19, 24, 388 | 25, 55, 52, 44, 6, 19, 24, 25, 389 | 55, 52, 46, 6, 19, 24, 25, 55, 390 | 54, 18, 6, 19, 24, 25, 55, 54, 391 | 20, 6, 21, 24, 25, 55, 48, 26, 392 | 6, 21, 24, 25, 55, 48, 30, 6, 393 | 21, 24, 25, 55, 48, 34, 6, 21, 394 | 24, 25, 55, 48, 38, 6, 21, 24, 395 | 25, 55, 50, 28, 6, 21, 24, 25, 396 | 55, 50, 32, 6, 21, 24, 25, 55, 397 | 50, 36, 6, 21, 24, 25, 55, 50, 398 | 40, 6, 21, 24, 25, 55, 52, 42, 399 | 6, 21, 24, 25, 55, 52, 44, 6, 400 | 21, 24, 25, 55, 52, 46, 6, 21, 401 | 24, 25, 55, 54, 18, 6, 21, 24, 402 | 25, 55, 54, 20, 6, 24, 25, 49, 403 | 54, 12, 10, 6, 24, 25, 49, 54, 404 | 16, 14, 6, 24, 25, 55, 54, 12, 405 | 10, 6, 24, 25, 55, 54, 16, 14, 406 | 6, 27, 24, 25, 49, 48, 26, 6, 407 | 27, 24, 25, 49, 48, 30, 6, 27, 408 | 24, 25, 49, 48, 34, 6, 27, 24, 409 | 25, 49, 48, 38, 6, 27, 24, 25, 410 | 49, 50, 28, 6, 27, 24, 25, 49, 411 | 50, 32, 6, 27, 24, 25, 49, 50, 412 | 36, 6, 27, 24, 25, 49, 50, 40, 413 | 6, 27, 24, 25, 49, 52, 42, 6, 414 | 27, 24, 25, 49, 52, 44, 6, 27, 415 | 24, 25, 49, 52, 46, 6, 27, 24, 416 | 25, 49, 54, 18, 6, 27, 24, 25, 417 | 49, 54, 20, 6, 31, 24, 25, 49, 418 | 48, 26, 6, 31, 24, 25, 49, 48, 419 | 30, 6, 31, 24, 25, 49, 48, 34, 420 | 6, 31, 24, 25, 49, 48, 38, 6, 421 | 31, 24, 25, 49, 50, 28, 6, 31, 422 | 24, 25, 49, 50, 32, 6, 31, 24, 423 | 25, 49, 50, 36, 6, 31, 24, 25, 424 | 49, 50, 40, 6, 31, 24, 25, 49, 425 | 52, 42, 6, 31, 24, 25, 49, 52, 426 | 44, 6, 31, 24, 25, 49, 52, 46, 427 | 6, 31, 24, 25, 49, 54, 18, 6, 428 | 31, 24, 25, 49, 54, 20, 6, 35, 429 | 24, 25, 49, 48, 26, 6, 35, 24, 430 | 25, 49, 48, 30, 6, 35, 24, 25, 431 | 49, 48, 34, 6, 35, 24, 25, 49, 432 | 48, 38, 6, 35, 24, 25, 49, 50, 433 | 28, 6, 35, 24, 25, 49, 50, 32, 434 | 6, 35, 24, 25, 49, 50, 36, 6, 435 | 35, 24, 25, 49, 50, 40, 6, 35, 436 | 24, 25, 49, 52, 42, 6, 35, 24, 437 | 25, 49, 52, 44, 6, 35, 24, 25, 438 | 49, 52, 46, 6, 35, 24, 25, 49, 439 | 54, 18, 6, 35, 24, 25, 49, 54, 440 | 20, 6, 39, 24, 25, 49, 48, 26, 441 | 6, 39, 24, 25, 49, 48, 30, 6, 442 | 39, 24, 25, 49, 48, 34, 6, 39, 443 | 24, 25, 49, 48, 38, 6, 39, 24, 444 | 25, 49, 50, 28, 6, 39, 24, 25, 445 | 49, 50, 32, 6, 39, 24, 25, 49, 446 | 50, 36, 6, 39, 24, 25, 49, 50, 447 | 40, 6, 39, 24, 25, 49, 52, 42, 448 | 6, 39, 24, 25, 49, 52, 44, 6, 449 | 39, 24, 25, 49, 52, 46, 6, 39, 450 | 24, 25, 49, 54, 18, 6, 39, 24, 451 | 25, 49, 54, 20, 6, 49, 54, 12, 452 | 10, 16, 14, 6, 51, 54, 12, 10, 453 | 16, 14, 6, 53, 54, 12, 10, 16, 454 | 14, 6, 55, 54, 12, 10, 16, 14, 455 | 6, 56, 54, 12, 10, 16, 14, 7, 456 | 1, 24, 25, 49, 54, 12, 10, 7, 457 | 1, 24, 25, 49, 54, 16, 14, 7, 458 | 1, 24, 25, 55, 54, 12, 10, 7, 459 | 1, 24, 25, 55, 54, 16, 14, 7, 460 | 1, 49, 54, 12, 10, 16, 14, 7, 461 | 1, 51, 54, 12, 10, 16, 14, 7, 462 | 1, 53, 54, 12, 10, 16, 14, 7, 463 | 1, 55, 54, 12, 10, 16, 14, 7, 464 | 11, 24, 25, 55, 54, 12, 10, 7, 465 | 11, 24, 25, 55, 54, 16, 14, 7, 466 | 13, 24, 25, 55, 54, 12, 10, 7, 467 | 13, 24, 25, 55, 54, 16, 14, 7, 468 | 17, 24, 25, 55, 54, 12, 10, 7, 469 | 19, 24, 25, 55, 54, 12, 10, 7, 470 | 19, 24, 25, 55, 54, 16, 14, 7, 471 | 21, 24, 25, 55, 54, 12, 10, 7, 472 | 21, 24, 25, 55, 54, 16, 14, 7, 473 | 27, 24, 25, 49, 54, 12, 10, 7, 474 | 27, 24, 25, 49, 54, 16, 14, 7, 475 | 29, 51, 54, 12, 10, 16, 14, 7, 476 | 31, 24, 25, 49, 54, 12, 10, 7, 477 | 31, 24, 25, 49, 54, 16, 14, 7, 478 | 33, 51, 54, 12, 10, 16, 14, 7, 479 | 35, 24, 25, 49, 54, 12, 10, 7, 480 | 35, 24, 25, 49, 54, 16, 14, 7, 481 | 37, 51, 54, 12, 10, 16, 14, 7, 482 | 39, 24, 25, 49, 54, 12, 10, 7, 483 | 39, 24, 25, 49, 54, 16, 14, 7, 484 | 41, 51, 54, 12, 10, 16, 14, 7, 485 | 43, 53, 54, 12, 10, 16, 14, 7, 486 | 45, 53, 54, 12, 10, 16, 14, 7, 487 | 47, 53, 54, 12, 10, 16, 14, 8, 488 | 24, 25, 49, 54, 12, 10, 16, 14, 489 | 8, 24, 25, 55, 54, 12, 10, 16, 490 | 14, 9, 1, 24, 25, 49, 54, 12, 491 | 10, 16, 14, 9, 1, 24, 25, 55, 492 | 54, 12, 10, 16, 14, 9, 11, 24, 493 | 25, 55, 54, 12, 10, 16, 14, 9, 494 | 19, 24, 25, 55, 54, 12, 10, 16, 495 | 14, 9, 21, 24, 25, 55, 54, 12, 496 | 10, 16, 14, 9, 27, 24, 25, 49, 497 | 54, 12, 10, 16, 14, 9, 31, 24, 498 | 25, 49, 54, 12, 10, 16, 14, 9, 499 | 35, 24, 25, 49, 54, 12, 10, 16, 500 | 14, 9, 39, 24, 25, 49, 54, 12, 501 | 10, 16, 14, 502 | } 503 | 504 | var _JDR_key_offsets []int16 = []int16{ 505 | 0, 0, 14, 28, 31, 33, 41, 49, 506 | 57, 68, 79, 81, 92, 94, 96, 107, 507 | 109, 111, 113, 124, 132, 140, 142, 150, 508 | 154, 163, 169, 175, 181, 187, 189, 203, 509 | 205, 207, 221, 223, 225, 227, 241, 267, 510 | 293, 319, 346, 373, 400, 427, 454, 481, 511 | 507, 533, 559, 585, 611, 637, 667, 695, 512 | 721, 750, 776, 802, 831, 858, 884, 910, 513 | 935, 961, 987, 1014, 1039, 1065, 1091, 1117, 514 | 1144, 1171, 1197, 1222, 1247, 1274, 1300, 1326, 515 | 1352, 516 | } 517 | 518 | var _JDR_trans_keys []byte = []byte{ 519 | 10, 13, 34, 92, 128, 191, 192, 223, 520 | 224, 239, 240, 247, 248, 255, 10, 13, 521 | 34, 92, 128, 191, 192, 223, 224, 239, 522 | 240, 247, 248, 255, 48, 49, 57, 48, 523 | 57, 95, 126, 48, 57, 65, 90, 97, 524 | 122, 95, 126, 48, 57, 65, 90, 97, 525 | 122, 95, 126, 48, 57, 65, 90, 97, 526 | 122, 96, 128, 191, 192, 223, 224, 239, 527 | 240, 247, 248, 255, 96, 128, 191, 192, 528 | 223, 224, 239, 240, 247, 248, 255, 128, 529 | 191, 96, 128, 191, 192, 223, 224, 239, 530 | 240, 247, 248, 255, 128, 191, 128, 191, 531 | 96, 128, 191, 192, 223, 224, 239, 240, 532 | 247, 248, 255, 128, 191, 128, 191, 128, 533 | 191, 96, 128, 191, 192, 223, 224, 239, 534 | 240, 247, 248, 255, 95, 126, 48, 57, 535 | 65, 90, 97, 122, 95, 126, 48, 57, 536 | 65, 90, 97, 122, 48, 57, 95, 126, 537 | 48, 57, 65, 90, 97, 122, 43, 45, 538 | 48, 57, 34, 47, 92, 98, 102, 110, 539 | 114, 116, 117, 48, 57, 65, 90, 97, 540 | 102, 48, 57, 65, 90, 97, 102, 48, 541 | 57, 65, 90, 97, 102, 48, 57, 65, 542 | 90, 97, 102, 128, 191, 10, 13, 34, 543 | 92, 128, 191, 192, 223, 224, 239, 240, 544 | 247, 248, 255, 128, 191, 128, 191, 10, 545 | 13, 34, 92, 128, 191, 192, 223, 224, 546 | 239, 240, 247, 248, 255, 128, 191, 128, 547 | 191, 128, 191, 10, 13, 34, 92, 128, 548 | 191, 192, 223, 224, 239, 240, 247, 248, 549 | 255, 9, 10, 13, 32, 34, 40, 41, 550 | 44, 45, 48, 58, 59, 60, 62, 91, 551 | 93, 96, 123, 125, 126, 49, 57, 65, 552 | 90, 95, 122, 9, 10, 13, 32, 34, 553 | 40, 41, 44, 45, 48, 58, 59, 60, 554 | 62, 91, 93, 96, 123, 125, 126, 49, 555 | 57, 65, 90, 95, 122, 9, 10, 13, 556 | 32, 34, 40, 41, 44, 45, 48, 58, 557 | 59, 60, 62, 91, 93, 96, 123, 125, 558 | 126, 49, 57, 65, 90, 95, 122, 9, 559 | 10, 13, 32, 34, 40, 41, 44, 45, 560 | 48, 58, 59, 60, 62, 64, 91, 93, 561 | 96, 123, 125, 126, 49, 57, 65, 90, 562 | 95, 122, 9, 10, 13, 32, 34, 40, 563 | 41, 44, 45, 48, 58, 59, 60, 62, 564 | 64, 91, 93, 96, 123, 125, 126, 49, 565 | 57, 65, 90, 95, 122, 9, 10, 13, 566 | 32, 34, 40, 41, 44, 45, 48, 58, 567 | 59, 60, 62, 64, 91, 93, 96, 123, 568 | 125, 126, 49, 57, 65, 90, 95, 122, 569 | 9, 10, 13, 32, 34, 40, 41, 44, 570 | 45, 48, 58, 59, 60, 62, 64, 91, 571 | 93, 96, 123, 125, 126, 49, 57, 65, 572 | 90, 95, 122, 9, 10, 13, 32, 34, 573 | 40, 41, 44, 45, 48, 58, 59, 60, 574 | 62, 64, 91, 93, 96, 123, 125, 126, 575 | 49, 57, 65, 90, 95, 122, 9, 10, 576 | 13, 32, 34, 40, 41, 44, 45, 48, 577 | 58, 59, 60, 62, 64, 91, 93, 96, 578 | 123, 125, 126, 49, 57, 65, 90, 95, 579 | 122, 9, 10, 13, 32, 34, 40, 41, 580 | 44, 45, 48, 58, 59, 60, 62, 91, 581 | 93, 96, 123, 125, 126, 49, 57, 65, 582 | 90, 95, 122, 9, 10, 13, 32, 34, 583 | 40, 41, 44, 45, 48, 58, 59, 60, 584 | 62, 91, 93, 96, 123, 125, 126, 49, 585 | 57, 65, 90, 95, 122, 9, 10, 13, 586 | 32, 34, 40, 41, 44, 45, 48, 58, 587 | 59, 60, 62, 91, 93, 96, 123, 125, 588 | 126, 49, 57, 65, 90, 95, 122, 9, 589 | 10, 13, 32, 34, 40, 41, 44, 45, 590 | 48, 58, 59, 60, 62, 91, 93, 96, 591 | 123, 125, 126, 49, 57, 65, 90, 95, 592 | 122, 9, 10, 13, 32, 34, 40, 41, 593 | 44, 45, 48, 58, 59, 60, 62, 91, 594 | 93, 96, 123, 125, 126, 49, 57, 65, 595 | 90, 95, 122, 9, 10, 13, 32, 34, 596 | 40, 41, 44, 45, 48, 58, 59, 60, 597 | 62, 91, 93, 96, 123, 125, 126, 49, 598 | 57, 65, 90, 95, 122, 9, 10, 13, 599 | 32, 34, 40, 41, 44, 45, 46, 48, 600 | 58, 59, 60, 62, 64, 69, 91, 93, 601 | 96, 101, 123, 125, 126, 49, 57, 65, 602 | 90, 95, 122, 9, 10, 13, 32, 34, 603 | 40, 41, 44, 45, 58, 59, 60, 62, 604 | 64, 69, 91, 93, 96, 101, 123, 125, 605 | 126, 48, 57, 65, 90, 95, 122, 9, 606 | 10, 13, 32, 34, 40, 41, 44, 45, 607 | 48, 58, 59, 60, 62, 91, 93, 96, 608 | 123, 125, 126, 49, 57, 65, 90, 95, 609 | 122, 9, 10, 13, 32, 34, 40, 41, 610 | 44, 45, 46, 58, 59, 60, 62, 64, 611 | 69, 91, 93, 96, 101, 123, 125, 126, 612 | 48, 57, 65, 90, 95, 122, 9, 10, 613 | 13, 32, 34, 40, 41, 44, 45, 58, 614 | 59, 60, 62, 64, 91, 93, 96, 123, 615 | 125, 126, 48, 57, 65, 90, 95, 122, 616 | 9, 10, 13, 32, 34, 40, 41, 44, 617 | 45, 48, 58, 59, 60, 62, 91, 93, 618 | 96, 123, 125, 126, 49, 57, 65, 90, 619 | 95, 122, 9, 10, 13, 32, 34, 40, 620 | 41, 44, 45, 46, 58, 59, 60, 62, 621 | 64, 69, 91, 93, 96, 101, 123, 125, 622 | 126, 48, 57, 65, 90, 95, 122, 9, 623 | 10, 13, 32, 34, 40, 41, 44, 45, 624 | 48, 58, 59, 60, 62, 64, 91, 93, 625 | 96, 123, 125, 126, 49, 57, 65, 90, 626 | 95, 122, 9, 10, 13, 32, 34, 40, 627 | 41, 44, 45, 48, 58, 59, 60, 62, 628 | 91, 93, 96, 123, 125, 126, 49, 57, 629 | 65, 90, 95, 122, 9, 10, 13, 32, 630 | 34, 40, 41, 44, 45, 58, 59, 60, 631 | 62, 64, 91, 93, 96, 123, 125, 126, 632 | 48, 57, 65, 90, 95, 122, 9, 10, 633 | 13, 32, 34, 40, 41, 44, 45, 58, 634 | 59, 60, 62, 91, 93, 96, 123, 125, 635 | 126, 48, 57, 65, 90, 95, 122, 9, 636 | 10, 13, 32, 34, 40, 41, 44, 45, 637 | 48, 58, 59, 60, 62, 91, 93, 96, 638 | 123, 125, 126, 49, 57, 65, 90, 95, 639 | 122, 9, 10, 13, 32, 34, 40, 41, 640 | 44, 45, 48, 58, 59, 60, 62, 91, 641 | 93, 96, 123, 125, 126, 49, 57, 65, 642 | 90, 95, 122, 9, 10, 13, 32, 34, 643 | 40, 41, 44, 45, 48, 58, 59, 60, 644 | 62, 64, 91, 93, 96, 123, 125, 126, 645 | 49, 57, 65, 90, 95, 122, 9, 10, 646 | 13, 32, 34, 40, 41, 44, 45, 58, 647 | 59, 60, 62, 91, 93, 96, 123, 125, 648 | 126, 48, 57, 65, 90, 95, 122, 9, 649 | 10, 13, 32, 34, 40, 41, 44, 45, 650 | 48, 58, 59, 60, 62, 91, 93, 96, 651 | 123, 125, 126, 49, 57, 65, 90, 95, 652 | 122, 9, 10, 13, 32, 34, 40, 41, 653 | 44, 45, 48, 58, 59, 60, 62, 91, 654 | 93, 96, 123, 125, 126, 49, 57, 65, 655 | 90, 95, 122, 9, 10, 13, 32, 34, 656 | 40, 41, 44, 45, 48, 58, 59, 60, 657 | 62, 91, 93, 96, 123, 125, 126, 49, 658 | 57, 65, 90, 95, 122, 9, 10, 13, 659 | 32, 34, 40, 41, 44, 45, 48, 58, 660 | 59, 60, 62, 64, 91, 93, 96, 123, 661 | 125, 126, 49, 57, 65, 90, 95, 122, 662 | 9, 10, 13, 32, 34, 40, 41, 44, 663 | 45, 48, 58, 59, 60, 62, 64, 91, 664 | 93, 96, 123, 125, 126, 49, 57, 65, 665 | 90, 95, 122, 9, 10, 13, 32, 34, 666 | 40, 41, 44, 45, 48, 58, 59, 60, 667 | 62, 91, 93, 96, 123, 125, 126, 49, 668 | 57, 65, 90, 95, 122, 9, 10, 13, 669 | 32, 34, 40, 41, 44, 45, 58, 59, 670 | 60, 62, 91, 93, 96, 123, 125, 126, 671 | 48, 57, 65, 90, 95, 122, 9, 10, 672 | 13, 32, 34, 40, 41, 44, 45, 58, 673 | 59, 60, 62, 91, 93, 96, 123, 125, 674 | 126, 48, 57, 65, 90, 95, 122, 9, 675 | 10, 13, 32, 34, 40, 41, 43, 44, 676 | 45, 58, 59, 60, 62, 64, 91, 93, 677 | 96, 123, 125, 126, 48, 57, 65, 90, 678 | 95, 122, 9, 10, 13, 32, 34, 40, 679 | 41, 44, 45, 58, 59, 60, 62, 64, 680 | 91, 93, 96, 123, 125, 126, 48, 57, 681 | 65, 90, 95, 122, 9, 10, 13, 32, 682 | 34, 40, 41, 44, 45, 58, 59, 60, 683 | 62, 64, 91, 93, 96, 123, 125, 126, 684 | 48, 57, 65, 90, 95, 122, 9, 10, 685 | 13, 32, 34, 40, 41, 44, 45, 58, 686 | 59, 60, 62, 64, 91, 93, 96, 123, 687 | 125, 126, 48, 57, 65, 90, 95, 122, 688 | 9, 10, 13, 32, 34, 40, 41, 44, 689 | 45, 46, 58, 59, 60, 62, 64, 69, 690 | 91, 93, 96, 101, 123, 125, 126, 48, 691 | 57, 65, 90, 95, 122, 692 | } 693 | 694 | var _JDR_single_lengths []byte = []byte{ 695 | 0, 4, 4, 1, 0, 2, 2, 2, 696 | 1, 1, 0, 1, 0, 0, 1, 0, 697 | 0, 0, 1, 2, 2, 0, 2, 2, 698 | 9, 0, 0, 0, 0, 0, 4, 0, 699 | 0, 4, 0, 0, 0, 4, 20, 20, 700 | 20, 21, 21, 21, 21, 21, 21, 20, 701 | 20, 20, 20, 20, 20, 24, 22, 20, 702 | 23, 20, 20, 23, 21, 20, 20, 19, 703 | 20, 20, 21, 19, 20, 20, 20, 21, 704 | 21, 20, 19, 19, 21, 20, 20, 20, 705 | 23, 706 | } 707 | 708 | var _JDR_range_lengths []byte = []byte{ 709 | 0, 5, 5, 1, 1, 3, 3, 3, 710 | 5, 5, 1, 5, 1, 1, 5, 1, 711 | 1, 1, 5, 3, 3, 1, 3, 1, 712 | 0, 3, 3, 3, 3, 1, 5, 1, 713 | 1, 5, 1, 1, 1, 5, 3, 3, 714 | 3, 3, 3, 3, 3, 3, 3, 3, 715 | 3, 3, 3, 3, 3, 3, 3, 3, 716 | 3, 3, 3, 3, 3, 3, 3, 3, 717 | 3, 3, 3, 3, 3, 3, 3, 3, 718 | 3, 3, 3, 3, 3, 3, 3, 3, 719 | 3, 720 | } 721 | 722 | var _JDR_index_offsets []int16 = []int16{ 723 | 0, 0, 10, 20, 23, 25, 31, 37, 724 | 43, 50, 57, 59, 66, 68, 70, 77, 725 | 79, 81, 83, 90, 96, 102, 104, 110, 726 | 114, 124, 128, 132, 136, 140, 142, 152, 727 | 154, 156, 166, 168, 170, 172, 182, 206, 728 | 230, 254, 279, 304, 329, 354, 379, 404, 729 | 428, 452, 476, 500, 524, 548, 576, 602, 730 | 626, 653, 677, 701, 728, 753, 777, 801, 731 | 824, 848, 872, 897, 920, 944, 968, 992, 732 | 1017, 1042, 1066, 1089, 1112, 1137, 1161, 1185, 733 | 1209, 734 | } 735 | 736 | var _JDR_trans_targs []byte = []byte{ 737 | 0, 0, 41, 24, 0, 29, 31, 34, 738 | 0, 2, 0, 0, 41, 24, 0, 29, 739 | 31, 34, 0, 2, 53, 80, 0, 54, 740 | 0, 57, 57, 57, 57, 57, 0, 63, 741 | 63, 63, 63, 63, 0, 67, 67, 67, 742 | 67, 67, 0, 71, 0, 10, 12, 15, 743 | 0, 9, 71, 0, 10, 12, 15, 0, 744 | 9, 11, 0, 71, 0, 10, 12, 15, 745 | 0, 9, 13, 0, 14, 0, 71, 0, 746 | 10, 12, 15, 0, 9, 16, 0, 17, 747 | 0, 18, 0, 71, 0, 10, 12, 15, 748 | 0, 9, 74, 74, 74, 74, 74, 0, 749 | 75, 75, 75, 75, 75, 0, 77, 0, 750 | 57, 57, 78, 57, 57, 0, 21, 21, 751 | 77, 0, 1, 1, 1, 1, 1, 1, 752 | 1, 1, 25, 0, 26, 26, 26, 0, 753 | 27, 27, 27, 0, 28, 28, 28, 0, 754 | 1, 1, 1, 0, 30, 0, 0, 0, 755 | 41, 24, 0, 29, 31, 34, 0, 2, 756 | 32, 0, 33, 0, 0, 0, 41, 24, 757 | 0, 29, 31, 34, 0, 2, 35, 0, 758 | 36, 0, 37, 0, 0, 0, 41, 24, 759 | 0, 29, 31, 34, 0, 2, 39, 40, 760 | 39, 39, 1, 44, 47, 50, 3, 56, 761 | 55, 58, 60, 61, 66, 70, 8, 72, 762 | 73, 62, 59, 62, 62, 0, 39, 40, 763 | 39, 39, 1, 44, 47, 50, 3, 56, 764 | 55, 58, 60, 61, 66, 70, 8, 72, 765 | 73, 62, 59, 62, 62, 0, 39, 40, 766 | 39, 39, 1, 44, 47, 50, 3, 56, 767 | 55, 58, 60, 61, 66, 70, 8, 72, 768 | 73, 62, 59, 62, 62, 0, 42, 43, 769 | 42, 42, 1, 44, 47, 50, 3, 56, 770 | 55, 58, 60, 61, 6, 66, 70, 8, 771 | 72, 73, 62, 59, 62, 62, 0, 42, 772 | 43, 42, 42, 1, 44, 47, 50, 3, 773 | 56, 55, 58, 60, 61, 6, 66, 70, 774 | 8, 72, 73, 62, 59, 62, 62, 0, 775 | 42, 43, 42, 42, 1, 44, 47, 50, 776 | 3, 56, 55, 58, 60, 61, 6, 66, 777 | 70, 8, 72, 73, 62, 59, 62, 62, 778 | 0, 45, 46, 45, 45, 1, 44, 47, 779 | 50, 3, 56, 55, 58, 60, 61, 7, 780 | 66, 70, 8, 72, 73, 62, 59, 62, 781 | 62, 0, 45, 46, 45, 45, 1, 44, 782 | 47, 50, 3, 56, 55, 58, 60, 61, 783 | 7, 66, 70, 8, 72, 73, 62, 59, 784 | 62, 62, 0, 45, 46, 45, 45, 1, 785 | 44, 47, 50, 3, 56, 55, 58, 60, 786 | 61, 7, 66, 70, 8, 72, 73, 62, 787 | 59, 62, 62, 0, 48, 49, 48, 48, 788 | 1, 44, 47, 50, 3, 56, 55, 58, 789 | 60, 61, 66, 70, 8, 72, 73, 62, 790 | 59, 62, 62, 0, 48, 49, 48, 48, 791 | 1, 44, 47, 50, 3, 56, 55, 58, 792 | 60, 61, 66, 70, 8, 72, 73, 62, 793 | 59, 62, 62, 0, 48, 49, 48, 48, 794 | 1, 44, 47, 50, 3, 56, 55, 58, 795 | 60, 61, 66, 70, 8, 72, 73, 62, 796 | 59, 62, 62, 0, 51, 52, 51, 51, 797 | 1, 44, 47, 50, 3, 56, 55, 58, 798 | 60, 61, 66, 70, 8, 72, 73, 62, 799 | 59, 62, 62, 0, 51, 52, 51, 51, 800 | 1, 44, 47, 50, 3, 56, 55, 58, 801 | 60, 61, 66, 70, 8, 72, 73, 62, 802 | 59, 62, 62, 0, 51, 52, 51, 51, 803 | 1, 44, 47, 50, 3, 56, 55, 58, 804 | 60, 61, 66, 70, 8, 72, 73, 62, 805 | 59, 62, 62, 0, 42, 43, 42, 42, 806 | 1, 44, 47, 50, 3, 4, 56, 55, 807 | 58, 60, 61, 6, 23, 66, 70, 8, 808 | 23, 72, 73, 62, 59, 62, 62, 0, 809 | 42, 43, 42, 42, 1, 44, 47, 50, 810 | 3, 55, 58, 60, 61, 6, 23, 66, 811 | 70, 8, 23, 72, 73, 62, 54, 62, 812 | 62, 0, 51, 52, 51, 51, 1, 44, 813 | 47, 50, 3, 56, 55, 58, 60, 61, 814 | 66, 70, 8, 72, 73, 62, 59, 62, 815 | 62, 0, 42, 43, 42, 42, 1, 44, 816 | 47, 50, 5, 4, 55, 58, 60, 61, 817 | 6, 76, 66, 70, 8, 76, 72, 73, 818 | 62, 62, 62, 62, 0, 42, 43, 42, 819 | 42, 1, 44, 47, 50, 3, 55, 58, 820 | 60, 61, 6, 66, 70, 8, 72, 73, 821 | 57, 57, 57, 57, 0, 51, 52, 51, 822 | 51, 1, 44, 47, 50, 3, 56, 55, 823 | 58, 60, 61, 66, 70, 8, 72, 73, 824 | 62, 59, 62, 62, 0, 42, 43, 42, 825 | 42, 1, 44, 47, 50, 5, 4, 55, 826 | 58, 60, 61, 6, 76, 66, 70, 8, 827 | 76, 72, 73, 62, 59, 62, 62, 0, 828 | 45, 46, 45, 45, 1, 44, 47, 50, 829 | 3, 56, 55, 58, 60, 61, 7, 66, 830 | 70, 8, 72, 73, 62, 59, 62, 62, 831 | 0, 48, 49, 48, 48, 1, 44, 47, 832 | 50, 3, 56, 55, 58, 60, 61, 66, 833 | 70, 8, 72, 73, 62, 59, 62, 62, 834 | 0, 42, 43, 42, 42, 1, 44, 47, 835 | 50, 5, 55, 58, 60, 61, 6, 66, 836 | 70, 8, 72, 73, 62, 62, 62, 62, 837 | 0, 64, 65, 64, 64, 1, 44, 47, 838 | 50, 20, 55, 58, 60, 61, 66, 70, 839 | 8, 72, 73, 63, 63, 63, 63, 0, 840 | 64, 65, 64, 64, 1, 44, 47, 50, 841 | 3, 56, 55, 58, 60, 61, 66, 70, 842 | 8, 72, 73, 62, 59, 62, 62, 0, 843 | 64, 65, 64, 64, 1, 44, 47, 50, 844 | 3, 56, 55, 58, 60, 61, 66, 70, 845 | 8, 72, 73, 62, 59, 62, 62, 0, 846 | 45, 46, 45, 45, 1, 44, 47, 50, 847 | 3, 56, 55, 58, 60, 61, 7, 66, 848 | 70, 8, 72, 73, 62, 59, 62, 62, 849 | 0, 68, 69, 68, 68, 1, 44, 47, 850 | 50, 19, 55, 58, 60, 61, 66, 70, 851 | 8, 72, 73, 67, 67, 67, 67, 0, 852 | 68, 69, 68, 68, 1, 44, 47, 50, 853 | 3, 56, 55, 58, 60, 61, 66, 70, 854 | 8, 72, 73, 62, 59, 62, 62, 0, 855 | 68, 69, 68, 68, 1, 44, 47, 50, 856 | 3, 56, 55, 58, 60, 61, 66, 70, 857 | 8, 72, 73, 62, 59, 62, 62, 0, 858 | 48, 49, 48, 48, 1, 44, 47, 50, 859 | 3, 56, 55, 58, 60, 61, 66, 70, 860 | 8, 72, 73, 62, 59, 62, 62, 0, 861 | 42, 43, 42, 42, 1, 44, 47, 50, 862 | 3, 56, 55, 58, 60, 61, 6, 66, 863 | 70, 8, 72, 73, 62, 59, 62, 62, 864 | 0, 45, 46, 45, 45, 1, 44, 47, 865 | 50, 3, 56, 55, 58, 60, 61, 7, 866 | 66, 70, 8, 72, 73, 62, 59, 62, 867 | 62, 0, 48, 49, 48, 48, 1, 44, 868 | 47, 50, 3, 56, 55, 58, 60, 61, 869 | 66, 70, 8, 72, 73, 62, 59, 62, 870 | 62, 0, 68, 69, 68, 68, 1, 44, 871 | 47, 50, 3, 55, 58, 60, 61, 66, 872 | 70, 8, 72, 73, 74, 74, 74, 74, 873 | 0, 64, 65, 64, 64, 1, 44, 47, 874 | 50, 3, 55, 58, 60, 61, 66, 70, 875 | 8, 72, 73, 75, 75, 75, 75, 0, 876 | 42, 43, 42, 42, 1, 44, 47, 21, 877 | 50, 22, 55, 58, 60, 61, 6, 66, 878 | 70, 8, 72, 73, 62, 79, 62, 62, 879 | 0, 42, 43, 42, 42, 1, 44, 47, 880 | 50, 3, 55, 58, 60, 61, 6, 66, 881 | 70, 8, 72, 73, 62, 77, 62, 62, 882 | 0, 42, 43, 42, 42, 1, 44, 47, 883 | 50, 3, 55, 58, 60, 61, 6, 66, 884 | 70, 8, 72, 73, 57, 78, 57, 57, 885 | 0, 42, 43, 42, 42, 1, 44, 47, 886 | 50, 5, 55, 58, 60, 61, 6, 66, 887 | 70, 8, 72, 73, 62, 79, 62, 62, 888 | 0, 42, 43, 42, 42, 1, 44, 47, 889 | 50, 3, 4, 55, 58, 60, 61, 6, 890 | 23, 66, 70, 8, 23, 72, 73, 62, 891 | 80, 62, 62, 0, 892 | } 893 | 894 | var _JDR_trans_actions []int16 = []int16{ 895 | 0, 0, 0, 0, 0, 9, 13, 17, 896 | 0, 5, 0, 0, 7, 7, 0, 75, 897 | 78, 81, 0, 72, 0, 0, 0, 0, 898 | 0, 0, 0, 0, 0, 0, 0, 0, 899 | 0, 0, 0, 0, 0, 0, 0, 0, 900 | 0, 0, 0, 0, 0, 9, 13, 17, 901 | 0, 5, 7, 0, 75, 78, 81, 0, 902 | 72, 0, 0, 11, 0, 87, 90, 93, 903 | 0, 84, 0, 0, 0, 0, 15, 0, 904 | 99, 102, 105, 0, 96, 0, 0, 0, 905 | 0, 0, 0, 19, 0, 111, 114, 117, 906 | 0, 108, 0, 0, 0, 0, 0, 0, 907 | 0, 0, 0, 0, 0, 0, 0, 0, 908 | 0, 0, 0, 0, 0, 0, 0, 0, 909 | 0, 0, 0, 0, 0, 0, 0, 0, 910 | 0, 0, 0, 0, 0, 0, 0, 0, 911 | 0, 0, 0, 0, 0, 0, 0, 0, 912 | 0, 0, 0, 0, 0, 0, 0, 0, 913 | 11, 11, 0, 87, 90, 93, 0, 84, 914 | 0, 0, 0, 0, 0, 0, 15, 15, 915 | 0, 99, 102, 105, 0, 96, 0, 0, 916 | 0, 0, 0, 0, 0, 0, 19, 19, 917 | 0, 111, 114, 117, 0, 108, 59, 255, 918 | 59, 59, 625, 581, 597, 613, 1538, 3081, 919 | 617, 621, 593, 609, 585, 601, 629, 589, 920 | 605, 1543, 3081, 1543, 1543, 0, 0, 1, 921 | 0, 0, 246, 204, 219, 234, 521, 1920, 922 | 237, 240, 213, 228, 207, 222, 249, 210, 923 | 225, 525, 1920, 525, 525, 0, 3, 63, 924 | 3, 3, 317, 261, 281, 301, 828, 2108, 925 | 305, 309, 273, 293, 265, 285, 321, 269, 926 | 289, 833, 2108, 833, 833, 0, 29, 144, 927 | 29, 29, 2556, 2479, 2507, 2535, 3192, 3392, 928 | 2542, 2549, 2500, 2528, 147, 2486, 2514, 2563, 929 | 2493, 2521, 3200, 3392, 3200, 3200, 0, 0, 930 | 1, 0, 0, 1800, 1734, 1758, 1782, 2675, 931 | 3353, 1788, 1794, 1752, 1776, 33, 1740, 1764, 932 | 1806, 1746, 1770, 2682, 3353, 2682, 2682, 0, 933 | 3, 63, 3, 3, 2094, 2017, 2045, 2073, 934 | 3104, 3372, 2080, 2087, 2038, 2066, 66, 2024, 935 | 2052, 2101, 2031, 2059, 3112, 3372, 3112, 3112, 936 | 0, 37, 159, 37, 37, 2766, 2689, 2717, 937 | 2745, 3224, 3412, 2752, 2759, 2710, 2738, 162, 938 | 2696, 2724, 2773, 2703, 2731, 3232, 3412, 3232, 939 | 3232, 0, 0, 1, 0, 0, 1722, 1656, 940 | 1680, 1704, 2661, 3344, 1710, 1716, 1674, 1698, 941 | 33, 1662, 1686, 1728, 1668, 1692, 2668, 3344, 942 | 2668, 2668, 0, 3, 63, 3, 3, 2003, 943 | 1926, 1954, 1982, 3088, 3362, 1989, 1996, 1947, 944 | 1975, 66, 1933, 1961, 2010, 1940, 1968, 3096, 945 | 3362, 3096, 3096, 0, 39, 165, 39, 39, 946 | 1098, 1043, 1063, 1083, 1818, 3240, 1088, 1093, 947 | 1058, 1078, 1048, 1068, 1103, 1053, 1073, 1824, 948 | 3240, 1824, 1824, 0, 0, 1, 0, 0, 949 | 461, 417, 433, 449, 1508, 3060, 453, 457, 950 | 429, 445, 421, 437, 465, 425, 441, 1513, 951 | 3060, 1513, 1513, 0, 3, 63, 3, 3, 952 | 753, 698, 718, 738, 1572, 3128, 743, 748, 953 | 713, 733, 703, 723, 758, 708, 728, 1578, 954 | 3128, 1578, 1578, 0, 53, 195, 53, 53, 955 | 1358, 1303, 1323, 1343, 1884, 3320, 1348, 1353, 956 | 1318, 1338, 1308, 1328, 1363, 1313, 1333, 1890, 957 | 3320, 1890, 1890, 0, 0, 1, 0, 0, 958 | 513, 469, 485, 501, 1518, 3067, 505, 509, 959 | 481, 497, 473, 489, 517, 477, 493, 1523, 960 | 3067, 1523, 1523, 0, 3, 63, 3, 3, 961 | 818, 763, 783, 803, 1584, 3136, 808, 813, 962 | 778, 798, 768, 788, 823, 773, 793, 1590, 963 | 3136, 1590, 1590, 0, 21, 120, 21, 21, 964 | 2192, 2115, 2143, 2171, 3152, 0, 3382, 2178, 965 | 2185, 2136, 2164, 123, 0, 2122, 2150, 2199, 966 | 0, 2129, 2157, 3160, 3382, 3160, 3160, 0, 967 | 23, 126, 23, 23, 2283, 2206, 2234, 2262, 968 | 3168, 2269, 2276, 2227, 2255, 129, 0, 2213, 969 | 2241, 2290, 0, 2220, 2248, 3176, 0, 3176, 970 | 3176, 0, 55, 198, 55, 55, 1423, 1368, 971 | 1388, 1408, 1896, 3328, 1413, 1418, 1383, 1403, 972 | 1373, 1393, 1428, 1378, 1398, 1902, 3328, 1902, 973 | 1902, 0, 21, 120, 21, 21, 2192, 2115, 974 | 2143, 2171, 0, 0, 2178, 2185, 2136, 2164, 975 | 123, 0, 2122, 2150, 2199, 0, 2129, 2157, 976 | 0, 0, 0, 0, 0, 27, 138, 27, 977 | 27, 2465, 2388, 2416, 2444, 3184, 2451, 2458, 978 | 2409, 2437, 141, 2395, 2423, 2472, 2402, 2430, 979 | 0, 0, 0, 0, 0, 57, 201, 57, 980 | 57, 1488, 1433, 1453, 1473, 1908, 3336, 1478, 981 | 1483, 1448, 1468, 1438, 1458, 1493, 1443, 1463, 982 | 1914, 3336, 1914, 1914, 0, 21, 120, 21, 983 | 21, 2192, 2115, 2143, 2171, 0, 0, 2178, 984 | 2185, 2136, 2164, 123, 0, 2122, 2150, 2199, 985 | 0, 2129, 2157, 0, 0, 0, 0, 0, 986 | 49, 186, 49, 49, 3039, 2962, 2990, 3018, 987 | 3296, 3442, 3025, 3032, 2983, 3011, 189, 2969, 988 | 2997, 3046, 2976, 3004, 3304, 3442, 3304, 3304, 989 | 0, 51, 192, 51, 51, 1293, 1238, 1258, 990 | 1278, 1872, 3312, 1283, 1288, 1253, 1273, 1243, 991 | 1263, 1298, 1248, 1268, 1878, 3312, 1878, 1878, 992 | 0, 25, 132, 25, 25, 2374, 2297, 2325, 993 | 2353, 0, 2360, 2367, 2318, 2346, 135, 2304, 994 | 2332, 2381, 2311, 2339, 0, 0, 0, 0, 995 | 0, 35, 156, 35, 35, 1023, 968, 988, 996 | 1008, 0, 1013, 1018, 983, 1003, 973, 993, 997 | 1028, 978, 998, 0, 0, 0, 0, 0, 998 | 0, 1, 0, 0, 573, 529, 545, 561, 999 | 1528, 3074, 565, 569, 541, 557, 533, 549, 1000 | 577, 537, 553, 1533, 3074, 1533, 1533, 0, 1001 | 3, 63, 3, 3, 893, 838, 858, 878, 1002 | 1596, 3144, 883, 888, 853, 873, 843, 863, 1003 | 898, 848, 868, 1602, 3144, 1602, 1602, 0, 1004 | 41, 168, 41, 41, 2857, 2780, 2808, 2836, 1005 | 3248, 3422, 2843, 2850, 2801, 2829, 171, 2787, 1006 | 2815, 2864, 2794, 2822, 3256, 3422, 3256, 3256, 1007 | 0, 35, 156, 35, 35, 958, 903, 923, 1008 | 943, 0, 948, 953, 918, 938, 908, 928, 1009 | 963, 913, 933, 0, 0, 0, 0, 0, 1010 | 0, 1, 0, 0, 409, 365, 381, 397, 1011 | 1498, 3053, 401, 405, 377, 393, 369, 385, 1012 | 413, 373, 389, 1503, 3053, 1503, 1503, 0, 1013 | 3, 63, 3, 3, 688, 633, 653, 673, 1014 | 1560, 3120, 678, 683, 648, 668, 638, 658, 1015 | 693, 643, 663, 1566, 3120, 1566, 1566, 0, 1016 | 43, 174, 43, 43, 1163, 1108, 1128, 1148, 1017 | 1836, 3264, 1153, 1158, 1123, 1143, 1113, 1133, 1018 | 1168, 1118, 1138, 1842, 3264, 1842, 1842, 0, 1019 | 31, 150, 31, 31, 2647, 2570, 2598, 2626, 1020 | 3208, 3402, 2633, 2640, 2591, 2619, 153, 2577, 1021 | 2605, 2654, 2584, 2612, 3216, 3402, 3216, 3216, 1022 | 0, 45, 177, 45, 45, 2948, 2871, 2899, 1023 | 2927, 3272, 3432, 2934, 2941, 2892, 2920, 180, 1024 | 2878, 2906, 2955, 2885, 2913, 3280, 3432, 3280, 1025 | 3280, 0, 47, 183, 47, 47, 1228, 1173, 1026 | 1193, 1213, 1854, 3288, 1218, 1223, 1188, 1208, 1027 | 1178, 1198, 1233, 1183, 1203, 1860, 3288, 1860, 1028 | 1860, 0, 35, 156, 35, 35, 958, 903, 1029 | 923, 943, 1644, 948, 953, 918, 938, 908, 1030 | 928, 963, 913, 933, 0, 0, 0, 0, 1031 | 0, 35, 156, 35, 35, 1023, 968, 988, 1032 | 1008, 1650, 1013, 1018, 983, 1003, 973, 993, 1033 | 1028, 978, 998, 0, 0, 0, 0, 0, 1034 | 25, 132, 25, 25, 2374, 2297, 2325, 0, 1035 | 2353, 0, 2360, 2367, 2318, 2346, 135, 2304, 1036 | 2332, 2381, 2311, 2339, 0, 0, 0, 0, 1037 | 0, 23, 126, 23, 23, 2283, 2206, 2234, 1038 | 2262, 3168, 2269, 2276, 2227, 2255, 129, 2213, 1039 | 2241, 2290, 2220, 2248, 3176, 0, 3176, 3176, 1040 | 0, 23, 126, 23, 23, 2283, 2206, 2234, 1041 | 2262, 3168, 2269, 2276, 2227, 2255, 129, 2213, 1042 | 2241, 2290, 2220, 2248, 0, 0, 0, 0, 1043 | 0, 23, 126, 23, 23, 2283, 2206, 2234, 1044 | 2262, 0, 2269, 2276, 2227, 2255, 129, 2213, 1045 | 2241, 2290, 2220, 2248, 0, 0, 0, 0, 1046 | 0, 21, 120, 21, 21, 2192, 2115, 2143, 1047 | 2171, 3152, 0, 2178, 2185, 2136, 2164, 123, 1048 | 0, 2122, 2150, 2199, 0, 2129, 2157, 3160, 1049 | 0, 3160, 3160, 0, 1050 | } 1051 | 1052 | var _JDR_eof_actions []int16 = []int16{ 1053 | 0, 0, 0, 0, 0, 0, 0, 0, 1054 | 0, 0, 0, 0, 0, 0, 0, 0, 1055 | 0, 0, 0, 0, 0, 0, 0, 0, 1056 | 0, 0, 0, 0, 0, 0, 0, 0, 1057 | 0, 0, 0, 0, 0, 0, 258, 61, 1058 | 69, 1632, 1038, 1554, 1812, 1033, 1548, 337, 1059 | 231, 297, 353, 243, 313, 1608, 1614, 357, 1060 | 1608, 1626, 361, 1608, 1866, 349, 1620, 333, 1061 | 252, 325, 1830, 329, 216, 277, 341, 1638, 1062 | 1848, 345, 329, 333, 1620, 1614, 1614, 1614, 1063 | 1608, 1064 | } 1065 | 1066 | const JDR_start int = 38 1067 | const JDR_first_final int = 38 1068 | const JDR_error int = 0 1069 | 1070 | const JDR_en_main int = 38 1071 | 1072 | // the public API function 1073 | // 1074 | 1075 | func JDRlexer(state *JDRstate) (err error) { 1076 | 1077 | data := state.jdr 1078 | var mark0 [64]int 1079 | cs, p, pe, eof := 0, 0, len(data), len(data) 1080 | 1081 | { 1082 | cs = JDR_start 1083 | } 1084 | 1085 | { 1086 | var _klen int 1087 | var _trans int 1088 | var _acts int 1089 | var _nacts uint 1090 | var _keys int 1091 | if p == pe { 1092 | goto _test_eof 1093 | } 1094 | if cs == 0 { 1095 | goto _out 1096 | } 1097 | _resume: 1098 | _keys = int(_JDR_key_offsets[cs]) 1099 | _trans = int(_JDR_index_offsets[cs]) 1100 | 1101 | _klen = int(_JDR_single_lengths[cs]) 1102 | if _klen > 0 { 1103 | _lower := int(_keys) 1104 | var _mid int 1105 | _upper := int(_keys + _klen - 1) 1106 | for { 1107 | if _upper < _lower { 1108 | break 1109 | } 1110 | 1111 | _mid = _lower + ((_upper - _lower) >> 1) 1112 | switch { 1113 | case data[p] < _JDR_trans_keys[_mid]: 1114 | _upper = _mid - 1 1115 | case data[p] > _JDR_trans_keys[_mid]: 1116 | _lower = _mid + 1 1117 | default: 1118 | _trans += int(_mid - int(_keys)) 1119 | goto _match 1120 | } 1121 | } 1122 | _keys += _klen 1123 | _trans += _klen 1124 | } 1125 | 1126 | _klen = int(_JDR_range_lengths[cs]) 1127 | if _klen > 0 { 1128 | _lower := int(_keys) 1129 | var _mid int 1130 | _upper := int(_keys + (_klen << 1) - 2) 1131 | for { 1132 | if _upper < _lower { 1133 | break 1134 | } 1135 | 1136 | _mid = _lower + (((_upper - _lower) >> 1) & ^1) 1137 | switch { 1138 | case data[p] < _JDR_trans_keys[_mid]: 1139 | _upper = _mid - 2 1140 | case data[p] > _JDR_trans_keys[_mid+1]: 1141 | _lower = _mid + 2 1142 | default: 1143 | _trans += int((_mid - int(_keys)) >> 1) 1144 | goto _match 1145 | } 1146 | } 1147 | _trans += _klen 1148 | } 1149 | 1150 | _match: 1151 | cs = int(_JDR_trans_targs[_trans]) 1152 | 1153 | if _JDR_trans_actions[_trans] == 0 { 1154 | goto _again 1155 | } 1156 | 1157 | _acts = int(_JDR_trans_actions[_trans]) 1158 | _nacts = uint(_JDR_actions[_acts]) 1159 | _acts++ 1160 | for ; _nacts > 0; _nacts-- { 1161 | _acts++ 1162 | switch _JDR_actions[_acts-1] { 1163 | case 0: 1164 | 1165 | mark0[JDRNL] = p 1166 | case 1: 1167 | 1168 | err = JDRonNL(data[mark0[JDRNL]:p], state) 1169 | if err != nil { 1170 | p++ 1171 | goto _out 1172 | 1173 | } 1174 | 1175 | case 2: 1176 | 1177 | mark0[JDRUtf8cp1] = p 1178 | case 3: 1179 | 1180 | err = JDRonUtf8cp1(data[mark0[JDRUtf8cp1]:p], state) 1181 | if err != nil { 1182 | p++ 1183 | goto _out 1184 | 1185 | } 1186 | 1187 | case 4: 1188 | 1189 | mark0[JDRUtf8cp2] = p 1190 | case 5: 1191 | 1192 | err = JDRonUtf8cp2(data[mark0[JDRUtf8cp2]:p], state) 1193 | if err != nil { 1194 | p++ 1195 | goto _out 1196 | 1197 | } 1198 | 1199 | case 6: 1200 | 1201 | mark0[JDRUtf8cp3] = p 1202 | case 7: 1203 | 1204 | err = JDRonUtf8cp3(data[mark0[JDRUtf8cp3]:p], state) 1205 | if err != nil { 1206 | p++ 1207 | goto _out 1208 | 1209 | } 1210 | 1211 | case 8: 1212 | 1213 | mark0[JDRUtf8cp4] = p 1214 | case 9: 1215 | 1216 | err = JDRonUtf8cp4(data[mark0[JDRUtf8cp4]:p], state) 1217 | if err != nil { 1218 | p++ 1219 | goto _out 1220 | 1221 | } 1222 | 1223 | case 10: 1224 | 1225 | mark0[JDRInt] = p 1226 | case 11: 1227 | 1228 | err = JDRonInt(data[mark0[JDRInt]:p], state) 1229 | if err != nil { 1230 | p++ 1231 | goto _out 1232 | 1233 | } 1234 | 1235 | case 12: 1236 | 1237 | mark0[JDRFloat] = p 1238 | case 13: 1239 | 1240 | err = JDRonFloat(data[mark0[JDRFloat]:p], state) 1241 | if err != nil { 1242 | p++ 1243 | goto _out 1244 | 1245 | } 1246 | 1247 | case 14: 1248 | 1249 | mark0[JDRTerm] = p 1250 | case 15: 1251 | 1252 | err = JDRonTerm(data[mark0[JDRTerm]:p], state) 1253 | if err != nil { 1254 | p++ 1255 | goto _out 1256 | 1257 | } 1258 | 1259 | case 16: 1260 | 1261 | mark0[JDRRef] = p 1262 | case 17: 1263 | 1264 | err = JDRonRef(data[mark0[JDRRef]:p], state) 1265 | if err != nil { 1266 | p++ 1267 | goto _out 1268 | 1269 | } 1270 | 1271 | case 18: 1272 | 1273 | mark0[JDRString] = p 1274 | case 19: 1275 | 1276 | err = JDRonString(data[mark0[JDRString]:p], state) 1277 | if err != nil { 1278 | p++ 1279 | goto _out 1280 | 1281 | } 1282 | 1283 | case 20: 1284 | 1285 | mark0[JDRMLString] = p 1286 | case 21: 1287 | 1288 | err = JDRonMLString(data[mark0[JDRMLString]:p], state) 1289 | if err != nil { 1290 | p++ 1291 | goto _out 1292 | 1293 | } 1294 | 1295 | case 22: 1296 | 1297 | mark0[JDRStamp] = p 1298 | case 23: 1299 | 1300 | err = JDRonStamp(data[mark0[JDRStamp]:p], state) 1301 | if err != nil { 1302 | p++ 1303 | goto _out 1304 | 1305 | } 1306 | 1307 | case 24: 1308 | 1309 | mark0[JDRNoStamp] = p 1310 | case 25: 1311 | 1312 | err = JDRonNoStamp(data[mark0[JDRNoStamp]:p], state) 1313 | if err != nil { 1314 | p++ 1315 | goto _out 1316 | 1317 | } 1318 | 1319 | case 26: 1320 | 1321 | mark0[JDROpenP] = p 1322 | case 27: 1323 | 1324 | err = JDRonOpenP(data[mark0[JDROpenP]:p], state) 1325 | if err != nil { 1326 | p++ 1327 | goto _out 1328 | 1329 | } 1330 | 1331 | case 28: 1332 | 1333 | mark0[JDRCloseP] = p 1334 | case 29: 1335 | 1336 | err = JDRonCloseP(data[mark0[JDRCloseP]:p], state) 1337 | if err != nil { 1338 | p++ 1339 | goto _out 1340 | 1341 | } 1342 | 1343 | case 30: 1344 | 1345 | mark0[JDROpenL] = p 1346 | case 31: 1347 | 1348 | err = JDRonOpenL(data[mark0[JDROpenL]:p], state) 1349 | if err != nil { 1350 | p++ 1351 | goto _out 1352 | 1353 | } 1354 | 1355 | case 32: 1356 | 1357 | mark0[JDRCloseL] = p 1358 | case 33: 1359 | 1360 | err = JDRonCloseL(data[mark0[JDRCloseL]:p], state) 1361 | if err != nil { 1362 | p++ 1363 | goto _out 1364 | 1365 | } 1366 | 1367 | case 34: 1368 | 1369 | mark0[JDROpenE] = p 1370 | case 35: 1371 | 1372 | err = JDRonOpenE(data[mark0[JDROpenE]:p], state) 1373 | if err != nil { 1374 | p++ 1375 | goto _out 1376 | 1377 | } 1378 | 1379 | case 36: 1380 | 1381 | mark0[JDRCloseE] = p 1382 | case 37: 1383 | 1384 | err = JDRonCloseE(data[mark0[JDRCloseE]:p], state) 1385 | if err != nil { 1386 | p++ 1387 | goto _out 1388 | 1389 | } 1390 | 1391 | case 38: 1392 | 1393 | mark0[JDROpenX] = p 1394 | case 39: 1395 | 1396 | err = JDRonOpenX(data[mark0[JDROpenX]:p], state) 1397 | if err != nil { 1398 | p++ 1399 | goto _out 1400 | 1401 | } 1402 | 1403 | case 40: 1404 | 1405 | mark0[JDRCloseX] = p 1406 | case 41: 1407 | 1408 | err = JDRonCloseX(data[mark0[JDRCloseX]:p], state) 1409 | if err != nil { 1410 | p++ 1411 | goto _out 1412 | 1413 | } 1414 | 1415 | case 42: 1416 | 1417 | mark0[JDRComma] = p 1418 | case 43: 1419 | 1420 | err = JDRonComma(data[mark0[JDRComma]:p], state) 1421 | if err != nil { 1422 | p++ 1423 | goto _out 1424 | 1425 | } 1426 | 1427 | case 44: 1428 | 1429 | mark0[JDRColon] = p 1430 | case 45: 1431 | 1432 | err = JDRonColon(data[mark0[JDRColon]:p], state) 1433 | if err != nil { 1434 | p++ 1435 | goto _out 1436 | 1437 | } 1438 | 1439 | case 46: 1440 | 1441 | mark0[JDRSemicolon] = p 1442 | case 47: 1443 | 1444 | err = JDRonSemicolon(data[mark0[JDRSemicolon]:p], state) 1445 | if err != nil { 1446 | p++ 1447 | goto _out 1448 | 1449 | } 1450 | 1451 | case 48: 1452 | 1453 | mark0[JDROpen] = p 1454 | case 49: 1455 | 1456 | err = JDRonOpen(data[mark0[JDROpen]:p], state) 1457 | if err != nil { 1458 | p++ 1459 | goto _out 1460 | 1461 | } 1462 | 1463 | case 50: 1464 | 1465 | mark0[JDRClose] = p 1466 | case 51: 1467 | 1468 | err = JDRonClose(data[mark0[JDRClose]:p], state) 1469 | if err != nil { 1470 | p++ 1471 | goto _out 1472 | 1473 | } 1474 | 1475 | case 52: 1476 | 1477 | mark0[JDRInter] = p 1478 | case 53: 1479 | 1480 | err = JDRonInter(data[mark0[JDRInter]:p], state) 1481 | if err != nil { 1482 | p++ 1483 | goto _out 1484 | 1485 | } 1486 | 1487 | case 54: 1488 | 1489 | mark0[JDRFIRST] = p 1490 | case 55: 1491 | 1492 | err = JDRonFIRST(data[mark0[JDRFIRST]:p], state) 1493 | if err != nil { 1494 | p++ 1495 | goto _out 1496 | 1497 | } 1498 | 1499 | case 56: 1500 | 1501 | mark0[JDRRoot] = p 1502 | 1503 | } 1504 | } 1505 | 1506 | _again: 1507 | if cs == 0 { 1508 | goto _out 1509 | } 1510 | p++ 1511 | if p != pe { 1512 | goto _resume 1513 | } 1514 | _test_eof: 1515 | { 1516 | } 1517 | if p == eof { 1518 | __acts := _JDR_eof_actions[cs] 1519 | __nacts := uint(_JDR_actions[__acts]) 1520 | __acts++ 1521 | for ; __nacts > 0; __nacts-- { 1522 | __acts++ 1523 | switch _JDR_actions[__acts-1] { 1524 | case 1: 1525 | 1526 | err = JDRonNL(data[mark0[JDRNL]:p], state) 1527 | if err != nil { 1528 | p++ 1529 | goto _out 1530 | 1531 | } 1532 | 1533 | case 11: 1534 | 1535 | err = JDRonInt(data[mark0[JDRInt]:p], state) 1536 | if err != nil { 1537 | p++ 1538 | goto _out 1539 | 1540 | } 1541 | 1542 | case 13: 1543 | 1544 | err = JDRonFloat(data[mark0[JDRFloat]:p], state) 1545 | if err != nil { 1546 | p++ 1547 | goto _out 1548 | 1549 | } 1550 | 1551 | case 15: 1552 | 1553 | err = JDRonTerm(data[mark0[JDRTerm]:p], state) 1554 | if err != nil { 1555 | p++ 1556 | goto _out 1557 | 1558 | } 1559 | 1560 | case 17: 1561 | 1562 | err = JDRonRef(data[mark0[JDRRef]:p], state) 1563 | if err != nil { 1564 | p++ 1565 | goto _out 1566 | 1567 | } 1568 | 1569 | case 19: 1570 | 1571 | err = JDRonString(data[mark0[JDRString]:p], state) 1572 | if err != nil { 1573 | p++ 1574 | goto _out 1575 | 1576 | } 1577 | 1578 | case 21: 1579 | 1580 | err = JDRonMLString(data[mark0[JDRMLString]:p], state) 1581 | if err != nil { 1582 | p++ 1583 | goto _out 1584 | 1585 | } 1586 | 1587 | case 23: 1588 | 1589 | err = JDRonStamp(data[mark0[JDRStamp]:p], state) 1590 | if err != nil { 1591 | p++ 1592 | goto _out 1593 | 1594 | } 1595 | 1596 | case 24: 1597 | 1598 | mark0[JDRNoStamp] = p 1599 | case 25: 1600 | 1601 | err = JDRonNoStamp(data[mark0[JDRNoStamp]:p], state) 1602 | if err != nil { 1603 | p++ 1604 | goto _out 1605 | 1606 | } 1607 | 1608 | case 27: 1609 | 1610 | err = JDRonOpenP(data[mark0[JDROpenP]:p], state) 1611 | if err != nil { 1612 | p++ 1613 | goto _out 1614 | 1615 | } 1616 | 1617 | case 29: 1618 | 1619 | err = JDRonCloseP(data[mark0[JDRCloseP]:p], state) 1620 | if err != nil { 1621 | p++ 1622 | goto _out 1623 | 1624 | } 1625 | 1626 | case 31: 1627 | 1628 | err = JDRonOpenL(data[mark0[JDROpenL]:p], state) 1629 | if err != nil { 1630 | p++ 1631 | goto _out 1632 | 1633 | } 1634 | 1635 | case 33: 1636 | 1637 | err = JDRonCloseL(data[mark0[JDRCloseL]:p], state) 1638 | if err != nil { 1639 | p++ 1640 | goto _out 1641 | 1642 | } 1643 | 1644 | case 35: 1645 | 1646 | err = JDRonOpenE(data[mark0[JDROpenE]:p], state) 1647 | if err != nil { 1648 | p++ 1649 | goto _out 1650 | 1651 | } 1652 | 1653 | case 37: 1654 | 1655 | err = JDRonCloseE(data[mark0[JDRCloseE]:p], state) 1656 | if err != nil { 1657 | p++ 1658 | goto _out 1659 | 1660 | } 1661 | 1662 | case 39: 1663 | 1664 | err = JDRonOpenX(data[mark0[JDROpenX]:p], state) 1665 | if err != nil { 1666 | p++ 1667 | goto _out 1668 | 1669 | } 1670 | 1671 | case 41: 1672 | 1673 | err = JDRonCloseX(data[mark0[JDRCloseX]:p], state) 1674 | if err != nil { 1675 | p++ 1676 | goto _out 1677 | 1678 | } 1679 | 1680 | case 43: 1681 | 1682 | err = JDRonComma(data[mark0[JDRComma]:p], state) 1683 | if err != nil { 1684 | p++ 1685 | goto _out 1686 | 1687 | } 1688 | 1689 | case 45: 1690 | 1691 | err = JDRonColon(data[mark0[JDRColon]:p], state) 1692 | if err != nil { 1693 | p++ 1694 | goto _out 1695 | 1696 | } 1697 | 1698 | case 47: 1699 | 1700 | err = JDRonSemicolon(data[mark0[JDRSemicolon]:p], state) 1701 | if err != nil { 1702 | p++ 1703 | goto _out 1704 | 1705 | } 1706 | 1707 | case 49: 1708 | 1709 | err = JDRonOpen(data[mark0[JDROpen]:p], state) 1710 | if err != nil { 1711 | p++ 1712 | goto _out 1713 | 1714 | } 1715 | 1716 | case 51: 1717 | 1718 | err = JDRonClose(data[mark0[JDRClose]:p], state) 1719 | if err != nil { 1720 | p++ 1721 | goto _out 1722 | 1723 | } 1724 | 1725 | case 53: 1726 | 1727 | err = JDRonInter(data[mark0[JDRInter]:p], state) 1728 | if err != nil { 1729 | p++ 1730 | goto _out 1731 | 1732 | } 1733 | 1734 | case 55: 1735 | 1736 | err = JDRonFIRST(data[mark0[JDRFIRST]:p], state) 1737 | if err != nil { 1738 | p++ 1739 | goto _out 1740 | 1741 | } 1742 | 1743 | case 56: 1744 | 1745 | mark0[JDRRoot] = p 1746 | case 57: 1747 | 1748 | err = JDRonRoot(data[mark0[JDRRoot]:p], state) 1749 | if err != nil { 1750 | p++ 1751 | goto _out 1752 | 1753 | } 1754 | 1755 | } 1756 | } 1757 | } 1758 | 1759 | _out: 1760 | { 1761 | } 1762 | } 1763 | //fmt.Println("?") 1764 | if p == len(data) && cs < JDR_first_final { 1765 | return ErrIncomplete 1766 | } 1767 | if p != len(data) || cs < JDR_first_final || err != nil { 1768 | if p > len(data) { 1769 | p = len(data) 1770 | } 1771 | state.jdr = state.jdr[p:] 1772 | if err == nil { 1773 | err = ErrBadJDRSyntax 1774 | } 1775 | } 1776 | return 1777 | } 1778 | --------------------------------------------------------------------------------