├── .coveralls.yml ├── .gitignore ├── .travis.yml ├── .vscode └── launch.json ├── LICENSE ├── NOTICE ├── README.md ├── batch.go ├── block.go ├── block_reader.go ├── block_writer.go ├── crypto └── mac.go ├── db.go ├── db_info.go ├── db_internal.go ├── db_sync.go ├── db_test.go ├── docs ├── about.md ├── img │ ├── architecture-overview.png │ └── memdb-upsert.png ├── usage.md └── utp.md ├── encoding ├── base32.go ├── base8.go └── encoding.go ├── entry.go ├── errors.go ├── examples ├── memdb │ └── main.go ├── sample │ └── main.go └── simple │ └── main.go ├── expiry_window.go ├── file.go ├── file_unix.go ├── file_windows.go ├── filter.go ├── filter ├── bloom.go └── filter.go ├── go.mod ├── go.sum ├── hash ├── consistent.go ├── consistent_test.go ├── hash.go ├── modulo.go └── rand.go ├── leasing.go ├── logger.go ├── memdb ├── README.md ├── batch.go ├── block.go ├── db.go ├── db_internal.go ├── db_test.go ├── errors.go ├── meter.go ├── options.go ├── query.go ├── recovery.go ├── time_lock.go ├── time_mark.go └── tiny_log.go ├── message ├── id.go └── topic.go ├── meter.go ├── metrics ├── counter.go ├── gauage.go ├── histogram.go ├── metrics.go ├── sample.go └── timeseries.go ├── mutex.go ├── options.go ├── query.go ├── recovery.go ├── server ├── common │ ├── conn.go │ └── encode.go ├── internal │ ├── batch.go │ ├── cluster.go │ ├── cluster_leader.go │ ├── config │ │ └── config.go │ ├── conn.go │ ├── conn_cache.go │ ├── db │ │ ├── adapter.go │ │ └── unitdb │ │ │ └── adapter.go │ ├── globals.go │ ├── hdl_conn.go │ ├── message │ │ ├── messageids.go │ │ ├── security │ │ │ └── key.go │ │ └── sub.go │ ├── monitor.go │ ├── net │ │ ├── hdl_grpc.go │ │ ├── hdl_grpc_web.go │ │ ├── hdl_tcp.go │ │ ├── listener │ │ │ ├── listener.go │ │ │ └── patricia.go │ │ ├── message.go │ │ └── server.go │ ├── pkg │ │ ├── collection │ │ │ ├── arrpool.go │ │ │ ├── bpool.go │ │ │ └── payload.go │ │ ├── crypto │ │ │ └── mac.go │ │ ├── encoding │ │ │ ├── base32.go │ │ │ ├── base8.go │ │ │ └── encoding.go │ │ ├── hash │ │ │ ├── hash.go │ │ │ └── ringhash.go │ │ ├── log │ │ │ └── log.go │ │ ├── metrics │ │ │ ├── counter.go │ │ │ ├── gauage.go │ │ │ ├── histogram.go │ │ │ ├── metrics.go │ │ │ ├── sample.go │ │ │ └── timeseries.go │ │ ├── stats │ │ │ ├── buffers.go │ │ │ ├── options.go │ │ │ ├── stats.go │ │ │ └── tags.go │ │ └── uid │ │ │ ├── clientid.go │ │ │ ├── connid.go │ │ │ └── uid.go │ ├── service.go │ ├── store │ │ └── store.go │ └── types │ │ └── types.go ├── main.go ├── proto │ ├── unitdb.pb.go │ └── unitdb.proto ├── unitdb.conf └── utp │ ├── connect.go │ ├── flow_control.go │ ├── message.go │ ├── publish.go │ ├── relay.go │ └── subscribe.go ├── time_window.go ├── time_window_reader.go ├── time_window_writer.go ├── trie.go ├── uid └── uid.go └── wal ├── file.go ├── header.go ├── reader.go ├── wal.go ├── wal_test.go └── writer.go /.coveralls.yml: -------------------------------------------------------------------------------- 1 | service_name: travis-pro 2 | repo_token: u5X3P8Pb7qcgwSvcml6yT1JkYBLUPks4T -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | cmd/benchmark/** 2 | *.data 3 | *.index 4 | *.win 5 | *.log 6 | *.lease 7 | *.filter 8 | *.lock 9 | *.exe 10 | *.out 11 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.13 4 | os: 5 | - osx 6 | env: 7 | global: 8 | - secure: U4CTf4UB88XHyVGUhiCUvk/NNjkfokI6KX1uQEXkHZOp9X4JOEO295vbLETRgMHO9l4lhAouKZOJ47G6RD4PggTcheVsNxvUVeOUZ3UPtKOzte2Qpou1PGQjdnPUMj8rC+c7Hgc5C8GK9B3o1OmNzyNtDpux+xzbxZUKpCl0kW1UKCOKKApx6WWZWBNt3QeHS8q/8MO/OM/jm1G9o1U0WPx8b2dnzibpY2LcboNAPOPK5grauHoIWF/ABpq2iInB1viOTX2Zl1F/tFAGQ2Hon3BAzan1g/bjXnUKaJwz4Y/kSCBYJMxR4Pn6U60qNXYfAExYdlBMcTlJkNSfSySbivs7qROKz7ncrak025IeXaUpnmllNh+nQAZtDJhIHXfwGg+beoX/ShpkFGN4bBJjEoRAe0gY7siG6Es4RfHNThDXeWR4mGq2x5SwQV7ALpTeLUQufpGOHF3cBIu6Kb355rMkLui5zjLt/qifMAhcDC10q1Dbk6bJlWK9YDeJGhfFUIXY6+twNUksol+GGWRUchWp1LCAI0n9aeQPMEXsDZ3H4kkaTkFodcI2zIO5qYoGSpoP6/1F1naQ1pY7z5Ps8er/BBWbvB9ctBDPCmTIjHh7MHzV8BV0xPV9/NVNu+gEqF6y0C/+e8/KVSsWaVYj5ghuLeotO60NHmGhOOwqH7c= 9 | before_install: 10 | - go get golang.org/x/tools/cmd/cover 11 | - go get github.com/mattn/goveralls 12 | jobs: 13 | include: 14 | - stage: test 15 | script: 16 | - go get github.com/mattn/goveralls 17 | - go get -u github.com/rakyll/gotest 18 | - gotest -v -covermode=count -coverprofile=coverage.out ./ ./wal/... ./memdb/... 19 | - "$GOPATH/bin/goveralls -coverprofile=coverage.out -service=travis-pro" -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Launch", 9 | "type": "go", 10 | "request": "launch", 11 | "mode": "auto", 12 | "program": "${workspaceFolder}/server", 13 | "env": {}, 14 | "args": [] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Unitdb includes zerolog under the MIT License. Copyright (c) 2017 Olivier Poitrey. (https://github.com/rs/zerolog) 2 | 3 | ------------------------------------------------- 4 | Unitdb includes derived work from go-anchorhash (https://github.com/wdamron/go-anchorhash) under the MIT License: 5 | 6 | // The MIT License (MIT) 7 | // 8 | // Copyright (c) 2019 West Damron 9 | // 10 | // Permission is hereby granted, free of charge, to any person obtaining a copy 11 | // of this software and associated documentation files (the "Software"), to deal 12 | // in the Software without restriction, including without limitation the rights 13 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14 | // copies of the Software, and to permit persons to whom the Software is 15 | // furnished to do so, subject to the following conditions: 16 | // 17 | // The above copyright notice and this permission notice shall be included in all 18 | // copies or substantial portions of the Software. 19 | // 20 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 26 | // SOFTWARE. 27 | 28 | The derived work can be found in the files: 29 | hash/consistent.go 30 | hash/consistent_test.go 31 | hash/modulo.go 32 | hash/rand.go 33 | 34 | ------------------------------------------------- 35 | Unitdb includes derived work from tinode/chat under the GNU GPL v3.0. (https://github.com/tinode/chat/blob/master/LICENSE) 36 | 37 | The derived work can be found in the files: 38 | server/internal/cluster.go 39 | server/internal/cluste_leaderr.go -------------------------------------------------------------------------------- /block.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "encoding/binary" 21 | "fmt" 22 | ) 23 | 24 | const ( 25 | blockSize int32 = 4096 26 | ) 27 | 28 | type ( 29 | _IndexEntry struct { 30 | seq uint64 31 | topicSize uint16 32 | valueSize uint32 33 | msgOffset int64 34 | 35 | cache []byte // block from memdb if it exist 36 | } 37 | _IndexBlock struct { 38 | entries [entriesPerIndexBlock]_IndexEntry 39 | baseSeq uint64 40 | entryIdx uint16 41 | 42 | dirty bool 43 | leased bool 44 | } 45 | ) 46 | 47 | func blockIndex(seq uint64) int32 { 48 | return int32(float64(seq-1) / float64(entriesPerIndexBlock)) 49 | } 50 | 51 | func blockOffset(idx int32) int64 { 52 | if idx == -1 { 53 | return int64(0) 54 | } 55 | return int64(blockSize * idx) 56 | } 57 | 58 | func (e _IndexEntry) mSize() uint32 { 59 | return idSize + uint32(e.topicSize) + e.valueSize 60 | } 61 | 62 | func (b _IndexBlock) validation(blockIdx int32) error { 63 | bIdx := blockIndex(b.entries[0].seq) 64 | if bIdx != blockIdx { 65 | return fmt.Errorf("validation failed blockIdx %d, startBlockIdx %d", blockIdx, bIdx) 66 | } 67 | return nil 68 | } 69 | 70 | // marshalBinary serialized entries block into binary data. 71 | func (b _IndexBlock) marshalBinary() []byte { 72 | buf := make([]byte, blockSize) 73 | data := buf 74 | 75 | b.baseSeq = b.entries[0].seq 76 | binary.LittleEndian.PutUint64(buf[:8], b.baseSeq) 77 | buf = buf[8:] 78 | for i := 0; i < entriesPerIndexBlock; i++ { 79 | s := b.entries[i] 80 | seq := uint16(0) 81 | if s.seq != 0 { 82 | seq = uint16(int16(s.seq-b.baseSeq) + entriesPerIndexBlock) 83 | } 84 | binary.LittleEndian.PutUint16(buf[:2], seq) // marshal relative seq 85 | binary.LittleEndian.PutUint16(buf[2:4], s.topicSize) 86 | binary.LittleEndian.PutUint32(buf[4:8], s.valueSize) 87 | binary.LittleEndian.PutUint64(buf[8:16], uint64(s.msgOffset)) 88 | buf = buf[16:] 89 | } 90 | binary.LittleEndian.PutUint16(buf[:2], b.entryIdx) 91 | return data 92 | } 93 | 94 | // unmarshalBinary de-serialized entries block from binary data. 95 | func (b *_IndexBlock) unmarshalBinary(data []byte) error { 96 | b.baseSeq = binary.LittleEndian.Uint64(data[:8]) 97 | data = data[8:] 98 | for i := 0; i < entriesPerIndexBlock; i++ { 99 | _ = data[16] // bounds check hint to compiler; see golang.org/issue/14808 100 | seq := int16(binary.LittleEndian.Uint16(data[:2])) 101 | if seq == 0 { 102 | b.entries[i].seq = uint64(seq) 103 | } else { 104 | b.entries[i].seq = b.baseSeq + uint64(seq) - entriesPerIndexBlock // unmarshal from relative sequence 105 | } 106 | b.entries[i].topicSize = binary.LittleEndian.Uint16(data[2:4]) 107 | b.entries[i].valueSize = binary.LittleEndian.Uint32(data[4:8]) 108 | b.entries[i].msgOffset = int64(binary.LittleEndian.Uint64(data[8:16])) 109 | data = data[16:] 110 | } 111 | b.entryIdx = binary.LittleEndian.Uint16(data[:2]) 112 | return nil 113 | } 114 | -------------------------------------------------------------------------------- /block_reader.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | type _BlockReader struct { 20 | indexBlock _IndexBlock 21 | fs *_FileSet 22 | indexFile, dataFile *_File 23 | offset int64 24 | } 25 | 26 | func newBlockReader(fs *_FileSet) *_BlockReader { 27 | r := &_BlockReader{fs: fs} 28 | 29 | indexFile, err := fs.getFile(_FileDesc{fileType: typeIndex}) 30 | if err != nil { 31 | return r 32 | } 33 | r.indexFile = indexFile 34 | 35 | dataFile, err := fs.getFile(_FileDesc{fileType: typeData}) 36 | if err != nil { 37 | return nil 38 | } 39 | r.dataFile = dataFile 40 | 41 | return r 42 | } 43 | 44 | func (r *_BlockReader) readIndexBlock() (_IndexBlock, error) { 45 | buf, err := r.indexFile.slice(r.offset, r.offset+int64(blockSize)) 46 | if err != nil { 47 | return _IndexBlock{}, err 48 | } 49 | if err := r.indexBlock.unmarshalBinary(buf); err != nil { 50 | return _IndexBlock{}, err 51 | } 52 | 53 | return r.indexBlock, nil 54 | } 55 | 56 | func (r *_BlockReader) readEntry(seq uint64) (_IndexEntry, error) { 57 | bIdx := blockIndex(seq) 58 | r.offset = blockOffset(bIdx) 59 | b, err := r.readIndexBlock() 60 | if err != nil { 61 | return _IndexEntry{}, err 62 | } 63 | entryIdx := -1 64 | for i := 0; i < entriesPerIndexBlock; i++ { 65 | e := b.entries[i] 66 | if e.seq == seq { //topic exist in db 67 | if e.msgOffset == -1 { 68 | return _IndexEntry{}, errMsgIDDeleted 69 | } 70 | entryIdx = i 71 | break 72 | } 73 | } 74 | if entryIdx == -1 { 75 | return _IndexEntry{}, errEntryInvalid 76 | } 77 | 78 | return b.entries[entryIdx], nil 79 | } 80 | 81 | func (r *_BlockReader) readMessage(e _IndexEntry) ([]byte, []byte, error) { 82 | if e.cache != nil { 83 | return e.cache[:idSize], e.cache[e.topicSize+idSize:], nil 84 | } 85 | message, err := r.dataFile.slice(e.msgOffset, e.msgOffset+int64(e.mSize())) 86 | if err != nil { 87 | return nil, nil, err 88 | } 89 | return message[:idSize], message[e.topicSize+idSize:], nil 90 | } 91 | 92 | func (r *_BlockReader) readTopic(e _IndexEntry) ([]byte, error) { 93 | if e.cache != nil { 94 | return e.cache[idSize : e.topicSize+idSize], nil 95 | } 96 | return r.dataFile.slice(e.msgOffset+int64(idSize), e.msgOffset+int64(e.topicSize)+int64(idSize)) 97 | } 98 | -------------------------------------------------------------------------------- /crypto/mac.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package crypto 18 | 19 | import ( 20 | "crypto/cipher" 21 | "errors" 22 | 23 | "github.com/unit-io/unitdb/hash" 24 | "golang.org/x/crypto/chacha20poly1305" 25 | ) 26 | 27 | const ( 28 | // EpochSize length of epoch 29 | EpochSize = 4 30 | // MessageOffset offset for the message without overhead 31 | MessageOffset = EpochSize + 4 32 | ) 33 | 34 | // MAC has the ability to encrypt and decrypt (short) messages as long as they 35 | // share the same key and the same epoch. 36 | type MAC struct { 37 | parent cipher.AEAD 38 | salt []byte 39 | } 40 | 41 | // New builds a new MAC using a 256-bit/32 byte encryption key, a numeric epoch 42 | // and numeric pseudo-random salt 43 | func New(key []byte) (*MAC, error) { 44 | parent, err := chacha20poly1305.New(key) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | mac := new(MAC) 50 | mac.salt = make([]byte, 4) 51 | mac.parent = parent 52 | for i := 0; i < 4; i++ { 53 | mac.salt[i] = (byte(key[(4*i)+0]) << 24) | 54 | (byte(key[(4*i)+1]) << 16) | 55 | (byte(key[(4*i)+2]) << 8) | 56 | byte(key[(4*i)+3]) 57 | } 58 | 59 | return mac, nil 60 | } 61 | 62 | // Overhead returns the maximum difference between the lengths of a 63 | // plaintext and its ciphertext. 64 | func (m *MAC) Overhead() int { return m.parent.Overhead() + EpochSize } 65 | 66 | // SignatureToUint32 returns uint32 for signature 67 | func SignatureToUint32(sig []byte) uint32 { 68 | return uint32(sig[0])<<24 | uint32(sig[1])<<16 | uint32(sig[2])<<8 | uint32(sig[3]) 69 | } 70 | 71 | // Signature signature is used in the message encryption to validate signature of the message 72 | // signature is added to the destination slice 73 | func Signature(value uint32) []byte { 74 | sig := make([]byte, 4) 75 | sig[0] = byte(value >> 24) 76 | sig[1] = byte(value >> 16) 77 | sig[2] = byte(value >> 8) 78 | sig[3] = byte(value) 79 | return sig 80 | } 81 | 82 | // Encrypt encrypts src and appends to dst, returning the 83 | // resulting byte slice 84 | func (m *MAC) Encrypt(dst, src []byte) []byte { 85 | //Copy first 4 bytes epoch from source 86 | dst = append(dst, src[:EpochSize]...) 87 | h := hash.New(src) 88 | dst = append(dst, Signature(h)...) 89 | nonce := append(m.salt, dst[:MessageOffset]...) 90 | return m.parent.Seal(dst, nonce, src[EpochSize:], nil) 91 | } 92 | 93 | // Decrypt decrypts src and appends to dst, returning the 94 | // resulting byte slice or an error if the input cannot be 95 | // authenticated. 96 | func (m *MAC) Decrypt(dst, src []byte) ([]byte, error) { 97 | 98 | if len(src) < m.Overhead() { 99 | return dst, errors.New("Authentication failed.") 100 | } 101 | 102 | nonce := append(m.salt, src[:MessageOffset]...) 103 | dst, err := m.parent.Open(dst, nonce, src[MessageOffset:], nil) 104 | if err != nil { 105 | return dst, errors.New("Authentication failed.") 106 | } 107 | // Append epoch to dst at the beginning 108 | dst = append(src[:EpochSize], dst...) 109 | return dst, nil 110 | } 111 | -------------------------------------------------------------------------------- /db_info.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "encoding/binary" 21 | ) 22 | 23 | var ( 24 | signature = [7]byte{'u', 'n', 'i', 't', 'd', 'b', '\x0e'} 25 | fixed = uint32(32) 26 | ) 27 | 28 | type ( 29 | _Header struct { 30 | signature [7]byte 31 | version uint32 32 | } 33 | _DBInfo struct { 34 | header _Header 35 | encryption int8 36 | sequence uint64 37 | count uint64 38 | } 39 | ) 40 | 41 | // MarshalBinary serializes db info into binary data. 42 | func (inf _DBInfo) MarshalBinary() ([]byte, error) { 43 | buf := make([]byte, fixed) 44 | copy(buf[:7], inf.header.signature[:]) 45 | binary.LittleEndian.PutUint32(buf[7:11], inf.header.version) 46 | buf[12] = uint8(inf.encryption) 47 | binary.LittleEndian.PutUint64(buf[12:20], inf.sequence) 48 | binary.LittleEndian.PutUint64(buf[20:28], inf.count) 49 | 50 | return buf, nil 51 | } 52 | 53 | // UnmarshalBinary de-serializes db info from binary data. 54 | func (inf *_DBInfo) UnmarshalBinary(data []byte) error { 55 | copy(inf.header.signature[:], data[:7]) 56 | inf.header.version = binary.LittleEndian.Uint32(data[7:11]) 57 | inf.encryption = int8(data[7]) 58 | inf.sequence = binary.LittleEndian.Uint64(data[12:20]) 59 | inf.count = binary.LittleEndian.Uint64(data[20:28]) 60 | 61 | return nil 62 | } 63 | -------------------------------------------------------------------------------- /docs/img/architecture-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unit-io/unitdb/2a4e641b8a6b68abbd43ad36a13c1605a1da2c4b/docs/img/architecture-overview.png -------------------------------------------------------------------------------- /docs/img/memdb-upsert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unit-io/unitdb/2a4e641b8a6b68abbd43ad36a13c1605a1da2c4b/docs/img/memdb-upsert.png -------------------------------------------------------------------------------- /encoding/base32.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | func init() { 4 | for i := 0; i < len(dec); i++ { 5 | dec[i] = 0xFF 6 | } 7 | for i := 0; i < len(encoding); i++ { 8 | dec[encoding[i]] = byte(i) 9 | } 10 | } 11 | 12 | // Encode32 encode by unrolling the stdlib base32 algorithm + removing all safe checks 13 | func Encode32(dst, id []byte) { 14 | dst[0] = encoding[id[0]>>3] 15 | dst[1] = encoding[(id[1]>>6)&0x1F|(id[0]<<2)&0x1F] 16 | dst[2] = encoding[(id[1]>>1)&0x1F] 17 | dst[3] = encoding[(id[2]>>4)&0x1F|(id[1]<<4)&0x1F] 18 | dst[4] = encoding[id[3]>>7|(id[2]<<1)&0x1F] 19 | dst[5] = encoding[(id[3]>>2)&0x1F] 20 | dst[6] = encoding[id[4]>>5|(id[3]<<3)&0x1F] 21 | dst[7] = encoding[id[4]&0x1F] 22 | dst[8] = encoding[id[5]>>3] 23 | dst[9] = encoding[(id[6]>>6)&0x1F|(id[5]<<2)&0x1F] 24 | dst[10] = encoding[(id[6]>>1)&0x1F] 25 | dst[11] = encoding[(id[7]>>4)&0x1F|(id[6]<<4)&0x1F] 26 | dst[12] = encoding[id[8]>>7|(id[7]<<1)&0x1F] 27 | dst[13] = encoding[(id[8]>>2)&0x1F] 28 | dst[14] = encoding[(id[9]>>5)|(id[8]<<3)&0x1F] 29 | dst[15] = encoding[id[9]&0x1F] 30 | dst[16] = encoding[id[10]>>3] 31 | dst[17] = encoding[(id[11]>>6)&0x1F|(id[10]<<2)&0x1F] 32 | dst[18] = encoding[(id[11]>>1)&0x1F] 33 | dst[19] = encoding[(id[12]>>4)&0x1F|(id[11]<<4)&0x1F] 34 | dst[20] = encoding[id[13]>>7|(id[12]<<1)&0x1F] 35 | dst[21] = encoding[(id[13]>>2)&0x1F] 36 | dst[22] = encoding[(id[14]>>5)|(id[13]<<3)&0x1F] 37 | dst[23] = encoding[id[14]&0x1F] 38 | dst[24] = encoding[id[15]>>3] 39 | dst[25] = encoding[(id[16]>>6)&0x1F|(id[15]<<2)&0x1F] 40 | dst[26] = encoding[(id[16]>>1)&0x1F] 41 | dst[27] = encoding[(id[17]>>4)&0x1F|(id[16]<<4)&0x1F] 42 | dst[28] = encoding[id[18]>>7|(id[17]<<1)&0x1F] 43 | dst[29] = encoding[(id[18]>>2)&0x1F] 44 | dst[30] = encoding[(id[19]>>5)|(id[18]<<3)&0x1F] 45 | dst[31] = encoding[id[19]&0x1F] 46 | dst[32] = encoding[id[20]>>3] 47 | dst[33] = encoding[(id[21]>>6)&0x1F|(id[20]<<2)&0x1F] 48 | dst[34] = encoding[(id[21]>>1)&0x1F] 49 | dst[35] = encoding[(id[22]>>4)&0x1F|(id[21]<<4)&0x1F] 50 | dst[36] = encoding[id[23]>>7|(id[22]<<1)&0x1F] 51 | dst[37] = encoding[(id[23]>>2)&0x1F] 52 | dst[38] = encoding[(id[24]>>5)|(id[23]<<3)&0x1F] 53 | dst[39] = encoding[id[24]&0x1F] 54 | dst[40] = encoding[id[25]>>3] 55 | dst[41] = encoding[(id[26]>>6)&0x1F|(id[25]<<2)&0x1F] 56 | dst[42] = encoding[(id[26]>>1)&0x1F] 57 | dst[43] = encoding[(id[27]>>4)&0x1F|(id[26]<<4)&0x1F] 58 | dst[44] = encoding[id[28]>>7|(id[27]<<1)&0x1F] 59 | dst[45] = encoding[(id[28]>>2)&0x1F] 60 | dst[46] = encoding[(id[29]>>5)|(id[28]<<3)&0x1F] 61 | dst[47] = encoding[id[29]&0x1F] 62 | dst[48] = encoding[id[30]>>3] 63 | dst[49] = encoding[(id[31]>>6)&0x1F|(id[30]<<2)&0x1F] 64 | dst[50] = encoding[(id[31]>>1)&0x1F] 65 | dst[51] = encoding[(id[31]<<4)&0x1F] 66 | } 67 | 68 | // Decode32 decode by unrolling the stdlib base32 algorithm + removing all safe checks 69 | func Decode32(id []byte, src []byte) { 70 | id[0] = dec[src[0]]<<3 | dec[src[1]]>>2 71 | id[1] = dec[src[1]]<<6 | dec[src[2]]<<1 | dec[src[3]]>>4 72 | id[2] = dec[src[3]]<<4 | dec[src[4]]>>1 73 | id[3] = dec[src[4]]<<7 | dec[src[5]]<<2 | dec[src[6]]>>3 74 | id[4] = dec[src[6]]<<5 | dec[src[7]] 75 | id[5] = dec[src[8]]<<3 | dec[src[9]]>>2 76 | id[6] = dec[src[9]]<<6 | dec[src[10]]<<1 | dec[src[11]]>>4 77 | id[7] = dec[src[11]]<<4 | dec[src[12]]>>1 78 | id[8] = dec[src[12]]<<7 | dec[src[13]]<<2 | dec[src[14]]>>3 79 | id[9] = dec[src[14]]<<5 | dec[src[15]] 80 | id[10] = dec[src[16]]<<3 | dec[src[17]]>>2 81 | id[11] = dec[src[17]]<<6 | dec[src[18]]<<1 | dec[src[19]]>>4 82 | id[12] = dec[src[19]]<<4 | dec[src[20]]>>1 83 | id[13] = dec[src[20]]<<7 | dec[src[21]]<<2 | dec[src[22]]>>3 84 | id[14] = dec[src[22]]<<5 | dec[src[23]] 85 | id[15] = dec[src[24]]<<3 | dec[src[25]]>>2 86 | id[16] = dec[src[25]]<<6 | dec[src[26]]<<1 | dec[src[27]]>>4 87 | id[17] = dec[src[27]]<<4 | dec[src[28]]>>1 88 | id[18] = dec[src[28]]<<7 | dec[src[29]]<<2 | dec[src[30]]>>3 89 | id[19] = dec[src[30]]<<5 | dec[src[31]] 90 | id[20] = dec[src[32]]<<3 | dec[src[33]]>>2 91 | id[21] = dec[src[33]]<<6 | dec[src[34]]<<1 | dec[src[35]]>>4 92 | id[22] = dec[src[35]]<<4 | dec[src[36]]>>1 93 | id[23] = dec[src[36]]<<7 | dec[src[37]]<<2 | dec[src[38]]>>3 94 | id[24] = dec[src[38]]<<5 | dec[src[39]] 95 | id[25] = dec[src[40]]<<3 | dec[src[41]]>>2 96 | id[26] = dec[src[41]]<<6 | dec[src[42]]<<1 | dec[src[43]]>>4 97 | id[27] = dec[src[43]]<<4 | dec[src[44]]>>1 98 | id[28] = dec[src[44]]<<7 | dec[src[45]]<<2 | dec[src[46]]>>3 99 | id[29] = dec[src[46]]<<5 | dec[src[47]] 100 | id[30] = dec[src[48]]<<3 | dec[src[49]]>>2 101 | id[31] = dec[src[49]]<<6 | dec[src[50]]<<1 | dec[src[51]]>>4 102 | } 103 | -------------------------------------------------------------------------------- /encoding/base8.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | func init() { 4 | for i := 0; i < len(dec); i++ { 5 | dec[i] = 0xFF 6 | } 7 | for i := 0; i < len(encoding); i++ { 8 | dec[encoding[i]] = byte(i) 9 | } 10 | } 11 | 12 | // Encode8 encode by unrolling the stdlib base32 algorithm + removing all safe checks 13 | func Encode8(dst, id []byte) { 14 | dst[0] = encoding[id[0]>>3] 15 | dst[1] = encoding[(id[1]>>6)&0x1F|(id[0]<<2)&0x1F] 16 | dst[2] = encoding[(id[1]>>1)&0x1F] 17 | dst[3] = encoding[(id[2]>>4)&0x1F|(id[1]<<4)&0x1F] 18 | dst[4] = encoding[id[3]>>7|(id[2]<<1)&0x1F] 19 | dst[5] = encoding[(id[3]>>2)&0x1F] 20 | dst[6] = encoding[id[4]>>5|(id[3]<<3)&0x1F] 21 | dst[7] = encoding[id[4]&0x1F] 22 | dst[8] = encoding[id[5]>>3] 23 | dst[9] = encoding[(id[6]>>6)&0x1F|(id[5]<<2)&0x1F] 24 | dst[10] = encoding[(id[6]>>1)&0x1F] 25 | dst[11] = encoding[(id[7]>>4)&0x1F|(id[6]<<4)&0x1F] 26 | dst[12] = encoding[(id[7]<<1)&0x1F] 27 | } 28 | 29 | // Decode8 decode by unrolling the stdlib base32 algorithm + removing all safe checks 30 | func Decode8(id []byte, src []byte) { 31 | id[0] = dec[src[0]]<<3 | dec[src[1]]>>2 32 | id[1] = dec[src[1]]<<6 | dec[src[2]]<<1 | dec[src[3]]>>4 33 | id[2] = dec[src[3]]<<4 | dec[src[4]]>>1 34 | id[3] = dec[src[4]]<<7 | dec[src[5]]<<2 | dec[src[6]]>>3 35 | id[4] = dec[src[6]]<<5 | dec[src[7]] 36 | id[5] = dec[src[8]]<<3 | dec[src[9]]>>2 37 | id[6] = dec[src[9]]<<6 | dec[src[10]]<<1 | dec[src[11]]>>4 38 | id[7] = dec[src[11]]<<4 | dec[src[12]]>>1 39 | } 40 | -------------------------------------------------------------------------------- /encoding/encoding.go: -------------------------------------------------------------------------------- 1 | package encoding 2 | 3 | const ( 4 | // encoding stores a custom version of the base32 encoding. 5 | encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" 6 | ) 7 | 8 | var ( 9 | // dec is the decoding map for base32 encoding 10 | dec [256]byte 11 | ) 12 | -------------------------------------------------------------------------------- /entry.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "encoding/binary" 21 | "strconv" 22 | "time" 23 | "unsafe" 24 | ) 25 | 26 | const ( 27 | entrySize = 26 28 | ) 29 | 30 | type ( 31 | _Entry struct { 32 | seq uint64 33 | topicSize uint16 34 | valueSize uint32 35 | expiresAt uint32 // expiresAt for recovery from log and not persisted to index file but persisted to the time window file. 36 | 37 | parsed bool 38 | topicHash uint64 // topicHash for recovery from log and not persisted to the DB. 39 | cache []byte // entry from memdb if it exist. 40 | } 41 | // Entry entry is a message entry structure. 42 | Entry struct { 43 | entry _Entry 44 | ID []byte // The ID of the message. 45 | Topic []byte // The topic of the message. 46 | Payload []byte // The payload of the message. 47 | ExpiresAt uint32 // The time expiry of the message. 48 | Contract uint32 // The contract is used to as salt to hash topic parts and also used as prefix in the message ID. 49 | Encryption bool 50 | } 51 | ) 52 | 53 | // NewEntry creates a new entry structure from the topic. 54 | func NewEntry(topic, payload []byte) *Entry { 55 | return &Entry{ 56 | Topic: topic, 57 | Payload: payload, 58 | } 59 | } 60 | 61 | // WithID sets entry ID. 62 | func (e *Entry) WithID(id []byte) *Entry { 63 | e.ID = id 64 | return e 65 | } 66 | 67 | // WithPayload sets payload to put entry into DB. 68 | func (e *Entry) WithPayload(payload []byte) *Entry { 69 | e.Payload = payload 70 | return e 71 | } 72 | 73 | // WithContract sets contract on entry. 74 | func (e *Entry) WithContract(contract uint32) *Entry { 75 | e.Contract = contract 76 | return e 77 | } 78 | 79 | // WithTTL sets TTL for message expiry for the entry. 80 | func (e *Entry) WithTTL(ttl string) *Entry { 81 | val, err := strconv.ParseInt(ttl, 10, 64) 82 | if err == nil { 83 | e.ExpiresAt = uint32(time.Now().Add(time.Duration(int(val)) * time.Second).Unix()) 84 | } 85 | var duration time.Duration 86 | duration, _ = time.ParseDuration(ttl) 87 | e.ExpiresAt = uint32(time.Now().Add(duration).Unix()) 88 | return e 89 | } 90 | 91 | // WithEncryption sets encryption on entry. 92 | func (e *Entry) WithEncryption() *Entry { 93 | e.Encryption = true 94 | return e 95 | } 96 | 97 | func (e *Entry) reset() { 98 | e.entry.seq = 0 99 | e.entry.topicSize = 0 100 | e.entry.cache = nil 101 | e.ID = nil 102 | e.Payload = nil 103 | } 104 | 105 | func (e _Entry) ExpiresAt() uint32 { 106 | return e.expiresAt 107 | } 108 | 109 | // MarshalBinary serialized entry into binary data. 110 | func (e _Entry) MarshalBinary() ([]byte, error) { 111 | buf := make([]byte, entrySize) 112 | data := buf 113 | binary.LittleEndian.PutUint64(buf[:8], e.seq) 114 | binary.LittleEndian.PutUint16(buf[8:10], e.topicSize) 115 | binary.LittleEndian.PutUint32(buf[10:14], e.valueSize) 116 | binary.LittleEndian.PutUint32(buf[14:18], e.expiresAt) 117 | binary.LittleEndian.PutUint64(buf[18:26], e.topicHash) 118 | return data, nil 119 | } 120 | 121 | // MarshalBinary de-serialized entry from binary data. 122 | func (e *_Entry) UnmarshalBinary(data []byte) error { 123 | e.seq = binary.LittleEndian.Uint64(data[:8]) 124 | e.topicSize = binary.LittleEndian.Uint16(data[8:10]) 125 | e.valueSize = binary.LittleEndian.Uint32(data[10:14]) 126 | e.expiresAt = binary.LittleEndian.Uint32(data[14:18]) 127 | e.topicHash = binary.LittleEndian.Uint64(data[18:26]) 128 | return nil 129 | } 130 | 131 | // unsafeToString is used to convert a slice 132 | // of bytes to a string without incurring overhead. 133 | func unsafeToString(bs []byte) string { 134 | return *(*string)(unsafe.Pointer(&bs)) 135 | } 136 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "errors" 21 | ) 22 | 23 | var ( 24 | errTopicEmpty = errors.New("Topic is empty") 25 | errMsgIDEmpty = errors.New("Message ID is empty") 26 | errMsgIDDeleted = errors.New("Message ID is deleted") 27 | errMsgIDDoesNotExist = errors.New("Message ID does not exist in database") 28 | errMsgIDPrefixMismatch = errors.New("Message ID does not match topic or Contract") 29 | errTtlTooLarge = errors.New("TTL is too large") 30 | errTopicTooLarge = errors.New("Topic is too large") 31 | errMsgExpired = errors.New("Message has expired") 32 | errValueEmpty = errors.New("Payload is empty") 33 | errValueTooLarge = errors.New("value is too large") 34 | errEntryInvalid = errors.New("entry is invalid") 35 | errEntryExist = errors.New("entry exist in database") 36 | errImmutable = errors.New("database is immutable") 37 | errFull = errors.New("database is full") 38 | errCorrupted = errors.New("database is corrupted") 39 | errLocked = errors.New("database is locked") 40 | errClosed = errors.New("database is closed") 41 | errBatchSeqComplete = errors.New("batch seq is complete") 42 | errWriteConflict = errors.New("batch write conflict") 43 | errBadRequest = errors.New("The request was invalid or cannot be otherwise served") 44 | errForbidden = errors.New("The request is understood, but it has been refused or access is not allowed") 45 | ) 46 | -------------------------------------------------------------------------------- /examples/memdb/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "time" 7 | 8 | "github.com/unit-io/unitdb/memdb" 9 | ) 10 | 11 | func main() { 12 | // Opening a database. 13 | db, err := memdb.Open(memdb.WithLogFilePath("example")) 14 | if err != nil { 15 | log.Fatal(err) 16 | return 17 | } 18 | defer db.Close() 19 | 20 | gets := func(start, end int) { 21 | for i := start; i <= end; i++ { 22 | // Get message 23 | if msg, err := db.Get(uint64(i)); err == nil { 24 | log.Printf("%s ", msg) 25 | } 26 | } 27 | } 28 | 29 | fmt.Println("main: Print recovered entries on DB open") 30 | gets(1, 10) 31 | 32 | puts := func(start, end int) { 33 | for i := start; i <= end; i++ { 34 | val := []byte(fmt.Sprintf("msg.%2d", i)) 35 | db.Put(uint64(i), val) 36 | } 37 | time.Sleep(100 * time.Millisecond) 38 | } 39 | 40 | deletes := func(start, end int) { 41 | for i := start; i <= end; i++ { 42 | if err := db.Delete(uint64(i)); err != nil { 43 | fmt.Println("error: ", err) 44 | } 45 | } 46 | } 47 | 48 | batch := func(start, end int) { 49 | err = db.Batch(func(b *memdb.Batch, completed <-chan struct{}) error { 50 | for i := start; i <= end; i++ { 51 | val := []byte(fmt.Sprintf("msg.%2d", i)) 52 | b.Put(uint64(i), val) 53 | } 54 | return nil 55 | }) 56 | } 57 | 58 | fmt.Println("main: Put 1-3") 59 | puts(1, 3) 60 | gets(1, 5) 61 | fmt.Println("main: Delete 1-2") 62 | deletes(1, 2) 63 | fmt.Println("main: Put 4-5") 64 | puts(4, 5) 65 | fmt.Println("main: Delete 3-5") 66 | deletes(3, 5) 67 | fmt.Println("main: Batch Put 4-10") 68 | batch(4, 10) 69 | fmt.Println("main: Delete 4-5") 70 | deletes(4, 5) 71 | gets(1, 10) 72 | // print stats 73 | if varz, err := db.Varz(); err == nil { 74 | fmt.Printf("%+v\n", varz) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /examples/simple/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/unit-io/unitdb" 7 | ) 8 | 9 | func main() { 10 | // Opening a database. 11 | // Open DB with Mutable flag to allow deleting messages 12 | db, err := unitdb.Open("example", unitdb.WithDefaultOptions(), unitdb.WithMutable()) 13 | if err != nil { 14 | log.Fatal(err) 15 | return 16 | } 17 | defer db.Close() 18 | 19 | topic := []byte("teams.private.sales.saleschannel.message") 20 | msg := []byte("msg for sales channel") 21 | db.Put(topic, msg) 22 | 23 | // Send message to all channels in team sales 24 | topic = []byte("teams.private.sales.*.message") 25 | msg = []byte("msg for team alpha channel1 all recipients") 26 | db.Put(topic, msg) 27 | 28 | // Send message to all private teams 29 | topic = []byte("teams.private...") 30 | msg = []byte("msg for team alpha all channels") 31 | db.Put(topic, msg) 32 | 33 | // Get message for sales channel 34 | if msgs, err := db.Get(unitdb.NewQuery([]byte("teams.private.sales.saleschannel.message?last=1h")).WithLimit(10)); err == nil { 35 | for _, msg := range msgs { 36 | log.Printf("%s ", msg) 37 | } 38 | } 39 | 40 | // Delete message 41 | messageId := db.NewID() 42 | entry := unitdb.NewEntry([]byte("teams.public.customersupport.connectchannel.message"), []byte("msg for customer connect channel")).WithID(messageId) 43 | db.PutEntry(entry) 44 | 45 | db.Sync() 46 | 47 | err = db.DeleteEntry(unitdb.NewEntry([]byte("teams.public.customersupport.connectchannel.message"), nil).WithID(messageId)) 48 | if err != nil { 49 | log.Fatal(err) 50 | return 51 | } 52 | 53 | // Get message for customer connect channel 54 | if msgs, err := db.Get(unitdb.NewQuery([]byte("teams.public.customersupport.connectchannel.message?last=1h")).WithLimit(10)); err == nil { 55 | for _, msg := range msgs { 56 | log.Printf("%s ", msg) 57 | } 58 | } 59 | 60 | // Topic isolation using contract 61 | contract, err := db.NewContract() 62 | 63 | db.PutEntry(unitdb.NewEntry([]byte("teams.public.customersupport.connectchannel.message"), []byte("msg for customer connect channel")).WithContract(contract)) 64 | 65 | // Get message for customer connect chanel with new contract 66 | if msgs, err := db.Get(unitdb.NewQuery([]byte("teams.public.customersupport.connectchannel.message?last=1h")).WithContract(contract).WithLimit(10)); err == nil { 67 | for _, msg := range msgs { 68 | log.Printf("%s ", msg) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /expiry_window.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "sort" 21 | "sync" 22 | "sync/atomic" 23 | "time" 24 | 25 | "github.com/unit-io/unitdb/hash" 26 | ) 27 | 28 | type ( 29 | _ExpiryWindowEntries []timeWindowEntry 30 | 31 | timeWindowEntry interface { 32 | expiryTime() uint32 33 | } 34 | 35 | _ExpiryWindow struct { 36 | windows map[int64]_ExpiryWindowEntries // map[expiryHash]windowEntries. 37 | 38 | mu sync.RWMutex // Read Write mutex, guards access to internal collection. 39 | } 40 | 41 | _ExpiryWindows struct { 42 | sync.RWMutex 43 | expiry []*_ExpiryWindow 44 | consistent *hash.Consistent 45 | } 46 | 47 | _ExpiryWindowBucket struct { 48 | sync.RWMutex 49 | expiryWindows *_ExpiryWindows 50 | 51 | expDurationType time.Duration 52 | maxExpDurations int 53 | backgroundKeyExpiry bool 54 | earliestExpiryHash int64 55 | } 56 | ) 57 | 58 | // newExpiryWindows creates a new concurrent expiryWindows. 59 | func newExpiryWindows() *_ExpiryWindows { 60 | w := &_ExpiryWindows{ 61 | expiry: make([]*_ExpiryWindow, nBlocks), 62 | consistent: hash.InitConsistent(nBlocks, nBlocks), 63 | } 64 | 65 | for i := 0; i < nBlocks; i++ { 66 | w.expiry[i] = &_ExpiryWindow{windows: make(map[int64]_ExpiryWindowEntries)} 67 | } 68 | 69 | return w 70 | } 71 | 72 | // getWindows returns shard under given key. 73 | func (w *_ExpiryWindows) getWindows(key uint64) *_ExpiryWindow { 74 | w.RLock() 75 | defer w.RUnlock() 76 | return w.expiry[w.consistent.FindBlock(key)] 77 | } 78 | 79 | func newExpiryWindowBucket(bgKeyExp bool, expDurType time.Duration, maxExpDur int) *_ExpiryWindowBucket { 80 | ex := &_ExpiryWindowBucket{backgroundKeyExpiry: bgKeyExp, expDurationType: expDurType, maxExpDurations: maxExpDur} 81 | ex.expiryWindows = newExpiryWindows() 82 | return ex 83 | } 84 | 85 | func (wb *_ExpiryWindowBucket) getExpiredEntries(maxResults int) []timeWindowEntry { 86 | if !wb.backgroundKeyExpiry { 87 | return nil 88 | } 89 | var expiredEntries []timeWindowEntry 90 | startTime := uint32(time.Now().Unix()) 91 | 92 | if atomic.LoadInt64(&wb.earliestExpiryHash) > int64(startTime) { 93 | return expiredEntries 94 | } 95 | 96 | for i := 0; i < wb.maxExpDurations; i++ { 97 | // get windows shard. 98 | ws := wb.expiryWindows.expiry[i] 99 | ws.mu.Lock() 100 | defer ws.mu.Unlock() 101 | windowTimes := make([]int64, 0, len(ws.windows)) 102 | for windowTime := range ws.windows { 103 | windowTimes = append(windowTimes, windowTime) 104 | } 105 | sort.Slice(windowTimes[:], func(i, j int) bool { return windowTimes[i] < windowTimes[j] }) 106 | for i := 0; i < len(windowTimes); i++ { 107 | if windowTimes[i] > int64(startTime) || len(expiredEntries) > maxResults { 108 | break 109 | } 110 | windowEntries := ws.windows[windowTimes[i]] 111 | expiredEntriesCount := 0 112 | for i := range windowEntries { 113 | entry := windowEntries[i] 114 | if entry.expiryTime() < startTime { 115 | expiredEntries = append(expiredEntries, entry) 116 | expiredEntriesCount++ 117 | } 118 | } 119 | if expiredEntriesCount == len(windowEntries) { 120 | delete(ws.windows, windowTimes[i]) 121 | } 122 | } 123 | } 124 | atomic.StoreInt64(&wb.earliestExpiryHash, 0) 125 | return expiredEntries 126 | } 127 | 128 | // addExpiry adds expiry for entries expiring. Entries expires in future are not added to expiry window. 129 | func (wb *_ExpiryWindowBucket) addExpiry(e timeWindowEntry) error { 130 | if !wb.backgroundKeyExpiry { 131 | return nil 132 | } 133 | timeExpiry := int64(time.Unix(int64(e.expiryTime()), 0).Truncate(wb.expDurationType).Add(1 * wb.expDurationType).Unix()) 134 | atomic.CompareAndSwapInt64(&wb.earliestExpiryHash, 0, timeExpiry) 135 | 136 | // get windows shard. 137 | ws := wb.expiryWindows.getWindows(uint64(e.expiryTime())) 138 | ws.mu.Lock() 139 | defer ws.mu.Unlock() 140 | if expiryWindow, ok := ws.windows[timeExpiry]; ok { 141 | expiryWindow = append(expiryWindow, e) 142 | ws.windows[timeExpiry] = expiryWindow 143 | } else { 144 | ws.windows[timeExpiry] = _ExpiryWindowEntries{e} 145 | } 146 | 147 | return nil 148 | } 149 | -------------------------------------------------------------------------------- /file_unix.go: -------------------------------------------------------------------------------- 1 | // +build !windows 2 | 3 | /* 4 | * Copyright 2020 Saffat Technologies, Ltd. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package unitdb 20 | 21 | import ( 22 | "os" 23 | "syscall" 24 | ) 25 | 26 | type _UnixFileLock struct { 27 | f *os.File 28 | name string 29 | } 30 | 31 | // Unlock removes the lock from file. 32 | func (fl *_UnixFileLock) unlock() error { 33 | if err := os.Remove(fl.name); err != nil { 34 | return err 35 | } 36 | return fl.f.Close() 37 | } 38 | 39 | func lockFile(f *os.File) error { 40 | if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { 41 | if err == syscall.EWOULDBLOCK { 42 | err = os.ErrExist 43 | } 44 | return err 45 | } 46 | return nil 47 | } 48 | 49 | func newLockFile(name string) (_LockFile, error) { 50 | f, err := os.OpenFile(name, os.O_RDWR|os.O_CREATE, 0666) 51 | if err != nil { 52 | return nil, err 53 | } 54 | if err := lockFile(f); err != nil { 55 | f.Close() 56 | return nil, err 57 | } 58 | return &_UnixFileLock{f, name}, nil 59 | } 60 | -------------------------------------------------------------------------------- /file_windows.go: -------------------------------------------------------------------------------- 1 | // +build windows 2 | 3 | /* 4 | * Copyright 2020 Saffat Technologies, Ltd. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package unitdb 20 | 21 | import ( 22 | "os" 23 | "syscall" 24 | "unsafe" 25 | ) 26 | 27 | var ( 28 | modkernel32 = syscall.NewLazyDLL("kernel32.dll") 29 | procLockFileEx = modkernel32.NewProc("LockFileEx") 30 | ) 31 | 32 | const ( 33 | errorLockViolation = 0x21 34 | lockfileExclusiveLock = 3 35 | ) 36 | 37 | type _WindowsFileLock struct { 38 | fd syscall.Handle 39 | name string 40 | } 41 | 42 | // unlock removes the lock from file. 43 | func (fl *_WindowsFileLock) unlock() error { 44 | if err := os.Remove(fl.name); err != nil { 45 | return err 46 | } 47 | return syscall.Close(fl.fd) 48 | } 49 | 50 | func lockFile(h syscall.Handle, flags, reserved, locklow, lockhigh uint32, ol *syscall.Overlapped) error { 51 | r1, _, err := syscall.Syscall6(procLockFileEx.Addr(), 6, uintptr(h), uintptr(flags), uintptr(reserved), uintptr(locklow), uintptr(lockhigh), uintptr(unsafe.Pointer(ol))) 52 | if r1 == 0 && (err == syscall.ERROR_FILE_EXISTS || err == errorLockViolation) { 53 | return os.ErrExist 54 | } 55 | return nil 56 | } 57 | 58 | func newLockFile(name string) (_LockFile, error) { 59 | path, err := syscall.UTF16PtrFromString(name) 60 | if err != nil { 61 | return nil, err 62 | } 63 | fd, err := syscall.CreateFile(path, 64 | syscall.GENERIC_READ|syscall.GENERIC_WRITE, 65 | syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, 66 | nil, 67 | syscall.CREATE_ALWAYS, 68 | syscall.FILE_ATTRIBUTE_NORMAL, 69 | 0) 70 | if err != nil { 71 | return nil, os.ErrExist 72 | } 73 | defer func() { 74 | if err != nil { 75 | syscall.Close(fd) 76 | } 77 | }() 78 | var ol syscall.Overlapped 79 | err = lockFile(fd, lockfileExclusiveLock, 0, 1, 0, &ol) 80 | if err != nil { 81 | return nil, err 82 | } 83 | return &_WindowsFileLock{fd, name}, nil 84 | } 85 | -------------------------------------------------------------------------------- /filter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "github.com/unit-io/unitdb/filter" 21 | "github.com/unit-io/unitdb/memdb" 22 | ) 23 | 24 | // Filter filter is bloom filter generator. 25 | type Filter struct { 26 | file _FileSet 27 | filterBlock *filter.Generator 28 | blockCache *memdb.DB 29 | cacheID uint64 30 | } 31 | 32 | // Append appends an entry to bloom filter. 33 | func (f *Filter) Append(h uint64) { 34 | f.filterBlock.Append(h) 35 | } 36 | 37 | // Test tests entry in bloom filter. It returns false if entry definitely does not exist or true may be entry exist in DB. 38 | func (f *Filter) Test(h uint64) bool { 39 | /// Test filter block for presence. 40 | fltr, _ := f.getFilterBlock(true) 41 | if fltr != nil && !fltr.Test(h) { 42 | return false 43 | } 44 | return true 45 | } 46 | 47 | // Close finalizes writing filter to file. 48 | func (f *Filter) close() error { 49 | f.writeFilterBlock() 50 | if err := f.file.Close(); err != nil { 51 | return err 52 | } 53 | return nil 54 | } 55 | 56 | // writeFilterBlock writes the filter block. 57 | func (f *Filter) writeFilterBlock() error { 58 | d := f.filterBlock.Finish() 59 | if _, err := f.file.WriteAt(d, 0); err != nil { 60 | return err 61 | } 62 | 63 | return nil 64 | } 65 | 66 | func (f *Filter) getFilterBlock(fillCache bool) (*filter.Block, error) { 67 | if f.file.currSize() <= 0 { 68 | return nil, nil 69 | } 70 | var cacheKey uint64 71 | if f.blockCache != nil { 72 | cacheKey = f.cacheID ^ uint64(f.file.currSize()) 73 | if data, err := f.blockCache.Get(cacheKey); data != nil { 74 | return filter.NewFilterBlock(data), err 75 | } 76 | } 77 | 78 | raw := make([]byte, f.file.currSize()) 79 | if _, err := f.file.ReadAt(raw, 0); err != nil { 80 | return nil, err 81 | } 82 | 83 | if f.blockCache != nil && fillCache { 84 | f.blockCache.Put(cacheKey, raw) 85 | } 86 | return filter.NewFilterBlock(raw), nil 87 | } 88 | -------------------------------------------------------------------------------- /filter/bloom.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "encoding/binary" 7 | "sync" 8 | ) 9 | 10 | const ( 11 | // MMin is the minimum Filter filter bits count. 12 | MMin = 2 13 | // KMin is the minimum number of keys. 14 | KMin = 1 15 | // Uint64Bytes is the number of bytes in type uint64. 16 | Uint64Bytes = 8 17 | ) 18 | 19 | // Filter is an opaque Filter filter type. 20 | type Filter struct { 21 | lock sync.RWMutex 22 | bits []uint64 23 | keys []uint64 24 | m uint64 // number of bits the "bits" field should recognize 25 | n uint64 // number of inserted elements 26 | } 27 | 28 | func newFilter(m, k uint64) *Filter { 29 | keys := make([]uint64, k) 30 | binary.Read(rand.Reader, binary.LittleEndian, keys) 31 | if m < MMin { 32 | return nil 33 | } 34 | return &Filter{ 35 | m: m, 36 | n: 0, 37 | bits: make([]uint64, (m+63)/64), 38 | keys: keys, 39 | } 40 | } 41 | 42 | func newFilterFromBytes(b []byte, m, k uint64) *Filter { 43 | if m < MMin { 44 | return nil 45 | } 46 | keys := make([]uint64, k) 47 | bits := make([]uint64, (m+63)/64) 48 | buf := bytes.NewBuffer(b) 49 | binary.Read(buf, binary.LittleEndian, keys) 50 | binary.Read(buf, binary.LittleEndian, bits) 51 | 52 | return &Filter{ 53 | m: m, 54 | n: 0, 55 | bits: bits, 56 | keys: keys, 57 | } 58 | } 59 | 60 | // Bytes returns the bytes backing the filter. 61 | func (b *Filter) Bytes() []byte { 62 | b.lock.RLock() 63 | defer b.lock.RUnlock() 64 | buf := new(bytes.Buffer) 65 | binary.Write(buf, binary.LittleEndian, b.keys) 66 | binary.Write(buf, binary.LittleEndian, b.bits) 67 | return buf.Bytes() 68 | } 69 | 70 | // Hashable -> hashes 71 | func (b *Filter) hash(h uint64) []uint64 { 72 | n := len(b.keys) 73 | hashes := make([]uint64, n) 74 | for i := 0; i < n; i++ { 75 | hashes[i] = h ^ b.keys[i] 76 | } 77 | return hashes 78 | } 79 | 80 | // Add adds `key` to the filter. 81 | func (b *Filter) Add(h uint64) { 82 | b.lock.Lock() 83 | defer b.lock.Unlock() 84 | 85 | for _, i := range b.hash(h) { 86 | i %= b.m 87 | b.bits[i>>6] |= 1 << uint(i&0x3f) 88 | } 89 | b.n++ 90 | } 91 | 92 | // Test returns whether `key` is found. 93 | func (b *Filter) Test(h uint64) bool { 94 | b.lock.RLock() 95 | defer b.lock.RUnlock() 96 | 97 | for _, i := range b.hash(h) { 98 | i %= b.m 99 | if (b.bits[i>>6]>>uint(i&0x3f))&1 == 0 { 100 | return false 101 | } 102 | } 103 | return true 104 | } 105 | -------------------------------------------------------------------------------- /filter/filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | const ( 4 | bloomHashes uint64 = 7 5 | bloomBits uint64 = 160000 6 | ) 7 | 8 | // Generator bloom filter generator. 9 | type Generator struct { 10 | filter *Filter 11 | } 12 | 13 | // NewFilterGenerator returns a new filter generator. 14 | func NewFilterGenerator() *Generator { 15 | return &Generator{filter: newFilter(bloomBits, bloomHashes)} 16 | } 17 | 18 | // Append adds a key to the filter block. 19 | func (b *Generator) Append(h uint64) { 20 | b.filter.Add(h) 21 | } 22 | 23 | // Finish finishes building the filter block and returns a slice to its contents. 24 | func (b *Generator) Finish() []byte { 25 | return b.filter.Bytes() 26 | } 27 | 28 | // Bytes returns a slice to filter block contents. 29 | func (b *Generator) Bytes() []byte { 30 | return b.filter.Bytes() 31 | } 32 | 33 | // Block is a filter block 34 | type Block struct { 35 | filter *Filter 36 | } 37 | 38 | // NewFilterBlock returns new filter block, it is used to test key presence in the filter. 39 | func NewFilterBlock(b []byte) *Block { 40 | return &Block{ 41 | filter: newFilterFromBytes(b, bloomBits, bloomHashes), 42 | } 43 | } 44 | 45 | // Test is used to test for key presence in the filter. 46 | func (b *Block) Test(h uint64) bool { 47 | return b.filter.Test(h) 48 | } 49 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/unit-io/unitdb 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/DisposaBoy/JsonConfigReader v0.0.0-20201129172854-99cf318d67e7 7 | github.com/golang/protobuf v1.5.2 8 | github.com/golang/snappy v0.0.3 9 | github.com/gorilla/websocket v1.4.2 10 | github.com/rs/zerolog v1.21.0 11 | github.com/unit-io/bpool v0.0.0-20200906005724-1643bbf59264 12 | golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 13 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 14 | google.golang.org/grpc v1.39.0 15 | ) 16 | -------------------------------------------------------------------------------- /hash/hash.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package hash 18 | 19 | import ( 20 | "encoding/binary" 21 | ) 22 | 23 | const ( 24 | offset32 uint32 = 0xcc9e2d51 25 | prime32 uint32 = 0x1b873593 26 | 27 | // Init is what 32 bits hash values should be initialized with. 28 | Init = offset32 29 | ) 30 | 31 | // WithSalt returns the hash of bytes. it uses salt to shuffle the slice before calculating hash. 32 | func WithSalt(text []byte, salt uint32) uint32 { 33 | b := shuffleInPlace(text, salt) 34 | return New(b) 35 | } 36 | 37 | // New returns the hash of bytes. 38 | func New(b []byte) uint32 { 39 | h := Init 40 | i := 0 41 | n := (len(b) / 8) * 8 42 | 43 | for i != n { 44 | h = (h ^ uint32(b[i])) * prime32 45 | h = (h ^ uint32(b[i+1])) * prime32 46 | h = (h ^ uint32(b[i+2])) * prime32 47 | h = (h ^ uint32(b[i+3])) * prime32 48 | h = (h ^ uint32(b[i+4])) * prime32 49 | h = (h ^ uint32(b[i+5])) * prime32 50 | h = (h ^ uint32(b[i+6])) * prime32 51 | h = (h ^ uint32(b[i+7])) * prime32 52 | i += 8 53 | } 54 | 55 | for _, c := range b[i:] { 56 | h = (h ^ uint32(c)) * prime32 57 | } 58 | 59 | return h 60 | } 61 | 62 | // shuffleInPlace shuffle the slice. 63 | func shuffleInPlace(text []byte, contract uint32) []byte { 64 | if contract == 0 { 65 | return text 66 | } 67 | salt := make([]byte, 4) 68 | binary.LittleEndian.PutUint32(salt[0:4], contract) 69 | 70 | result := duplicateSlice(text) 71 | for i, v, p := len(result)-1, 0, 0; i > 0; i-- { 72 | p += int(salt[v]) 73 | j := (int(salt[v]) + v + p) % i 74 | result[i], result[j] = result[j], result[i] 75 | v = (v + 1) % len(salt) 76 | } 77 | return result 78 | } 79 | 80 | // duplicateSlice get a copy of slice. 81 | func duplicateSlice(data []byte) []byte { 82 | result := make([]byte, len(data)) 83 | copy(result, data) 84 | return result 85 | } 86 | -------------------------------------------------------------------------------- /hash/modulo.go: -------------------------------------------------------------------------------- 1 | package hash 2 | 3 | // FastMod see "A fast alternative to the modulo reduction" (Lemire, 2016) 4 | // https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ 5 | func FastMod(x, m uint64) uint32 { return uint32((x * m) >> 32) } 6 | -------------------------------------------------------------------------------- /hash/rand.go: -------------------------------------------------------------------------------- 1 | package hash 2 | 3 | import "math/bits" 4 | 5 | const ( 6 | fleaSeed = uint32(0xf1ea5eed) 7 | fleaRot1 = 27 8 | fleaRot2 = 17 9 | fleaInitRounds = 3 // initializing with 3 rounds works well enough in practice 10 | ) 11 | 12 | // "A small noncryptographic PRNG" (Jenkins, 2007) 13 | // * http://burtleburtle.net/bob/rand/smallprng.html 14 | // * https://groups.google.com/d/msg/sci.crypt.random-numbers/LAuBGOErdrk/xrMBr3guA7IJ 15 | // 16 | // Also known as FLEA 17 | func fleaInit(key uint64) (a, b, c, d uint32) { 18 | seed := uint32((key >> 32) ^ key) 19 | a, b, c, d = fleaSeed, seed, seed, seed 20 | i := 0 21 | // Functions containing for-loops cannot currently be inlined. 22 | // See https://github.com/golang/go/issues/14768 23 | loop: 24 | e := a - bits.RotateLeft32(b, fleaRot1) 25 | a = b ^ bits.RotateLeft32(c, fleaRot2) 26 | b = c + d 27 | c = d + e 28 | d = e + a 29 | i++ 30 | if i < fleaInitRounds { 31 | goto loop 32 | } 33 | return 34 | } 35 | 36 | func fleaRound(a, b, c, d uint32) (uint32, uint32, uint32, uint32) { 37 | e := a - bits.RotateLeft32(b, fleaRot1) 38 | a = b ^ bits.RotateLeft32(c, fleaRot2) 39 | b = c + d 40 | c = d + e 41 | d = e + a 42 | return a, b, c, d 43 | } 44 | -------------------------------------------------------------------------------- /logger.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "os" 21 | "strings" 22 | 23 | "github.com/rs/zerolog" 24 | ) 25 | 26 | // Logger is logger to use in application. 27 | var logger = zerolog.New(os.Stderr).With().Timestamp().Logger() 28 | 29 | // Info logs the action with a tag. 30 | func Info(context, action string) { 31 | logger.Info().Str("context", context).Msg(action) 32 | } 33 | 34 | // Fatal logs the fatal error messages. 35 | func Fatal(context, msg string, err error) { 36 | logger.Fatal(). 37 | Err(err). 38 | Str("context", context).Msg(msg) 39 | } 40 | 41 | // Debug logs the debug message with tag if it is turned on. 42 | func Debug(context, msg string) { 43 | logger.Debug().Str("context", context).Msg(msg) 44 | } 45 | 46 | // ParseLevel parses a string which represents a log level and returns 47 | // a zerolog.Level. 48 | func ParseLevel(level string, defaultLevel zerolog.Level) zerolog.Level { 49 | l := defaultLevel 50 | switch strings.ToLower(level) { 51 | case "0", "debug": 52 | l = zerolog.DebugLevel 53 | case "1", "info": 54 | l = zerolog.InfoLevel 55 | case "2", "warn": 56 | l = zerolog.WarnLevel 57 | case "3", "error": 58 | l = zerolog.ErrorLevel 59 | case "4", "fatal": 60 | l = zerolog.FatalLevel 61 | } 62 | return l 63 | } 64 | -------------------------------------------------------------------------------- /memdb/README.md: -------------------------------------------------------------------------------- 1 | # memdb [![GoDoc](https://godoc.org/github.com/unit-io/unitdb/memdb?status.svg)](https://pkg.go.dev/github.com/unit-io/unitdb/memdb) [![Go Report Card](https://goreportcard.com/badge/github.com/unit-io/unitdb/memdb)](https://goreportcard.com/report/github.com/unit-io/unitdb/memdb) 2 | 3 | The memdb is blazing fast specialized in memory key-value store for time-series database. The in-memory key-value data store persist entries into a WAL for immediate durability. The Write Ahead Log (WAL) retains memdb data when the db restarts. The WAL ensures data is durable in case of an unexpected failure. 4 | 5 | # About memdb 6 | 7 | ## Key characteristics 8 | - 100% Go 9 | - Optimized for fast lookups and writes 10 | - All DB methods are safe for concurrent use by multiple goroutines. 11 | 12 | ## Quick Start 13 | To build memdb from source code use go get command. 14 | 15 | > go get -u github.com/unit-io/unitdb/memdb 16 | 17 | ## Usage 18 | Detailed API documentation is available using the [go.dev](https://pkg.go.dev/github.com/unit-io/unitdb/memdb) service. 19 | 20 | Make use of the client by importing it in your Go client source code. For example, 21 | 22 | > import "github.com/unit-io/unitdb/memdb" 23 | 24 | The memdb supports Get, Set, Delete operations. It also supports batch operations. 25 | 26 | Samples are available in the examples directory for reference. 27 | 28 | ### Opening a database 29 | To open or create a new database, use the memdb.Open() function: 30 | 31 | ``` 32 | package main 33 | 34 | import ( 35 | "log" 36 | 37 | "github.com/unit-io/unitdb/memdb" 38 | ) 39 | 40 | func main() { 41 | // Opening a database. 42 | // Open DB with reset flag to to skip recovery from log 43 | db, err := memdb.Open(memdb.WithLogFilePath("unitdb"), memdb.WithLogReset()) 44 | if err != nil { 45 | log.Fatal(err) 46 | return 47 | } 48 | defer db.Close() 49 | } 50 | 51 | ``` 52 | 53 | ### Writing to a database 54 | 55 | #### Store a message 56 | Use DB.Put() function to insert a new key-value pair. 57 | Note, if key exists then it overrides key-value when the writes happen within same timeID (see timeRecordInterval option) or it insert a new key-value pair in different timeID. 58 | 59 | ``` 60 | if timeID, err := db.Put(1, []byte("msg 1")); err != nil { 61 | log.Fatal(err) 62 | return 63 | } 64 | 65 | ``` 66 | 67 | #### Read message 68 | Use DB.Get() function to read inserted value. It gets entry from most recent timeID for the provided key. 69 | 70 | ``` 71 | if val, err := db.Get(1); err == nil { 72 | log.Printf("%s ", val) 73 | } 74 | 75 | ``` 76 | 77 | #### Lookup message 78 | Use DB.Lookup() function to lookup entry for faster read. 79 | 80 | ``` 81 | timeID, err := db.Put(1, []byte("msg 1")) 82 | if err != nil { 83 | log.Fatal(err) 84 | return 85 | } 86 | 87 | if val, err := db.Lookup(timeID, 1); err == nil { 88 | log.Printf("%s ", val) 89 | } 90 | 91 | ``` 92 | 93 | #### Deleting a message 94 | use DB.Delete() function to delete a key-value pair. 95 | 96 | ``` 97 | if err := db.Delete(1); err != nil { 98 | fmt.Println("error: ", err) 99 | } 100 | 101 | ``` 102 | 103 | ### Batch operation 104 | Use batch operation to bulk insert records into memdb or bulk delete records from memdb. See examples under examples/memdb folder. 105 | 106 | #### Writing to a batch 107 | Use Batch.Put() to insert a new key-value or Batch.Delete() to delete a key-value from DB. 108 | 109 | ``` 110 | db.Batch(func(b *memdb.Batch, completed <-chan struct{}) error { 111 | for i := 1; i <= 10; i++ { 112 | val := []byte(fmt.Sprintf("msg.%2d", i)) 113 | b.Put(uint64(i), val) 114 | } 115 | return nil 116 | }) 117 | 118 | ``` 119 | 120 | ## Contributing 121 | If you'd like to contribute, please fork the repository and use a feature branch. Pull requests are welcome. 122 | 123 | ## Licensing 124 | This project is licensed under [Apache-2.0 License](https://github.com/unit-io/unitdb/blob/master/LICENSE). 125 | -------------------------------------------------------------------------------- /memdb/batch.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package memdb 18 | 19 | import ( 20 | "fmt" 21 | "time" 22 | ) 23 | 24 | // Batch is a write batch. 25 | type Batch struct { 26 | db *DB 27 | opts *_Options 28 | managed bool 29 | 30 | //tiny Log 31 | tinyLog *_TinyLog 32 | writeLockC chan struct{} 33 | batchGroup []_TimeID 34 | 35 | // commitComplete is used to signal if batch commit is complete and batch is fully written to WAL. 36 | commitComplete chan struct{} 37 | } 38 | 39 | func (b *Batch) newTinyLog() { 40 | timeID := _TimeID(time.Now().UTC().UnixNano()) 41 | b.db.addTimeBlock(timeID) 42 | b.tinyLog = &_TinyLog{id: timeID, _TimeID: timeID, managed: true, doneChan: make(chan struct{})} 43 | } 44 | 45 | // batch starts a new batch. 46 | func (db *DB) batch() *Batch { 47 | b := &Batch{db: db, writeLockC: make(chan struct{}, 1), commitComplete: make(chan struct{})} 48 | b.newTinyLog() 49 | 50 | return b 51 | } 52 | 53 | // TimeID returns time ID for the batch. 54 | func (b *Batch) TimeID() int64 { 55 | return int64(b.tinyLog.timeID()) 56 | } 57 | 58 | // Put adds a new key-value pair to the batch. 59 | func (b *Batch) Put(key uint64, data []byte) error { 60 | if err := b.db.ok(); err != nil { 61 | return err 62 | } 63 | 64 | block, ok := b.db.timeBlock(b.tinyLog.timeID()) 65 | if !ok { 66 | return errForbidden 67 | } 68 | 69 | block.Lock() 70 | defer block.Unlock() 71 | ikey := iKey(false, key) 72 | if err := block.put(ikey, data); err != nil { 73 | return err 74 | } 75 | b.db.addTimeFilter(b.tinyLog.timeID(), key) 76 | 77 | b.db.internal.meter.Puts.Inc(1) 78 | 79 | return nil 80 | } 81 | 82 | // Write starts writing entries into DB. 83 | func (b *Batch) Write() error { 84 | b.writeLockC <- struct{}{} 85 | defer func() { 86 | <-b.writeLockC 87 | }() 88 | b.batchGroup = append(b.batchGroup, b.tinyLog.timeID()) 89 | b.db.internal.logManager.writeWait(b.tinyLog) 90 | b.newTinyLog() 91 | 92 | return nil 93 | } 94 | 95 | // Commit commits changes to the DB. In batch operation commit is managed and client is not allowed to call Commit. 96 | // On Commit complete batch operation signal to the caller if the batch is fully committed to DB. 97 | func (b *Batch) Commit() error { 98 | _assert(!b.managed, "managed batch commit not allowed") 99 | defer func() { 100 | close(b.commitComplete) 101 | b.Abort() 102 | }() 103 | 104 | // Write batch entries. 105 | if err := b.Write(); err != nil { 106 | return err 107 | } 108 | 109 | for _, timeID := range b.batchGroup { 110 | b.db.internal.timeMark.add(timeID) 111 | b.db.internal.timeMark.release(timeID) 112 | } 113 | 114 | b.batchGroup = b.batchGroup[:0] 115 | 116 | return nil 117 | } 118 | 119 | //Abort aborts batch or perform cleanup operation on batch complete. 120 | func (b *Batch) Abort() error { 121 | _assert(!b.managed, "managed batch abort not allowed") 122 | for _, ID := range b.batchGroup { 123 | if err := b.db.releaseLog(ID); err != nil { 124 | return err 125 | } 126 | } 127 | b.db = nil 128 | 129 | return nil 130 | } 131 | 132 | // _assert will panic with a given formatted message if the given condition is false. 133 | func _assert(condition bool, msg string, v ...interface{}) { 134 | if !condition { 135 | panic(fmt.Sprintf("assertion failed: "+msg, v...)) 136 | } 137 | } 138 | 139 | // setManaged sets batch managed. 140 | func (b *Batch) setManaged() { 141 | b.managed = true 142 | } 143 | 144 | // unsetManaged sets batch unmanaged. 145 | func (b *Batch) unsetManaged() { 146 | b.managed = false 147 | } 148 | -------------------------------------------------------------------------------- /memdb/block.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package memdb 18 | 19 | import ( 20 | "encoding/binary" 21 | "sync" 22 | 23 | "github.com/unit-io/bpool" 24 | "github.com/unit-io/unitdb/filter" 25 | ) 26 | 27 | // To avoid lock bottlenecks block cache is divided into several (nShards) shards. 28 | type ( 29 | _TimeID int64 30 | _TimeFilter struct { 31 | timeRecords map[_TimeID]*filter.Block 32 | // bloom filter adds keys to the filter for all entries in a time block. 33 | // filter is checked during get or delete operation 34 | // to indicate key definitely not exist in the time block. 35 | filter *filter.Generator 36 | sync.RWMutex // Read Write mutex, guards access to internal map. 37 | } 38 | _TimeBlocks map[_TimeID]*_Block 39 | ) 40 | 41 | type ( 42 | // _Key is an internal key that includes deleted flag for the key. 43 | _Key struct { 44 | delFlag uint8 // deleted flag 45 | key uint64 46 | } 47 | _BlockKey uint16 48 | _Block struct { 49 | sync.RWMutex // Read Write mutex, guards access to internal map. 50 | count int64 51 | data *bpool.Buffer 52 | records map[_Key]int64 // map[key]offset 53 | 54 | timeRefs []_TimeID 55 | lastOffset int64 // last offset of block data written to the log 56 | } 57 | ) 58 | 59 | // iKey an internal key includes deleted flag. 60 | func iKey(delFlag bool, k uint64) _Key { 61 | dFlag := uint8(0) 62 | if delFlag { 63 | dFlag = 1 64 | } 65 | return _Key{delFlag: dFlag, key: k} 66 | } 67 | 68 | func (b *_Block) get(off int64) ([]byte, error) { 69 | scratch, err := b.data.Slice(off, off+4) // read data length. 70 | if err != nil { 71 | return nil, err 72 | } 73 | dataLen := int64(binary.LittleEndian.Uint32(scratch[:4])) 74 | data, err := b.data.Slice(off, off+dataLen) 75 | if err != nil { 76 | return nil, err 77 | } 78 | 79 | return data[8+1+4:], nil 80 | } 81 | 82 | func (b *_Block) put(ikey _Key, data []byte) error { 83 | dataLen := int64(len(data) + 8 + 1 + 4) // data len + key len + flag bit + scratch len 84 | off, err := b.data.Extend(dataLen) 85 | if err != nil { 86 | return err 87 | } 88 | var scratch [4]byte 89 | binary.LittleEndian.PutUint32(scratch[0:4], uint32(dataLen)) 90 | if _, err := b.data.WriteAt(scratch[:], off); err != nil { 91 | return err 92 | } 93 | 94 | // k with flag bit 95 | var k [9]byte 96 | k[0] = ikey.delFlag 97 | binary.LittleEndian.PutUint64(k[1:], ikey.key) 98 | if _, err := b.data.WriteAt(k[:], off+4); err != nil { 99 | return err 100 | } 101 | if _, err := b.data.WriteAt(data, off+8+1+4); err != nil { 102 | return err 103 | } 104 | if ikey.delFlag == 0 { 105 | b.count++ 106 | } 107 | b.records[ikey] = off 108 | 109 | return nil 110 | } 111 | 112 | func (b *_Block) delete(key uint64) error { 113 | ikey := iKey(false, key) 114 | off := b.records[ikey] 115 | // k with flag bit 116 | var k [9]byte 117 | k[0] = 1 118 | binary.LittleEndian.PutUint64(k[1:], key) 119 | if _, err := b.data.WriteAt(k[:], off+4); err != nil { 120 | return err 121 | } 122 | 123 | delete(b.records, ikey) 124 | b.count-- 125 | 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /memdb/errors.go: -------------------------------------------------------------------------------- 1 | package memdb 2 | 3 | import "errors" 4 | 5 | var ( 6 | errEntryDeleted = errors.New("Entry is deleted") 7 | errEntryDoesNotExist = errors.New("Entry does not exist in memdb") 8 | errValueEmpty = errors.New("Payload is empty") 9 | errValueTooLarge = errors.New("value is too large") 10 | errEntryInvalid = errors.New("Entry is invalid") 11 | errClosed = errors.New("The memdb is closed") 12 | errBadRequest = errors.New("The request was invalid or cannot be otherwise served") 13 | errForbidden = errors.New("The request is understood, but it has been refused or access is not allowed") 14 | ) 15 | -------------------------------------------------------------------------------- /memdb/options.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package memdb 18 | 19 | import ( 20 | "time" 21 | ) 22 | 23 | type _Options struct { 24 | logFilePath string 25 | 26 | // memdbSize sets maximum size of DB. 27 | memdbSize int64 28 | 29 | // bufferSize sets size of buffer to use for buffer pooling. 30 | bufferSize int64 31 | 32 | // logResetFlag flag to skips log recovery on DB open and reset WAL. 33 | logResetFlag bool 34 | 35 | logInterval time.Duration 36 | 37 | timeBlockDuration time.Duration 38 | } 39 | 40 | // Options it contains configurable options and flags for DB. 41 | type Options interface { 42 | set(*_Options) 43 | } 44 | 45 | // fOption wraps a function that modifies options and flags into an 46 | // implementation of the Options interface. 47 | type fOption struct { 48 | f func(*_Options) 49 | } 50 | 51 | func (fo *fOption) set(o *_Options) { 52 | fo.f(o) 53 | } 54 | 55 | func newFuncOption(f func(*_Options)) *fOption { 56 | return &fOption{ 57 | f: f, 58 | } 59 | } 60 | 61 | // WithDefaultOptions will open DB with some default values. 62 | func WithDefaultOptions() Options { 63 | return newFuncOption(func(o *_Options) { 64 | if o.logFilePath == "" { 65 | o.logFilePath = "/tmp/unitdb" 66 | } 67 | if o.memdbSize == 0 { 68 | o.memdbSize = defaultMemdbSize 69 | } 70 | if o.bufferSize == 0 { 71 | o.bufferSize = defaultBufferSize 72 | } 73 | if o.logInterval == 0 { 74 | o.logInterval = 15 * time.Millisecond 75 | } 76 | if o.timeBlockDuration == 0 { 77 | o.timeBlockDuration = 1 * time.Second 78 | } 79 | }) 80 | } 81 | 82 | // WithLogFilePath sets database directory for storing logs. 83 | func WithLogFilePath(path string) Options { 84 | return newFuncOption(func(o *_Options) { 85 | o.logFilePath = path 86 | }) 87 | } 88 | 89 | // WithMemdbSize sets max size of DB. 90 | func WithMemdbSize(size int64) Options { 91 | return newFuncOption(func(o *_Options) { 92 | o.memdbSize = size 93 | }) 94 | } 95 | 96 | // WithBufferSize sets max size of buffer to use for buffer pooling. 97 | func WithBufferSize(size int64) Options { 98 | return newFuncOption(func(o *_Options) { 99 | o.bufferSize = size 100 | }) 101 | } 102 | 103 | // WithLogReset flag to skip recovery on DB open and reset WAL. 104 | func WithLogReset() Options { 105 | return newFuncOption(func(o *_Options) { 106 | o.logResetFlag = true 107 | }) 108 | } 109 | 110 | // WithLogInterval sets interval for a time block. Block is pushed to the queue to write it to the log file. 111 | func WithLogInterval(dur time.Duration) Options { 112 | return newFuncOption(func(o *_Options) { 113 | o.logInterval = dur 114 | }) 115 | } 116 | 117 | // WithTimeBlockInterval sets interval for a time block. Block is pushed to the queue to write it to the log file. 118 | func WithTimeBlockInterval(dur time.Duration) Options { 119 | return newFuncOption(func(o *_Options) { 120 | o.timeBlockDuration = dur 121 | }) 122 | } 123 | -------------------------------------------------------------------------------- /memdb/query.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package memdb 18 | 19 | type _QueryManager struct { 20 | timeRcord _TimeID 21 | timeFilters map[_BlockKey]*_TimeFilter 22 | timeBlocks _TimeBlocks 23 | 24 | cutoff _TimeID 25 | limit int 26 | } 27 | -------------------------------------------------------------------------------- /memdb/time_lock.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package memdb 18 | 19 | import ( 20 | "sync" 21 | 22 | "github.com/unit-io/unitdb/hash" 23 | ) 24 | 25 | // _TimeLock mutex to perform time based lock/unlock. 26 | type ( 27 | _Internal struct { 28 | *sync.RWMutex 29 | } 30 | _TimeLock struct { 31 | locks []_Internal 32 | consistent *hash.Consistent 33 | } 34 | ) 35 | 36 | // newTimeLock creates mutex to perform time based lock/unlock. 37 | func newTimeLock() _TimeLock { 38 | timeLock := _TimeLock{ 39 | locks: make([]_Internal, nLocks), 40 | consistent: hash.InitConsistent(int(nLocks), int(nLocks)), 41 | } 42 | 43 | for i := 0; i < nLocks; i++ { 44 | timeLock.locks[i] = _Internal{new(sync.RWMutex)} 45 | } 46 | 47 | return timeLock 48 | } 49 | 50 | // getTimeLock returns mutex for the provided time ID 51 | func (tl *_TimeLock) getTimeLock(timeID _TimeID) _Internal { 52 | return tl.locks[tl.consistent.FindBlock(uint64(timeID))] 53 | } 54 | -------------------------------------------------------------------------------- /memdb/time_mark.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package memdb 18 | 19 | import ( 20 | "sort" 21 | "sync" 22 | ) 23 | 24 | type ( 25 | _TimeRecord struct { 26 | refs int 27 | lastUnref _TimeID 28 | } 29 | 30 | _TimeMark struct { 31 | sync.RWMutex 32 | records map[_TimeID]_TimeRecord 33 | releasedRecords map[_TimeID]_TimeRecord 34 | } 35 | ) 36 | 37 | func newTimeMark() *_TimeMark { 38 | return &_TimeMark{records: make(map[_TimeID]_TimeRecord), releasedRecords: make(map[_TimeID]_TimeRecord)} 39 | } 40 | 41 | func (tm *_TimeMark) add(timeID _TimeID) { 42 | tm.Lock() 43 | defer tm.Unlock() 44 | if r, ok := tm.records[timeID]; ok { 45 | r.refs++ 46 | } 47 | tm.records[timeID] = _TimeRecord{refs: 1} 48 | } 49 | 50 | func (tm *_TimeMark) release(timeID _TimeID) { 51 | tm.Lock() 52 | defer tm.Unlock() 53 | 54 | timeMark, ok := tm.records[timeID] 55 | if !ok { 56 | return 57 | } 58 | timeMark.refs-- 59 | if timeMark.refs > 0 { 60 | tm.records[timeID] = timeMark 61 | } else { 62 | delete(tm.records, timeID) 63 | timeMark.lastUnref = timeID 64 | tm.releasedRecords[timeID] = timeMark 65 | } 66 | } 67 | 68 | func (tm *_TimeMark) timeRefs(timeRef _TimeID) (timeIDs []_TimeID) { 69 | tm.RLock() 70 | defer tm.RUnlock() 71 | for timeID, r := range tm.releasedRecords { 72 | if r.lastUnref > 0 && r.lastUnref < timeRef { 73 | timeIDs = append(timeIDs, timeID) 74 | } 75 | } 76 | sort.Slice(timeIDs[:], func(i, j int) bool { 77 | return timeIDs[i] < timeIDs[j] 78 | }) 79 | 80 | return timeIDs 81 | } 82 | 83 | func (tm *_TimeMark) allRefs() (timeIDs []_TimeID) { 84 | tm.RLock() 85 | defer tm.RUnlock() 86 | for timeID, _ := range tm.releasedRecords { 87 | timeIDs = append(timeIDs, timeID) 88 | } 89 | sort.Slice(timeIDs[:], func(i, j int) bool { 90 | return timeIDs[i] < timeIDs[j] 91 | }) 92 | 93 | return timeIDs 94 | } 95 | 96 | func (tm *_TimeMark) timeUnref(timeID _TimeID) { 97 | tm.Lock() 98 | defer tm.Unlock() 99 | 100 | delete(tm.releasedRecords, timeID) 101 | } 102 | -------------------------------------------------------------------------------- /message/id.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package message 18 | 19 | import ( 20 | "encoding/binary" 21 | 22 | "github.com/unit-io/unitdb/uid" 23 | ) 24 | 25 | // Various constant parts of the ID. 26 | const ( 27 | // MasterContract contract is default contract used for topics if client program does not specify Contract in the request. 28 | MasterContract = uint32(3376684800) 29 | 30 | fixed = 16 31 | ) 32 | 33 | // Prefix generates unique ID from topic parts and concatenate contract as first part of the topic. 34 | func Prefix(parts []Part) uint64 { 35 | if len(parts) == 1 { 36 | return uint64(parts[0].Hash) 37 | } 38 | return uint64(parts[1].Hash)<<32 + uint64(parts[0].Hash) 39 | } 40 | 41 | // ID represents a message ID and lexigraphically sortable. 42 | type ID []byte 43 | 44 | // NewID generates a new message identifier with a prefix. Master contract is adde to the ID and actual Contract is set later. 45 | func NewID(seq uint64) ID { 46 | id := make(ID, fixed) 47 | binary.LittleEndian.PutUint32(id[0:4], uid.NewApoch()) 48 | binary.LittleEndian.PutUint32(id[4:8], MasterContract) 49 | binary.LittleEndian.PutUint64(id[8:16], seq) 50 | 51 | return id 52 | } 53 | 54 | // Size return fixed size of the ID. 55 | func (id ID) Size() int { 56 | return fixed 57 | } 58 | 59 | // Sequence gets the seq for the id. 60 | func (id ID) Sequence() uint64 { 61 | return binary.LittleEndian.Uint64(id[8:16]) 62 | } 63 | 64 | // SetContract sets Contract on ID. 65 | func (id *ID) SetContract(contract uint32) { 66 | newid := make(ID, fixed) 67 | copy(newid[:fixed], *id) 68 | binary.LittleEndian.PutUint32(newid[4:8], contract) 69 | *id = newid 70 | } 71 | 72 | // Prefix return message ID only containing prefix. 73 | func (id ID) Prefix() ID { 74 | prefix := make(ID, 8) 75 | copy(prefix, id[:8]) 76 | return prefix 77 | } 78 | 79 | // EvalPrefix matches the prefix with the cutoff time. 80 | func (id ID) EvalPrefix(contract uint32, cutoff int64) bool { 81 | // wild card topic (i.e. "*" or "...") will match first 4 byte of contract was added to the ID. 82 | if cutoff > 0 { 83 | return uid.Time(id[0:4]) >= cutoff && binary.LittleEndian.Uint32(id[4:8]) == contract 84 | } 85 | return binary.LittleEndian.Uint32(id[4:8]) == contract 86 | } 87 | -------------------------------------------------------------------------------- /metrics/counter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import "sync/atomic" 20 | 21 | // Counter hold an int64 value that can be incremented and decremented. 22 | type Counter interface { 23 | Reset() 24 | Count() int64 25 | Dec(int64) 26 | Inc(int64) 27 | Snapshot() Counter 28 | } 29 | 30 | // GetOrRegisterCounter returns an existing Counter or constructs and registers 31 | // a new StandardCounter. 32 | func GetOrRegisterCounter(name string, r Metrics) Counter { 33 | return r.GetOrRegister(name, NewCounter).(Counter) 34 | } 35 | 36 | // NewCounter constructs a new StandardCounter. 37 | func NewCounter() Counter { 38 | return &_Counter{0} 39 | } 40 | 41 | // CounterSnapshot is a read-only copy of another Counter. 42 | type CounterSnapshot int64 43 | 44 | // Reset reset clear panics. 45 | func (CounterSnapshot) Reset() { 46 | panic("Clear called on a CounterSnapshot") 47 | } 48 | 49 | // Count returns the count at the time the snapshot was taken. 50 | func (c CounterSnapshot) Count() int64 { return int64(c) } 51 | 52 | // Dec panics. 53 | func (CounterSnapshot) Dec(int64) { 54 | panic("Dec called on a CounterSnapshot") 55 | } 56 | 57 | // Inc panics. 58 | func (CounterSnapshot) Inc(int64) { 59 | panic("Inc called on a CounterSnapshot") 60 | } 61 | 62 | // Snapshot returns the snapshot. 63 | func (c CounterSnapshot) Snapshot() Counter { return c } 64 | 65 | // StandardCounter is the standard implementation of a Counter and uses the 66 | // sync/atomic package to manage a single int64 value. 67 | type _Counter struct { 68 | count int64 69 | } 70 | 71 | // Clear sets the counter to zero. 72 | func (c *_Counter) Reset() { 73 | atomic.StoreInt64(&c.count, 0) 74 | } 75 | 76 | // Count returns the current count. 77 | func (c *_Counter) Count() int64 { 78 | return atomic.LoadInt64(&c.count) 79 | } 80 | 81 | // Dec decrements the counter by the given amount. 82 | func (c *_Counter) Dec(i int64) { 83 | atomic.AddInt64(&c.count, -i) 84 | } 85 | 86 | // Inc increments the counter by the given amount. 87 | func (c *_Counter) Inc(i int64) { 88 | atomic.AddInt64(&c.count, i) 89 | } 90 | 91 | // Snapshot returns a read-only copy of the counter. 92 | func (c *_Counter) Snapshot() Counter { 93 | return CounterSnapshot(c.Count()) 94 | } 95 | -------------------------------------------------------------------------------- /metrics/gauage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import "sync/atomic" 20 | 21 | // Gauge hold an int64 value that can be set arbitrarily. 22 | type Gauge interface { 23 | Snapshot() Gauge 24 | Update(int64) 25 | Value() int64 26 | } 27 | 28 | // GetOrRegisterGauge returns an existing Gauge or constructs and registers a 29 | // new StandardGauge. 30 | func GetOrRegisterGauge(name string, r Metrics) Gauge { 31 | return r.GetOrRegister(name, NewGauge).(Gauge) 32 | } 33 | 34 | // NewGauge constructs a new StandardGauge. 35 | func NewGauge() Gauge { 36 | return &_Gauge{0} 37 | } 38 | 39 | // GaugeSnapshot is a read-only copy of another Gauge. 40 | type GaugeSnapshot int64 41 | 42 | // Snapshot returns the snapshot. 43 | func (g GaugeSnapshot) Snapshot() Gauge { return g } 44 | 45 | // Update panics. 46 | func (GaugeSnapshot) Update(int64) { 47 | panic("Update called on a GaugeSnapshot") 48 | } 49 | 50 | // Value returns the value at the time the snapshot was taken. 51 | func (g GaugeSnapshot) Value() int64 { return int64(g) } 52 | 53 | // StandardGauge is the standard implementation of a Gauge and uses the 54 | // sync/atomic package to manage a single int64 value. 55 | type _Gauge struct { 56 | value int64 57 | } 58 | 59 | // Snapshot returns a read-only copy of the gauge. 60 | func (g *_Gauge) Snapshot() Gauge { 61 | return GaugeSnapshot(g.Value()) 62 | } 63 | 64 | // Update updates the gauge's value. 65 | func (g *_Gauge) Update(v int64) { 66 | atomic.StoreInt64(&g.value, v) 67 | } 68 | 69 | // Value returns the gauge's current value. 70 | func (g *_Gauge) Value() int64 { 71 | return atomic.LoadInt64(&g.value) 72 | } 73 | -------------------------------------------------------------------------------- /metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "fmt" 21 | "reflect" 22 | "sync" 23 | ) 24 | 25 | // Metrics implementation is copied from github.com/rcrowley/go-metrics and it is simplified for tracking Trace metrics. 26 | 27 | // DuplicateMetric is the error returned by Registry.Register when a metric 28 | // already exists. If you mean to Register that metric you must first 29 | // Unregister the existing metric. 30 | type DuplicateMetric string 31 | 32 | func (err DuplicateMetric) Error() string { 33 | return fmt.Sprintf("duplicate metric: %s", string(err)) 34 | } 35 | 36 | // Metrics A Metrics holds registry references to a set of metrics by name and can iterate 37 | // over them, calling callback functions provided by the user. 38 | // 39 | // This is an interface so as to encourage other structs to implement 40 | // the Metrics API as appropriate. 41 | type Metrics interface { 42 | 43 | // Gets an existing metric or registers the given one. 44 | // The interface can be the metric to register if not found in registry, 45 | // or a function returning the metric for lazy instantiation. 46 | GetOrRegister(string, interface{}) interface{} 47 | 48 | // Unregister the metric with the given name. 49 | Unregister(string) 50 | 51 | // Unregister all metrics. (Mostly for testing.) 52 | UnregisterAll() 53 | } 54 | 55 | // StandardMetrics the standard implementation of a Registry is a mutex-protected map 56 | // of names to metrics. 57 | type StandardMetrics struct { 58 | metrics map[string]interface{} 59 | mutex sync.RWMutex 60 | } 61 | 62 | // NewMetrics new metrics create a new registry. 63 | func NewMetrics() Metrics { 64 | return &StandardMetrics{metrics: make(map[string]interface{})} 65 | } 66 | 67 | // GetOrRegister gets an existing metric or creates and registers a new one. Threadsafe 68 | // alternative to calling Get and Register on failure. 69 | // The interface can be the metric to register if not found in registry, 70 | // or a function returning the metric for lazy instantiation. 71 | func (m *StandardMetrics) GetOrRegister(name string, i interface{}) interface{} { 72 | // access the read lock first which should be re-entrant 73 | m.mutex.RLock() 74 | metric, ok := m.metrics[name] 75 | m.mutex.RUnlock() 76 | if ok { 77 | return metric 78 | } 79 | 80 | // only take the write lock if we'll be modifying the metrics map 81 | m.mutex.Lock() 82 | defer m.mutex.Unlock() 83 | if metric, ok := m.metrics[name]; ok { 84 | return metric 85 | } 86 | if v := reflect.ValueOf(i); v.Kind() == reflect.Func { 87 | i = v.Call(nil)[0].Interface() 88 | } 89 | m.register(name, i) 90 | return i 91 | } 92 | 93 | // Unregister the metric with the given name. 94 | func (m *StandardMetrics) Unregister(name string) { 95 | m.mutex.Lock() 96 | defer m.mutex.Unlock() 97 | m.stop(name) 98 | delete(m.metrics, name) 99 | } 100 | 101 | // UnregisterAll unregisters all metrics. (Mostly for testing.) 102 | func (m *StandardMetrics) UnregisterAll() { 103 | m.mutex.Lock() 104 | defer m.mutex.Unlock() 105 | for name := range m.metrics { 106 | m.stop(name) 107 | delete(m.metrics, name) 108 | } 109 | } 110 | 111 | func (m *StandardMetrics) register(name string, i interface{}) error { 112 | if _, ok := m.metrics[name]; ok { 113 | return DuplicateMetric(name) 114 | } 115 | switch i.(type) { 116 | case Counter, Gauge: 117 | m.metrics[name] = i 118 | } 119 | return nil 120 | } 121 | 122 | func (m *StandardMetrics) stop(name string) { 123 | if i, ok := m.metrics[name]; ok { 124 | if s, ok := i.(Stoppable); ok { 125 | s.Stop() 126 | } 127 | } 128 | } 129 | 130 | // Stoppable defines the metrics which has to be stopped. 131 | type Stoppable interface { 132 | Stop() 133 | } 134 | -------------------------------------------------------------------------------- /mutex.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "sync" 21 | 22 | "github.com/unit-io/unitdb/hash" 23 | ) 24 | 25 | // mutex mutex to lock/unlock. 26 | type _Mutex struct { 27 | internal []*sync.RWMutex 28 | consistent *hash.Consistent 29 | } 30 | 31 | // newMutex creates mutex to lock/unlock. 32 | func newMutex() _Mutex { 33 | mu := _Mutex{ 34 | internal: make([]*sync.RWMutex, nBlocks), 35 | consistent: hash.InitConsistent(int(nBlocks), int(nBlocks)), 36 | } 37 | 38 | for i := 0; i < nBlocks; i++ { 39 | mu.internal[i] = new(sync.RWMutex) 40 | } 41 | 42 | return mu 43 | } 44 | 45 | // getMutex returns mutex under given blockID 46 | func (mu *_Mutex) getMutex(blockID uint64) *sync.RWMutex { 47 | return mu.internal[mu.consistent.FindBlock(blockID)] 48 | } 49 | -------------------------------------------------------------------------------- /query.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "time" 21 | 22 | "github.com/unit-io/unitdb/message" 23 | ) 24 | 25 | // Query represents a topic to query and optional contract information. 26 | type ( 27 | _Query struct { 28 | topicHash uint64 29 | seq uint64 30 | } 31 | _InternalQuery struct { 32 | parts []message.Part // The parts represents a topic which contains a contract and a list of hashes for various parts of the topic. 33 | depth uint8 34 | topicType uint8 35 | prefix uint64 // The prefix is generated from contract and first of the topic. 36 | cutoff int64 // The cutoff is time limit check on message IDs. 37 | winEntries []_Query 38 | 39 | opts *_QueryOptions 40 | } 41 | Query struct { 42 | internal _InternalQuery 43 | Topic []byte // The topic of the message. 44 | Contract uint32 // The contract is used as prefix in the message ID. 45 | Limit int // The maximum number of elements to return. 46 | } 47 | ) 48 | 49 | // NewQuery creates a new query structure from the topic. 50 | func NewQuery(topic []byte) *Query { 51 | opts := &_Options{} 52 | WithDefaultQueryOptions().set(opts) 53 | return &Query{ 54 | internal: _InternalQuery{opts: &opts.queryOptions}, 55 | Topic: topic, 56 | } 57 | } 58 | 59 | // WithContract sets contract on query. 60 | func (q *Query) WithContract(contract uint32) *Query { 61 | q.Contract = contract 62 | return q 63 | } 64 | 65 | // WithLimit sets query limit. 66 | func (q *Query) WithLimit(limit int) *Query { 67 | q.Limit = limit 68 | return q 69 | } 70 | 71 | // WithLast sets query duration to fetch stored messages. 72 | func (q *Query) WithLast(dur string) *Query { 73 | base := time.Now() 74 | duration, err := time.ParseDuration(dur) 75 | if err != nil { 76 | return q 77 | } 78 | // In case of last, include it to the query. 79 | q.internal.cutoff = base.Add(-duration).Unix() 80 | switch { 81 | case (q.Limit == 0): 82 | q.Limit = q.internal.opts.defaultQueryLimit 83 | case q.Limit > q.internal.opts.maxQueryLimit: 84 | q.Limit = q.internal.opts.maxQueryLimit 85 | } 86 | 87 | return q 88 | } 89 | 90 | func (q *Query) parse() error { 91 | if q.Contract == 0 { 92 | q.Contract = message.MasterContract 93 | } 94 | topic := new(message.Topic) 95 | //Parse the Key. 96 | topic.ParseKey(q.Topic) 97 | // Parse the topic. 98 | topic.Parse(q.Contract, true) 99 | if topic.TopicType == message.TopicInvalid { 100 | return errBadRequest 101 | } 102 | topic.AddContract(q.Contract) 103 | q.internal.parts = topic.Parts 104 | q.internal.depth = topic.Depth 105 | q.internal.topicType = topic.TopicType 106 | q.internal.prefix = message.Prefix(q.internal.parts) 107 | // In case of last, include it to the query. 108 | if from, limit, ok := topic.Last(); ok { 109 | q.internal.cutoff = from.Unix() 110 | switch { 111 | case (q.Limit == 0 && limit == 0): 112 | q.Limit = q.internal.opts.defaultQueryLimit 113 | case q.Limit > q.internal.opts.maxQueryLimit || limit > q.internal.opts.maxQueryLimit: 114 | q.Limit = q.internal.opts.maxQueryLimit 115 | case limit > q.Limit: 116 | q.Limit = limit 117 | } 118 | } 119 | if q.Limit == 0 { 120 | q.Limit = q.internal.opts.defaultQueryLimit 121 | } 122 | return nil 123 | } 124 | -------------------------------------------------------------------------------- /server/common/encode.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package common 18 | 19 | import ( 20 | "github.com/golang/protobuf/proto" 21 | ) 22 | 23 | const ( 24 | MaxMessageSize = 1 << 19 25 | ) 26 | 27 | // Encoder encodes a byte slice to write into the destination proto.Message. 28 | // You do not need to copy the slice; you may use it directly. 29 | // 30 | // You do not have to encode the full byte slice in one packet. You can 31 | // choose to chunk your packets by returning 0 < n < len(p) and the 32 | // Conn will repeatedly send subsequent messages by slicing into the 33 | // byte slice. 34 | type Encoder func(proto.Message, []byte) (int, error) 35 | 36 | // Decode is given a Response value and expects you to decode the 37 | // response value into the byte slice given. You MUST decode up to 38 | // len(p) if available. 39 | // 40 | // This should return the data slice directly from m. The length of this 41 | // is used to determine if there is more data and the offset for the next 42 | // read. 43 | type Decoder func(m proto.Message, offset int, p []byte) ([]byte, error) 44 | 45 | // Encode is the easiest way to generate an Encoder for a proto.Message. 46 | // You just give it a callback that gets the pointer to the byte slice field 47 | // and a valid encoder will be generated. 48 | // 49 | // Example: given a structure that has a field "Data []byte", you could: 50 | // 51 | // Encode(func(msg proto.Message) *[]byte { 52 | // return &msg.(*pbx.Packet).Data 53 | // }) 54 | // 55 | func Encode(f func(proto.Message) *[]byte) Encoder { 56 | return func(msg proto.Message, p []byte) (int, error) { 57 | bytePtr := f(msg) 58 | *bytePtr = p 59 | return len(p), nil 60 | } 61 | } 62 | 63 | // SimpleDecoder is the easiest way to generate a Decoder for a proto.Message. 64 | // Provide a callback that gets the pointer to the byte slice field and a 65 | // valid decoder will be generated. 66 | func Decode(f func(proto.Message) *[]byte) Decoder { 67 | return func(msg proto.Message, offset int, p []byte) ([]byte, error) { 68 | bytePtr := f(msg) 69 | copy(p, (*bytePtr)[offset:]) 70 | return *bytePtr, nil 71 | } 72 | } 73 | 74 | // ChunkedEncoder ensures that data to encode is chunked at the proper size. 75 | func ChunkedEncoder(enc Encoder, size int) Encoder { 76 | return func(msg proto.Message, p []byte) (int, error) { 77 | if len(p) > size { 78 | p = p[:size] 79 | } 80 | 81 | return enc(msg, p) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /server/internal/config/config.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package config 18 | 19 | import ( 20 | "encoding/json" 21 | 22 | "github.com/unit-io/unitdb/server/internal/pkg/log" 23 | ) 24 | 25 | const ( 26 | MaxMessageSize = 65536 // Maximum message size allowed from/to the peer. 27 | ) 28 | 29 | // Config represents main configuration. 30 | type Config struct { 31 | // Default HTTP(S) address:port to listen on for websocket. Either a 32 | // numeric or a canonical name, e.g. ":80" or ":https". Could include a host name, e.g. 33 | // "localhost:80". 34 | // Could be blank: if TLS is not configured, will use ":80", otherwise ":443". 35 | // Can be overridden from the command line, see option --listen. 36 | Listen string `json:"listen"` 37 | 38 | // Default HTTP(S) address:port to listen on for grpc. Either a 39 | // numeric or a canonical name, e.g. ":80" or ":https". Could include a host name, e.g. 40 | // "localhost:80". 41 | // Could be blank: if TLS is not configured, will use ":80", otherwise ":443". 42 | // Can be overridden from the command line, see option --listen. 43 | GrpcListen string `json:"grpc_listen"` 44 | 45 | // Default logging level is "InfoLevel" so to enable the debug log set the "LogLevel" to "DebugLevel". 46 | LoggingLevel string `json:"logging_level"` 47 | 48 | // MaxMessageSize int `json:"max_message_size"` 49 | // // Maximum number of topic subscribers. 50 | // MaxSubscriberCount int `json:"max_subscriber_count"` 51 | 52 | EncryptionConfig json.RawMessage `json:"encryption_config"` 53 | 54 | // Configs for subsystems 55 | Cluster json.RawMessage `json:"cluster_config"` 56 | 57 | // Config for database store 58 | DBPath string `json:"db_path"` 59 | StoreConfig json.RawMessage `json:"store_config"` 60 | 61 | // Config to expose runtime stats 62 | VarzPath string `json:"varz_path"` 63 | } 64 | 65 | // EncryptionConfig represents the configuration for the encryption. 66 | type EncryptionConfig struct { 67 | 68 | // chacha20poly1305 encryption key for client Ids and topic keys. 32 random bytes base64-encoded. 69 | Key string `json:"key,omitempty"` 70 | 71 | // Key identifier. it is useful when you use multiple keys. 72 | Identifier string `json:"identifier"` 73 | 74 | // slealed flag tells if key in the configuration is sealed. 75 | Sealed bool `json:"slealed"` 76 | 77 | // timestamp is helpful to determine the latest key in case of keyroll over. 78 | Timestamp uint32 `json:"timestamp,omitempty"` 79 | } 80 | 81 | func (c *Config) Encryption(encrConfig json.RawMessage) EncryptionConfig { 82 | var encr EncryptionConfig 83 | if err := json.Unmarshal(encrConfig, &encr); err != nil { 84 | log.Fatal("config.Encryption", "error in parsing encryption config", err) 85 | } 86 | 87 | return encr 88 | } 89 | 90 | // StoreConfig represents the configuration for the store. 91 | type StoreConfig struct { 92 | // Reset resets message store on service restart 93 | Reset bool `json:"reset"` 94 | } 95 | 96 | func (c *Config) Store(storeConfig json.RawMessage) StoreConfig { 97 | var store StoreConfig 98 | if err := json.Unmarshal(storeConfig, &store); err != nil { 99 | log.Fatal("config.Encryption", "error in parsing encryption config", err) 100 | } 101 | 102 | return store 103 | } 104 | -------------------------------------------------------------------------------- /server/internal/conn_cache.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package internal 18 | 19 | import ( 20 | "sync" 21 | 22 | "github.com/unit-io/unitdb/server/internal/pkg/uid" 23 | ) 24 | 25 | type _ConnCache struct { 26 | sync.RWMutex 27 | connections map[uid.LID]*_Conn 28 | } 29 | 30 | func NewConnCache() *_ConnCache { 31 | cache := &_ConnCache{ 32 | connections: make(map[uid.LID]*_Conn), 33 | } 34 | 35 | return cache 36 | } 37 | 38 | func (cc *_ConnCache) add(conn *_Conn) { 39 | cc.Lock() 40 | defer cc.Unlock() 41 | cc.connections[conn.connID] = conn 42 | } 43 | 44 | // get fetches a connection from cache by connection ID. 45 | func (cc *_ConnCache) get(connID uid.LID) *_Conn { 46 | cc.Lock() 47 | defer cc.Unlock() 48 | if conn := cc.connections[connID]; conn != nil { 49 | return conn 50 | } 51 | 52 | return nil 53 | } 54 | 55 | func (cc *_ConnCache) delete(connID uid.LID) { 56 | cc.Lock() 57 | defer cc.Unlock() 58 | delete(cc.connections, connID) 59 | } 60 | -------------------------------------------------------------------------------- /server/internal/db/adapter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package adapter 18 | 19 | import ( 20 | "errors" 21 | ) 22 | 23 | var ( 24 | errNotFound = errors.New("no messages were found") 25 | ) 26 | 27 | // Adapter represents a message storage contract that message storage provides 28 | // must fulfill. 29 | type Adapter interface { 30 | // General 31 | 32 | // Open and configure the adapter 33 | Open(path, config string, reset bool) error 34 | // Close the adapter 35 | Close() error 36 | // IsOpen checks if the adapter is ready for use 37 | IsOpen() bool 38 | // // CheckDbVersion checks if the actual database version matches adapter version. 39 | // CheckDbVersion() error 40 | // GetName returns the name of the adapter 41 | GetName() string 42 | 43 | // Put is used to store a message, the SSID provided must be a full SSID 44 | // SSID, where first element should be a contract ID. The time resolution 45 | // for TTL will be in seconds. The function is executed synchronously and 46 | // it returns an error if some error was encountered during storage. 47 | Put(contract uint32, topic string, payload []byte, ttl string) error 48 | 49 | // PutWithID is used to store a message using a pre generated ID, the SSID provided must be a full SSID 50 | // SSID, where first element should be a contract ID. The time resolution 51 | // for TTL will be in seconds. The function is executed synchronously and 52 | // it returns an error if some error was encountered during storage. 53 | PutWithID(contract uint32, messageId []byte, topic string, payload []byte, ttl string) error 54 | 55 | // Get performs a query and attempts to fetch last messages where 56 | // last is specified by last duration argument. 57 | Get(contract uint32, topic string, last string) ([][]byte, error) 58 | 59 | // NewID generate messageId that can later used to store and delete message from message store 60 | NewID() ([]byte, error) 61 | 62 | // Delete is used to delete entry, the SSID provided must be a full SSID 63 | // SSID, where first element should be a contract ID. The function is executed synchronously and 64 | // it returns an error if some error was encountered during delete. 65 | Delete(contract uint32, messageId []byte, topic string) error 66 | 67 | // PutMessage is used to store a message. 68 | // it returns an error if some error was encountered during storage. 69 | PutMessage(key uint64, payload []byte) error 70 | 71 | // GetMessage performs a query and attempts to fetch message for the given key 72 | GetMessage(key uint64) ([]byte, error) 73 | 74 | // DeleteMessage is used to delete message. 75 | // it returns an error if some error was encountered during delete. 76 | DeleteMessage(key uint64) error 77 | 78 | // Keys performs a query and attempts to fetch all keys. 79 | Keys() []uint64 80 | } 81 | -------------------------------------------------------------------------------- /server/internal/globals.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package internal 18 | 19 | var Globals struct { 20 | Cluster *Cluster 21 | connCache *_ConnCache 22 | Service *_Service 23 | } 24 | -------------------------------------------------------------------------------- /server/internal/message/messageids.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package message 18 | 19 | import ( 20 | "sync" 21 | 22 | "github.com/unit-io/unitdb/server/utp" 23 | ) 24 | 25 | // MID is 32-bit local message identifier 26 | type MID int32 27 | 28 | type MessageIds struct { 29 | sync.RWMutex 30 | id MID 31 | resume map[MID]struct{} 32 | index map[MID]utp.MessageType // map[MID]PacketType 33 | } 34 | 35 | func NewMessageIds() MessageIds { 36 | return MessageIds{ 37 | resume: make(map[MID]struct{}), 38 | index: make(map[MID]utp.MessageType), 39 | } 40 | } 41 | 42 | func (mids *MessageIds) Reset() { 43 | mids.Lock() 44 | defer mids.Unlock() 45 | mids.id = 0 46 | } 47 | 48 | func (mids *MessageIds) ResumeID(id MID) { 49 | mids.Lock() 50 | defer mids.Unlock() 51 | mids.resume[id] = struct{}{} 52 | } 53 | 54 | func (mids *MessageIds) FreeID(id MID) { 55 | mids.Lock() 56 | defer mids.Unlock() 57 | delete(mids.index, id) 58 | } 59 | 60 | func (mids *MessageIds) NextID(pktType utp.MessageType) MID { 61 | mids.Lock() 62 | defer mids.Unlock() 63 | mids.id++ 64 | if _, ok := mids.resume[mids.id]; ok { 65 | mids.NextID(pktType) 66 | } 67 | mids.index[mids.id] = pktType 68 | return mids.id 69 | } 70 | 71 | func (mids *MessageIds) GetType(id MID) utp.MessageType { 72 | mids.RLock() 73 | defer mids.RUnlock() 74 | return mids.index[id] 75 | } 76 | -------------------------------------------------------------------------------- /server/internal/message/sub.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package message 18 | 19 | import ( 20 | "sync" 21 | ) 22 | 23 | const ( 24 | CONNECT = uint8(iota + 1) 25 | PUBLISH 26 | SUBSCRIBE 27 | UNSUBSCRIBE 28 | PINGREQ 29 | PINGRESP 30 | DISCONNECT 31 | 32 | fixed = 16 33 | 34 | Contract = uint32(3376684800) 35 | ) 36 | 37 | // ------------------------------------------------------------------------------------ 38 | 39 | // SubscriberType represents a type of subscriber 40 | type SubscriberType uint8 41 | 42 | type TopicAnyCount uint8 43 | 44 | // Subscriber types 45 | const ( 46 | SubscriberDirect = SubscriberType(iota) 47 | SubscriberRemote 48 | ) 49 | 50 | // Subscriber is a value associated with a subscription. 51 | type Subscriber interface { 52 | ID() string 53 | Type() SubscriberType 54 | SendMessage(*Message) bool 55 | } 56 | 57 | // // ------------------------------------------------------------------------------------ 58 | 59 | // Message represents a message which has to be forwarded or stored. 60 | type Message struct { 61 | MessageID uint16 `json:"message_id,omitempty"` // The ID of the message 62 | DeliveryMode uint8 `json:"delivery_mode,omitempty"` // The delivery mode of the message 63 | Delay int32 `json:"delay,omitempty"` // The time in milliseconds to delay the delivery of the message 64 | Topic string `json:"topic,omitempty"` // The topic of the message 65 | Payload []byte `json:"data,omitempty"` // The payload of the message 66 | TTL int64 `json:"ttl,omitempty"` // The time-to-live of the message 67 | } 68 | 69 | // Size returns the byte size of the message. 70 | func (m *Message) Size() int64 { 71 | return int64(len(m.Payload)) 72 | } 73 | 74 | // // ------------------------------------------------------------------------------------ 75 | 76 | // Stats represents a subscription map. 77 | type Stats struct { 78 | sync.Mutex 79 | stats map[string]*Stat 80 | } 81 | 82 | type Stat struct { 83 | ID []byte 84 | Topic string 85 | Counter int 86 | } 87 | 88 | // NewStats creates a new container. 89 | func NewStats() *Stats { 90 | return &Stats{ 91 | stats: make(map[string]*Stat), 92 | } 93 | } 94 | 95 | // Increment adds the subscription to the stats. 96 | func (s *Stats) Increment(topic string, key string, id []byte) (first bool) { 97 | s.Lock() 98 | defer s.Unlock() 99 | 100 | stat, exists := s.stats[key] 101 | if !exists { 102 | stat = &Stat{ 103 | ID: id, 104 | Topic: topic, 105 | } 106 | } 107 | stat.Counter++ 108 | s.stats[key] = stat 109 | return stat.Counter == 1 110 | } 111 | 112 | // Decrement remove a subscription from the stats. 113 | func (s *Stats) Decrement(topic string, key string) (last bool, id []byte) { 114 | s.Lock() 115 | defer s.Unlock() 116 | 117 | if stat, exists := s.stats[key]; exists { 118 | stat.Counter-- 119 | // Remove if there's no subscribers left 120 | if stat.Counter <= 0 { 121 | delete(s.stats, key) 122 | return true, stat.ID 123 | } 124 | } 125 | 126 | return false, nil 127 | } 128 | 129 | // Get gets subscription from the stats. 130 | func (s *Stats) Exist(key string) (ok bool) { 131 | s.Lock() 132 | defer s.Unlock() 133 | 134 | if _, exists := s.stats[key]; exists { 135 | return true 136 | } 137 | return false 138 | } 139 | 140 | // All gets the all subscriptions from the stats. 141 | func (s *Stats) All() []Stat { 142 | s.Lock() 143 | defer s.Unlock() 144 | 145 | stats := make([]Stat, 0, len(s.stats)) 146 | for _, stat := range s.stats { 147 | stats = append(stats, *stat) 148 | } 149 | 150 | return stats 151 | } 152 | -------------------------------------------------------------------------------- /server/internal/net/hdl_grpc.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package net 18 | 19 | import ( 20 | "log" 21 | "net" 22 | "time" 23 | 24 | "github.com/golang/protobuf/proto" 25 | "github.com/unit-io/unitdb/server/common" 26 | pbx "github.com/unit-io/unitdb/server/proto" 27 | "google.golang.org/grpc" 28 | "google.golang.org/grpc/credentials" 29 | "google.golang.org/grpc/keepalive" 30 | ) 31 | 32 | type GrpcServer server 33 | 34 | func NewGrpcServer(opts ...Options) *GrpcServer { 35 | srv := &GrpcServer{ 36 | opts: new(options), 37 | } 38 | WithDefaultOptions().set(srv.opts) 39 | for _, opt := range opts { 40 | opt.set(srv.opts) 41 | } 42 | return srv 43 | } 44 | 45 | func StreamConn( 46 | stream grpc.Stream, 47 | ) *common.Conn { 48 | packetFunc := func(msg proto.Message) *[]byte { 49 | return &msg.(*pbx.Packet).Data 50 | } 51 | return &common.Conn{ 52 | Stream: stream, 53 | InMsg: &pbx.Packet{}, 54 | OutMsg: &pbx.Packet{}, 55 | Encode: common.Encode(packetFunc), 56 | Decode: common.Decode(packetFunc), 57 | } 58 | } 59 | 60 | // Stream implements duplex unitdb.Stream 61 | func (s *GrpcServer) Stream(stream pbx.Unitdb_StreamServer) error { 62 | conn := StreamConn(stream) 63 | defer conn.Close() 64 | 65 | go s.Handler(conn) 66 | <-stream.Context().Done() 67 | return nil 68 | } 69 | 70 | func (s *GrpcServer) Serve(list net.Listener) error { 71 | secure := "" 72 | var opts []grpc.ServerOption 73 | opts = append(opts, grpc.MaxRecvMsgSize(int(MaxMessageSize))) 74 | if s.opts.TLSConfig != nil { 75 | opts = append(opts, grpc.Creds(credentials.NewTLS(s.opts.TLSConfig))) 76 | secure = " secure" 77 | } 78 | 79 | if s.opts.KeepAlive { 80 | kepConfig := keepalive.EnforcementPolicy{ 81 | MinTime: 1 * time.Second, // If a client pings more than once every second, terminate the connection 82 | PermitWithoutStream: true, // Allow pings even when there are no active streams 83 | } 84 | opts = append(opts, grpc.KeepaliveEnforcementPolicy(kepConfig)) 85 | 86 | kpConfig := keepalive.ServerParameters{ 87 | Time: 60 * time.Second, // Ping the client if it is idle for 60 seconds to ensure the connection is still active 88 | Timeout: 20 * time.Second, // Wait 20 second for the ping ack before assuming the connection is dead 89 | } 90 | opts = append(opts, grpc.KeepaliveParams(kpConfig)) 91 | } 92 | 93 | srv := grpc.NewServer(opts...) 94 | pbx.RegisterUnitdbServer(srv, s) 95 | log.Printf("gRPC/%s%s server is registered", grpc.Version, secure) 96 | go func() { 97 | if err := srv.Serve(list); err != nil { 98 | log.Println("gRPC server failed:", err) 99 | } 100 | }() 101 | return nil 102 | } 103 | 104 | var _ pbx.UnitdbServer = (*GrpcServer)(nil) 105 | -------------------------------------------------------------------------------- /server/internal/net/hdl_tcp.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package net 18 | 19 | import ( 20 | "net" 21 | "time" 22 | ) 23 | 24 | // //onAccept is a callback which get called when a connection is accepted 25 | // type TcpHandler func(c net.Conn, proto Proto) 26 | 27 | type TcpServer server 28 | 29 | func NewTcpServer(opts ...Options) *TcpServer { 30 | srv := &TcpServer{ 31 | opts: new(options), 32 | } 33 | WithDefaultOptions().set(srv.opts) 34 | for _, opt := range opts { 35 | opt.set(srv.opts) 36 | } 37 | return srv 38 | } 39 | 40 | func (s *TcpServer) Serve(list net.Listener) error { 41 | defer list.Close() 42 | 43 | var tempDelay time.Duration // how long to sleep on accept failure 44 | for { 45 | conn, err := list.Accept() 46 | if err != nil { 47 | select { 48 | case <-signalHandler(): 49 | return ErrServerClosed 50 | default: 51 | } 52 | 53 | if netErr, ok := err.(net.Error); ok && netErr.Temporary() { 54 | if tempDelay == 0 { 55 | tempDelay = 5 * time.Millisecond 56 | } else { 57 | tempDelay *= 2 58 | } 59 | if max := 1 * time.Second; tempDelay > max { 60 | tempDelay = max 61 | } 62 | 63 | time.Sleep(tempDelay) 64 | continue 65 | } 66 | return err 67 | } 68 | 69 | tempDelay = 0 70 | go s.Handler(conn) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /server/internal/net/listener/patricia.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package listener 18 | 19 | import ( 20 | "bytes" 21 | "io" 22 | ) 23 | 24 | // patriciaTree is a simple patricia tree that handles []byte instead of string 25 | // and cannot be changed after instantiation. 26 | type patriciaTree struct { 27 | root *ptNode 28 | maxDepth int // max depth of the tree. 29 | } 30 | 31 | func newPatriciaTree(bs ...[]byte) *patriciaTree { 32 | max := 0 33 | for _, b := range bs { 34 | if max < len(b) { 35 | max = len(b) 36 | } 37 | } 38 | return &patriciaTree{ 39 | root: newNode(bs), 40 | maxDepth: max + 1, 41 | } 42 | } 43 | 44 | func newPatriciaTreeString(strs ...string) *patriciaTree { 45 | b := make([][]byte, len(strs)) 46 | for i, s := range strs { 47 | b[i] = []byte(s) 48 | } 49 | return newPatriciaTree(b...) 50 | } 51 | 52 | func (t *patriciaTree) matchPrefix(r io.Reader) bool { 53 | buf := make([]byte, t.maxDepth) 54 | n, _ := io.ReadFull(r, buf) 55 | return t.root.match(buf[:n], true) 56 | } 57 | 58 | func (t *patriciaTree) match(r io.Reader) bool { 59 | buf := make([]byte, t.maxDepth) 60 | n, _ := io.ReadFull(r, buf) 61 | return t.root.match(buf[:n], false) 62 | } 63 | 64 | type ptNode struct { 65 | prefix []byte 66 | next map[byte]*ptNode 67 | terminal bool 68 | } 69 | 70 | func newNode(strs [][]byte) *ptNode { 71 | if len(strs) == 0 { 72 | return &ptNode{ 73 | prefix: []byte{}, 74 | terminal: true, 75 | } 76 | } 77 | 78 | if len(strs) == 1 { 79 | return &ptNode{ 80 | prefix: strs[0], 81 | terminal: true, 82 | } 83 | } 84 | 85 | p, strs := splitPrefix(strs) 86 | n := &ptNode{ 87 | prefix: p, 88 | } 89 | 90 | nexts := make(map[byte][][]byte) 91 | for _, s := range strs { 92 | if len(s) == 0 { 93 | n.terminal = true 94 | continue 95 | } 96 | nexts[s[0]] = append(nexts[s[0]], s[1:]) 97 | } 98 | 99 | n.next = make(map[byte]*ptNode) 100 | for first, rests := range nexts { 101 | n.next[first] = newNode(rests) 102 | } 103 | 104 | return n 105 | } 106 | 107 | func splitPrefix(bss [][]byte) (prefix []byte, rest [][]byte) { 108 | if len(bss) == 0 || len(bss[0]) == 0 { 109 | return prefix, bss 110 | } 111 | 112 | if len(bss) == 1 { 113 | return bss[0], [][]byte{{}} 114 | } 115 | 116 | for i := 0; ; i++ { 117 | var cur byte 118 | eq := true 119 | for j, b := range bss { 120 | if len(b) <= i { 121 | eq = false 122 | break 123 | } 124 | 125 | if j == 0 { 126 | cur = b[i] 127 | continue 128 | } 129 | 130 | if cur != b[i] { 131 | eq = false 132 | break 133 | } 134 | } 135 | 136 | if !eq { 137 | break 138 | } 139 | 140 | prefix = append(prefix, cur) 141 | } 142 | 143 | rest = make([][]byte, 0, len(bss)) 144 | for _, b := range bss { 145 | rest = append(rest, b[len(prefix):]) 146 | } 147 | 148 | return prefix, rest 149 | } 150 | 151 | func (n *ptNode) match(b []byte, prefix bool) bool { 152 | l := len(n.prefix) 153 | if l > 0 { 154 | if l > len(b) { 155 | l = len(b) 156 | } 157 | if !bytes.Equal(b[:l], n.prefix) { 158 | return false 159 | } 160 | } 161 | 162 | if n.terminal && (prefix || len(n.prefix) == len(b)) { 163 | return true 164 | } 165 | 166 | if l >= len(b) { 167 | return false 168 | } 169 | 170 | nextN, ok := n.next[b[l]] 171 | if !ok { 172 | return false 173 | } 174 | 175 | if l == len(b) { 176 | b = b[l:l] 177 | } else { 178 | b = b[l+1:] 179 | } 180 | return nextN.match(b, prefix) 181 | } 182 | -------------------------------------------------------------------------------- /server/internal/net/message.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package net 18 | 19 | import ( 20 | "bytes" 21 | "fmt" 22 | "io" 23 | 24 | "github.com/unit-io/unitdb/server/utp" 25 | ) 26 | 27 | // MessagePack is the interface for all Messages 28 | type MessagePack interface { 29 | ToBinary() (bytes.Buffer, error) 30 | FromBinary(fh utp.FixedHeader, data []byte) 31 | Type() utp.MessageType 32 | Info() utp.Info 33 | } 34 | 35 | // Read unpacks the packet from the provided reader. 36 | func Read(r io.Reader) (MessagePack, error) { 37 | var fh utp.FixedHeader 38 | if err := fh.FromBinary(r); err != nil { 39 | return nil, err 40 | } 41 | 42 | // Check for empty Messages 43 | switch fh.MessageType { 44 | case 0: 45 | // TODO fixe zero length message issue. 46 | // return &utp.Pingreq{}, nil 47 | fmt.Println("message::Read: Invalid zero-length packet type ", fh.MessageType) 48 | return nil, fmt.Errorf("message::Read: Invalid zero-length packet type %d", fh.MessageType) 49 | case utp.PINGREQ: 50 | return &utp.Pingreq{}, nil 51 | case utp.DISCONNECT: 52 | return &utp.Disconnect{}, nil 53 | } 54 | 55 | rawMsg := make([]byte, fh.MessageLength) 56 | _, err := io.ReadFull(r, rawMsg) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | // unpack the body 62 | var pack MessagePack 63 | if fh.FlowControl != utp.NONE { 64 | pack = &utp.ControlMessage{} 65 | pack.FromBinary(fh, rawMsg) 66 | return pack, nil 67 | } 68 | switch fh.MessageType { 69 | case utp.CONNECT: 70 | pack = &utp.Connect{} 71 | case utp.PUBLISH: 72 | pack = &utp.Publish{} 73 | case utp.RELAY: 74 | pack = &utp.Relay{} 75 | case utp.SUBSCRIBE: 76 | pack = &utp.Subscribe{} 77 | case utp.UNSUBSCRIBE: 78 | pack = &utp.Unsubscribe{} 79 | default: 80 | return nil, fmt.Errorf("message::Read: Invalid zero-length packet type %d", fh.MessageType) 81 | } 82 | 83 | pack.FromBinary(fh, rawMsg) 84 | return pack, nil 85 | } 86 | 87 | // Encode encodes the message into binary data 88 | func Encode(pack MessagePack) (bytes.Buffer, error) { 89 | switch pack.Type() { 90 | case utp.DISCONNECT: 91 | return pack.(*utp.Disconnect).ToBinary() 92 | case utp.PUBLISH: 93 | return pack.(*utp.Publish).ToBinary() 94 | case utp.FLOWCONTROL: 95 | return pack.(*utp.ControlMessage).ToBinary() 96 | default: 97 | return bytes.Buffer{}, fmt.Errorf("message::Encode: Invalid zero-length packet type %d", pack.Type()) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /server/internal/net/server.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package net 18 | 19 | import ( 20 | "crypto/tls" 21 | "errors" 22 | "fmt" 23 | "net" 24 | "os" 25 | "os/signal" 26 | "sync" 27 | "syscall" 28 | ) 29 | 30 | const ( 31 | MaxMessageSize = 1 << 19 32 | ) 33 | 34 | // ErrServerClosed occurs when a tcp server is closed. 35 | var ErrServerClosed = errors.New("Server closed") 36 | 37 | // Proto represents the type of connection 38 | type Proto int 39 | 40 | const ( 41 | UTP Proto = iota // Unit Transport Protocol 42 | UNITQL // Unit Query Language 43 | ) 44 | 45 | //Handler is a callback which get called when a tcp, websocket connection is established or a grpc stream is established 46 | type Handler func(c net.Conn) 47 | 48 | type options struct { 49 | TLSConfig *tls.Config 50 | KeepAlive bool 51 | } 52 | 53 | // Options it contains configurable options for client 54 | type Options interface { 55 | set(*options) 56 | } 57 | 58 | // fOption wraps a function that modifies options into an 59 | // implementation of the Option interface. 60 | type fOption struct { 61 | f func(*options) 62 | } 63 | 64 | func (fo *fOption) set(o *options) { 65 | fo.f(o) 66 | } 67 | 68 | func newFuncOption(f func(*options)) *fOption { 69 | return &fOption{ 70 | f: f, 71 | } 72 | } 73 | 74 | // WithDefaultOptions will create client connection with some default values. 75 | // KeepAlive: true 76 | // TlsConfig: nil 77 | func WithDefaultOptions() Options { 78 | return newFuncOption(func(o *options) { 79 | o.KeepAlive = true 80 | o.TLSConfig = nil 81 | }) 82 | } 83 | 84 | // WithTLSConfig will set an SSL/TLS configuration to be used when connecting 85 | // to server. 86 | func WithTLSConfig(t *tls.Config) Options { 87 | return newFuncOption(func(o *options) { 88 | o.TLSConfig = t 89 | }) 90 | } 91 | 92 | type Server interface { 93 | // Serve serve the requests if type tcp, websocket or grpc stream 94 | Serve(net.Listener) error 95 | } 96 | 97 | type server struct { 98 | sync.Mutex 99 | opts *options 100 | Handler Handler //The handler to invoke when a connection is accepted 101 | } 102 | 103 | func signalHandler() <-chan bool { 104 | stop := make(chan bool) 105 | 106 | signchan := make(chan os.Signal, 1) 107 | signal.Notify(signchan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) 108 | 109 | go func() { 110 | // Wait for a signal. Don't care which signal it is 111 | sig := <-signchan 112 | fmt.Printf("Signal received: '%s', shutting down\n", sig) 113 | stop <- true 114 | }() 115 | 116 | return stop 117 | } 118 | -------------------------------------------------------------------------------- /server/internal/pkg/collection/arrpool.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package collection 18 | 19 | import ( 20 | "encoding/binary" 21 | "sync" 22 | ) 23 | 24 | var arrayPool = &sync.Pool{ 25 | New: func() interface{} { 26 | return &Array{ 27 | buf: make([]byte, 0, 500), 28 | } 29 | }, 30 | } 31 | 32 | // Array is used to prepopulate an array of items 33 | // which can be re-used to add to log messages. 34 | type Array struct { 35 | buf []byte 36 | } 37 | 38 | func putArray(arr *Array) { 39 | // Proper usage of a sync.Pool requires each entry to have approximately 40 | // the same memory cost. To obtain this property when the stored type 41 | // contains a variably-sized buffer, we add a hard limit on the maximum buffer 42 | // to place back in the pool. 43 | // 44 | // See https://golang.org/issue/23199 45 | const maxSize = 1 << 16 // 64KiB 46 | if cap(arr.buf) > maxSize { 47 | return 48 | } 49 | arrayPool.Put(arr) 50 | } 51 | 52 | // Arr creates an array to be added to an Event or Context. 53 | func Arr() *Array { 54 | arr := arrayPool.Get().(*Array) 55 | arr.buf = arr.buf[:0] 56 | return arr 57 | } 58 | 59 | // MarshalZerologArray method here is no-op - since data is 60 | // already in the needed format. 61 | func (*Array) MarshalZerologArray(*Array) { 62 | } 63 | 64 | func (arr *Array) write(dst []byte) []byte { 65 | sizebuf := make([]byte, 4) 66 | binary.LittleEndian.PutUint32(sizebuf[:4], uint32(len(arr.buf))) 67 | if len(arr.buf) > 0 { 68 | dst = append(append(dst, sizebuf...)) 69 | dst = append(append(dst, arr.buf...)) 70 | } 71 | putArray(arr) 72 | return dst 73 | } 74 | -------------------------------------------------------------------------------- /server/internal/pkg/collection/bpool.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package collection 18 | 19 | import ( 20 | "bytes" 21 | "sync" 22 | ) 23 | 24 | // BufferPool represents a thread safe buffer pool 25 | type BufferPool struct { 26 | internal sync.Pool 27 | } 28 | 29 | // NewBufferPool creates a new BufferPool bounded to the given size. 30 | func NewBufferPool() (bp *BufferPool) { 31 | return &BufferPool{ 32 | sync.Pool{ 33 | New: func() interface{} { 34 | return new(bytes.Buffer) 35 | }, 36 | }, 37 | } 38 | } 39 | 40 | // Get gets a Buffer from the SizedBufferPool, or creates a new one if none are 41 | // available in the pool. Buffers have a pre-allocated capacity. 42 | func (pool *BufferPool) Get() (buffer *bytes.Buffer) { 43 | return pool.internal.Get().(*bytes.Buffer) 44 | } 45 | 46 | // Put returns the given Buffer to the SizedBufferPool. 47 | func (pool *BufferPool) Put(buffer *bytes.Buffer) { 48 | // See https://golang.org/issue/23199 49 | const maxSize = 1 << 16 // 64KiB 50 | if buffer.Len() > maxSize { 51 | return 52 | } 53 | buffer.Reset() 54 | pool.internal.Put(buffer) 55 | } 56 | -------------------------------------------------------------------------------- /server/internal/pkg/collection/payload.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package collection 18 | 19 | type Payload []byte 20 | -------------------------------------------------------------------------------- /server/internal/pkg/crypto/mac.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package crypto 18 | 19 | import ( 20 | "crypto/cipher" 21 | "errors" 22 | 23 | "github.com/unit-io/unitdb/server/internal/pkg/hash" 24 | "golang.org/x/crypto/chacha20poly1305" 25 | ) 26 | 27 | const ( 28 | EpochSize = 4 29 | MessageOffset = EpochSize + 4 30 | ) 31 | 32 | // MAC has the ability to encrypt and decrypt (short) messages as long as they 33 | // share the same key and the same epoch. 34 | type MAC struct { 35 | parent cipher.AEAD 36 | salt []byte 37 | } 38 | 39 | // New builds a new MAC using a 256-bit/32 byte encryption key, a numeric epoch 40 | // and numeric pseudo-random salt 41 | func New(key []byte) (*MAC, error) { 42 | parent, err := chacha20poly1305.New(key) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | mac := new(MAC) 48 | mac.salt = make([]byte, 4) 49 | mac.parent = parent 50 | for i := 0; i < 4; i++ { 51 | mac.salt[i] = (byte(key[(4*i)+0]) << 24) | 52 | (byte(key[(4*i)+1]) << 16) | 53 | (byte(key[(4*i)+2]) << 8) | 54 | byte(key[(4*i)+3]) 55 | } 56 | 57 | return mac, nil 58 | } 59 | 60 | // Overhead returns the maximum difference between the lengths of a 61 | // plaintext and its ciphertext. 62 | func (m *MAC) Overhead() int { return m.parent.Overhead() + EpochSize } 63 | 64 | func SignatureToUint32(sig []byte) uint32 { 65 | return uint32(sig[0])<<24 | uint32(sig[1])<<16 | uint32(sig[2])<<8 | uint32(sig[3]) 66 | } 67 | 68 | func Signature(value uint32) []byte { 69 | sig := make([]byte, 4) 70 | sig[0] = byte(value >> 24) 71 | sig[1] = byte(value >> 16) 72 | sig[2] = byte(value >> 8) 73 | sig[3] = byte(value) 74 | return sig 75 | } 76 | 77 | // Encrypt encrypts src and appends to dst, returning the 78 | // resulting byte slice 79 | func (m *MAC) Encrypt(dst, src []byte) []byte { 80 | //Copy first 4 bytes epoch from source 81 | dst = append(dst, src[:EpochSize]...) 82 | h := hash.New(src) 83 | dst = append(dst, Signature(h)...) 84 | nonce := append(m.salt, dst[:MessageOffset]...) 85 | return m.parent.Seal(dst, nonce, src[EpochSize:], nil) 86 | } 87 | 88 | // Decrypt decrypts src and appends to dst, returning the 89 | // resulting byte slice or an error if the input cannot be 90 | // authenticated. 91 | func (m *MAC) Decrypt(dst, src []byte) ([]byte, error) { 92 | 93 | if len(src) < m.Overhead() { 94 | return dst, errors.New("Authentication failed.") 95 | } 96 | 97 | nonce := append(m.salt, src[:MessageOffset]...) 98 | dst, err := m.parent.Open(dst, nonce, src[MessageOffset:], nil) 99 | if err != nil { 100 | return dst, errors.New("Authentication failed.") 101 | } 102 | // Append epoch to dst at the begining 103 | dst = append(src[:EpochSize], dst...) 104 | return dst, nil 105 | } 106 | -------------------------------------------------------------------------------- /server/internal/pkg/encoding/base8.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package encoding 18 | 19 | func init() { 20 | for i := 0; i < len(dec); i++ { 21 | dec[i] = 0xFF 22 | } 23 | for i := 0; i < len(encoding); i++ { 24 | dec[encoding[i]] = byte(i) 25 | } 26 | } 27 | 28 | // encode by unrolling the stdlib base32 algorithm + removing all safe checks 29 | func Encode8(dst, id []byte) { 30 | dst[0] = encoding[id[0]>>3] 31 | dst[1] = encoding[(id[1]>>6)&0x1F|(id[0]<<2)&0x1F] 32 | dst[2] = encoding[(id[1]>>1)&0x1F] 33 | dst[3] = encoding[(id[2]>>4)&0x1F|(id[1]<<4)&0x1F] 34 | dst[4] = encoding[id[3]>>7|(id[2]<<1)&0x1F] 35 | dst[5] = encoding[(id[3]>>2)&0x1F] 36 | dst[6] = encoding[id[4]>>5|(id[3]<<3)&0x1F] 37 | dst[7] = encoding[id[4]&0x1F] 38 | dst[8] = encoding[id[5]>>3] 39 | dst[9] = encoding[(id[6]>>6)&0x1F|(id[5]<<2)&0x1F] 40 | dst[10] = encoding[(id[6]>>1)&0x1F] 41 | dst[11] = encoding[(id[7]>>4)&0x1F|(id[6]<<4)&0x1F] 42 | dst[12] = encoding[(id[7]<<1)&0x1F] 43 | } 44 | 45 | // decode by unrolling the stdlib base32 algorithm + removing all safe checks 46 | func Decode8(id []byte, src []byte) { 47 | id[0] = dec[src[0]]<<3 | dec[src[1]]>>2 48 | id[1] = dec[src[1]]<<6 | dec[src[2]]<<1 | dec[src[3]]>>4 49 | id[2] = dec[src[3]]<<4 | dec[src[4]]>>1 50 | id[3] = dec[src[4]]<<7 | dec[src[5]]<<2 | dec[src[6]]>>3 51 | id[4] = dec[src[6]]<<5 | dec[src[7]] 52 | id[5] = dec[src[8]]<<3 | dec[src[9]]>>2 53 | id[6] = dec[src[9]]<<6 | dec[src[10]]<<1 | dec[src[11]]>>4 54 | id[7] = dec[src[11]]<<4 | dec[src[12]]>>1 55 | } 56 | -------------------------------------------------------------------------------- /server/internal/pkg/encoding/encoding.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package encoding 18 | 19 | const ( 20 | // encoding stores a custom version of the base32 encoding. 21 | encoding = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" 22 | ) 23 | 24 | var ( 25 | // dec is the decoding map for base32 encoding 26 | dec [256]byte 27 | ) 28 | -------------------------------------------------------------------------------- /server/internal/pkg/hash/hash.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package hash 18 | 19 | import ( 20 | "encoding/binary" 21 | ) 22 | 23 | const ( 24 | offset32 uint32 = 0xcc9e2d51 25 | prime32 uint32 = 0x1b873593 26 | 27 | // Init is what 32 bits hash values should be initialized with. 28 | Init = offset32 29 | ) 30 | 31 | // Of returns the hash of bytes. it uses salt to shuffle the slice before calculating hash 32 | func WithSalt(text []byte, salt uint32) uint32 { 33 | b := shuffleInPlace(text, salt) 34 | return New(b) 35 | } 36 | 37 | // New returns the hash of bytes. 38 | func New(b []byte) uint32 { 39 | h := Init 40 | i := 0 41 | n := (len(b) / 8) * 8 42 | 43 | for i != n { 44 | h = (h ^ uint32(b[i])) * prime32 45 | h = (h ^ uint32(b[i+1])) * prime32 46 | h = (h ^ uint32(b[i+2])) * prime32 47 | h = (h ^ uint32(b[i+3])) * prime32 48 | h = (h ^ uint32(b[i+4])) * prime32 49 | h = (h ^ uint32(b[i+5])) * prime32 50 | h = (h ^ uint32(b[i+6])) * prime32 51 | h = (h ^ uint32(b[i+7])) * prime32 52 | i += 8 53 | } 54 | 55 | for _, c := range b[i:] { 56 | h = (h ^ uint32(c)) * prime32 57 | } 58 | 59 | return h 60 | } 61 | 62 | // shuffleInPlace shuffle the slice 63 | func shuffleInPlace(text []byte, contract uint32) []byte { 64 | if contract == 0 { 65 | return text 66 | } 67 | salt := make([]byte, 4) 68 | binary.BigEndian.PutUint32(salt[0:4], contract) 69 | 70 | result := duplicateSlice(text) 71 | for i, v, p := len(result)-1, 0, 0; i > 0; i-- { 72 | p += int(salt[v]) 73 | j := (int(salt[v]) + v + p) % i 74 | result[i], result[j] = result[j], result[i] 75 | v = (v + 1) % len(salt) 76 | } 77 | return result 78 | } 79 | 80 | // duplicateSlice get a copy of slice 81 | func duplicateSlice(data []byte) []byte { 82 | result := make([]byte, len(data)) 83 | copy(result, data) 84 | return result 85 | } 86 | -------------------------------------------------------------------------------- /server/internal/pkg/hash/ringhash.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // Package ringhash implementats a consistent ring hash: 18 | // https://en.wikipedia.org/wiki/Consistent_hashing 19 | package hash 20 | 21 | import ( 22 | "encoding/ascii85" 23 | "hash/fnv" 24 | "log" 25 | "sort" 26 | "strconv" 27 | ) 28 | 29 | // Hash is a signature of a hash function used by the package. 30 | type Hash func(data []byte) uint32 31 | 32 | type elem struct { 33 | key string 34 | hash uint32 35 | } 36 | 37 | type sortable []elem 38 | 39 | func (k sortable) Len() int { return len(k) } 40 | func (k sortable) Swap(i, j int) { k[i], k[j] = k[j], k[i] } 41 | func (k sortable) Less(i, j int) bool { 42 | // Weak hash function may cause collisions. 43 | if k[i].hash < k[j].hash { 44 | return true 45 | } 46 | if k[i].hash == k[j].hash { 47 | return k[i].key < k[j].key 48 | } 49 | return false 50 | } 51 | 52 | // Ring is the definition of the ringhash. 53 | type Ring struct { 54 | keys []elem // Sorted list of keys. 55 | 56 | signature string 57 | replicas int 58 | hashfunc Hash 59 | } 60 | 61 | // New initializes an empty ringhash with the given number of replicas and a hash function. 62 | // If the hash function is nil, fnv.New32a() is used. 63 | func NewRing(replicas int, fn Hash) *Ring { 64 | ring := &Ring{ 65 | replicas: replicas, 66 | hashfunc: fn, 67 | } 68 | if ring.hashfunc == nil { 69 | ring.hashfunc = func(data []byte) uint32 { 70 | hash := fnv.New32a() 71 | hash.Write(data) 72 | return hash.Sum32() 73 | } 74 | } 75 | return ring 76 | } 77 | 78 | // Len returns the number of keys in the ring. 79 | func (ring *Ring) Len() int { 80 | return len(ring.keys) 81 | } 82 | 83 | // Add adds keys to the ring. 84 | func (ring *Ring) Add(keys ...string) { 85 | for _, key := range keys { 86 | for i := 0; i < ring.replicas; i++ { 87 | ring.keys = append(ring.keys, elem{ 88 | hash: ring.hashfunc([]byte(strconv.Itoa(i) + key)), 89 | key: key}) 90 | } 91 | } 92 | sort.Sort(sortable(ring.keys)) 93 | 94 | // Calculate signature 95 | hash := fnv.New128a() 96 | b := make([]byte, 4) 97 | for _, key := range ring.keys { 98 | b[0] = byte(key.hash) 99 | b[1] = byte(key.hash >> 8) 100 | b[2] = byte(key.hash >> 16) 101 | b[3] = byte(key.hash >> 24) 102 | hash.Write(b) 103 | hash.Write([]byte(key.key)) 104 | } 105 | 106 | b = []byte{} 107 | b = hash.Sum(b) 108 | dst := make([]byte, ascii85.MaxEncodedLen(len(b))) 109 | ascii85.Encode(dst, b) 110 | ring.signature = string(dst) 111 | } 112 | 113 | // Get returns the closest item in the ring to the provided key. 114 | func (ring *Ring) Get(key string) string { 115 | 116 | if ring.Len() == 0 { 117 | return "" 118 | } 119 | 120 | hash := ring.hashfunc([]byte(key)) 121 | 122 | // Binary search for appropriate replica. 123 | idx := sort.Search(len(ring.keys), func(i int) bool { 124 | el := ring.keys[i] 125 | return (el.hash > hash) || (el.hash == hash && el.key >= key) 126 | }) 127 | 128 | // Means we have cycled back to the first replica. 129 | if idx == len(ring.keys) { 130 | idx = 0 131 | } 132 | 133 | return ring.keys[idx].key 134 | } 135 | 136 | // Signature returns the ring's hash signature. Two identical ringhashes 137 | // will have the same signature. Two hashes with different 138 | // number of keys or replicas or hash functions will have different 139 | // signatures. 140 | func (ring *Ring) Signature() string { 141 | return ring.signature 142 | } 143 | 144 | func (ring *Ring) dump() { 145 | for _, e := range ring.keys { 146 | log.Println("key %s hash %d", e.key, e.hash) 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /server/internal/pkg/log/log.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package log 18 | 19 | import ( 20 | "os" 21 | "strings" 22 | 23 | "github.com/rs/zerolog" 24 | ) 25 | 26 | // ConnLogger is the logger to use in application. 27 | var ConnLogger = zerolog.New(os.Stderr).With().Timestamp().Logger() 28 | 29 | // ErrLogger is the error logger to use in application. 30 | var ErrLogger = zerolog.New(os.Stderr).With().Timestamp().Logger() 31 | 32 | // errlogger is the error logger with caller to use in application. 33 | var errlogger = zerolog.New(os.Stderr).With().Timestamp().Caller().Logger() 34 | 35 | // Info logs the conn or sub/unsub action with a tag. 36 | func Info(context, action string) { 37 | ConnLogger.Info().Str("context", context).Msg(action) 38 | } 39 | 40 | // Error logs the error messages. 41 | func Error(context, err string) { 42 | errlogger.Error().Str("context", context).Msg(err) 43 | } 44 | 45 | // Fatal logs the fatal error messages. 46 | func Fatal(context, msg string, err error) { 47 | errlogger.Fatal(). 48 | Err(err). 49 | Str("context", context).Msg(msg) 50 | } 51 | 52 | // Debug logs the debug message with tag if it is turned on. 53 | func Debug(context, msg string) { 54 | ErrLogger.Debug().Str("context", context).Msg(msg) 55 | } 56 | 57 | // ParseLevel parses a string which represents a log level and returns 58 | // a zerolog.Level. 59 | func ParseLevel(level string, defaultLevel zerolog.Level) zerolog.Level { 60 | l := defaultLevel 61 | switch strings.ToLower(level) { 62 | case "0", "debug": 63 | l = zerolog.DebugLevel 64 | case "1", "info": 65 | l = zerolog.InfoLevel 66 | case "2", "warn": 67 | l = zerolog.WarnLevel 68 | case "3", "error": 69 | l = zerolog.ErrorLevel 70 | case "4", "fatal": 71 | l = zerolog.FatalLevel 72 | } 73 | return l 74 | } 75 | -------------------------------------------------------------------------------- /server/internal/pkg/metrics/counter.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import "sync/atomic" 20 | 21 | // Counters hold an int64 value that can be incremented and decremented. 22 | type Counter interface { 23 | Reset() 24 | Count() int64 25 | Dec(int64) 26 | Inc(int64) 27 | Snapshot() Counter 28 | } 29 | 30 | // GetOrRegisterCounter returns an existing Counter or constructs and registers 31 | // a new StandardCounter. 32 | func GetOrRegisterCounter(name string, r Metrics) Counter { 33 | return r.GetOrRegister(name, NewCounter).(Counter) 34 | } 35 | 36 | // NewCounter constructs a new StandardCounter. 37 | func NewCounter() Counter { 38 | return &counter{0} 39 | } 40 | 41 | // CounterSnapshot is a read-only copy of another Counter. 42 | type CounterSnapshot int64 43 | 44 | // Clear panics. 45 | func (CounterSnapshot) Reset() { 46 | panic("Clear called on a CounterSnapshot") 47 | } 48 | 49 | // Count returns the count at the time the snapshot was taken. 50 | func (c CounterSnapshot) Count() int64 { return int64(c) } 51 | 52 | // Dec panics. 53 | func (CounterSnapshot) Dec(int64) { 54 | panic("Dec called on a CounterSnapshot") 55 | } 56 | 57 | // Inc panics. 58 | func (CounterSnapshot) Inc(int64) { 59 | panic("Inc called on a CounterSnapshot") 60 | } 61 | 62 | // Snapshot returns the snapshot. 63 | func (c CounterSnapshot) Snapshot() Counter { return c } 64 | 65 | // StandardCounter is the standard implementation of a Counter and uses the 66 | // sync/atomic package to manage a single int64 value. 67 | type counter struct { 68 | count int64 69 | } 70 | 71 | // Clear sets the counter to zero. 72 | func (c *counter) Reset() { 73 | atomic.StoreInt64(&c.count, 0) 74 | } 75 | 76 | // Count returns the current count. 77 | func (c *counter) Count() int64 { 78 | return atomic.LoadInt64(&c.count) 79 | } 80 | 81 | // Dec decrements the counter by the given amount. 82 | func (c *counter) Dec(i int64) { 83 | atomic.AddInt64(&c.count, -i) 84 | } 85 | 86 | // Inc increments the counter by the given amount. 87 | func (c *counter) Inc(i int64) { 88 | atomic.AddInt64(&c.count, i) 89 | } 90 | 91 | // Snapshot returns a read-only copy of the counter. 92 | func (c *counter) Snapshot() Counter { 93 | return CounterSnapshot(c.Count()) 94 | } 95 | -------------------------------------------------------------------------------- /server/internal/pkg/metrics/gauage.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import "sync/atomic" 20 | 21 | // Gauges hold an int64 value that can be set arbitrarily. 22 | type Gauge interface { 23 | Snapshot() Gauge 24 | Update(int64) 25 | Value() int64 26 | } 27 | 28 | // GetOrRegisterGauge returns an existing Gauge or constructs and registers a 29 | // new StandardGauge. 30 | func GetOrRegisterGauge(name string, r Metrics) Gauge { 31 | return r.GetOrRegister(name, NewGauge).(Gauge) 32 | } 33 | 34 | // NewGauge constructs a new StandardGauge. 35 | func NewGauge() Gauge { 36 | return &gauge{0} 37 | } 38 | 39 | // GaugeSnapshot is a read-only copy of another Gauge. 40 | type GaugeSnapshot int64 41 | 42 | // Snapshot returns the snapshot. 43 | func (g GaugeSnapshot) Snapshot() Gauge { return g } 44 | 45 | // Update panics. 46 | func (GaugeSnapshot) Update(int64) { 47 | panic("Update called on a GaugeSnapshot") 48 | } 49 | 50 | // Value returns the value at the time the snapshot was taken. 51 | func (g GaugeSnapshot) Value() int64 { return int64(g) } 52 | 53 | // StandardGauge is the standard implementation of a Gauge and uses the 54 | // sync/atomic package to manage a single int64 value. 55 | type gauge struct { 56 | value int64 57 | } 58 | 59 | // Snapshot returns a read-only copy of the gauge. 60 | func (g *gauge) Snapshot() Gauge { 61 | return GaugeSnapshot(g.Value()) 62 | } 63 | 64 | // Update updates the gauge's value. 65 | func (g *gauge) Update(v int64) { 66 | atomic.StoreInt64(&g.value, v) 67 | } 68 | 69 | // Value returns the gauge's current value. 70 | func (g *gauge) Value() int64 { 71 | return atomic.LoadInt64(&g.value) 72 | } 73 | -------------------------------------------------------------------------------- /server/internal/pkg/metrics/metrics.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package metrics 18 | 19 | import ( 20 | "fmt" 21 | "reflect" 22 | "sync" 23 | ) 24 | 25 | // Metrics implementation is copied from github.com/rcrowley/go-metrics and it is simplified for tracking Trace metrics 26 | 27 | // DuplicateMetric is the error returned by Registry.Register when a metric 28 | // already exists. If you mean to Register that metric you must first 29 | // Unregister the existing metric. 30 | type DuplicateMetric string 31 | 32 | func (err DuplicateMetric) Error() string { 33 | return fmt.Sprintf("duplicate metric: %s", string(err)) 34 | } 35 | 36 | // A Metrics holds registry references to a set of metrics by name and can iterate 37 | // over them, calling callback functions provided by the user. 38 | // 39 | // This is an interface so as to encourage other structs to implement 40 | // the Metrics API as appropriate. 41 | type Metrics interface { 42 | 43 | // Gets an existing metric or registers the given one. 44 | // The interface can be the metric to register if not found in registry, 45 | // or a function returning the metric for lazy instantiation. 46 | GetOrRegister(string, interface{}) interface{} 47 | 48 | // Unregister the metric with the given name. 49 | Unregister(string) 50 | 51 | // Unregister all metrics. (Mostly for testing.) 52 | UnregisterAll() 53 | } 54 | 55 | // The standard implementation of a Registry is a mutex-protected map 56 | // of names to metrics. 57 | type StandardMetrics struct { 58 | metrics map[string]interface{} 59 | mutex sync.RWMutex 60 | } 61 | 62 | // Create a new registry. 63 | func NewMetrics() Metrics { 64 | return &StandardMetrics{metrics: make(map[string]interface{})} 65 | } 66 | 67 | // Gets an existing metric or creates and registers a new one. Threadsafe 68 | // alternative to calling Get and Register on failure. 69 | // The interface can be the metric to register if not found in registry, 70 | // or a function returning the metric for lazy instantiation. 71 | func (m *StandardMetrics) GetOrRegister(name string, i interface{}) interface{} { 72 | // access the read lock first which should be re-entrant 73 | m.mutex.RLock() 74 | metric, ok := m.metrics[name] 75 | m.mutex.RUnlock() 76 | if ok { 77 | return metric 78 | } 79 | 80 | // only take the write lock if we'll be modifying the metrics map 81 | m.mutex.Lock() 82 | defer m.mutex.Unlock() 83 | if metric, ok := m.metrics[name]; ok { 84 | return metric 85 | } 86 | if v := reflect.ValueOf(i); v.Kind() == reflect.Func { 87 | i = v.Call(nil)[0].Interface() 88 | } 89 | m.register(name, i) 90 | return i 91 | } 92 | 93 | // Unregister the metric with the given name. 94 | func (m *StandardMetrics) Unregister(name string) { 95 | m.mutex.Lock() 96 | defer m.mutex.Unlock() 97 | m.stop(name) 98 | delete(m.metrics, name) 99 | } 100 | 101 | // Unregister all metrics. (Mostly for testing.) 102 | func (m *StandardMetrics) UnregisterAll() { 103 | m.mutex.Lock() 104 | defer m.mutex.Unlock() 105 | for name, _ := range m.metrics { 106 | m.stop(name) 107 | delete(m.metrics, name) 108 | } 109 | } 110 | 111 | func (m *StandardMetrics) register(name string, i interface{}) error { 112 | if _, ok := m.metrics[name]; ok { 113 | return DuplicateMetric(name) 114 | } 115 | switch i.(type) { 116 | case Counter, Gauge: 117 | m.metrics[name] = i 118 | } 119 | return nil 120 | } 121 | 122 | func (m *StandardMetrics) stop(name string) { 123 | if i, ok := m.metrics[name]; ok { 124 | if s, ok := i.(Stoppable); ok { 125 | s.Stop() 126 | } 127 | } 128 | } 129 | 130 | // Stoppable defines the metrics which has to be stopped. 131 | type Stoppable interface { 132 | Stop() 133 | } 134 | -------------------------------------------------------------------------------- /server/internal/pkg/stats/buffers.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import "sync/atomic" 20 | 21 | // checkBuf checks current buffer for overflow, and flushes buffer up to lastLen bytes on overflow 22 | // 23 | // overflow part is preserved in flushBuf 24 | func (t *transport) checkBuf(lastLen int) { 25 | if len(t.buf) > t.maxPacketSize { 26 | t.flushBuf(lastLen) 27 | } 28 | } 29 | 30 | // flushBuf sends buffer to the queue and initializes new buffer 31 | func (t *transport) flushBuf(length int) { 32 | sendBuf := t.buf[0:length] 33 | tail := t.buf[length:len(t.buf)] 34 | 35 | // get new buffer 36 | select { 37 | case t.buf = <-t.bufPool: 38 | t.buf = t.buf[0:0] 39 | default: 40 | t.buf = make([]byte, 0, t.bufSize) 41 | } 42 | 43 | // copy tail to the new buffer 44 | t.buf = append(t.buf, tail...) 45 | 46 | // flush current buffer 47 | select { 48 | case t.sendQueue <- sendBuf: 49 | default: 50 | // flush failed, we lost some data 51 | atomic.AddInt64(&t.lostPacketsPeriod, 1) 52 | atomic.AddInt64(&t.lostPacketsOverall, 1) 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /server/internal/pkg/stats/tags.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package stats 18 | 19 | import "strconv" 20 | 21 | // Tag placement constants 22 | const ( 23 | TagPlacementName = iota 24 | TagPlacementSuffix 25 | ) 26 | 27 | // TagFormat controls tag formatting style 28 | type TagFormat struct { 29 | // FirstSeparator is put after metric name and before first tag 30 | FirstSeparator string 31 | // Placement specifies part of the message to append tags to 32 | Placement byte 33 | // OtherSeparator separates 2nd and subsequent tags from each other 34 | OtherSeparator byte 35 | // KeyValueSeparator separates tag name and tag value 36 | KeyValueSeparator byte 37 | } 38 | 39 | // Tag types 40 | const ( 41 | typeString = iota 42 | typeInt64 43 | ) 44 | 45 | // Tag is metric-specific tag 46 | type Tag struct { 47 | name string 48 | strvalue string 49 | intvalue int64 50 | typ byte 51 | } 52 | 53 | // Append formats tag and appends it to the buffer 54 | func (tag Tag) Append(buf []byte, style *TagFormat) []byte { 55 | buf = append(buf, []byte(tag.name)...) 56 | buf = append(buf, style.KeyValueSeparator) 57 | if tag.typ == typeString { 58 | return append(buf, []byte(tag.strvalue)...) 59 | } 60 | return strconv.AppendInt(buf, tag.intvalue, 10) 61 | } 62 | 63 | // StringTag creates Tag with string value 64 | func StringTag(name, value string) Tag { 65 | return Tag{name: name, strvalue: value, typ: typeString} 66 | } 67 | 68 | // IntTag creates Tag with integer value 69 | func IntTag(name string, value int) Tag { 70 | return Tag{name: name, intvalue: int64(value), typ: typeInt64} 71 | } 72 | 73 | // Int64Tag creates Tag with integer value 74 | func Int64Tag(name string, value int64) Tag { 75 | return Tag{name: name, intvalue: value, typ: typeInt64} 76 | } 77 | 78 | func (s *Stats) formatTags(buf []byte, tags []Tag) []byte { 79 | tagsLen := len(s.defaultTags) + len(tags) 80 | if tagsLen == 0 { 81 | return buf 82 | } 83 | 84 | buf = append(buf, []byte(s.trans.tagFormat.FirstSeparator)...) 85 | for i := range s.defaultTags { 86 | buf = s.defaultTags[i].Append(buf, s.trans.tagFormat) 87 | if i != tagsLen-1 { 88 | buf = append(buf, s.trans.tagFormat.OtherSeparator) 89 | } 90 | } 91 | 92 | for i := range tags { 93 | buf = tags[i].Append(buf, s.trans.tagFormat) 94 | if i+len(s.defaultTags) != tagsLen-1 { 95 | buf = append(buf, s.trans.tagFormat.OtherSeparator) 96 | } 97 | } 98 | 99 | return buf 100 | } 101 | 102 | var ( 103 | // TagFormatInfluxDB is format for InfluxDB StatsD telegraf plugin 104 | // 105 | // Docs: https://github.com/influxdata/telegraf/tree/master/plugins/inputs/statsd 106 | TagFormatInfluxDB = &TagFormat{ 107 | Placement: TagPlacementName, 108 | FirstSeparator: ",", 109 | OtherSeparator: ',', 110 | KeyValueSeparator: '=', 111 | } 112 | 113 | // TagFormatDatadog is format for DogStatsD (Datadog Agent) 114 | // 115 | // Docs: https://docs.datadoghq.com/developers/dogstatsd/#datagram-format 116 | TagFormatDatadog = &TagFormat{ 117 | Placement: TagPlacementSuffix, 118 | FirstSeparator: "|#", 119 | OtherSeparator: ',', 120 | KeyValueSeparator: ':', 121 | } 122 | ) 123 | -------------------------------------------------------------------------------- /server/internal/pkg/uid/connid.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package uid 18 | 19 | import ( 20 | "sync/atomic" 21 | ) 22 | 23 | // LID represents a process-wide unique ID. 24 | type LID uint32 25 | 26 | // NewID generates a new, process-wide unique ID. 27 | func NewLID() LID { 28 | return LID(atomic.AddUint32(&Next, 1)) 29 | } 30 | -------------------------------------------------------------------------------- /server/internal/pkg/uid/uid.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package uid 18 | 19 | import ( 20 | "encoding/binary" 21 | "math" 22 | "math/rand" 23 | "time" 24 | ) 25 | 26 | const ( 27 | Offset = 1555770000 28 | ) 29 | 30 | var ( 31 | // next is the next identifier. It is time in millsecond 32 | // to avoid collisions of ids between process restarts. 33 | Next = uint32( 34 | time.Date(2070, 1, 1, 0, 0, 0, 0, time.UTC).Sub(TimeNow()), 35 | ) 36 | ) 37 | 38 | func NewApoch() uint32 { 39 | now := uint32(TimeNow().Unix() - Offset) 40 | return math.MaxUint32 - now 41 | } 42 | 43 | func NewUnique() uint32 { 44 | b := make([]byte, 4) 45 | random := rand.New(rand.NewSource(int64(NewApoch()))) 46 | random.Read(b) 47 | return binary.BigEndian.Uint32(b) 48 | } 49 | 50 | // TimeNow returns current wall time in UTC rounded to milliseconds. 51 | func TimeNow() time.Time { 52 | return time.Now().UTC().Round(time.Millisecond) 53 | } 54 | -------------------------------------------------------------------------------- /server/internal/types/types.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package types 18 | 19 | import ( 20 | "github.com/unit-io/unitdb/server/internal/message/security" 21 | ) 22 | 23 | // Error represents an event code which provides a more details. 24 | type Error struct { 25 | ReturnCode uint8 26 | Status int `json:"status"` 27 | Message string `json:"message"` 28 | ID int `json:"id,omitempty"` 29 | } 30 | 31 | // Error implements error interface. 32 | func (e *Error) Error() string { return e.Message } 33 | 34 | // ErrorCode implements error interface. 35 | func (e *Error) ErrrorCode() uint8 { return e.ReturnCode } 36 | 37 | // Represents a set of errors used in the handlers. 38 | var ( 39 | ErrInvalidProto = &Error{ReturnCode: 0x01, Status: 401, Message: "Unacceptable proto version. The proto version is invalid."} 40 | ErrInvalidClientID = &Error{ReturnCode: 0x02, Status: 401, Message: "Identifier rejected. The client ID is invalid or missing. Use a valid client Id or use an auto generated client ID in the connection request."} 41 | ErrClientIdForbidden = &Error{ReturnCode: 0x03, Status: 403, Message: "Unacceptable identifier, access not allowed use primary client Id to request a secondary client ID."} 42 | ErrUnauthorized = &Error{ReturnCode: 0x04, Status: 401, Message: "Security key rejected. The security key provided is not authorized to perform this operation."} 43 | ErrServerError = &Error{ReturnCode: 0x05, Status: 500, Message: "An unexpected condition was encountered."} 44 | ErrBadToken = &Error{ReturnCode: 0x06, Status: 403, Message: "Authentication failed."} 45 | ErrForbidden = &Error{ReturnCode: 0x07, Status: 403, Message: "The request is understood, but it has been refused or access is not allowed."} 46 | ErrSessionExist = &Error{ReturnCode: 0x08, Status: 403, Message: "Another connection using the same session ID has an active connection causing this connection to be closed."} 47 | ErrUnknownEpoch = &Error{ReturnCode: 0x09, Status: 403, Message: "Unknown authentication epoch."} 48 | ErrTimteout = &Error{ReturnCode: 0x10, Status: 504, Message: "The network connection timeout."} 49 | ErrNotFound = &Error{ReturnCode: 0x11, Status: 404, Message: "The resource requested does not exist."} 50 | ErrBadRequest = &Error{ReturnCode: 0x12, Status: 400, Message: "The request was invalid or cannot be otherwise served."} 51 | ErrTargetTooLong = &Error{ReturnCode: 0x13, Status: 400, Message: "Topic can not have more than 23 parts."} 52 | ErrNotImplemented = &Error{ReturnCode: 0x14, Status: 501, Message: "The server does not recognize the request method."} 53 | ) 54 | 55 | type KeyGenRequest struct { 56 | Topic string `json:"topic"` 57 | Type string `json:"type"` 58 | } 59 | 60 | func (m *KeyGenRequest) Access() uint32 { 61 | required := security.AllowNone 62 | 63 | for i := 0; i < len(m.Type); i++ { 64 | switch c := m.Type[i]; c { 65 | case 'o': 66 | required |= security.AllowOwner | security.AllowAdmin | security.AllowReadWrite 67 | case 'a': 68 | required |= security.AllowAdmin | security.AllowReadWrite 69 | case 'r': 70 | required |= security.AllowRead 71 | case 'w': 72 | required |= security.AllowWrite 73 | } 74 | } 75 | 76 | return required 77 | } 78 | 79 | type KeyGenResponse struct { 80 | Status int `json:"status"` 81 | Key string `json:"key"` 82 | Topic string `json:"topic"` 83 | } 84 | 85 | type ClientIdResponse struct { 86 | Status int `json:"status"` 87 | ClientId string `json:"key"` 88 | } 89 | -------------------------------------------------------------------------------- /server/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package main 18 | 19 | import ( 20 | "encoding/json" 21 | "flag" 22 | "os" 23 | "path/filepath" 24 | "time" 25 | 26 | jcr "github.com/DisposaBoy/JsonConfigReader" 27 | "github.com/rs/zerolog" 28 | "github.com/unit-io/unitdb/server/internal" 29 | "github.com/unit-io/unitdb/server/internal/config" 30 | "github.com/unit-io/unitdb/server/internal/pkg/log" 31 | ) 32 | 33 | func main() { 34 | // Get the directory of the process 35 | exe, err := os.Executable() 36 | if err != nil { 37 | panic(err.Error()) 38 | } 39 | 40 | var configfile = flag.String("config", "unitdb.conf", "Path to config file.") 41 | var listenOn = flag.String("listen", "", "Override address and port to listen on for HTTP(S) clients.") 42 | var listenGrpcOn = flag.String("grpc_listen", "", "Override address and port to listen on for GRPC clients.") 43 | var clusterSelf = flag.String("cluster_self", "", "Override the name of the current cluster node") 44 | var dbPath = flag.String("db_path", "/tmp/unitdb", "Override the db path.") 45 | var varzPath = flag.String("varz", "/varz", "Expose runtime stats at the given endpoint, e.g. /varz. Disabled if not set") 46 | flag.Parse() 47 | 48 | // Default level for is fatal, unless debug flag is present 49 | zerolog.SetGlobalLevel(zerolog.InfoLevel) 50 | 51 | //*configfile = toAbsolutePath(rootpath, *configfile) 52 | *configfile = filepath.Join(filepath.Dir(exe), *configfile) 53 | log.Debug("main", "Using config from "+*configfile) 54 | var cfg *config.Config 55 | if file, err := os.Open(*configfile); err != nil { 56 | log.Fatal("main", "Failed to read config file", err) 57 | } else if err = json.NewDecoder(jcr.New(file)).Decode(&cfg); err != nil { 58 | log.Fatal("main", "Failed to parse config file", err) 59 | } 60 | 61 | zerolog.DurationFieldUnit = time.Nanosecond 62 | if cfg.LoggingLevel != "" { 63 | l := log.ParseLevel(cfg.LoggingLevel, zerolog.InfoLevel) 64 | zerolog.SetGlobalLevel(l) 65 | } 66 | 67 | if *listenOn != "" { 68 | cfg.Listen = *listenOn 69 | } 70 | 71 | // Set up gRPC server, if one is configured 72 | if *listenGrpcOn != "" { 73 | cfg.GrpcListen = *listenGrpcOn 74 | } 75 | 76 | if *dbPath != "" { 77 | cfg.DBPath = *dbPath 78 | } 79 | 80 | if *varzPath != "" { 81 | cfg.VarzPath = *varzPath 82 | } 83 | 84 | // Initialize cluster and receive calculated workerId. 85 | // Cluster won't be started here yet. 86 | internal.ClusterInit(cfg.Cluster, clusterSelf) 87 | 88 | svc, err := internal.NewService(cfg) 89 | if err != nil { 90 | panic(err.Error()) 91 | } 92 | 93 | // Start accepting cluster traffic. 94 | if internal.Globals.Cluster != nil { 95 | internal.Globals.Cluster.Start() 96 | } 97 | 98 | internal.Globals.Service = svc 99 | 100 | // Listen and serve 101 | svc.Listen() 102 | log.Info("main", "Service is running at port "+cfg.Listen) 103 | } 104 | -------------------------------------------------------------------------------- /server/proto/unitdb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package unitdb.schema; 4 | 5 | option go_package='.;schema'; 6 | 7 | // Unitdb server interface 8 | service Unitdb { 9 | rpc Stream (stream Packet) returns (stream Packet); 10 | } 11 | 12 | message Empty { 13 | } 14 | 15 | message Packet { 16 | bytes data=1; 17 | } 18 | 19 | enum FlowControl { 20 | NONE=0; 21 | ACKNOWLEDGE=1; 22 | NOTIFY=2; 23 | RECEIVE=3; 24 | RECEIPT=4; 25 | COMPLETE=5; 26 | } 27 | 28 | enum MessageType { 29 | RERSERVED=0; 30 | CONNECT=1; 31 | PUBLISH=2; 32 | RELAY=3; 33 | SUBSCRIBE=4; 34 | UNSUBSCRIBE=5; 35 | PINGREQ=6; 36 | DISCONNECT=7; 37 | } 38 | 39 | message FixedHeader { 40 | MessageType MessageType=1; 41 | FlowControl FlowControl=2; 42 | int32 MessageLength=3; 43 | } 44 | 45 | // Connect represents a CONNECT Message type. 46 | message Connect { 47 | int32 Version=1; 48 | bool InsecureFlag=2; 49 | string ClientID=3; 50 | int32 KeepAlive=4; 51 | bool CleanSessFlag=5; 52 | int32 SessKey=6; 53 | string Username=7; 54 | bytes Password=8; 55 | int32 BatchDuration=9; 56 | int32 BatchByteThreshold=10; 57 | int32 BatchCountThreshold=11; 58 | } 59 | 60 | // ConnectAcknowledge represents a CONNECT Acknowledge Message type. 61 | // 0x00 connection accepted 62 | // 0x01 refused: unacceptable proto version 63 | // 0x02 refused: identifier rejected 64 | // 0x03 refused: unacceptable identifier, access not allowed 65 | // 0x04 refused server unavailiable 66 | // 0x05 not authorized 67 | // 0x06 bad request 68 | message ConnectAcknowledge { 69 | int32 ReturnCode=1; 70 | int32 Epoch=2; 71 | int32 ConnID=3; 72 | } 73 | 74 | // PingRequest is a keepalive 75 | message PingRequest { 76 | } 77 | 78 | // Disconnect is to signal client want to cease communications with the server. 79 | message Disconnect { 80 | int32 MessageID=1; 81 | } 82 | 83 | message PublishMessage { 84 | string Topic=1; 85 | bytes Payload=2; 86 | string Ttl=3; 87 | } 88 | 89 | // Publish represents a PUBREQ Message type. It supports following delivery mode. 90 | // 0 EXPRESS 91 | // 1 RELIEABLE 92 | // 2 BATCH 93 | message Publish { 94 | int32 MessageID=1; 95 | int32 DeliveryMode=2; 96 | repeated PublishMessage Messages=3; 97 | } 98 | 99 | // RelayRequest is pairing the Topic and Last parameter together 100 | message RelayRequest { 101 | string Topic=1; 102 | string Last=2; 103 | } 104 | 105 | // Relay tells the server which topics and last durations the client would like get data persisted on Servr. The Delivery Mode for relay is EXPRESS. 106 | message Relay { 107 | int32 MessageID=1; 108 | repeated RelayRequest relayRequests=2; 109 | } 110 | 111 | // Subscription is pairing the Delivery Mode and Topic together 112 | // for the delivery mode's pairs in unsubscribe and subscribe. 113 | // Delay in Millisecond to delay the messsage delivery if DeliveryMode is RELIEABLE or BATCH. 114 | // 0 EXPRESS 115 | // 1 RELIEABLE 116 | // 2 BATCH 117 | message Subscription { 118 | int32 DeliveryMode=1; 119 | int32 Delay=2; 120 | string Topic=3; 121 | } 122 | 123 | // Subscribe tells the server which topics the client would like to subscribe to and choose a Delivery Mode. 124 | message Subscribe { 125 | int32 MessageID=1; 126 | repeated Subscription Subscriptions=2; 127 | } 128 | 129 | // Unsubscribe is the Message to send if you don't want to subscribe to a topic anymore. 130 | message Unsubscribe { 131 | int32 MessageID=1; 132 | repeated Subscription Subscriptions=2; 133 | } 134 | 135 | // ControlMessage is a Flow Control Message. 136 | message ControlMessage { 137 | int32 MessageID=1; 138 | // Optional Control Message bytes 139 | bytes Message=2; 140 | } -------------------------------------------------------------------------------- /server/unitdb.conf: -------------------------------------------------------------------------------- 1 | { 2 | // Default HTTP(S) address:port to listen on for websocket. Either a 3 | // numeric or a canonical name, e.g. ":80" or ":https". Could include a host name, e.g. 4 | // "localhost:80". 5 | // Could be blank: if TLS is not configured, will use ":80", otherwise ":443". 6 | // Can be overridden from the command line, see option --listen. 7 | "listen": ":6060", 8 | 9 | // Default HTTP(S) address:port to listen on for grpc. Either a 10 | // numeric or a canonical name, e.g. ":80" or ":https". Could include a host name, e.g. 11 | // "localhost:80". 12 | // Could be blank: if TLS is not configured, will use ":80", otherwise ":443". 13 | // Can be overridden from the command line, see option --listen. 14 | "grpc_listen": ":6080", 15 | 16 | // Default logging level is "InfoLevel" so to enable the debug log set the "LogLevel" to "DebugLevel". 17 | "logging_level": "Error", 18 | 19 | // Maximum message size allowed from client in bytes (262144 = 256KB). 20 | // Intended to prevent malicious clients from sending very large messages inband (does 21 | // not affect out-of-band large files). 22 | "max_message_size": 262144, 23 | 24 | // Maximum number of subscribers per group topic. 25 | "max_subscriber_count": 128, 26 | 27 | // Encryption configuration 28 | "encryption_config": { 29 | // chacha20poly1305 encryption key for client Ids and topic keys. 32 random bytes base64-encoded. 30 | // Generate your own and keep it secret. 31 | "key": "4BWm1vZletvrCDGWsF6mex8oBSd59m6I", 32 | // Key identifier. it is useful when you use multiple keys. 33 | "identifier":"local", 34 | // slealed flag tells if key in the configuration is sealed. 35 | "sealed":false, 36 | // timestamp is helpful to determine the latest key in case of keyroll over. 37 | "timestamp":1522325758 38 | }, 39 | 40 | // Cluster-mode configuration. 41 | "cluster_config": { 42 | // Name of this node. Can be assigned from the command line. 43 | // Empty string disables clustering. 44 | "self": "", 45 | 46 | // List of available nodes. 47 | "nodes": [ 48 | // Name and TCP address of every node in the cluster. 49 | {"name": "one", "addr":"localhost:12001"}, 50 | {"name": "two", "addr":"localhost:12002"}, 51 | {"name": "three", "addr":"localhost:12003"} 52 | ], 53 | 54 | // Failover config. 55 | "failover": { 56 | // Failover is enabled. 57 | "enabled": true, 58 | // Time in milliseconds between heartbeats. 59 | "heartbeat": 100, 60 | // Initiate leader election when the leader is not available for this many heartbeats. 61 | "vote_after": 8, 62 | // Consider node failed when it missed this many heartbeats. 63 | "node_fail_after": 16 64 | } 65 | }, 66 | 67 | // Database configuration 68 | "store_config": { 69 | // reset message store on service restart 70 | "reset": false, 71 | // Configurations of individual adapters. 72 | "adapters": { 73 | // unitdb configuration. 74 | "unitdb": { 75 | // Name of the database. 76 | "database": "unitdb", 77 | // Memdb message store size 78 | "mem_size": 500000000 79 | } 80 | } 81 | } 82 | } -------------------------------------------------------------------------------- /server/utp/flow_control.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package utp 18 | 19 | import ( 20 | "bytes" 21 | 22 | "github.com/golang/protobuf/proto" 23 | pbx "github.com/unit-io/unitdb/server/proto" 24 | ) 25 | 26 | // FlowControl represents FlowControl Message type 27 | type FlowControl uint8 28 | 29 | const ( 30 | // Flow Control 31 | NONE FlowControl = iota 32 | ACKNOWLEDGE 33 | NOTIFY 34 | RECEIVE 35 | RECEIPT 36 | COMPLETE 37 | ) 38 | 39 | type ControlMessage struct { 40 | MessageID uint16 41 | MessageType MessageType 42 | FlowControl FlowControl 43 | Message []byte 44 | } 45 | 46 | // ToBinary encodes the Control Message into binary data 47 | func (c *ControlMessage) ToBinary() (bytes.Buffer, error) { 48 | var buf bytes.Buffer 49 | var err error 50 | var fh FixedHeader 51 | ctrl := pbx.ControlMessage{ 52 | MessageID: int32(c.MessageID), 53 | Message: c.Message, 54 | } 55 | rawMsg, err := proto.Marshal(&ctrl) 56 | if err != nil { 57 | return buf, err 58 | } 59 | switch c.FlowControl { 60 | case ACKNOWLEDGE: 61 | switch c.MessageType { 62 | case CONNECT: 63 | fh = FixedHeader{MessageType: CONNECT, FlowControl: ACKNOWLEDGE, MessageLength: len(rawMsg)} 64 | case PUBLISH: 65 | fh = FixedHeader{MessageType: PUBLISH, FlowControl: ACKNOWLEDGE, MessageLength: len(rawMsg)} 66 | case RELAY: 67 | fh = FixedHeader{MessageType: RELAY, FlowControl: ACKNOWLEDGE, MessageLength: len(rawMsg)} 68 | case SUBSCRIBE: 69 | fh = FixedHeader{MessageType: SUBSCRIBE, FlowControl: ACKNOWLEDGE, MessageLength: len(rawMsg)} 70 | case UNSUBSCRIBE: 71 | fh = FixedHeader{MessageType: UNSUBSCRIBE, FlowControl: ACKNOWLEDGE, MessageLength: len(rawMsg)} 72 | case PINGREQ: 73 | fh = FixedHeader{MessageType: PINGREQ, FlowControl: ACKNOWLEDGE, MessageLength: len(rawMsg)} 74 | } 75 | case NOTIFY: 76 | fh = FixedHeader{MessageType: PUBLISH, FlowControl: NOTIFY, MessageLength: len(rawMsg)} 77 | case RECEIVE: 78 | fh = FixedHeader{MessageType: PUBLISH, FlowControl: RECEIVE, MessageLength: len(rawMsg)} 79 | case RECEIPT: 80 | fh = FixedHeader{MessageType: PUBLISH, FlowControl: RECEIPT, MessageLength: len(rawMsg)} 81 | case COMPLETE: 82 | fh = FixedHeader{MessageType: PUBLISH, FlowControl: COMPLETE, MessageLength: len(rawMsg)} 83 | } 84 | buf = fh.pack() 85 | _, err = buf.Write(rawMsg) 86 | return buf, err 87 | } 88 | 89 | func (c *ControlMessage) FromBinary(fh FixedHeader, data []byte) { 90 | var ctrl pbx.ControlMessage 91 | proto.Unmarshal(data, &ctrl) 92 | 93 | c.MessageID = uint16(ctrl.MessageID) 94 | c.MessageType = fh.MessageType 95 | c.FlowControl = fh.FlowControl 96 | c.Message = ctrl.Message 97 | } 98 | 99 | // Type returns the Message type. 100 | func (c *ControlMessage) Type() MessageType { 101 | return FLOWCONTROL 102 | } 103 | 104 | // Info returns DeliveryMode and MessageID of this Message. 105 | func (c *ControlMessage) Info() Info { 106 | return Info{DeliveryMode: 1, MessageID: c.MessageID} 107 | } 108 | -------------------------------------------------------------------------------- /server/utp/message.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package utp 18 | 19 | import ( 20 | "bytes" 21 | "io" 22 | 23 | "github.com/golang/protobuf/proto" 24 | pbx "github.com/unit-io/unitdb/server/proto" 25 | ) 26 | 27 | // MessageType represents a Message type 28 | type MessageType uint8 29 | 30 | const ( 31 | // Message 32 | CONNECT MessageType = iota + 1 33 | PUBLISH 34 | RELAY 35 | SUBSCRIBE 36 | UNSUBSCRIBE 37 | PINGREQ 38 | DISCONNECT 39 | FLOWCONTROL 40 | ) 41 | 42 | // Below are the const definitions for error codes returned by 43 | // Connect() 44 | const ( 45 | Accepted = 0x00 46 | ErrRefusedBadProtocolVersion = 0x01 47 | ErrRefusedIDRejected = 0x02 48 | ErrRefusedbADID = 0x03 49 | ErrRefusedServerUnavailable = 0x04 50 | ErrNotAuthorised = 0x05 51 | ErrBadRequest = 0x06 52 | ) 53 | 54 | // FixedHeader 55 | type FixedHeader struct { 56 | MessageType MessageType 57 | FlowControl FlowControl 58 | MessageLength int 59 | } 60 | 61 | // Info returns Qos and MessageID by the Info() function called on the Packet 62 | type Info struct { 63 | DeliveryMode uint8 64 | MessageID uint16 65 | } 66 | 67 | func (fh *FixedHeader) pack() bytes.Buffer { 68 | var head bytes.Buffer 69 | ph := pbx.FixedHeader{ 70 | MessageType: pbx.MessageType(fh.MessageType), 71 | FlowControl: pbx.FlowControl(fh.FlowControl), 72 | MessageLength: int32(fh.MessageLength), 73 | } 74 | h, err := proto.Marshal(&ph) 75 | if err != nil { 76 | return head 77 | } 78 | size := encodeLength(len(h)) 79 | head.Write(size) 80 | head.Write(h) 81 | return head 82 | } 83 | 84 | func (fh *FixedHeader) FromBinary(r io.Reader) error { 85 | fhSize, err := decodeLength(r) 86 | if err != nil { 87 | return err 88 | } 89 | 90 | // read FixedHeader 91 | head := make([]byte, fhSize) 92 | if _, err := io.ReadFull(r, head); err != nil { 93 | return err 94 | } 95 | 96 | var h pbx.FixedHeader 97 | proto.Unmarshal(head, &h) 98 | 99 | *fh = FixedHeader{ 100 | MessageType: MessageType(h.MessageType), 101 | FlowControl: FlowControl(h.FlowControl), 102 | MessageLength: int(h.MessageLength), 103 | } 104 | 105 | return nil 106 | } 107 | 108 | func encodeLength(length int) []byte { 109 | var encLength []byte 110 | for { 111 | digit := byte(length % 128) 112 | length /= 128 113 | if length > 0 { 114 | digit |= 0x80 115 | } 116 | encLength = append(encLength, digit) 117 | if length == 0 { 118 | break 119 | } 120 | } 121 | return encLength 122 | } 123 | 124 | func decodeLength(r io.Reader) (int, error) { 125 | var rLength uint32 126 | var multiplier uint32 127 | b := make([]byte, 1) 128 | for multiplier < 27 { //fix: Infinite '(digit & 128) == 1' will cause the dead loop 129 | _, err := io.ReadFull(r, b) 130 | if err != nil { 131 | return 0, err 132 | } 133 | 134 | digit := b[0] 135 | rLength |= uint32(digit&127) << multiplier 136 | if (digit & 128) == 0 { 137 | break 138 | } 139 | multiplier += 7 140 | } 141 | return int(rLength), nil 142 | } 143 | -------------------------------------------------------------------------------- /server/utp/publish.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package utp 18 | 19 | import ( 20 | "bytes" 21 | 22 | "github.com/golang/protobuf/proto" 23 | pbx "github.com/unit-io/unitdb/server/proto" 24 | ) 25 | 26 | // PublishMessage reprensents a publish Message 27 | type PublishMessage struct { 28 | Topic string 29 | Payload []byte 30 | Ttl string 31 | } 32 | 33 | // Publish represents a publish Messages. 34 | type Publish struct { 35 | IsForwarded bool 36 | MessageID uint16 37 | DeliveryMode uint8 38 | Messages []*PublishMessage 39 | } 40 | 41 | func (p *Publish) ToBinary() (bytes.Buffer, error) { 42 | var buf bytes.Buffer 43 | 44 | var protoMessages []*pbx.PublishMessage 45 | for _, pubMsg := range p.Messages { 46 | protoMsg := &pbx.PublishMessage{ 47 | Topic: string(pubMsg.Topic), 48 | Payload: pubMsg.Payload, 49 | Ttl: pubMsg.Ttl, 50 | } 51 | protoMessages = append(protoMessages, protoMsg) 52 | } 53 | pub := pbx.Publish{ 54 | MessageID: int32(p.MessageID), 55 | DeliveryMode: int32(p.DeliveryMode), 56 | Messages: protoMessages, 57 | } 58 | rawMsg, err := proto.Marshal(&pub) 59 | if err != nil { 60 | return buf, err 61 | } 62 | fh := FixedHeader{MessageType: PUBLISH, MessageLength: len(rawMsg)} 63 | buf = fh.pack() 64 | _, err = buf.Write(rawMsg) 65 | return buf, err 66 | } 67 | 68 | func (p *Publish) FromBinary(fh FixedHeader, data []byte) { 69 | var pub pbx.Publish 70 | proto.Unmarshal(data, &pub) 71 | var pubMessages []*PublishMessage 72 | for _, protoMsg := range pub.Messages { 73 | pubMsg := &PublishMessage{ 74 | Topic: protoMsg.Topic, 75 | Payload: protoMsg.Payload, 76 | Ttl: protoMsg.Ttl, 77 | } 78 | pubMessages = append(pubMessages, pubMsg) 79 | } 80 | p.MessageID = uint16(pub.MessageID) 81 | p.DeliveryMode = uint8(pub.DeliveryMode) 82 | p.Messages = pubMessages 83 | } 84 | 85 | // Type returns the Message type. 86 | func (p *Publish) Type() MessageType { 87 | return PUBLISH 88 | } 89 | 90 | // Info returns DeliveryMode and MessageID of this Message. 91 | func (p *Publish) Info() Info { 92 | return Info{DeliveryMode: p.DeliveryMode, MessageID: p.MessageID} 93 | } 94 | -------------------------------------------------------------------------------- /server/utp/relay.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package utp 18 | 19 | import ( 20 | // "bytes" 21 | 22 | "bytes" 23 | 24 | "github.com/golang/protobuf/proto" 25 | pbx "github.com/unit-io/unitdb/server/proto" 26 | ) 27 | 28 | // RelayRequest is pairing the Topic and Last parameter together 29 | type RelayRequest struct { 30 | Topic string 31 | Last string 32 | } 33 | 34 | // Relay tells the server which topics and last durations the client would like get data. The Delivery Mode for relay is EXPRESS. 35 | type Relay struct { 36 | IsForwarded bool 37 | MessageID uint16 38 | RelayRequests []*RelayRequest 39 | } 40 | 41 | func (r *Relay) ToBinary() (bytes.Buffer, error) { 42 | var buf bytes.Buffer 43 | var reqs []*pbx.RelayRequest 44 | for _, req := range r.RelayRequests { 45 | r := pbx.RelayRequest{ 46 | Topic: string(req.Topic), 47 | Last: req.Last, 48 | } 49 | reqs = append(reqs, &r) 50 | } 51 | rel := pbx.Relay{ 52 | MessageID: int32(r.MessageID), 53 | RelayRequests: reqs, 54 | } 55 | rawMsg, err := proto.Marshal(&rel) 56 | if err != nil { 57 | return buf, err 58 | } 59 | fh := FixedHeader{MessageType: RELAY, MessageLength: len(rawMsg)} 60 | buf = fh.pack() 61 | _, err = buf.Write(rawMsg) 62 | return buf, err 63 | } 64 | 65 | func (r *Relay) FromBinary(fh FixedHeader, data []byte) { 66 | var rel pbx.Relay 67 | proto.Unmarshal(data, &rel) 68 | var reqs []*RelayRequest 69 | for _, req := range rel.RelayRequests { 70 | r := &RelayRequest{ 71 | Topic: req.Topic, 72 | Last: req.Last, 73 | } 74 | reqs = append(reqs, r) 75 | } 76 | 77 | r.MessageID = uint16(rel.MessageID) 78 | r.RelayRequests = reqs 79 | } 80 | 81 | // Type returns the Message type. 82 | func (r *Relay) Type() MessageType { 83 | return RELAY 84 | } 85 | 86 | // Info returns DeliveryMode and MessageID of this Message. 87 | func (r *Relay) Info() Info { 88 | return Info{DeliveryMode: 1, MessageID: r.MessageID} 89 | } 90 | -------------------------------------------------------------------------------- /server/utp/subscribe.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package utp 18 | 19 | import ( 20 | // "bytes" 21 | 22 | "bytes" 23 | 24 | "github.com/golang/protobuf/proto" 25 | pbx "github.com/unit-io/unitdb/server/proto" 26 | ) 27 | 28 | // Subscription is a struct for pairing the delivery mode and topic together 29 | // for the delivery mode's pairs in unsubscribe and subscribe 30 | type Subscription struct { 31 | DeliveryMode uint8 32 | Delay int32 33 | Topic string 34 | } 35 | 36 | // Subscribe tells the server which topics the client would like to subscribe to 37 | type Subscribe struct { 38 | IsForwarded bool 39 | MessageID uint16 40 | Subscriptions []*Subscription 41 | } 42 | 43 | // Unsubscribe is the Message to send if you don't want to subscribe to a topic anymore 44 | type Unsubscribe struct { 45 | IsForwarded bool 46 | MessageID uint16 47 | Subscriptions []*Subscription 48 | } 49 | 50 | func (s *Subscribe) ToBinary() (bytes.Buffer, error) { 51 | var buf bytes.Buffer 52 | var subs []*pbx.Subscription 53 | for _, subscription := range s.Subscriptions { 54 | sub := pbx.Subscription{ 55 | DeliveryMode: int32(subscription.DeliveryMode), 56 | Delay: subscription.Delay, 57 | Topic: string(subscription.Topic), 58 | } 59 | subs = append(subs, &sub) 60 | } 61 | sub := pbx.Subscribe{ 62 | MessageID: int32(s.MessageID), 63 | Subscriptions: subs, 64 | } 65 | rawMsg, err := proto.Marshal(&sub) 66 | if err != nil { 67 | return buf, err 68 | } 69 | fh := FixedHeader{MessageType: SUBSCRIBE, MessageLength: len(rawMsg)} 70 | buf = fh.pack() 71 | _, err = buf.Write(rawMsg) 72 | return buf, err 73 | } 74 | 75 | func (s *Subscribe) FromBinary(fh FixedHeader, data []byte) { 76 | var sub pbx.Subscribe 77 | proto.Unmarshal(data, &sub) 78 | var subs []*Subscription 79 | for _, subscription := range sub.Subscriptions { 80 | sub := &Subscription{ 81 | DeliveryMode: uint8(subscription.DeliveryMode), 82 | Delay: subscription.Delay, 83 | Topic: subscription.Topic, 84 | } 85 | subs = append(subs, sub) 86 | } 87 | 88 | s.MessageID = uint16(sub.MessageID) 89 | s.Subscriptions = subs 90 | } 91 | 92 | // Type returns the Message type. 93 | func (s *Subscribe) Type() MessageType { 94 | return SUBSCRIBE 95 | } 96 | 97 | // Info returns DeliveryMode and MessageID of this Message. 98 | func (s *Subscribe) Info() Info { 99 | return Info{DeliveryMode: 1, MessageID: s.MessageID} 100 | } 101 | 102 | func (u *Unsubscribe) ToBinary() (bytes.Buffer, error) { 103 | var buf bytes.Buffer 104 | var subs []*pbx.Subscription 105 | for _, subscription := range u.Subscriptions { 106 | sub := pbx.Subscription{ 107 | DeliveryMode: int32(subscription.DeliveryMode), 108 | Delay: subscription.Delay, 109 | Topic: string(subscription.Topic), 110 | } 111 | subs = append(subs, &sub) 112 | } 113 | unsub := pbx.Unsubscribe{ 114 | MessageID: int32(u.MessageID), 115 | Subscriptions: subs, 116 | } 117 | rawMsg, err := proto.Marshal(&unsub) 118 | if err != nil { 119 | return buf, err 120 | } 121 | fh := FixedHeader{MessageType: UNSUBSCRIBE, MessageLength: len(rawMsg)} 122 | buf = fh.pack() 123 | _, err = buf.Write(rawMsg) 124 | return buf, err 125 | } 126 | 127 | func (u *Unsubscribe) FromBinary(fh FixedHeader, data []byte) { 128 | var unsub pbx.Unsubscribe 129 | proto.Unmarshal(data, &unsub) 130 | var subs []*Subscription 131 | for _, subscription := range unsub.Subscriptions { 132 | sub := &Subscription{ 133 | DeliveryMode: uint8(subscription.DeliveryMode), 134 | Delay: subscription.Delay, 135 | Topic: subscription.Topic, 136 | } 137 | subs = append(subs, sub) 138 | } 139 | 140 | u.MessageID = uint16(unsub.MessageID) 141 | u.Subscriptions = subs 142 | } 143 | 144 | // Type returns the Message type. 145 | func (u *Unsubscribe) Type() MessageType { 146 | return UNSUBSCRIBE 147 | } 148 | 149 | // Info returns DeliveryMode and MessageID of this Message. 150 | func (u *Unsubscribe) Info() Info { 151 | return Info{DeliveryMode: 1, MessageID: u.MessageID} 152 | } 153 | -------------------------------------------------------------------------------- /time_window_reader.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package unitdb 18 | 19 | import ( 20 | "io" 21 | ) 22 | 23 | type _WindowReader struct { 24 | winBlock _WinBlock 25 | windowIdx int32 26 | fs *_FileSet 27 | winFile *_File 28 | offset int64 29 | } 30 | 31 | func newWindowReader(fs *_FileSet) *_WindowReader { 32 | w := &_WindowReader{windowIdx: -1, fs: fs} 33 | winFile, err := fs.getFile(_FileDesc{fileType: typeTimeWindow}) 34 | if err != nil { 35 | return w 36 | } 37 | w.winFile = winFile 38 | 39 | if winFile.currSize() > 0 { 40 | w.windowIdx = int32(winFile.currSize() / int64(blockSize)) 41 | } 42 | return w 43 | } 44 | 45 | func (r *_WindowReader) readWindowBlock() (_WinBlock, error) { 46 | buf, err := r.winFile.slice(r.offset, r.offset+int64(blockSize)) 47 | if err != nil { 48 | return _WinBlock{}, err 49 | } 50 | if err := r.winBlock.unmarshalBinary(buf); err != nil { 51 | return _WinBlock{}, err 52 | } 53 | 54 | return r.winBlock, nil 55 | } 56 | 57 | // blockIterator iterates all window blocks from disk. 58 | func (r *_WindowReader) blockIterator(f func(startSeq, topicHash uint64, off int64) (bool, error)) (err error) { 59 | windowIdx := int32(0) 60 | nBlocks := r.windowIdx 61 | for windowIdx <= nBlocks { 62 | r.offset = winBlockOffset(windowIdx) 63 | b, err := r.readWindowBlock() 64 | if err != nil { 65 | if err == io.EOF { 66 | return nil 67 | } 68 | return err 69 | } 70 | windowIdx++ 71 | if b.entryIdx == 0 || b.next != 0 { 72 | continue 73 | } 74 | // fmt.Println("timeWindow.blockIterator: topicHash, seq ", b.topicHash, b.entries[0].sequence) 75 | if stop, err := f(b.entries[0].sequence, b.topicHash, r.offset); stop || err != nil { 76 | return err 77 | } 78 | } 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /uid/uid.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package uid 18 | 19 | import ( 20 | "encoding/binary" 21 | "math" 22 | "math/rand" 23 | "sync/atomic" 24 | "time" 25 | ) 26 | 27 | const ( 28 | // Offset is used to create new apoch from current time. 29 | Offset = 1555770000 30 | ) 31 | 32 | var ( 33 | // Next is the next identifier. It is time in seconds 34 | // to avoid collisions of ids between process restarts. 35 | Next = uint32( 36 | time.Date(2070, 1, 1, 0, 0, 0, 0, time.UTC).Sub(time.Now()).Seconds(), 37 | ) 38 | ) 39 | 40 | // LID represents a process-wide unique ID. 41 | type LID uint64 42 | 43 | // NewApoch creates an appoch to generate unique id. 44 | func NewApoch() uint32 { 45 | now := uint32(time.Now().Unix() - Offset) 46 | return math.MaxUint32 - now 47 | } 48 | 49 | // NewUnique return unique value to use generating unique id. 50 | func NewUnique() uint32 { 51 | b := make([]byte, 4) 52 | random := rand.New(rand.NewSource(int64(NewApoch()))) 53 | random.Read(b) 54 | u := uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(0) 55 | return u 56 | } 57 | 58 | // Time returns time portion of ID. 59 | func Time(id []byte) int64 { 60 | return int64(math.MaxUint32-binary.LittleEndian.Uint32(id)) + Offset 61 | } 62 | 63 | // NewLID generates a new, process-wide unique ID. 64 | func NewLID() LID { 65 | return LID(atomic.AddUint32(&Next, 1)) 66 | } 67 | -------------------------------------------------------------------------------- /wal/header.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package wal 18 | 19 | import ( 20 | "encoding/binary" 21 | ) 22 | 23 | var ( 24 | logHeaderSize = 18 25 | ) 26 | 27 | type _LogInfo struct { 28 | version uint16 29 | timeID int64 30 | count uint32 31 | size uint32 32 | 33 | _ [28]byte 34 | } 35 | 36 | // MarshalBinary serialized logInfo into binary data. 37 | func (l _LogInfo) MarshalBinary() ([]byte, error) { 38 | buf := make([]byte, logHeaderSize) 39 | binary.LittleEndian.PutUint16(buf[:2], l.version) 40 | binary.LittleEndian.PutUint64(buf[2:10], uint64(l.timeID)) 41 | binary.LittleEndian.PutUint32(buf[10:14], l.count) 42 | binary.LittleEndian.PutUint32(buf[14:18], l.size) 43 | 44 | return buf, nil 45 | } 46 | 47 | // UnmarshalBinary deserialized logInfo from binary data. 48 | func (l *_LogInfo) UnmarshalBinary(data []byte) error { 49 | l.version = binary.LittleEndian.Uint16(data[:2]) 50 | l.timeID = int64(binary.LittleEndian.Uint64(data[2:10])) 51 | l.count = binary.LittleEndian.Uint32(data[10:14]) 52 | l.size = binary.LittleEndian.Uint32(data[14:18]) 53 | 54 | return nil 55 | } 56 | -------------------------------------------------------------------------------- /wal/reader.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package wal 18 | 19 | import ( 20 | "encoding/binary" 21 | "errors" 22 | 23 | "github.com/unit-io/bpool" 24 | "github.com/unit-io/unitdb/uid" 25 | ) 26 | 27 | // Reader reads logs from WAL. 28 | // Reader reader is a simple iterator over log data. 29 | type Reader struct { 30 | Id uid.LID 31 | offset int64 32 | entryCount uint32 33 | buffer *bpool.Buffer 34 | 35 | wal *WAL 36 | } 37 | 38 | // NewReader returns new log reader to read the logs from the WAL. 39 | func (wal *WAL) NewReader() (*Reader, error) { 40 | if err := wal.ok(); err != nil { 41 | return &Reader{wal: wal}, err 42 | } 43 | r := &Reader{ 44 | Id: uid.NewLID(), 45 | wal: wal, 46 | } 47 | 48 | return r, nil 49 | } 50 | 51 | // Iterator iterates the pending logs from the WAL. 52 | func (r *Reader) Iterator(f func(timeID int64) (bool, error)) (err error) { 53 | r.wal.mu.RLock() 54 | r.buffer = r.wal.bufPool.Get() 55 | defer func() { 56 | r.wal.recoveredTimeIDs = r.wal.recoveredTimeIDs[:0] 57 | r.wal.bufPool.Put(r.buffer) 58 | r.wal.mu.RUnlock() 59 | }() 60 | 61 | for _, timeID := range r.wal.recoveredTimeIDs { 62 | r.offset = 0 63 | r.buffer.Reset() 64 | info := r.wal.logStore.read(timeID, r.buffer) 65 | r.entryCount = info.count 66 | if stop, err := f(timeID); stop || err != nil { 67 | return err 68 | } 69 | } 70 | 71 | return nil 72 | } 73 | 74 | // Count returns entry count for the current interation. 75 | func (r *Reader) Count() uint32 { 76 | return r.entryCount 77 | } 78 | 79 | // Next returns next record from the iterator or false if iteration is done. 80 | func (r *Reader) Next() ([]byte, bool, error) { 81 | if r.entryCount == 0 { 82 | return nil, false, nil 83 | } 84 | r.entryCount-- 85 | scratch, _ := r.buffer.Slice(r.offset, r.offset+4) 86 | dataLen := binary.LittleEndian.Uint32(scratch) 87 | data, err := r.buffer.Slice(r.offset+4, r.offset+int64(dataLen)) 88 | if err != nil { 89 | return nil, false, errors.New("error reading log") 90 | } 91 | r.offset += int64(dataLen) 92 | return data, true, nil 93 | } 94 | -------------------------------------------------------------------------------- /wal/wal.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package wal 18 | 19 | import ( 20 | "errors" 21 | "sync" 22 | "sync/atomic" 23 | 24 | "github.com/unit-io/bpool" 25 | ) 26 | 27 | const ( 28 | version = 1 // file format version 29 | 30 | logExt = ".log" 31 | tmpExt = ".tmp" 32 | corruptExt = ".CORRUPT" 33 | ) 34 | 35 | type ( 36 | // WALInfo provides WAL stats. 37 | WALInfo struct { 38 | logCountWritten int64 39 | logCountApplied int64 40 | entriesWritten int64 41 | entriesApplied int64 42 | } 43 | // WAL write ahead logs to recover db commit failure dues to db crash or other unexpected errors. 44 | WAL struct { 45 | // wg is a WaitGroup that allows us to wait for the syncThread to finish to 46 | // ensure a clean shutdown. 47 | wg sync.WaitGroup 48 | mu sync.RWMutex 49 | 50 | WALInfo 51 | recoveredTimeIDs []int64 52 | 53 | bufPool *bpool.BufferPool 54 | logStore *_FileStore 55 | 56 | opts Options 57 | 58 | // close 59 | closed uint32 60 | } 61 | 62 | // Options wal options to create new WAL. WAL logs uses cyclic rotation to avoid fragmentation. 63 | // It allocates free blocks only when log reaches target size. 64 | Options struct { 65 | Path string 66 | BufferSize int64 67 | Reset bool 68 | } 69 | ) 70 | 71 | func newWal(opts Options) (wal *WAL, err error) { 72 | wal = &WAL{ 73 | bufPool: bpool.NewBufferPool(opts.BufferSize, nil), 74 | opts: opts, 75 | } 76 | wal.logStore, err = openFile(opts.Path, opts.BufferSize) 77 | if err != nil { 78 | return wal, err 79 | } 80 | 81 | if opts.Reset { 82 | wal.logStore.reset() 83 | return wal, nil 84 | } 85 | 86 | wal.recoverWal() 87 | 88 | return wal, nil 89 | } 90 | 91 | // recoverWal recovers a WAL for the log written but not released. It also updates free blocks. 92 | func (wal *WAL) recoverWal() { 93 | wal.recoveredTimeIDs = wal.logStore.all() 94 | } 95 | 96 | // Close closes the wal, frees used resources and checks for active 97 | // logs. 98 | func (wal *WAL) Close() error { 99 | if !wal.setClosed() { 100 | return errors.New("wal is closed") 101 | } 102 | 103 | // Make sure sync thread isn't running. 104 | wal.wg.Wait() 105 | 106 | // fmt.Println("wal.close: WALInfo ", wal.WALInfo) 107 | wal.logStore.close() 108 | 109 | return nil 110 | } 111 | 112 | func (wal *WAL) put(log _LogInfo, data *bpool.Buffer) error { 113 | log.version = version 114 | wal.logCountWritten++ 115 | wal.entriesWritten += int64(log.count) 116 | 117 | return wal.logStore.put(log, data) 118 | } 119 | 120 | // SignalLogApplied informs the WAL that it is safe to reuse blocks. 121 | func (wal *WAL) SignalLogApplied(timeID int64) error { 122 | wal.mu.RLock() 123 | wal.wg.Add(1) 124 | defer func() { 125 | wal.wg.Done() 126 | wal.mu.RUnlock() 127 | }() 128 | 129 | wal.logCountApplied++ 130 | wal.logStore.del(timeID) 131 | 132 | return nil 133 | } 134 | 135 | // Reset removes all persistested logs from log store. 136 | func (wal *WAL) Reset() { 137 | wal.logStore.reset() 138 | } 139 | 140 | // setClosed flag; return true if not already closed. 141 | func (wal *WAL) setClosed() bool { 142 | if wal == nil { 143 | return false 144 | } 145 | return atomic.CompareAndSwapUint32(&wal.closed, 0, 1) 146 | } 147 | 148 | // isClosed Checks whether WAL was closed. 149 | func (wal *WAL) isClosed() bool { 150 | return atomic.LoadUint32(&wal.closed) != 0 151 | } 152 | 153 | // Ok checks read ok status. 154 | func (wal *WAL) ok() error { 155 | if wal.isClosed() { 156 | return errors.New("wal is closed.") 157 | } 158 | return nil 159 | } 160 | 161 | // New will open a WAL. If the previous run did not shut down cleanly, a set of 162 | // log entries will be returned which got committed successfully to the WAL, but 163 | // were never signaled as fully completed. 164 | // 165 | // If no WAL exists, a new one will be created. 166 | // 167 | func New(opts Options) (*WAL, error) { 168 | // Create a wal 169 | return newWal(opts) 170 | } 171 | -------------------------------------------------------------------------------- /wal/wal_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package wal 18 | 19 | import ( 20 | "errors" 21 | "fmt" 22 | "os" 23 | "testing" 24 | ) 25 | 26 | var ( 27 | dbPath = "test" 28 | logDir = "logs" 29 | ) 30 | 31 | func newTestWal(del bool) (*WAL, error) { 32 | logOpts := Options{Path: dbPath + "/" + logDir, BufferSize: 1 << 8} 33 | if del { 34 | os.RemoveAll(dbPath) 35 | } 36 | // Make sure we have a directory. 37 | if err := os.MkdirAll(dbPath, 0777); err != nil { 38 | return nil, errors.New("newTestWal, Unable to create dir") 39 | } 40 | return New(logOpts) 41 | } 42 | 43 | func TestEmptyLog(t *testing.T) { 44 | wal, err := newTestWal(true) 45 | if len(wal.recoveredTimeIDs) != 0 || err != nil { 46 | t.Fatal(err) 47 | } 48 | defer wal.Close() 49 | } 50 | 51 | func TestRecovery(t *testing.T) { 52 | wal, err := newTestWal(true) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | defer wal.Close() 57 | 58 | if len(wal.recoveredTimeIDs) != 0 { 59 | t.Fatalf("Write ahead log non-empty") 60 | } 61 | 62 | var i uint16 63 | var n uint16 = 1000 64 | 65 | logWriter, err := wal.NewWriter() 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | 70 | for i = 0; i < n; i++ { 71 | val := []byte(fmt.Sprintf("msg.%2d", i)) 72 | if err := <-logWriter.Append(val); err != nil { 73 | t.Fatal(err) 74 | } 75 | } 76 | 77 | if err := <-logWriter.SignalInitWrite(int64(n)); err != nil { 78 | t.Fatal(err) 79 | } 80 | 81 | if err := wal.Close(); err != nil { 82 | t.Fatal(err) 83 | } 84 | 85 | wal, err = newTestWal(false) 86 | if len(wal.recoveredTimeIDs) == 0 || err != nil { 87 | t.Fatal(err) 88 | } 89 | } 90 | 91 | func TestLogApplied(t *testing.T) { 92 | wal, err := newTestWal(true) 93 | if err != nil { 94 | t.Fatal(err) 95 | } 96 | defer wal.Close() 97 | var i uint16 98 | var n uint16 = 1000 99 | 100 | logWriter, err := wal.NewWriter() 101 | if err != nil { 102 | t.Fatal(err) 103 | } 104 | 105 | for i = 0; i < n; i++ { 106 | val := []byte(fmt.Sprintf("msg.%2d", i)) 107 | if err := <-logWriter.Append(val); err != nil { 108 | t.Fatal(err) 109 | } 110 | } 111 | 112 | if err := <-logWriter.SignalInitWrite(int64(n)); err != nil { 113 | t.Fatal(err) 114 | } 115 | 116 | if err := wal.Close(); err != nil { 117 | t.Fatal(err) 118 | } 119 | wal, err = newTestWal(false) 120 | if len(wal.recoveredTimeIDs) == 0 || err != nil { 121 | t.Fatal(err) 122 | } 123 | 124 | r, err := wal.NewReader() 125 | if err != nil { 126 | t.Fatal(err) 127 | } 128 | err = r.Iterator(func(timeID int64) (bool, error) { 129 | for { 130 | _, ok, err := r.Next() 131 | if !ok || err != nil { 132 | break 133 | } 134 | } 135 | return false, nil 136 | }) 137 | if err != nil { 138 | t.Fatal(err) 139 | } 140 | 141 | if err := wal.Close(); err != nil { 142 | t.Fatal(err) 143 | } 144 | } 145 | 146 | func TestSimple(t *testing.T) { 147 | wal, err := newTestWal(true) 148 | if err != nil { 149 | t.Fatal(err) 150 | } 151 | defer wal.Close() 152 | 153 | var i uint16 154 | var n uint16 = 1000 155 | 156 | logWriter, err := wal.NewWriter() 157 | if err != nil { 158 | t.Fatal(err) 159 | } 160 | 161 | for i = 0; i < n; i++ { 162 | val := []byte(fmt.Sprintf("msg.%2d", i)) 163 | if err := <-logWriter.Append(val); err != nil { 164 | t.Fatal(err) 165 | } 166 | } 167 | 168 | if err := <-logWriter.SignalInitWrite(int64(n)); err != nil { 169 | t.Fatal(err) 170 | } 171 | 172 | if err := wal.SignalLogApplied(int64(n)); err != nil { 173 | t.Fatal(err) 174 | } 175 | 176 | } 177 | -------------------------------------------------------------------------------- /wal/writer.go: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Saffat Technologies, Ltd. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package wal 18 | 19 | import ( 20 | "encoding/binary" 21 | "errors" 22 | 23 | "github.com/unit-io/bpool" 24 | "github.com/unit-io/unitdb/uid" 25 | ) 26 | 27 | // Writer writes entries to the write ahead log. 28 | // Thread-safe. 29 | type Writer struct { 30 | Id uid.LID 31 | writeComplete bool 32 | releaseComplete bool 33 | 34 | count uint32 35 | 36 | buffer *bpool.Buffer 37 | logSize uint32 38 | 39 | wal *WAL 40 | 41 | // writeCompleted is used to signal if log is fully written. 42 | writeCompleted chan struct{} 43 | } 44 | 45 | // NewWriter returns new log writer to write to WAL. 46 | func (wal *WAL) NewWriter() (*Writer, error) { 47 | if err := wal.ok(); err != nil { 48 | return &Writer{wal: wal}, err 49 | } 50 | w := &Writer{ 51 | Id: uid.NewLID(), 52 | wal: wal, 53 | writeCompleted: make(chan struct{}, 1), 54 | } 55 | 56 | w.buffer = wal.bufPool.Get() 57 | return w, nil 58 | } 59 | 60 | func (w *Writer) append(data []byte) error { 61 | if len(data) == 0 { 62 | return nil 63 | } 64 | 65 | w.count++ 66 | 67 | var scratch [4]byte 68 | dataLen := uint32(len(data) + 4) 69 | binary.LittleEndian.PutUint32(scratch[0:4], dataLen) 70 | 71 | if _, err := w.buffer.Write(scratch[:]); err != nil { 72 | return err 73 | } 74 | w.logSize += dataLen 75 | 76 | if _, err := w.buffer.Write(data); err != nil { 77 | return err 78 | } 79 | 80 | return nil 81 | } 82 | 83 | // Append appends records to the WAL. 84 | func (w *Writer) Append(data []byte) <-chan error { 85 | done := make(chan error, 1) 86 | 87 | if w.writeComplete || w.releaseComplete { 88 | done <- errors.New("logWriter error - can't append to log once it is written/released") 89 | return done 90 | } 91 | go func() { 92 | done <- w.append(data) 93 | }() 94 | return done 95 | } 96 | 97 | // writeLog writes log by setting correct header and status. 98 | func (w *Writer) writeLog(timeID int64) error { 99 | w.writeCompleted <- struct{}{} 100 | w.wal.mu.Lock() 101 | defer func() { 102 | w.wal.bufPool.Put(w.buffer) 103 | w.wal.wg.Done() 104 | w.wal.mu.Unlock() 105 | <-w.writeCompleted 106 | }() 107 | 108 | if w.logSize == 0 { 109 | return nil 110 | } 111 | dataLen := w.logSize 112 | info := _LogInfo{ 113 | timeID: timeID, 114 | count: w.count, 115 | size: dataLen, 116 | } 117 | if err := w.wal.put(info, w.buffer); err != nil { 118 | return err 119 | } 120 | 121 | w.writeComplete = true 122 | 123 | return nil 124 | } 125 | 126 | // SignalInitWrite will signal to the WAL that log append has 127 | // completed, and that the WAL can safely write log and being 128 | // applied atomically. 129 | func (w *Writer) SignalInitWrite(timeID int64) <-chan error { 130 | done := make(chan error, 1) 131 | if w.writeComplete || w.releaseComplete { 132 | done <- errors.New("misuse of log write - call each of the signaling methods exactly ones, in serial, in order") 133 | return done 134 | } 135 | 136 | // Write the log non-blocking. 137 | w.wal.wg.Add(1) 138 | go func() { 139 | done <- w.writeLog(timeID) 140 | }() 141 | return done 142 | } 143 | --------------------------------------------------------------------------------