├── .gitattributes ├── Dockerfile ├── LICENSE ├── README.md ├── clients ├── go │ └── README.md ├── java │ ├── .gitignore │ ├── README.txt │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── nl │ │ │ └── prymr │ │ │ └── rochefort │ │ │ ├── Client.java │ │ │ ├── Proto.java │ │ │ └── Util.java │ │ └── test │ │ └── java │ │ └── nl │ │ └── prymr │ │ └── rochefort │ │ └── ClientTest.java ├── js │ ├── .gitignore │ ├── README.txt │ ├── index.js │ ├── package.json │ └── test.js └── ruby │ ├── README.txt │ ├── Rakefile │ ├── lib │ └── rochefort.rb │ ├── rochefort.gemspec │ └── test │ └── test_rochefort.rb ├── compact_test.go ├── input.pb ├── input.pb.go ├── input.proto ├── main.go ├── query.go ├── query_test.go └── test_everything.sh /.gitattributes: -------------------------------------------------------------------------------- 1 | *.java linguist-detectable=false 2 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.9 2 | 3 | ARG CACHEBUST=1 4 | 5 | RUN git clone https://github.com/jackdoe/rochefort clone && cd clone && git checkout tags/v2.5 && go get github.com/gogo/protobuf/proto && go get github.com/dgryski/go-metro && go build -o rochefort . && mv rochefort /usr/bin/rochefort && cd - && rm -rf clone /go 6 | 7 | CMD /usr/bin/rochefort -root ${ROOT:-/tmp/rochefort_files} -bind ${BIND:-:8000} $ROCHEFORT_OPTS 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Borislav Nikolov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rochefort - poor man's kafka (with in-place modifications and inverted index search) 2 | 3 | ### PUSH DATA, GET FILE OFFSET; no shenanigans 4 | #### (if you can afford to lose data and do your own replication) 5 | 6 | * **disk write speed** storage service that returns offsets to stored values 7 | * if you are ok with losing some data (does not fsync on write) 8 | * supports: **append, multiappend, modify, get, multiget, close, query, compact** 9 | * clients: [go](https://github.com/jackdoe/go-rochefort-client), [java](https://github.com/jackdoe/rochefort/tree/master/clients/java) 10 | 11 | ## turns out when you are fine with losing some data, things are much faster and simpler :) 12 | 13 | --- 14 | 15 | ## run in docker 16 | 17 | run with docker: jackdoe/rochefort:2.5 18 | 19 | ``` 20 | docker run -e BIND=":8000" \ 21 | -e ROOT="/tmp/rochefort" \ 22 | -p 8000:8000 \ 23 | jackdoe/rochefort:2.5 24 | ``` 25 | 26 | ### breaking change between 0.5 and 1.0 27 | 28 | * added 4 more bytes in the header 29 | * the -buckets parameter is gone, so everything is appended in one file per namespace 30 | 31 | you can migrate your data by doing: 32 | 33 | ``` 34 | oldServer.scan(namespace: ns) do |offset, v| 35 | newServer.append(namespace:ns, data: v) 36 | end 37 | ``` 38 | 39 | ### breaking change between 1.x and 2.0 40 | 41 | * moved get/multiget/append to protobuf 42 | * moved delete/close to protobuf 43 | 44 | 45 | ### parameters 46 | * root: root directory, files will be created at `root/namespace||default/append.raw` 47 | * bind: address to bind to (default :8000) 48 | 49 | dont forget to mount persisted root directory 50 | 51 | 52 | ## compile from source 53 | 54 | ``` 55 | $ go run main.go query.go input.pb.go -bind :8000 -root /tmp 56 | 2018/02/10 12:06:21 starting http server on :8000 57 | .... 58 | 59 | ``` 60 | 61 | ## APPEND/MULTI APPEND 62 | 63 | ``` 64 | res, err := r.Set(&AppendInput{ 65 | AppendPayload: []*Append{{ 66 | Namespace: ns, 67 | Data: []byte("abc"), 68 | AllocSize: 10, // so you can do inplace modification 69 | Tags: []string{"a","b","c"} // so you can search it 70 | }, { 71 | Namespace: ns, 72 | Data: []byte("zxc"), 73 | }}, 74 | }) 75 | ``` 76 | 77 | 78 | you can always do inplace modifications to an object, and you can also reserve some space to add more stuff to the same offset later 79 | 80 | the searchable tags are sanitized as all non alphanumeric characters(excluding _) `[^a-zA-Z0-9_]+` are removed 81 | 82 | ### inverted index 83 | passing tags a,b,c will create postings lists in the namespace 84 | a.postings, b.postings and c.postings, later you can query only specific tags with /query 85 | 86 | ## MODIFY 87 | 88 | ``` 89 | _, err = r.Set(&AppendInput{ 90 | ModifyPayload: []*Modify{{ 91 | Namespace: ns, 92 | Offset: off, 93 | Pos: 1, 94 | Data: []byte("zxcv"), 95 | }}, 96 | }) 97 | ``` 98 | 99 | inplace modifies position, for example if we want to replace 'abc' 100 | with 'szze' in the blob we appended at offset 0, we modify rochefort 101 | offset 0 with 'zz' from position 1 102 | If you pass Pos: -1 it will append to the previous end of the blob 103 | 104 | in AppendInput you can mix modify and append commands 105 | 106 | ## GET/MULTI GET 107 | 108 | ``` 109 | fetched, err := r.Get(&GetInput{ 110 | GetPayload: []*Get{{ 111 | Namespace: "example", 112 | Offset: offset1, 113 | }, { 114 | Namespace: "example, 115 | Offset: offset12, 116 | }}, 117 | }) 118 | 119 | ``` 120 | 121 | output is GetOutput which is just array of arrays of byte, so fetched[0] is array of bytes holding the first blob and fetched[1] is the second blob 122 | 123 | ## NAMESPACE 124 | you can also pass "namespace" parameter and this will create different directories per namespace, for example 125 | 126 | ``` 127 | namespace: events_from_20171111 128 | namespace: events_from_20171112 129 | ``` 130 | will crete {root_directory}/events_from_20171111/... and {root_directory}/events_from_20171112/... 131 | 132 | and then you simply delete the directories you don't need (after closing them) 133 | 134 | ## CLOSE/DELETE 135 | Closes a namespace so it can be deleted (or you can directly delete it with DELETE) 136 | 137 | ## STORAGE FORMAT 138 | 139 | ``` 140 | header is 16 bytes 141 | D: data length: 4 bytes 142 | R: reserved: 8 bytes 143 | A: allocSize: 4 bytes 144 | C: crc32(length, time): 4 bytes 145 | V: the stored value 146 | 147 | DDDDRRRRRRRRAAAACCCCVVVVVVVVVVVVVVVVVVVV...DDDDRRRRRRRRAAAACCCCVVVVVV.... 148 | 149 | ``` 150 | 151 | as you can see the value is not included in the checksum, I am 152 | checking only the header as my usecase is quite ok with 153 | missing/corrupting the data itself, but it is not ok if corrupted 154 | header makes us allocate 10gb in `output := make([]byte, dataLen)` 155 | 156 | 157 | ## SCAN 158 | 159 | scans the file 160 | 161 | ``` 162 | $ curl http://localhost:8000/scan?namespace=someStoragePrefix > dump.txt 163 | ``` 164 | 165 | the format is 166 | [len 4 bytes(little endian)][offset 8 bytes little endian)]data...[len][offset]data 167 | 168 | ## SEARCH 169 | 170 | you can search all tagged blobs, the dsl is fairly simple, post/get json blob to /query 171 | 172 | * basic tag query 173 | 174 | ``` 175 | {"tag":"xyz"} 176 | ``` 177 | 178 | * basic OR query 179 | 180 | ``` 181 | {"or": [... subqueries ...]} 182 | ``` 183 | 184 | 185 | * basic AND query 186 | 187 | ``` 188 | {"and": [... subqueries ...]} 189 | ``` 190 | 191 | 192 | example: 193 | 194 | ``` 195 | curl -XGET -d '{"and":[{"tag":"c"},{"or":[{"tag":"b"},{"tag":"c"}]}]}' 'http://localhost:8000/query' 196 | ``` 197 | 198 | it spits out the output in same format as /scan, so the result of the query can be very big 199 | but it is streamed 200 | 201 | 202 | ## LICENSE 203 | 204 | MIT 205 | 206 | 207 | ## naming rochefort 208 | 209 | [Rochefort Trappistes 10](https://www.ratebeer.com/beer/rochefort-trappistes-10/2360/) is my favorite beer and I was drinking it 210 | while doing the initial implementation at sunday night 211 | 212 | 213 | ## losing data + NIH 214 | You can lose data on crash and there is no replication, so you have 215 | to orchestrate that yourself doing double writes or something. 216 | 217 | The super simple architecture allows for all kinds of hacks to do 218 | backups/replication/sharding but you have to do those yourself. 219 | 220 | My usecase is ok with losing some data, and we dont have money to pay 221 | for kafka+zk+monitoring(kafka,zk), nor time to learn how to optimize 222 | it for our quite big write and very big multi-read load. 223 | 224 | Keep in mind that there is some not-invented-here syndrome involved 225 | into making it, but I use the service in production and it works very 226 | nice :) 227 | 228 | ### non atomic modify 229 | there is race between reading and modification from the client prespective 230 | 231 | 232 | 233 | ## TODO 234 | 235 | * travis-ci 236 | * perl client 237 | * make c client that can be used from ruby/perl 238 | * javadoc for the java client 239 | * publish the java client on maven central 240 | -------------------------------------------------------------------------------- /clients/go/README.md: -------------------------------------------------------------------------------- 1 | # lives at [jackdoe/go-rochefort-client](https://github.com/jackdoe/go-rochefort-client) 2 | 3 | example 4 | 5 | ``` 6 | $ go get https://github.com/jackdoe/go-rochefort-client 7 | ``` 8 | 9 | main.go 10 | 11 | ``` 12 | package main 13 | 14 | import ( 15 | "github.com/jackdoe/go-rochefort-client" 16 | "log" 17 | ) 18 | 19 | func main() { 20 | r := rochefort.NewClient("http://localhost:8002", nil) 21 | 22 | offset, _ := r.Append("namespace", nil, 0, []byte("some data")) 23 | fetched, _ := r.Get("namespace", offset) 24 | log.Printf("fetched: %s", string(fetched)) 25 | } 26 | 27 | ``` 28 | 29 | read the godoc for more info 30 | -------------------------------------------------------------------------------- /clients/java/.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.ear 17 | *.zip 18 | *.tar.gz 19 | *.rar 20 | 21 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 22 | hs_err_pid* 23 | 24 | *.iml 25 | target/ 26 | .idea 27 | -------------------------------------------------------------------------------- /clients/java/README.txt: -------------------------------------------------------------------------------- 1 | java client rochefort 2 | 3 | install: 4 | 5 | mvn install 6 | 7 | example usage: 8 | 9 | import nl.prymr.rochefort.Client; 10 | 11 | // the client is threadsafe, so make only 1 12 | public static final Client rochefort = new Client("http://localhost:8000"); 13 | 14 | // stores data for "exampleKey" 15 | long offset = rochefort.append("exampleKey", new byte[]{0,1,2,3,4,5}); 16 | 17 | // fetches data for "exampleKey" with the appropriate offset 18 | byte[] data = rochefort.get(offset); 19 | // data is now new byte[]{0,1,2,3,4,5} 20 | 21 | enjoy 22 | 23 | dependencies: 24 | 25 | 26 | com.mashape.unirest 27 | unirest-java 28 | 1.4.9 29 | 30 | 31 | Unirest is quite cool, check it out on: http://unirest.io/java.html 32 | -------------------------------------------------------------------------------- /clients/java/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | nl.prymr.rochefort 5 | client 6 | jar 7 | 1.0-SNAPSHOT 8 | client 9 | http://maven.apache.org 10 | 11 | 1.7 12 | 1.7 13 | UTF-8 14 | 15 | 16 | 17 | com.mashape.unirest 18 | unirest-java 19 | 1.4.9 20 | 21 | 22 | 23 | com.google.protobuf 24 | protobuf-java 25 | 3.5.1 26 | 27 | 28 | junit 29 | junit 30 | 3.8.1 31 | test 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /clients/java/src/main/java/nl/prymr/rochefort/Client.java: -------------------------------------------------------------------------------- 1 | package nl.prymr.rochefort; 2 | 3 | import com.google.protobuf.ByteString; 4 | import com.mashape.unirest.http.HttpResponse; 5 | import com.mashape.unirest.http.Unirest; 6 | import org.json.JSONObject; 7 | 8 | import java.io.*; 9 | import java.net.HttpURLConnection; 10 | import java.net.URL; 11 | import java.util.ArrayList; 12 | import java.util.Arrays; 13 | import java.util.List; 14 | 15 | import static nl.prymr.rochefort.Util.convertStreamToString; 16 | 17 | public class Client { 18 | private String urlGet, urlSet, urlScan, urlStats, urlQuery, urlDelete, urlCompact; 19 | 20 | public Client(String url) throws Exception { 21 | this(new URL(url)); 22 | } 23 | 24 | public Client(URL url) { 25 | String prefix; 26 | 27 | if (url.getPort() != -1) { 28 | prefix = url.getProtocol() + "://" + url.getHost() + ":" + url.getPort(); 29 | } else { 30 | prefix = url.getProtocol() + "://" + url.getHost(); 31 | } 32 | prefix += url.getPath(); 33 | if (!url.getPath().endsWith("/")) { 34 | prefix += "/"; 35 | } 36 | 37 | this.urlGet = prefix + "get"; 38 | this.urlSet = prefix + "set"; 39 | this.urlScan = prefix + "scan"; 40 | this.urlQuery = prefix + "query"; 41 | this.urlStats = prefix + "stat"; 42 | this.urlDelete = prefix + "delete"; 43 | this.urlCompact = prefix + "compact"; 44 | } 45 | 46 | public static Proto.AppendOutput append(String urlSet, Proto.AppendInput input) throws Exception { 47 | HttpResponse response = Unirest.post(urlSet).body(input.toByteArray()).asBinary(); 48 | if (response.getStatus() != 200) { 49 | throw new Exception( 50 | "status code " 51 | + response.getStatus() 52 | + " url: " 53 | + urlSet 54 | + " body: " 55 | + convertStreamToString(response.getRawBody())); 56 | } 57 | 58 | Proto.AppendOutput out = Proto.AppendOutput.parseFrom(response.getBody()); 59 | return out; 60 | } 61 | 62 | public static long append( 63 | String urlSet, String namespace, String[] tags, int allocSize, byte[] data) throws Exception { 64 | Proto.Append appendPayload = 65 | Proto.Append.newBuilder() 66 | .addAllTags(Arrays.asList(tags == null ? new String[0] : tags)) 67 | .setData(ByteString.copyFrom(data)) 68 | .setNamespace(namespace) 69 | .setAllocSize(allocSize) 70 | .build(); 71 | Proto.AppendInput input = 72 | Proto.AppendInput.newBuilder().addAppendPayload(appendPayload).build(); 73 | return append(urlSet, input).getOffset(0); 74 | } 75 | 76 | public static boolean modify( 77 | String urlSet, String namespace, long offset, int position, boolean resetLength, byte[] data) 78 | throws Exception { 79 | Proto.Modify modifyPayload = 80 | Proto.Modify.newBuilder() 81 | .setData(ByteString.copyFrom(data)) 82 | .setNamespace(namespace) 83 | .setOffset(offset) 84 | .setPos(position) 85 | .setResetLength(resetLength) 86 | .build(); 87 | Proto.AppendInput input = 88 | Proto.AppendInput.newBuilder().addModifyPayload(modifyPayload).build(); 89 | 90 | HttpResponse response = Unirest.post(urlSet).body(input.toByteArray()).asBinary(); 91 | if (response.getStatus() != 200) { 92 | throw new Exception( 93 | "status code " 94 | + response.getStatus() 95 | + " url: " 96 | + urlSet 97 | + " body: " 98 | + convertStreamToString(response.getRawBody())); 99 | } 100 | 101 | Proto.AppendOutput out = Proto.AppendOutput.parseFrom(response.getBody()); 102 | return out.getModifiedCount() > 0; 103 | } 104 | 105 | public static byte[] get(String urlGet, String namespace, long offset) throws Exception { 106 | long[] offsets = new long[1]; 107 | offsets[0] = offset; 108 | return get(urlGet, namespace, offsets).get(0); 109 | } 110 | 111 | public static List get(String urlGet, String namespace, long[] offsets) throws Exception { 112 | List payloads = new ArrayList<>(offsets.length); 113 | for (long offset : offsets) { 114 | payloads.add(Proto.Get.newBuilder().setNamespace(namespace).setOffset(offset).build()); 115 | } 116 | Proto.GetInput input = Proto.GetInput.newBuilder().addAllGetPayload(payloads).build(); 117 | HttpResponse response = Unirest.post(urlGet).body(input.toByteArray()).asBinary(); 118 | if (response.getStatus() != 200) { 119 | throw new Exception( 120 | "status code " 121 | + response.getStatus() 122 | + " url: " 123 | + urlGet 124 | + " body: " 125 | + convertStreamToString(response.getRawBody())); 126 | } 127 | 128 | List out = new ArrayList<>(); 129 | Proto.GetOutput ret = Proto.GetOutput.parseFrom(response.getBody()); 130 | for (ByteString b : ret.getDataList()) { 131 | out.add(b.toByteArray()); 132 | } 133 | return out; 134 | } 135 | 136 | public static boolean compact(String urlCompact, String namespace) throws Exception { 137 | Proto.NamespaceInput input = Proto.NamespaceInput.newBuilder().setNamespace(namespace).build(); 138 | 139 | HttpResponse response = 140 | Unirest.post(urlCompact).body(input.toByteArray()).asBinary(); 141 | if (response.getStatus() != 200) { 142 | throw new Exception( 143 | "status code " 144 | + response.getStatus() 145 | + " url: " 146 | + urlCompact 147 | + " namespace: " 148 | + namespace 149 | + " body: " 150 | + convertStreamToString(response.getRawBody())); 151 | } 152 | 153 | Proto.SuccessOutput out = Proto.SuccessOutput.parseFrom(response.getBody()); 154 | return out.getSuccess(); 155 | } 156 | 157 | public static boolean delete(String urlDelete, String namespace) throws Exception { 158 | Proto.NamespaceInput input = Proto.NamespaceInput.newBuilder().setNamespace(namespace).build(); 159 | 160 | HttpResponse response = 161 | Unirest.post(urlDelete).body(input.toByteArray()).asBinary(); 162 | if (response.getStatus() != 200) { 163 | throw new Exception( 164 | "status code " 165 | + response.getStatus() 166 | + " url: " 167 | + urlDelete 168 | + " namespace: " 169 | + namespace 170 | + " body: " 171 | + convertStreamToString(response.getRawBody())); 172 | } 173 | 174 | Proto.SuccessOutput out = Proto.SuccessOutput.parseFrom(response.getBody()); 175 | return out.getSuccess(); 176 | } 177 | 178 | public static void scan(String urlQuery, String namespace, Query query, ScanConsumer consumer) 179 | throws Exception { 180 | URL url = new URL((urlQuery + "?namespace=" + namespace)); 181 | 182 | // XXX: Unirest reads the whole body, which makes the scan useless 183 | HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 184 | 185 | connection.setRequestMethod("POST"); 186 | connection.setReadTimeout(consumer.getReadTimeout()); 187 | connection.setConnectTimeout(consumer.getConnectTimeout()); 188 | connection.setRequestProperty("Connection", "close"); 189 | InputStream inputStream = null; 190 | InputStreamReader reader = null; 191 | 192 | try { 193 | if (query != null) { 194 | connection.setDoOutput(true); 195 | connection.setRequestProperty("Content-Type", "application/json"); 196 | JSONObject json = new JSONObject(query); 197 | OutputStream os = connection.getOutputStream(); 198 | OutputStreamWriter writer = new OutputStreamWriter(os); 199 | json.write(writer); 200 | writer.flush(); 201 | writer.close(); 202 | os.close(); 203 | } else { 204 | connection.setDoOutput(true); 205 | } 206 | 207 | inputStream = connection.getInputStream(); 208 | byte[] buffer = new byte[65535]; 209 | byte[] header = new byte[12]; 210 | DataInputStream is = new DataInputStream(inputStream); 211 | 212 | while (true) { 213 | try { 214 | is.readFully(header, 0, header.length); 215 | } catch (EOFException e) { 216 | break; 217 | } 218 | int len = Util.aByteToInt(header, 0); 219 | long offset = Util.abyteToLong(header, 4); 220 | if (buffer.length < len) { 221 | buffer = new byte[len]; 222 | } 223 | is.readFully(buffer, 0, len); 224 | consumer.accept(buffer, len, offset); 225 | } 226 | 227 | } catch (Exception e) { 228 | int code = connection.getResponseCode(); 229 | throw new Exception( 230 | "status code " 231 | + code 232 | + " url: " 233 | + urlQuery 234 | + " namespace: " 235 | + namespace 236 | + " exception: " 237 | + e.getMessage()); 238 | } finally { 239 | if (reader != null) { 240 | reader.close(); 241 | } 242 | if (inputStream != null) { 243 | inputStream.close(); 244 | } 245 | connection.disconnect(); 246 | } 247 | } 248 | 249 | public static Proto.StatsOutput stats(String urlStats, String namespace) throws Exception { 250 | Proto.NamespaceInput input = Proto.NamespaceInput.newBuilder().setNamespace(namespace).build(); 251 | 252 | HttpResponse response = 253 | Unirest.post(urlStats).body(input.toByteArray()).asBinary(); 254 | if (response.getStatus() != 200) { 255 | throw new Exception( 256 | "status code " 257 | + response.getStatus() 258 | + " url: " 259 | + urlStats 260 | + " namespace: " 261 | + namespace 262 | + " body: " 263 | + convertStreamToString(response.getRawBody())); 264 | } 265 | 266 | Proto.StatsOutput out = Proto.StatsOutput.parseFrom(response.getBody()); 267 | return out; 268 | } 269 | 270 | public boolean modify(long offset, int position, boolean resetLength, byte[] data) 271 | throws Exception { 272 | return modify("", offset, position, resetLength, data); 273 | } 274 | 275 | public boolean modify( 276 | String namespace, long offset, int position, boolean resetLength, byte[] data) 277 | throws Exception { 278 | return modify(this.urlSet, namespace, offset, position, resetLength, data); 279 | } 280 | 281 | public long append(byte[] data) throws Exception { 282 | return append("", null, 0, data); 283 | } 284 | 285 | public long append(String[] tags, byte[] data) throws Exception { 286 | return append("", tags, 0, data); 287 | } 288 | 289 | public long append(int allocSize, byte[] data) throws Exception { 290 | return append("", null, allocSize, data); 291 | } 292 | 293 | public long append(String[] tags, int allocSize, byte[] data) throws Exception { 294 | return append("", tags, allocSize, data); 295 | } 296 | 297 | public long append(String namespace, byte[] data) throws Exception { 298 | return append(namespace, null, 0, data); 299 | } 300 | 301 | public long append(String namespace, String[] tags, byte[] data) throws Exception { 302 | return append(namespace, tags, 0, data); 303 | } 304 | 305 | public long append(String namespace, String[] tags, int allocSize, byte[] data) throws Exception { 306 | return append(this.urlSet, namespace, tags, allocSize, data); 307 | } 308 | 309 | public Proto.AppendOutput append(Proto.AppendInput input) throws Exception { 310 | return append(this.urlSet, input); 311 | } 312 | 313 | public byte[] get(long offset) throws Exception { 314 | return get("", offset); 315 | } 316 | 317 | public byte[] get(String namespace, long offset) throws Exception { 318 | return get(this.urlGet, namespace, offset); 319 | } 320 | 321 | public List getMulti(long[] listOfOffsets) throws Exception { 322 | return getMulti("", listOfOffsets); 323 | } 324 | 325 | public List getMulti(byte[] encodedListOfOffsets) throws Exception { 326 | return getMulti("", encodedListOfOffsets); 327 | } 328 | 329 | public List getMulti(String namespace, long[] listOfOffsets) throws Exception { 330 | return get(this.urlGet, namespace, listOfOffsets); 331 | } 332 | 333 | public List getMulti(String namespace, byte[] encodedListOfOffsets) throws Exception { 334 | return get(this.urlGet, namespace, Util.listOfBytesToLongs(encodedListOfOffsets)); 335 | } 336 | 337 | public void scan(ScanConsumer consumer) throws Exception { 338 | scan(this.urlScan, "", null, consumer); 339 | } 340 | 341 | public void scan(String namespace, ScanConsumer consumer) throws Exception { 342 | scan(this.urlScan, namespace, null, consumer); 343 | } 344 | 345 | public void search(Query query, ScanConsumer consumer) throws Exception { 346 | scan(this.urlQuery, "", query, consumer); 347 | } 348 | 349 | public void search(String namespace, Query query, ScanConsumer consumer) throws Exception { 350 | scan(this.urlQuery, namespace, query, consumer); 351 | } 352 | 353 | public Proto.StatsOutput stats(String namespace) throws Exception { 354 | return stats(this.urlStats, namespace); 355 | } 356 | 357 | public boolean delete(String namespace) throws Exception { 358 | return delete(this.urlDelete, namespace); 359 | } 360 | 361 | public boolean compact(String namespace) throws Exception { 362 | return compact(this.urlCompact, namespace); 363 | } 364 | 365 | public abstract static class ScanConsumer { 366 | public int getConnectTimeout() { 367 | return 1000; 368 | } 369 | 370 | public int getReadTimeout() { 371 | return 1000; 372 | } 373 | 374 | public abstract void accept(byte[] buffer, int length, long rochefortOffset) throws Exception; 375 | } 376 | 377 | public static final class Query { 378 | public List and; 379 | public List or; 380 | public String tag; 381 | 382 | public Query(String t) { 383 | this.tag = t; 384 | } 385 | 386 | public Query() {} 387 | 388 | public String getTag() { 389 | return tag; 390 | } 391 | 392 | public List getAnd() { 393 | return and; 394 | } 395 | 396 | public List getOr() { 397 | return or; 398 | } 399 | 400 | public Query and(Query... qq) { 401 | if (this.and == null) this.and = new ArrayList<>(); 402 | 403 | for (Query q : qq) this.and.add(q); 404 | return this; 405 | } 406 | 407 | public Query or(Query... qq) { 408 | if (this.or == null) this.or = new ArrayList<>(); 409 | 410 | for (Query q : qq) this.or.add(q); 411 | return this; 412 | } 413 | } 414 | } 415 | -------------------------------------------------------------------------------- /clients/java/src/main/java/nl/prymr/rochefort/Util.java: -------------------------------------------------------------------------------- 1 | package nl.prymr.rochefort; 2 | 3 | import java.io.EOFException; 4 | import java.io.IOException; 5 | import java.io.InputStream; 6 | import java.util.Arrays; 7 | 8 | public class Util { 9 | public static byte[] listOfLongsToBytes(long[] longs) { 10 | byte[] bytes = new byte[longs.length * 8]; 11 | for (int i = 0, j = 0; i < longs.length; i++, j += 8) { 12 | bytes[j + 7] = (byte) (longs[i] >> 56); 13 | bytes[j + 6] = (byte) (longs[i] >> 48); 14 | bytes[j + 5] = (byte) (longs[i] >> 40); 15 | bytes[j + 4] = (byte) (longs[i] >> 32); 16 | bytes[j + 3] = (byte) (longs[i] >> 24); 17 | bytes[j + 2] = (byte) (longs[i] >> 16); 18 | bytes[j + 1] = (byte) (longs[i] >> 8); 19 | bytes[j + 0] = (byte) (longs[i]); 20 | } 21 | 22 | return bytes; 23 | } 24 | 25 | public static long abyteToLong(byte[] bytes, int off) { 26 | long r = 27 | ((long) (bytes[off + 7] & 0xFF)) << 56L 28 | | ((long) (bytes[off + 6] & 0xFF)) << 48L 29 | | ((long) (bytes[off + 5] & 0xFF)) << 40L 30 | | ((long) (bytes[off + 4] & 0xFF)) << 32L 31 | | ((long) (bytes[off + 3] & 0xFF)) << 24L 32 | | ((long) (bytes[off + 2] & 0xFF)) << 16L 33 | | ((long) (bytes[off + 1] & 0xFF)) << 8L 34 | | ((long) (bytes[off + 0] & 0xFF)); 35 | return r; 36 | } 37 | 38 | public static long[] listOfBytesToLongs(byte[] bytes) { 39 | long[] longs = new long[bytes.length / 8]; 40 | for (int i = 0, j = 0; i < bytes.length; i += 8, j++) { 41 | longs[j] = abyteToLong(bytes, i); 42 | } 43 | return longs; 44 | } 45 | 46 | public static long[] listOfBytesToLongsPlusExtraOne(byte[] bytes) { 47 | long[] longs = new long[(bytes.length / 8) + 1]; 48 | for (int i = 0, j = 0; i < bytes.length; i += 8, j++) { 49 | longs[j] = abyteToLong(bytes, i); 50 | } 51 | return longs; 52 | } 53 | 54 | public static int aByteToInt(byte[] bytes, int off) { 55 | int r = 56 | ((int) ((int) (bytes[off + 3] & 0xFF)) << 24L 57 | | ((int) (bytes[off + 2] & 0xFF)) << 16L 58 | | ((int) (bytes[off + 1] & 0xFF)) << 8L 59 | | ((int) (bytes[off + 0] & 0xFF))); 60 | 61 | return r; 62 | } 63 | 64 | public static String convertStreamToString(java.io.InputStream is) { 65 | java.util.Scanner s = new java.util.Scanner(is).useDelimiter("\\A"); 66 | return s.hasNext() ? s.next() : ""; 67 | } 68 | 69 | public static byte[] readFully(InputStream is, int length, boolean readAll) throws IOException { 70 | byte[] output = {}; 71 | if (length == -1) length = Integer.MAX_VALUE; 72 | int pos = 0; 73 | while (pos < length) { 74 | int bytesToRead; 75 | if (pos >= output.length) { // Only expand when there's no room 76 | bytesToRead = Math.min(length - pos, output.length + 1024); 77 | if (output.length < pos + bytesToRead) { 78 | output = Arrays.copyOf(output, pos + bytesToRead); 79 | } 80 | } else { 81 | bytesToRead = output.length - pos; 82 | } 83 | int cc = is.read(output, pos, bytesToRead); 84 | if (cc < 0) { 85 | if (readAll && length != Integer.MAX_VALUE) { 86 | throw new EOFException("Detect premature EOF"); 87 | } else { 88 | if (output.length != pos) { 89 | output = Arrays.copyOf(output, pos); 90 | } 91 | break; 92 | } 93 | } 94 | pos += cc; 95 | } 96 | return output; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /clients/java/src/test/java/nl/prymr/rochefort/ClientTest.java: -------------------------------------------------------------------------------- 1 | package nl.prymr.rochefort; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | import java.io.ByteArrayOutputStream; 8 | import java.util.*; 9 | import java.util.concurrent.*; 10 | 11 | public class ClientTest extends TestCase { 12 | public static final String[] namespaces = new String[] {"", "some-very-long-name", "example"}; 13 | public static Map> lookupAllOffsets = new ConcurrentHashMap<>(); 14 | Client client; 15 | 16 | public ClientTest(String testName) { 17 | super(testName); 18 | } 19 | 20 | public static Test suite() { 21 | return new TestSuite(ClientTest.class); 22 | } 23 | 24 | @Override 25 | public void setUp() throws Exception { 26 | client = new Client(System.getenv("ROCHEFORT_TEST")); 27 | for (final String s : namespaces) { 28 | lookupAllOffsets.put(s, new ConcurrentHashMap()); 29 | client.scan( 30 | s, 31 | new Client.ScanConsumer() { 32 | @Override 33 | public int getReadTimeout() { 34 | return 5000; 35 | } 36 | 37 | @Override 38 | public void accept(byte[] buffer, int length, long offset) { 39 | byte[] tmp = Arrays.copyOf(buffer, length); 40 | lookupAllOffsets.get(s).put(offset, tmp); 41 | } 42 | }); 43 | } 44 | } 45 | 46 | public void testSearch() throws Exception { 47 | client.append("search", new String[] {"jaz"}, "abc".getBytes()); 48 | client.append("search", new String[] {"jaz"}, "zzz".getBytes()); 49 | client.append("search", new String[] {"jaz"}, "zzz2".getBytes()); 50 | 51 | final List matching = new ArrayList<>(); 52 | client.search( 53 | "search", 54 | new Client.Query() 55 | .or( 56 | new Client.Query("jaz"), 57 | new Client.Query("jaz"), 58 | new Client.Query().and(new Client.Query("jaz"))), 59 | new Client.ScanConsumer() { 60 | @Override 61 | public void accept(byte[] buffer, int length, long rochefortOffset) throws Exception { 62 | matching.add(new String(Arrays.copyOfRange(buffer, 0, length))); 63 | } 64 | }); 65 | 66 | assertEquals(matching.get(0), "abc"); 67 | assertEquals(matching.get(1), "zzz"); 68 | 69 | Proto.StatsOutput s = client.stats("search"); 70 | assertEquals(Long.valueOf(3), s.getTagsMap().get("jaz")); 71 | } 72 | 73 | public void testModify() throws Exception { 74 | long offset = client.append(1024, "abc".getBytes()); 75 | 76 | assertEquals(new String(client.get(offset)), "abc"); 77 | client.modify(offset, 1, false, "xyz".getBytes()); 78 | 79 | assertEquals(new String(client.get(offset)), "axyz"); 80 | 81 | client.modify(offset, 0, true, "xyz".getBytes()); 82 | assertEquals(new String(client.get(offset)), "xyzz"); 83 | 84 | client.modify(offset, 0, true, "xyz".getBytes()); 85 | assertEquals(new String(client.get(offset)), "xyz"); 86 | 87 | lookupAllOffsets.get("").put(offset, client.get(offset)); 88 | } 89 | 90 | public void testApp() throws Exception { 91 | Random random = new Random(System.currentTimeMillis()); 92 | 93 | for (int attempt = 0; attempt < 2; attempt++) { 94 | 95 | for (final String namespace : namespaces) { 96 | List everything = new ArrayList<>(); 97 | List allOffsets = new ArrayList<>(); 98 | 99 | for (String id : new String[] {"abc", "abcd", "abcdef"}) { 100 | List offsets = new ArrayList(); 101 | List stored = new ArrayList(); 102 | for (int size : 103 | new int[] { 104 | 1, 1, 1, 1, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 105 | 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 106 | 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 1000, 1000, 107 | 10000, 100000, 1000000 108 | }) { 109 | 110 | ByteArrayOutputStream bos = new ByteArrayOutputStream(); 111 | for (int i = 0; i < size; i++) { 112 | bos.write(random.nextInt()); 113 | } 114 | bos.flush(); 115 | 116 | byte[] data = bos.toByteArray(); 117 | long offset = client.append(namespace, data); 118 | offsets.add(offset); 119 | stored.add(data); 120 | 121 | // make sure we never get the same offset twice 122 | assertNull( 123 | "we already have offset " + offset, lookupAllOffsets.get(namespace).get(offset)); 124 | lookupAllOffsets.get(namespace).put(offset, data); 125 | 126 | byte[] fetchedData = client.get(namespace, offset); 127 | assertTrue(Arrays.equals(data, fetchedData)); 128 | 129 | long[] loffsets = new long[offsets.size()]; 130 | for (int i = 0; i < offsets.size(); i++) loffsets[i] = offsets.get(i); 131 | List fetched = client.getMulti(namespace, loffsets); 132 | 133 | assertFalse(stored.size() == 0); 134 | assertEquals(stored.size(), fetched.size()); 135 | for (int i = 0; i < stored.size(); i++) { 136 | assertTrue(Arrays.equals(stored.get(i), fetched.get(i))); 137 | } 138 | } 139 | 140 | everything.addAll(stored); 141 | allOffsets.addAll(offsets); 142 | } 143 | 144 | long[] loffsets = new long[allOffsets.size()]; 145 | for (int i = 0; i < allOffsets.size(); i++) loffsets[i] = allOffsets.get(i); 146 | List fetched = client.getMulti(namespace, loffsets); 147 | 148 | assertFalse(fetched.size() == 0); 149 | assertEquals(everything.size(), fetched.size()); 150 | for (int i = 0; i < everything.size(); i++) { 151 | assertTrue(Arrays.equals(everything.get(i), fetched.get(i))); 152 | } 153 | } 154 | } 155 | } 156 | 157 | public void testManyAsync() throws Exception { 158 | int threadCount = 5; 159 | Callable task = 160 | new Callable() { 161 | @Override 162 | public Long call() { 163 | try { 164 | testApp(); 165 | return 1L; 166 | } catch (Exception e) { 167 | throw new RuntimeException(e); 168 | } 169 | } 170 | }; 171 | List> tasks = Collections.nCopies(threadCount, task); 172 | ExecutorService executorService = Executors.newFixedThreadPool(threadCount); 173 | List> futures = executorService.invokeAll(tasks); 174 | long sum = 0; 175 | for (Future f : futures) { 176 | sum += f.get(); 177 | } 178 | assertEquals(sum, threadCount); 179 | } 180 | 181 | @Override 182 | public void tearDown() throws Exception { 183 | for (final String namespace : namespaces) { 184 | client.scan( 185 | namespace, 186 | new Client.ScanConsumer() { 187 | @Override 188 | public void accept(byte[] buffer, int length, long offset) { 189 | assertNotNull( 190 | "missing offset " + offset + " on namespace " + namespace, 191 | lookupAllOffsets.get(namespace).get(offset)); 192 | byte[] tmp = Arrays.copyOf(buffer, length); 193 | byte[] stored = lookupAllOffsets.get(namespace).get(offset); 194 | 195 | assertTrue(Arrays.equals(tmp, stored)); 196 | } 197 | }); 198 | } 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /clients/js/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock* 3 | -------------------------------------------------------------------------------- /clients/js/README.txt: -------------------------------------------------------------------------------- 1 | ** works with rochefort < 2.0 ** 2 | 3 | $ npm install rochefort 4 | 5 | example: 6 | 7 | var data = new Buffer('asdasljdasjhd') 8 | var r = new rochefort('http://localhost:8002') 9 | var stored_offset 10 | r.append({data: data}).then(offset => { 11 | // append example returns the stored offset 12 | // since the offset is 64 bit, rochefort also returns a string (offset_str) 13 | // and we use the string version 14 | stored_offset = offset 15 | return r.get({offset: offset}) 16 | }).then(value => { 17 | // get examplem returns the stored buffer 18 | console.log(value) 19 | }).then(() => { 20 | // again offsets is array of strings returned from append() 21 | return r.getMulti({offsets: [stored_offset,stored_offset,stored_offset,stored_offset]}) 22 | }).then(values => { 23 | // multi get example 24 | // returns array of Buffers 25 | console.log(values) 26 | }) 27 | -------------------------------------------------------------------------------- /clients/js/index.js: -------------------------------------------------------------------------------- 1 | 'use strict;' 2 | const { URL } = require('url') 3 | const axios = require('axios') 4 | 5 | const append = function append(args) { 6 | return axios({ 7 | method: 'get', 8 | url: new URL('append',args.url).toString(), 9 | params: { 10 | namespace: args.namespace, 11 | allocSize: args.allocSize, 12 | tags: (args.tags || []).join(","), 13 | }, 14 | timeout: args.timeout || 1000, 15 | data: args.data 16 | }).then((r) => { 17 | return r.data.offset 18 | }) 19 | } 20 | 21 | 22 | const modify = function append(args) { 23 | return axios({ 24 | method: 'get', 25 | url: new URL('modify',args.url).toString(), 26 | params: { 27 | namespace: args.namespace, 28 | pos: args.position, 29 | offset: args.offset, 30 | }, 31 | timeout: args.timeout || 1000, 32 | data: args.data 33 | }).then((r) => { 34 | return r.data.success 35 | }) 36 | } 37 | 38 | 39 | const get = function get(args) { 40 | return axios({ 41 | method: 'get', 42 | url: new URL("get",args.url).toString(), 43 | params: { 44 | namespace: args.namespace, 45 | offset: args.offset || 0, 46 | }, 47 | responseType: 'arraybuffer', 48 | timeout: args.timeout || 1000, 49 | }).then((r) => { 50 | return r.data 51 | }) 52 | } 53 | const Buffer = require('buffer').Buffer; 54 | 55 | const getMulti = function get(args) { 56 | var raw = Buffer.alloc(args.offsets.length * 8) 57 | var offsets = args.offsets 58 | for (let i = 0; i < offsets.length; i++) { 59 | var num = offsets[i] 60 | var bufferOffset = i * 8; 61 | 62 | var lo = num | 0; 63 | if (lo < 0) 64 | lo += 4294967296; 65 | 66 | var hi = num - lo; 67 | hi /= 4294967296; 68 | raw.writeUInt32LE(lo, bufferOffset) 69 | raw.writeUInt32LE(hi, bufferOffset + 4) 70 | } 71 | 72 | return axios({ 73 | url:new URL("getMulti",args.url).toString(), 74 | params: { 75 | namespace: args.namespace, 76 | }, 77 | responseType: 'arraybuffer', 78 | data: raw, 79 | timeout: args.timeout || 1000, 80 | }).then((r) => { 81 | var out = [] 82 | var offset = 0 83 | var data = r.data 84 | while(offset < data.length) { 85 | var len = data.readUInt32LE(offset) 86 | out.push(data.slice(offset + 4, offset + len + 4)) 87 | offset += 4 + len 88 | } 89 | return out 90 | }) 91 | } 92 | 93 | const rochefort = function(url) { 94 | this.url = url 95 | this.get = function(args) { 96 | return get({...args, url: this.url}) 97 | } 98 | 99 | this.append = function(args) { 100 | return append({...args, url: this.url}) 101 | } 102 | 103 | this.modify = function(args) { 104 | return modify({...args, url: this.url}) 105 | } 106 | 107 | this.getMulti = function(args) { 108 | return getMulti({...args, url: this.url}) 109 | } 110 | } 111 | 112 | module.exports = rochefort 113 | -------------------------------------------------------------------------------- /clients/js/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "rochefort", 3 | "version": "2.0.1", 4 | "description": "client for rochefort - fast data append service that returns offsets to be indexed", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "node test.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/jackdoe/rochefort.git" 12 | }, 13 | "keywords": [ 14 | "rochefort", 15 | "store" 16 | ], 17 | "author": "Borislav Nikolov", 18 | "license": "MIT", 19 | "bugs": { 20 | "url": "https://github.com/jackdoe/rochefort/issues" 21 | }, 22 | "homepage": "https://github.com/jackdoe/rochefort#readme", 23 | "dependencies": { 24 | "axios": ">=0.18.1" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /clients/js/test.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | const rochefort = require('./index.js') 3 | var o 4 | 5 | data = new Buffer.alloc(1024 * 1024 * 10) 6 | r = new rochefort('http://localhost:8000') 7 | r.append({data: data, tags: ['a','b','c']}).then(offset => { 8 | o = offset 9 | return r.get({offset: offset}) 10 | }).then(v => { 11 | assert.equal(0, Buffer.compare(data,v)) 12 | }).then(() => { 13 | return r.getMulti({offsets: [o,o,o,o]}) 14 | }).then(v => { 15 | assert.equal(4, v.length) 16 | for (var i = 0; i < v.length; i++) { 17 | assert.equal(0, Buffer.compare(v[i],data)) 18 | } 19 | }).catch(e => { 20 | console.log(e) 21 | }) 22 | 23 | 24 | var o1 25 | r.append({data: 'abcd', allocSize: 5}).then(offset => { 26 | o1 = offset 27 | return r.modify({ 28 | data: 'xyz', 29 | offset: offset, 30 | position: 1 31 | }) 32 | }).then(() => { 33 | return r.get({offset: o1}) 34 | }).then(value => { 35 | assert.equal(value, 'axyz') 36 | }).catch(e => { 37 | console.log(e) 38 | }) 39 | -------------------------------------------------------------------------------- /clients/ruby/README.txt: -------------------------------------------------------------------------------- 1 | ** works with rochefort < 2.0 ** 2 | 3 | $ gem install rochefort 4 | 5 | example usage: 6 | 7 | 8 | require 'rochefort' 9 | 10 | r = Rochefort.new("http://localhost:8001") 11 | 12 | # append data 13 | offset = r.append(id: '123',data: 'example data') 14 | offset2 = r.append(id: '123',data: 'some other data') 15 | 16 | # get the data stored at offset 17 | fetched = r.get(offset: offset) 18 | 19 | # get multi 20 | many = r.getMulti(offsets: [offset,offset2]) 21 | 22 | timeouts: 23 | you can pass read_timeout and open_timeout 24 | 25 | if you want to namespace your ids into different directories use the 26 | parameter namespace 27 | 28 | e.g.: 29 | offset = r.append(namespace: 'today', id: '123',data: 'example data') 30 | -------------------------------------------------------------------------------- /clients/ruby/Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake/testtask' 2 | 3 | Rake::TestTask.new do |t| 4 | t.libs << 'test' 5 | end 6 | 7 | desc "Run tests" 8 | task :default => :test 9 | -------------------------------------------------------------------------------- /clients/ruby/lib/rochefort.rb: -------------------------------------------------------------------------------- 1 | require 'rest-client' 2 | require 'uri' 3 | require 'json' 4 | require 'input_pb' 5 | 6 | # client for https://github.com/jackdoe/rochefort - disk speed append + offset service (poor man's kafka), 7 | # example usage 8 | # r = new Rochefort("http://localhost:8001") 9 | # offset = r.append(namespace: "example-namespace", data:"example-data") 10 | # p r.get(namespace: "example-namespace", offset: offset) 11 | class Rochefort 12 | # @param url - the rochefort url (String) 13 | def initialize(url) 14 | @urlAppend = URI::join(url,'append').to_s 15 | @urlModify = URI::join(url,'modify').to_s 16 | @urlGet = URI::join(url,'get').to_s 17 | @urlScan = URI::join(url,'scan').to_s 18 | @urlQuery = URI::join(url,'query').to_s 19 | @urlGetMulti = URI::join(url,'getMulti').to_s 20 | end 21 | 22 | # append to rochefort, and returns the stored offset 23 | # r = Rochefort.new(url) 24 | # r.append( 25 | # namespace: "ns", # default nil (nil means the default namespace) 26 | # data: "some data", # the data you want to append 27 | # alloc_size: 1024, # you can allocate more space and then modify it, or 0 for len(data) 28 | # read_timeout: 1, # default 1 29 | # open_timeout: 1, # default 1 30 | # ) 31 | # @return the offset at which the data was stored 32 | def append(opts) 33 | tags = opts[:tags] || [] 34 | data = RestClient::Request.execute(method: :post, 35 | url: "#{@urlAppend}?namespace=#{opts[:namespace]}&allocSize=#{opts[:alloc_size]}&tags=#{tags.join(",")}", 36 | payload: opts[:data], 37 | read_timeout: opts[:read_timeout] || 1, 38 | open_timeout: opts[:open_timeout] || 1) 39 | out = JSON.parse(data.body) 40 | return out["offset"] 41 | end 42 | 43 | 44 | # modify inplace 45 | # r = Rochefort.new(url) 46 | # r.modify( 47 | # namespace: "ns", # default nil (nil means the default namespace) 48 | # data: "some data", # the data you want to append 49 | # offset: 1024, # appended offset you want to modify 50 | # position: 10, # position within the blob you want to replace with 'data' 51 | # read_timeout: 1, # default 1 52 | # open_timeout: 1, # default 1 53 | # ) 54 | # @return the offset at which the data was stored 55 | def modify(opts) 56 | data = RestClient::Request.execute(method: :post, 57 | url: "#{@urlModify}?namespace=#{opts[:namespace]}&offset=#{opts[:offset]}&pos=#{opts[:position]}", 58 | payload: opts[:data], 59 | read_timeout: opts[:read_timeout] || 1, 60 | open_timeout: opts[:open_timeout] || 1) 61 | out = JSON.parse(data.body) 62 | return out["success"] 63 | end 64 | 65 | 66 | # get data from rochefort 67 | # r = Rochefort.new(url) 68 | # r.get( 69 | # namespace: "ns", # default nil (nil means the default namespace) 70 | # offset: 0, # the offset returned from append() 71 | # read_timeout: 1, # default 1 72 | # open_timeout: 1, # default 1 73 | # ) 74 | # @return the stored data (String) 75 | def get(opts) 76 | data = RestClient::Request.execute(method: :get, 77 | url: "#{@urlGet}?namespace=#{opts[:namespace]}&offset=#{opts[:offset]}", 78 | read_timeout: opts[:read_timeout] || 1, 79 | open_timeout: opts[:open_timeout] || 1).body 80 | return data 81 | end 82 | 83 | # get multiple items from rochefort, (@see #get) (@see #append) 84 | # r = Rochefort.new(url) 85 | # r.getMulti( 86 | # namespace: "ns", # default nil (nil means the default namespace) 87 | # offsets: [], # array of offsets 88 | # read_timeout: 1, # default 1 89 | # open_timeout: 1, # default 1 90 | # ) 91 | # @return array of stored elements (array of strings) 92 | def getMulti(opts) 93 | data = RestClient::Request.execute(method: :get, 94 | url: "#{@urlGetMulti}?namespace=#{opts[:namespace]}", 95 | payload: opts[:offsets].pack("q<*"), 96 | read_timeout: opts[:read_timeout] || 1, 97 | open_timeout: opts[:open_timeout] || 1).body 98 | offset = 0 99 | out = [] 100 | while offset != data.length 101 | len = data.unpack('l<')[0] 102 | out << data[offset + 4, len] 103 | offset += 4 + len 104 | end 105 | return out 106 | end 107 | 108 | 109 | def scanParser(input_block) 110 | block = proc do |response| 111 | buffer = "" 112 | header_len = 12 113 | need = header_len 114 | 115 | waiting_for_header = true 116 | 117 | len = 0 118 | rochefort_offset = 0 119 | 120 | response.read_body do |chunk| 121 | buffer << chunk 122 | while buffer.length >= need 123 | if waiting_for_header 124 | h = buffer.unpack('l= need 133 | input_block.call(rochefort_offset, buffer[0, len]) 134 | buffer = buffer[len, buffer.length - len] 135 | need = header_len 136 | waiting_for_header = true 137 | end 138 | end 139 | end 140 | end 141 | return block 142 | end 143 | 144 | 145 | # scans a namespace, reading from a stream, so the namespace can be very big 146 | # r = Rochefort.new(url) 147 | # r.scan(namespace: ns) do |offset, value| 148 | # puts value 149 | # end 150 | # @return calls the block for each item 151 | def scan(opts,&input_block) 152 | RestClient::Request.execute(method: :get, 153 | url: "#{@urlScan}?namespace=#{opts[:namespace]}", 154 | read_timeout: opts[:read_timeout] || 1, 155 | open_timeout: opts[:open_timeout] || 1, 156 | block_response: scanParser(input_block)) 157 | end 158 | 159 | # searches a namespace, for the tagged /append blobs, reading from a stream, so the namespace can be very big 160 | # r = Rochefort.new(url) 161 | # r.search(query: {tag: 'a'}, namespace: ns) do |offset, value| 162 | # puts value 163 | # end 164 | # 165 | # the dsl is fairly simple: (A OR B OR (C AND D) 166 | # { 167 | # or: [ { tag: 'a' }, { tag: 'a' }, { and: [ {tag: 'c'}, {tag:'d'} ] } ] 168 | # } 169 | # @return calls the block for each item 170 | def search(opts,&input_block) 171 | RestClient::Request.execute(method: :post, 172 | payload: JSON.generate(opts[:query]), 173 | url: "#{@urlQuery}?namespace=#{opts[:namespace]}", 174 | read_timeout: opts[:read_timeout] || 1, 175 | open_timeout: opts[:open_timeout] || 1, 176 | block_response: scanParser(input_block)) 177 | end 178 | end 179 | -------------------------------------------------------------------------------- /clients/ruby/rochefort.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = 'rochefort' 3 | s.version = '0.1.2' 4 | s.date = '2018-02-15' 5 | s.summary = "client for rochefort storage service" 6 | s.description = "rochefort is fast data append service that returns offsets to be indexed" 7 | s.authors = ["Borislav Nikolov"] 8 | s.email = 'jack@sofialondonmoskva.com' 9 | s.files = ["lib/rochefort.rb"] 10 | s.homepage = 'https://github.com/jackdoe/rochefort' 11 | s.license = 'MIT' 12 | s.add_runtime_dependency('rest-client','~> 2.0') 13 | s.add_development_dependency('rake') 14 | end 15 | -------------------------------------------------------------------------------- /clients/ruby/test/test_rochefort.rb: -------------------------------------------------------------------------------- 1 | require 'minitest/autorun' 2 | require 'rochefort' 3 | require 'securerandom' 4 | class RochefortTest < Minitest::Unit::TestCase 5 | def test_empty 6 | url = ENV["ROCHEFORT_TEST"] 7 | if url 8 | r = Rochefort.new(url) 9 | offset = r.append(namespace: 'empty', alloc_size: 1024, data: '') 10 | fetched = r.get(namespace: 'empty', offset: offset) 11 | assert_equal fetched,'' 12 | end 13 | end 14 | 15 | def test_search 16 | url = ENV["ROCHEFORT_TEST"] 17 | if url 18 | r = Rochefort.new(url) 19 | 20 | a = [] 21 | b = [] 22 | ns = 'search' 23 | a << r.append(namespace: ns, data: 'aaa', tags: ['a']) 24 | a << r.append(namespace: ns, data: 'aaa2', tags: ['a','b']) 25 | a << r.append(namespace: ns, data: 'aaa3', tags: ['a','b','c']) 26 | 27 | b << r.append(namespace: ns, data: 'bbb', tags: ['b']) 28 | b << r.append(namespace: ns, data: 'bbb2', tags: ['a','b']) 29 | b << r.append(namespace: ns, data: 'bbb3', tags: ['a','b','c']) 30 | 31 | 32 | aa = [] 33 | bb = [] 34 | ab = [] 35 | 36 | r.search(query: {tag: 'a'}, namespace: ns) do |offset, v| 37 | aa << v 38 | end 39 | 40 | r.search(query: {tag: 'b'}, namespace: ns) do |offset, v| 41 | bb << v 42 | end 43 | 44 | r.search(query: {or: [{tag: 'a'}, {tag: 'b'}]}, namespace: ns) do |offset, v| 45 | ab << v 46 | end 47 | 48 | aabb = [] 49 | r.search(query: {and: [{tag: 'a'}, {tag: 'b'}]}, namespace: ns) do |offset, v| 50 | aabb << v 51 | end 52 | 53 | 54 | assert_equal(aa, ['aaa','aaa2','aaa3','bbb2','bbb3']) 55 | assert_equal(aabb, ['aaa2','aaa3','bbb2','bbb3']) 56 | assert_equal(bb, ['aaa2','aaa3','bbb','bbb2','bbb3']) 57 | assert_equal(ab, ['aaa','aaa2','aaa3','bbb','bbb2','bbb3']) 58 | end 59 | end 60 | 61 | 62 | def test_rochefort 63 | url = ENV["ROCHEFORT_TEST"] 64 | r = Rochefort.new(url) 65 | if url 66 | [nil,"atext","exam,ple"].each do |ns| 67 | everything_so_far = {} 68 | r.scan(namespace: ns) do |offset, data| 69 | everything_so_far[offset] = data 70 | end 71 | 72 | 1.upto(10) do |id| 73 | 1.upto(10) do |suffix| 74 | data = SecureRandom.random_bytes(suffix * 100) 75 | offset = r.append(namespace: ns, alloc_size: data.length * 2, data: data) 76 | fetched = r.get(namespace: ns, offset: offset) 77 | 78 | assert_equal data,fetched 79 | many = r.getMulti(namespace: ns,offsets: [offset,offset,offset,offset]) 80 | assert_equal many,[data,data,data,data] 81 | 82 | # test modification head 83 | r.modify(namespace: ns, offset: offset, position: 0, data: "abc") 84 | fetchedAfter = r.get(namespace: ns, offset: offset) 85 | fetched[0,3] = 'abc' 86 | assert_equal fetched,fetchedAfter 87 | 88 | 89 | # test modification tail 90 | r.modify(namespace: ns, offset: offset, position: data.length - 4, data: "abcd") 91 | fetchedAfter = r.get(namespace: ns, offset: offset) 92 | fetched[fetched.length - 4, 4] = 'abcd' 93 | assert_equal fetched,fetchedAfter 94 | 95 | # test modification outside 96 | r.modify(namespace: ns, offset: offset, position: -1, data: "abcde") 97 | fetchedAfter = r.get(namespace: ns, offset: offset) 98 | fetched << 'abcde' 99 | assert_equal fetched,fetchedAfter 100 | 101 | 102 | # test modification near end of allocSize 103 | r.modify(namespace: ns, offset: offset, position: (data.length * 2) - 3, data: "zxc") 104 | fetchedAfter = r.get(namespace: ns, offset: offset) 105 | fetched[fetched.length, (data.length * 2) - fetched.length - 3] = "\x00" * ((data.length * 2) - fetched.length - 3) 106 | fetched << 'zxc' 107 | assert_equal fetched,fetchedAfter 108 | 109 | 110 | everything_so_far[offset] = fetched 111 | 112 | matching = 0 113 | r.scan(namespace: ns, open_timeout: 10) do |soffset, v| 114 | if !everything_so_far[soffset] 115 | raise "expected ${soffset}" 116 | end 117 | matching += 1 118 | end 119 | 120 | assert_equal matching, everything_so_far.length 121 | end 122 | end 123 | end 124 | end 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /compact_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | mr "math/rand" 7 | "os" 8 | "path" 9 | "testing" 10 | ) 11 | 12 | func TestCompact(t *testing.T) { 13 | path := path.Join(os.TempDir(), "rochefort_compact_test") 14 | os.RemoveAll(path) 15 | 16 | storage := NewStorage(path) 17 | 18 | data := []byte{1, 2, 3, 4} 19 | compactedOffset := uint32(0) 20 | uncompactedOffset := uint32(0) 21 | for i := 0; i < 100; i++ { 22 | allocSize := uint32(1024 + i) 23 | 24 | storage.append(allocSize, data) 25 | 26 | compactedOffset += uint32(len(data)) + headerLen 27 | uncompactedOffset += headerLen + allocSize 28 | } 29 | 30 | if uncompactedOffset != uint32(storage.offset) { 31 | t.Log("uncompactedOffset != stored.offset") 32 | t.FailNow() 33 | } 34 | 35 | if compactedOffset == uncompactedOffset { 36 | t.Log("compactedOffset == uncompactedOffset") 37 | t.FailNow() 38 | } 39 | 40 | for i := 0; i < 10; i++ { 41 | storage.compact() 42 | if compactedOffset != uint32(storage.offset) { 43 | t.Log("compactedOffset != stored.offset") 44 | t.FailNow() 45 | } 46 | } 47 | fi, err := storage.descriptor.Stat() 48 | if err != nil { 49 | t.Log(err) 50 | t.FailNow() 51 | } 52 | if uint32(fi.Size()) != compactedOffset { 53 | t.Log("size !+ compactedOffset") 54 | t.FailNow() 55 | 56 | } 57 | os.RemoveAll(path) 58 | } 59 | 60 | type Thing struct { 61 | data []byte 62 | allocSize uint32 63 | } 64 | 65 | func NewThing() *Thing { 66 | size := mr.Int31n(10240) + 1024 67 | allocSize := size * 2 68 | return &Thing{ 69 | allocSize: uint32(allocSize), 70 | } 71 | } 72 | 73 | func (t *Thing) mutate() { 74 | t.data = make([]byte, (mr.Int31n(int32(t.allocSize)))) 75 | mr.Read(t.data) 76 | } 77 | 78 | type Corruptor struct { 79 | storage *StoreItem 80 | things map[uint64]*Thing 81 | progressA string 82 | } 83 | 84 | func NewCorruptor(n int, storage *StoreItem) *Corruptor { 85 | return &Corruptor{ 86 | storage: storage, 87 | things: map[uint64]*Thing{}, 88 | progressA: fmt.Sprintf("[c: %d]", n), 89 | } 90 | } 91 | 92 | func (c *Corruptor) Smash(done chan bool) { 93 | bytesSoFar := 0 94 | modifiedSoFar := 0 95 | for i := 0; i < 100; i++ { 96 | t := NewThing() 97 | t.mutate() 98 | off, err := c.storage.append(t.allocSize, t.data) 99 | bytesSoFar += len(t.data) 100 | if err != nil { 101 | panic(err) 102 | } 103 | c.things[off] = t 104 | 105 | for off, t := range c.things { 106 | data, err := c.storage.read(off) 107 | if err != nil { 108 | panic(err) 109 | } 110 | 111 | if !bytes.Equal(data, t.data) { 112 | panic("not equals") 113 | } 114 | t.mutate() 115 | modifiedSoFar += len(t.data) 116 | err = c.storage.modify(off, 0, t.data, true) 117 | if err != nil { 118 | panic(err) 119 | } 120 | } 121 | if i%10 == 0 { 122 | fmt.Printf("%s: %d, bytesSoFar: %.2fMB, modifiedSoFar: %.2fMB\n", c.progressA, i, float32(bytesSoFar)/1024/1024, float32(modifiedSoFar)/1024/1024) 123 | } 124 | } 125 | 126 | done <- true 127 | } 128 | 129 | func (c *Corruptor) Test(relocationMap map[uint64]uint64) { 130 | i := 0 131 | for off, t := range c.things { 132 | 133 | data, err := c.storage.read(relocationMap[off]) 134 | if err != nil { 135 | panic(err) 136 | } 137 | 138 | if !bytes.Equal(data, t.data) { 139 | panic("not equals") 140 | } 141 | i++ 142 | if i%100 == 0 { 143 | fmt.Printf("[sample]%s: offset %d moved to %d after compaction\n", c.progressA, off, relocationMap[off]) 144 | } 145 | } 146 | } 147 | 148 | func TestForever(t *testing.T) { 149 | path := path.Join(os.TempDir(), "smash") 150 | os.RemoveAll(path) 151 | 152 | for k := 0; k < 10; k++ { 153 | done := make(chan bool, 1) 154 | n := 100 155 | storage := NewStorage(path) 156 | corruptors := []*Corruptor{} 157 | for i := 0; i < n; i++ { 158 | c := NewCorruptor(i, storage) 159 | corruptors = append(corruptors, c) 160 | go c.Smash(done) 161 | } 162 | i := 0 163 | for { 164 | <-done 165 | i++ 166 | if i == n { 167 | break 168 | } 169 | } 170 | 171 | relocationMap, err := storage.compact() 172 | if err != nil { 173 | panic(err) 174 | } 175 | 176 | for _, c := range corruptors { 177 | c.Test(relocationMap) 178 | } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /input.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jackdoe/rochefort/9f0de5d0f3e06f8279ddd60d157dc0c117d79de3/input.pb -------------------------------------------------------------------------------- /input.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-gogo. DO NOT EDIT. 2 | // source: input.proto 3 | 4 | /* 5 | Package main is a generated protocol buffer package. 6 | 7 | It is generated from these files: 8 | input.proto 9 | 10 | It has these top-level messages: 11 | Modify 12 | Append 13 | AppendInput 14 | AppendOutput 15 | NamespaceInput 16 | SuccessOutput 17 | Get 18 | GetInput 19 | ScanOutput 20 | GetOutput 21 | StatsOutput 22 | */ 23 | package main 24 | 25 | import proto "github.com/gogo/protobuf/proto" 26 | import fmt "fmt" 27 | import math "math" 28 | 29 | import bytes "bytes" 30 | 31 | import strings "strings" 32 | import reflect "reflect" 33 | import sortkeys "github.com/gogo/protobuf/sortkeys" 34 | 35 | import io "io" 36 | 37 | // Reference imports to suppress errors if they are not otherwise used. 38 | var _ = proto.Marshal 39 | var _ = fmt.Errorf 40 | var _ = math.Inf 41 | 42 | // This is a compile-time assertion to ensure that this generated file 43 | // is compatible with the proto package it is being compiled against. 44 | // A compilation error at this line likely means your copy of the 45 | // proto package needs to be updated. 46 | const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package 47 | 48 | type Modify struct { 49 | Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` 50 | Pos int32 `protobuf:"varint,2,opt,name=pos,proto3" json:"pos,omitempty"` 51 | Offset uint64 `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"` 52 | Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` 53 | ResetLength bool `protobuf:"varint,5,opt,name=resetLength,proto3" json:"resetLength,omitempty"` 54 | } 55 | 56 | func (m *Modify) Reset() { *m = Modify{} } 57 | func (*Modify) ProtoMessage() {} 58 | func (*Modify) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{0} } 59 | 60 | func (m *Modify) GetNamespace() string { 61 | if m != nil { 62 | return m.Namespace 63 | } 64 | return "" 65 | } 66 | 67 | func (m *Modify) GetPos() int32 { 68 | if m != nil { 69 | return m.Pos 70 | } 71 | return 0 72 | } 73 | 74 | func (m *Modify) GetOffset() uint64 { 75 | if m != nil { 76 | return m.Offset 77 | } 78 | return 0 79 | } 80 | 81 | func (m *Modify) GetData() []byte { 82 | if m != nil { 83 | return m.Data 84 | } 85 | return nil 86 | } 87 | 88 | func (m *Modify) GetResetLength() bool { 89 | if m != nil { 90 | return m.ResetLength 91 | } 92 | return false 93 | } 94 | 95 | type Append struct { 96 | Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` 97 | AllocSize uint32 `protobuf:"varint,2,opt,name=allocSize,proto3" json:"allocSize,omitempty"` 98 | Tags []string `protobuf:"bytes,4,rep,name=tags" json:"tags,omitempty"` 99 | Data []byte `protobuf:"bytes,5,opt,name=data,proto3" json:"data,omitempty"` 100 | } 101 | 102 | func (m *Append) Reset() { *m = Append{} } 103 | func (*Append) ProtoMessage() {} 104 | func (*Append) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{1} } 105 | 106 | func (m *Append) GetNamespace() string { 107 | if m != nil { 108 | return m.Namespace 109 | } 110 | return "" 111 | } 112 | 113 | func (m *Append) GetAllocSize() uint32 { 114 | if m != nil { 115 | return m.AllocSize 116 | } 117 | return 0 118 | } 119 | 120 | func (m *Append) GetTags() []string { 121 | if m != nil { 122 | return m.Tags 123 | } 124 | return nil 125 | } 126 | 127 | func (m *Append) GetData() []byte { 128 | if m != nil { 129 | return m.Data 130 | } 131 | return nil 132 | } 133 | 134 | type AppendInput struct { 135 | AppendPayload []*Append `protobuf:"bytes,1,rep,name=appendPayload" json:"appendPayload,omitempty"` 136 | ModifyPayload []*Modify `protobuf:"bytes,2,rep,name=modifyPayload" json:"modifyPayload,omitempty"` 137 | } 138 | 139 | func (m *AppendInput) Reset() { *m = AppendInput{} } 140 | func (*AppendInput) ProtoMessage() {} 141 | func (*AppendInput) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{2} } 142 | 143 | func (m *AppendInput) GetAppendPayload() []*Append { 144 | if m != nil { 145 | return m.AppendPayload 146 | } 147 | return nil 148 | } 149 | 150 | func (m *AppendInput) GetModifyPayload() []*Modify { 151 | if m != nil { 152 | return m.ModifyPayload 153 | } 154 | return nil 155 | } 156 | 157 | type AppendOutput struct { 158 | Offset []uint64 `protobuf:"varint,1,rep,packed,name=offset" json:"offset,omitempty"` 159 | ModifiedCount uint64 `protobuf:"varint,2,opt,name=modifiedCount,proto3" json:"modifiedCount,omitempty"` 160 | } 161 | 162 | func (m *AppendOutput) Reset() { *m = AppendOutput{} } 163 | func (*AppendOutput) ProtoMessage() {} 164 | func (*AppendOutput) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{3} } 165 | 166 | func (m *AppendOutput) GetOffset() []uint64 { 167 | if m != nil { 168 | return m.Offset 169 | } 170 | return nil 171 | } 172 | 173 | func (m *AppendOutput) GetModifiedCount() uint64 { 174 | if m != nil { 175 | return m.ModifiedCount 176 | } 177 | return 0 178 | } 179 | 180 | type NamespaceInput struct { 181 | Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` 182 | } 183 | 184 | func (m *NamespaceInput) Reset() { *m = NamespaceInput{} } 185 | func (*NamespaceInput) ProtoMessage() {} 186 | func (*NamespaceInput) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{4} } 187 | 188 | func (m *NamespaceInput) GetNamespace() string { 189 | if m != nil { 190 | return m.Namespace 191 | } 192 | return "" 193 | } 194 | 195 | type SuccessOutput struct { 196 | Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` 197 | } 198 | 199 | func (m *SuccessOutput) Reset() { *m = SuccessOutput{} } 200 | func (*SuccessOutput) ProtoMessage() {} 201 | func (*SuccessOutput) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{5} } 202 | 203 | func (m *SuccessOutput) GetSuccess() bool { 204 | if m != nil { 205 | return m.Success 206 | } 207 | return false 208 | } 209 | 210 | type Get struct { 211 | Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` 212 | Offset uint64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` 213 | } 214 | 215 | func (m *Get) Reset() { *m = Get{} } 216 | func (*Get) ProtoMessage() {} 217 | func (*Get) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{6} } 218 | 219 | func (m *Get) GetNamespace() string { 220 | if m != nil { 221 | return m.Namespace 222 | } 223 | return "" 224 | } 225 | 226 | func (m *Get) GetOffset() uint64 { 227 | if m != nil { 228 | return m.Offset 229 | } 230 | return 0 231 | } 232 | 233 | type GetInput struct { 234 | GetPayload []*Get `protobuf:"bytes,1,rep,name=getPayload" json:"getPayload,omitempty"` 235 | } 236 | 237 | func (m *GetInput) Reset() { *m = GetInput{} } 238 | func (*GetInput) ProtoMessage() {} 239 | func (*GetInput) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{7} } 240 | 241 | func (m *GetInput) GetGetPayload() []*Get { 242 | if m != nil { 243 | return m.GetPayload 244 | } 245 | return nil 246 | } 247 | 248 | type ScanOutput struct { 249 | Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` 250 | Offset uint64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` 251 | } 252 | 253 | func (m *ScanOutput) Reset() { *m = ScanOutput{} } 254 | func (*ScanOutput) ProtoMessage() {} 255 | func (*ScanOutput) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{8} } 256 | 257 | func (m *ScanOutput) GetData() []byte { 258 | if m != nil { 259 | return m.Data 260 | } 261 | return nil 262 | } 263 | 264 | func (m *ScanOutput) GetOffset() uint64 { 265 | if m != nil { 266 | return m.Offset 267 | } 268 | return 0 269 | } 270 | 271 | type GetOutput struct { 272 | Data [][]byte `protobuf:"bytes,1,rep,name=data" json:"data,omitempty"` 273 | } 274 | 275 | func (m *GetOutput) Reset() { *m = GetOutput{} } 276 | func (*GetOutput) ProtoMessage() {} 277 | func (*GetOutput) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{9} } 278 | 279 | func (m *GetOutput) GetData() [][]byte { 280 | if m != nil { 281 | return m.Data 282 | } 283 | return nil 284 | } 285 | 286 | type StatsOutput struct { 287 | Tags map[string]uint64 `protobuf:"bytes,1,rep,name=tags" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` 288 | Offset uint64 `protobuf:"varint,2,opt,name=offset,proto3" json:"offset,omitempty"` 289 | File string `protobuf:"bytes,3,opt,name=file,proto3" json:"file,omitempty"` 290 | } 291 | 292 | func (m *StatsOutput) Reset() { *m = StatsOutput{} } 293 | func (*StatsOutput) ProtoMessage() {} 294 | func (*StatsOutput) Descriptor() ([]byte, []int) { return fileDescriptorInput, []int{10} } 295 | 296 | func (m *StatsOutput) GetTags() map[string]uint64 { 297 | if m != nil { 298 | return m.Tags 299 | } 300 | return nil 301 | } 302 | 303 | func (m *StatsOutput) GetOffset() uint64 { 304 | if m != nil { 305 | return m.Offset 306 | } 307 | return 0 308 | } 309 | 310 | func (m *StatsOutput) GetFile() string { 311 | if m != nil { 312 | return m.File 313 | } 314 | return "" 315 | } 316 | 317 | func init() { 318 | proto.RegisterType((*Modify)(nil), "main.Modify") 319 | proto.RegisterType((*Append)(nil), "main.Append") 320 | proto.RegisterType((*AppendInput)(nil), "main.AppendInput") 321 | proto.RegisterType((*AppendOutput)(nil), "main.AppendOutput") 322 | proto.RegisterType((*NamespaceInput)(nil), "main.NamespaceInput") 323 | proto.RegisterType((*SuccessOutput)(nil), "main.SuccessOutput") 324 | proto.RegisterType((*Get)(nil), "main.Get") 325 | proto.RegisterType((*GetInput)(nil), "main.GetInput") 326 | proto.RegisterType((*ScanOutput)(nil), "main.ScanOutput") 327 | proto.RegisterType((*GetOutput)(nil), "main.GetOutput") 328 | proto.RegisterType((*StatsOutput)(nil), "main.StatsOutput") 329 | } 330 | func (this *Modify) Equal(that interface{}) bool { 331 | if that == nil { 332 | return this == nil 333 | } 334 | 335 | that1, ok := that.(*Modify) 336 | if !ok { 337 | that2, ok := that.(Modify) 338 | if ok { 339 | that1 = &that2 340 | } else { 341 | return false 342 | } 343 | } 344 | if that1 == nil { 345 | return this == nil 346 | } else if this == nil { 347 | return false 348 | } 349 | if this.Namespace != that1.Namespace { 350 | return false 351 | } 352 | if this.Pos != that1.Pos { 353 | return false 354 | } 355 | if this.Offset != that1.Offset { 356 | return false 357 | } 358 | if !bytes.Equal(this.Data, that1.Data) { 359 | return false 360 | } 361 | if this.ResetLength != that1.ResetLength { 362 | return false 363 | } 364 | return true 365 | } 366 | func (this *Append) Equal(that interface{}) bool { 367 | if that == nil { 368 | return this == nil 369 | } 370 | 371 | that1, ok := that.(*Append) 372 | if !ok { 373 | that2, ok := that.(Append) 374 | if ok { 375 | that1 = &that2 376 | } else { 377 | return false 378 | } 379 | } 380 | if that1 == nil { 381 | return this == nil 382 | } else if this == nil { 383 | return false 384 | } 385 | if this.Namespace != that1.Namespace { 386 | return false 387 | } 388 | if this.AllocSize != that1.AllocSize { 389 | return false 390 | } 391 | if len(this.Tags) != len(that1.Tags) { 392 | return false 393 | } 394 | for i := range this.Tags { 395 | if this.Tags[i] != that1.Tags[i] { 396 | return false 397 | } 398 | } 399 | if !bytes.Equal(this.Data, that1.Data) { 400 | return false 401 | } 402 | return true 403 | } 404 | func (this *AppendInput) Equal(that interface{}) bool { 405 | if that == nil { 406 | return this == nil 407 | } 408 | 409 | that1, ok := that.(*AppendInput) 410 | if !ok { 411 | that2, ok := that.(AppendInput) 412 | if ok { 413 | that1 = &that2 414 | } else { 415 | return false 416 | } 417 | } 418 | if that1 == nil { 419 | return this == nil 420 | } else if this == nil { 421 | return false 422 | } 423 | if len(this.AppendPayload) != len(that1.AppendPayload) { 424 | return false 425 | } 426 | for i := range this.AppendPayload { 427 | if !this.AppendPayload[i].Equal(that1.AppendPayload[i]) { 428 | return false 429 | } 430 | } 431 | if len(this.ModifyPayload) != len(that1.ModifyPayload) { 432 | return false 433 | } 434 | for i := range this.ModifyPayload { 435 | if !this.ModifyPayload[i].Equal(that1.ModifyPayload[i]) { 436 | return false 437 | } 438 | } 439 | return true 440 | } 441 | func (this *AppendOutput) Equal(that interface{}) bool { 442 | if that == nil { 443 | return this == nil 444 | } 445 | 446 | that1, ok := that.(*AppendOutput) 447 | if !ok { 448 | that2, ok := that.(AppendOutput) 449 | if ok { 450 | that1 = &that2 451 | } else { 452 | return false 453 | } 454 | } 455 | if that1 == nil { 456 | return this == nil 457 | } else if this == nil { 458 | return false 459 | } 460 | if len(this.Offset) != len(that1.Offset) { 461 | return false 462 | } 463 | for i := range this.Offset { 464 | if this.Offset[i] != that1.Offset[i] { 465 | return false 466 | } 467 | } 468 | if this.ModifiedCount != that1.ModifiedCount { 469 | return false 470 | } 471 | return true 472 | } 473 | func (this *NamespaceInput) Equal(that interface{}) bool { 474 | if that == nil { 475 | return this == nil 476 | } 477 | 478 | that1, ok := that.(*NamespaceInput) 479 | if !ok { 480 | that2, ok := that.(NamespaceInput) 481 | if ok { 482 | that1 = &that2 483 | } else { 484 | return false 485 | } 486 | } 487 | if that1 == nil { 488 | return this == nil 489 | } else if this == nil { 490 | return false 491 | } 492 | if this.Namespace != that1.Namespace { 493 | return false 494 | } 495 | return true 496 | } 497 | func (this *SuccessOutput) Equal(that interface{}) bool { 498 | if that == nil { 499 | return this == nil 500 | } 501 | 502 | that1, ok := that.(*SuccessOutput) 503 | if !ok { 504 | that2, ok := that.(SuccessOutput) 505 | if ok { 506 | that1 = &that2 507 | } else { 508 | return false 509 | } 510 | } 511 | if that1 == nil { 512 | return this == nil 513 | } else if this == nil { 514 | return false 515 | } 516 | if this.Success != that1.Success { 517 | return false 518 | } 519 | return true 520 | } 521 | func (this *Get) Equal(that interface{}) bool { 522 | if that == nil { 523 | return this == nil 524 | } 525 | 526 | that1, ok := that.(*Get) 527 | if !ok { 528 | that2, ok := that.(Get) 529 | if ok { 530 | that1 = &that2 531 | } else { 532 | return false 533 | } 534 | } 535 | if that1 == nil { 536 | return this == nil 537 | } else if this == nil { 538 | return false 539 | } 540 | if this.Namespace != that1.Namespace { 541 | return false 542 | } 543 | if this.Offset != that1.Offset { 544 | return false 545 | } 546 | return true 547 | } 548 | func (this *GetInput) Equal(that interface{}) bool { 549 | if that == nil { 550 | return this == nil 551 | } 552 | 553 | that1, ok := that.(*GetInput) 554 | if !ok { 555 | that2, ok := that.(GetInput) 556 | if ok { 557 | that1 = &that2 558 | } else { 559 | return false 560 | } 561 | } 562 | if that1 == nil { 563 | return this == nil 564 | } else if this == nil { 565 | return false 566 | } 567 | if len(this.GetPayload) != len(that1.GetPayload) { 568 | return false 569 | } 570 | for i := range this.GetPayload { 571 | if !this.GetPayload[i].Equal(that1.GetPayload[i]) { 572 | return false 573 | } 574 | } 575 | return true 576 | } 577 | func (this *ScanOutput) Equal(that interface{}) bool { 578 | if that == nil { 579 | return this == nil 580 | } 581 | 582 | that1, ok := that.(*ScanOutput) 583 | if !ok { 584 | that2, ok := that.(ScanOutput) 585 | if ok { 586 | that1 = &that2 587 | } else { 588 | return false 589 | } 590 | } 591 | if that1 == nil { 592 | return this == nil 593 | } else if this == nil { 594 | return false 595 | } 596 | if !bytes.Equal(this.Data, that1.Data) { 597 | return false 598 | } 599 | if this.Offset != that1.Offset { 600 | return false 601 | } 602 | return true 603 | } 604 | func (this *GetOutput) Equal(that interface{}) bool { 605 | if that == nil { 606 | return this == nil 607 | } 608 | 609 | that1, ok := that.(*GetOutput) 610 | if !ok { 611 | that2, ok := that.(GetOutput) 612 | if ok { 613 | that1 = &that2 614 | } else { 615 | return false 616 | } 617 | } 618 | if that1 == nil { 619 | return this == nil 620 | } else if this == nil { 621 | return false 622 | } 623 | if len(this.Data) != len(that1.Data) { 624 | return false 625 | } 626 | for i := range this.Data { 627 | if !bytes.Equal(this.Data[i], that1.Data[i]) { 628 | return false 629 | } 630 | } 631 | return true 632 | } 633 | func (this *StatsOutput) Equal(that interface{}) bool { 634 | if that == nil { 635 | return this == nil 636 | } 637 | 638 | that1, ok := that.(*StatsOutput) 639 | if !ok { 640 | that2, ok := that.(StatsOutput) 641 | if ok { 642 | that1 = &that2 643 | } else { 644 | return false 645 | } 646 | } 647 | if that1 == nil { 648 | return this == nil 649 | } else if this == nil { 650 | return false 651 | } 652 | if len(this.Tags) != len(that1.Tags) { 653 | return false 654 | } 655 | for i := range this.Tags { 656 | if this.Tags[i] != that1.Tags[i] { 657 | return false 658 | } 659 | } 660 | if this.Offset != that1.Offset { 661 | return false 662 | } 663 | if this.File != that1.File { 664 | return false 665 | } 666 | return true 667 | } 668 | func (this *Modify) GoString() string { 669 | if this == nil { 670 | return "nil" 671 | } 672 | s := make([]string, 0, 9) 673 | s = append(s, "&main.Modify{") 674 | s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") 675 | s = append(s, "Pos: "+fmt.Sprintf("%#v", this.Pos)+",\n") 676 | s = append(s, "Offset: "+fmt.Sprintf("%#v", this.Offset)+",\n") 677 | s = append(s, "Data: "+fmt.Sprintf("%#v", this.Data)+",\n") 678 | s = append(s, "ResetLength: "+fmt.Sprintf("%#v", this.ResetLength)+",\n") 679 | s = append(s, "}") 680 | return strings.Join(s, "") 681 | } 682 | func (this *Append) GoString() string { 683 | if this == nil { 684 | return "nil" 685 | } 686 | s := make([]string, 0, 8) 687 | s = append(s, "&main.Append{") 688 | s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") 689 | s = append(s, "AllocSize: "+fmt.Sprintf("%#v", this.AllocSize)+",\n") 690 | s = append(s, "Tags: "+fmt.Sprintf("%#v", this.Tags)+",\n") 691 | s = append(s, "Data: "+fmt.Sprintf("%#v", this.Data)+",\n") 692 | s = append(s, "}") 693 | return strings.Join(s, "") 694 | } 695 | func (this *AppendInput) GoString() string { 696 | if this == nil { 697 | return "nil" 698 | } 699 | s := make([]string, 0, 6) 700 | s = append(s, "&main.AppendInput{") 701 | if this.AppendPayload != nil { 702 | s = append(s, "AppendPayload: "+fmt.Sprintf("%#v", this.AppendPayload)+",\n") 703 | } 704 | if this.ModifyPayload != nil { 705 | s = append(s, "ModifyPayload: "+fmt.Sprintf("%#v", this.ModifyPayload)+",\n") 706 | } 707 | s = append(s, "}") 708 | return strings.Join(s, "") 709 | } 710 | func (this *AppendOutput) GoString() string { 711 | if this == nil { 712 | return "nil" 713 | } 714 | s := make([]string, 0, 6) 715 | s = append(s, "&main.AppendOutput{") 716 | s = append(s, "Offset: "+fmt.Sprintf("%#v", this.Offset)+",\n") 717 | s = append(s, "ModifiedCount: "+fmt.Sprintf("%#v", this.ModifiedCount)+",\n") 718 | s = append(s, "}") 719 | return strings.Join(s, "") 720 | } 721 | func (this *NamespaceInput) GoString() string { 722 | if this == nil { 723 | return "nil" 724 | } 725 | s := make([]string, 0, 5) 726 | s = append(s, "&main.NamespaceInput{") 727 | s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") 728 | s = append(s, "}") 729 | return strings.Join(s, "") 730 | } 731 | func (this *SuccessOutput) GoString() string { 732 | if this == nil { 733 | return "nil" 734 | } 735 | s := make([]string, 0, 5) 736 | s = append(s, "&main.SuccessOutput{") 737 | s = append(s, "Success: "+fmt.Sprintf("%#v", this.Success)+",\n") 738 | s = append(s, "}") 739 | return strings.Join(s, "") 740 | } 741 | func (this *Get) GoString() string { 742 | if this == nil { 743 | return "nil" 744 | } 745 | s := make([]string, 0, 6) 746 | s = append(s, "&main.Get{") 747 | s = append(s, "Namespace: "+fmt.Sprintf("%#v", this.Namespace)+",\n") 748 | s = append(s, "Offset: "+fmt.Sprintf("%#v", this.Offset)+",\n") 749 | s = append(s, "}") 750 | return strings.Join(s, "") 751 | } 752 | func (this *GetInput) GoString() string { 753 | if this == nil { 754 | return "nil" 755 | } 756 | s := make([]string, 0, 5) 757 | s = append(s, "&main.GetInput{") 758 | if this.GetPayload != nil { 759 | s = append(s, "GetPayload: "+fmt.Sprintf("%#v", this.GetPayload)+",\n") 760 | } 761 | s = append(s, "}") 762 | return strings.Join(s, "") 763 | } 764 | func (this *ScanOutput) GoString() string { 765 | if this == nil { 766 | return "nil" 767 | } 768 | s := make([]string, 0, 6) 769 | s = append(s, "&main.ScanOutput{") 770 | s = append(s, "Data: "+fmt.Sprintf("%#v", this.Data)+",\n") 771 | s = append(s, "Offset: "+fmt.Sprintf("%#v", this.Offset)+",\n") 772 | s = append(s, "}") 773 | return strings.Join(s, "") 774 | } 775 | func (this *GetOutput) GoString() string { 776 | if this == nil { 777 | return "nil" 778 | } 779 | s := make([]string, 0, 5) 780 | s = append(s, "&main.GetOutput{") 781 | s = append(s, "Data: "+fmt.Sprintf("%#v", this.Data)+",\n") 782 | s = append(s, "}") 783 | return strings.Join(s, "") 784 | } 785 | func (this *StatsOutput) GoString() string { 786 | if this == nil { 787 | return "nil" 788 | } 789 | s := make([]string, 0, 7) 790 | s = append(s, "&main.StatsOutput{") 791 | keysForTags := make([]string, 0, len(this.Tags)) 792 | for k, _ := range this.Tags { 793 | keysForTags = append(keysForTags, k) 794 | } 795 | sortkeys.Strings(keysForTags) 796 | mapStringForTags := "map[string]uint64{" 797 | for _, k := range keysForTags { 798 | mapStringForTags += fmt.Sprintf("%#v: %#v,", k, this.Tags[k]) 799 | } 800 | mapStringForTags += "}" 801 | if this.Tags != nil { 802 | s = append(s, "Tags: "+mapStringForTags+",\n") 803 | } 804 | s = append(s, "Offset: "+fmt.Sprintf("%#v", this.Offset)+",\n") 805 | s = append(s, "File: "+fmt.Sprintf("%#v", this.File)+",\n") 806 | s = append(s, "}") 807 | return strings.Join(s, "") 808 | } 809 | func valueToGoStringInput(v interface{}, typ string) string { 810 | rv := reflect.ValueOf(v) 811 | if rv.IsNil() { 812 | return "nil" 813 | } 814 | pv := reflect.Indirect(rv).Interface() 815 | return fmt.Sprintf("func(v %v) *%v { return &v } ( %#v )", typ, typ, pv) 816 | } 817 | func (m *Modify) Marshal() (dAtA []byte, err error) { 818 | size := m.Size() 819 | dAtA = make([]byte, size) 820 | n, err := m.MarshalTo(dAtA) 821 | if err != nil { 822 | return nil, err 823 | } 824 | return dAtA[:n], nil 825 | } 826 | 827 | func (m *Modify) MarshalTo(dAtA []byte) (int, error) { 828 | var i int 829 | _ = i 830 | var l int 831 | _ = l 832 | if len(m.Namespace) > 0 { 833 | dAtA[i] = 0xa 834 | i++ 835 | i = encodeVarintInput(dAtA, i, uint64(len(m.Namespace))) 836 | i += copy(dAtA[i:], m.Namespace) 837 | } 838 | if m.Pos != 0 { 839 | dAtA[i] = 0x10 840 | i++ 841 | i = encodeVarintInput(dAtA, i, uint64(m.Pos)) 842 | } 843 | if m.Offset != 0 { 844 | dAtA[i] = 0x18 845 | i++ 846 | i = encodeVarintInput(dAtA, i, uint64(m.Offset)) 847 | } 848 | if len(m.Data) > 0 { 849 | dAtA[i] = 0x22 850 | i++ 851 | i = encodeVarintInput(dAtA, i, uint64(len(m.Data))) 852 | i += copy(dAtA[i:], m.Data) 853 | } 854 | if m.ResetLength { 855 | dAtA[i] = 0x28 856 | i++ 857 | if m.ResetLength { 858 | dAtA[i] = 1 859 | } else { 860 | dAtA[i] = 0 861 | } 862 | i++ 863 | } 864 | return i, nil 865 | } 866 | 867 | func (m *Append) Marshal() (dAtA []byte, err error) { 868 | size := m.Size() 869 | dAtA = make([]byte, size) 870 | n, err := m.MarshalTo(dAtA) 871 | if err != nil { 872 | return nil, err 873 | } 874 | return dAtA[:n], nil 875 | } 876 | 877 | func (m *Append) MarshalTo(dAtA []byte) (int, error) { 878 | var i int 879 | _ = i 880 | var l int 881 | _ = l 882 | if len(m.Namespace) > 0 { 883 | dAtA[i] = 0xa 884 | i++ 885 | i = encodeVarintInput(dAtA, i, uint64(len(m.Namespace))) 886 | i += copy(dAtA[i:], m.Namespace) 887 | } 888 | if m.AllocSize != 0 { 889 | dAtA[i] = 0x10 890 | i++ 891 | i = encodeVarintInput(dAtA, i, uint64(m.AllocSize)) 892 | } 893 | if len(m.Tags) > 0 { 894 | for _, s := range m.Tags { 895 | dAtA[i] = 0x22 896 | i++ 897 | l = len(s) 898 | for l >= 1<<7 { 899 | dAtA[i] = uint8(uint64(l)&0x7f | 0x80) 900 | l >>= 7 901 | i++ 902 | } 903 | dAtA[i] = uint8(l) 904 | i++ 905 | i += copy(dAtA[i:], s) 906 | } 907 | } 908 | if len(m.Data) > 0 { 909 | dAtA[i] = 0x2a 910 | i++ 911 | i = encodeVarintInput(dAtA, i, uint64(len(m.Data))) 912 | i += copy(dAtA[i:], m.Data) 913 | } 914 | return i, nil 915 | } 916 | 917 | func (m *AppendInput) Marshal() (dAtA []byte, err error) { 918 | size := m.Size() 919 | dAtA = make([]byte, size) 920 | n, err := m.MarshalTo(dAtA) 921 | if err != nil { 922 | return nil, err 923 | } 924 | return dAtA[:n], nil 925 | } 926 | 927 | func (m *AppendInput) MarshalTo(dAtA []byte) (int, error) { 928 | var i int 929 | _ = i 930 | var l int 931 | _ = l 932 | if len(m.AppendPayload) > 0 { 933 | for _, msg := range m.AppendPayload { 934 | dAtA[i] = 0xa 935 | i++ 936 | i = encodeVarintInput(dAtA, i, uint64(msg.Size())) 937 | n, err := msg.MarshalTo(dAtA[i:]) 938 | if err != nil { 939 | return 0, err 940 | } 941 | i += n 942 | } 943 | } 944 | if len(m.ModifyPayload) > 0 { 945 | for _, msg := range m.ModifyPayload { 946 | dAtA[i] = 0x12 947 | i++ 948 | i = encodeVarintInput(dAtA, i, uint64(msg.Size())) 949 | n, err := msg.MarshalTo(dAtA[i:]) 950 | if err != nil { 951 | return 0, err 952 | } 953 | i += n 954 | } 955 | } 956 | return i, nil 957 | } 958 | 959 | func (m *AppendOutput) Marshal() (dAtA []byte, err error) { 960 | size := m.Size() 961 | dAtA = make([]byte, size) 962 | n, err := m.MarshalTo(dAtA) 963 | if err != nil { 964 | return nil, err 965 | } 966 | return dAtA[:n], nil 967 | } 968 | 969 | func (m *AppendOutput) MarshalTo(dAtA []byte) (int, error) { 970 | var i int 971 | _ = i 972 | var l int 973 | _ = l 974 | if len(m.Offset) > 0 { 975 | dAtA2 := make([]byte, len(m.Offset)*10) 976 | var j1 int 977 | for _, num := range m.Offset { 978 | for num >= 1<<7 { 979 | dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) 980 | num >>= 7 981 | j1++ 982 | } 983 | dAtA2[j1] = uint8(num) 984 | j1++ 985 | } 986 | dAtA[i] = 0xa 987 | i++ 988 | i = encodeVarintInput(dAtA, i, uint64(j1)) 989 | i += copy(dAtA[i:], dAtA2[:j1]) 990 | } 991 | if m.ModifiedCount != 0 { 992 | dAtA[i] = 0x10 993 | i++ 994 | i = encodeVarintInput(dAtA, i, uint64(m.ModifiedCount)) 995 | } 996 | return i, nil 997 | } 998 | 999 | func (m *NamespaceInput) Marshal() (dAtA []byte, err error) { 1000 | size := m.Size() 1001 | dAtA = make([]byte, size) 1002 | n, err := m.MarshalTo(dAtA) 1003 | if err != nil { 1004 | return nil, err 1005 | } 1006 | return dAtA[:n], nil 1007 | } 1008 | 1009 | func (m *NamespaceInput) MarshalTo(dAtA []byte) (int, error) { 1010 | var i int 1011 | _ = i 1012 | var l int 1013 | _ = l 1014 | if len(m.Namespace) > 0 { 1015 | dAtA[i] = 0xa 1016 | i++ 1017 | i = encodeVarintInput(dAtA, i, uint64(len(m.Namespace))) 1018 | i += copy(dAtA[i:], m.Namespace) 1019 | } 1020 | return i, nil 1021 | } 1022 | 1023 | func (m *SuccessOutput) Marshal() (dAtA []byte, err error) { 1024 | size := m.Size() 1025 | dAtA = make([]byte, size) 1026 | n, err := m.MarshalTo(dAtA) 1027 | if err != nil { 1028 | return nil, err 1029 | } 1030 | return dAtA[:n], nil 1031 | } 1032 | 1033 | func (m *SuccessOutput) MarshalTo(dAtA []byte) (int, error) { 1034 | var i int 1035 | _ = i 1036 | var l int 1037 | _ = l 1038 | if m.Success { 1039 | dAtA[i] = 0x8 1040 | i++ 1041 | if m.Success { 1042 | dAtA[i] = 1 1043 | } else { 1044 | dAtA[i] = 0 1045 | } 1046 | i++ 1047 | } 1048 | return i, nil 1049 | } 1050 | 1051 | func (m *Get) Marshal() (dAtA []byte, err error) { 1052 | size := m.Size() 1053 | dAtA = make([]byte, size) 1054 | n, err := m.MarshalTo(dAtA) 1055 | if err != nil { 1056 | return nil, err 1057 | } 1058 | return dAtA[:n], nil 1059 | } 1060 | 1061 | func (m *Get) MarshalTo(dAtA []byte) (int, error) { 1062 | var i int 1063 | _ = i 1064 | var l int 1065 | _ = l 1066 | if len(m.Namespace) > 0 { 1067 | dAtA[i] = 0xa 1068 | i++ 1069 | i = encodeVarintInput(dAtA, i, uint64(len(m.Namespace))) 1070 | i += copy(dAtA[i:], m.Namespace) 1071 | } 1072 | if m.Offset != 0 { 1073 | dAtA[i] = 0x10 1074 | i++ 1075 | i = encodeVarintInput(dAtA, i, uint64(m.Offset)) 1076 | } 1077 | return i, nil 1078 | } 1079 | 1080 | func (m *GetInput) Marshal() (dAtA []byte, err error) { 1081 | size := m.Size() 1082 | dAtA = make([]byte, size) 1083 | n, err := m.MarshalTo(dAtA) 1084 | if err != nil { 1085 | return nil, err 1086 | } 1087 | return dAtA[:n], nil 1088 | } 1089 | 1090 | func (m *GetInput) MarshalTo(dAtA []byte) (int, error) { 1091 | var i int 1092 | _ = i 1093 | var l int 1094 | _ = l 1095 | if len(m.GetPayload) > 0 { 1096 | for _, msg := range m.GetPayload { 1097 | dAtA[i] = 0xa 1098 | i++ 1099 | i = encodeVarintInput(dAtA, i, uint64(msg.Size())) 1100 | n, err := msg.MarshalTo(dAtA[i:]) 1101 | if err != nil { 1102 | return 0, err 1103 | } 1104 | i += n 1105 | } 1106 | } 1107 | return i, nil 1108 | } 1109 | 1110 | func (m *ScanOutput) Marshal() (dAtA []byte, err error) { 1111 | size := m.Size() 1112 | dAtA = make([]byte, size) 1113 | n, err := m.MarshalTo(dAtA) 1114 | if err != nil { 1115 | return nil, err 1116 | } 1117 | return dAtA[:n], nil 1118 | } 1119 | 1120 | func (m *ScanOutput) MarshalTo(dAtA []byte) (int, error) { 1121 | var i int 1122 | _ = i 1123 | var l int 1124 | _ = l 1125 | if len(m.Data) > 0 { 1126 | dAtA[i] = 0xa 1127 | i++ 1128 | i = encodeVarintInput(dAtA, i, uint64(len(m.Data))) 1129 | i += copy(dAtA[i:], m.Data) 1130 | } 1131 | if m.Offset != 0 { 1132 | dAtA[i] = 0x10 1133 | i++ 1134 | i = encodeVarintInput(dAtA, i, uint64(m.Offset)) 1135 | } 1136 | return i, nil 1137 | } 1138 | 1139 | func (m *GetOutput) Marshal() (dAtA []byte, err error) { 1140 | size := m.Size() 1141 | dAtA = make([]byte, size) 1142 | n, err := m.MarshalTo(dAtA) 1143 | if err != nil { 1144 | return nil, err 1145 | } 1146 | return dAtA[:n], nil 1147 | } 1148 | 1149 | func (m *GetOutput) MarshalTo(dAtA []byte) (int, error) { 1150 | var i int 1151 | _ = i 1152 | var l int 1153 | _ = l 1154 | if len(m.Data) > 0 { 1155 | for _, b := range m.Data { 1156 | dAtA[i] = 0xa 1157 | i++ 1158 | i = encodeVarintInput(dAtA, i, uint64(len(b))) 1159 | i += copy(dAtA[i:], b) 1160 | } 1161 | } 1162 | return i, nil 1163 | } 1164 | 1165 | func (m *StatsOutput) Marshal() (dAtA []byte, err error) { 1166 | size := m.Size() 1167 | dAtA = make([]byte, size) 1168 | n, err := m.MarshalTo(dAtA) 1169 | if err != nil { 1170 | return nil, err 1171 | } 1172 | return dAtA[:n], nil 1173 | } 1174 | 1175 | func (m *StatsOutput) MarshalTo(dAtA []byte) (int, error) { 1176 | var i int 1177 | _ = i 1178 | var l int 1179 | _ = l 1180 | if len(m.Tags) > 0 { 1181 | for k, _ := range m.Tags { 1182 | dAtA[i] = 0xa 1183 | i++ 1184 | v := m.Tags[k] 1185 | mapSize := 1 + len(k) + sovInput(uint64(len(k))) + 1 + sovInput(uint64(v)) 1186 | i = encodeVarintInput(dAtA, i, uint64(mapSize)) 1187 | dAtA[i] = 0xa 1188 | i++ 1189 | i = encodeVarintInput(dAtA, i, uint64(len(k))) 1190 | i += copy(dAtA[i:], k) 1191 | dAtA[i] = 0x10 1192 | i++ 1193 | i = encodeVarintInput(dAtA, i, uint64(v)) 1194 | } 1195 | } 1196 | if m.Offset != 0 { 1197 | dAtA[i] = 0x10 1198 | i++ 1199 | i = encodeVarintInput(dAtA, i, uint64(m.Offset)) 1200 | } 1201 | if len(m.File) > 0 { 1202 | dAtA[i] = 0x1a 1203 | i++ 1204 | i = encodeVarintInput(dAtA, i, uint64(len(m.File))) 1205 | i += copy(dAtA[i:], m.File) 1206 | } 1207 | return i, nil 1208 | } 1209 | 1210 | func encodeVarintInput(dAtA []byte, offset int, v uint64) int { 1211 | for v >= 1<<7 { 1212 | dAtA[offset] = uint8(v&0x7f | 0x80) 1213 | v >>= 7 1214 | offset++ 1215 | } 1216 | dAtA[offset] = uint8(v) 1217 | return offset + 1 1218 | } 1219 | func (m *Modify) Size() (n int) { 1220 | var l int 1221 | _ = l 1222 | l = len(m.Namespace) 1223 | if l > 0 { 1224 | n += 1 + l + sovInput(uint64(l)) 1225 | } 1226 | if m.Pos != 0 { 1227 | n += 1 + sovInput(uint64(m.Pos)) 1228 | } 1229 | if m.Offset != 0 { 1230 | n += 1 + sovInput(uint64(m.Offset)) 1231 | } 1232 | l = len(m.Data) 1233 | if l > 0 { 1234 | n += 1 + l + sovInput(uint64(l)) 1235 | } 1236 | if m.ResetLength { 1237 | n += 2 1238 | } 1239 | return n 1240 | } 1241 | 1242 | func (m *Append) Size() (n int) { 1243 | var l int 1244 | _ = l 1245 | l = len(m.Namespace) 1246 | if l > 0 { 1247 | n += 1 + l + sovInput(uint64(l)) 1248 | } 1249 | if m.AllocSize != 0 { 1250 | n += 1 + sovInput(uint64(m.AllocSize)) 1251 | } 1252 | if len(m.Tags) > 0 { 1253 | for _, s := range m.Tags { 1254 | l = len(s) 1255 | n += 1 + l + sovInput(uint64(l)) 1256 | } 1257 | } 1258 | l = len(m.Data) 1259 | if l > 0 { 1260 | n += 1 + l + sovInput(uint64(l)) 1261 | } 1262 | return n 1263 | } 1264 | 1265 | func (m *AppendInput) Size() (n int) { 1266 | var l int 1267 | _ = l 1268 | if len(m.AppendPayload) > 0 { 1269 | for _, e := range m.AppendPayload { 1270 | l = e.Size() 1271 | n += 1 + l + sovInput(uint64(l)) 1272 | } 1273 | } 1274 | if len(m.ModifyPayload) > 0 { 1275 | for _, e := range m.ModifyPayload { 1276 | l = e.Size() 1277 | n += 1 + l + sovInput(uint64(l)) 1278 | } 1279 | } 1280 | return n 1281 | } 1282 | 1283 | func (m *AppendOutput) Size() (n int) { 1284 | var l int 1285 | _ = l 1286 | if len(m.Offset) > 0 { 1287 | l = 0 1288 | for _, e := range m.Offset { 1289 | l += sovInput(uint64(e)) 1290 | } 1291 | n += 1 + sovInput(uint64(l)) + l 1292 | } 1293 | if m.ModifiedCount != 0 { 1294 | n += 1 + sovInput(uint64(m.ModifiedCount)) 1295 | } 1296 | return n 1297 | } 1298 | 1299 | func (m *NamespaceInput) Size() (n int) { 1300 | var l int 1301 | _ = l 1302 | l = len(m.Namespace) 1303 | if l > 0 { 1304 | n += 1 + l + sovInput(uint64(l)) 1305 | } 1306 | return n 1307 | } 1308 | 1309 | func (m *SuccessOutput) Size() (n int) { 1310 | var l int 1311 | _ = l 1312 | if m.Success { 1313 | n += 2 1314 | } 1315 | return n 1316 | } 1317 | 1318 | func (m *Get) Size() (n int) { 1319 | var l int 1320 | _ = l 1321 | l = len(m.Namespace) 1322 | if l > 0 { 1323 | n += 1 + l + sovInput(uint64(l)) 1324 | } 1325 | if m.Offset != 0 { 1326 | n += 1 + sovInput(uint64(m.Offset)) 1327 | } 1328 | return n 1329 | } 1330 | 1331 | func (m *GetInput) Size() (n int) { 1332 | var l int 1333 | _ = l 1334 | if len(m.GetPayload) > 0 { 1335 | for _, e := range m.GetPayload { 1336 | l = e.Size() 1337 | n += 1 + l + sovInput(uint64(l)) 1338 | } 1339 | } 1340 | return n 1341 | } 1342 | 1343 | func (m *ScanOutput) Size() (n int) { 1344 | var l int 1345 | _ = l 1346 | l = len(m.Data) 1347 | if l > 0 { 1348 | n += 1 + l + sovInput(uint64(l)) 1349 | } 1350 | if m.Offset != 0 { 1351 | n += 1 + sovInput(uint64(m.Offset)) 1352 | } 1353 | return n 1354 | } 1355 | 1356 | func (m *GetOutput) Size() (n int) { 1357 | var l int 1358 | _ = l 1359 | if len(m.Data) > 0 { 1360 | for _, b := range m.Data { 1361 | l = len(b) 1362 | n += 1 + l + sovInput(uint64(l)) 1363 | } 1364 | } 1365 | return n 1366 | } 1367 | 1368 | func (m *StatsOutput) Size() (n int) { 1369 | var l int 1370 | _ = l 1371 | if len(m.Tags) > 0 { 1372 | for k, v := range m.Tags { 1373 | _ = k 1374 | _ = v 1375 | mapEntrySize := 1 + len(k) + sovInput(uint64(len(k))) + 1 + sovInput(uint64(v)) 1376 | n += mapEntrySize + 1 + sovInput(uint64(mapEntrySize)) 1377 | } 1378 | } 1379 | if m.Offset != 0 { 1380 | n += 1 + sovInput(uint64(m.Offset)) 1381 | } 1382 | l = len(m.File) 1383 | if l > 0 { 1384 | n += 1 + l + sovInput(uint64(l)) 1385 | } 1386 | return n 1387 | } 1388 | 1389 | func sovInput(x uint64) (n int) { 1390 | for { 1391 | n++ 1392 | x >>= 7 1393 | if x == 0 { 1394 | break 1395 | } 1396 | } 1397 | return n 1398 | } 1399 | func sozInput(x uint64) (n int) { 1400 | return sovInput(uint64((x << 1) ^ uint64((int64(x) >> 63)))) 1401 | } 1402 | func (this *Modify) String() string { 1403 | if this == nil { 1404 | return "nil" 1405 | } 1406 | s := strings.Join([]string{`&Modify{`, 1407 | `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, 1408 | `Pos:` + fmt.Sprintf("%v", this.Pos) + `,`, 1409 | `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, 1410 | `Data:` + fmt.Sprintf("%v", this.Data) + `,`, 1411 | `ResetLength:` + fmt.Sprintf("%v", this.ResetLength) + `,`, 1412 | `}`, 1413 | }, "") 1414 | return s 1415 | } 1416 | func (this *Append) String() string { 1417 | if this == nil { 1418 | return "nil" 1419 | } 1420 | s := strings.Join([]string{`&Append{`, 1421 | `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, 1422 | `AllocSize:` + fmt.Sprintf("%v", this.AllocSize) + `,`, 1423 | `Tags:` + fmt.Sprintf("%v", this.Tags) + `,`, 1424 | `Data:` + fmt.Sprintf("%v", this.Data) + `,`, 1425 | `}`, 1426 | }, "") 1427 | return s 1428 | } 1429 | func (this *AppendInput) String() string { 1430 | if this == nil { 1431 | return "nil" 1432 | } 1433 | s := strings.Join([]string{`&AppendInput{`, 1434 | `AppendPayload:` + strings.Replace(fmt.Sprintf("%v", this.AppendPayload), "Append", "Append", 1) + `,`, 1435 | `ModifyPayload:` + strings.Replace(fmt.Sprintf("%v", this.ModifyPayload), "Modify", "Modify", 1) + `,`, 1436 | `}`, 1437 | }, "") 1438 | return s 1439 | } 1440 | func (this *AppendOutput) String() string { 1441 | if this == nil { 1442 | return "nil" 1443 | } 1444 | s := strings.Join([]string{`&AppendOutput{`, 1445 | `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, 1446 | `ModifiedCount:` + fmt.Sprintf("%v", this.ModifiedCount) + `,`, 1447 | `}`, 1448 | }, "") 1449 | return s 1450 | } 1451 | func (this *NamespaceInput) String() string { 1452 | if this == nil { 1453 | return "nil" 1454 | } 1455 | s := strings.Join([]string{`&NamespaceInput{`, 1456 | `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, 1457 | `}`, 1458 | }, "") 1459 | return s 1460 | } 1461 | func (this *SuccessOutput) String() string { 1462 | if this == nil { 1463 | return "nil" 1464 | } 1465 | s := strings.Join([]string{`&SuccessOutput{`, 1466 | `Success:` + fmt.Sprintf("%v", this.Success) + `,`, 1467 | `}`, 1468 | }, "") 1469 | return s 1470 | } 1471 | func (this *Get) String() string { 1472 | if this == nil { 1473 | return "nil" 1474 | } 1475 | s := strings.Join([]string{`&Get{`, 1476 | `Namespace:` + fmt.Sprintf("%v", this.Namespace) + `,`, 1477 | `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, 1478 | `}`, 1479 | }, "") 1480 | return s 1481 | } 1482 | func (this *GetInput) String() string { 1483 | if this == nil { 1484 | return "nil" 1485 | } 1486 | s := strings.Join([]string{`&GetInput{`, 1487 | `GetPayload:` + strings.Replace(fmt.Sprintf("%v", this.GetPayload), "Get", "Get", 1) + `,`, 1488 | `}`, 1489 | }, "") 1490 | return s 1491 | } 1492 | func (this *ScanOutput) String() string { 1493 | if this == nil { 1494 | return "nil" 1495 | } 1496 | s := strings.Join([]string{`&ScanOutput{`, 1497 | `Data:` + fmt.Sprintf("%v", this.Data) + `,`, 1498 | `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, 1499 | `}`, 1500 | }, "") 1501 | return s 1502 | } 1503 | func (this *GetOutput) String() string { 1504 | if this == nil { 1505 | return "nil" 1506 | } 1507 | s := strings.Join([]string{`&GetOutput{`, 1508 | `Data:` + fmt.Sprintf("%v", this.Data) + `,`, 1509 | `}`, 1510 | }, "") 1511 | return s 1512 | } 1513 | func (this *StatsOutput) String() string { 1514 | if this == nil { 1515 | return "nil" 1516 | } 1517 | keysForTags := make([]string, 0, len(this.Tags)) 1518 | for k, _ := range this.Tags { 1519 | keysForTags = append(keysForTags, k) 1520 | } 1521 | sortkeys.Strings(keysForTags) 1522 | mapStringForTags := "map[string]uint64{" 1523 | for _, k := range keysForTags { 1524 | mapStringForTags += fmt.Sprintf("%v: %v,", k, this.Tags[k]) 1525 | } 1526 | mapStringForTags += "}" 1527 | s := strings.Join([]string{`&StatsOutput{`, 1528 | `Tags:` + mapStringForTags + `,`, 1529 | `Offset:` + fmt.Sprintf("%v", this.Offset) + `,`, 1530 | `File:` + fmt.Sprintf("%v", this.File) + `,`, 1531 | `}`, 1532 | }, "") 1533 | return s 1534 | } 1535 | func valueToStringInput(v interface{}) string { 1536 | rv := reflect.ValueOf(v) 1537 | if rv.IsNil() { 1538 | return "nil" 1539 | } 1540 | pv := reflect.Indirect(rv).Interface() 1541 | return fmt.Sprintf("*%v", pv) 1542 | } 1543 | func (m *Modify) Unmarshal(dAtA []byte) error { 1544 | l := len(dAtA) 1545 | iNdEx := 0 1546 | for iNdEx < l { 1547 | preIndex := iNdEx 1548 | var wire uint64 1549 | for shift := uint(0); ; shift += 7 { 1550 | if shift >= 64 { 1551 | return ErrIntOverflowInput 1552 | } 1553 | if iNdEx >= l { 1554 | return io.ErrUnexpectedEOF 1555 | } 1556 | b := dAtA[iNdEx] 1557 | iNdEx++ 1558 | wire |= (uint64(b) & 0x7F) << shift 1559 | if b < 0x80 { 1560 | break 1561 | } 1562 | } 1563 | fieldNum := int32(wire >> 3) 1564 | wireType := int(wire & 0x7) 1565 | if wireType == 4 { 1566 | return fmt.Errorf("proto: Modify: wiretype end group for non-group") 1567 | } 1568 | if fieldNum <= 0 { 1569 | return fmt.Errorf("proto: Modify: illegal tag %d (wire type %d)", fieldNum, wire) 1570 | } 1571 | switch fieldNum { 1572 | case 1: 1573 | if wireType != 2 { 1574 | return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) 1575 | } 1576 | var stringLen uint64 1577 | for shift := uint(0); ; shift += 7 { 1578 | if shift >= 64 { 1579 | return ErrIntOverflowInput 1580 | } 1581 | if iNdEx >= l { 1582 | return io.ErrUnexpectedEOF 1583 | } 1584 | b := dAtA[iNdEx] 1585 | iNdEx++ 1586 | stringLen |= (uint64(b) & 0x7F) << shift 1587 | if b < 0x80 { 1588 | break 1589 | } 1590 | } 1591 | intStringLen := int(stringLen) 1592 | if intStringLen < 0 { 1593 | return ErrInvalidLengthInput 1594 | } 1595 | postIndex := iNdEx + intStringLen 1596 | if postIndex > l { 1597 | return io.ErrUnexpectedEOF 1598 | } 1599 | m.Namespace = string(dAtA[iNdEx:postIndex]) 1600 | iNdEx = postIndex 1601 | case 2: 1602 | if wireType != 0 { 1603 | return fmt.Errorf("proto: wrong wireType = %d for field Pos", wireType) 1604 | } 1605 | m.Pos = 0 1606 | for shift := uint(0); ; shift += 7 { 1607 | if shift >= 64 { 1608 | return ErrIntOverflowInput 1609 | } 1610 | if iNdEx >= l { 1611 | return io.ErrUnexpectedEOF 1612 | } 1613 | b := dAtA[iNdEx] 1614 | iNdEx++ 1615 | m.Pos |= (int32(b) & 0x7F) << shift 1616 | if b < 0x80 { 1617 | break 1618 | } 1619 | } 1620 | case 3: 1621 | if wireType != 0 { 1622 | return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) 1623 | } 1624 | m.Offset = 0 1625 | for shift := uint(0); ; shift += 7 { 1626 | if shift >= 64 { 1627 | return ErrIntOverflowInput 1628 | } 1629 | if iNdEx >= l { 1630 | return io.ErrUnexpectedEOF 1631 | } 1632 | b := dAtA[iNdEx] 1633 | iNdEx++ 1634 | m.Offset |= (uint64(b) & 0x7F) << shift 1635 | if b < 0x80 { 1636 | break 1637 | } 1638 | } 1639 | case 4: 1640 | if wireType != 2 { 1641 | return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) 1642 | } 1643 | var byteLen int 1644 | for shift := uint(0); ; shift += 7 { 1645 | if shift >= 64 { 1646 | return ErrIntOverflowInput 1647 | } 1648 | if iNdEx >= l { 1649 | return io.ErrUnexpectedEOF 1650 | } 1651 | b := dAtA[iNdEx] 1652 | iNdEx++ 1653 | byteLen |= (int(b) & 0x7F) << shift 1654 | if b < 0x80 { 1655 | break 1656 | } 1657 | } 1658 | if byteLen < 0 { 1659 | return ErrInvalidLengthInput 1660 | } 1661 | postIndex := iNdEx + byteLen 1662 | if postIndex > l { 1663 | return io.ErrUnexpectedEOF 1664 | } 1665 | m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) 1666 | if m.Data == nil { 1667 | m.Data = []byte{} 1668 | } 1669 | iNdEx = postIndex 1670 | case 5: 1671 | if wireType != 0 { 1672 | return fmt.Errorf("proto: wrong wireType = %d for field ResetLength", wireType) 1673 | } 1674 | var v int 1675 | for shift := uint(0); ; shift += 7 { 1676 | if shift >= 64 { 1677 | return ErrIntOverflowInput 1678 | } 1679 | if iNdEx >= l { 1680 | return io.ErrUnexpectedEOF 1681 | } 1682 | b := dAtA[iNdEx] 1683 | iNdEx++ 1684 | v |= (int(b) & 0x7F) << shift 1685 | if b < 0x80 { 1686 | break 1687 | } 1688 | } 1689 | m.ResetLength = bool(v != 0) 1690 | default: 1691 | iNdEx = preIndex 1692 | skippy, err := skipInput(dAtA[iNdEx:]) 1693 | if err != nil { 1694 | return err 1695 | } 1696 | if skippy < 0 { 1697 | return ErrInvalidLengthInput 1698 | } 1699 | if (iNdEx + skippy) > l { 1700 | return io.ErrUnexpectedEOF 1701 | } 1702 | iNdEx += skippy 1703 | } 1704 | } 1705 | 1706 | if iNdEx > l { 1707 | return io.ErrUnexpectedEOF 1708 | } 1709 | return nil 1710 | } 1711 | func (m *Append) Unmarshal(dAtA []byte) error { 1712 | l := len(dAtA) 1713 | iNdEx := 0 1714 | for iNdEx < l { 1715 | preIndex := iNdEx 1716 | var wire uint64 1717 | for shift := uint(0); ; shift += 7 { 1718 | if shift >= 64 { 1719 | return ErrIntOverflowInput 1720 | } 1721 | if iNdEx >= l { 1722 | return io.ErrUnexpectedEOF 1723 | } 1724 | b := dAtA[iNdEx] 1725 | iNdEx++ 1726 | wire |= (uint64(b) & 0x7F) << shift 1727 | if b < 0x80 { 1728 | break 1729 | } 1730 | } 1731 | fieldNum := int32(wire >> 3) 1732 | wireType := int(wire & 0x7) 1733 | if wireType == 4 { 1734 | return fmt.Errorf("proto: Append: wiretype end group for non-group") 1735 | } 1736 | if fieldNum <= 0 { 1737 | return fmt.Errorf("proto: Append: illegal tag %d (wire type %d)", fieldNum, wire) 1738 | } 1739 | switch fieldNum { 1740 | case 1: 1741 | if wireType != 2 { 1742 | return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) 1743 | } 1744 | var stringLen uint64 1745 | for shift := uint(0); ; shift += 7 { 1746 | if shift >= 64 { 1747 | return ErrIntOverflowInput 1748 | } 1749 | if iNdEx >= l { 1750 | return io.ErrUnexpectedEOF 1751 | } 1752 | b := dAtA[iNdEx] 1753 | iNdEx++ 1754 | stringLen |= (uint64(b) & 0x7F) << shift 1755 | if b < 0x80 { 1756 | break 1757 | } 1758 | } 1759 | intStringLen := int(stringLen) 1760 | if intStringLen < 0 { 1761 | return ErrInvalidLengthInput 1762 | } 1763 | postIndex := iNdEx + intStringLen 1764 | if postIndex > l { 1765 | return io.ErrUnexpectedEOF 1766 | } 1767 | m.Namespace = string(dAtA[iNdEx:postIndex]) 1768 | iNdEx = postIndex 1769 | case 2: 1770 | if wireType != 0 { 1771 | return fmt.Errorf("proto: wrong wireType = %d for field AllocSize", wireType) 1772 | } 1773 | m.AllocSize = 0 1774 | for shift := uint(0); ; shift += 7 { 1775 | if shift >= 64 { 1776 | return ErrIntOverflowInput 1777 | } 1778 | if iNdEx >= l { 1779 | return io.ErrUnexpectedEOF 1780 | } 1781 | b := dAtA[iNdEx] 1782 | iNdEx++ 1783 | m.AllocSize |= (uint32(b) & 0x7F) << shift 1784 | if b < 0x80 { 1785 | break 1786 | } 1787 | } 1788 | case 4: 1789 | if wireType != 2 { 1790 | return fmt.Errorf("proto: wrong wireType = %d for field Tags", wireType) 1791 | } 1792 | var stringLen uint64 1793 | for shift := uint(0); ; shift += 7 { 1794 | if shift >= 64 { 1795 | return ErrIntOverflowInput 1796 | } 1797 | if iNdEx >= l { 1798 | return io.ErrUnexpectedEOF 1799 | } 1800 | b := dAtA[iNdEx] 1801 | iNdEx++ 1802 | stringLen |= (uint64(b) & 0x7F) << shift 1803 | if b < 0x80 { 1804 | break 1805 | } 1806 | } 1807 | intStringLen := int(stringLen) 1808 | if intStringLen < 0 { 1809 | return ErrInvalidLengthInput 1810 | } 1811 | postIndex := iNdEx + intStringLen 1812 | if postIndex > l { 1813 | return io.ErrUnexpectedEOF 1814 | } 1815 | m.Tags = append(m.Tags, string(dAtA[iNdEx:postIndex])) 1816 | iNdEx = postIndex 1817 | case 5: 1818 | if wireType != 2 { 1819 | return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) 1820 | } 1821 | var byteLen int 1822 | for shift := uint(0); ; shift += 7 { 1823 | if shift >= 64 { 1824 | return ErrIntOverflowInput 1825 | } 1826 | if iNdEx >= l { 1827 | return io.ErrUnexpectedEOF 1828 | } 1829 | b := dAtA[iNdEx] 1830 | iNdEx++ 1831 | byteLen |= (int(b) & 0x7F) << shift 1832 | if b < 0x80 { 1833 | break 1834 | } 1835 | } 1836 | if byteLen < 0 { 1837 | return ErrInvalidLengthInput 1838 | } 1839 | postIndex := iNdEx + byteLen 1840 | if postIndex > l { 1841 | return io.ErrUnexpectedEOF 1842 | } 1843 | m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) 1844 | if m.Data == nil { 1845 | m.Data = []byte{} 1846 | } 1847 | iNdEx = postIndex 1848 | default: 1849 | iNdEx = preIndex 1850 | skippy, err := skipInput(dAtA[iNdEx:]) 1851 | if err != nil { 1852 | return err 1853 | } 1854 | if skippy < 0 { 1855 | return ErrInvalidLengthInput 1856 | } 1857 | if (iNdEx + skippy) > l { 1858 | return io.ErrUnexpectedEOF 1859 | } 1860 | iNdEx += skippy 1861 | } 1862 | } 1863 | 1864 | if iNdEx > l { 1865 | return io.ErrUnexpectedEOF 1866 | } 1867 | return nil 1868 | } 1869 | func (m *AppendInput) Unmarshal(dAtA []byte) error { 1870 | l := len(dAtA) 1871 | iNdEx := 0 1872 | for iNdEx < l { 1873 | preIndex := iNdEx 1874 | var wire uint64 1875 | for shift := uint(0); ; shift += 7 { 1876 | if shift >= 64 { 1877 | return ErrIntOverflowInput 1878 | } 1879 | if iNdEx >= l { 1880 | return io.ErrUnexpectedEOF 1881 | } 1882 | b := dAtA[iNdEx] 1883 | iNdEx++ 1884 | wire |= (uint64(b) & 0x7F) << shift 1885 | if b < 0x80 { 1886 | break 1887 | } 1888 | } 1889 | fieldNum := int32(wire >> 3) 1890 | wireType := int(wire & 0x7) 1891 | if wireType == 4 { 1892 | return fmt.Errorf("proto: AppendInput: wiretype end group for non-group") 1893 | } 1894 | if fieldNum <= 0 { 1895 | return fmt.Errorf("proto: AppendInput: illegal tag %d (wire type %d)", fieldNum, wire) 1896 | } 1897 | switch fieldNum { 1898 | case 1: 1899 | if wireType != 2 { 1900 | return fmt.Errorf("proto: wrong wireType = %d for field AppendPayload", wireType) 1901 | } 1902 | var msglen int 1903 | for shift := uint(0); ; shift += 7 { 1904 | if shift >= 64 { 1905 | return ErrIntOverflowInput 1906 | } 1907 | if iNdEx >= l { 1908 | return io.ErrUnexpectedEOF 1909 | } 1910 | b := dAtA[iNdEx] 1911 | iNdEx++ 1912 | msglen |= (int(b) & 0x7F) << shift 1913 | if b < 0x80 { 1914 | break 1915 | } 1916 | } 1917 | if msglen < 0 { 1918 | return ErrInvalidLengthInput 1919 | } 1920 | postIndex := iNdEx + msglen 1921 | if postIndex > l { 1922 | return io.ErrUnexpectedEOF 1923 | } 1924 | m.AppendPayload = append(m.AppendPayload, &Append{}) 1925 | if err := m.AppendPayload[len(m.AppendPayload)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { 1926 | return err 1927 | } 1928 | iNdEx = postIndex 1929 | case 2: 1930 | if wireType != 2 { 1931 | return fmt.Errorf("proto: wrong wireType = %d for field ModifyPayload", wireType) 1932 | } 1933 | var msglen int 1934 | for shift := uint(0); ; shift += 7 { 1935 | if shift >= 64 { 1936 | return ErrIntOverflowInput 1937 | } 1938 | if iNdEx >= l { 1939 | return io.ErrUnexpectedEOF 1940 | } 1941 | b := dAtA[iNdEx] 1942 | iNdEx++ 1943 | msglen |= (int(b) & 0x7F) << shift 1944 | if b < 0x80 { 1945 | break 1946 | } 1947 | } 1948 | if msglen < 0 { 1949 | return ErrInvalidLengthInput 1950 | } 1951 | postIndex := iNdEx + msglen 1952 | if postIndex > l { 1953 | return io.ErrUnexpectedEOF 1954 | } 1955 | m.ModifyPayload = append(m.ModifyPayload, &Modify{}) 1956 | if err := m.ModifyPayload[len(m.ModifyPayload)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { 1957 | return err 1958 | } 1959 | iNdEx = postIndex 1960 | default: 1961 | iNdEx = preIndex 1962 | skippy, err := skipInput(dAtA[iNdEx:]) 1963 | if err != nil { 1964 | return err 1965 | } 1966 | if skippy < 0 { 1967 | return ErrInvalidLengthInput 1968 | } 1969 | if (iNdEx + skippy) > l { 1970 | return io.ErrUnexpectedEOF 1971 | } 1972 | iNdEx += skippy 1973 | } 1974 | } 1975 | 1976 | if iNdEx > l { 1977 | return io.ErrUnexpectedEOF 1978 | } 1979 | return nil 1980 | } 1981 | func (m *AppendOutput) Unmarshal(dAtA []byte) error { 1982 | l := len(dAtA) 1983 | iNdEx := 0 1984 | for iNdEx < l { 1985 | preIndex := iNdEx 1986 | var wire uint64 1987 | for shift := uint(0); ; shift += 7 { 1988 | if shift >= 64 { 1989 | return ErrIntOverflowInput 1990 | } 1991 | if iNdEx >= l { 1992 | return io.ErrUnexpectedEOF 1993 | } 1994 | b := dAtA[iNdEx] 1995 | iNdEx++ 1996 | wire |= (uint64(b) & 0x7F) << shift 1997 | if b < 0x80 { 1998 | break 1999 | } 2000 | } 2001 | fieldNum := int32(wire >> 3) 2002 | wireType := int(wire & 0x7) 2003 | if wireType == 4 { 2004 | return fmt.Errorf("proto: AppendOutput: wiretype end group for non-group") 2005 | } 2006 | if fieldNum <= 0 { 2007 | return fmt.Errorf("proto: AppendOutput: illegal tag %d (wire type %d)", fieldNum, wire) 2008 | } 2009 | switch fieldNum { 2010 | case 1: 2011 | if wireType == 0 { 2012 | var v uint64 2013 | for shift := uint(0); ; shift += 7 { 2014 | if shift >= 64 { 2015 | return ErrIntOverflowInput 2016 | } 2017 | if iNdEx >= l { 2018 | return io.ErrUnexpectedEOF 2019 | } 2020 | b := dAtA[iNdEx] 2021 | iNdEx++ 2022 | v |= (uint64(b) & 0x7F) << shift 2023 | if b < 0x80 { 2024 | break 2025 | } 2026 | } 2027 | m.Offset = append(m.Offset, v) 2028 | } else if wireType == 2 { 2029 | var packedLen int 2030 | for shift := uint(0); ; shift += 7 { 2031 | if shift >= 64 { 2032 | return ErrIntOverflowInput 2033 | } 2034 | if iNdEx >= l { 2035 | return io.ErrUnexpectedEOF 2036 | } 2037 | b := dAtA[iNdEx] 2038 | iNdEx++ 2039 | packedLen |= (int(b) & 0x7F) << shift 2040 | if b < 0x80 { 2041 | break 2042 | } 2043 | } 2044 | if packedLen < 0 { 2045 | return ErrInvalidLengthInput 2046 | } 2047 | postIndex := iNdEx + packedLen 2048 | if postIndex > l { 2049 | return io.ErrUnexpectedEOF 2050 | } 2051 | for iNdEx < postIndex { 2052 | var v uint64 2053 | for shift := uint(0); ; shift += 7 { 2054 | if shift >= 64 { 2055 | return ErrIntOverflowInput 2056 | } 2057 | if iNdEx >= l { 2058 | return io.ErrUnexpectedEOF 2059 | } 2060 | b := dAtA[iNdEx] 2061 | iNdEx++ 2062 | v |= (uint64(b) & 0x7F) << shift 2063 | if b < 0x80 { 2064 | break 2065 | } 2066 | } 2067 | m.Offset = append(m.Offset, v) 2068 | } 2069 | } else { 2070 | return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) 2071 | } 2072 | case 2: 2073 | if wireType != 0 { 2074 | return fmt.Errorf("proto: wrong wireType = %d for field ModifiedCount", wireType) 2075 | } 2076 | m.ModifiedCount = 0 2077 | for shift := uint(0); ; shift += 7 { 2078 | if shift >= 64 { 2079 | return ErrIntOverflowInput 2080 | } 2081 | if iNdEx >= l { 2082 | return io.ErrUnexpectedEOF 2083 | } 2084 | b := dAtA[iNdEx] 2085 | iNdEx++ 2086 | m.ModifiedCount |= (uint64(b) & 0x7F) << shift 2087 | if b < 0x80 { 2088 | break 2089 | } 2090 | } 2091 | default: 2092 | iNdEx = preIndex 2093 | skippy, err := skipInput(dAtA[iNdEx:]) 2094 | if err != nil { 2095 | return err 2096 | } 2097 | if skippy < 0 { 2098 | return ErrInvalidLengthInput 2099 | } 2100 | if (iNdEx + skippy) > l { 2101 | return io.ErrUnexpectedEOF 2102 | } 2103 | iNdEx += skippy 2104 | } 2105 | } 2106 | 2107 | if iNdEx > l { 2108 | return io.ErrUnexpectedEOF 2109 | } 2110 | return nil 2111 | } 2112 | func (m *NamespaceInput) Unmarshal(dAtA []byte) error { 2113 | l := len(dAtA) 2114 | iNdEx := 0 2115 | for iNdEx < l { 2116 | preIndex := iNdEx 2117 | var wire uint64 2118 | for shift := uint(0); ; shift += 7 { 2119 | if shift >= 64 { 2120 | return ErrIntOverflowInput 2121 | } 2122 | if iNdEx >= l { 2123 | return io.ErrUnexpectedEOF 2124 | } 2125 | b := dAtA[iNdEx] 2126 | iNdEx++ 2127 | wire |= (uint64(b) & 0x7F) << shift 2128 | if b < 0x80 { 2129 | break 2130 | } 2131 | } 2132 | fieldNum := int32(wire >> 3) 2133 | wireType := int(wire & 0x7) 2134 | if wireType == 4 { 2135 | return fmt.Errorf("proto: NamespaceInput: wiretype end group for non-group") 2136 | } 2137 | if fieldNum <= 0 { 2138 | return fmt.Errorf("proto: NamespaceInput: illegal tag %d (wire type %d)", fieldNum, wire) 2139 | } 2140 | switch fieldNum { 2141 | case 1: 2142 | if wireType != 2 { 2143 | return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) 2144 | } 2145 | var stringLen uint64 2146 | for shift := uint(0); ; shift += 7 { 2147 | if shift >= 64 { 2148 | return ErrIntOverflowInput 2149 | } 2150 | if iNdEx >= l { 2151 | return io.ErrUnexpectedEOF 2152 | } 2153 | b := dAtA[iNdEx] 2154 | iNdEx++ 2155 | stringLen |= (uint64(b) & 0x7F) << shift 2156 | if b < 0x80 { 2157 | break 2158 | } 2159 | } 2160 | intStringLen := int(stringLen) 2161 | if intStringLen < 0 { 2162 | return ErrInvalidLengthInput 2163 | } 2164 | postIndex := iNdEx + intStringLen 2165 | if postIndex > l { 2166 | return io.ErrUnexpectedEOF 2167 | } 2168 | m.Namespace = string(dAtA[iNdEx:postIndex]) 2169 | iNdEx = postIndex 2170 | default: 2171 | iNdEx = preIndex 2172 | skippy, err := skipInput(dAtA[iNdEx:]) 2173 | if err != nil { 2174 | return err 2175 | } 2176 | if skippy < 0 { 2177 | return ErrInvalidLengthInput 2178 | } 2179 | if (iNdEx + skippy) > l { 2180 | return io.ErrUnexpectedEOF 2181 | } 2182 | iNdEx += skippy 2183 | } 2184 | } 2185 | 2186 | if iNdEx > l { 2187 | return io.ErrUnexpectedEOF 2188 | } 2189 | return nil 2190 | } 2191 | func (m *SuccessOutput) Unmarshal(dAtA []byte) error { 2192 | l := len(dAtA) 2193 | iNdEx := 0 2194 | for iNdEx < l { 2195 | preIndex := iNdEx 2196 | var wire uint64 2197 | for shift := uint(0); ; shift += 7 { 2198 | if shift >= 64 { 2199 | return ErrIntOverflowInput 2200 | } 2201 | if iNdEx >= l { 2202 | return io.ErrUnexpectedEOF 2203 | } 2204 | b := dAtA[iNdEx] 2205 | iNdEx++ 2206 | wire |= (uint64(b) & 0x7F) << shift 2207 | if b < 0x80 { 2208 | break 2209 | } 2210 | } 2211 | fieldNum := int32(wire >> 3) 2212 | wireType := int(wire & 0x7) 2213 | if wireType == 4 { 2214 | return fmt.Errorf("proto: SuccessOutput: wiretype end group for non-group") 2215 | } 2216 | if fieldNum <= 0 { 2217 | return fmt.Errorf("proto: SuccessOutput: illegal tag %d (wire type %d)", fieldNum, wire) 2218 | } 2219 | switch fieldNum { 2220 | case 1: 2221 | if wireType != 0 { 2222 | return fmt.Errorf("proto: wrong wireType = %d for field Success", wireType) 2223 | } 2224 | var v int 2225 | for shift := uint(0); ; shift += 7 { 2226 | if shift >= 64 { 2227 | return ErrIntOverflowInput 2228 | } 2229 | if iNdEx >= l { 2230 | return io.ErrUnexpectedEOF 2231 | } 2232 | b := dAtA[iNdEx] 2233 | iNdEx++ 2234 | v |= (int(b) & 0x7F) << shift 2235 | if b < 0x80 { 2236 | break 2237 | } 2238 | } 2239 | m.Success = bool(v != 0) 2240 | default: 2241 | iNdEx = preIndex 2242 | skippy, err := skipInput(dAtA[iNdEx:]) 2243 | if err != nil { 2244 | return err 2245 | } 2246 | if skippy < 0 { 2247 | return ErrInvalidLengthInput 2248 | } 2249 | if (iNdEx + skippy) > l { 2250 | return io.ErrUnexpectedEOF 2251 | } 2252 | iNdEx += skippy 2253 | } 2254 | } 2255 | 2256 | if iNdEx > l { 2257 | return io.ErrUnexpectedEOF 2258 | } 2259 | return nil 2260 | } 2261 | func (m *Get) Unmarshal(dAtA []byte) error { 2262 | l := len(dAtA) 2263 | iNdEx := 0 2264 | for iNdEx < l { 2265 | preIndex := iNdEx 2266 | var wire uint64 2267 | for shift := uint(0); ; shift += 7 { 2268 | if shift >= 64 { 2269 | return ErrIntOverflowInput 2270 | } 2271 | if iNdEx >= l { 2272 | return io.ErrUnexpectedEOF 2273 | } 2274 | b := dAtA[iNdEx] 2275 | iNdEx++ 2276 | wire |= (uint64(b) & 0x7F) << shift 2277 | if b < 0x80 { 2278 | break 2279 | } 2280 | } 2281 | fieldNum := int32(wire >> 3) 2282 | wireType := int(wire & 0x7) 2283 | if wireType == 4 { 2284 | return fmt.Errorf("proto: Get: wiretype end group for non-group") 2285 | } 2286 | if fieldNum <= 0 { 2287 | return fmt.Errorf("proto: Get: illegal tag %d (wire type %d)", fieldNum, wire) 2288 | } 2289 | switch fieldNum { 2290 | case 1: 2291 | if wireType != 2 { 2292 | return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) 2293 | } 2294 | var stringLen uint64 2295 | for shift := uint(0); ; shift += 7 { 2296 | if shift >= 64 { 2297 | return ErrIntOverflowInput 2298 | } 2299 | if iNdEx >= l { 2300 | return io.ErrUnexpectedEOF 2301 | } 2302 | b := dAtA[iNdEx] 2303 | iNdEx++ 2304 | stringLen |= (uint64(b) & 0x7F) << shift 2305 | if b < 0x80 { 2306 | break 2307 | } 2308 | } 2309 | intStringLen := int(stringLen) 2310 | if intStringLen < 0 { 2311 | return ErrInvalidLengthInput 2312 | } 2313 | postIndex := iNdEx + intStringLen 2314 | if postIndex > l { 2315 | return io.ErrUnexpectedEOF 2316 | } 2317 | m.Namespace = string(dAtA[iNdEx:postIndex]) 2318 | iNdEx = postIndex 2319 | case 2: 2320 | if wireType != 0 { 2321 | return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) 2322 | } 2323 | m.Offset = 0 2324 | for shift := uint(0); ; shift += 7 { 2325 | if shift >= 64 { 2326 | return ErrIntOverflowInput 2327 | } 2328 | if iNdEx >= l { 2329 | return io.ErrUnexpectedEOF 2330 | } 2331 | b := dAtA[iNdEx] 2332 | iNdEx++ 2333 | m.Offset |= (uint64(b) & 0x7F) << shift 2334 | if b < 0x80 { 2335 | break 2336 | } 2337 | } 2338 | default: 2339 | iNdEx = preIndex 2340 | skippy, err := skipInput(dAtA[iNdEx:]) 2341 | if err != nil { 2342 | return err 2343 | } 2344 | if skippy < 0 { 2345 | return ErrInvalidLengthInput 2346 | } 2347 | if (iNdEx + skippy) > l { 2348 | return io.ErrUnexpectedEOF 2349 | } 2350 | iNdEx += skippy 2351 | } 2352 | } 2353 | 2354 | if iNdEx > l { 2355 | return io.ErrUnexpectedEOF 2356 | } 2357 | return nil 2358 | } 2359 | func (m *GetInput) Unmarshal(dAtA []byte) error { 2360 | l := len(dAtA) 2361 | iNdEx := 0 2362 | for iNdEx < l { 2363 | preIndex := iNdEx 2364 | var wire uint64 2365 | for shift := uint(0); ; shift += 7 { 2366 | if shift >= 64 { 2367 | return ErrIntOverflowInput 2368 | } 2369 | if iNdEx >= l { 2370 | return io.ErrUnexpectedEOF 2371 | } 2372 | b := dAtA[iNdEx] 2373 | iNdEx++ 2374 | wire |= (uint64(b) & 0x7F) << shift 2375 | if b < 0x80 { 2376 | break 2377 | } 2378 | } 2379 | fieldNum := int32(wire >> 3) 2380 | wireType := int(wire & 0x7) 2381 | if wireType == 4 { 2382 | return fmt.Errorf("proto: GetInput: wiretype end group for non-group") 2383 | } 2384 | if fieldNum <= 0 { 2385 | return fmt.Errorf("proto: GetInput: illegal tag %d (wire type %d)", fieldNum, wire) 2386 | } 2387 | switch fieldNum { 2388 | case 1: 2389 | if wireType != 2 { 2390 | return fmt.Errorf("proto: wrong wireType = %d for field GetPayload", wireType) 2391 | } 2392 | var msglen int 2393 | for shift := uint(0); ; shift += 7 { 2394 | if shift >= 64 { 2395 | return ErrIntOverflowInput 2396 | } 2397 | if iNdEx >= l { 2398 | return io.ErrUnexpectedEOF 2399 | } 2400 | b := dAtA[iNdEx] 2401 | iNdEx++ 2402 | msglen |= (int(b) & 0x7F) << shift 2403 | if b < 0x80 { 2404 | break 2405 | } 2406 | } 2407 | if msglen < 0 { 2408 | return ErrInvalidLengthInput 2409 | } 2410 | postIndex := iNdEx + msglen 2411 | if postIndex > l { 2412 | return io.ErrUnexpectedEOF 2413 | } 2414 | m.GetPayload = append(m.GetPayload, &Get{}) 2415 | if err := m.GetPayload[len(m.GetPayload)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { 2416 | return err 2417 | } 2418 | iNdEx = postIndex 2419 | default: 2420 | iNdEx = preIndex 2421 | skippy, err := skipInput(dAtA[iNdEx:]) 2422 | if err != nil { 2423 | return err 2424 | } 2425 | if skippy < 0 { 2426 | return ErrInvalidLengthInput 2427 | } 2428 | if (iNdEx + skippy) > l { 2429 | return io.ErrUnexpectedEOF 2430 | } 2431 | iNdEx += skippy 2432 | } 2433 | } 2434 | 2435 | if iNdEx > l { 2436 | return io.ErrUnexpectedEOF 2437 | } 2438 | return nil 2439 | } 2440 | func (m *ScanOutput) Unmarshal(dAtA []byte) error { 2441 | l := len(dAtA) 2442 | iNdEx := 0 2443 | for iNdEx < l { 2444 | preIndex := iNdEx 2445 | var wire uint64 2446 | for shift := uint(0); ; shift += 7 { 2447 | if shift >= 64 { 2448 | return ErrIntOverflowInput 2449 | } 2450 | if iNdEx >= l { 2451 | return io.ErrUnexpectedEOF 2452 | } 2453 | b := dAtA[iNdEx] 2454 | iNdEx++ 2455 | wire |= (uint64(b) & 0x7F) << shift 2456 | if b < 0x80 { 2457 | break 2458 | } 2459 | } 2460 | fieldNum := int32(wire >> 3) 2461 | wireType := int(wire & 0x7) 2462 | if wireType == 4 { 2463 | return fmt.Errorf("proto: ScanOutput: wiretype end group for non-group") 2464 | } 2465 | if fieldNum <= 0 { 2466 | return fmt.Errorf("proto: ScanOutput: illegal tag %d (wire type %d)", fieldNum, wire) 2467 | } 2468 | switch fieldNum { 2469 | case 1: 2470 | if wireType != 2 { 2471 | return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) 2472 | } 2473 | var byteLen int 2474 | for shift := uint(0); ; shift += 7 { 2475 | if shift >= 64 { 2476 | return ErrIntOverflowInput 2477 | } 2478 | if iNdEx >= l { 2479 | return io.ErrUnexpectedEOF 2480 | } 2481 | b := dAtA[iNdEx] 2482 | iNdEx++ 2483 | byteLen |= (int(b) & 0x7F) << shift 2484 | if b < 0x80 { 2485 | break 2486 | } 2487 | } 2488 | if byteLen < 0 { 2489 | return ErrInvalidLengthInput 2490 | } 2491 | postIndex := iNdEx + byteLen 2492 | if postIndex > l { 2493 | return io.ErrUnexpectedEOF 2494 | } 2495 | m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) 2496 | if m.Data == nil { 2497 | m.Data = []byte{} 2498 | } 2499 | iNdEx = postIndex 2500 | case 2: 2501 | if wireType != 0 { 2502 | return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) 2503 | } 2504 | m.Offset = 0 2505 | for shift := uint(0); ; shift += 7 { 2506 | if shift >= 64 { 2507 | return ErrIntOverflowInput 2508 | } 2509 | if iNdEx >= l { 2510 | return io.ErrUnexpectedEOF 2511 | } 2512 | b := dAtA[iNdEx] 2513 | iNdEx++ 2514 | m.Offset |= (uint64(b) & 0x7F) << shift 2515 | if b < 0x80 { 2516 | break 2517 | } 2518 | } 2519 | default: 2520 | iNdEx = preIndex 2521 | skippy, err := skipInput(dAtA[iNdEx:]) 2522 | if err != nil { 2523 | return err 2524 | } 2525 | if skippy < 0 { 2526 | return ErrInvalidLengthInput 2527 | } 2528 | if (iNdEx + skippy) > l { 2529 | return io.ErrUnexpectedEOF 2530 | } 2531 | iNdEx += skippy 2532 | } 2533 | } 2534 | 2535 | if iNdEx > l { 2536 | return io.ErrUnexpectedEOF 2537 | } 2538 | return nil 2539 | } 2540 | func (m *GetOutput) Unmarshal(dAtA []byte) error { 2541 | l := len(dAtA) 2542 | iNdEx := 0 2543 | for iNdEx < l { 2544 | preIndex := iNdEx 2545 | var wire uint64 2546 | for shift := uint(0); ; shift += 7 { 2547 | if shift >= 64 { 2548 | return ErrIntOverflowInput 2549 | } 2550 | if iNdEx >= l { 2551 | return io.ErrUnexpectedEOF 2552 | } 2553 | b := dAtA[iNdEx] 2554 | iNdEx++ 2555 | wire |= (uint64(b) & 0x7F) << shift 2556 | if b < 0x80 { 2557 | break 2558 | } 2559 | } 2560 | fieldNum := int32(wire >> 3) 2561 | wireType := int(wire & 0x7) 2562 | if wireType == 4 { 2563 | return fmt.Errorf("proto: GetOutput: wiretype end group for non-group") 2564 | } 2565 | if fieldNum <= 0 { 2566 | return fmt.Errorf("proto: GetOutput: illegal tag %d (wire type %d)", fieldNum, wire) 2567 | } 2568 | switch fieldNum { 2569 | case 1: 2570 | if wireType != 2 { 2571 | return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) 2572 | } 2573 | var byteLen int 2574 | for shift := uint(0); ; shift += 7 { 2575 | if shift >= 64 { 2576 | return ErrIntOverflowInput 2577 | } 2578 | if iNdEx >= l { 2579 | return io.ErrUnexpectedEOF 2580 | } 2581 | b := dAtA[iNdEx] 2582 | iNdEx++ 2583 | byteLen |= (int(b) & 0x7F) << shift 2584 | if b < 0x80 { 2585 | break 2586 | } 2587 | } 2588 | if byteLen < 0 { 2589 | return ErrInvalidLengthInput 2590 | } 2591 | postIndex := iNdEx + byteLen 2592 | if postIndex > l { 2593 | return io.ErrUnexpectedEOF 2594 | } 2595 | m.Data = append(m.Data, make([]byte, postIndex-iNdEx)) 2596 | copy(m.Data[len(m.Data)-1], dAtA[iNdEx:postIndex]) 2597 | iNdEx = postIndex 2598 | default: 2599 | iNdEx = preIndex 2600 | skippy, err := skipInput(dAtA[iNdEx:]) 2601 | if err != nil { 2602 | return err 2603 | } 2604 | if skippy < 0 { 2605 | return ErrInvalidLengthInput 2606 | } 2607 | if (iNdEx + skippy) > l { 2608 | return io.ErrUnexpectedEOF 2609 | } 2610 | iNdEx += skippy 2611 | } 2612 | } 2613 | 2614 | if iNdEx > l { 2615 | return io.ErrUnexpectedEOF 2616 | } 2617 | return nil 2618 | } 2619 | func (m *StatsOutput) Unmarshal(dAtA []byte) error { 2620 | l := len(dAtA) 2621 | iNdEx := 0 2622 | for iNdEx < l { 2623 | preIndex := iNdEx 2624 | var wire uint64 2625 | for shift := uint(0); ; shift += 7 { 2626 | if shift >= 64 { 2627 | return ErrIntOverflowInput 2628 | } 2629 | if iNdEx >= l { 2630 | return io.ErrUnexpectedEOF 2631 | } 2632 | b := dAtA[iNdEx] 2633 | iNdEx++ 2634 | wire |= (uint64(b) & 0x7F) << shift 2635 | if b < 0x80 { 2636 | break 2637 | } 2638 | } 2639 | fieldNum := int32(wire >> 3) 2640 | wireType := int(wire & 0x7) 2641 | if wireType == 4 { 2642 | return fmt.Errorf("proto: StatsOutput: wiretype end group for non-group") 2643 | } 2644 | if fieldNum <= 0 { 2645 | return fmt.Errorf("proto: StatsOutput: illegal tag %d (wire type %d)", fieldNum, wire) 2646 | } 2647 | switch fieldNum { 2648 | case 1: 2649 | if wireType != 2 { 2650 | return fmt.Errorf("proto: wrong wireType = %d for field Tags", wireType) 2651 | } 2652 | var msglen int 2653 | for shift := uint(0); ; shift += 7 { 2654 | if shift >= 64 { 2655 | return ErrIntOverflowInput 2656 | } 2657 | if iNdEx >= l { 2658 | return io.ErrUnexpectedEOF 2659 | } 2660 | b := dAtA[iNdEx] 2661 | iNdEx++ 2662 | msglen |= (int(b) & 0x7F) << shift 2663 | if b < 0x80 { 2664 | break 2665 | } 2666 | } 2667 | if msglen < 0 { 2668 | return ErrInvalidLengthInput 2669 | } 2670 | postIndex := iNdEx + msglen 2671 | if postIndex > l { 2672 | return io.ErrUnexpectedEOF 2673 | } 2674 | if m.Tags == nil { 2675 | m.Tags = make(map[string]uint64) 2676 | } 2677 | var mapkey string 2678 | var mapvalue uint64 2679 | for iNdEx < postIndex { 2680 | entryPreIndex := iNdEx 2681 | var wire uint64 2682 | for shift := uint(0); ; shift += 7 { 2683 | if shift >= 64 { 2684 | return ErrIntOverflowInput 2685 | } 2686 | if iNdEx >= l { 2687 | return io.ErrUnexpectedEOF 2688 | } 2689 | b := dAtA[iNdEx] 2690 | iNdEx++ 2691 | wire |= (uint64(b) & 0x7F) << shift 2692 | if b < 0x80 { 2693 | break 2694 | } 2695 | } 2696 | fieldNum := int32(wire >> 3) 2697 | if fieldNum == 1 { 2698 | var stringLenmapkey uint64 2699 | for shift := uint(0); ; shift += 7 { 2700 | if shift >= 64 { 2701 | return ErrIntOverflowInput 2702 | } 2703 | if iNdEx >= l { 2704 | return io.ErrUnexpectedEOF 2705 | } 2706 | b := dAtA[iNdEx] 2707 | iNdEx++ 2708 | stringLenmapkey |= (uint64(b) & 0x7F) << shift 2709 | if b < 0x80 { 2710 | break 2711 | } 2712 | } 2713 | intStringLenmapkey := int(stringLenmapkey) 2714 | if intStringLenmapkey < 0 { 2715 | return ErrInvalidLengthInput 2716 | } 2717 | postStringIndexmapkey := iNdEx + intStringLenmapkey 2718 | if postStringIndexmapkey > l { 2719 | return io.ErrUnexpectedEOF 2720 | } 2721 | mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) 2722 | iNdEx = postStringIndexmapkey 2723 | } else if fieldNum == 2 { 2724 | for shift := uint(0); ; shift += 7 { 2725 | if shift >= 64 { 2726 | return ErrIntOverflowInput 2727 | } 2728 | if iNdEx >= l { 2729 | return io.ErrUnexpectedEOF 2730 | } 2731 | b := dAtA[iNdEx] 2732 | iNdEx++ 2733 | mapvalue |= (uint64(b) & 0x7F) << shift 2734 | if b < 0x80 { 2735 | break 2736 | } 2737 | } 2738 | } else { 2739 | iNdEx = entryPreIndex 2740 | skippy, err := skipInput(dAtA[iNdEx:]) 2741 | if err != nil { 2742 | return err 2743 | } 2744 | if skippy < 0 { 2745 | return ErrInvalidLengthInput 2746 | } 2747 | if (iNdEx + skippy) > postIndex { 2748 | return io.ErrUnexpectedEOF 2749 | } 2750 | iNdEx += skippy 2751 | } 2752 | } 2753 | m.Tags[mapkey] = mapvalue 2754 | iNdEx = postIndex 2755 | case 2: 2756 | if wireType != 0 { 2757 | return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) 2758 | } 2759 | m.Offset = 0 2760 | for shift := uint(0); ; shift += 7 { 2761 | if shift >= 64 { 2762 | return ErrIntOverflowInput 2763 | } 2764 | if iNdEx >= l { 2765 | return io.ErrUnexpectedEOF 2766 | } 2767 | b := dAtA[iNdEx] 2768 | iNdEx++ 2769 | m.Offset |= (uint64(b) & 0x7F) << shift 2770 | if b < 0x80 { 2771 | break 2772 | } 2773 | } 2774 | case 3: 2775 | if wireType != 2 { 2776 | return fmt.Errorf("proto: wrong wireType = %d for field File", wireType) 2777 | } 2778 | var stringLen uint64 2779 | for shift := uint(0); ; shift += 7 { 2780 | if shift >= 64 { 2781 | return ErrIntOverflowInput 2782 | } 2783 | if iNdEx >= l { 2784 | return io.ErrUnexpectedEOF 2785 | } 2786 | b := dAtA[iNdEx] 2787 | iNdEx++ 2788 | stringLen |= (uint64(b) & 0x7F) << shift 2789 | if b < 0x80 { 2790 | break 2791 | } 2792 | } 2793 | intStringLen := int(stringLen) 2794 | if intStringLen < 0 { 2795 | return ErrInvalidLengthInput 2796 | } 2797 | postIndex := iNdEx + intStringLen 2798 | if postIndex > l { 2799 | return io.ErrUnexpectedEOF 2800 | } 2801 | m.File = string(dAtA[iNdEx:postIndex]) 2802 | iNdEx = postIndex 2803 | default: 2804 | iNdEx = preIndex 2805 | skippy, err := skipInput(dAtA[iNdEx:]) 2806 | if err != nil { 2807 | return err 2808 | } 2809 | if skippy < 0 { 2810 | return ErrInvalidLengthInput 2811 | } 2812 | if (iNdEx + skippy) > l { 2813 | return io.ErrUnexpectedEOF 2814 | } 2815 | iNdEx += skippy 2816 | } 2817 | } 2818 | 2819 | if iNdEx > l { 2820 | return io.ErrUnexpectedEOF 2821 | } 2822 | return nil 2823 | } 2824 | func skipInput(dAtA []byte) (n int, err error) { 2825 | l := len(dAtA) 2826 | iNdEx := 0 2827 | for iNdEx < l { 2828 | var wire uint64 2829 | for shift := uint(0); ; shift += 7 { 2830 | if shift >= 64 { 2831 | return 0, ErrIntOverflowInput 2832 | } 2833 | if iNdEx >= l { 2834 | return 0, io.ErrUnexpectedEOF 2835 | } 2836 | b := dAtA[iNdEx] 2837 | iNdEx++ 2838 | wire |= (uint64(b) & 0x7F) << shift 2839 | if b < 0x80 { 2840 | break 2841 | } 2842 | } 2843 | wireType := int(wire & 0x7) 2844 | switch wireType { 2845 | case 0: 2846 | for shift := uint(0); ; shift += 7 { 2847 | if shift >= 64 { 2848 | return 0, ErrIntOverflowInput 2849 | } 2850 | if iNdEx >= l { 2851 | return 0, io.ErrUnexpectedEOF 2852 | } 2853 | iNdEx++ 2854 | if dAtA[iNdEx-1] < 0x80 { 2855 | break 2856 | } 2857 | } 2858 | return iNdEx, nil 2859 | case 1: 2860 | iNdEx += 8 2861 | return iNdEx, nil 2862 | case 2: 2863 | var length int 2864 | for shift := uint(0); ; shift += 7 { 2865 | if shift >= 64 { 2866 | return 0, ErrIntOverflowInput 2867 | } 2868 | if iNdEx >= l { 2869 | return 0, io.ErrUnexpectedEOF 2870 | } 2871 | b := dAtA[iNdEx] 2872 | iNdEx++ 2873 | length |= (int(b) & 0x7F) << shift 2874 | if b < 0x80 { 2875 | break 2876 | } 2877 | } 2878 | iNdEx += length 2879 | if length < 0 { 2880 | return 0, ErrInvalidLengthInput 2881 | } 2882 | return iNdEx, nil 2883 | case 3: 2884 | for { 2885 | var innerWire uint64 2886 | var start int = iNdEx 2887 | for shift := uint(0); ; shift += 7 { 2888 | if shift >= 64 { 2889 | return 0, ErrIntOverflowInput 2890 | } 2891 | if iNdEx >= l { 2892 | return 0, io.ErrUnexpectedEOF 2893 | } 2894 | b := dAtA[iNdEx] 2895 | iNdEx++ 2896 | innerWire |= (uint64(b) & 0x7F) << shift 2897 | if b < 0x80 { 2898 | break 2899 | } 2900 | } 2901 | innerWireType := int(innerWire & 0x7) 2902 | if innerWireType == 4 { 2903 | break 2904 | } 2905 | next, err := skipInput(dAtA[start:]) 2906 | if err != nil { 2907 | return 0, err 2908 | } 2909 | iNdEx = start + next 2910 | } 2911 | return iNdEx, nil 2912 | case 4: 2913 | return iNdEx, nil 2914 | case 5: 2915 | iNdEx += 4 2916 | return iNdEx, nil 2917 | default: 2918 | return 0, fmt.Errorf("proto: illegal wireType %d", wireType) 2919 | } 2920 | } 2921 | panic("unreachable") 2922 | } 2923 | 2924 | var ( 2925 | ErrInvalidLengthInput = fmt.Errorf("proto: negative length found during unmarshaling") 2926 | ErrIntOverflowInput = fmt.Errorf("proto: integer overflow") 2927 | ) 2928 | 2929 | func init() { proto.RegisterFile("input.proto", fileDescriptorInput) } 2930 | 2931 | var fileDescriptorInput = []byte{ 2932 | // 527 bytes of a gzipped FileDescriptorProto 2933 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x53, 0xb1, 0x8e, 0xd3, 0x40, 2934 | 0x10, 0xcd, 0xc6, 0x4e, 0x88, 0x27, 0x09, 0x42, 0x2b, 0x84, 0x2c, 0x38, 0x2d, 0x96, 0x45, 0xe1, 2935 | 0x6b, 0x8c, 0x74, 0x08, 0x71, 0x82, 0x8a, 0x43, 0x28, 0x42, 0x3a, 0xe0, 0xb4, 0xe1, 0x07, 0x16, 2936 | 0x67, 0x93, 0xb3, 0x70, 0xbc, 0x96, 0xbd, 0x46, 0x0a, 0x15, 0x15, 0x35, 0xff, 0x40, 0xc3, 0xa7, 2937 | 0x50, 0x5e, 0x49, 0x49, 0x4c, 0x43, 0x79, 0x9f, 0x80, 0x76, 0xd7, 0x8e, 0x1d, 0xa1, 0xe3, 0xba, 2938 | 0x99, 0xd9, 0x99, 0x79, 0x6f, 0xde, 0xce, 0xc0, 0x38, 0x4e, 0xb3, 0x52, 0x86, 0x59, 0x2e, 0xa4, 2939 | 0xc0, 0xf6, 0x9a, 0xc5, 0xa9, 0xff, 0x05, 0xc1, 0xf0, 0xb5, 0x58, 0xc4, 0xcb, 0x0d, 0x3e, 0x00, 2940 | 0x27, 0x65, 0x6b, 0x5e, 0x64, 0x2c, 0xe2, 0x2e, 0xf2, 0x50, 0xe0, 0xd0, 0x36, 0x80, 0x6f, 0x81, 2941 | 0x95, 0x89, 0xc2, 0xed, 0x7b, 0x28, 0x18, 0x50, 0x65, 0xe2, 0x3b, 0x30, 0x14, 0xcb, 0x65, 0xc1, 2942 | 0xa5, 0x6b, 0x79, 0x28, 0xb0, 0x69, 0xed, 0x61, 0x0c, 0xf6, 0x82, 0x49, 0xe6, 0xda, 0x1e, 0x0a, 2943 | 0x26, 0x54, 0xdb, 0xd8, 0x83, 0x71, 0xce, 0x0b, 0x2e, 0x4f, 0x79, 0xba, 0x92, 0xe7, 0xee, 0xc0, 2944 | 0x43, 0xc1, 0x88, 0x76, 0x43, 0x7e, 0x02, 0xc3, 0xe7, 0x59, 0xc6, 0xd3, 0xc5, 0x35, 0x3c, 0x0e, 2945 | 0xc0, 0x61, 0x49, 0x22, 0xa2, 0x79, 0xfc, 0x89, 0x6b, 0x36, 0x53, 0xda, 0x06, 0x14, 0xb6, 0x64, 2946 | 0xab, 0xc2, 0xb5, 0x3d, 0x2b, 0x70, 0xa8, 0xb6, 0x77, 0x7c, 0x06, 0x2d, 0x1f, 0xbf, 0x84, 0xb1, 2947 | 0x41, 0x7b, 0xa5, 0x14, 0xc1, 0x47, 0x30, 0x65, 0xda, 0x3d, 0x63, 0x9b, 0x44, 0xb0, 0x85, 0x8b, 2948 | 0x3c, 0x2b, 0x18, 0x1f, 0x4d, 0x42, 0xa5, 0x51, 0x68, 0x32, 0xe9, 0x7e, 0x8a, 0xaa, 0x59, 0x6b, 2949 | 0xe1, 0x9a, 0x9a, 0x7e, 0xb7, 0xc6, 0x68, 0x4a, 0xf7, 0x53, 0xfc, 0x53, 0x98, 0x98, 0x66, 0x6f, 2950 | 0x4b, 0xa9, 0x70, 0x5b, 0x09, 0x15, 0x60, 0x2b, 0xe1, 0x83, 0xba, 0x77, 0xcc, 0x17, 0x2f, 0x44, 2951 | 0x99, 0x4a, 0x3d, 0xa8, 0x4d, 0xf7, 0x83, 0x7e, 0x08, 0x37, 0xdf, 0x34, 0xba, 0x98, 0x39, 0xfe, 2952 | 0x2b, 0x9d, 0x7f, 0x08, 0xd3, 0x79, 0x19, 0x45, 0xbc, 0x28, 0x6a, 0x78, 0x17, 0x6e, 0x14, 0x26, 2953 | 0xa0, 0x93, 0x47, 0xb4, 0x71, 0xfd, 0x67, 0x60, 0xcd, 0xf8, 0x35, 0xfd, 0x3a, 0xec, 0xfb, 0xdd, 2954 | 0x05, 0xf0, 0x1f, 0xc3, 0x68, 0xc6, 0xa5, 0x61, 0x74, 0x08, 0xb0, 0xe2, 0x72, 0x5f, 0x56, 0xc7, 2955 | 0x48, 0x34, 0xe3, 0x92, 0x76, 0x1e, 0xfd, 0x63, 0x80, 0x79, 0xc4, 0xd2, 0x9a, 0x5b, 0xf3, 0x6b, 2956 | 0xa8, 0xb3, 0x45, 0x57, 0x01, 0xde, 0x07, 0x67, 0xc6, 0xe5, 0x3f, 0x85, 0xd6, 0xee, 0xbb, 0xbf, 2957 | 0x21, 0x18, 0xcf, 0x25, 0x93, 0xcd, 0xe0, 0x0f, 0xeb, 0x35, 0x31, 0x7c, 0xee, 0x19, 0x3e, 0x9d, 2958 | 0x84, 0xf0, 0x1d, 0x5b, 0x15, 0x2f, 0x53, 0x99, 0x6f, 0xea, 0x1d, 0xba, 0x02, 0x59, 0x81, 0x2d, 2959 | 0xe3, 0x84, 0xeb, 0x0b, 0x70, 0xa8, 0xb6, 0xef, 0x3e, 0x01, 0x67, 0x57, 0xae, 0xce, 0xe6, 0x03, 2960 | 0xdf, 0xd4, 0xda, 0x29, 0x13, 0xdf, 0x86, 0xc1, 0x47, 0x96, 0x94, 0xbc, 0xee, 0x64, 0x9c, 0xa7, 2961 | 0xfd, 0x63, 0x74, 0x32, 0xbb, 0xd8, 0x92, 0xde, 0xcf, 0x2d, 0xe9, 0x5d, 0x6e, 0x09, 0xfa, 0x5c, 2962 | 0x11, 0xf4, 0xbd, 0x22, 0xe8, 0x47, 0x45, 0xd0, 0x45, 0x45, 0xd0, 0xaf, 0x8a, 0xa0, 0x3f, 0x15, 2963 | 0xe9, 0x5d, 0x56, 0x04, 0x7d, 0xfd, 0x4d, 0x7a, 0x80, 0xd3, 0x24, 0xcc, 0xf2, 0xcd, 0x3a, 0x0f, 2964 | 0x73, 0x11, 0x9d, 0xf3, 0xa5, 0xc8, 0xe5, 0xc9, 0xe0, 0x4c, 0x9d, 0xf6, 0xfb, 0xa1, 0xbe, 0xf0, 2965 | 0x47, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xd5, 0xaf, 0xb0, 0x4c, 0xf0, 0x03, 0x00, 0x00, 2966 | } 2967 | -------------------------------------------------------------------------------- /input.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package main; 4 | 5 | 6 | option java_package = "nl.prymr.rochefort"; 7 | option java_outer_classname = "Proto"; 8 | 9 | message Modify { 10 | string namespace = 1; 11 | int32 pos = 2; 12 | uint64 offset = 3; 13 | bytes data = 4; 14 | bool resetLength = 5; 15 | } 16 | 17 | message Append { 18 | string namespace = 1; 19 | uint32 allocSize = 2; 20 | repeated string tags = 4; 21 | bytes data = 5; 22 | } 23 | 24 | message AppendInput { 25 | repeated Append appendPayload = 1; 26 | repeated Modify modifyPayload = 2; 27 | } 28 | 29 | message AppendOutput { 30 | repeated uint64 offset = 1; 31 | uint64 modifiedCount = 2; 32 | } 33 | 34 | message NamespaceInput { 35 | string namespace = 1; 36 | } 37 | 38 | message SuccessOutput { 39 | bool success = 1; 40 | } 41 | 42 | message Get { 43 | string namespace = 1; 44 | uint64 offset = 2; 45 | 46 | } 47 | 48 | message GetInput { 49 | repeated Get getPayload = 1; 50 | } 51 | 52 | message ScanOutput { 53 | bytes data = 1; 54 | uint64 offset = 2; 55 | } 56 | 57 | message GetOutput { 58 | repeated bytes data = 1; 59 | } 60 | 61 | message StatsOutput { 62 | map tags = 1; 63 | uint64 offset = 2; 64 | string file = 3; 65 | } 66 | 67 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "errors" 7 | "flag" 8 | "fmt" 9 | "github.com/dgryski/go-metro" 10 | "io" 11 | "io/ioutil" 12 | "log" 13 | "net/http" 14 | "os" 15 | "os/signal" 16 | "path" 17 | "regexp" 18 | "strings" 19 | "sync" 20 | "sync/atomic" 21 | "syscall" 22 | "time" 23 | ) 24 | 25 | type PostingsList struct { 26 | descriptor *os.File 27 | offset uint64 28 | } 29 | 30 | func (this *PostingsList) newTermQuery() *Term { 31 | postings := make([]byte, this.offset) 32 | n, err := this.descriptor.ReadAt(postings, 0) 33 | if n != len(postings) && err != nil { 34 | postings = []byte{} 35 | } 36 | 37 | longed := make([]int64, len(postings)/8) 38 | j := 0 39 | for i := 0; i < len(postings); i += 8 { 40 | longed[j] = int64(binary.LittleEndian.Uint64(postings[i:])) 41 | j++ 42 | } 43 | return NewTerm(longed) 44 | } 45 | 46 | type StoreItem struct { 47 | path string 48 | root string 49 | descriptor *os.File 50 | index map[string]*PostingsList 51 | offset uint64 52 | sync.RWMutex 53 | } 54 | 55 | var nonAlphaNumeric = regexp.MustCompile("[^a-zA-Z0-9_]+") 56 | 57 | func sanitize(s string) string { 58 | return nonAlphaNumeric.ReplaceAllLiteralString(s, "") 59 | } 60 | 61 | func openAtEnd(filePath string) (*os.File, uint64) { 62 | f, err := os.OpenFile(filePath, os.O_CREATE|os.O_RDWR, 0600) 63 | if err != nil { 64 | panic(err) 65 | } 66 | offset, err := f.Seek(0, 2) 67 | 68 | if err != nil { 69 | panic(err) 70 | } 71 | 72 | log.Printf("openning: %s with offset: %d", filePath, offset) 73 | return f, uint64(offset) 74 | } 75 | 76 | func NewStorage(root string) *StoreItem { 77 | os.MkdirAll(root, 0700) 78 | 79 | filePath := path.Join(root, "append.raw") 80 | f, offset := openAtEnd(filePath) 81 | 82 | si := &StoreItem{ 83 | offset: uint64(offset), 84 | path: filePath, 85 | index: map[string]*PostingsList{}, 86 | descriptor: f, 87 | root: root, 88 | } 89 | 90 | files, err := ioutil.ReadDir(root) 91 | if err != nil { 92 | panic(err) 93 | } 94 | for _, dirFile := range files { 95 | if strings.HasSuffix(dirFile.Name(), ".postings") { 96 | dot := strings.IndexRune(dirFile.Name(), '.') 97 | idxName := dirFile.Name()[:dot] 98 | si.CreatePostingsList(idxName) 99 | } 100 | } 101 | 102 | return si 103 | } 104 | 105 | func (this *StoreItem) GetPostingsList(name string) *PostingsList { 106 | name = sanitize(name) 107 | this.RLock() 108 | defer this.RUnlock() 109 | 110 | if p, ok := this.index[name]; ok { 111 | return p 112 | } 113 | return nil 114 | } 115 | 116 | func (this *StoreItem) CreatePostingsList(name string) *PostingsList { 117 | name = sanitize(name) 118 | this.RLock() 119 | 120 | if p, ok := this.index[name]; ok { 121 | this.RUnlock() 122 | return p 123 | } 124 | this.RUnlock() 125 | this.Lock() 126 | defer this.Unlock() 127 | 128 | if p, ok := this.index[name]; ok { 129 | return p 130 | } 131 | 132 | f, offset := openAtEnd(path.Join(this.root, fmt.Sprintf("%s.postings", name))) 133 | p := &PostingsList{ 134 | descriptor: f, 135 | offset: offset, 136 | } 137 | this.index[name] = p 138 | return p 139 | } 140 | 141 | func (this *StoreItem) stats() *StatsOutput { 142 | out := &StatsOutput{ 143 | Tags: make(map[string]uint64), 144 | Offset: this.offset, 145 | File: this.path, 146 | } 147 | 148 | this.RLock() 149 | defer this.RUnlock() 150 | for name, index := range this.index { 151 | out.Tags[name] = index.offset / 8 152 | } 153 | 154 | return out 155 | } 156 | 157 | func (this *StoreItem) scan(cb func(uint64, []byte) bool) { 158 | SCAN: 159 | for offset := uint64(0); offset < this.offset; { 160 | // this is lockless, which means we could read a header, 161 | // but the data might be incomplete 162 | 163 | dataLen, allocSize, err := readHeader(this.descriptor, offset) 164 | if err != nil { 165 | break SCAN 166 | } 167 | output := make([]byte, dataLen) 168 | _, err = this.descriptor.ReadAt(output, int64(offset)+int64(headerLen)) 169 | if err != nil { 170 | break SCAN 171 | } 172 | 173 | if !cb(offset, output) { 174 | break SCAN 175 | } 176 | 177 | offset += uint64(allocSize) + uint64(headerLen) 178 | } 179 | } 180 | 181 | func (this *StoreItem) compact() (map[uint64]uint64, error) { 182 | this.Lock() 183 | defer this.Unlock() 184 | 185 | relocationMap := map[uint64]uint64{} 186 | 187 | if len(this.index) > 0 { 188 | return nil, errors.New("can not compact indexed items") 189 | } 190 | if this.offset <= headerLen { 191 | return nil, errors.New("data is too small, nothing to compact") 192 | } 193 | 194 | endOffet := this.offset - uint64(headerLen) 195 | needCompaction := false 196 | for offset := uint64(0); offset < endOffet; { 197 | // this is lockless, which means we could read a header, 198 | // but the data might be incomplete 199 | uncorruptedOffset, dataLen, allocSize, err := gotoNextValidHeader(this.descriptor, offset, endOffet) 200 | if err != nil { 201 | needCompaction = true 202 | break 203 | } 204 | if offset != uncorruptedOffset { 205 | needCompaction = true 206 | break 207 | } 208 | if dataLen != allocSize { 209 | needCompaction = true 210 | break 211 | } 212 | offset += uint64(allocSize) + uint64(headerLen) 213 | } 214 | 215 | if !needCompaction { 216 | log.Printf("%s no compaction needed", this.root) 217 | return nil, nil 218 | } 219 | 220 | actualOffset := uint64(0) 221 | for offset := uint64(0); offset < endOffet; { 222 | // this is lockless, which means we could read a header, 223 | // but the data might be incomplete 224 | uncorruptedOffset, dataLen, allocSize, err := gotoNextValidHeader(this.descriptor, offset, endOffet) 225 | if err != nil { 226 | log.Printf("%s failed to read header at %d, err: %s", this.root, offset, err.Error()) 227 | break 228 | } 229 | if offset != uncorruptedOffset { 230 | log.Printf("%s found corrupt header, skipped from %d to %d dataLen: %d, allocSize: %d, end: %d", this.root, offset, uncorruptedOffset, dataLen, allocSize, endOffet) 231 | } 232 | offset = uncorruptedOffset 233 | 234 | storedData := make([]byte, dataLen) 235 | _, err = this.descriptor.ReadAt(storedData, int64(offset)+int64(headerLen)) 236 | if err != nil { 237 | log.Printf("%s failed to read data at %d, err: %s", this.root, offset+headerLen, err.Error()) 238 | break 239 | } 240 | 241 | this.writeHeader(actualOffset, dataLen, dataLen) 242 | relocationMap[offset] = actualOffset 243 | 244 | _, err = this.descriptor.WriteAt(storedData, int64(actualOffset)+int64(headerLen)) 245 | if err != nil { 246 | log.Printf("%s failed to write data at %d, err: %s", this.root, int64(actualOffset)+int64(headerLen), err.Error()) 247 | break 248 | } 249 | actualOffset += uint64(dataLen) + uint64(headerLen) 250 | offset += uint64(allocSize) + uint64(headerLen) 251 | } 252 | 253 | // this will lose data if something was actually written in the end of the file 254 | // we will also truncate it 255 | // make sure you are compacting data you are no longer writing to 256 | err := this.descriptor.Truncate(int64(actualOffset)) 257 | if err != nil { 258 | log.Fatalf("failed to truncate file to %d, err: %s", actualOffset, err.Error()) 259 | } 260 | 261 | log.Printf("compaction %s done, old size: %d, new size: %d", this.root, this.offset, actualOffset) 262 | atomic.StoreUint64(&this.offset, actualOffset) 263 | 264 | return relocationMap, nil 265 | } 266 | 267 | func (this *StoreItem) ExecuteQuery(query Query, cb func(uint64, []byte) bool) { 268 | for query.Next() != NO_MORE { 269 | offset := uint64(query.GetDocId()) 270 | dataLen, _, err := readHeader(this.descriptor, offset) 271 | if err != nil { 272 | break 273 | } 274 | 275 | output := make([]byte, dataLen) 276 | _, err = this.descriptor.ReadAt(output, int64(offset)+int64(headerLen)) 277 | if err != nil { 278 | break 279 | } 280 | 281 | if !cb(offset, output) { 282 | break 283 | } 284 | } 285 | } 286 | 287 | const headerLen = 4 + 8 + 4 + 4 288 | 289 | var wrongChecksumError = errors.New("wrong checksum") 290 | var noValidHeaderFoundError = errors.New("no valid header found") 291 | 292 | func gotoNextValidHeader(file *os.File, offset, endOffset uint64) (uint64, uint32, uint32, error) { 293 | for start := offset; start < endOffset; start++ { 294 | dataLen, allocSize, err := readHeader(file, start) 295 | if err == nil { 296 | return start, dataLen, allocSize, nil 297 | } else { 298 | if err == io.EOF { 299 | return 0, 0, 0, noValidHeaderFoundError 300 | } 301 | } 302 | } 303 | 304 | return 0, 0, 0, noValidHeaderFoundError 305 | 306 | } 307 | func readHeader(file *os.File, offset uint64) (uint32, uint32, error) { 308 | headerBytes := make([]byte, headerLen) 309 | _, err := file.ReadAt(headerBytes, int64(offset)) 310 | if err != nil { 311 | return 0, 0, err 312 | } 313 | dataLen := binary.LittleEndian.Uint32(headerBytes[0:]) 314 | 315 | allocSize := binary.LittleEndian.Uint32(headerBytes[12:]) 316 | checksum := binary.LittleEndian.Uint32(headerBytes[16:]) 317 | computedChecksum := crc(headerBytes[0:16]) 318 | if checksum != computedChecksum { 319 | return 0, 0, wrongChecksumError 320 | } 321 | return dataLen, allocSize, nil 322 | } 323 | 324 | func (this *StoreItem) writeHeader(currentOffset uint64, dataLen uint32, allocSize uint32) { 325 | header := make([]byte, headerLen) 326 | 327 | binary.LittleEndian.PutUint32(header[0:], uint32(dataLen)) 328 | binary.LittleEndian.PutUint64(header[4:], uint64(time.Now().UnixNano())) 329 | binary.LittleEndian.PutUint32(header[12:], allocSize) 330 | 331 | checksum := crc(header[0:16]) 332 | binary.LittleEndian.PutUint32(header[16:], checksum) 333 | 334 | _, err := this.descriptor.WriteAt(header, int64(currentOffset)) 335 | 336 | if err != nil { 337 | log.Fatal(err) 338 | } 339 | } 340 | 341 | func (this *StoreItem) appendPostings(name string, value uint64) { 342 | p := this.CreatePostingsList(name) 343 | 344 | data := make([]byte, 8) 345 | binary.LittleEndian.PutUint64(data, value) 346 | 347 | // add it to the end 348 | offset := atomic.AddUint64(&p.offset, uint64(8)) - 8 349 | p.descriptor.WriteAt(data, int64(offset)) 350 | } 351 | 352 | func (this *StoreItem) read(offset uint64) ([]byte, error) { 353 | // lockless read 354 | dataLen, _, err := readHeader(this.descriptor, offset) 355 | if err != nil { 356 | return nil, err 357 | } 358 | 359 | output := make([]byte, dataLen) 360 | _, err = this.descriptor.ReadAt(output, int64(offset)+int64(headerLen)) 361 | if err != nil { 362 | return nil, err 363 | } 364 | return output, nil 365 | } 366 | 367 | func (this *StoreItem) append(allocSize uint32, dataRaw []byte) (uint64, error) { 368 | if len(dataRaw) > int(allocSize) { 369 | allocSize = uint32(len(dataRaw)) 370 | } 371 | 372 | offset := atomic.AddUint64(&this.offset, uint64(allocSize+headerLen)) 373 | 374 | currentOffset := offset - uint64(allocSize+headerLen) 375 | _, err := this.descriptor.WriteAt(dataRaw, int64(currentOffset+headerLen)) 376 | if err != nil { 377 | panic(err) 378 | } 379 | 380 | this.writeHeader(currentOffset, uint32(len(dataRaw)), allocSize) 381 | 382 | return currentOffset, nil 383 | } 384 | 385 | func (this *StoreItem) modify(offset uint64, pos int32, dataRaw []byte, resetLength bool) error { 386 | oldDataLen, allocSize, err := readHeader(this.descriptor, offset) 387 | if err != nil { 388 | return err 389 | } 390 | 391 | if pos < 0 { 392 | pos = int32(oldDataLen) 393 | } 394 | 395 | end := uint32(pos) + uint32(len(dataRaw)) 396 | 397 | if end > allocSize { 398 | return errors.New("pos+len > allocSize") 399 | } 400 | 401 | _, err = this.descriptor.WriteAt(dataRaw, int64(offset+uint64(headerLen)+uint64(pos))) 402 | if err != nil { 403 | panic(err) 404 | } 405 | 406 | if end > oldDataLen || resetLength { 407 | // need to recompute the header 408 | this.writeHeader(offset, end, allocSize) 409 | } 410 | return nil 411 | } 412 | 413 | func crc(b []byte) uint32 { 414 | return uint32(metro.Hash64(b, 0) >> uint64(32)) 415 | } 416 | 417 | type MultiStore struct { 418 | stores map[string]*StoreItem 419 | root string 420 | sync.RWMutex 421 | } 422 | 423 | func (this *MultiStore) find(storageIdentifier string) *StoreItem { 424 | if storageIdentifier == "" { 425 | storageIdentifier = "default" 426 | } 427 | 428 | this.RLock() 429 | storage, ok := this.stores[storageIdentifier] 430 | this.RUnlock() 431 | if !ok { 432 | this.Lock() 433 | defer this.Unlock() 434 | storage, ok = this.stores[storageIdentifier] 435 | 436 | if !ok { 437 | storage = NewStorage(path.Join(this.root, storageIdentifier)) 438 | this.stores[storageIdentifier] = storage 439 | } 440 | } 441 | return storage 442 | } 443 | 444 | func (this *MultiStore) close(storageIdentifier string) { 445 | this.Lock() 446 | defer this.Unlock() 447 | if storageIdentifier == "" { 448 | storageIdentifier = "default" 449 | } 450 | storage, ok := this.stores[storageIdentifier] 451 | if ok { 452 | storage.descriptor.Close() 453 | storage.Lock() 454 | for name, i := range storage.index { 455 | log.Printf("closing: %s/%s.postings", storage.root, name) 456 | i.descriptor.Close() 457 | } 458 | storage.index = make(map[string]*PostingsList) 459 | storage.Unlock() 460 | log.Printf("closing: %s", storage.path) 461 | } 462 | delete(this.stores, storageIdentifier) 463 | } 464 | 465 | func (this *MultiStore) delete(storageIdentifier string) { 466 | this.Lock() 467 | defer this.Unlock() 468 | if storageIdentifier == "" { 469 | storageIdentifier = "default" 470 | } 471 | storage, ok := this.stores[storageIdentifier] 472 | if ok { 473 | storage.descriptor.Close() 474 | storage.Lock() 475 | for name, i := range storage.index { 476 | log.Printf("closing (tobe deleted): %s/%s.postings", storage.root, name) 477 | i.descriptor.Close() 478 | } 479 | storage.index = make(map[string]*PostingsList) 480 | storage.Unlock() 481 | log.Printf("closing (tobe deleted): %s", storage.path) 482 | os.RemoveAll(storage.root) 483 | } 484 | delete(this.stores, storageIdentifier) 485 | } 486 | 487 | func (this *MultiStore) stats(storageIdentifier string) *StatsOutput { 488 | return this.find(storageIdentifier).stats() 489 | } 490 | 491 | func (this *MultiStore) scan(storageIdentifier string, cb func(uint64, []byte) bool) { 492 | this.find(storageIdentifier).scan(cb) 493 | } 494 | 495 | func (this *MultiStore) compact(storageIdentifier string) error { 496 | _, e := this.find(storageIdentifier).compact() 497 | return e 498 | } 499 | 500 | func (this *MultiStore) ExecuteQuery(storageIdentifier string, query Query, cb func(uint64, []byte) bool) { 501 | this.find(storageIdentifier).ExecuteQuery(query, cb) 502 | } 503 | 504 | func makeTimestamp() int64 { 505 | return time.Now().UnixNano() / int64(time.Millisecond) 506 | } 507 | 508 | func Log(handler http.Handler, tookThresh int64) http.Handler { 509 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 510 | t0 := makeTimestamp() 511 | handler.ServeHTTP(w, r) 512 | took := makeTimestamp() - t0 513 | if took > tookThresh { 514 | log.Printf("%s %s %s took: %d", r.RemoteAddr, r.Method, r.URL, took) 515 | } 516 | }) 517 | } 518 | 519 | const namespaceKey = "namespace" 520 | 521 | func main() { 522 | var pbind = flag.String("bind", ":8000", "address to bind to") 523 | var proot = flag.String("root", "/tmp/rochefort", "root directory") 524 | var ptookThresh = flag.Int("logSlowerThan", 5, "only log queries slower than N milliseconds") 525 | flag.Parse() 526 | 527 | multiStore := &MultiStore{ 528 | stores: make(map[string]*StoreItem), 529 | root: *proot, 530 | } 531 | 532 | os.MkdirAll(*proot, 0700) 533 | namespaces, err := ioutil.ReadDir(*proot) 534 | if err != nil { 535 | panic(err) 536 | } 537 | // open all files in the root 538 | for _, namespace := range namespaces { 539 | if namespace.IsDir() { 540 | // XXX: race against sigterm 541 | go func(namespaceName string) { 542 | files, err := ioutil.ReadDir(path.Join(*proot, namespaceName)) 543 | if err == nil { 544 | for _, file := range files { 545 | if strings.HasSuffix(file.Name(), ".raw") { 546 | multiStore.find(namespaceName) 547 | return 548 | } 549 | } 550 | } 551 | }(namespace.Name()) 552 | 553 | } 554 | } 555 | 556 | sigs := make(chan os.Signal, 1) 557 | signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) 558 | go func() { 559 | <-sigs 560 | log.Printf("\nReceived an interrupt, stopping services...\n") 561 | multiStore.Lock() // dont unlock it 562 | for _, storage := range multiStore.stores { 563 | storage.Lock() // dont unlock it 564 | storage.descriptor.Close() 565 | for name, i := range storage.index { 566 | log.Printf("closing: %s/%s.postings", storage.root, name) 567 | i.descriptor.Close() 568 | } 569 | log.Printf("closing: %s", storage.path) 570 | } 571 | os.Exit(0) 572 | 573 | }() 574 | 575 | unmarshalNamespaceInput := func(w http.ResponseWriter, r *http.Request) (*NamespaceInput, bool) { 576 | defer r.Body.Close() 577 | dataRaw, err := ioutil.ReadAll(r.Body) 578 | if err != nil { 579 | w.WriteHeader(http.StatusInternalServerError) 580 | w.Write([]byte(err.Error())) 581 | return nil, false 582 | } 583 | input := &NamespaceInput{} 584 | err = input.Unmarshal(dataRaw) 585 | if err != nil { 586 | w.WriteHeader(http.StatusInternalServerError) 587 | w.Write([]byte(err.Error())) 588 | return nil, false 589 | } 590 | return input, true 591 | 592 | } 593 | 594 | http.HandleFunc("/close", func(w http.ResponseWriter, r *http.Request) { 595 | input, success := unmarshalNamespaceInput(w, r) 596 | if !success { 597 | return 598 | } 599 | 600 | multiStore.close(input.Namespace) 601 | 602 | w.Header().Set("Content-Type", "application/protobuf") 603 | out := &SuccessOutput{} 604 | m, err := out.Marshal() 605 | if err != nil { 606 | w.WriteHeader(http.StatusInternalServerError) 607 | w.Write([]byte(err.Error())) 608 | return 609 | } 610 | w.Write(m) 611 | }) 612 | 613 | http.HandleFunc("/delete", func(w http.ResponseWriter, r *http.Request) { 614 | input, success := unmarshalNamespaceInput(w, r) 615 | if !success { 616 | return 617 | } 618 | 619 | multiStore.delete(input.Namespace) 620 | 621 | w.Header().Set("Content-Type", "application/protobuf") 622 | out := &SuccessOutput{} 623 | m, err := out.Marshal() 624 | if err != nil { 625 | w.WriteHeader(http.StatusInternalServerError) 626 | w.Write([]byte(err.Error())) 627 | return 628 | } 629 | w.Write(m) 630 | }) 631 | 632 | http.HandleFunc("/compact", func(w http.ResponseWriter, r *http.Request) { 633 | input, success := unmarshalNamespaceInput(w, r) 634 | if !success { 635 | return 636 | } 637 | 638 | multiStore.compact(input.Namespace) 639 | 640 | w.Header().Set("Content-Type", "application/protobuf") 641 | out := &SuccessOutput{} 642 | m, err := out.Marshal() 643 | if err != nil { 644 | w.WriteHeader(http.StatusInternalServerError) 645 | w.Write([]byte(err.Error())) 646 | return 647 | } 648 | w.Write(m) 649 | }) 650 | 651 | http.HandleFunc("/set", func(w http.ResponseWriter, r *http.Request) { 652 | defer r.Body.Close() 653 | dataRaw, err := ioutil.ReadAll(r.Body) 654 | if err != nil { 655 | w.WriteHeader(http.StatusInternalServerError) 656 | w.Write([]byte(err.Error())) 657 | return 658 | } 659 | input := AppendInput{} 660 | err = input.Unmarshal(dataRaw) 661 | if err != nil { 662 | w.WriteHeader(http.StatusInternalServerError) 663 | w.Write([]byte(err.Error())) 664 | return 665 | } 666 | var last *StoreItem 667 | lastns := "" 668 | out := &AppendOutput{} 669 | 670 | if input.AppendPayload != nil { 671 | out.Offset = make([]uint64, len(input.AppendPayload)) 672 | for idx, item := range input.AppendPayload { 673 | if last == nil || lastns != item.Namespace { 674 | lastns = item.Namespace 675 | last = multiStore.find(item.Namespace) 676 | } 677 | offset, err := last.append(item.AllocSize, item.Data) 678 | if err != nil { 679 | w.WriteHeader(http.StatusInternalServerError) 680 | w.Write([]byte(err.Error())) 681 | return 682 | } 683 | 684 | out.Offset[idx] = offset 685 | if item.Tags != nil { 686 | for _, t := range item.Tags { 687 | last.appendPostings(t, offset) 688 | } 689 | } 690 | } 691 | } 692 | 693 | if input.ModifyPayload != nil { 694 | for _, item := range input.ModifyPayload { 695 | if last == nil || lastns != item.Namespace { 696 | lastns = item.Namespace 697 | last = multiStore.find(item.Namespace) 698 | } 699 | err := last.modify(item.Offset, item.Pos, item.Data, item.ResetLength) 700 | if err != nil { 701 | w.WriteHeader(http.StatusInternalServerError) 702 | w.Write([]byte(err.Error())) 703 | return 704 | } 705 | out.ModifiedCount++ 706 | } 707 | } 708 | m, err := out.Marshal() 709 | if err != nil { 710 | w.WriteHeader(http.StatusInternalServerError) 711 | w.Write([]byte(err.Error())) 712 | return 713 | } 714 | 715 | w.Header().Set("Content-Type", "application/protobuf") 716 | w.Write(m) 717 | 718 | }) 719 | 720 | http.HandleFunc("/get", func(w http.ResponseWriter, r *http.Request) { 721 | defer r.Body.Close() 722 | dataRaw, err := ioutil.ReadAll(r.Body) 723 | if err != nil { 724 | w.WriteHeader(http.StatusInternalServerError) 725 | w.Write([]byte(err.Error())) 726 | return 727 | } 728 | input := GetInput{} 729 | err = input.Unmarshal(dataRaw) 730 | if err != nil { 731 | w.WriteHeader(http.StatusInternalServerError) 732 | w.Write([]byte(err.Error())) 733 | return 734 | } 735 | if input.GetPayload != nil { 736 | var last *StoreItem 737 | lastns := "" 738 | out := &GetOutput{ 739 | Data: make([][]byte, len(input.GetPayload)), 740 | } 741 | 742 | for idx, item := range input.GetPayload { 743 | if last == nil || lastns != item.Namespace { 744 | lastns = item.Namespace 745 | last = multiStore.find(item.Namespace) 746 | } 747 | item, err := last.read(item.Offset) 748 | if err != nil { 749 | w.WriteHeader(http.StatusInternalServerError) 750 | w.Write([]byte(err.Error())) 751 | return 752 | } 753 | 754 | out.Data[idx] = item 755 | } 756 | 757 | m, err := out.Marshal() 758 | if err != nil { 759 | w.WriteHeader(http.StatusInternalServerError) 760 | w.Write([]byte(err.Error())) 761 | return 762 | } 763 | 764 | w.Header().Set("Content-Type", "application/protobuf") 765 | w.Write(m) 766 | } 767 | }) 768 | 769 | http.HandleFunc("/scan", func(w http.ResponseWriter, r *http.Request) { 770 | w.Header().Set("Content-Type", "application/octet-stream") 771 | 772 | header := make([]byte, 12) 773 | cb := func(offset uint64, data []byte) bool { 774 | binary.LittleEndian.PutUint32(header[0:], uint32(len(data))) 775 | binary.LittleEndian.PutUint64(header[4:], offset) 776 | 777 | _, err := w.Write(header) 778 | if err != nil { 779 | return false 780 | } 781 | _, err = w.Write(data) 782 | if err != nil { 783 | return false 784 | } 785 | return true 786 | } 787 | multiStore.scan(r.URL.Query().Get(namespaceKey), cb) 788 | }) 789 | 790 | http.HandleFunc("/query", func(w http.ResponseWriter, r *http.Request) { 791 | w.Header().Set("Content-Type", "application/octet-stream") 792 | 793 | defer r.Body.Close() 794 | body, err := ioutil.ReadAll(r.Body) 795 | if err != nil { 796 | w.WriteHeader(http.StatusInternalServerError) 797 | w.Write([]byte(err.Error())) 798 | return 799 | } 800 | 801 | var decoded map[string]interface{} 802 | err = json.Unmarshal(body, &decoded) 803 | if err != nil { 804 | w.WriteHeader(http.StatusInternalServerError) 805 | w.Write([]byte(err.Error())) 806 | return 807 | } 808 | stored := multiStore.find(r.URL.Query().Get(namespaceKey)) 809 | 810 | query, err := fromJSON(stored, decoded) 811 | if err != nil { 812 | w.WriteHeader(http.StatusInternalServerError) 813 | w.Write([]byte(err.Error())) 814 | return 815 | } 816 | 817 | header := make([]byte, 12) 818 | cb := func(offset uint64, data []byte) bool { 819 | binary.LittleEndian.PutUint32(header[0:], uint32(len(data))) 820 | binary.LittleEndian.PutUint64(header[4:], offset) 821 | 822 | _, err := w.Write(header) 823 | if err != nil { 824 | return false 825 | } 826 | _, err = w.Write(data) 827 | if err != nil { 828 | return false 829 | } 830 | return true 831 | } 832 | stored.ExecuteQuery(query, cb) 833 | }) 834 | 835 | http.HandleFunc("/stat", func(w http.ResponseWriter, r *http.Request) { 836 | input, success := unmarshalNamespaceInput(w, r) 837 | if !success { 838 | return 839 | } 840 | 841 | stats := multiStore.stats(input.Namespace) 842 | m, err := stats.Marshal() 843 | if err != nil { 844 | w.WriteHeader(http.StatusInternalServerError) 845 | w.Write([]byte(err.Error())) 846 | return 847 | } 848 | 849 | w.Header().Set("Content-Type", "application/protobuf") 850 | w.Write(m) 851 | }) 852 | 853 | log.Printf("starting http server on %s", *pbind) 854 | err = http.ListenAndServe(*pbind, Log(http.DefaultServeMux, int64(*ptookThresh))) 855 | if err != nil { 856 | log.Fatal(err) 857 | } 858 | } 859 | -------------------------------------------------------------------------------- /query.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "math" 6 | ) 7 | 8 | const ( 9 | NO_MORE = int64(math.MaxInt64) 10 | NOT_READY = int64(-1) 11 | ) 12 | 13 | type Query interface { 14 | advance(int64) int64 15 | Next() int64 16 | GetDocId() int64 17 | } 18 | 19 | type QueryBase struct { 20 | docId int64 21 | } 22 | 23 | func (q *QueryBase) GetDocId() int64 { 24 | return q.docId 25 | } 26 | 27 | type Term struct { 28 | cursor int 29 | postings []int64 30 | QueryBase 31 | } 32 | 33 | func NewTerm(postings []int64) *Term { 34 | return &Term{ 35 | cursor: -1, 36 | postings: postings, 37 | QueryBase: QueryBase{NOT_READY}, 38 | } 39 | 40 | } 41 | func (t *Term) advance(target int64) int64 { 42 | if t.docId == NO_MORE || t.docId == target || target == NO_MORE { 43 | t.docId = target 44 | return t.docId 45 | } 46 | if t.cursor < 0 { 47 | t.cursor = 0 48 | } 49 | 50 | start := t.cursor 51 | end := len(t.postings) 52 | 53 | for start < end { 54 | mid := start + ((end - start) / 2) 55 | current := t.postings[mid] 56 | if current == target { 57 | t.cursor = mid 58 | t.docId = target 59 | return target 60 | } 61 | 62 | if current < target { 63 | start = mid + 1 64 | } else { 65 | end = mid 66 | } 67 | } 68 | if start >= len(t.postings) { 69 | t.docId = NO_MORE 70 | return NO_MORE 71 | } 72 | t.cursor = start 73 | t.docId = t.postings[start] 74 | return t.docId 75 | } 76 | 77 | func (t *Term) Next() int64 { 78 | t.cursor++ 79 | if t.cursor >= len(t.postings) { 80 | t.docId = NO_MORE 81 | } else { 82 | t.docId = t.postings[t.cursor] 83 | } 84 | return t.docId 85 | } 86 | 87 | type BoolQueryBase struct { 88 | queries []Query 89 | } 90 | 91 | func (q *BoolQueryBase) AddSubQuery(sub Query) { 92 | q.queries = append(q.queries, sub) 93 | } 94 | 95 | type BoolOrQuery struct { 96 | BoolQueryBase 97 | QueryBase 98 | } 99 | 100 | func NewBoolOrQuery(queries ...Query) *BoolOrQuery { 101 | return &BoolOrQuery{ 102 | BoolQueryBase: BoolQueryBase{queries}, 103 | QueryBase: QueryBase{NOT_READY}, 104 | } 105 | } 106 | 107 | func (q *BoolOrQuery) advance(target int64) int64 { 108 | new_doc := NO_MORE 109 | n := len(q.queries) 110 | for i := 0; i < n; i++ { 111 | sub_query := q.queries[i] 112 | cur_doc := sub_query.GetDocId() 113 | if cur_doc < target { 114 | cur_doc = sub_query.advance(target) 115 | } 116 | 117 | if cur_doc < new_doc { 118 | new_doc = cur_doc 119 | } 120 | } 121 | q.docId = new_doc 122 | return q.docId 123 | } 124 | 125 | func (q *BoolOrQuery) Next() int64 { 126 | new_doc := NO_MORE 127 | n := len(q.queries) 128 | for i := 0; i < n; i++ { 129 | sub_query := q.queries[i] 130 | cur_doc := sub_query.GetDocId() 131 | if cur_doc == q.docId { 132 | cur_doc = sub_query.Next() 133 | } 134 | 135 | if cur_doc < new_doc { 136 | new_doc = cur_doc 137 | } 138 | } 139 | q.docId = new_doc 140 | return new_doc 141 | } 142 | 143 | type BoolAndQuery struct { 144 | BoolQueryBase 145 | QueryBase 146 | } 147 | 148 | func NewBoolAndQuery(queries ...Query) *BoolAndQuery { 149 | return &BoolAndQuery{ 150 | BoolQueryBase: BoolQueryBase{queries}, 151 | QueryBase: QueryBase{NOT_READY}, 152 | } 153 | } 154 | 155 | func (q *BoolAndQuery) nextAndedDoc(target int64) int64 { 156 | // initial iteration skips queries[0] 157 | n := len(q.queries) 158 | for i := 1; i < n; i++ { 159 | sub_query := q.queries[i] 160 | if sub_query.GetDocId() < target { 161 | sub_query.advance(target) 162 | } 163 | 164 | if sub_query.GetDocId() == target { 165 | continue 166 | } 167 | 168 | target = q.queries[0].advance(sub_query.GetDocId()) 169 | i = 0 //restart the loop from the first query 170 | } 171 | q.docId = target 172 | return q.docId 173 | } 174 | 175 | func (q *BoolAndQuery) advance(target int64) int64 { 176 | if len(q.queries) == 0 { 177 | q.docId = NO_MORE 178 | return NO_MORE 179 | } 180 | 181 | return q.nextAndedDoc(q.queries[0].advance(target)) 182 | } 183 | 184 | func (q *BoolAndQuery) Next() int64 { 185 | if len(q.queries) == 0 { 186 | q.docId = NO_MORE 187 | return NO_MORE 188 | } 189 | 190 | // XXX: pick cheapest leading query 191 | return q.nextAndedDoc(q.queries[0].Next()) 192 | } 193 | 194 | /* 195 | 196 | { 197 | and: [{"or": [{"tag":"b"}]}] 198 | } 199 | 200 | */ 201 | 202 | func fromJSON(store *StoreItem, input interface{}) (Query, error) { 203 | mapped, ok := input.(map[string]interface{}) 204 | queries := []Query{} 205 | if ok { 206 | 207 | if v, ok := mapped["tag"]; ok && v != nil { 208 | value, ok := v.(string) 209 | if !ok { 210 | return nil, errors.New("[tag] must be a string") 211 | } 212 | pl := store.GetPostingsList(value) 213 | if pl == nil { 214 | queries = append(queries, NewTerm([]int64{})) 215 | } else { 216 | queries = append(queries, pl.newTermQuery()) 217 | } 218 | } 219 | if v, ok := mapped["and"]; ok && v != nil { 220 | list, ok := v.([]interface{}) 221 | if ok { 222 | and := NewBoolAndQuery([]Query{}...) 223 | for _, subQuery := range list { 224 | q, err := fromJSON(store, subQuery) 225 | if err != nil { 226 | return nil, err 227 | } 228 | and.AddSubQuery(q) 229 | } 230 | queries = append(queries, and) 231 | } else { 232 | return nil, errors.New("[or] takes array of subqueries") 233 | } 234 | } 235 | 236 | if v, ok := mapped["or"]; ok && v != nil { 237 | list, ok := v.([]interface{}) 238 | if ok { 239 | or := NewBoolOrQuery([]Query{}...) 240 | for _, subQuery := range list { 241 | q, err := fromJSON(store, subQuery) 242 | if err != nil { 243 | return nil, err 244 | } 245 | or.AddSubQuery(q) 246 | } 247 | queries = append(queries, or) 248 | } else { 249 | return nil, errors.New("[and] takes array of subqueries") 250 | } 251 | } 252 | } 253 | 254 | if len(queries) == 1 { 255 | return queries[0], nil 256 | } 257 | 258 | return NewBoolAndQuery(queries...), nil 259 | } 260 | -------------------------------------------------------------------------------- /query_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func postingsList(n int) []int64 { 8 | list := make([]int64, n) 9 | for i := 0; i < n; i++ { 10 | list[i] = int64(i) * 3 11 | } 12 | return list 13 | } 14 | 15 | func query(query Query) []int64 { 16 | out := []int64{} 17 | for query.Next() != NO_MORE { 18 | out = append(out, query.GetDocId()) 19 | } 20 | return out 21 | } 22 | 23 | func eq(t *testing.T, a, b []int64) { 24 | if len(a) != len(b) { 25 | t.Logf("len(a) != len(b) ; len(a) = %d, len(b) = %d", len(a), len(b)) 26 | t.FailNow() 27 | } 28 | 29 | for i, _ := range a { 30 | if a[i] != b[i] { 31 | t.Log("a[i] != b[i]") 32 | t.FailNow() 33 | } 34 | } 35 | } 36 | 37 | func BenchmarkNext1000(b *testing.B) { 38 | x := postingsList(1000) 39 | 40 | for n := 0; n < b.N; n++ { 41 | sum := int64(0) 42 | q := NewTerm(x) 43 | for q.Next() != NO_MORE { 44 | sum += q.GetDocId() 45 | } 46 | } 47 | } 48 | 49 | func BenchmarkOr1000(b *testing.B) { 50 | x := postingsList(1000) 51 | y := postingsList(1000) 52 | 53 | for n := 0; n < b.N; n++ { 54 | sum := int64(0) 55 | q := NewBoolOrQuery( 56 | NewTerm(x), 57 | NewTerm(y), 58 | ) 59 | 60 | for q.Next() != NO_MORE { 61 | sum += q.GetDocId() 62 | } 63 | } 64 | } 65 | 66 | func BenchmarkAnd1000(b *testing.B) { 67 | x := postingsList(1000000) 68 | y := postingsList(1000) 69 | 70 | for n := 0; n < b.N; n++ { 71 | sum := int64(0) 72 | q := NewBoolAndQuery( 73 | NewTerm(x), 74 | NewTerm(y), 75 | ) 76 | 77 | for q.Next() != NO_MORE { 78 | sum += q.GetDocId() 79 | } 80 | } 81 | } 82 | 83 | func TestModify(t *testing.T) { 84 | a := postingsList(100) 85 | b := postingsList(1000) 86 | c := postingsList(10000) 87 | d := postingsList(100000) 88 | e := postingsList(1000000) 89 | 90 | eq(t, a, query(NewTerm(a))) 91 | eq(t, b, query(NewTerm(b))) 92 | eq(t, c, query(NewTerm(c))) 93 | eq(t, d, query(NewTerm(d))) 94 | eq(t, e, query(NewTerm(e))) 95 | 96 | eq(t, b, query(NewBoolOrQuery( 97 | NewTerm(a), 98 | NewTerm(b), 99 | ))) 100 | 101 | eq(t, c, query(NewBoolOrQuery( 102 | NewTerm(a), 103 | NewTerm(b), 104 | NewTerm(c), 105 | ))) 106 | 107 | eq(t, e, query(NewBoolOrQuery( 108 | NewTerm(a), 109 | NewTerm(b), 110 | NewTerm(c), 111 | NewTerm(d), 112 | NewTerm(e), 113 | ))) 114 | 115 | eq(t, a, query(NewBoolAndQuery( 116 | NewTerm(a), 117 | NewTerm(b), 118 | NewTerm(c), 119 | NewTerm(d), 120 | NewTerm(e), 121 | ))) 122 | 123 | eq(t, a, query(NewBoolAndQuery( 124 | NewTerm(a), 125 | NewTerm(b), 126 | NewTerm(c), 127 | NewTerm(d), 128 | NewTerm(e), 129 | ))) 130 | 131 | eq(t, b, query(NewBoolAndQuery( 132 | NewBoolOrQuery( 133 | NewTerm(a), 134 | NewTerm(b), 135 | ), 136 | NewTerm(b), 137 | NewTerm(c), 138 | NewTerm(d), 139 | NewTerm(e), 140 | ))) 141 | 142 | eq(t, c, query(NewBoolAndQuery( 143 | NewBoolOrQuery( 144 | NewTerm(a), 145 | NewTerm(b), 146 | NewBoolAndQuery( 147 | NewTerm(c), 148 | NewTerm(d), 149 | ), 150 | ), 151 | NewTerm(d), 152 | NewTerm(e), 153 | ))) 154 | } 155 | -------------------------------------------------------------------------------- /test_everything.sh: -------------------------------------------------------------------------------- 1 | export ROCHEFORT_TEST=http://localhost:8000 2 | cd clients && cd java && mvn test && cd ../ruby && rake test && cd ../js/ && npm test 3 | --------------------------------------------------------------------------------