├── .gitignore
├── clients
└── py
│ ├── leveldb_server_client.egg-info
│ ├── dependency_links.txt
│ ├── top_level.txt
│ ├── SOURCES.txt
│ └── PKG-INFO
│ ├── dist
│ └── leveldb_server_client-0.1.0-py2.6.egg
│ ├── leveldbClient
│ ├── __init__.py
│ └── database.py
│ ├── build
│ └── lib.linux-i686-2.6
│ │ └── leveldbClient
│ │ ├── __init__.py
│ │ └── database.py
│ └── setup.py
├── license.txt
├── README.textile
└── leveldb-server.py
/.gitignore:
--------------------------------------------------------------------------------
1 | level.db
2 | *.pyc
3 | clients/py/build/lib
4 |
--------------------------------------------------------------------------------
/clients/py/leveldb_server_client.egg-info/dependency_links.txt:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/clients/py/leveldb_server_client.egg-info/top_level.txt:
--------------------------------------------------------------------------------
1 | leveldbClient
2 |
--------------------------------------------------------------------------------
/clients/py/dist/leveldb_server_client-0.1.0-py2.6.egg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/srinikom/leveldb-server/HEAD/clients/py/dist/leveldb_server_client-0.1.0-py2.6.egg
--------------------------------------------------------------------------------
/clients/py/leveldb_server_client.egg-info/SOURCES.txt:
--------------------------------------------------------------------------------
1 | setup.py
2 | leveldbClient/__init__.py
3 | leveldbClient/database.py
4 | leveldb_server_client.egg-info/PKG-INFO
5 | leveldb_server_client.egg-info/SOURCES.txt
6 | leveldb_server_client.egg-info/dependency_links.txt
7 | leveldb_server_client.egg-info/top_level.txt
--------------------------------------------------------------------------------
/clients/py/leveldbClient/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | #Copyright (c) 2011 Fabula Solutions. All rights reserved.
3 | #Use of this source code is governed by a BSD-style license that can be
4 | #found in the license.txt file.
5 |
6 | """The client for leveldb-server."""
7 |
8 | version = "0.1.0"
9 | version_info = (0, 1, 0)
10 |
--------------------------------------------------------------------------------
/clients/py/leveldb_server_client.egg-info/PKG-INFO:
--------------------------------------------------------------------------------
1 | Metadata-Version: 1.0
2 | Name: leveldb-server-client
3 | Version: 0.1.0
4 | Summary: client for leveldb-server
5 | Home-page: https://github.com/srinikom/leveldb-server
6 | Author: Srini
7 | Author-email: srini@fabulasolutions.com
8 | License: BSD
9 | Description: UNKNOWN
10 | Platform: UNKNOWN
11 |
--------------------------------------------------------------------------------
/clients/py/build/lib.linux-i686-2.6/leveldbClient/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | #Copyright (c) 2011 Fabula Solutions. All rights reserved.
3 | #Use of this source code is governed by a BSD-style license that can be
4 | #found in the license.txt file.
5 |
6 | """The client for leveldb-server."""
7 |
8 | version = "0.1.0"
9 | version_info = (0, 1, 0)
10 |
--------------------------------------------------------------------------------
/clients/py/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | #Copyright (c) 2011 Fabula Solutions. All rights reserved.
3 | #Use of this source code is governed by a BSD-style license that can be
4 | #found in the license.txt file.
5 |
6 | import distutils.core
7 | import sys
8 | # Importing setuptools adds some features like "setup.py develop", but
9 | # it's optional so swallow the error if it's not there.
10 | try:
11 | import setuptools
12 | except ImportError:
13 | pass
14 |
15 | kwargs = {}
16 |
17 | version = "0.1.0"
18 |
19 | distutils.core.setup(
20 | name="leveldb-server client",
21 | version=version,
22 | packages = ["leveldbClient"],
23 | author="Srini",
24 | author_email="srini@fabulasolutions.com",
25 | url="https://github.com/srinikom/leveldb-server",
26 | license="BSD",
27 | description="client for leveldb-server",
28 | **kwargs
29 | )
30 |
--------------------------------------------------------------------------------
/clients/py/leveldbClient/database.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | #Copyright (c) 2011 Fabula Solutions. All rights reserved.
3 | #Use of this source code is governed by a BSD-style license that can be
4 | #found in the license.txt file.
5 |
6 | # leveldb client
7 | import zmq
8 | import threading
9 | import time
10 | import json
11 |
12 | class leveldb(object):
13 | """leveldb client"""
14 | def __init__(self, host="tcp://127.0.0.1:5147", timeout=10*1000):
15 | self.host = host
16 | self.timeout = timeout
17 | self.connect()
18 |
19 | def connect(self):
20 | self.context = zmq.Context()
21 | self.socket = self.context.socket(zmq.XREQ)
22 | self.socket.connect(self.host)
23 |
24 | def get(self, key):
25 | self.socket.send_multipart(['get', json.dumps(key)])
26 | return self.socket.recv_multipart()[0]
27 |
28 | def put(self, key, value):
29 | self.socket.send_multipart(['put', json.dumps([key, value])])
30 | return self.socket.recv_multipart()[0]
31 |
32 | def delete(self, key):
33 | self.socket.send_multipart(['delete', json.dumps(key)])
34 | return self.socket.recv_multipart()[0]
35 |
36 | def range(self, start=None, end=None):
37 | self.socket.send_multipart(['range', json.dumps([start, end])])
38 | return self.socket.recv_multipart()[0]
39 |
40 | def close(self):
41 | self.socket.close()
42 | self.context.term()
43 |
44 |
--------------------------------------------------------------------------------
/clients/py/build/lib.linux-i686-2.6/leveldbClient/database.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | #Copyright (c) 2011 Fabula Solutions. All rights reserved.
3 | #Use of this source code is governed by a BSD-style license that can be
4 | #found in the license.txt file.
5 |
6 | # leveldb client
7 | import zmq
8 | import threading
9 | import time
10 | import json
11 |
12 | class leveldb(object):
13 | """leveldb client"""
14 | def __init__(self, host="tcp://127.0.0.1:5147", timeout=10*1000):
15 | self.host = host
16 | self.timeout = timeout
17 | self.connect()
18 |
19 | def connect(self):
20 | self.context = zmq.Context()
21 | self.socket = self.context.socket(zmq.XREQ)
22 | self.socket.connect(self.host)
23 |
24 | def get(self, key):
25 | self.socket.send_multipart(['get', json.dumps(key)])
26 | return self.socket.recv_multipart()[0]
27 |
28 | def put(self, key, value):
29 | self.socket.send_multipart(['put', json.dumps([key, value])])
30 | return self.socket.recv_multipart()[0]
31 |
32 | def delete(self, key):
33 | self.socket.send_multipart(['delete', json.dumps(key)])
34 | return self.socket.recv_multipart()[0]
35 |
36 | def range(self, start=None, end=None):
37 | self.socket.send_multipart(['range', json.dumps([start, end])])
38 | return self.socket.recv_multipart()[0]
39 |
40 | def close(self):
41 | self.socket.close()
42 | self.context.term()
43 |
44 |
--------------------------------------------------------------------------------
/license.txt:
--------------------------------------------------------------------------------
1 | Copyright (c) 2011 Fabula Solutions. All rights reserved.
2 |
3 | Redistribution and use in source and binary forms, with or without
4 | modification, are permitted provided that the following conditions are
5 | met:
6 |
7 | * Redistributions of source code must retain the above copyright
8 | notice, this list of conditions and the following disclaimer.
9 | * Redistributions in binary form must reproduce the above
10 | copyright notice, this list of conditions and the following disclaimer
11 | in the documentation and/or other materials provided with the
12 | distribution.
13 | * Neither the name of Fabula Solutions nor the names of its
14 | contributors may be used to endorse or promote products derived from
15 | this software without specific prior written permission.
16 |
17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 |
29 |
--------------------------------------------------------------------------------
/README.textile:
--------------------------------------------------------------------------------
1 | h2. leveldb-server
2 |
3 | * Async leveldb server and client based on zeromq
4 | * Storage engine *"leveldb":http://code.google.com/p/leveldb/*
5 | * Networking library *"zeromq":http://www.zeromq.org/*
6 | * We use leveldb-server at *"Safebox":http://safebox.fabulasolutions.com/*
7 |
8 | h3. License
9 |
10 | New BSD license. Please see license.txt for more details.
11 |
12 | h2. Features
13 |
14 | * Very simple key-value storage
15 | * Data is sorted by key - allows @range@ queries
16 | * Data is automatically compressed
17 | * Can act as persistent cache
18 | * For our use at *"Safebox":http://safebox.fabulasolutions.com/* it replaced memcached+mysql
19 | * Simple backups @cp -rf level.db backup.db@
20 | * Networking/wiring from @zeromq@ messaging library - allows many topologies
21 | * Async server for scalability and capacity
22 | * Sync client for easy coding
23 | * Easy polyglot client bindings. See *"zmq bindings":http://www.zeromq.org/bindings:_start*
24 |
25 |
26 | >>> db.put("k3", "v3")
27 | 'True'
28 | >>> db.get("k3")
29 | 'v3'
30 | >>> db.range()
31 | '[{"k1": "v1"}, {"k2": "v2"}, {"k3": "v3"}]'
32 | >>> db.range("k1", "k2")
33 | '[{"k1": "v1"}, {"k2": "v2"}]'
34 | >>> db.delete('k1')
35 | >>>
36 |
37 |
38 | Will be adding high availability, replication and autosharding using the same zeromq framework.
39 |
40 | h3. Dependencies
41 |
42 |
43 | python 2.6+ (older versions with simplejson)
44 | zmq
45 | pyzmq
46 | leveldb
47 | pyleveldb
48 |
49 |
50 | h2. Getting Started
51 |
52 | Instructions for an EC2 Ubuntu box.
53 |
54 | h3. Installing zeromq
55 |
56 |
57 | wget http://download.zeromq.org/zeromq-2.1.10.tar.gz
58 | tar xvfz zeromq-2.1.10.tar.gz
59 | cd zeromq-2.1.10
60 | sudo ./configure
61 | sudo make
62 | sudo make install
63 |
64 |
65 | h3. Installing pyzmq
66 |
67 |
68 | wget https://github.com/zeromq/pyzmq/downloads/pyzmq-2.1.10.tar.gz
69 | tar xvfz pyzmq-2.1.10.tar.gz
70 | cd pyzmq-2.1.10/
71 | sudo python setup.py configure --zmq=/usr/local/lib/
72 | sudo python setup.py install
73 |
74 |
75 | h3. Installing leveldb and pyleveldb
76 |
77 |
78 | svn checkout http://py-leveldb.googlecode.com/svn/trunk/ py-leveldb-read-only
79 | cd py-leveldb-read-only
80 | sudo compile_leveldb.sh
81 | sudo python setup.py install
82 |
83 |
84 | h3. Starting the "leveldb-server":https://github.com/srinikom/leveldb-server/blob/master/leveldb-server.py
85 |
86 |
87 | > python leveldb-server.py -h
88 | Usage: leveldb-server.py
89 | -p [port and host settings] Default: tcp://127.0.0.1:5147
90 | -f [database file name] Default: level.db
91 |
92 | leveldb-server
93 |
94 | Options:
95 | --version show program's version number and exit
96 | -h, --help show this help message and exit
97 | -p HOST, --host=HOST
98 | -d DBFILE, --dbfile=DBFILE
99 | > python leveldb-server.py
100 |
101 |
102 | h3. Using the "leveldb-client-py":https://github.com/srinikom/leveldb-server/blob/master/clients/py/leveldbClient/database.py
103 |
104 |
105 | > cd clients/py/
106 | > sudo python setup.py install
107 | > python
108 | >>> from leveldbClient import database
109 | >>> db = database.leveldb()
110 | >>> db.get("Key")
111 | >>> db.put("K", "V")
112 | >>> db.range()
113 | >>> db.range(start, end)
114 | >>> db.delete("K")
115 |
116 |
117 | h2. Backups
118 |
119 |
120 | > cp -rpf level.db backup.db
121 |
122 |
123 | h2. Known issues and work in progress
124 |
125 | Would love your pull requests on
126 | * Benchmarking and performance analysis
127 | * client libraries for other languages
128 | * [issue] zeromq performance issues with 1M+ inserts at a time
129 | * [feature] timeouts in client library
130 | * [feature] support for counters
131 | * [feature] limit support in range queries
132 | * Serializing and seperate threads for get/put/range in leveldb-server
133 | * HA/replication/autosharding and possibly pub-sub for replication
134 |
135 | h2. Thanks
136 |
137 | Thanks to all the folks who have contributed to all the dependencies. Special thanks to pyzmq/examples/mongo* author for inspiration.
138 |
--------------------------------------------------------------------------------
/leveldb-server.py:
--------------------------------------------------------------------------------
1 | #Copyright (c) 2011 Fabula Solutions. All rights reserved.
2 | #Use of this source code is governed by a BSD-style license that can be
3 | #found in the license.txt file.
4 |
5 | # leveldb server
6 | import threading
7 | import zmq
8 | import leveldb
9 | import json
10 | import optparse
11 | from time import sleep
12 |
13 | class workerThread(threading.Thread):
14 | """workerThread"""
15 | def __init__(self, context, db):
16 | threading.Thread.__init__ (self)
17 | self.context = context
18 | self.db = db
19 | self.running = True
20 | self.processing = False
21 | self.socket = self.context.socket(zmq.XREQ)
22 |
23 | def run(self):
24 | self.socket.connect('inproc://backend')
25 | while self.running:
26 | try:
27 | msg = self.socket.recv_multipart()
28 | except zmq.ZMQError:
29 | self.running = False
30 | continue
31 |
32 | self.processing = True
33 | if len(msg) != 3:
34 | value = 'None'
35 | reply = [msg[0], value]
36 | self.socket.send_multipart(reply)
37 | continue
38 | id = msg[0]
39 | op = msg[1]
40 | data = json.loads(msg[2])
41 | reply = [id]
42 | if op == 'get':
43 | try:
44 | value = self.db.Get(data)
45 | except:
46 | value = ""
47 | reply.append(value)
48 |
49 | elif op == 'put':
50 | try:
51 | self.db.Put(data[0], data[1])
52 | value = "True"
53 | except:
54 | value = ""
55 | reply.append(value)
56 |
57 | elif op == 'delete':
58 | self.db.Delete(data)
59 | value = ""
60 | reply.append(value)
61 |
62 | elif op == 'range':
63 | start = data[0]
64 | end = data[1]
65 | if start and end:
66 | try:
67 | arr = []
68 | for value in self.db.RangeIter(start, end):
69 | arr.append({value[0]: value[1]})
70 | reply.append(json.dumps(arr))
71 | except:
72 | value = ""
73 | reply.append(value)
74 | else:
75 | try:
76 | arr = []
77 | for value in self.db.RangeIter():
78 | arr.append({value[0]: value[1]})
79 | reply.append(json.dumps(arr))
80 | except:
81 | value = ""
82 | reply.append(value)
83 | else:
84 | value = ""
85 | reply.append(value)
86 | self.socket.send_multipart(reply)
87 | self.processing = False
88 |
89 | def close(self):
90 | self.running = False
91 | while self.processing:
92 | sleep(1)
93 | self.socket.close()
94 |
95 | if __name__ == "__main__":
96 | optparser = optparse.OptionParser(
97 | prog='leveldb-server.py',
98 | version='0.1.1',
99 | description='leveldb-server',
100 | usage='%prog \n\t-p [port and host settings] Default: tcp://127.0.0.1:5147\n' + \
101 | '\t-f [database file name] Default: level.db')
102 | optparser.add_option('--host', '-p', dest='host',
103 | default='tcp://127.0.0.1:5147')
104 | optparser.add_option('--dbfile', '-d', dest='dbfile',
105 | default='level.db')
106 | options, arguments = optparser.parse_args()
107 |
108 | if not (options.host and options.dbfile):
109 | optparser.print_help()
110 |
111 | print "Starting leveldb-server %s" % options.host
112 | context = zmq.Context()
113 | frontend = context.socket(zmq.XREP)
114 | frontend.bind(options.host)
115 | backend = context.socket(zmq.XREQ)
116 | backend.bind('inproc://backend')
117 |
118 | poll = zmq.Poller()
119 | poll.register(frontend, zmq.POLLIN)
120 | poll.register(backend, zmq.POLLIN)
121 |
122 | db = leveldb.LevelDB(options.dbfile)
123 |
124 | workers = []
125 | for i in xrange(3):
126 | worker = workerThread(context, db)
127 | worker.start()
128 | workers.append(worker)
129 |
130 | try:
131 | while True:
132 | sockets = dict(poll.poll())
133 | if frontend in sockets:
134 | if sockets[frontend] == zmq.POLLIN:
135 | msg = frontend.recv_multipart()
136 | backend.send_multipart(msg)
137 |
138 | if backend in sockets:
139 | if sockets[backend] == zmq.POLLIN:
140 | msg = backend.recv_multipart()
141 | frontend.send_multipart(msg)
142 | except KeyboardInterrupt:
143 | for worker in workers:
144 | worker.close()
145 | frontend.close()
146 | backend.close()
147 | context.term()
148 |
149 |
--------------------------------------------------------------------------------