├── weave
├── minimal
│ ├── __init__.py
│ ├── constants.py
│ ├── compat.py
│ ├── misc.py
│ ├── user.py
│ ├── utils.py
│ └── storage.py
├── static
│ └── scratch.png
└── __init__.py
├── MANIFEST.in
├── site
├── setup.png
└── index.html
├── .gitignore
├── setup.py
├── LICENSE
└── README.md
/weave/minimal/__init__.py:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include weave/static/scratch.png
2 |
--------------------------------------------------------------------------------
/site/setup.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/posativ/weave-minimal/HEAD/site/setup.png
--------------------------------------------------------------------------------
/weave/static/scratch.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/posativ/weave-minimal/HEAD/weave/static/scratch.png
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | *.pyc
2 | *.egg-*
3 |
4 | /.Python
5 | /bin/
6 | /build/
7 | /dist/
8 | /include/
9 | /lib/
10 | /share/
11 |
--------------------------------------------------------------------------------
/site/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Firefox Sync Setup with weave-minimal
5 |
6 |
7 |
8 |
14 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/weave/minimal/constants.py:
--------------------------------------------------------------------------------
1 | # -*- encoding: utf-8 -*-
2 |
3 | WEAVE_INVALID_USER = "3" # Invalid/missing username
4 | WEAVE_INVALID_WRITE = "4" # Attempt to overwrite data that can't be
5 | WEAVE_MALFORMED_JSON = "6" # Json parse failure
6 | WEAVE_MISSING_PASSWORD = "7" # Missing password field
7 | WEAVE_INVALID_WBO = "8" # Invalid Weave Basic Object
8 | WEAVE_WEAK_PASSWORD = "9" # Requested password not strong enough
9 |
--------------------------------------------------------------------------------
/weave/minimal/compat.py:
--------------------------------------------------------------------------------
1 | # -*- encoding: utf-8 -*-
2 | #
3 | # Copyright 2013 Armin Ronacher . All rights reserved.
4 | # License: BSD Style, 2 clauses -- see LICENSE.
5 | #
6 | # http://lucumr.pocoo.org/2013/5/21/porting-to-python-3-redux/
7 |
8 | import sys
9 | PY2K = sys.version_info[0] == 2
10 |
11 | if not PY2K:
12 | iterkeys = lambda d: iter(d.keys())
13 | iteritems = lambda d: iter(d.items())
14 | else:
15 | iterkeys = lambda d: d.iterkeys()
16 | iteritems = lambda d: d.iteritems()
17 |
--------------------------------------------------------------------------------
/weave/minimal/misc.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- encoding: utf-8 -*-
3 |
4 | from werkzeug.wrappers import Response
5 |
6 |
7 | def index(app, environ, request):
8 | return Response("It works!", 200)
9 |
10 |
11 | def captcha_html(app, environ, request, version):
12 | path = environ.get("HTTP_X_SCRIPT_NAME", "/").rstrip("/")
13 | return Response("".join([
14 | '
'
17 | ]), content_type="text/html")
18 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- encoding: utf-8 -*-
3 |
4 | from sys import version_info
5 | from setuptools import setup, find_packages
6 |
7 | if version_info < (3, 0):
8 | requires = ['werkzeug>=0.8']
9 | if version_info < (2, 7):
10 | requires += ["argparse"]
11 | else:
12 | requires = ['werkzeug>=0.9']
13 |
14 | setup(
15 | name='weave-minimal',
16 | version='1.5',
17 | author='Martin Zimmermann',
18 | author_email='info@posativ.org',
19 | packages=find_packages(),
20 | zip_safe=True,
21 | url='https://github.com/posativ/weave-minimal/',
22 | license='BSD revised',
23 | description='lightweight firefox weave/sync server',
24 | classifiers=[
25 | "Development Status :: 5 - Production/Stable",
26 | "Topic :: Internet",
27 | "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
28 | "Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
29 | "License :: OSI Approved :: BSD License",
30 | "Programming Language :: Python :: 2.6",
31 | "Programming Language :: Python :: 2.7",
32 | "Programming Language :: Python :: 3.3"
33 | ],
34 | install_requires=requires,
35 | entry_points={
36 | 'console_scripts':
37 | ['weave-minimal = weave:main'],
38 | },
39 | )
40 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) 2013, Martin Zimmermann and Contributors
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without
5 | modification, are permitted provided that the following conditions are met:
6 |
7 | 1. Redistributions of source code must retain the above copyright notice,
8 | this list of conditions and the following disclaimer.
9 |
10 | 2. Redistributions in binary form must reproduce the above copyright
11 | notice, this list of conditions and the following disclaimer in the
12 | documentation and/or other materials provided with the distribution.
13 |
14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
15 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
18 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
20 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
21 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
23 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 |
25 | The views and conclusions contained in the software and documentation are
26 | those of the authors and should not be interpreted as representing official
27 | policies, either expressed or implied, of Martin Zimmermann .
--------------------------------------------------------------------------------
/weave/minimal/user.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- encoding: utf-8 -*-
3 |
4 | import os
5 | import sqlite3
6 |
7 | from werkzeug.wrappers import Response
8 |
9 | from weave.minimal.utils import login, BadRequest
10 | from weave.minimal.constants import (
11 | WEAVE_INVALID_WRITE, WEAVE_MISSING_PASSWORD,
12 | WEAVE_WEAK_PASSWORD, WEAVE_INVALID_USER)
13 |
14 |
15 | @login(['DELETE', 'POST'])
16 | def index(app, environ, request, version, uid):
17 |
18 | # Returns 1 if the uid is in use, 0 if it is available.
19 | if request.method in ['HEAD']:
20 | return Response('', 200)
21 |
22 | elif request.method in ['GET']:
23 | if not [p for p in os.listdir(app.data_dir) if p.split('.', 1)[0] == uid]:
24 | code = '0' if app.registration else '1'
25 | else:
26 | code = '1'
27 | return Response(code, 200)
28 |
29 | # Requests that an account be created for uid
30 | elif request.method == 'PUT':
31 | if app.registration and not [p for p in os.listdir(app.data_dir) if p.startswith(uid)]:
32 |
33 | try:
34 | passwd = request.get_json()['password']
35 | except (KeyError, TypeError):
36 | raise BadRequest(WEAVE_MISSING_PASSWORD)
37 |
38 | try:
39 | con = sqlite3.connect(app.dbpath(uid, passwd))
40 | con.commit()
41 | con.close()
42 | except IOError:
43 | raise BadRequest(WEAVE_INVALID_WRITE)
44 | return Response(uid, 200)
45 |
46 | raise BadRequest(WEAVE_INVALID_WRITE)
47 |
48 | elif request.method == 'POST':
49 | return Response('Not Implemented', 501)
50 |
51 | elif request.method == 'DELETE':
52 | if request.authorization.username != uid:
53 | return Response('Not Authorized', 401)
54 |
55 | try:
56 | os.remove(app.dbpath(uid, request.authorization.password))
57 | except OSError:
58 | pass
59 | return Response('0', 200)
60 |
61 |
62 | @login(['POST'])
63 | def change_password(app, environ, request, version, uid):
64 | """POST https://server/pathname/version/username/password"""
65 |
66 | if not [p for p in os.listdir(app.data_dir) if p.split('.', 1)[0] == uid]:
67 | return Response(WEAVE_INVALID_USER, 404)
68 |
69 | if len(request.get_data(as_text=True)) == 0:
70 | return Response(WEAVE_MISSING_PASSWORD, 400)
71 | elif len(request.get_data(as_text=True)) < 4:
72 | return Response(WEAVE_WEAK_PASSWORD, 400)
73 |
74 | old_dbpath = app.dbpath(uid, request.authorization.password)
75 | new_dbpath = app.dbpath(uid, request.get_data(as_text=True))
76 | try:
77 | os.rename(old_dbpath, new_dbpath)
78 | except OSError:
79 | return Response(WEAVE_INVALID_WRITE, 503)
80 |
81 | return Response('success', 200)
82 |
--------------------------------------------------------------------------------
/weave/minimal/utils.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- encoding: utf-8 -*-
3 |
4 | import pkg_resources
5 | werkzeug = pkg_resources.get_distribution("werkzeug")
6 |
7 | import re
8 | import json
9 | import base64
10 | import struct
11 |
12 | from os.path import isfile
13 | from hashlib import sha1
14 |
15 | from werkzeug.wrappers import Request as _Request, Response
16 | from werkzeug.exceptions import BadRequest as _BadRequest
17 |
18 | from weave.minimal.compat import iterkeys
19 | from weave.minimal.constants import WEAVE_MALFORMED_JSON, WEAVE_INVALID_WBO
20 |
21 |
22 | class BadRequest(_BadRequest):
23 | """Remove fancy HTML from exceptions."""
24 |
25 | def get_headers(self, environ):
26 | return [("Content-Type", "text/plain")]
27 |
28 | def get_body(self, environ):
29 | return self.description
30 |
31 |
32 | class Request(_Request):
33 |
34 | max_content_length = 512 * 1024
35 |
36 | if werkzeug.version.startswith("0.8"):
37 | def get_data(self, **kw):
38 | return self.data.decode('utf-8')
39 |
40 | def get_json(self):
41 | try:
42 | data = json.loads(self.get_data(as_text=True))
43 | except ValueError:
44 | raise BadRequest(WEAVE_MALFORMED_JSON)
45 | else:
46 | if not isinstance(data, (dict, list)):
47 | raise BadRequest(WEAVE_INVALID_WBO)
48 | return data
49 |
50 |
51 | def encode(uid):
52 | if re.search('[^A-Z0-9._-]', uid, re.I):
53 | return base64.b32encode(sha1(uid).digest()).lower()
54 | return uid
55 |
56 |
57 | class login:
58 | """login decorator using HTTP Basic Authentication. Pattern based on
59 | http://flask.pocoo.org/docs/patterns/viewdecorators/"""
60 |
61 | def __init__(self, methods=['GET', 'POST', 'DELETE', 'PUT']):
62 | self.methods = methods
63 |
64 | def __call__(self, f):
65 |
66 | def dec(app, env, req, *args, **kwargs):
67 | """This decorater function will send an authenticate header, if none
68 | is present and denies access, if HTTP Basic Auth fails."""
69 | if req.method not in self.methods:
70 | return f(app, env, req, *args, **kwargs)
71 | if not req.authorization:
72 | response = Response('Unauthorized', 401)
73 | response.www_authenticate.set_basic('Weave')
74 | return response
75 | else:
76 | user = req.authorization.username
77 | passwd = req.authorization.password
78 | if not isfile(app.dbpath(user, passwd)):
79 | return Response('Unauthorized', 401) # kinda stupid
80 | return f(app, env, req, *args, **kwargs)
81 | return dec
82 |
83 |
84 | def wbo2dict(query):
85 | """converts sqlite table to WBO (dict [json-parsable])"""
86 |
87 | res = {'id': query[0], 'modified': round(query[1], 2),
88 | 'sortindex': query[2], 'payload': query[3],
89 | 'parentid': query[4], 'predecessorid': query[5], 'ttl': query[6]}
90 |
91 | for key in list(iterkeys(res)):
92 | if res[key] is None:
93 | res.pop(key)
94 |
95 | return res
96 |
97 |
98 | def convert(value, mime):
99 | """post processor producing lists in application/newlines format."""
100 |
101 | if mime and mime.endswith(('/newlines', '/whoisi')):
102 | try:
103 | value = value["items"]
104 | except (KeyError, TypeError):
105 | pass
106 |
107 | if mime.endswith('/whoisi'):
108 | res = []
109 | for record in value:
110 | js = json.dumps(record)
111 | res.append(struct.pack('!I', len(js)) + js.encode('utf-8'))
112 | rv = b''.join(res)
113 | else:
114 | rv = '\n'.join(json.dumps(item).replace('\n', '\000a') for item in value)
115 | else:
116 |
117 | rv, mime = json.dumps(value), 'application/json'
118 |
119 | return rv, mime, len(value)
120 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | weave-minimal: a Firefox Sync Server that just works™
2 | =====================================================
3 |
4 | **DOES NOT WORK WITH FIREFOX 29 OR LATER**
5 |
6 | This is a lightweight implementation of Mozillas' [User API v1.0][1] and
7 | [Storage API v1.1][2] without LDAP, MySQL, Redis etc. overhead. It is multi
8 | users capable and depends only on [werkzeug][3].
9 |
10 | I mean, *really* lightweight and *really* simple to install. No hg-attack clone
11 | fetch fail apt-get install. It just works.
12 |
13 | Note, that the name originates from the deprecated [Weave Minimal Server][4],
14 | but shares nothing beside the name; see [FSyncMS][5] for a still working, PHP
15 | based sync server.
16 |
17 | [1]: http://docs.services.mozilla.com/reg/apis.html
18 | [2]: http://docs.services.mozilla.com/storage/apis-1.1.html
19 | [3]: http://werkzeug.pocoo.org/
20 | [4]: https://tobyelliott.wordpress.com/2011/03/25/updating-and-deprecating-the-weave-minimal-server/
21 | [5]: https://github.com/balu-/FSyncMS/
22 |
23 | Setup and Configuration
24 | -----------------------
25 |
26 | You need Python 2.6, 2.7 or 3.3. See `weave-minimal --help` for a list of
27 | parameters and a short description.
28 |
29 | $ pip install weave-minimal
30 | $ weave-minimal --enable-registration
31 | * Running on http://127.0.0.1:8080/
32 |
33 | To run weave-minimal as a daemon, consider this SysVinit script:
34 |
35 | ```bash
36 | $ cat /etc/init.d/weave-minimal
37 | #!/bin/sh
38 |
39 | NAME=weave-minimal
40 | USER=www
41 | CMD=/usr/local/bin/weave-minimal
42 |
43 | PORT=8080
44 | DBPATH=/var/lib/weave-minimal/
45 |
46 | if [ ! -d $DBPATH ]; then
47 | mkdir /var/lib/weave-minimal
48 | chown $USER /var/lib/weave-minimal
49 | fi
50 |
51 | case $1 in
52 | start)
53 | echo -n "Starting $NAME."
54 | start-stop-daemon --start --background --pidfile /var/run/$NAME.pid \
55 | --chuid $USER --make-pidfile --exec $CMD -- --data-dir=$DBPATH \
56 | --port=$PORT --enable-registration
57 | ;;
58 | stop) start-stop-daemon --stop --pidfile /var/run/$NAME.pid
59 | ;;
60 | esac
61 | $ chmod +x /etc/init.d/weave-minimal
62 | $ sudo update-rc.d weave-minimal defaults 99
63 | $ sudo invoke-rc.d weave-minimal start
64 | ```
65 |
66 | ### See also
67 |
68 | * [EN: Firefox Sync server right on router][6]
69 | * [DE: Uberspace und dein Firefox Sync Server][7], [FastCGI wrapper][8]
70 |
71 | [6]: http://forums.smallnetbuilder.com/showthread.php?t=10797
72 | [7]: http://christoph-polcin.com/2012/12/31/firefox-minimal-weave-auf-uberspace/
73 | [8]: https://github.com/oa/weave-minimal-uberspace
74 |
75 | Setting up Firefox
76 | ------------------
77 |
78 | 0. **Migrate from the official servers**: write down your email address and sync
79 | key (you can reset your password anyway) and unlink your client. If you want
80 | to keep your previous sync key, enter the key in the advanced settings.
81 |
82 | 1. **Create a new account** in the sync preferences. Choose a valid email
83 | address and password and enter the custom url into the server location
84 | (leave the trailing slash!). If you get an error, check the SSL certificate
85 | first.
86 |
87 | 2. If no errors come up, click continue and wait a minute. If you sync tabs,
88 | quit, re-open and manually sync otherwise you'll get an empty tab list.
89 |
90 | 3. **Connect other clients** is as easy as with the mozilla servers (the client
91 | actually uses mozilla's servers for this): click *I already have an account*
92 | and write the three codes into an already linked browser using *Pair Device*.
93 | Optionally you can use the manual prodecure but the you have to enter your
94 | sync key by hand.
95 |
96 | 4. If you have connected your clients, you can close the registration by running
97 | `weave-minimal` without the `--enable-registration` flag.
98 |
99 | **Q:** Is this implementation standard compliant?
100 | **A:** Yes, it passes the official functional test suite (with [minor
101 | modifications][9]).
102 |
103 | **Q:** Is it compatible with the latest version of Firefox?
104 | **A:** Not guaranteed. There is a new API draft, but not used in
105 | Firefox/Firefox ESR before 2014.
106 |
107 | **Q:** Can I use a custom certificate for HTTPS?
108 | **A:** Yes, but import the CA or visit the url before you enable syncing.
109 | Firefox will show you a misleading error "invalid url" if you did not
110 | accept this cert before!
111 | If you are using Firefox on Android, you have to accept the certificate
112 | with the default Android Browser (called "Browser").
113 | Also [see here](#ssl-and-firefox-for-android) for
114 | information on a bug in Firefox for Android that might
115 | cause you troubles.
116 |
117 | **Q:** It does not sync!
118 | **A:** Make sure, that `$ curl http://example.tld/prefix/user/1.0/example/node/weave`
119 | returns the correct sync url. Next, try to restart your browser. If that
120 | doesn't help, please file a bug report.
121 |
122 | [9]: https://github.com/posativ/weave-minimal/issues/4#issuecomment-8268947
123 |
124 | ### Using a Custom Username
125 |
126 | Mozilla assumes, you're using their services, therefore you can not enter a
127 | non-valid email-address and Firefox will prevent you from doing this. But
128 | there's an alternate way:
129 |
130 | Instead of entering all your details into the first screen of "Firefox Sync
131 | Setup" you click on "I already have a Firefox Sync Account". Before you can go
132 | to the next step, you have to set up a user account in weave.
133 |
134 | $ weave-minimal --register bob:secret123
135 | [info] database for `bob` created at `.data/bob.c203011d1453ba7c`
136 |
137 | Now you can continue your Firefox Sync Setup and click "I don't have the device
138 | with me" and enter your username, password, "use custom server" -> url and
139 | secret passphrase. That's all.
140 |
141 |
142 | Webserver Configuration
143 | -----------------------
144 |
145 | ### using lighttpd and mod_proxy
146 |
147 | To run weave-minimal using [lighttpd][10] and mod_proxy you need to pass the
148 | public url, e.g. `weave-minimal --base-url=http://example.org/sync ...`
149 |
150 | $HTTP["url"] =~ "^/weave/" {
151 | proxy.server = ("" =>
152 | (("host" => "127.0.0.1", "port" => 8080)))
153 | setenv.add-request-header = ("X-Forwarded-Proto" => "https") # optionally for HTTPS
154 | }
155 |
156 | [10]: http://www.lighttpd.net/
157 |
158 | ### nginx
159 |
160 | location ^~ /weave/ {
161 | proxy_set_header Host $host;
162 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
163 | proxy_set_header X-Forwarded-Proto $scheme;
164 | proxy_set_header X-Script-Name /weave;
165 | proxy_pass http://127.0.0.1:8080;
166 | }
167 |
168 | ### using apache and mod_proxy, with SSL
169 |
170 |
171 | ProxyPass http://127.0.0.1:8080
172 | RequestHeader set X-Forwarded-Proto "https"
173 |
174 |
175 | You can skip `RequestHeader`, if apache proxies the service on regular `http`.
176 |
177 | ### SSL and Firefox for Android
178 |
179 | If Firefox for Android fails to sync properly, yet doesn't give any error
180 | messages either, check out `adb logcat`. You might get an error similar to
181 | this:
182 |
183 | javax.net.ssl.SSLPeerUnverifiedException: No peer certificate
184 |
185 | That's because [Firefox for Android has a nasty
186 | bug](https://bugzilla.mozilla.org/show_bug.cgi?id=756763) that doesn't allow
187 | you to use self-signed certificates or certain ciphers.
188 |
189 | For some reason, this error occurs not because your certificate is untrusted,
190 | but because of ciphers. Adding RC4+RSA to the server's cipher list "fixes" that
191 | issue.
192 |
193 | For nginx the code to add would be:
194 |
195 | server {
196 | # ...
197 | # this makes ssl connections less secure!
198 | ssl_ciphers HIGH:!aNULL:!MD5:RC4+RSA;
199 | # ...
200 | }
201 |
202 | Deployment
203 | ----------
204 |
205 | For higher concurrency (if possible at all with SQLite), gevent will be used if
206 | installed (`pip install gevent`). Furthermore, `weave-minimal` exports an
207 | *application* object for uWSGI and Gunicorn, e.g.:
208 |
209 | ```bash
210 | $ env ENABLE_REGISTRATION=1 DATA_DIR=/var/lib/... gunicorn weave -b localhost:1234
211 | ```
212 |
213 | Do *not* use multiple processes to run `weave-minimal`. The code does not
214 | acquire inter-process locks on the database and I have no plans to add an IPC
215 | concurrency pattern to the rather simple code base (programmer's lame excuse,
216 | I know).
217 |
--------------------------------------------------------------------------------
/weave/__init__.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- encoding: utf-8 -*-
3 | #
4 | # Copyright 2013 Martin Zimmermann . All rights reserved.
5 | #
6 | # Redistribution and use in source and binary forms, with or without
7 | # modification, are permitted provided that the following conditions are met:
8 | #
9 | # 1. Redistributions of source code must retain the above copyright notice,
10 | # this list of conditions and the following disclaimer.
11 | #
12 | # 2. Redistributions in binary form must reproduce the above copyright
13 | # notice, this list of conditions and the following disclaimer in the
14 | # documentation and/or other materials provided with the distribution.
15 | #
16 | # The views and conclusions contained in the software and documentation are
17 | # those of the authors and should not be interpreted as representing official
18 | # policies, either expressed or implied, of Martin Zimmermann .
19 | #
20 | # lightweight firefox weave/sync server
21 |
22 | from __future__ import print_function
23 |
24 | import pkg_resources
25 | dist = pkg_resources.get_distribution("weave-minimal")
26 |
27 | try:
28 | import gevent.monkey; gevent.monkey.patch_all()
29 | except ImportError:
30 | pass
31 |
32 | import sys
33 |
34 | if sys.version_info < (2, 7):
35 | reload(sys)
36 | sys.setdefaultencoding("utf-8") # yolo
37 |
38 | import os
39 | import errno
40 | import hashlib
41 | import sqlite3
42 | import logging
43 |
44 | from os.path import join, dirname
45 | from argparse import ArgumentParser, HelpFormatter, SUPPRESS
46 |
47 | try:
48 | from urllib.parse import urlsplit
49 | except ImportError:
50 | from urlparse import urlsplit
51 |
52 | from werkzeug.routing import Map, Rule, BaseConverter
53 | from werkzeug.serving import run_simple
54 | from werkzeug.wrappers import Response
55 | from werkzeug.exceptions import HTTPException, NotFound, NotImplemented
56 |
57 | from werkzeug.wsgi import SharedDataMiddleware
58 |
59 | from weave.minimal import user, storage, misc
60 | from weave.minimal.utils import encode, Request
61 |
62 | logging.basicConfig(
63 | level=logging.INFO,
64 | format="%(asctime)s %(levelname)s: %(message)s")
65 |
66 | logger = logging.getLogger("weave-minimal")
67 |
68 |
69 | class RegexConverter(BaseConverter):
70 | def __init__(self, url_map, *items):
71 | super(RegexConverter, self).__init__(url_map)
72 | self.regex = items[0]
73 |
74 |
75 | url_map = Map([
76 | # reg-server
77 | Rule('/user//', endpoint=user.index,
78 | methods=['GET', 'HEAD', 'PUT', 'DELETE']),
79 | Rule('/user///password',
80 | endpoint=user.change_password, methods=['POST']),
81 | Rule('/user///node/weave',
82 | endpoint=lambda app, env, req, version, uid: Response(req.url_root, 200)),
83 | Rule('/user///password_reset',
84 | endpoint=lambda *args, **kw: NotImplemented()),
85 | Rule('/user///email',
86 | endpoint=lambda *args, **kw: NotImplemented()),
87 |
88 | # some useless UI stuff, not working
89 | Rule('/weave-password-reset', methods=['GET', 'HEAD', 'POST'],
90 | endpoint=lambda app, env, req: NotImplemented()),
91 | Rule('/misc//captcha_html',
92 | endpoint=misc.captcha_html),
93 | Rule('/media/', endpoint=lambda app, env, req: NotImplemented()),
94 |
95 | # info
96 | Rule('/', endpoint=misc.index),
97 | Rule('///info/collections',
98 | endpoint=storage.get_collections_info),
99 | Rule('///info/collection_counts',
100 | endpoint=storage.get_collection_counts),
101 | Rule('///info/collection_usage',
102 | endpoint=storage.get_collection_usage),
103 | Rule('///info/quota',
104 | endpoint=storage.get_quota),
105 |
106 | # storage
107 | Rule('///storage',
108 | endpoint=storage.storage, methods=['DELETE']),
109 | Rule('///storage/',
110 | endpoint=storage.collection, methods=['GET', 'HEAD', 'PUT', 'POST', 'DELETE']),
111 | Rule('///storage//',
112 | endpoint=storage.item, methods=['GET', 'HEAD', 'PUT', 'DELETE']),
113 | ], converters={'re': RegexConverter}, strict_slashes=False)
114 |
115 |
116 | class ReverseProxied(object):
117 | """
118 | Handle X-Script-Name and X-Forwarded-Proto. E.g.:
119 |
120 | location /weave {
121 | proxy_pass http://localhost:8080;
122 | proxy_set_header X-Script-Name /weave;
123 | proxy_set_header X-Forwarded-Proto $scheme;
124 | }
125 |
126 | :param app: the WSGI application
127 | """
128 |
129 | def __init__(self, app, base_url):
130 | self.app = app
131 | self.base_url = base_url
132 |
133 | def __call__(self, environ, start_response):
134 |
135 | prefix = None
136 |
137 | if self.base_url is not None:
138 | scheme, host, prefix, x, y = urlsplit(self.base_url)
139 |
140 | environ['wsgi.url_scheme'] = scheme
141 | environ['HTTP_HOST'] = host
142 |
143 | if 'HTTP_X_FORWARDED_PROTO' in environ:
144 | environ['wsgi.url_scheme'] = environ['HTTP_X_FORWARDED_PROTO']
145 |
146 | script_name = environ.get('HTTP_X_SCRIPT_NAME', prefix)
147 | if script_name:
148 | environ['SCRIPT_NAME'] = script_name
149 | path_info = environ['PATH_INFO']
150 | if path_info.startswith(script_name):
151 | environ['PATH_INFO'] = path_info[len(script_name):]
152 |
153 | return self.app(environ, start_response)
154 |
155 |
156 | class Weave(object):
157 |
158 | salt = r'\x14Q\xd4JbDk\x1bN\x84J\xd0\x05\x8a\x1b\x8b\xa6&V\x1b\xc5\x91\x97\xc4'
159 |
160 | def __init__(self, data_dir, registration):
161 |
162 | try:
163 | os.makedirs(data_dir)
164 | except OSError as ex:
165 | if ex.errno != errno.EEXIST:
166 | raise
167 |
168 | self.data_dir = data_dir
169 | self.registration = registration
170 |
171 | def crypt(self, password):
172 | return hashlib.sha1((self.salt+password).encode('utf-8')).hexdigest()[:16]
173 |
174 | def dbpath(self, user, password):
175 | return join(self.data_dir, (user + '.' + self.crypt(password)))
176 |
177 | def initialize(self, uid, password):
178 |
179 | dbpath = self.dbpath(uid, password)
180 |
181 | try:
182 | os.unlink(dbpath)
183 | except OSError:
184 | pass
185 |
186 | with sqlite3.connect(dbpath) as con:
187 | con.commit()
188 |
189 | logger.info("database for `%s` created at `%s`", uid, dbpath)
190 |
191 | def dispatch(self, request, start_response):
192 | adapter = url_map.bind_to_environ(request.environ)
193 | try:
194 | handler, values = adapter.match()
195 | return handler(self, request.environ, request, **values)
196 | except NotFound:
197 | return Response('Not Found', 404)
198 | except HTTPException as e:
199 | return e
200 |
201 | def wsgi_app(self, environ, start_response):
202 | request = Request(environ)
203 | response = self.dispatch(request, start_response)
204 | if hasattr(response, 'headers'):
205 | response.headers['X-Weave-Backoff'] = 0 # we have no load!1
206 | return response(environ, start_response)
207 |
208 | def __call__(self, environ, start_response):
209 | return self.wsgi_app(environ, start_response)
210 |
211 |
212 | def make_app(data_dir='.data/', base_url=None, register=False):
213 | application = Weave(data_dir, register)
214 | application.wsgi_app = SharedDataMiddleware(application.wsgi_app, {
215 | "/static": join(dirname(__file__), "static")})
216 | application.wsgi_app = ReverseProxied(application.wsgi_app, base_url)
217 | return application
218 |
219 |
220 | def main():
221 |
222 | fmt = lambda prog: HelpFormatter(prog, max_help_position=28)
223 | desc = u"A lightweight Firefox Sync server, that just works™. If it " \
224 | u"doesn't just work for you, please file a bug: " \
225 | u"https://github.com/posativ/weave-minimal/issues"
226 |
227 | parser = ArgumentParser(description=desc, formatter_class=fmt)
228 | option = parser.add_argument
229 |
230 | option("--host", dest="host", default="127.0.0.1", type=str,
231 | metavar="127.0.0.1", help="host interface")
232 | option("--port", dest="port", default=8080, type=int, metavar="8080",
233 | help="port to listen on")
234 | option("--log-file", dest="logfile", default=None, type=str,
235 | metavar="FILE", help="log to a file")
236 |
237 | option("--data-dir", dest="data_dir", default=".data/", metavar="/var/...",
238 | help="directory to store sync data, defaults to .data/")
239 | option("--enable-registration", dest="registration", action="store_true",
240 | help="enable public registration"),
241 | option("--base-url", dest="base_url", default=None, metavar="URL",
242 | help="public URL, e.g. https://example.org/weave/")
243 | option("--register", dest="creds", default=None, metavar="user:pass",
244 | help="register a new user and exit")
245 |
246 | option("--use-reloader", action="store_true", dest="reloader",
247 | help=SUPPRESS, default=False)
248 |
249 | option("--version", action="store_true", dest="version",
250 | help=SUPPRESS, default=False)
251 |
252 | options = parser.parse_args()
253 |
254 | if options.version:
255 | print('weave-minimal', dist.version, end=' ')
256 | print('(Storage API 1.1, User API 1.0)')
257 | sys.exit(0)
258 |
259 | if options.logfile:
260 | handler = logging.FileHandler(options.logfile)
261 |
262 | logger.addHandler(handler)
263 | logging.getLogger('werkzeug').addHandler(handler)
264 |
265 | logger.propagate = False
266 | logging.getLogger('werkzeug').propagate = False
267 |
268 | app = make_app(options.data_dir, options.base_url, options.registration)
269 |
270 | if options.creds:
271 |
272 | try:
273 | username, passwd = options.creds.split(':', 1)
274 | except ValueError:
275 | logger.error("provide credentials as `user:pass`!")
276 | sys.exit(os.EX_DATAERR)
277 |
278 | if len(passwd) < 8:
279 | logger.error("password too short, minimum length is 8")
280 | sys.exit(os.EX_DATAERR)
281 |
282 | app.initialize(encode(username), passwd)
283 | sys.exit(os.EX_OK)
284 |
285 | try:
286 | from gevent.pywsgi import WSGIServer
287 | WSGIServer((options.host, options.port), app).serve_forever()
288 | except ImportError:
289 | run_simple(options.host, options.port, app, use_reloader=options.reloader, threaded=True)
290 |
291 |
292 | if sys.argv[0].endswith(("gunicorn", "uwsgi")):
293 | application = make_app(
294 | data_dir=os.environ.get("DATA_DIR", ".data/"),
295 | base_url=os.environ.get("BASE_URL", None),
296 | register=bool(os.environ.get("ENABLE_REGISTRATION", "0")))
297 |
--------------------------------------------------------------------------------
/weave/minimal/storage.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | # -*- encoding: utf-8 -*-
3 |
4 | import sys
5 | import math
6 | import time
7 | import json
8 | import sqlite3
9 |
10 | from werkzeug.wrappers import Response
11 | from werkzeug.exceptions import PreconditionFailed
12 |
13 | if sys.version_info < (2, 7):
14 | import __builtin__
15 |
16 | class Float(float):
17 |
18 | def __new__(self, x, i):
19 | self.i = i
20 | return float.__new__(self, x)
21 |
22 | def __repr__(self):
23 | return (('%%.%sf' % self.i ) % self)
24 |
25 | defaultround = round
26 | setattr(__builtin__, 'round', lambda x, i: Float(defaultround(x, 2), i))
27 |
28 | from weave.minimal.utils import login, wbo2dict, convert, BadRequest
29 | from weave.minimal.compat import iteritems
30 | from weave.minimal.constants import WEAVE_INVALID_WBO
31 |
32 | FIELDS = ['id', 'modified', 'sortindex', 'payload', 'parentid', 'predecessorid', 'ttl']
33 |
34 |
35 | def iter_collections(dbpath):
36 | """iters all available collection_ids"""
37 | with sqlite3.connect(dbpath) as db:
38 | res = db.execute("SELECT name FROM sqlite_master WHERE type='table';").fetchall()
39 | return [x[0] for x in res]
40 |
41 |
42 | def expire(dbpath, cid):
43 | try:
44 | with sqlite3.connect(dbpath) as db:
45 | db.execute("DELETE FROM %s WHERE (%s - modified) > ttl" % (cid, time.time()))
46 | except sqlite3.OperationalError:
47 | pass
48 |
49 |
50 | def has_modified(since, dbpath, cid):
51 | """On any write transaction (PUT, POST, DELETE), if the collection to be acted
52 | on has been modified since the provided timestamp, the request will fail with
53 | an HTTP 412 Precondition Failed status."""
54 |
55 | with sqlite3.connect(dbpath) as db:
56 |
57 | try:
58 | sql = 'SELECT MAX(modified) FROM %s' % cid
59 | rv = db.execute(sql).fetchone()
60 | except sqlite3.OperationalError:
61 | return False
62 |
63 | return rv and since < rv[0]
64 |
65 |
66 | def set_item(dbpath, uid, cid, data):
67 |
68 | obj = {'id': data['id']}
69 | obj['modified'] = round(time.time(), 2)
70 | obj['payload'] = data.get('payload', None)
71 | obj['payload_size'] = len(obj['payload']) if obj['payload'] else 0
72 | obj['sortindex'] = data.get('sortindex', None)
73 | obj['parentid'] = data.get('parentid', None)
74 | obj['predecessorid'] = data.get('predecessorid', None)
75 | obj['ttl'] = data.get('ttl', None)
76 |
77 | if obj['sortindex']:
78 | try:
79 | obj['sortindex'] = int(math.floor(float(obj['sortindex'])))
80 | except ValueError:
81 | return obj
82 |
83 | with sqlite3.connect(dbpath) as db:
84 | sql = ('main.%s (id VARCHAR(64) PRIMARY KEY, modified FLOAT,'
85 | 'sortindex INTEGER, payload VARCHAR(256),'
86 | 'payload_size INTEGER, parentid VARCHAR(64),'
87 | 'predecessorid VARCHAR(64), ttl INTEGER)') % cid
88 | db.execute("CREATE table IF NOT EXISTS %s;" % sql)
89 |
90 | into = []; values = []
91 | for k,v in iteritems(obj):
92 | into.append(k); values.append(v)
93 |
94 | try:
95 | db.execute("INSERT INTO %s (%s) VALUES (%s);" % \
96 | (cid, ', '.join(into), ','.join(['?' for x in values])), values)
97 | except sqlite3.IntegrityError:
98 | for k,v in iteritems(obj):
99 | if v is None: continue
100 | db.execute('UPDATE %s SET %s=? WHERE id=?;' % (cid, k), [v, obj['id']])
101 | except sqlite3.InterfaceError:
102 | raise ValueError
103 |
104 | return obj
105 |
106 |
107 | @login(['GET', 'HEAD'])
108 | def get_collections_info(app, environ, request, version, uid):
109 | """Returns a hash of collections associated with the account,
110 | Along with the last modified timestamp for each collection.
111 | """
112 | if request.method == 'HEAD' or request.authorization.username != uid:
113 | return Response('Not Authorized', 401)
114 |
115 | dbpath = app.dbpath(uid, request.authorization.password)
116 | ids = iter_collections(dbpath); collections = {}
117 |
118 | with sqlite3.connect(dbpath) as db:
119 | for id in ids:
120 | x = db.execute('SELECT id, MAX(modified) FROM %s;' % id).fetchall()
121 | for k,v in x:
122 | if not k:
123 | continue # XXX: why None, None yields here?
124 | collections[id] = round(v, 2)
125 |
126 | return Response(json.dumps(collections), 200, content_type='application/json',
127 | headers={'X-Weave-Records': str(len(collections))})
128 |
129 |
130 | @login(['GET', 'HEAD'])
131 | def get_collection_counts(app, environ, request, version, uid):
132 | """Returns a hash of collections associated with the account,
133 | Along with the total number of items for each collection.
134 | """
135 | if request.method == 'HEAD' or request.authorization.username != uid:
136 | return Response('Not Authorized', 401)
137 |
138 | dbpath = app.dbpath(uid, request.authorization.password)
139 | ids = iter_collections(dbpath); collections = {}
140 |
141 | with sqlite3.connect(dbpath) as db:
142 | for id in ids:
143 | cur = db.execute('SELECT id FROM %s;' % id)
144 | collections[id] = len(cur.fetchall())
145 |
146 | return Response(json.dumps(collections), 200, content_type='application/json',
147 | headers={'X-Weave-Records': str(len(collections))})
148 |
149 |
150 | @login(['GET', 'HEAD'])
151 | def get_collection_usage(app, environ, request, version, uid):
152 | """Returns a hash of collections associated with the account, along with
153 | the data volume used for each (in K).
154 | """
155 | if request.method == 'HEAD' or request.authorization.username != uid:
156 | return Response('Not Authorized', 401)
157 |
158 | dbpath = app.dbpath(uid, request.authorization.password)
159 | with sqlite3.connect(dbpath) as db:
160 | res = {}
161 | for table in iter_collections(dbpath):
162 | v = db.execute('SELECT SUM(payload_size) FROM %s' % table).fetchone()[0] or 0
163 | res[table] = v/1024.0
164 |
165 | js = json.dumps(res)
166 | return Response(js, 200, content_type='application/json',
167 | headers={'X-Weave-Records': str(len(js))})
168 |
169 |
170 | @login(['GET', 'HEAD'])
171 | def get_quota(app, environ, request, version, uid):
172 | if request.method == 'HEAD' or request.authorization.username != uid:
173 | return Response('Not Authorized', 401)
174 |
175 | dbpath = app.dbpath(uid, request.authorization.password)
176 | with sqlite3.connect(dbpath) as db:
177 | sum = 0
178 | for table in iter_collections(dbpath):
179 | sum += db.execute('SELECT SUM(payload_size) FROM %s' % table).fetchone()[0] or 0
180 | # sum = os.path.getsize(dbpath) # -- real usage
181 |
182 | js = json.dumps([sum/1024.0, None])
183 | return Response(js, 200, content_type='application/json',
184 | headers={'X-Weave-Records': str(len(js))})
185 |
186 |
187 | def storage(app, environ, request, version, uid):
188 |
189 | if request.method == 'DELETE':
190 | if request.headers.get('X-Confirm-Delete', '0') == '1':
191 | app.initialize(uid, request.authorization.password)
192 |
193 | return Response(json.dumps(time.time()), 200)
194 |
195 | return Response('Precondition Failed', 412)
196 |
197 |
198 | @login(['GET', 'HEAD', 'POST', 'PUT', 'DELETE'])
199 | def collection(app, environ, request, version, uid, cid):
200 | """///storage/"""
201 |
202 | if request.method == 'HEAD' or request.authorization.username != uid:
203 | return Response('Not Authorized', 401)
204 |
205 | dbpath = app.dbpath(uid, request.authorization.password)
206 | expire(dbpath, cid)
207 |
208 | ids = request.args.get('ids', None)
209 | offset = request.args.get('offset', None)
210 | older = request.args.get('older', None)
211 | newer = request.args.get('newer', None)
212 | full = request.args.get('full', False)
213 | index_above = request.args.get('index_above', None)
214 | index_below = request.args.get('index_below', None)
215 | limit = request.args.get('limit', None)
216 | offset = request.args.get('offset', None)
217 | sort = request.args.get('sort', None)
218 | parentid = request.args.get('parentid', None)
219 | predecessorid = request.args.get('predecessorid', None)
220 |
221 | try:
222 | older and float(older)
223 | newer and float(newer)
224 | limit and int(limit)
225 | offset and int(offset)
226 | index_above and int(index_above)
227 | index_below and int(index_below)
228 | except ValueError:
229 | raise BadRequest
230 |
231 | if limit is not None:
232 | limit = int(limit)
233 |
234 | if offset is not None:
235 | # we need both
236 | if limit is None:
237 | offset = None
238 | else:
239 | offset = int(offset)
240 |
241 | if not full:
242 | fields = ['id']
243 | else:
244 | fields = FIELDS
245 |
246 | # filters used in WHERE clause
247 | filters = {}
248 | if ids is not None:
249 | filters['id'] = 'IN', '(%s)' % ",".join("'" + x.strip() + "'" for x in ids.split(","))
250 | if older is not None:
251 | filters['modified'] = '<', float(older)
252 | if newer is not None:
253 | filters['modified'] = '>', float(newer)
254 | if index_above is not None:
255 | filters['sortindex'] = '>', int(index_above)
256 | if index_below is not None:
257 | filters['sortindex'] = '<', int(index_below)
258 | if parentid is not None:
259 | filters['parentid'] = '=', "'%s'" % parentid
260 | if predecessorid is not None:
261 | filters['predecessorid'] = '=', "'%s'" % predecessorid
262 |
263 | filter_query, sort_query, limit_query = '', '', ''
264 |
265 | # ORDER BY x ASC|DESC
266 | if sort is not None:
267 | if sort == 'index':
268 | sort_query = ' ORDER BY sortindex DESC'
269 | elif sort == 'oldest':
270 | sort_query = ' ORDER BY modified ASC'
271 | elif sort == 'newest':
272 | sort_query = ' ORDER BY modified DESC'
273 |
274 | # WHERE x 
10 |
11 |
13 |