├── .gitignore ├── conf ├── ffsync ├── ffsync.logrotate ├── nginx.conf └── syncserver.ini ├── manifest.json ├── scripts ├── install ├── remove └── upgrade └── sources ├── .gitignore ├── .travis.yml ├── Dockerfile ├── MANIFEST.in ├── Makefile ├── README.rst ├── requirements.txt ├── setup.py ├── syncserver.ini ├── syncserver.wsgi └── syncserver ├── __init__.py ├── staticnode.py └── tests.ini /.gitignore: -------------------------------------------------------------------------------- 1 | # From kateproject 2 | .kateproject 3 | .kateproject.d 4 | .directory 5 | 6 | -------------------------------------------------------------------------------- /conf/ffsync: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # /etc/init.d/sync 3 | # version 0.1 2013-03-12 (YYYY-MM-DD) 4 | 5 | ### BEGIN INIT INFO 6 | # Provides: sync 7 | # Required-Start: $local_fs $remote_fs 8 | # Required-Stop: $local_fs $remote_fs 9 | # Should-Start: $network 10 | # Should-Stop: $network 11 | # Default-Start: 2 3 4 5 12 | # Default-Stop: 0 1 6 13 | # Short-Description: Mozilla Sync server 14 | # Description: Starts the mozilla sync server 15 | ### END INIT INFO 16 | 17 | # Source function library. 18 | #. /etc/rc.d/init.d/functions 19 | 20 | prog=sync 21 | SYNC_USER=ffsync 22 | SYNC_HOME=/opt/yunohost/ffsync 23 | CPU_COUNT=2 24 | pidfile=/tmp/sync.pid 25 | lockfile=/var/run/sync.lock 26 | conffile=${SYNC_HOME}/syncserver.ini 27 | GUNICORN=${SYNC_HOME}/local/bin/gunicorn 28 | GUNICORN_ARGS="--paste $conffile --access-logfile /var/log/ffsync.log --daemon -p $pidfile" 29 | 30 | start () { 31 | echo -n "Starting $prog" 32 | start-stop-daemon --start -c ${SYNC_USER} --exec $GUNICORN -- $GUNICORN_ARGS 33 | RETVAL=$? 34 | echo 35 | [ $RETVAL = 0 ] && touch ${lockfile} 36 | return $RETVAL 37 | } 38 | 39 | stop() { 40 | echo "Stopping $prog" 41 | start-stop-daemon --stop --quiet --oknodo --pidfile ${pidfile} 42 | #log_end_msg $? 43 | rm -f ${pidfile} 44 | } 45 | 46 | status(){ 47 | if [[ -f ${pidfile} ]]; then 48 | echo "Status: running." 49 | exit 0; 50 | else 51 | echo "Status: not running." 52 | exit 1; 53 | fi 54 | } 55 | 56 | case "$1" in 57 | start) 58 | start 59 | ;; 60 | stop) 61 | stop 62 | ;; 63 | status) 64 | status 65 | ;; 66 | restart) 67 | stop 68 | start 69 | ;; 70 | *) 71 | echo $"Usage: $prog {start|stop|restart|help}" 72 | RETVAL=2 73 | esac 74 | 75 | exit $RETVAL 76 | -------------------------------------------------------------------------------- /conf/ffsync.logrotate: -------------------------------------------------------------------------------- 1 | "/var/log/ffsync.log" { 2 | copytruncate 3 | daily 4 | rotate 7 5 | compress 6 | delaycompress 7 | missingok 8 | notifempty 9 | } 10 | -------------------------------------------------------------------------------- /conf/nginx.conf: -------------------------------------------------------------------------------- 1 | location PATHTOCHANGE { 2 | if ($scheme = http) { 3 | rewrite ^ https://$server_name$request_uri? permanent; 4 | } 5 | proxy_set_header Host $http_host; 6 | proxy_set_header X-Forwarded-Proto $scheme; 7 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 8 | proxy_set_header X-Real-IP $remote_addr; 9 | proxy_redirect off; 10 | proxy_read_timeout 120; 11 | proxy_connect_timeout 10; 12 | proxy_pass http://127.0.0.1:5000/; 13 | 14 | include conf.d/yunohost_panel.conf.inc; 15 | } 16 | -------------------------------------------------------------------------------- /conf/syncserver.ini: -------------------------------------------------------------------------------- 1 | [server:main] 2 | use = egg:gunicorn 3 | host = 0.0.0.0 4 | port = 5000 5 | workers = 1 6 | timeout = 30 7 | 8 | [app:main] 9 | use = egg:syncserver 10 | 11 | [syncserver] 12 | # This must be edited to point to the public URL of your server, 13 | # i.e. the URL as seen by Firefox. 14 | public_url = https://ynhbaseurl/ 15 | 16 | # This defines the database in which to store all server data. 17 | sqluri = pymysql://yunouser:yunopass@localhost/yunobase 18 | 19 | # This is a secret key used for signing authentication tokens. 20 | # It should be long and randomly-generated. 21 | # The following command will give a suitable value on *nix systems: 22 | # 23 | # head -c 20 /dev/urandom | sha1sum 24 | # 25 | # If not specified then the server will generate a temporary one at startup. 26 | secret = changesecret 27 | 28 | # Set this to "false" to disable new-user signups on the server. 29 | # Only request by existing accounts will be honoured. 30 | allow_new_users = true 31 | 32 | # Set this to "true" to work around a mismatch between public_url and 33 | # the application URL as seen by python, which can happen in certain reverse- 34 | # proxy hosting setups. It will overwrite the WSGI environ dict with the 35 | # details from public_url. This could have security implications if e.g. 36 | # you tell the app that it's on HTTPS but it's really on HTTP, so it should 37 | # only be used as a last resort and after careful checking of server config. 38 | force_wsgi_environ = true 39 | 40 | # Uncomment and edit the following to use a local BrowserID verifier 41 | # rather than posting assertions to the mozilla-hosted verifier. 42 | # Audiences should be set to your public_url without a trailing slash. 43 | #[browserid] 44 | #backend = tokenserver.verifiers.LocalVerifier 45 | #audiences = https://localhost:5000 46 | -------------------------------------------------------------------------------- /manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Firefox Sync Server", 3 | "id": "ffsync", 4 | "description": { 5 | "en": "", 6 | "fr": "" 7 | }, 8 | "developer": { 9 | "name": "beudbeud", 10 | "email": "beudbeud@beudibox.fr", 11 | "url": "https://github.com/balu-/FSyncMS" 12 | }, 13 | "multi_instance": "false", 14 | "arguments": { 15 | "install" : [ 16 | { 17 | "name": "domain", 18 | "ask": { 19 | "en": "Choose a domain for Firefox-Sync Server" 20 | }, 21 | "example": "domain.org" 22 | }, 23 | { 24 | "name": "path", 25 | "ask": { 26 | "en": "Choose a path for Firefox-Sync Server" 27 | }, 28 | "example": "/ffsync", 29 | "default": "/ffsync" 30 | } 31 | ] 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /scripts/install: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Retrieve arguments 4 | domain=$1 5 | path=$2 6 | 7 | # Check domain/path availability 8 | sudo yunohost app checkurl $domain$path -a ffsync 9 | if [[ ! $? -eq 0 ]]; then 10 | exit 1 11 | fi 12 | 13 | # Generate random password 14 | db_pwd=$(head -c 8 /dev/urandom | sha1sum | cut -d " " -f1) 15 | 16 | # Use 'FSyncMS' as database name and user 17 | db_user=ffsync 18 | 19 | # Initialize database and store mysql password for upgrade 20 | sudo yunohost app initdb $db_user -p $db_pwd 21 | sudo yunohost app setting ffsync mysqlpwd -v $db_pwd 22 | 23 | # Generate random password and save 24 | secret=$(head -c 20 /dev/urandom | sha1sum | cut -d " " -f1) 25 | sudo yunohost app setting ffsync secret -v $secret 26 | 27 | # Check depends installation 28 | sudo apt-get install make python-dev python-virtualenv -y 29 | 30 | # Check Swap 31 | if [ $(sudo swapon -s | wc -l) = 1 ]; 32 | then 33 | # It is NOT possible to setup a swap file on a tmpfs filesystem 34 | mount | grep /tmp | grep tmpfs > /dev/null 2>&1 35 | if [ $? = 1 ]; 36 | then 37 | tmp_swap_file=/tmp/ffsync_swapfile 38 | else 39 | tmp_swap_file=/var/cache/ffsync_swapfile 40 | fi 41 | sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256 42 | sudo chmod 600 $tmp_swap_file 43 | sudo mkswap $tmp_swap_file 44 | sudo swapon $tmp_swap_file 45 | fi 46 | 47 | # Copy files to the right place 48 | final_path=/opt/yunohost/ffsync 49 | sudo mkdir -p $final_path 50 | sudo cp -a ../sources/* $final_path 51 | sudo cp ../conf/ffsync /etc/init.d/ 52 | sudo cp ../conf/ffsync.logrotate /etc/logrotate.d/ffsync 53 | sudo touch /var/log/ffsync.log 54 | 55 | # Set permissions to ffsync directory 56 | sudo useradd ffsync -d $final_path 57 | sudo chown ffsync:ffsync -R $final_path 58 | sudo chown ffsync /var/log/ffsync.log 59 | 60 | # Modify Nginx configuration file and copy it to Nginx conf directory 61 | sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf 62 | sed -i "s@ALIASTOCHANGE@$final_path/@g" ../conf/nginx.conf 63 | sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf 64 | sudo cp ../conf/syncserver.ini $final_path/syncserver.ini 65 | sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini 66 | sudo sed -i -e "s@changesecret@$secret@g" $final_path/syncserver.ini 67 | sudo sed -i "s/yunouser/$db_user/g" $final_path/syncserver.ini 68 | sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini 69 | sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini 70 | 71 | # Init virtualenv 72 | cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn 73 | 74 | # Disable swapfile 75 | if [ -z ${tmp_swap_file+x} ]; 76 | then 77 | sudo swapoff $tmp_swap_file 78 | sudo rm -f $tmp_swap_file 79 | fi 80 | 81 | # Fix permission 82 | sudo find $final_path/ -type d -exec chmod 2755 {} \; 83 | sudo find $final_path/ -type f -exec chmod g+r,o+r {} \; 84 | 85 | #enable services 86 | sudo chmod +x /etc/init.d/ffsync 87 | sudo update-rc.d ffsync defaults 88 | sudo service ffsync restart 89 | sudo service ffsync restart 90 | sudo service ffsync restart 91 | 92 | # Reload Nginx and regenerate SSOwat conf 93 | sudo yunohost app ssowatconf 94 | sudo service nginx restart 95 | sudo yunohost service add ffsync -l /var/log/ffsync.log 96 | sudo yunohost app setting ffsync skipped_uris -v "/" 97 | -------------------------------------------------------------------------------- /scripts/remove: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | db_user=ffsync 4 | db_name=ffsync 5 | root_pwd=$(sudo cat /etc/yunohost/mysql) 6 | domain=$(sudo yunohost app setting ffsync domain) 7 | 8 | mysql -u root -p$root_pwd -e "DROP DATABASE $db_name ; DROP USER $db_user@localhost ;" 9 | sudo rm -rf /opt/yunohost/ffsync 10 | sudo rm -f /etc/nginx/conf.d/$domain.d/ffsync.conf 11 | sudo service ffsync stop 12 | sudo update-rc.d ffsync remove 13 | sudo rm /etc/init.d/ffsync 14 | sudo rm /etc/logrotate.d/ffsync 15 | sudo yunohost service remove ffsync 16 | 17 | sudo service nginx reload 18 | sudo userdel ffsync 19 | sudo delgroup ffsync 20 | -------------------------------------------------------------------------------- /scripts/upgrade: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Retrieve arguments 4 | domain=$(sudo yunohost app setting ffsync domain) 5 | path=$(sudo yunohost app setting ffsync path) 6 | db_pwd=$(sudo yunohost app setting ffsync mysqlpwd) 7 | db_user=ffsync 8 | final_path=/opt/yunohost/ffsync 9 | 10 | # Get secret variable 11 | secret=$(sudo yunohost app setting ffsync secret) 12 | # Get from conf file if not defined 13 | if [[ -z $secret ]] 14 | then 15 | secret=$(sudo grep "secret =" $final_path/syncserver.ini | cut -d" " -f3) 16 | sudo yunohost app setting ffsync secret -v $secret 17 | fi 18 | 19 | # Check Swap 20 | if [ $(sudo swapon -s | wc -l) = 1 ]; 21 | then 22 | # It is NOT possible to setup a swap file on a tmpfs filesystem 23 | mount | grep /tmp | grep tmpfs > /dev/null 2>&1 24 | if [ $? = 1 ]; 25 | then 26 | tmp_swap_file=/tmp/ffsync_swapfile 27 | else 28 | tmp_swap_file=/var/cache/ffsync_swapfile 29 | fi 30 | sudo dd if=/dev/zero of=$tmp_swap_file bs=1M count=256 31 | sudo chmod 600 $tmp_swap_file 32 | sudo mkswap $tmp_swap_file 33 | sudo swapon $tmp_swap_file 34 | fi 35 | 36 | # Copy files to the right place 37 | sudo mkdir -p $final_path 38 | sudo cp -a ../sources/* $final_path 39 | sudo cp ../conf/ffsync /etc/init.d/ 40 | sudo cp ../conf/ffsync.logrotate /etc/logrotate.d/ffsync 41 | 42 | # Set permissions to ffsync directory 43 | sudo useradd ffsync -d $final_path 44 | sudo chown ffsync:ffsync -R $final_path 45 | 46 | # Modify Nginx configuration file and copy it to Nginx conf directory 47 | sed -i "s@PATHTOCHANGE@$path@g" ../conf/nginx.conf 48 | sed -i "s@ALIASTOCHANGE@$final_path@g" ../conf/nginx.conf 49 | sudo cp ../conf/nginx.conf /etc/nginx/conf.d/$domain.d/ffsync.conf 50 | sudo cp ../conf/syncserver.ini $final_path/syncserver.ini 51 | sudo sed -i -e "s@ynhbaseurl@$domain$path@g" $final_path/syncserver.ini 52 | sudo sed -i -e "s@changesecret@$secret@g" $final_path/syncserver.ini 53 | sudo sed -i "s/yunouser/$db_user/g" $final_path/syncserver.ini 54 | sudo sed -i "s/yunopass/$db_pwd/g" $final_path/syncserver.ini 55 | sudo sed -i "s/yunobase/$db_user/g" $final_path/syncserver.ini 56 | 57 | # stop service before upgrade 58 | sudo service ffsync stop 59 | 60 | # Init virtualenv 61 | cd $final_path && sudo make build && sudo ./local/bin/easy_install gunicorn 62 | 63 | # Disable swapfile 64 | if [ -z ${tmp_swap_file+x} ]; 65 | then 66 | sudo swapoff $tmp_swap_file 67 | sudo rm -f $tmp_swap_file 68 | fi 69 | 70 | # Fix permission 71 | sudo find $final_path/ -type d -exec chmod 2755 {} \; 72 | sudo find $final_path/ -type f -exec chmod g+r,o+r {} \; 73 | sudo usermod -a -G ffsync www-data 74 | 75 | #enable services 76 | sudo chmod +x /etc/init.d/ffsync 77 | sudo update-rc.d ffsync defaults 78 | sudo service ffsync restart 79 | sudo service ffsync restart 80 | sudo service ffsync restart 81 | 82 | # Reload Nginx and regenerate SSOwat conf 83 | sudo service nginx reload 84 | sudo yunohost app setting ffsync skipped_uris -v "/" 85 | sudo yunohost app ssowatconf 86 | -------------------------------------------------------------------------------- /sources/.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.mako.py 3 | local 4 | *.egg-info 5 | *.swp 6 | \.coverage 7 | *~ 8 | nosetests.xml 9 | syncserver.db 10 | -------------------------------------------------------------------------------- /sources/.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | 3 | python: 4 | - "2.6" 5 | - "2.7" 6 | 7 | notifications: 8 | email: 9 | - rfkelly@mozilla.com 10 | irc: 11 | channels: 12 | - "irc.mozilla.org#services-dev" 13 | use_notice: false 14 | skip_join: false 15 | 16 | install: 17 | - make build 18 | 19 | script: 20 | - make test 21 | -------------------------------------------------------------------------------- /sources/Dockerfile: -------------------------------------------------------------------------------- 1 | ########################################################## 2 | # /!\ WARNING /!\ # 3 | # This is completely experimental. Use at your own risk. # 4 | # Also, learn you some docker: # 5 | # http://docker.io/gettingstarted # 6 | ########################################################## 7 | 8 | FROM debian:7.4 9 | MAINTAINER Dan Callahan 10 | 11 | # Base system setup 12 | 13 | RUN DEBIAN_FRONTEND=noninteractive apt-get update \ 14 | && apt-get install --no-install-recommends -y \ 15 | vim locales \ 16 | && apt-get clean 17 | 18 | RUN locale-gen C.UTF-8 && LANG=C.UTF-8 /usr/sbin/update-locale 19 | 20 | ENV LANG C.UTF-8 21 | 22 | RUN useradd --create-home app 23 | 24 | # Build the Sync server 25 | 26 | RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ 27 | ca-certificates \ 28 | build-essential \ 29 | libzmq-dev \ 30 | python-dev \ 31 | python-virtualenv \ 32 | && apt-get clean 33 | 34 | USER app 35 | 36 | RUN mkdir -p /home/app/syncserver 37 | ADD Makefile *.ini *.wsgi *.rst *.txt *.py /home/app/syncserver/ 38 | ADD ./syncserver/ /home/app/syncserver/syncserver/ 39 | WORKDIR /home/app/syncserver 40 | 41 | RUN make build 42 | 43 | # Run the Sync server 44 | 45 | EXPOSE 5000 46 | 47 | ENTRYPOINT ["/usr/bin/make"] 48 | CMD ["serve"] 49 | -------------------------------------------------------------------------------- /sources/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include syncserver.ini 2 | include syncserver.wsgi 3 | include syncserver/tests.ini 4 | -------------------------------------------------------------------------------- /sources/Makefile: -------------------------------------------------------------------------------- 1 | SYSTEMPYTHON = `which python2 python | head -n 1` 2 | VIRTUALENV = virtualenv --python=$(SYSTEMPYTHON) 3 | ENV = ./local 4 | TOOLS := $(addprefix $(ENV)/bin/,flake8 nosetests) 5 | 6 | # Hackety-hack around OSX system python bustage. 7 | # The need for this should go away with a future osx/xcode update. 8 | ARCHFLAGS = -Wno-error=unused-command-line-argument-hard-error-in-future 9 | 10 | # Hackety-hack around errors duing compile of ultramemcached. 11 | CFLAGS = "-Wno-error -Wno-error=format-security" 12 | 13 | INSTALL = CFLAGS=$(CFLAGS) ARCHFLAGS=$(ARCHFLAGS) $(ENV)/bin/pip install 14 | 15 | 16 | .PHONY: all 17 | all: build 18 | 19 | .PHONY: build 20 | build: | $(ENV)/COMPLETE 21 | $(ENV)/COMPLETE: requirements.txt 22 | $(VIRTUALENV) --no-site-packages $(ENV) 23 | $(INSTALL) -r requirements.txt 24 | $(ENV)/bin/python ./setup.py develop 25 | touch $(ENV)/COMPLETE 26 | 27 | .PHONY: test 28 | test: | $(TOOLS) 29 | $(ENV)/bin/flake8 ./syncserver 30 | $(ENV)/bin/nosetests -s syncstorage.tests 31 | # Tokenserver tests currently broken due to incorrect file paths 32 | # $(ENV)/bin/nosetests -s tokenserver.tests 33 | 34 | # Test against a running server 35 | $(ENV)/bin/gunicorn --paste syncserver/tests.ini 2> /dev/null & SERVER_PID=$$!; \ 36 | sleep 2; \ 37 | $(ENV)/bin/python -m syncstorage.tests.functional.test_storage \ 38 | --use-token-server http://localhost:5000/token/1.0/sync/1.5; \ 39 | kill $$SERVER_PID 40 | 41 | $(TOOLS): | $(ENV)/COMPLETE 42 | $(INSTALL) nose flake8 43 | 44 | .PHONY: serve 45 | serve: | $(ENV)/COMPLETE 46 | $(ENV)/bin/gunicorn --paste ./syncserver.ini 47 | 48 | .PHONY: clean 49 | clean: 50 | rm -rf $(ENV) 51 | -------------------------------------------------------------------------------- /sources/README.rst: -------------------------------------------------------------------------------- 1 | Run-Your-Own Firefox Sync Server 2 | ================================ 3 | 4 | This is an all-in-one package for running a self-hosted Firefox Sync server. 5 | If bundles the "tokenserver" project for authentication and the "syncstorage" 6 | project for storage, produce a single stand-alone webapp. 7 | 8 | Complete installation instructions are available at: 9 | 10 | https://docs.services.mozilla.com/howtos/run-sync-1.5.html 11 | 12 | 13 | Quickstart 14 | ---------- 15 | 16 | The Sync Server software runs using **python 2.6** or later, and the build 17 | process requires **make** and **virtualenv**. You will need to have the 18 | following packages (or similar, depending on your operating system) installed: 19 | 20 | - python2.7 21 | - python2.7-dev 22 | - python-virtualenv 23 | - make 24 | 25 | Take a checkout of this repository, then run "make build" to pull in the 26 | necessary python package dependencies:: 27 | 28 | $ git clone https://github.com/mozilla-services/syncserver 29 | $ cd syncserver 30 | $ make build 31 | 32 | To sanity-check that things got installed correctly, do the following:: 33 | 34 | $ make test 35 | 36 | Now you can run the server:: 37 | 38 | $ make serve 39 | 40 | This should start a server on http://localhost:5000/. 41 | 42 | Now go into Firefox's `about:config` page, search for a setting named 43 | "tokenServerURI", and change it to point to your server:: 44 | 45 | services.sync.tokenServerURI: http://localhost:5000/token/1.0/sync/1.5 46 | 47 | Firefox should now sync against your local server rather than the default 48 | Mozilla-hosted servers. 49 | 50 | For more details on setting up a stable deployment, see: 51 | 52 | https://docs.services.mozilla.com/howtos/run-sync-1.5.html 53 | 54 | 55 | Customization 56 | ------------- 57 | 58 | All customization of the server can be done by editing the file 59 | "syncserver.ini", which contains lots of comments to help you on 60 | your way. Things you might like to change include: 61 | 62 | * The client-visible hostname for your server. Edit the "public_url" 63 | key under the [syncstorage] section. 64 | 65 | * The database in which to store sync data. Edit the "sqluri" setting 66 | under the [syncstorage] section. 67 | 68 | 69 | Questions, Feedback 70 | ------------------- 71 | 72 | - IRC channel: #sync. See http://irc.mozilla.org/ 73 | - Mailing list: https://mail.mozilla.org/listinfo/services-dev 74 | -------------------------------------------------------------------------------- /sources/requirements.txt: -------------------------------------------------------------------------------- 1 | cornice==0.16.2 2 | gunicorn==19.1.1 3 | pyramid==1.5 4 | requests==2.7 5 | simplejson==3.4 6 | SQLAlchemy==0.9.4 7 | unittest2==0.5.1 8 | zope.component==4.2.1 9 | configparser==3.5.0b2 10 | https://github.com/mozilla-services/mozservices/archive/e00e1b68130423ad98d0f6185655bde650443da8.zip 11 | https://github.com/mozilla-services/tokenserver/archive/d7e513e8a4f5c588b70d685a8df1d2e508c341c0.zip 12 | http://github.com/mozilla-services/server-syncstorage/archive/1.5.5.zip 13 | -------------------------------------------------------------------------------- /sources/setup.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | from setuptools import setup 5 | 6 | entry_points = """ 7 | [paste.app_factory] 8 | main = syncserver:main 9 | """ 10 | 11 | setup( 12 | name='syncserver', 13 | version="1.5.2", 14 | packages=['syncserver'], 15 | entry_points=entry_points 16 | ) 17 | -------------------------------------------------------------------------------- /sources/syncserver.ini: -------------------------------------------------------------------------------- 1 | [server:main] 2 | use = egg:gunicorn 3 | host = 0.0.0.0 4 | port = 5000 5 | workers = 1 6 | timeout = 30 7 | 8 | [app:main] 9 | use = egg:syncserver 10 | 11 | [syncserver] 12 | # This must be edited to point to the public URL of your server, 13 | # i.e. the URL as seen by Firefox. 14 | public_url = http://localhost:5000/ 15 | 16 | # This defines the database in which to store all server data. 17 | #sqluri = sqlite:////tmp/syncserver.db 18 | 19 | # This is a secret key used for signing authentication tokens. 20 | # It should be long and randomly-generated. 21 | # The following command will give a suitable value on *nix systems: 22 | # 23 | # head -c 20 /dev/urandom | sha1sum 24 | # 25 | # If not specified then the server will generate a temporary one at startup. 26 | #secret = INSERT_SECRET_KEY_HERE 27 | 28 | # Set this to "false" to disable new-user signups on the server. 29 | # Only request by existing accounts will be honoured. 30 | # allow_new_users = false 31 | 32 | # Set this to "true" to work around a mismatch between public_url and 33 | # the application URL as seen by python, which can happen in certain reverse- 34 | # proxy hosting setups. It will overwrite the WSGI environ dict with the 35 | # details from public_url. This could have security implications if e.g. 36 | # you tell the app that it's on HTTPS but it's really on HTTP, so it should 37 | # only be used as a last resort and after careful checking of server config. 38 | force_wsgi_environ = false 39 | 40 | # Uncomment and edit the following to use a local BrowserID verifier 41 | # rather than posting assertions to the mozilla-hosted verifier. 42 | # Audiences should be set to your public_url without a trailing slash. 43 | #[browserid] 44 | #backend = tokenserver.verifiers.LocalVerifier 45 | #audiences = https://localhost:5000 46 | -------------------------------------------------------------------------------- /sources/syncserver.wsgi: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import os 6 | import sys 7 | import site 8 | from logging.config import fileConfig 9 | from ConfigParser import NoSectionError 10 | 11 | # detecting if virtualenv was used in this dir 12 | _CURDIR = os.path.dirname(os.path.abspath(__file__)) 13 | _PY_VER = sys.version.split()[0][:3] 14 | _SITE_PKG = os.path.join(_CURDIR, 'local', 'lib', 'python' + _PY_VER, 'site-packages') 15 | 16 | # adding virtualenv's site-package and ordering paths 17 | saved = sys.path[:] 18 | 19 | if os.path.exists(_SITE_PKG): 20 | site.addsitedir(_SITE_PKG) 21 | 22 | for path in sys.path: 23 | if path not in saved: 24 | saved.insert(0, path) 25 | 26 | sys.path[:] = saved 27 | 28 | # setting up the egg cache to a place where apache can write 29 | os.environ['PYTHON_EGG_CACHE'] = '/tmp/python-eggs' 30 | 31 | # setting up logging 32 | ini_file = os.path.join(_CURDIR, 'syncserver.ini') 33 | try: 34 | fileConfig(ini_file) 35 | except NoSectionError: 36 | pass 37 | 38 | # running the app using Paste 39 | from paste.deploy import loadapp 40 | application = loadapp('config:%s'% ini_file) 41 | -------------------------------------------------------------------------------- /sources/syncserver/__init__.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | 5 | import os 6 | import logging 7 | from urlparse import urlparse, urlunparse 8 | 9 | from pyramid.response import Response 10 | from pyramid.events import NewRequest, subscriber 11 | 12 | import mozsvc.config 13 | 14 | from tokenserver.util import _JSONError 15 | 16 | logger = logging.getLogger("syncserver") 17 | 18 | 19 | def includeme(config): 20 | """Install SyncServer application into the given Pyramid configurator.""" 21 | # Set the umask so that files are created with secure permissions. 22 | # Necessary for e.g. created-on-demand sqlite database files. 23 | os.umask(0077) 24 | 25 | # Sanity-check the deployment settings and provide sensible defaults. 26 | settings = config.registry.settings 27 | public_url = settings.get("syncserver.public_url") 28 | if public_url is None: 29 | raise RuntimeError("you much configure syncserver.public_url") 30 | public_url = public_url.rstrip("/") 31 | settings["syncserver.public_url"] = public_url 32 | 33 | secret = settings.get("syncserver.secret") 34 | if secret is None: 35 | secret = os.urandom(32).encode("hex") 36 | sqluri = settings.get("syncserver.sqluri") 37 | if sqluri is None: 38 | rootdir = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) 39 | sqluri = "sqlite:///" + os.path.join(rootdir, "syncserver.db") 40 | 41 | # Configure app-specific defaults based on top-level configuration. 42 | settings.pop("config", None) 43 | if "tokenserver.backend" not in settings: 44 | # Default to our simple static node-assignment backend 45 | settings["tokenserver.backend"] =\ 46 | "syncserver.staticnode.StaticNodeAssignment" 47 | settings["tokenserver.sqluri"] = sqluri 48 | settings["tokenserver.node_url"] = public_url 49 | settings["endpoints.sync-1.5"] = "{node}/storage/1.5/{uid}" 50 | if "tokenserver.monkey_patch_gevent" not in settings: 51 | # Default to no gevent monkey-patching 52 | settings["tokenserver.monkey_patch_gevent"] = False 53 | if "tokenserver.applications" not in settings: 54 | # Default to just the sync-1.5 application 55 | settings["tokenserver.applications"] = "sync-1.5" 56 | if "tokenserver.secrets.backend" not in settings: 57 | # Default to a single fixed signing secret 58 | settings["tokenserver.secrets.backend"] = "mozsvc.secrets.FixedSecrets" 59 | settings["tokenserver.secrets.secrets"] = [secret] 60 | if "tokenserver.allow_new_users" not in settings: 61 | allow_new_users = settings.get("syncserver.allow_new_users") 62 | if allow_new_users is not None: 63 | settings["tokenserver.allow_new_users"] = allow_new_users 64 | if "hawkauth.secrets.backend" not in settings: 65 | # Default to the same secrets backend as the tokenserver 66 | for key in settings.keys(): 67 | if key.startswith("tokenserver.secrets."): 68 | newkey = "hawkauth" + key[len("tokenserver"):] 69 | settings[newkey] = settings[key] 70 | if "storage.backend" not in settings: 71 | # Default to sql syncstorage backend 72 | settings["storage.backend"] = "syncstorage.storage.sql.SQLStorage" 73 | settings["storage.sqluri"] = sqluri 74 | settings["storage.create_tables"] = True 75 | if "browserid.backend" not in settings: 76 | # Default to remote verifier, with base of public_url as only audience 77 | audience = urlunparse(urlparse(public_url)._replace(path="")) 78 | settings["browserid.backend"] = "tokenserver.verifiers.RemoteVerifier" 79 | settings["browserid.audiences"] = audience 80 | if "loggers" not in settings: 81 | # Default to basic logging config. 82 | root_logger = logging.getLogger("") 83 | if not root_logger.handlers: 84 | logging.basicConfig(level=logging.INFO) 85 | 86 | # Include the relevant sub-packages. 87 | config.scan("syncserver") 88 | config.include("syncstorage", route_prefix="/storage") 89 | config.include("tokenserver", route_prefix="/token") 90 | 91 | # Add a top-level "it works!" view. 92 | def itworks(request): 93 | return Response("it works!") 94 | 95 | config.add_route('itworks', '/') 96 | config.add_view(itworks, route_name='itworks') 97 | 98 | 99 | @subscriber(NewRequest) 100 | def reconcile_wsgi_environ_with_public_url(event): 101 | """Event-listener that checks and tweaks WSGI environ based on public_url. 102 | 103 | This is a simple trick to help ensure that the configured public_url 104 | matches the actual deployed address. It fixes fixes parts of the WSGI 105 | environ where it makes sense (e.g. SCRIPT_NAME) and warns about any parts 106 | that seem obviously mis-configured (e.g. http:// versus https://). 107 | 108 | It's very important to get public_url and WSGI environ matching exactly, 109 | since they're used for browserid audience checking and HAWK signature 110 | validation, so mismatches can easily cause strange and cryptic errors. 111 | """ 112 | request = event.request 113 | public_url = request.registry.settings["syncserver.public_url"] 114 | p_public_url = urlparse(public_url) 115 | # If we don't have a SCRIPT_NAME, take it from the public_url. 116 | # This is often the case if we're behind e.g. an nginx proxy that 117 | # is serving us at some sub-path. 118 | if not request.script_name: 119 | request.script_name = p_public_url.path.rstrip("/") 120 | # If the environ does not match public_url, requests are almost certainly 121 | # going to fail due to auth errors. We can either bail out early, or we 122 | # can forcibly clobber the WSGI environ with the values from public_url. 123 | # This is a security risk if you've e.g. mis-configured the server, so 124 | # it's not enabled by default. 125 | application_url = request.application_url 126 | if public_url != application_url: 127 | if not request.registry.settings.get("syncserver.force_wsgi_environ"): 128 | msg = "\n".join(( 129 | "The public_url setting doesn't match the application url.", 130 | "This will almost certainly cause authentication failures!", 131 | " public_url setting is: %s" % (public_url,), 132 | " application url is: %s" % (application_url,), 133 | "You can disable this check by setting the force_wsgi_environ", 134 | "option in your config file, but do so at your own risk.", 135 | )) 136 | logger.error(msg) 137 | raise _JSONError([msg], status_code=500) 138 | request.scheme = p_public_url.scheme 139 | request.host = p_public_url.netloc 140 | request.script_name = p_public_url.path.rstrip("/") 141 | 142 | 143 | def get_configurator(global_config, **settings): 144 | """Load a SyncStorge configurator object from deployment settings.""" 145 | config = mozsvc.config.get_configurator(global_config, **settings) 146 | config.begin() 147 | try: 148 | config.include(includeme) 149 | finally: 150 | config.end() 151 | return config 152 | 153 | 154 | def main(global_config, **settings): 155 | """Load a SyncStorage WSGI app from deployment settings.""" 156 | config = get_configurator(global_config, **settings) 157 | return config.make_wsgi_app() 158 | -------------------------------------------------------------------------------- /sources/syncserver/staticnode.py: -------------------------------------------------------------------------------- 1 | # This Source Code Form is subject to the terms of the Mozilla Public 2 | # License, v. 2.0. If a copy of the MPL was not distributed with this file, 3 | # You can obtain one at http://mozilla.org/MPL/2.0/. 4 | """ 5 | Simple node-assignment backend using a single, static node. 6 | 7 | This is a greatly-simplified node-assignment backend. It keeps user records 8 | in an SQL database, but does not attempt to do any node management. All users 9 | are implicitly assigned to a single, static node. 10 | 11 | XXX TODO: move this into the tokenserver repo. 12 | 13 | """ 14 | import time 15 | import urlparse 16 | from mozsvc.exceptions import BackendError 17 | 18 | from sqlalchemy import Column, Integer, String, BigInteger, Index 19 | from sqlalchemy import create_engine, Table, MetaData 20 | from sqlalchemy.pool import QueuePool 21 | from sqlalchemy.sql import text as sqltext 22 | from sqlalchemy.exc import IntegrityError 23 | 24 | from tokenserver.assignment import INodeAssignment 25 | from zope.interface import implements 26 | 27 | 28 | metadata = MetaData() 29 | 30 | 31 | users = Table( 32 | "users", 33 | metadata, 34 | Column("uid", Integer(), primary_key=True, autoincrement=True, 35 | nullable=False), 36 | Column("service", String(32), nullable=False), 37 | Column("email", String(255), nullable=False), 38 | Column("generation", BigInteger(), nullable=False), 39 | Column("client_state", String(32), nullable=False), 40 | Column("created_at", BigInteger(), nullable=False), 41 | Column("replaced_at", BigInteger(), nullable=True), 42 | Index('lookup_idx', 'email', 'service', 'created_at'), 43 | Index('clientstate_idx', 'email', 'service', 'client_state', unique=True), 44 | ) 45 | 46 | 47 | _GET_USER_RECORDS = sqltext("""\ 48 | select 49 | uid, generation, client_state 50 | from 51 | users 52 | where 53 | email = :email 54 | and 55 | service = :service 56 | order by 57 | created_at desc, uid desc 58 | limit 59 | 20 60 | """) 61 | 62 | 63 | _CREATE_USER_RECORD = sqltext("""\ 64 | insert into 65 | users 66 | (service, email, generation, client_state, created_at, replaced_at) 67 | values 68 | (:service, :email, :generation, :client_state, :timestamp, NULL) 69 | """) 70 | 71 | 72 | _UPDATE_GENERATION_NUMBER = sqltext("""\ 73 | update 74 | users 75 | set 76 | generation = :generation 77 | where 78 | service = :service and email = :email and 79 | generation < :generation and replaced_at is null 80 | """) 81 | 82 | 83 | _REPLACE_USER_RECORDS = sqltext("""\ 84 | update 85 | users 86 | set 87 | replaced_at = :timestamp 88 | where 89 | service = :service and email = :email 90 | and replaced_at is null and created_at < :timestamp 91 | """) 92 | 93 | 94 | def get_timestamp(): 95 | """Get current timestamp in milliseconds.""" 96 | return int(time.time() * 1000) 97 | 98 | 99 | class StaticNodeAssignment(object): 100 | implements(INodeAssignment) 101 | 102 | def __init__(self, sqluri, node_url, **kw): 103 | self.sqluri = sqluri 104 | self.node_url = node_url 105 | self.driver = urlparse.urlparse(sqluri).scheme.lower() 106 | sqlkw = { 107 | "logging_name": "syncserver", 108 | "connect_args": {}, 109 | "poolclass": QueuePool, 110 | "pool_reset_on_return": True, 111 | } 112 | if self.driver == "sqlite": 113 | # We must mark it as safe to share sqlite connections between 114 | # threads. The pool will ensure there's on race conditions. 115 | sqlkw["connect_args"]["check_same_thread"] = False 116 | # If using a :memory: database, we must use a QueuePool of size 117 | # 1 so that a single connection is shared by all threads. 118 | if urlparse.urlparse(sqluri).path.lower() in ("/", "/:memory:"): 119 | sqlkw["pool_size"] = 1 120 | sqlkw["max_overflow"] = 0 121 | if "mysql" in self.driver: 122 | # Guard against the db closing idle conections. 123 | sqlkw["pool_recycle"] = 3600 124 | self._engine = create_engine(sqluri, **sqlkw) 125 | users.create(self._engine, checkfirst=True) 126 | 127 | def get_user(self, service, email): 128 | params = {'service': service, 'email': email} 129 | res = self._engine.execute(_GET_USER_RECORDS, **params) 130 | try: 131 | row = res.fetchone() 132 | if row is None: 133 | return None 134 | # The first row is the most up-to-date user record. 135 | user = { 136 | 'email': email, 137 | 'uid': row.uid, 138 | 'node': self.node_url, 139 | 'generation': row.generation, 140 | 'client_state': row.client_state, 141 | 'old_client_states': {} 142 | } 143 | # Any subsequent rows are due to old client-state values. 144 | row = res.fetchone() 145 | while row is not None: 146 | user['old_client_states'][row.client_state] = True 147 | row = res.fetchone() 148 | return user 149 | finally: 150 | res.close() 151 | 152 | def allocate_user(self, service, email, generation=0, client_state=''): 153 | params = { 154 | 'service': service, 'email': email, 'generation': generation, 155 | 'client_state': client_state, 'timestamp': get_timestamp() 156 | } 157 | try: 158 | res = self._engine.execute(_CREATE_USER_RECORD, **params) 159 | except IntegrityError: 160 | raise 161 | return self.get_user(service, email) 162 | else: 163 | res.close() 164 | return { 165 | 'email': email, 166 | 'uid': res.lastrowid, 167 | 'node': self.node_url, 168 | 'generation': generation, 169 | 'client_state': client_state, 170 | 'old_client_states': {} 171 | } 172 | 173 | def update_user(self, service, user, generation=None, client_state=None): 174 | if client_state is None: 175 | # uid can stay the same, just update the generation number. 176 | if generation is not None: 177 | params = { 178 | 'service': service, 179 | 'email': user['email'], 180 | 'generation': generation, 181 | } 182 | res = self._engine.execute(_UPDATE_GENERATION_NUMBER, **params) 183 | res.close() 184 | user['generation'] = max(generation, user['generation']) 185 | else: 186 | # reject previously-seen client-state strings. 187 | if client_state == user['client_state']: 188 | raise BackendError('previously seen client-state string') 189 | if client_state in user['old_client_states']: 190 | raise BackendError('previously seen client-state string') 191 | # need to create a new record for new client_state. 192 | if generation is not None: 193 | generation = max(user['generation'], generation) 194 | else: 195 | generation = user['generation'] 196 | now = get_timestamp() 197 | params = { 198 | 'service': service, 'email': user['email'], 199 | 'generation': generation, 'client_state': client_state, 200 | 'timestamp': now, 201 | } 202 | try: 203 | res = self._engine.execute(_CREATE_USER_RECORD, **params) 204 | except IntegrityError: 205 | user.update(self.get_user(service, user['email'])) 206 | else: 207 | self.get_user(service, user['email']) 208 | user['uid'] = res.lastrowid 209 | user['generation'] = generation 210 | user['old_client_states'][user['client_state']] = True 211 | user['client_state'] = client_state 212 | res.close() 213 | # mark old records as having been replaced. 214 | # if we crash here, they are unmarked and we may fail to 215 | # garbage collect them for a while, but the active state 216 | # will be undamaged. 217 | params = { 218 | 'service': service, 'email': user['email'], 'timestamp': now 219 | } 220 | res = self._engine.execute(_REPLACE_USER_RECORDS, **params) 221 | res.close() 222 | -------------------------------------------------------------------------------- /sources/syncserver/tests.ini: -------------------------------------------------------------------------------- 1 | [server:main] 2 | use = egg:gunicorn 3 | host = 0.0.0.0 4 | port = 5000 5 | workers = 1 6 | timeout = 30 7 | 8 | [app:main] 9 | use = egg:SyncServer 10 | 11 | [syncserver] 12 | # This must be edited to point to the public URL of your server. 13 | public_url = http://localhost:5000/ 14 | 15 | # This defines the database in which to store all server data. 16 | #sqluri = sqlite:////tmp/syncserver.db 17 | 18 | # This is a secret key used for signing authentication tokens. 19 | #secret = INSERT_SECRET_KEY_HERE 20 | --------------------------------------------------------------------------------