├── addon ├── config.json └── __init__.py ├── tests ├── helpers │ ├── __init__.py │ ├── mock_servers.py │ ├── collection_utils.py │ ├── db_utils.py │ ├── file_utils.py │ ├── server_utils.py │ └── monkey_patches.py ├── assets │ ├── blue.jpg │ └── test.conf ├── test_web_hostkey.py ├── collection_test_base.py ├── test_full_sync.py ├── test_collection_wrappers.py ├── test_media.py ├── sync_app_functional_test_base.py ├── test_sync_app.py ├── test_sessions.py ├── test_users.py └── test_web_media.py ├── .gitignore ├── .gitmodules ├── ankisyncd ├── __main__.py ├── __init__.py ├── config.py ├── full_sync.py ├── media.py ├── collection.py ├── sessions.py ├── thread.py ├── users.py └── sync_app.py ├── ankisyncd.conf ├── ankisyncctl.py ├── utils └── migrate_user_tables.py ├── README.md └── COPYING /addon/config.json: -------------------------------------------------------------------------------- 1 | {"profiles":{}} 2 | -------------------------------------------------------------------------------- /tests/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | from . import db_utils -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.db 3 | 4 | /ankisyncd/_version.py 5 | /collections 6 | /venv 7 | -------------------------------------------------------------------------------- /tests/assets/blue.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tsdko/anki-sync-server/HEAD/tests/assets/blue.jpg -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "anki-bundled"] 2 | path = anki-bundled 3 | url = https://github.com/dae/anki.git 4 | -------------------------------------------------------------------------------- /tests/assets/test.conf: -------------------------------------------------------------------------------- 1 | [sync_app] 2 | host = 127.0.0.1 3 | port = 27701 4 | data_root = ./collections 5 | base_url = /sync/ 6 | base_media_url = /msync/ 7 | auth_db_path = ./auth.db 8 | session_db_path = ./session.db 9 | -------------------------------------------------------------------------------- /ankisyncd/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | if __package__ is None and not hasattr(sys, "frozen"): 4 | import os.path 5 | path = os.path.realpath(os.path.abspath(__file__)) 6 | sys.path.insert(0, os.path.dirname(os.path.dirname(path))) 7 | 8 | import ankisyncd.sync_app 9 | 10 | if __name__ == "__main__": 11 | ankisyncd.sync_app.main() 12 | -------------------------------------------------------------------------------- /tests/test_web_hostkey.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from sync_app_functional_test_base import SyncAppFunctionalTestBase 3 | 4 | 5 | class SyncAppFunctionalHostKeyTest(SyncAppFunctionalTestBase): 6 | def setUp(self): 7 | SyncAppFunctionalTestBase.setUp(self) 8 | self.server = self.mock_remote_server 9 | 10 | def tearDown(self): 11 | self.server = None 12 | SyncAppFunctionalTestBase.tearDown(self) 13 | 14 | # First breakage: c7d7ff3e858415ec12785579882a4c7586cbfce4 15 | def test_login(self): 16 | self.assertIsNotNone(self.server.hostKey("testuser", "testpassword")) 17 | self.assertIsNone(self.server.hostKey("testuser", "wrongpassword")) 18 | self.assertIsNone(self.server.hostKey("wronguser", "wrongpassword")) 19 | self.assertIsNone(self.server.hostKey("wronguser", "testpassword")) 20 | 21 | -------------------------------------------------------------------------------- /ankisyncd/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | sys.path.insert(0, "/usr/share/anki") 5 | sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(__file__)), "anki-bundled")) 6 | 7 | _homepage = "https://github.com/tsudoko/anki-sync-server" 8 | _unknown_version = "[unknown version]" 9 | 10 | 11 | def _get_version(): 12 | try: 13 | from ankisyncd._version import version 14 | 15 | return version 16 | except ImportError: 17 | pass 18 | 19 | import subprocess 20 | 21 | try: 22 | return ( 23 | subprocess.run( 24 | ["git", "describe", "--always"], 25 | stdout=subprocess.PIPE, 26 | stderr=subprocess.PIPE, 27 | ) 28 | .stdout.strip() 29 | .decode() 30 | or _unknown_version 31 | ) 32 | except (FileNotFoundError, subprocess.CalledProcessError): 33 | return _unknown_version 34 | -------------------------------------------------------------------------------- /ankisyncd.conf: -------------------------------------------------------------------------------- 1 | [sync_app] 2 | # change to 127.0.0.1 if you don't want the server to be accessible from the internet 3 | host = 0.0.0.0 4 | port = 27701 5 | data_root = ./collections 6 | base_url = /sync/ 7 | base_media_url = /msync/ 8 | auth_db_path = ./auth.db 9 | # optional, for session persistence between restarts 10 | session_db_path = ./session.db 11 | 12 | # optional, for overriding the default managers and wrappers 13 | # # must inherit from ankisyncd.full_sync.FullSyncManager, e.g, 14 | # full_sync_manager = great_stuff.postgres.PostgresFullSyncManager 15 | # # must inherit from ankisyncd.session.SimpleSessionManager, e.g, 16 | # session_manager = great_stuff.postgres.PostgresSessionManager 17 | # # must inherit from ankisyncd.users.SimpleUserManager, e.g, 18 | # user_manager = great_stuff.postgres.PostgresUserManager 19 | # # must inherit from ankisyncd.collection.CollectionWrapper, e.g, 20 | # collection_wrapper = great_stuff.postgres.PostgresCollectionWrapper 21 | -------------------------------------------------------------------------------- /ankisyncd/config.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | import logging 3 | import os 4 | from os.path import dirname, realpath 5 | 6 | logger = logging.getLogger("ankisyncd") 7 | 8 | paths = [ 9 | "/etc/ankisyncd/ankisyncd.conf", 10 | os.environ.get("XDG_CONFIG_HOME") and 11 | (os.path.join(os.environ['XDG_CONFIG_HOME'], "ankisyncd", "ankisyncd.conf")) or 12 | os.path.join(os.path.expanduser("~"), ".config", "ankisyncd", "ankisyncd.conf"), 13 | os.path.join(dirname(dirname(realpath(__file__))), "ankisyncd.conf"), 14 | ] 15 | 16 | # Get values from ENV and update the config. To use this prepend `ANKISYNCD_` 17 | # to the uppercase form of the key. E.g, `ANKISYNCD_SESSION_MANAGER` to set 18 | # `session_manager` 19 | def load_from_env(conf): 20 | logger.debug("Loading/overriding config values from ENV") 21 | for env in os.environ: 22 | if env.startswith('ANKISYNCD_'): 23 | config_key = env[10:].lower() 24 | conf[config_key] = os.getenv(env) 25 | logger.info("Setting {} from ENV".format(config_key)) 26 | 27 | def load(path=None): 28 | choices = paths 29 | parser = configparser.ConfigParser() 30 | if path: 31 | choices = [path] 32 | for path in choices: 33 | logger.debug("config.location: trying", path) 34 | try: 35 | parser.read(path) 36 | conf = parser['sync_app'] 37 | logger.info("Loaded config from {}".format(path)) 38 | load_from_env(conf) 39 | return conf 40 | except KeyError: 41 | pass 42 | raise Exception("No config found, looked for {}".format(", ".join(choices))) 43 | -------------------------------------------------------------------------------- /tests/collection_test_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import unittest 3 | import tempfile 4 | import os 5 | from unittest.mock import MagicMock 6 | import shutil 7 | 8 | import anki 9 | import anki.storage 10 | 11 | 12 | class CollectionTestBase(unittest.TestCase): 13 | """Parent class for tests that need a collection set up and torn down.""" 14 | 15 | def setUp(self): 16 | self.temp_dir = tempfile.mkdtemp() 17 | self.collection_path = os.path.join(self.temp_dir, 'collection.anki2'); 18 | self.collection = anki.storage.Collection(self.collection_path) 19 | self.mock_app = MagicMock() 20 | 21 | def tearDown(self): 22 | self.collection.close() 23 | self.collection = None 24 | shutil.rmtree(self.temp_dir) 25 | self.mock_app.reset_mock() 26 | 27 | # TODO: refactor into some kind of utility 28 | def add_note(self, data): 29 | from anki.notes import Note 30 | 31 | model = self.collection.models.byName(data['model']) 32 | 33 | note = Note(self.collection, model) 34 | for name, value in data['fields'].items(): 35 | note[name] = value 36 | 37 | if 'tags' in data: 38 | note.setTagsFromStr(data['tags']) 39 | 40 | self.collection.addNote(note) 41 | 42 | # TODO: refactor into a parent class 43 | def add_default_note(self, count=1): 44 | data = { 45 | 'model': 'Basic', 46 | 'fields': { 47 | 'Front': 'The front', 48 | 'Back': 'The back', 49 | }, 50 | 'tags': "Tag1 Tag2", 51 | } 52 | for idx in range(0, count): 53 | self.add_note(data) 54 | -------------------------------------------------------------------------------- /tests/test_full_sync.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import unittest 5 | import configparser 6 | 7 | from ankisyncd.full_sync import FullSyncManager, get_full_sync_manager 8 | 9 | import helpers.server_utils 10 | 11 | class FakeFullSyncManager(FullSyncManager): 12 | def __init__(self, config): 13 | pass 14 | 15 | class BadFullSyncManager: 16 | pass 17 | 18 | class FullSyncManagerFactoryTest(unittest.TestCase): 19 | def test_get_full_sync_manager(self): 20 | # Get absolute path to development ini file. 21 | script_dir = os.path.dirname(os.path.realpath(__file__)) 22 | ini_file_path = os.path.join(script_dir, 23 | "assets", 24 | "test.conf") 25 | 26 | # Create temporary files and dirs the server will use. 27 | server_paths = helpers.server_utils.create_server_paths() 28 | 29 | config = configparser.ConfigParser() 30 | config.read(ini_file_path) 31 | 32 | # Use custom files and dirs in settings. Should be PersistenceManager 33 | config['sync_app'].update(server_paths) 34 | self.assertTrue(type(get_full_sync_manager(config['sync_app']) == FullSyncManager)) 35 | 36 | # A conf-specified FullSyncManager is loaded 37 | config.set("sync_app", "full_sync_manager", 'test_full_sync.FakeFullSyncManager') 38 | self.assertTrue(type(get_full_sync_manager(config['sync_app'])) == FakeFullSyncManager) 39 | 40 | # Should fail at load time if the class doesn't inherit from FullSyncManager 41 | config.set("sync_app", "full_sync_manager", 'test_full_sync.BadFullSyncManager') 42 | with self.assertRaises(TypeError): 43 | pm = get_full_sync_manager(config['sync_app']) 44 | 45 | -------------------------------------------------------------------------------- /tests/test_collection_wrappers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import unittest 5 | import configparser 6 | 7 | from ankisyncd.collection import CollectionWrapper 8 | from ankisyncd.collection import get_collection_wrapper 9 | 10 | import helpers.server_utils 11 | 12 | class FakeCollectionWrapper(CollectionWrapper): 13 | def __init__(self, config, path, setup_new_collection=None): 14 | self. _CollectionWrapper__col = None 15 | pass 16 | 17 | class BadCollectionWrapper: 18 | pass 19 | 20 | class CollectionWrapperFactoryTest(unittest.TestCase): 21 | def test_get_collection_wrapper(self): 22 | # Get absolute path to development ini file. 23 | script_dir = os.path.dirname(os.path.realpath(__file__)) 24 | ini_file_path = os.path.join(script_dir, 25 | "assets", 26 | "test.conf") 27 | 28 | # Create temporary files and dirs the server will use. 29 | server_paths = helpers.server_utils.create_server_paths() 30 | 31 | config = configparser.ConfigParser() 32 | config.read(ini_file_path) 33 | path = os.path.realpath('fake/collection.anki2') 34 | 35 | # Use custom files and dirs in settings. Should be CollectionWrapper 36 | config['sync_app'].update(server_paths) 37 | self.assertTrue(type(get_collection_wrapper(config['sync_app'], path) == CollectionWrapper)) 38 | 39 | # A conf-specified CollectionWrapper is loaded 40 | config.set("sync_app", "collection_wrapper", 'test_collection_wrappers.FakeCollectionWrapper') 41 | self.assertTrue(type(get_collection_wrapper(config['sync_app'], path)) == FakeCollectionWrapper) 42 | 43 | # Should fail at load time if the class doesn't inherit from CollectionWrapper 44 | config.set("sync_app", "collection_wrapper", 'test_collection_wrappers.BadCollectionWrapper') 45 | with self.assertRaises(TypeError): 46 | pm = get_collection_wrapper(config['sync_app'], path) 47 | 48 | -------------------------------------------------------------------------------- /tests/helpers/mock_servers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import io 3 | import logging 4 | import types 5 | 6 | from anki.sync import HttpSyncer, RemoteServer, RemoteMediaServer 7 | 8 | 9 | class MockServerConnection: 10 | """ 11 | Mock for HttpSyncer's client attribute, an AnkiRequestsClient. All requests 12 | that would normally got to the remote server will be redirected to our 13 | server_app_to_test object. 14 | """ 15 | 16 | def __init__(self, server_app_to_test): 17 | self.test_app = server_app_to_test 18 | 19 | def post(self, url, data, headers): 20 | logging.debug("Posting to URI '{}'.".format(url)) 21 | r = self.test_app.post(url, params=data.read(), headers=headers, status="*") 22 | return types.SimpleNamespace(status_code=r.status_int, body=r.body) 23 | 24 | 25 | def streamContent(self, r): 26 | return r.body 27 | 28 | 29 | class MockRemoteServer(RemoteServer): 30 | """ 31 | Mock for RemoteServer. All communication to our remote counterpart is 32 | routed to our TestApp object. 33 | """ 34 | 35 | def __init__(self, hkey, server_test_app): 36 | # Create a custom connection object we will use to communicate with our 37 | # 'remote' server counterpart. 38 | connection = MockServerConnection(server_test_app) 39 | HttpSyncer.__init__(self, hkey, connection) 40 | 41 | def syncURL(self): # Overrides RemoteServer.syncURL(). 42 | return "/sync/" 43 | 44 | 45 | class MockRemoteMediaServer(RemoteMediaServer): 46 | """ 47 | Mock for RemoteMediaServer. All communication to our remote counterpart is 48 | routed to our TestApp object. 49 | """ 50 | 51 | def __init__(self, col, hkey, server_test_app): 52 | # Create a custom connection object we will use to communicate with our 53 | # 'remote' server counterpart. 54 | connection = MockServerConnection(server_test_app) 55 | HttpSyncer.__init__(self, hkey, connection) 56 | 57 | def syncURL(self): # Overrides RemoteServer.syncURL(). 58 | return "/msync/" 59 | -------------------------------------------------------------------------------- /ankisyncd/full_sync.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | from sqlite3 import dbapi2 as sqlite 5 | 6 | import anki.db 7 | 8 | class FullSyncManager: 9 | def upload(self, col, data, session): 10 | # Verify integrity of the received database file before replacing our 11 | # existing db. 12 | temp_db_path = session.get_collection_path() + ".tmp" 13 | with open(temp_db_path, 'wb') as f: 14 | f.write(data) 15 | 16 | try: 17 | with anki.db.DB(temp_db_path) as test_db: 18 | if test_db.scalar("pragma integrity_check") != "ok": 19 | raise HTTPBadRequest("Integrity check failed for uploaded " 20 | "collection database file.") 21 | except sqlite.Error as e: 22 | raise HTTPBadRequest("Uploaded collection database file is " 23 | "corrupt.") 24 | 25 | # Overwrite existing db. 26 | col.close() 27 | try: 28 | os.replace(temp_db_path, session.get_collection_path()) 29 | finally: 30 | col.reopen() 31 | col.load() 32 | 33 | return "OK" 34 | 35 | 36 | def download(self, col, session): 37 | col.close() 38 | try: 39 | data = open(session.get_collection_path(), 'rb').read() 40 | finally: 41 | col.reopen() 42 | col.load() 43 | return data 44 | 45 | 46 | def get_full_sync_manager(config): 47 | if "full_sync_manager" in config and config["full_sync_manager"]: # load from config 48 | import importlib 49 | import inspect 50 | module_name, class_name = config['full_sync_manager'].rsplit('.', 1) 51 | module = importlib.import_module(module_name.strip()) 52 | class_ = getattr(module, class_name.strip()) 53 | 54 | if not FullSyncManager in inspect.getmro(class_): 55 | raise TypeError('''"full_sync_manager" found in the conf file but it doesn''t 56 | inherit from FullSyncManager''') 57 | return class_(config) 58 | else: 59 | return FullSyncManager() 60 | -------------------------------------------------------------------------------- /tests/helpers/collection_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import shutil 4 | import tempfile 5 | 6 | from anki import Collection 7 | 8 | 9 | class CollectionUtils: 10 | """ 11 | Provides utility methods for creating, inspecting and manipulating anki 12 | collections. 13 | """ 14 | 15 | def __init__(self): 16 | self.collections_to_close = [] 17 | self.tempdir = tempfile.mkdtemp(prefix="CollectionUtils") 18 | self.master_db_path = None 19 | 20 | def __create_master_col(self): 21 | """ 22 | Creates an empty master anki db that will be copied on each request 23 | for a new db. This is more efficient than initializing a new db each 24 | time. 25 | """ 26 | 27 | file_path = os.path.join(self.tempdir, "collection.anki2") 28 | master_col = Collection(file_path) 29 | master_col.db.close() 30 | self.master_db_path = file_path 31 | 32 | def __enter__(self): 33 | return self 34 | 35 | def __exit__(self, exc_type, exc_value, traceback): 36 | self.clean_up() 37 | 38 | def __mark_collection_for_closing(self, collection): 39 | self.collections_to_close.append(collection) 40 | 41 | def clean_up(self): 42 | """ 43 | Removes all files created by the Collection objects we issued and the 44 | master db file. 45 | """ 46 | 47 | # Close collections. 48 | for col in self.collections_to_close: 49 | col.close() # This also closes the media col. 50 | self.collections_to_close = [] 51 | shutil.rmtree(self.tempdir) 52 | self.master_db_path = None 53 | 54 | def create_empty_col(self): 55 | """ 56 | Returns a Collection object using a copy of our master db file. 57 | """ 58 | 59 | if self.master_db_path is None: 60 | self.__create_master_col() 61 | 62 | file_descriptor, file_path = tempfile.mkstemp(dir=self.tempdir, suffix=".anki2") 63 | # Overwrite temp file with a copy of our master db. 64 | shutil.copy(self.master_db_path, file_path) 65 | collection = Collection(file_path) 66 | 67 | return collection 68 | -------------------------------------------------------------------------------- /tests/test_media.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import unittest 3 | 4 | import ankisyncd.media 5 | import helpers.collection_utils 6 | 7 | 8 | class ServerMediaManagerTest(unittest.TestCase): 9 | @classmethod 10 | def setUpClass(cls): 11 | cls.colutils = helpers.collection_utils.CollectionUtils() 12 | 13 | @classmethod 14 | def tearDownClass(cls): 15 | cls.colutils.clean_up() 16 | cls.colutils = None 17 | 18 | def test_upgrade(self): 19 | col = self.colutils.create_empty_col() 20 | cm = col.media 21 | 22 | fpath = os.path.join(cm.dir(), "file") 23 | with open(fpath + "A", "w") as f: 24 | f.write("some contents") 25 | with open(fpath + "B", "w") as f: 26 | f.write("other contents") 27 | cm._logChanges() 28 | 29 | self.assertEqual( 30 | set(cm.db.execute("SELECT fname, csum FROM media")), 31 | { 32 | ("fileA", "53059abba1a72c7aff34a3eaf7fef10ed65541ce"), 33 | ("fileB", "a5ae546046d09559399c80fa7076fb10f1ce4bcd"), 34 | }, 35 | ) 36 | cm.setLastUsn(161) 37 | 38 | sm = ankisyncd.media.ServerMediaManager(col) 39 | self.assertEqual( 40 | list(sm.db.execute("SELECT fname, csum FROM media")), 41 | list(cm.db.execute("SELECT fname, csum FROM media")), 42 | ) 43 | self.assertEqual(cm.lastUsn(), sm.lastUsn()) 44 | self.assertEqual(list(sm.db.execute("SELECT usn FROM media")), [(161,), (161,)]) 45 | 46 | def test_mediaChanges_lastUsn_order(self): 47 | col = self.colutils.create_empty_col() 48 | col.media = ankisyncd.media.ServerMediaManager(col) 49 | mh = ankisyncd.sync_app.SyncMediaHandler(col) 50 | mh.col.media.db.execute(""" 51 | INSERT INTO media (fname, usn, csum) 52 | VALUES 53 | ('fileA', 101, '53059abba1a72c7aff34a3eaf7fef10ed65541ce'), 54 | ('fileB', 100, 'a5ae546046d09559399c80fa7076fb10f1ce4bcd') 55 | """) 56 | 57 | # anki assumes mh.col.media.lastUsn() == mh.mediaChanges()['data'][-1][1] 58 | # ref: anki/sync.py:720 (commit cca3fcb2418880d0430a5c5c2e6b81ba260065b7) 59 | self.assertEqual(mh.mediaChanges(lastUsn=99)['data'][-1][1], mh.col.media.lastUsn()) 60 | -------------------------------------------------------------------------------- /tests/helpers/db_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import sqlite3 4 | import subprocess 5 | 6 | 7 | def from_sql(path, sql): 8 | """ 9 | Creates an SQLite db and executes the passed sql statements on it. 10 | 11 | :param path: the path to the created db file 12 | :param sql: the sql statements to execute on the newly created db 13 | """ 14 | 15 | connection = sqlite3.connect(path) 16 | cursor = connection.cursor() 17 | cursor.executescript(sql) 18 | connection.commit() 19 | connection.close() 20 | 21 | 22 | def to_sql(database): 23 | """ 24 | Returns a string containing the sql export of the database. Used for 25 | debugging. 26 | 27 | :param database: either the path to the SQLite db file or an open 28 | connection to it 29 | :return: a string representing the sql export of the database 30 | """ 31 | 32 | if type(database) == str: 33 | connection = sqlite3.connect(database) 34 | else: 35 | connection = database 36 | 37 | res = '\n'.join(connection.iterdump()) 38 | 39 | if type(database) == str: 40 | connection.close() 41 | 42 | return res 43 | 44 | 45 | def diff(left_db_path, right_db_path): 46 | """ 47 | Uses the sqldiff cli tool to compare two sqlite files for equality. 48 | Returns True if the databases differ, False if they don't. 49 | 50 | :param left_db_path: path to the left db file 51 | :param right_db_path: path to the right db file 52 | :return: True if the specified databases differ, False else 53 | """ 54 | 55 | command = ["sqldiff", left_db_path, right_db_path] 56 | 57 | child_process = subprocess.Popen(command, 58 | shell=False, 59 | stdout=subprocess.PIPE) 60 | stdout, stderr = child_process.communicate() 61 | exit_code = child_process.returncode 62 | 63 | if exit_code != 0 or stderr is not None: 64 | raise RuntimeError("Command {} encountered an error, exit " 65 | "code: {}, stderr: {}" 66 | .format(" ".join(command), 67 | exit_code, 68 | stderr)) 69 | 70 | # Any output from sqldiff means the databases differ. 71 | return stdout != "" 72 | -------------------------------------------------------------------------------- /ankisyncctl.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | import getpass 5 | 6 | import ankisyncd.config 7 | from ankisyncd.users import get_user_manager 8 | 9 | 10 | config = ankisyncd.config.load() 11 | 12 | def usage(): 13 | print("usage: {} []".format(sys.argv[0])) 14 | print() 15 | print("Commands:") 16 | print(" adduser - add a new user") 17 | print(" deluser - delete a user") 18 | print(" lsuser - list users") 19 | print(" passwd - change password of a user") 20 | 21 | def adduser(username): 22 | password = getpass.getpass("Enter password for {}: ".format(username)) 23 | 24 | user_manager = get_user_manager(config) 25 | user_manager.add_user(username, password) 26 | 27 | def deluser(username): 28 | user_manager = get_user_manager(config) 29 | try: 30 | user_manager.del_user(username) 31 | except ValueError as error: 32 | print("Could not delete user {}: {}".format(username, error), file=sys.stderr) 33 | 34 | def lsuser(): 35 | user_manager = get_user_manager(config) 36 | try: 37 | users = user_manager.user_list() 38 | for username in users: 39 | print(username) 40 | except ValueError as error: 41 | print("Could not list users: {}".format(error), file=sys.stderr) 42 | 43 | def passwd(username): 44 | user_manager = get_user_manager(config) 45 | 46 | if username not in user_manager.user_list(): 47 | print("User {} doesn't exist".format(username)) 48 | return 49 | 50 | password = getpass.getpass("Enter password for {}: ".format(username)) 51 | try: 52 | user_manager.set_password_for_user(username, password) 53 | except ValueError as error: 54 | print("Could not set password for user {}: {}".format(username, error), file=sys.stderr) 55 | 56 | def main(): 57 | argc = len(sys.argv) 58 | 59 | cmds = { 60 | "adduser": adduser, 61 | "deluser": deluser, 62 | "lsuser": lsuser, 63 | "passwd": passwd, 64 | } 65 | 66 | if argc < 2: 67 | usage() 68 | exit(1) 69 | 70 | c = sys.argv[1] 71 | try: 72 | if argc > 2: 73 | for arg in sys.argv[2:]: 74 | cmds[c](arg) 75 | else: 76 | cmds[c]() 77 | except KeyError: 78 | usage() 79 | exit(1) 80 | 81 | if __name__ == "__main__": 82 | main() 83 | -------------------------------------------------------------------------------- /ankisyncd/media.py: -------------------------------------------------------------------------------- 1 | # Based on anki.media.MediaManager, © Ankitects Pty Ltd and contributors 2 | # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html 3 | # Original source: https://raw.githubusercontent.com/dae/anki/62481ddc1aa78430cb8114cbf00a7739824318a8/anki/media.py 4 | 5 | import logging 6 | import re 7 | import os 8 | import os.path 9 | 10 | import anki.db 11 | 12 | logger = logging.getLogger("ankisyncd.media") 13 | 14 | 15 | class ServerMediaManager: 16 | def __init__(self, col): 17 | self._dir = re.sub(r"(?i)\.(anki2)$", ".media", col.path) 18 | self.connect() 19 | 20 | def connect(self): 21 | path = self.dir() + ".server.db" 22 | create = not os.path.exists(path) 23 | self.db = anki.db.DB(path) 24 | if create: 25 | self.db.executescript( 26 | """CREATE TABLE media ( 27 | fname TEXT NOT NULL PRIMARY KEY, 28 | usn INT NOT NULL, 29 | csum TEXT -- null if deleted 30 | ); 31 | CREATE INDEX idx_media_usn ON media (usn);""" 32 | ) 33 | oldpath = self.dir() + ".db2" 34 | if os.path.exists(oldpath): 35 | logger.info("Found client media database, migrating contents") 36 | self.db.execute("ATTACH ? AS old", oldpath) 37 | self.db.execute( 38 | "INSERT INTO media SELECT fname, lastUsn, csum FROM old.media, old.meta" 39 | ) 40 | self.db.commit() 41 | self.db.execute("DETACH old") 42 | 43 | def close(self): 44 | self.db.close() 45 | 46 | def dir(self): 47 | return self._dir 48 | 49 | def lastUsn(self): 50 | return self.db.scalar("SELECT max(usn) FROM media") or 0 51 | 52 | def mediaCount(self): 53 | return self.db.scalar("SELECT count() FROM media WHERE csum IS NOT NULL") 54 | 55 | # used only in unit tests 56 | def syncInfo(self, fname): 57 | return self.db.first("SELECT csum, 0 FROM media WHERE fname=?", fname) 58 | 59 | def syncDelete(self, fname): 60 | fpath = os.path.join(self.dir(), fname) 61 | if os.path.exists(fpath): 62 | os.remove(fpath) 63 | self.db.execute( 64 | "UPDATE media SET csum = NULL, usn = ? WHERE fname = ?", 65 | self.lastUsn() + 1, 66 | fname, 67 | ) 68 | -------------------------------------------------------------------------------- /tests/sync_app_functional_test_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import unittest 4 | from webtest import TestApp 5 | 6 | import helpers.server_utils 7 | from ankisyncd.users import SqliteUserManager 8 | from helpers.collection_utils import CollectionUtils 9 | from helpers.mock_servers import MockRemoteServer 10 | from helpers.monkey_patches import monkeypatch_db, unpatch_db 11 | 12 | 13 | class SyncAppFunctionalTestBase(unittest.TestCase): 14 | 15 | @classmethod 16 | def setUpClass(cls): 17 | cls.colutils = CollectionUtils() 18 | 19 | @classmethod 20 | def tearDownClass(cls): 21 | cls.colutils.clean_up() 22 | cls.colutils = None 23 | 24 | def setUp(self): 25 | monkeypatch_db() 26 | 27 | # Create temporary files and dirs the server will use. 28 | self.server_paths = helpers.server_utils.create_server_paths() 29 | 30 | # Add a test user to the temp auth db the server will use. 31 | self.user_manager = SqliteUserManager(self.server_paths['auth_db_path'], 32 | self.server_paths['data_root']) 33 | self.user_manager.add_user('testuser', 'testpassword') 34 | 35 | # Get absolute path to development ini file. 36 | script_dir = os.path.dirname(os.path.realpath(__file__)) 37 | ini_file_path = os.path.join(script_dir, 38 | "assets", 39 | "test.conf") 40 | 41 | # Create SyncApp instance using the dev ini file and the temporary 42 | # paths. 43 | self.server_app = helpers.server_utils.create_sync_app(self.server_paths, 44 | ini_file_path) 45 | 46 | # Wrap the SyncApp object in TestApp instance for testing. 47 | self.server_test_app = TestApp(self.server_app) 48 | 49 | # MockRemoteServer instance needed for testing normal collection 50 | # syncing and for retrieving hkey for other tests. 51 | self.mock_remote_server = MockRemoteServer(hkey=None, 52 | server_test_app=self.server_test_app) 53 | 54 | def tearDown(self): 55 | self.server_paths = {} 56 | self.user_manager = None 57 | 58 | # Shut down server. 59 | self.server_app.collection_manager.shutdown() 60 | self.server_app = None 61 | 62 | self.client_server_connection = None 63 | 64 | unpatch_db() 65 | -------------------------------------------------------------------------------- /tests/helpers/file_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from io import BytesIO 3 | import json 4 | import logging 5 | import logging.config 6 | import os 7 | import random 8 | import shutil 9 | import tempfile 10 | import unicodedata 11 | import zipfile 12 | 13 | from anki.consts import SYNC_ZIP_SIZE 14 | 15 | 16 | def create_named_file(filename, file_contents=None): 17 | """ 18 | Creates a temporary file with a custom name within a new temporary 19 | directory and marks that parent dir for recursive deletion method. 20 | """ 21 | 22 | # We need to create a parent directory for the file so we can freely 23 | # choose the file name . 24 | temp_file_parent_dir = tempfile.mkdtemp(prefix="named_file") 25 | 26 | file_path = os.path.join(temp_file_parent_dir, filename) 27 | 28 | if file_contents is not None: 29 | with open(file_path, "w") as f: 30 | f.write(file_contents) 31 | 32 | return file_path 33 | 34 | 35 | def create_zip_with_existing_files(file_paths): 36 | """ 37 | The method zips existing files and returns the zip data. Logic is 38 | adapted from Anki Desktop's MediaManager.mediaChangesZip(). 39 | 40 | :param file_paths: the paths of the files to include in the zip 41 | :type file_paths: list 42 | :return: the data of the created zip file 43 | """ 44 | 45 | buf = BytesIO() 46 | with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as z: 47 | meta = [] 48 | size = 0 49 | 50 | for index, path in enumerate(file_paths): 51 | z.write(path, str(index)) 52 | normname = unicodedata.normalize("NFC", os.path.basename(path)) 53 | meta.append((normname, str(index))) 54 | 55 | size += os.path.getsize(path) 56 | if size >= SYNC_ZIP_SIZE: 57 | break 58 | 59 | z.writestr("_meta", json.dumps(meta)) 60 | 61 | return buf.getvalue() 62 | 63 | 64 | def get_asset_path(relative_file_path): 65 | """ 66 | Retrieves the path of a file for testing from the "assets" directory. 67 | 68 | :param relative_file_path: the name of the file to retrieve, relative 69 | to the "assets" directory 70 | :return: the absolute path to the file in the "assets" directory. 71 | """ 72 | 73 | join = os.path.join 74 | 75 | script_dir = os.path.dirname(os.path.realpath(__file__)) 76 | support_dir = join(script_dir, os.pardir, "assets") 77 | res = join(support_dir, relative_file_path) 78 | return res 79 | -------------------------------------------------------------------------------- /tests/test_sync_app.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import sqlite3 4 | import tempfile 5 | import unittest 6 | 7 | from anki.consts import SYNC_VER 8 | 9 | from ankisyncd.sync_app import SyncCollectionHandler 10 | from ankisyncd.sync_app import SyncUserSession 11 | 12 | from collection_test_base import CollectionTestBase 13 | 14 | 15 | class SyncCollectionHandlerTest(CollectionTestBase): 16 | def setUp(self): 17 | CollectionTestBase.setUp(self) 18 | self.syncCollectionHandler = SyncCollectionHandler(self.collection) 19 | 20 | def tearDown(self): 21 | CollectionTestBase.tearDown(self) 22 | self.syncCollectionHandler = None 23 | 24 | def test_old_client(self): 25 | old = ( 26 | ','.join(('ankidesktop', '2.0.12', 'lin::')), 27 | ','.join(('ankidesktop', '2.0.26', 'lin::')), 28 | ','.join(('ankidroid', '2.1', '')), 29 | ','.join(('ankidroid', '2.2', '')), 30 | ','.join(('ankidroid', '2.2.2', '')), 31 | ','.join(('ankidroid', '2.3alpha3', '')), 32 | ) 33 | 34 | current = ( 35 | None, 36 | ','.join(('ankidesktop', '2.0.27', 'lin::')), 37 | ','.join(('ankidesktop', '2.0.32', 'lin::')), 38 | ','.join(('ankidesktop', '2.1.0', 'lin::')), 39 | ','.join(('ankidesktop', '2.1.6-beta2', 'lin::')), 40 | ','.join(('ankidesktop', '2.1.9 (dev)', 'lin::')), 41 | ','.join(('ankidroid', '2.2.3', '')), 42 | ','.join(('ankidroid', '2.3alpha4', '')), 43 | ','.join(('ankidroid', '2.3alpha5', '')), 44 | ','.join(('ankidroid', '2.3beta1', '')), 45 | ','.join(('ankidroid', '2.3', '')), 46 | ','.join(('ankidroid', '2.9', '')), 47 | ) 48 | 49 | for cv in old: 50 | if not SyncCollectionHandler._old_client(cv): 51 | raise AssertionError("old_client(\"%s\") is False" % cv) 52 | 53 | for cv in current: 54 | if SyncCollectionHandler._old_client(cv): 55 | raise AssertionError("old_client(\"%s\") is True" % cv) 56 | 57 | def test_meta(self): 58 | meta = self.syncCollectionHandler.meta(v=SYNC_VER) 59 | self.assertEqual(meta['scm'], self.collection.scm) 60 | self.assertTrue((type(meta['ts']) == int) and meta['ts'] > 0) 61 | self.assertEqual(meta['mod'], self.collection.mod) 62 | self.assertEqual(meta['usn'], self.collection._usn) 63 | self.assertEqual(meta['musn'], self.collection.media.lastUsn()) 64 | self.assertEqual(meta['msg'], '') 65 | self.assertEqual(meta['cont'], True) 66 | 67 | 68 | class SyncAppTest(unittest.TestCase): 69 | pass 70 | -------------------------------------------------------------------------------- /utils/migrate_user_tables.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | """ 4 | This script updates the auth and session sqlite3 databases to use the 5 | more compatible `username` column instead of `user`, which is a reserved 6 | word in many other SQL dialects. 7 | """ 8 | import os 9 | import sys 10 | path = os.path.realpath(os.path.abspath(os.path.join(__file__, '../'))) 11 | sys.path.insert(0, os.path.dirname(path)) 12 | 13 | import sqlite3 14 | import ankisyncd.config 15 | conf = ankisyncd.config.load() 16 | 17 | 18 | def main(): 19 | 20 | if os.path.isfile(conf["auth_db_path"]): 21 | conn = sqlite3.connect(conf["auth_db_path"]) 22 | 23 | cursor = conn.cursor() 24 | cursor.execute("SELECT * FROM sqlite_master " 25 | "WHERE sql LIKE '%user VARCHAR PRIMARY KEY%' " 26 | "AND tbl_name = 'auth'") 27 | res = cursor.fetchone() 28 | 29 | if res is not None: 30 | cursor.execute("ALTER TABLE auth RENAME TO auth_old") 31 | cursor.execute("CREATE TABLE auth (username VARCHAR PRIMARY KEY, hash VARCHAR)") 32 | cursor.execute("INSERT INTO auth (username, hash) SELECT user, hash FROM auth_old") 33 | cursor.execute("DROP TABLE auth_old") 34 | conn.commit() 35 | print("Successfully updated table 'auth'") 36 | else: 37 | print("No outdated 'auth' table found.") 38 | 39 | conn.close() 40 | else: 41 | print("No auth DB found at the configured 'auth_db_path' path.") 42 | 43 | if os.path.isfile(conf["session_db_path"]): 44 | conn = sqlite3.connect(conf["session_db_path"]) 45 | 46 | cursor = conn.cursor() 47 | cursor.execute("SELECT * FROM sqlite_master " 48 | "WHERE sql LIKE '%user VARCHAR%' " 49 | "AND tbl_name = 'session'") 50 | res = cursor.fetchone() 51 | 52 | if res is not None: 53 | cursor.execute("ALTER TABLE session RENAME TO session_old") 54 | cursor.execute("CREATE TABLE session (hkey VARCHAR PRIMARY KEY, skey VARCHAR, " 55 | "username VARCHAR, path VARCHAR)") 56 | cursor.execute("INSERT INTO session (hkey, skey, username, path) " 57 | "SELECT hkey, skey, user, path FROM session_old") 58 | cursor.execute("DROP TABLE session_old") 59 | conn.commit() 60 | print("Successfully updated table 'session'") 61 | else: 62 | print("No outdated 'session' table found.") 63 | 64 | conn.close() 65 | else: 66 | print("No session DB found at the configured 'session_db_path' path.") 67 | 68 | 69 | if __name__ == "__main__": 70 | main() 71 | -------------------------------------------------------------------------------- /addon/__init__.py: -------------------------------------------------------------------------------- 1 | from PyQt5.Qt import Qt, QCheckBox, QLabel, QHBoxLayout, QLineEdit 2 | from aqt.forms import preferences 3 | from anki.hooks import wrap, addHook 4 | import aqt 5 | import anki.consts 6 | import anki.sync 7 | 8 | DEFAULT_ADDR = "http://localhost:27701/" 9 | config = aqt.mw.addonManager.getConfig(__name__) 10 | 11 | # TODO: force the user to log out before changing any of the settings 12 | 13 | def addui(self, _): 14 | self = self.form 15 | parent_w = self.tab_2 16 | parent_l = self.vboxlayout 17 | self.useCustomServer = QCheckBox(parent_w) 18 | self.useCustomServer.setText("Use custom sync server") 19 | parent_l.addWidget(self.useCustomServer) 20 | cshl = QHBoxLayout() 21 | parent_l.addLayout(cshl) 22 | 23 | self.serverAddrLabel = QLabel(parent_w) 24 | self.serverAddrLabel.setText("Server address") 25 | cshl.addWidget(self.serverAddrLabel) 26 | self.customServerAddr = QLineEdit(parent_w) 27 | self.customServerAddr.setPlaceholderText(DEFAULT_ADDR) 28 | cshl.addWidget(self.customServerAddr) 29 | 30 | pconfig = getprofileconfig() 31 | if pconfig["enabled"]: 32 | self.useCustomServer.setCheckState(Qt.Checked) 33 | if pconfig["addr"]: 34 | self.customServerAddr.setText(pconfig["addr"]) 35 | 36 | self.customServerAddr.textChanged.connect(lambda text: updateserver(self, text)) 37 | def onchecked(state): 38 | pconfig["enabled"] = state == Qt.Checked 39 | updateui(self, state) 40 | updateserver(self, self.customServerAddr.text()) 41 | self.useCustomServer.stateChanged.connect(onchecked) 42 | 43 | updateui(self, self.useCustomServer.checkState()) 44 | 45 | def updateserver(self, text): 46 | pconfig = getprofileconfig() 47 | if pconfig['enabled']: 48 | addr = text or self.customServerAddr.placeholderText() 49 | pconfig['addr'] = addr 50 | setserver() 51 | aqt.mw.addonManager.writeConfig(__name__, config) 52 | 53 | def updateui(self, state): 54 | self.serverAddrLabel.setEnabled(state == Qt.Checked) 55 | self.customServerAddr.setEnabled(state == Qt.Checked) 56 | 57 | def setserver(): 58 | pconfig = getprofileconfig() 59 | if pconfig['enabled']: 60 | aqt.mw.pm.profile['hostNum'] = None 61 | anki.sync.SYNC_BASE = "%s" + pconfig['addr'] 62 | else: 63 | anki.sync.SYNC_BASE = anki.consts.SYNC_BASE 64 | 65 | def getprofileconfig(): 66 | if aqt.mw.pm.name not in config["profiles"]: 67 | # inherit global settings if present (used in earlier versions of the addon) 68 | config["profiles"][aqt.mw.pm.name] = { 69 | "enabled": config.get("enabled", False), 70 | "addr": config.get("addr", DEFAULT_ADDR), 71 | } 72 | aqt.mw.addonManager.writeConfig(__name__, config) 73 | return config["profiles"][aqt.mw.pm.name] 74 | 75 | addHook("profileLoaded", setserver) 76 | aqt.preferences.Preferences.__init__ = wrap(aqt.preferences.Preferences.__init__, addui, "after") 77 | -------------------------------------------------------------------------------- /tests/helpers/server_utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import configparser 3 | import logging 4 | import os 5 | import shutil 6 | import tempfile 7 | 8 | import anki.utils 9 | 10 | from ankisyncd.sync_app import SyncApp, SyncCollectionHandler, SyncMediaHandler 11 | 12 | 13 | def create_server_paths(): 14 | """ 15 | Creates temporary files and dirs for our app to use during tests. 16 | """ 17 | dir = tempfile.mkdtemp(prefix="ServerUtils") 18 | os.mkdir(os.path.join(dir, "data")) 19 | 20 | return { 21 | "auth_db_path": os.path.join(dir, "auth.db"), 22 | "session_db_path": os.path.join(dir, "session.db"), 23 | "data_root": os.path.join(dir, "data"), 24 | } 25 | 26 | def create_sync_app(server_paths, config_path): 27 | config = configparser.ConfigParser() 28 | config.read(config_path) 29 | 30 | # Use custom files and dirs in settings. 31 | config['sync_app'].update(server_paths) 32 | 33 | return SyncApp(config['sync_app']) 34 | 35 | def get_session_for_hkey(server, hkey): 36 | return server.session_manager.load(hkey) 37 | 38 | def get_thread_for_hkey(server, hkey): 39 | session = get_session_for_hkey(server, hkey) 40 | thread = session.get_thread() 41 | return thread 42 | 43 | def get_col_wrapper_for_hkey(server, hkey): 44 | thread = get_thread_for_hkey(server, hkey) 45 | col_wrapper = thread.wrapper 46 | return col_wrapper 47 | 48 | def get_col_for_hkey(server, hkey): 49 | col_wrapper = get_col_wrapper_for_hkey(server, hkey) 50 | col_wrapper.open() # Make sure the col is opened. 51 | return col_wrapper._CollectionWrapper__col 52 | 53 | def get_col_db_path_for_hkey(server, hkey): 54 | col = get_col_for_hkey(server, hkey) 55 | return col.db._path 56 | 57 | def get_syncer_for_hkey(server, hkey, syncer_type='collection'): 58 | col = get_col_for_hkey(server, hkey) 59 | 60 | session = get_session_for_hkey(server, hkey) 61 | 62 | syncer_type = syncer_type.lower() 63 | if syncer_type == 'collection': 64 | handler_method = SyncCollectionHandler.operations[0] 65 | elif syncer_type == 'media': 66 | handler_method = SyncMediaHandler.operations[0] 67 | 68 | return session.get_handler_for_operation(handler_method, col) 69 | 70 | def add_files_to_client_mediadb(media, filepaths, update_db=False): 71 | for filepath in filepaths: 72 | logging.debug("Adding file '{}' to client media DB".format(filepath)) 73 | # Import file into media dir. 74 | media.addFile(filepath) 75 | 76 | if update_db: 77 | media.findChanges() # Write changes to db. 78 | 79 | def add_files_to_server_mediadb(media, filepaths): 80 | for filepath in filepaths: 81 | logging.debug("Adding file '{}' to server media DB".format(filepath)) 82 | fname = os.path.basename(filepath) 83 | with open(filepath, 'rb') as infile: 84 | data = infile.read() 85 | csum = anki.utils.checksum(data) 86 | 87 | with open(os.path.join(media.dir(), fname), 'wb') as f: 88 | f.write(data) 89 | media.db.execute("INSERT INTO media VALUES (?, ?, ?)", fname, media.lastUsn() + 1, csum) 90 | media.db.commit() 91 | -------------------------------------------------------------------------------- /tests/helpers/monkey_patches.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import sqlite3 as sqlite 4 | from anki.media import MediaManager 5 | from anki.storage import DB 6 | 7 | mediamanager_orig_funcs = { 8 | "findChanges": None, 9 | "mediaChangesZip": None, 10 | "addFilesFromZip": None, 11 | "syncDelete": None, 12 | "_logChanges": None, 13 | } 14 | 15 | db_orig_funcs = { 16 | "__init__": None 17 | } 18 | 19 | 20 | def monkeypatch_mediamanager(): 21 | """ 22 | Monkey patches anki.media.MediaManager's methods so they chdir to 23 | self.dir() before acting on its media directory and chdir back to the 24 | original cwd after finishing. 25 | """ 26 | 27 | def make_cwd_safe(original_func): 28 | mediamanager_orig_funcs["findChanges"] = MediaManager.findChanges 29 | mediamanager_orig_funcs["mediaChangesZip"] = MediaManager.mediaChangesZip 30 | mediamanager_orig_funcs["addFilesFromZip"] = MediaManager.addFilesFromZip 31 | mediamanager_orig_funcs["syncDelete"] = MediaManager.syncDelete 32 | mediamanager_orig_funcs["_logChanges"] = MediaManager._logChanges 33 | 34 | def wrapper(instance, *args): 35 | old_cwd = os.getcwd() 36 | os.chdir(instance.dir()) 37 | 38 | res = original_func(instance, *args) 39 | 40 | os.chdir(old_cwd) 41 | return res 42 | return wrapper 43 | 44 | MediaManager.findChanges = make_cwd_safe(MediaManager.findChanges) 45 | MediaManager.mediaChangesZip = make_cwd_safe(MediaManager.mediaChangesZip) 46 | MediaManager.addFilesFromZip = make_cwd_safe(MediaManager.addFilesFromZip) 47 | MediaManager.syncDelete = make_cwd_safe(MediaManager.syncDelete) 48 | MediaManager._logChanges = make_cwd_safe(MediaManager._logChanges) 49 | 50 | 51 | def unpatch_mediamanager(): 52 | """Undoes monkey patches to Anki's MediaManager.""" 53 | 54 | MediaManager.findChanges = mediamanager_orig_funcs["findChanges"] 55 | MediaManager.mediaChangesZip = mediamanager_orig_funcs["mediaChangesZip"] 56 | MediaManager.addFilesFromZip = mediamanager_orig_funcs["addFilesFromZip"] 57 | MediaManager.syncDelete = mediamanager_orig_funcs["syncDelete"] 58 | MediaManager._logChanges = mediamanager_orig_funcs["_logChanges"] 59 | 60 | mediamanager_orig_funcs["findChanges"] = None 61 | mediamanager_orig_funcs["mediaChangesZip"] = None 62 | mediamanager_orig_funcs["mediaChangesZip"] = None 63 | mediamanager_orig_funcs["mediaChangesZip"] = None 64 | mediamanager_orig_funcs["_logChanges"] = None 65 | 66 | 67 | def monkeypatch_db(): 68 | """ 69 | Monkey patches Anki's DB.__init__ to connect to allow access to the db 70 | connection from more than one thread, so that we can inspect and modify 71 | the db created in the app in our test code. 72 | """ 73 | db_orig_funcs["__init__"] = DB.__init__ 74 | 75 | def patched___init__(self, path, text=None, timeout=0): 76 | # Code taken from Anki's DB.__init__() 77 | # Allow more than one thread to use this connection. 78 | self._db = sqlite.connect(path, 79 | timeout=timeout, 80 | check_same_thread=False) 81 | if text: 82 | self._db.text_factory = text 83 | self._path = path 84 | self.echo = os.environ.get("DBECHO") # echo db modifications 85 | self.mod = False # flag that db has been modified? 86 | 87 | DB.__init__ = patched___init__ 88 | 89 | 90 | def unpatch_db(): 91 | """Undoes monkey patches to Anki's DB.""" 92 | 93 | DB.__init__ = db_orig_funcs["__init__"] 94 | db_orig_funcs["__init__"] = None 95 | -------------------------------------------------------------------------------- /tests/test_sessions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import os 4 | import tempfile 5 | import sqlite3 6 | import unittest 7 | import configparser 8 | 9 | from ankisyncd.sessions import SimpleSessionManager 10 | from ankisyncd.sessions import SqliteSessionManager 11 | from ankisyncd.sessions import get_session_manager 12 | 13 | from ankisyncd.sync_app import SyncUserSession 14 | 15 | import helpers.server_utils 16 | 17 | class FakeSessionManager(SimpleSessionManager): 18 | def __init__(self, config): 19 | pass 20 | 21 | class BadSessionManager: 22 | pass 23 | 24 | class SessionManagerFactoryTest(unittest.TestCase): 25 | def test_get_session_manager(self): 26 | # Get absolute path to development ini file. 27 | script_dir = os.path.dirname(os.path.realpath(__file__)) 28 | ini_file_path = os.path.join(script_dir, 29 | "assets", 30 | "test.conf") 31 | 32 | # Create temporary files and dirs the server will use. 33 | server_paths = helpers.server_utils.create_server_paths() 34 | 35 | config = configparser.ConfigParser() 36 | config.read(ini_file_path) 37 | 38 | # Use custom files and dirs in settings. Should be SqliteSessionManager 39 | config['sync_app'].update(server_paths) 40 | self.assertTrue(type(get_session_manager(config['sync_app']) == SqliteSessionManager)) 41 | 42 | # No value defaults to SimpleSessionManager 43 | config.remove_option("sync_app", "session_db_path") 44 | self.assertTrue(type(get_session_manager(config['sync_app'])) == SimpleSessionManager) 45 | 46 | # A conf-specified SessionManager is loaded 47 | config.set("sync_app", "session_manager", 'test_sessions.FakeSessionManager') 48 | self.assertTrue(type(get_session_manager(config['sync_app'])) == FakeSessionManager) 49 | 50 | # Should fail at load time if the class doesn't inherit from SimpleSessionManager 51 | config.set("sync_app", "session_manager", 'test_sessions.BadSessionManager') 52 | with self.assertRaises(TypeError): 53 | sm = get_session_manager(config['sync_app']) 54 | 55 | # Add the session_db_path back, it should take precedence over BadSessionManager 56 | config['sync_app'].update(server_paths) 57 | self.assertTrue(type(get_session_manager(config['sync_app'])) == SqliteSessionManager) 58 | 59 | 60 | class SimpleSessionManagerTest(unittest.TestCase): 61 | test_hkey = '1234567890' 62 | sdir = tempfile.mkdtemp(suffix="_session") 63 | os.rmdir(sdir) 64 | test_session = SyncUserSession('testName', sdir, None, None) 65 | 66 | def setUp(self): 67 | self.sessionManager = SimpleSessionManager() 68 | 69 | def tearDown(self): 70 | self.sessionManager = None 71 | 72 | def test_save(self): 73 | self.sessionManager.save(self.test_hkey, self.test_session) 74 | self.assertEqual(self.sessionManager.sessions[self.test_hkey].name, 75 | self.test_session.name) 76 | self.assertEqual(self.sessionManager.sessions[self.test_hkey].path, 77 | self.test_session.path) 78 | 79 | def test_delete(self): 80 | self.sessionManager.save(self.test_hkey, self.test_session) 81 | self.assertTrue(self.test_hkey in self.sessionManager.sessions) 82 | 83 | self.sessionManager.delete(self.test_hkey) 84 | 85 | self.assertTrue(self.test_hkey not in self.sessionManager.sessions) 86 | 87 | def test_load(self): 88 | self.sessionManager.save(self.test_hkey, self.test_session) 89 | self.assertTrue(self.test_hkey in self.sessionManager.sessions) 90 | 91 | loaded_session = self.sessionManager.load(self.test_hkey) 92 | self.assertEqual(loaded_session.name, self.test_session.name) 93 | self.assertEqual(loaded_session.path, self.test_session.path) 94 | 95 | 96 | class SqliteSessionManagerTest(SimpleSessionManagerTest): 97 | file_descriptor, _test_sess_db_path = tempfile.mkstemp(suffix=".db") 98 | os.close(file_descriptor) 99 | os.unlink(_test_sess_db_path) 100 | 101 | def setUp(self): 102 | self.sessionManager = SqliteSessionManager(self._test_sess_db_path) 103 | 104 | def tearDown(self): 105 | if os.path.exists(self._test_sess_db_path): 106 | os.remove(self._test_sess_db_path) 107 | 108 | def test_save(self): 109 | SimpleSessionManagerTest.test_save(self) 110 | self.assertTrue(os.path.exists(self._test_sess_db_path)) 111 | 112 | conn = sqlite3.connect(self._test_sess_db_path) 113 | cursor = conn.cursor() 114 | cursor.execute("SELECT username, path FROM session WHERE hkey=?", 115 | (self.test_hkey,)) 116 | res = cursor.fetchone() 117 | conn.close() 118 | 119 | self.assertEqual(res[0], self.test_session.name) 120 | self.assertEqual(res[1], self.test_session.path) 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /ankisyncd/collection.py: -------------------------------------------------------------------------------- 1 | import anki 2 | import anki.storage 3 | 4 | import ankisyncd.media 5 | 6 | import os, errno 7 | import logging 8 | 9 | logger = logging.getLogger("ankisyncd.collection") 10 | 11 | 12 | class CollectionWrapper: 13 | """A simple wrapper around an anki.storage.Collection object. 14 | 15 | This allows us to manage and refer to the collection, whether it's open or not. It 16 | also provides a special "continuation passing" interface for executing functions 17 | on the collection, which makes it easy to switch to a threading mode. 18 | 19 | See ThreadingCollectionWrapper for a version that maintains a seperate thread for 20 | interacting with the collection. 21 | """ 22 | 23 | def __init__(self, _config, path, setup_new_collection=None): 24 | self.path = os.path.realpath(path) 25 | self.username = os.path.basename(os.path.dirname(self.path)) 26 | self.setup_new_collection = setup_new_collection 27 | self.__col = None 28 | 29 | def __del__(self): 30 | """Close the collection if the user forgot to do so.""" 31 | self.close() 32 | 33 | def execute(self, func, args=[], kw={}, waitForReturn=True): 34 | """ Executes the given function with the underlying anki.storage.Collection 35 | object as the first argument and any additional arguments specified by *args 36 | and **kw. 37 | 38 | If 'waitForReturn' is True, then it will block until the function has 39 | executed and return its return value. If False, the function MAY be 40 | executed some time later and None will be returned. 41 | """ 42 | 43 | # Open the collection and execute the function 44 | self.open() 45 | args = [self.__col] + args 46 | ret = func(*args, **kw) 47 | 48 | # Only return the value if they requested it, so the interface remains 49 | # identical between this class and ThreadingCollectionWrapper 50 | if waitForReturn: 51 | return ret 52 | 53 | def __create_collection(self): 54 | """Creates a new collection and runs any special setup.""" 55 | 56 | # mkdir -p the path, because it might not exist 57 | os.makedirs(os.path.dirname(self.path), exist_ok=True) 58 | 59 | col = self._get_collection() 60 | 61 | # Do any special setup 62 | if self.setup_new_collection is not None: 63 | self.setup_new_collection(col) 64 | 65 | return col 66 | 67 | def _get_collection(self): 68 | col = anki.storage.Collection(self.path) 69 | 70 | # Ugly hack, replace default media manager with our custom one 71 | col.media.close() 72 | col.media = ankisyncd.media.ServerMediaManager(col) 73 | 74 | return col 75 | 76 | def open(self): 77 | """Open the collection, or create it if it doesn't exist.""" 78 | if self.__col is None: 79 | if os.path.exists(self.path): 80 | self.__col = self._get_collection() 81 | else: 82 | self.__col = self.__create_collection() 83 | 84 | def close(self): 85 | """Close the collection if opened.""" 86 | if not self.opened(): 87 | return 88 | 89 | self.__col.close() 90 | self.__col = None 91 | 92 | def opened(self): 93 | """Returns True if the collection is open, False otherwise.""" 94 | return self.__col is not None 95 | 96 | class CollectionManager: 97 | """Manages a set of CollectionWrapper objects.""" 98 | 99 | collection_wrapper = CollectionWrapper 100 | 101 | def __init__(self, config): 102 | self.collections = {} 103 | self.config = config 104 | 105 | def get_collection(self, path, setup_new_collection=None): 106 | """Gets a CollectionWrapper for the given path.""" 107 | 108 | path = os.path.realpath(path) 109 | 110 | try: 111 | col = self.collections[path] 112 | except KeyError: 113 | col = self.collections[path] = self.collection_wrapper(self.config, path, setup_new_collection) 114 | 115 | return col 116 | 117 | def shutdown(self): 118 | """Close all CollectionWrappers managed by this object.""" 119 | for path, col in list(self.collections.items()): 120 | del self.collections[path] 121 | col.close() 122 | 123 | def get_collection_wrapper(config, path, setup_new_collection = None): 124 | if "collection_wrapper" in config and config["collection_wrapper"]: 125 | logger.info("Found collection_wrapper in config, using {} for " 126 | "user data persistence".format(config['collection_wrapper'])) 127 | import importlib 128 | import inspect 129 | module_name, class_name = config['collection_wrapper'].rsplit('.', 1) 130 | module = importlib.import_module(module_name.strip()) 131 | class_ = getattr(module, class_name.strip()) 132 | 133 | if not CollectionWrapper in inspect.getmro(class_): 134 | raise TypeError('''"collection_wrapper" found in the conf file but it doesn''t 135 | inherit from CollectionWrapper''') 136 | return class_(config, path, setup_new_collection) 137 | else: 138 | return CollectionWrapper(config, path, setup_new_collection) 139 | -------------------------------------------------------------------------------- /ankisyncd/sessions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import logging 4 | from sqlite3 import dbapi2 as sqlite 5 | 6 | logger = logging.getLogger("ankisyncd.sessions") 7 | 8 | 9 | class SimpleSessionManager: 10 | """A simple session manager that keeps the sessions in memory.""" 11 | 12 | def __init__(self): 13 | self.sessions = {} 14 | 15 | def load(self, hkey, session_factory=None): 16 | return self.sessions.get(hkey) 17 | 18 | def load_from_skey(self, skey, session_factory=None): 19 | for i in self.sessions: 20 | if self.sessions[i].skey == skey: 21 | return self.sessions[i] 22 | 23 | def save(self, hkey, session): 24 | self.sessions[hkey] = session 25 | 26 | def delete(self, hkey): 27 | del self.sessions[hkey] 28 | 29 | 30 | class SqliteSessionManager(SimpleSessionManager): 31 | """Stores sessions in a SQLite database to prevent the user from being logged out 32 | everytime the SyncApp is restarted.""" 33 | 34 | def __init__(self, session_db_path): 35 | SimpleSessionManager.__init__(self) 36 | 37 | self.session_db_path = os.path.realpath(session_db_path) 38 | self._ensure_schema_up_to_date() 39 | 40 | def _ensure_schema_up_to_date(self): 41 | if not os.path.exists(self.session_db_path): 42 | return True 43 | 44 | conn = self._conn() 45 | cursor = conn.cursor() 46 | cursor.execute("SELECT * FROM sqlite_master " 47 | "WHERE sql LIKE '%user VARCHAR PRIMARY KEY%' " 48 | "AND tbl_name = 'session'") 49 | res = cursor.fetchone() 50 | conn.close() 51 | if res is not None: 52 | raise Exception("Outdated database schema, run utils/migrate_user_tables.py") 53 | 54 | def _conn(self): 55 | new = not os.path.exists(self.session_db_path) 56 | conn = sqlite.connect(self.session_db_path) 57 | if new: 58 | cursor = conn.cursor() 59 | cursor.execute("CREATE TABLE session (hkey VARCHAR PRIMARY KEY, skey VARCHAR, username VARCHAR, path VARCHAR)") 60 | return conn 61 | 62 | # Default to using sqlite3 syntax but overridable for sub-classes using other 63 | # DB API 2 driver variants 64 | @staticmethod 65 | def fs(sql): 66 | return sql 67 | 68 | def load(self, hkey, session_factory=None): 69 | session = SimpleSessionManager.load(self, hkey) 70 | if session is not None: 71 | return session 72 | 73 | conn = self._conn() 74 | cursor = conn.cursor() 75 | 76 | cursor.execute(self.fs("SELECT skey, username, path FROM session WHERE hkey=?"), (hkey,)) 77 | res = cursor.fetchone() 78 | 79 | if res is not None: 80 | session = self.sessions[hkey] = session_factory(res[1], res[2]) 81 | session.skey = res[0] 82 | return session 83 | 84 | def load_from_skey(self, skey, session_factory=None): 85 | session = SimpleSessionManager.load_from_skey(self, skey) 86 | if session is not None: 87 | return session 88 | 89 | conn = self._conn() 90 | cursor = conn.cursor() 91 | 92 | cursor.execute(self.fs("SELECT hkey, username, path FROM session WHERE skey=?"), (skey,)) 93 | res = cursor.fetchone() 94 | 95 | if res is not None: 96 | session = self.sessions[res[0]] = session_factory(res[1], res[2]) 97 | session.skey = skey 98 | return session 99 | 100 | def save(self, hkey, session): 101 | SimpleSessionManager.save(self, hkey, session) 102 | 103 | conn = self._conn() 104 | cursor = conn.cursor() 105 | 106 | cursor.execute("INSERT OR REPLACE INTO session (hkey, skey, username, path) VALUES (?, ?, ?, ?)", 107 | (hkey, session.skey, session.name, session.path)) 108 | 109 | conn.commit() 110 | 111 | def delete(self, hkey): 112 | SimpleSessionManager.delete(self, hkey) 113 | 114 | conn = self._conn() 115 | cursor = conn.cursor() 116 | 117 | cursor.execute(self.fs("DELETE FROM session WHERE hkey=?"), (hkey,)) 118 | conn.commit() 119 | 120 | def get_session_manager(config): 121 | if "session_db_path" in config and config["session_db_path"]: 122 | logger.info("Found session_db_path in config, using SqliteSessionManager for auth") 123 | return SqliteSessionManager(config['session_db_path']) 124 | elif "session_manager" in config and config["session_manager"]: # load from config 125 | logger.info("Found session_manager in config, using {} for persisting sessions".format( 126 | config['session_manager']) 127 | ) 128 | import importlib 129 | import inspect 130 | module_name, class_name = config['session_manager'].rsplit('.', 1) 131 | module = importlib.import_module(module_name.strip()) 132 | class_ = getattr(module, class_name.strip()) 133 | 134 | if not SimpleSessionManager in inspect.getmro(class_): 135 | raise TypeError('''"session_manager" found in the conf file but it doesn''t 136 | inherit from SimpleSessionManager''') 137 | return class_(config) 138 | else: 139 | logger.warning("Neither session_db_path nor session_manager set, " 140 | "ankisyncd will lose sessions on application restart") 141 | return SimpleSessionManager() 142 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ankisyncd 2 | ========= 3 | 4 | [Anki][] is a powerful open source flashcard application, which helps you 5 | quickly and easily memorize facts over the long term utilizing a spaced 6 | repetition algorithm. Anki's main form is a desktop application (for Windows, 7 | Linux and macOS) which can sync to a web version (AnkiWeb) and mobile 8 | versions for Android and iOS. 9 | 10 | This is a personal Anki server, which you can sync against instead of 11 | AnkiWeb. It was originally developed by [David Snopek](https://github.com/dsnopek) 12 | to support the flashcard functionality on Bibliobird, a web application for 13 | language learning. 14 | 15 | This version is a fork of [jdoe0/ankisyncd](https://github.com/jdoe0/ankisyncd). 16 | It supports Python 3 and Anki 2.1. 17 | 18 | [Anki]: https://apps.ankiweb.net/ 19 | [dsnopek's Anki Sync Server]: https://github.com/dsnopek/anki-sync-server 20 | 21 |
Contents 22 | 23 | - [Installing](#installing) 24 | - [Installing (Docker)](#installing-docker) 25 | - [Setting up Anki](#setting-up-anki) 26 | - [Anki 2.1](#anki-21) 27 | - [Anki 2.0](#anki-20) 28 | - [AnkiDroid](#ankidroid) 29 | - [Running `ankisyncd` without `pyaudio`](#running-ankisyncd-without-pyaudio) 30 | - [Anki ≥2.1.9](#anki-219) 31 | - [Older versions](#older-versions) 32 | - [ENVVAR configuration overrides](#envvar-configuration-overrides) 33 | - [Support for other database backends](#support-for-other-database-backends) 34 |
35 | 36 | Installing 37 | ---------- 38 | 39 | 0. Install Anki. The currently supported version range is 2.1.1〜2.1.11, with the 40 | exception of 2.1.9[1](#readme-fn-01). (Keep in 41 | mind this range only applies to the Anki used by the server, clients can be 42 | as old as 2.0.27 and still work.) Running the server with other versions might 43 | work as long as they're not 2.0.x, but things might break, so do it at your 44 | own risk. If for some reason you can't get the supported Anki version easily 45 | on your system, you can use `anki-bundled` from this repo: 46 | 47 | $ git submodule update --init 48 | $ cd anki-bundled 49 | $ pip install -r requirements.txt 50 | 51 | Keep in mind `pyaudio`, a dependency of Anki, requires development headers for 52 | Python 3 and PortAudio to be present before running `pip`. If you can't or 53 | don't want to install these, you can try [patching Anki](#running-ankisyncd-without-pyaudio). 54 | 55 | 1. Install the dependencies: 56 | 57 | $ pip install webob 58 | 59 | 2. Modify ankisyncd.conf according to your needs 60 | 61 | 3. Create user: 62 | 63 | $ ./ankisyncctl.py adduser 64 | 65 | 4. Run ankisyncd: 66 | 67 | $ python -m ankisyncd 68 | 69 | --- 70 | 71 | 72 | 1. 2.1.9 is not supported due to [commit `95ccbfdd3679`][] introducing the 73 | dependency on the `aqt` module, which depends on PyQt5. The server should 74 | still work fine if you have PyQt5 installed. This has been fixed in 75 | [commit `a389b8b4a0e2`][], which is a part of the 2.1.10 release. 76 | [↑](#readme-fn-01b) 77 | 78 | [commit `95ccbfdd3679`]: https://github.com/dae/anki/commit/95ccbfdd3679dd46f22847c539c7fddb8fa904ea 79 | [commit `a389b8b4a0e2`]: https://github.com/dae/anki/commit/a389b8b4a0e209023c4533a7ee335096a704079c 80 | 81 | Installing (Docker) 82 | ------------------- 83 | 84 | Follow [these instructions](https://github.com/kuklinistvan/docker-anki-sync-server#usage). 85 | 86 | Setting up Anki 87 | --------------- 88 | 89 | ### Anki 2.1 90 | 91 | Create a new directory in [the add-ons folder][addons21] (name it something 92 | like ankisyncd), create a file named `__init__.py` containing the code below 93 | and put it in the `ankisyncd` directory. 94 | 95 | import anki.sync, anki.hooks, aqt 96 | 97 | addr = "http://127.0.0.1:27701/" # put your server address here 98 | anki.sync.SYNC_BASE = "%s" + addr 99 | def resetHostNum(): 100 | aqt.mw.pm.profile['hostNum'] = None 101 | anki.hooks.addHook("profileLoaded", resetHostNum) 102 | 103 | ### Anki 2.0 104 | 105 | Create a file (name it something like ankisyncd.py) containing the code below 106 | and put it in `~/Anki/addons`. 107 | 108 | import anki.sync 109 | 110 | addr = "http://127.0.0.1:27701/" # put your server address here 111 | anki.sync.SYNC_BASE = addr 112 | anki.sync.SYNC_MEDIA_BASE = addr + "msync/" 113 | 114 | [addons21]: https://apps.ankiweb.net/docs/addons.html#_add_on_folders 115 | 116 | ### AnkiDroid 117 | 118 | Advanced → Custom sync server 119 | 120 | Unless you have set up a reverse proxy to handle encrypted connections, use 121 | `http` as the protocol. The port will be either the default, 27701, or 122 | whatever you have specified in `ankisyncd.conf` (or, if using a reverse proxy, 123 | whatever port you configured to accept the front-end connection). 124 | 125 | **Do not use trailing slashes.** 126 | 127 | Even though the AnkiDroid interface will request an email address, this is not 128 | required; it will simply be the username you configured with `ankisyncctl.py 129 | adduser`. 130 | 131 | Running `ankisyncd` without `pyaudio` 132 | ------------------------------------- 133 | 134 | `ankisyncd` doesn't use the audio recording feature of Anki, so if you don't 135 | want to install PortAudio, you can edit some files in the `anki-bundled` 136 | directory to exclude `pyaudio`: 137 | 138 | ### Anki ≥2.1.9 139 | 140 | Just remove "pyaudio" from requirements.txt and you're done. This change has 141 | been introduced in [commit `ca710ab3f1c1`][]. 142 | 143 | [commit `ca710ab3f1c1`]: https://github.com/dae/anki/commit/ca710ab3f1c1174469a3b48f1257c0fc0ce624bf 144 | 145 | ### Older versions 146 | 147 | First go to `anki-bundled`, then follow one of the instructions below. They all 148 | do the same thing, you can pick whichever one you're most comfortable with. 149 | 150 | Manual version: remove every line past "# Packaged commands" in anki/sound.py, 151 | remove every line starting with "pyaudio" in requirements.txt 152 | 153 | `ed` version: 154 | 155 | $ echo '/# Packaged commands/,$d;w' | tr ';' '\n' | ed anki/sound.py 156 | $ echo '/^pyaudio/d;w' | tr ';' '\n' | ed requirements.txt 157 | 158 | `sed -i` version: 159 | 160 | $ sed -i '/# Packaged commands/,$d' anki/sound.py 161 | $ sed -i '/^pyaudio/d' requirements.txt 162 | 163 | ENVVAR configuration overrides 164 | ------------------------------ 165 | 166 | Configuration values can be set via environment variables using `ANKISYNCD_` prepended 167 | to the uppercase form of the configuration value. E.g. the environment variable, 168 | `ANKISYNCD_AUTH_DB_PATH` will set the configuration value `auth_db_path` 169 | 170 | Environment variables override the values set in the `ankisyncd.conf`. 171 | 172 | Support for other database backends 173 | ----------------------------------- 174 | 175 | sqlite3 is used by default for user data, authentication and session persistence. 176 | 177 | `ankisyncd` supports loading classes defined via config that manage most 178 | persistence requirements (the media DB and files are being worked on). All that is 179 | required is to extend one of the existing manager classes and then reference those 180 | classes in the config file. See ankisyncd.conf for example config. 181 | -------------------------------------------------------------------------------- /ankisyncd/thread.py: -------------------------------------------------------------------------------- 1 | from ankisyncd.collection import CollectionManager, get_collection_wrapper 2 | 3 | from threading import Thread 4 | from queue import Queue 5 | 6 | import time, logging 7 | 8 | def short_repr(obj, logger=logging.getLogger(), maxlen=80): 9 | """Like repr, but shortens strings and bytestrings if logger's logging level 10 | is above DEBUG. Currently shallow and very limited, only implemented for 11 | dicts and lists.""" 12 | if logger.isEnabledFor(logging.DEBUG): 13 | return repr(obj) 14 | 15 | def shorten(s): 16 | if isinstance(s, (bytes, str)) and len(s) > maxlen: 17 | return s[:maxlen] + ("..." if isinstance(s, str) else b"...") 18 | else: 19 | return s 20 | 21 | o = obj.copy() 22 | if isinstance(o, dict): 23 | for k in o: 24 | o[k] = shorten(o[k]) 25 | elif isinstance(o, list): 26 | for k in range(len(o)): 27 | o[k] = shorten(o[k]) 28 | 29 | return repr(o) 30 | 31 | class ThreadingCollectionWrapper: 32 | """Provides the same interface as CollectionWrapper, but it creates a new Thread to 33 | interact with the collection.""" 34 | 35 | def __init__(self, config, path, setup_new_collection=None): 36 | self.path = path 37 | self.wrapper = get_collection_wrapper(config, path, setup_new_collection) 38 | self.logger = logging.getLogger("ankisyncd." + str(self)) 39 | 40 | self._queue = Queue() 41 | self._thread = None 42 | self._running = False 43 | self.last_timestamp = time.time() 44 | 45 | self.start() 46 | 47 | def __str__(self): 48 | return "CollectionThread[{}]".format(self.wrapper.username) 49 | 50 | @property 51 | def running(self): 52 | return self._running 53 | 54 | def qempty(self): 55 | return self._queue.empty() 56 | 57 | def current(self): 58 | from threading import current_thread 59 | return current_thread() == self._thread 60 | 61 | def execute(self, func, args=[], kw={}, waitForReturn=True): 62 | """ Executes a given function on this thread with the *args and **kw. 63 | 64 | If 'waitForReturn' is True, then it will block until the function has 65 | executed and return its return value. If False, it will return None 66 | immediately and the function will be executed sometime later. 67 | """ 68 | 69 | if waitForReturn: 70 | return_queue = Queue() 71 | else: 72 | return_queue = None 73 | 74 | self._queue.put((func, args, kw, return_queue)) 75 | 76 | if return_queue is not None: 77 | ret = return_queue.get(True) 78 | if isinstance(ret, Exception): 79 | raise ret 80 | return ret 81 | 82 | def _run(self): 83 | self.logger.info("Starting...") 84 | 85 | try: 86 | while self._running: 87 | func, args, kw, return_queue = self._queue.get(True) 88 | 89 | if hasattr(func, '__name__'): 90 | func_name = func.__name__ 91 | else: 92 | func_name = func.__class__.__name__ 93 | 94 | self.logger.info("Running %s(*%s, **%s)", func_name, short_repr(args, self.logger), short_repr(kw, self.logger)) 95 | self.last_timestamp = time.time() 96 | 97 | try: 98 | ret = self.wrapper.execute(func, args, kw, return_queue) 99 | except Exception as e: 100 | self.logger.error("Unable to %s(*%s, **%s): %s", 101 | func_name, repr(args), repr(kw), e, exc_info=True) 102 | # we return the Exception which will be raise'd on the other end 103 | ret = e 104 | 105 | if return_queue is not None: 106 | return_queue.put(ret) 107 | except Exception as e: 108 | self.logger.error("Thread crashed! Exception: %s", e, exc_info=True) 109 | finally: 110 | self.wrapper.close() 111 | # clean out old thread object 112 | self._thread = None 113 | # in case we got here via an exception 114 | self._running = False 115 | 116 | self.logger.info("Stopped!") 117 | 118 | def start(self): 119 | if not self._running: 120 | self._running = True 121 | assert self._thread is None 122 | self._thread = Thread(target=self._run) 123 | self._thread.start() 124 | 125 | def stop(self): 126 | def _stop(col): 127 | self._running = False 128 | self.execute(_stop, waitForReturn=False) 129 | 130 | def stop_and_wait(self): 131 | """ Tell the thread to stop and wait for it to happen. """ 132 | self.stop() 133 | if self._thread is not None: 134 | self._thread.join() 135 | 136 | # 137 | # Mimic the CollectionWrapper interface 138 | # 139 | 140 | def open(self): 141 | """Non-op. The collection will be opened on demand.""" 142 | pass 143 | 144 | def close(self): 145 | """Closes the underlying collection without stopping the thread.""" 146 | 147 | def _close(col): 148 | self.wrapper.close() 149 | self.execute(_close, waitForReturn=False) 150 | 151 | def opened(self): 152 | return self.wrapper.opened() 153 | 154 | class ThreadingCollectionManager(CollectionManager): 155 | """Manages a set of ThreadingCollectionWrapper objects.""" 156 | 157 | collection_wrapper = ThreadingCollectionWrapper 158 | 159 | def __init__(self, config): 160 | super(ThreadingCollectionManager, self).__init__(config) 161 | 162 | self.monitor_frequency = 15 163 | self.monitor_inactivity = 90 164 | self.logger = logging.getLogger("ankisyncd.ThreadingCollectionManager") 165 | 166 | monitor = Thread(target=self._monitor_run) 167 | monitor.daemon = True 168 | monitor.start() 169 | self._monitor_thread = monitor 170 | 171 | # TODO: we should raise some error if a collection is started on a manager that has already been shutdown! 172 | # or maybe we could support being restarted? 173 | 174 | # TODO: it would be awesome to have a safe way to stop inactive threads completely! 175 | # TODO: we need a way to inform other code that the collection has been closed 176 | def _monitor_run(self): 177 | """ Monitors threads for inactivity and closes the collection on them 178 | (leaves the thread itself running -- hopefully waiting peacefully with only a 179 | small memory footprint!) """ 180 | while True: 181 | cur = time.time() 182 | for path, thread in self.collections.items(): 183 | if thread.running and thread.wrapper.opened() and thread.qempty() and cur - thread.last_timestamp >= self.monitor_inactivity: 184 | self.logger.info("Monitor is closing collection on inactive %s", thread) 185 | thread.close() 186 | time.sleep(self.monitor_frequency) 187 | 188 | def shutdown(self): 189 | # TODO: stop the monitor thread! 190 | 191 | # stop all the threads 192 | for path, col in list(self.collections.items()): 193 | del self.collections[path] 194 | col.stop() 195 | 196 | # let the parent do whatever else it might want to do... 197 | super(ThreadingCollectionManager, self).shutdown() 198 | 199 | # 200 | # For working with the global ThreadingCollectionManager: 201 | # 202 | 203 | collection_manager = None 204 | 205 | def get_collection_manager(config): 206 | """Return the global ThreadingCollectionManager for this process.""" 207 | global collection_manager 208 | if collection_manager is None: 209 | collection_manager = ThreadingCollectionManager(config) 210 | return collection_manager 211 | 212 | def shutdown(): 213 | """If the global ThreadingCollectionManager exists, shut it down.""" 214 | global collection_manager 215 | if collection_manager is not None: 216 | collection_manager.shutdown() 217 | collection_manager = None 218 | 219 | -------------------------------------------------------------------------------- /ankisyncd/users.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import binascii 3 | import hashlib 4 | import logging 5 | import os 6 | import sqlite3 as sqlite 7 | 8 | logger = logging.getLogger("ankisyncd.users") 9 | 10 | 11 | class SimpleUserManager: 12 | """A simple user manager that always allows any user.""" 13 | 14 | def __init__(self, collection_path=''): 15 | self.collection_path = collection_path 16 | 17 | def authenticate(self, username, password): 18 | """ 19 | Returns True if this username is allowed to connect with this password. 20 | False otherwise. Override this to change how users are authenticated. 21 | """ 22 | 23 | return True 24 | 25 | def userdir(self, username): 26 | """ 27 | Returns the directory name for the given user. By default, this is just 28 | the username. Override this to adjust the mapping between users and 29 | their directory. 30 | """ 31 | 32 | return username 33 | 34 | def _create_user_dir(self, username): 35 | user_dir_path = os.path.join(self.collection_path, username) 36 | if not os.path.isdir(user_dir_path): 37 | logger.info("Creating collection directory for user '{}' at {}" 38 | .format(username, user_dir_path)) 39 | os.makedirs(user_dir_path) 40 | 41 | 42 | class SqliteUserManager(SimpleUserManager): 43 | """Authenticates users against a SQLite database.""" 44 | 45 | def __init__(self, auth_db_path, collection_path=None): 46 | SimpleUserManager.__init__(self, collection_path) 47 | 48 | self.auth_db_path = os.path.realpath(auth_db_path) 49 | self._ensure_schema_up_to_date() 50 | 51 | def _ensure_schema_up_to_date(self): 52 | if not self.auth_db_exists(): 53 | return True 54 | 55 | conn = self._conn() 56 | cursor = conn.cursor() 57 | cursor.execute("SELECT * FROM sqlite_master " 58 | "WHERE sql LIKE '%user VARCHAR PRIMARY KEY%' " 59 | "AND tbl_name = 'auth'") 60 | res = cursor.fetchone() 61 | conn.close() 62 | if res is not None: 63 | raise Exception("Outdated database schema, run utils/migrate_user_tables.py") 64 | 65 | # Default to using sqlite3 but overridable for sub-classes using other 66 | # DB API 2 driver variants 67 | def auth_db_exists(self): 68 | return os.path.isfile(self.auth_db_path) 69 | 70 | # Default to using sqlite3 but overridable for sub-classes using other 71 | # DB API 2 driver variants 72 | def _conn(self): 73 | return sqlite.connect(self.auth_db_path) 74 | 75 | # Default to using sqlite3 syntax but overridable for sub-classes using other 76 | # DB API 2 driver variants 77 | @staticmethod 78 | def fs(sql): 79 | return sql 80 | 81 | def user_list(self): 82 | if not self.auth_db_exists(): 83 | raise ValueError("Auth DB {} doesn't exist".format(self.auth_db_path)) 84 | 85 | conn = self._conn() 86 | cursor = conn.cursor() 87 | cursor.execute(self.fs("SELECT username FROM auth")) 88 | rows = cursor.fetchall() 89 | conn.commit() 90 | conn.close() 91 | 92 | return [row[0] for row in rows] 93 | 94 | def user_exists(self, username): 95 | users = self.user_list() 96 | return username in users 97 | 98 | def del_user(self, username): 99 | # Warning, this doesn't remove the user directory or clean it 100 | if not self.auth_db_exists(): 101 | raise ValueError("Auth DB {} doesn't exist".format(self.auth_db_path)) 102 | 103 | conn = self._conn() 104 | cursor = conn.cursor() 105 | logger.info("Removing user '{}' from auth db".format(username)) 106 | cursor.execute(self.fs("DELETE FROM auth WHERE username=?"), (username,)) 107 | conn.commit() 108 | conn.close() 109 | 110 | def add_user(self, username, password): 111 | self._add_user_to_auth_db(username, password) 112 | self._create_user_dir(username) 113 | 114 | def add_users(self, users_data): 115 | for username, password in users_data: 116 | self.add_user(username, password) 117 | 118 | def _add_user_to_auth_db(self, username, password): 119 | if not self.auth_db_exists(): 120 | self.create_auth_db() 121 | 122 | pass_hash = self._create_pass_hash(username, password) 123 | 124 | conn = self._conn() 125 | cursor = conn.cursor() 126 | logger.info("Adding user '{}' to auth db.".format(username)) 127 | cursor.execute(self.fs("INSERT INTO auth VALUES (?, ?)"), 128 | (username, pass_hash)) 129 | conn.commit() 130 | conn.close() 131 | 132 | def set_password_for_user(self, username, new_password): 133 | if not self.auth_db_exists(): 134 | raise ValueError("Auth DB {} doesn't exist".format(self.auth_db_path)) 135 | elif not self.user_exists(username): 136 | raise ValueError("User {} doesn't exist".format(username)) 137 | 138 | hash = self._create_pass_hash(username, new_password) 139 | 140 | conn = self._conn() 141 | cursor = conn.cursor() 142 | cursor.execute(self.fs("UPDATE auth SET hash=? WHERE username=?"), (hash, username)) 143 | conn.commit() 144 | conn.close() 145 | 146 | logger.info("Changed password for user {}".format(username)) 147 | 148 | def authenticate(self, username, password): 149 | """Returns True if this username is allowed to connect with this password. False otherwise.""" 150 | 151 | conn = self._conn() 152 | cursor = conn.cursor() 153 | param = (username,) 154 | cursor.execute(self.fs("SELECT hash FROM auth WHERE username=?"), param) 155 | db_hash = cursor.fetchone() 156 | conn.close() 157 | 158 | if db_hash is None: 159 | logger.info("Authentication failed for nonexistent user {}." 160 | .format(username)) 161 | return False 162 | 163 | expected_value = str(db_hash[0]) 164 | salt = self._extract_salt(expected_value) 165 | 166 | hashobj = hashlib.sha256() 167 | hashobj.update((username + password + salt).encode()) 168 | actual_value = hashobj.hexdigest() + salt 169 | 170 | if actual_value == expected_value: 171 | logger.info("Authentication succeeded for user {}".format(username)) 172 | return True 173 | else: 174 | logger.info("Authentication failed for user {}".format(username)) 175 | return False 176 | 177 | @staticmethod 178 | def _extract_salt(hash): 179 | return hash[-16:] 180 | 181 | @staticmethod 182 | def _create_pass_hash(username, password): 183 | salt = binascii.b2a_hex(os.urandom(8)) 184 | pass_hash = (hashlib.sha256((username + password).encode() + salt).hexdigest() + 185 | salt.decode()) 186 | return pass_hash 187 | 188 | def create_auth_db(self): 189 | conn = self._conn() 190 | cursor = conn.cursor() 191 | logger.info("Creating auth db at {}." 192 | .format(self.auth_db_path)) 193 | cursor.execute(self.fs("""CREATE TABLE IF NOT EXISTS auth 194 | (username VARCHAR PRIMARY KEY, hash VARCHAR)""")) 195 | conn.commit() 196 | conn.close() 197 | 198 | 199 | def get_user_manager(config): 200 | if "auth_db_path" in config and config["auth_db_path"]: 201 | logger.info("Found auth_db_path in config, using SqliteUserManager for auth") 202 | return SqliteUserManager(config['auth_db_path'], config['data_root']) 203 | elif "user_manager" in config and config["user_manager"]: # load from config 204 | logger.info("Found user_manager in config, using {} for auth".format(config['user_manager'])) 205 | import importlib 206 | import inspect 207 | module_name, class_name = config['user_manager'].rsplit('.', 1) 208 | module = importlib.import_module(module_name.strip()) 209 | class_ = getattr(module, class_name.strip()) 210 | 211 | if not SimpleUserManager in inspect.getmro(class_): 212 | raise TypeError('''"user_manager" found in the conf file but it doesn''t 213 | inherit from SimpleUserManager''') 214 | return class_(config) 215 | else: 216 | logger.warning("neither auth_db_path nor user_manager set, ankisyncd will accept any password") 217 | return SimpleUserManager() 218 | -------------------------------------------------------------------------------- /tests/test_users.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import shutil 4 | import tempfile 5 | import unittest 6 | import configparser 7 | 8 | from ankisyncd.users import SimpleUserManager, SqliteUserManager 9 | from ankisyncd.users import get_user_manager 10 | 11 | import helpers.server_utils 12 | 13 | class FakeUserManager(SimpleUserManager): 14 | def __init__(self, config): 15 | pass 16 | 17 | class BadUserManager: 18 | pass 19 | 20 | class UserManagerFactoryTest(unittest.TestCase): 21 | def test_get_user_manager(self): 22 | # Get absolute path to development ini file. 23 | script_dir = os.path.dirname(os.path.realpath(__file__)) 24 | ini_file_path = os.path.join(script_dir, 25 | "assets", 26 | "test.conf") 27 | 28 | # Create temporary files and dirs the server will use. 29 | server_paths = helpers.server_utils.create_server_paths() 30 | 31 | config = configparser.ConfigParser() 32 | config.read(ini_file_path) 33 | 34 | # Use custom files and dirs in settings. Should be SqliteUserManager 35 | config['sync_app'].update(server_paths) 36 | self.assertTrue(type(get_user_manager(config['sync_app']) == SqliteUserManager)) 37 | 38 | # No value defaults to SimpleUserManager 39 | config.remove_option("sync_app", "auth_db_path") 40 | self.assertTrue(type(get_user_manager(config['sync_app'])) == SimpleUserManager) 41 | 42 | # A conf-specified UserManager is loaded 43 | config.set("sync_app", "user_manager", 'test_users.FakeUserManager') 44 | self.assertTrue(type(get_user_manager(config['sync_app'])) == FakeUserManager) 45 | 46 | # Should fail at load time if the class doesn't inherit from SimpleUserManager 47 | config.set("sync_app", "user_manager", 'test_users.BadUserManager') 48 | with self.assertRaises(TypeError): 49 | um = get_user_manager(config['sync_app']) 50 | 51 | # Add the auth_db_path back, it should take precedence over BadUserManager 52 | config['sync_app'].update(server_paths) 53 | self.assertTrue(type(get_user_manager(config['sync_app']) == SqliteUserManager)) 54 | 55 | 56 | class SimpleUserManagerTest(unittest.TestCase): 57 | def setUp(self): 58 | self.user_manager = SimpleUserManager() 59 | 60 | def tearDown(self): 61 | self._user_manager = None 62 | 63 | def test_authenticate(self): 64 | good_test_un = 'username' 65 | good_test_pw = 'password' 66 | bad_test_un = 'notAUsername' 67 | bad_test_pw = 'notAPassword' 68 | 69 | self.assertTrue(self.user_manager.authenticate(good_test_un, 70 | good_test_pw)) 71 | self.assertTrue(self.user_manager.authenticate(bad_test_un, 72 | bad_test_pw)) 73 | self.assertTrue(self.user_manager.authenticate(good_test_un, 74 | bad_test_pw)) 75 | self.assertTrue(self.user_manager.authenticate(bad_test_un, 76 | good_test_pw)) 77 | 78 | def test_userdir(self): 79 | username = 'my_username' 80 | dirname = self.user_manager.userdir(username) 81 | self.assertEqual(dirname, username) 82 | 83 | 84 | class SqliteUserManagerTest(unittest.TestCase): 85 | def setUp(self): 86 | basedir = tempfile.mkdtemp(prefix=self.__class__.__name__) 87 | self.basedir = basedir 88 | self.auth_db_path = os.path.join(basedir, "auth.db") 89 | self.collection_path = os.path.join(basedir, "collections") 90 | self.user_manager = SqliteUserManager(self.auth_db_path, 91 | self.collection_path) 92 | 93 | def tearDown(self): 94 | shutil.rmtree(self.basedir) 95 | self.user_manager = None 96 | 97 | def test_auth_db_exists(self): 98 | self.assertFalse(self.user_manager.auth_db_exists()) 99 | 100 | self.user_manager.create_auth_db() 101 | self.assertTrue(self.user_manager.auth_db_exists()) 102 | 103 | os.unlink(self.auth_db_path) 104 | self.assertFalse(self.user_manager.auth_db_exists()) 105 | 106 | def test_user_list(self): 107 | username = "my_username" 108 | password = "my_password" 109 | self.user_manager.create_auth_db() 110 | 111 | self.assertEqual(self.user_manager.user_list(), []) 112 | 113 | self.user_manager.add_user(username, password) 114 | self.assertEqual(self.user_manager.user_list(), [username]) 115 | 116 | def test_user_exists(self): 117 | username = "my_username" 118 | password = "my_password" 119 | self.user_manager.create_auth_db() 120 | self.user_manager.add_user(username, password) 121 | self.assertTrue(self.user_manager.user_exists(username)) 122 | 123 | self.user_manager.del_user(username) 124 | self.assertFalse(self.user_manager.user_exists(username)) 125 | 126 | def test_del_user(self): 127 | username = "my_username" 128 | password = "my_password" 129 | collection_dir_path = os.path.join(self.collection_path, username) 130 | self.user_manager.create_auth_db() 131 | self.user_manager.add_user(username, password) 132 | self.user_manager.del_user(username) 133 | 134 | # User should be gone. 135 | self.assertFalse(self.user_manager.user_exists(username)) 136 | # User's collection dir should still be there. 137 | self.assertTrue(os.path.isdir(collection_dir_path)) 138 | 139 | def test_add_user(self): 140 | username = "my_username" 141 | password = "my_password" 142 | expected_dir_path = os.path.join(self.collection_path, username) 143 | self.user_manager.create_auth_db() 144 | 145 | self.assertFalse(os.path.exists(expected_dir_path)) 146 | 147 | self.user_manager.add_user(username, password) 148 | 149 | # User db entry and collection dir should be present. 150 | self.assertTrue(self.user_manager.user_exists(username)) 151 | self.assertTrue(os.path.isdir(expected_dir_path)) 152 | 153 | def test_add_users(self): 154 | users_data = [("my_first_username", "my_first_password"), 155 | ("my_second_username", "my_second_password")] 156 | self.user_manager.create_auth_db() 157 | self.user_manager.add_users(users_data) 158 | 159 | user_list = self.user_manager.user_list() 160 | self.assertIn("my_first_username", user_list) 161 | self.assertIn("my_second_username", user_list) 162 | self.assertTrue(os.path.isdir(os.path.join(self.collection_path, 163 | "my_first_username"))) 164 | self.assertTrue(os.path.isdir(os.path.join(self.collection_path, 165 | "my_second_username"))) 166 | 167 | def test__add_user_to_auth_db(self): 168 | username = "my_username" 169 | password = "my_password" 170 | self.user_manager.create_auth_db() 171 | self.user_manager.add_user(username, password) 172 | 173 | self.assertTrue(self.user_manager.user_exists(username)) 174 | 175 | def test_create_auth_db(self): 176 | self.assertFalse(os.path.exists(self.auth_db_path)) 177 | self.user_manager.create_auth_db() 178 | self.assertTrue(os.path.isfile(self.auth_db_path)) 179 | 180 | def test__create_user_dir(self): 181 | username = "my_username" 182 | expected_dir_path = os.path.join(self.collection_path, username) 183 | self.assertFalse(os.path.exists(expected_dir_path)) 184 | self.user_manager._create_user_dir(username) 185 | self.assertTrue(os.path.isdir(expected_dir_path)) 186 | 187 | def test_authenticate(self): 188 | username = "my_username" 189 | password = "my_password" 190 | 191 | self.user_manager.create_auth_db() 192 | self.user_manager.add_user(username, password) 193 | 194 | self.assertTrue(self.user_manager.authenticate(username, 195 | password)) 196 | 197 | def test_set_password_for_user(self): 198 | username = "my_username" 199 | password = "my_password" 200 | new_password = "my_new_password" 201 | 202 | self.user_manager.create_auth_db() 203 | self.user_manager.add_user(username, password) 204 | 205 | self.user_manager.set_password_for_user(username, new_password) 206 | self.assertFalse(self.user_manager.authenticate(username, 207 | password)) 208 | self.assertTrue(self.user_manager.authenticate(username, 209 | new_password)) 210 | 211 | -------------------------------------------------------------------------------- /tests/test_web_media.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import tempfile 3 | import filecmp 4 | import sqlite3 5 | import os 6 | import shutil 7 | 8 | import helpers.file_utils 9 | import helpers.server_utils 10 | import helpers.db_utils 11 | import anki.utils 12 | from anki.sync import MediaSyncer 13 | from helpers.mock_servers import MockRemoteMediaServer 14 | from helpers.monkey_patches import monkeypatch_mediamanager, unpatch_mediamanager 15 | from sync_app_functional_test_base import SyncAppFunctionalTestBase 16 | 17 | 18 | class SyncAppFunctionalMediaTest(SyncAppFunctionalTestBase): 19 | def setUp(self): 20 | SyncAppFunctionalTestBase.setUp(self) 21 | 22 | monkeypatch_mediamanager() 23 | self.tempdir = tempfile.mkdtemp(prefix=self.__class__.__name__) 24 | self.hkey = self.mock_remote_server.hostKey("testuser", "testpassword") 25 | client_collection = self.colutils.create_empty_col() 26 | self.client_syncer = self.create_client_syncer(client_collection, 27 | self.hkey, 28 | self.server_test_app) 29 | 30 | def tearDown(self): 31 | self.hkey = None 32 | self.client_syncer = None 33 | unpatch_mediamanager() 34 | SyncAppFunctionalTestBase.tearDown(self) 35 | 36 | @staticmethod 37 | def create_client_syncer(collection, hkey, server_test_app): 38 | mock_remote_server = MockRemoteMediaServer(col=collection, 39 | hkey=hkey, 40 | server_test_app=server_test_app) 41 | media_syncer = MediaSyncer(col=collection, 42 | server=mock_remote_server) 43 | return media_syncer 44 | 45 | @staticmethod 46 | def file_checksum(fname): 47 | with open(fname, "rb") as f: 48 | return anki.utils.checksum(f.read()) 49 | 50 | def media_dbs_differ(self, left_db_path, right_db_path, compare_timestamps=False): 51 | """ 52 | Compares two media sqlite database files for equality. mtime and dirMod 53 | timestamps are not considered when comparing. 54 | 55 | :param left_db_path: path to the left db file 56 | :param right_db_path: path to the right db file 57 | :param compare_timestamps: flag determining if timestamp values 58 | (media.mtime and meta.dirMod) are included 59 | in the comparison 60 | :return: True if the specified databases differ, False else 61 | """ 62 | 63 | if not os.path.isfile(right_db_path): 64 | raise IOError("file '" + left_db_path + "' does not exist") 65 | elif not os.path.isfile(right_db_path): 66 | raise IOError("file '" + right_db_path + "' does not exist") 67 | 68 | # Create temporary copies of the files to act on. 69 | newleft = os.path.join(self.tempdir, left_db_path) + ".tmp" 70 | shutil.copyfile(left_db_path, newleft) 71 | left_db_path = newleft 72 | 73 | newright = os.path.join(self.tempdir, left_db_path) + ".tmp" 74 | shutil.copyfile(right_db_path, newright) 75 | right_db_path = newright 76 | 77 | if not compare_timestamps: 78 | # Set all timestamps that are not NULL to 0. 79 | for dbPath in [left_db_path, right_db_path]: 80 | connection = sqlite3.connect(dbPath) 81 | 82 | connection.execute("""UPDATE media SET mtime=0 83 | WHERE mtime IS NOT NULL""") 84 | 85 | connection.execute("""UPDATE meta SET dirMod=0 86 | WHERE rowid=1""") 87 | connection.commit() 88 | connection.close() 89 | 90 | return helpers.db_utils.diff(left_db_path, right_db_path) 91 | 92 | def test_sync_empty_media_dbs(self): 93 | # With both the client and the server having no media to sync, 94 | # syncing should change nothing. 95 | self.assertEqual('noChanges', self.client_syncer.sync()) 96 | self.assertEqual('noChanges', self.client_syncer.sync()) 97 | 98 | def test_sync_file_from_server(self): 99 | """ 100 | Adds a file on the server. After syncing, client and server should have 101 | the identical file in their media directories and media databases. 102 | """ 103 | client = self.client_syncer 104 | server = helpers.server_utils.get_syncer_for_hkey(self.server_app, 105 | self.hkey, 106 | 'media') 107 | 108 | # Create a test file. 109 | temp_file_path = helpers.file_utils.create_named_file("foo.jpg", "hello") 110 | 111 | # Add the test file to the server's collection. 112 | helpers.server_utils.add_files_to_server_mediadb(server.col.media, [temp_file_path]) 113 | 114 | # Syncing should work. 115 | self.assertEqual(client.sync(), 'OK') 116 | 117 | # The test file should be present in the server's and in the client's 118 | # media directory. 119 | self.assertTrue( 120 | filecmp.cmp(os.path.join(client.col.media.dir(), "foo.jpg"), 121 | os.path.join(server.col.media.dir(), "foo.jpg"))) 122 | 123 | # Further syncing should do nothing. 124 | self.assertEqual(client.sync(), 'noChanges') 125 | 126 | def test_sync_file_from_client(self): 127 | """ 128 | Adds a file on the client. After syncing, client and server should have 129 | the identical file in their media directories and media databases. 130 | """ 131 | join = os.path.join 132 | client = self.client_syncer 133 | server = helpers.server_utils.get_syncer_for_hkey(self.server_app, 134 | self.hkey, 135 | 'media') 136 | 137 | # Create a test file. 138 | temp_file_path = helpers.file_utils.create_named_file("foo.jpg", "hello") 139 | 140 | # Add the test file to the client's media collection. 141 | helpers.server_utils.add_files_to_client_mediadb(client.col.media, 142 | [temp_file_path], 143 | update_db=True) 144 | 145 | # Syncing should work. 146 | self.assertEqual(client.sync(), 'OK') 147 | 148 | # The same file should be present in both the client's and the server's 149 | # media directory. 150 | self.assertTrue(filecmp.cmp(join(client.col.media.dir(), "foo.jpg"), 151 | join(server.col.media.dir(), "foo.jpg"))) 152 | 153 | # Further syncing should do nothing. 154 | self.assertEqual(client.sync(), 'noChanges') 155 | 156 | # The media data of client and server should be identical. 157 | self.assertEqual( 158 | list(client.col.media.db.execute("SELECT fname, csum FROM media")), 159 | list(server.col.media.db.execute("SELECT fname, csum FROM media")) 160 | ) 161 | self.assertEqual(client.col.media.lastUsn(), server.col.media.lastUsn()) 162 | 163 | def test_sync_different_files(self): 164 | """ 165 | Adds a file on the client and a file with different name and content on 166 | the server. After syncing, both client and server should have both 167 | files in their media directories and databases. 168 | """ 169 | join = os.path.join 170 | isfile = os.path.isfile 171 | client = self.client_syncer 172 | server = helpers.server_utils.get_syncer_for_hkey(self.server_app, 173 | self.hkey, 174 | 'media') 175 | 176 | # Create two files and add one to the server and one to the client. 177 | file_for_client = helpers.file_utils.create_named_file("foo.jpg", "hello") 178 | file_for_server = helpers.file_utils.create_named_file("bar.jpg", "goodbye") 179 | 180 | helpers.server_utils.add_files_to_client_mediadb(client.col.media, 181 | [file_for_client], 182 | update_db=True) 183 | helpers.server_utils.add_files_to_server_mediadb(server.col.media, [file_for_server]) 184 | 185 | # Syncing should work. 186 | self.assertEqual(client.sync(), 'OK') 187 | 188 | # Both files should be present in the client's and in the server's 189 | # media directories. 190 | self.assertTrue(isfile(join(client.col.media.dir(), "foo.jpg"))) 191 | self.assertTrue(isfile(join(server.col.media.dir(), "foo.jpg"))) 192 | self.assertTrue(filecmp.cmp( 193 | join(client.col.media.dir(), "foo.jpg"), 194 | join(server.col.media.dir(), "foo.jpg")) 195 | ) 196 | self.assertTrue(isfile(join(client.col.media.dir(), "bar.jpg"))) 197 | self.assertTrue(isfile(join(server.col.media.dir(), "bar.jpg"))) 198 | self.assertTrue(filecmp.cmp( 199 | join(client.col.media.dir(), "bar.jpg"), 200 | join(server.col.media.dir(), "bar.jpg")) 201 | ) 202 | 203 | # Further syncing should change nothing. 204 | self.assertEqual(client.sync(), 'noChanges') 205 | 206 | def test_sync_different_contents(self): 207 | """ 208 | Adds a file to the client and a file with identical name but different 209 | contents to the server. After syncing, both client and server should 210 | have the server's version of the file in their media directories and 211 | databases. 212 | """ 213 | join = os.path.join 214 | isfile = os.path.isfile 215 | client = self.client_syncer 216 | server = helpers.server_utils.get_syncer_for_hkey(self.server_app, 217 | self.hkey, 218 | 'media') 219 | 220 | # Create two files with identical names but different contents and 221 | # checksums. Add one to the server and one to the client. 222 | file_for_client = helpers.file_utils.create_named_file("foo.jpg", "hello") 223 | file_for_server = helpers.file_utils.create_named_file("foo.jpg", "goodbye") 224 | 225 | helpers.server_utils.add_files_to_client_mediadb(client.col.media, 226 | [file_for_client], 227 | update_db=True) 228 | helpers.server_utils.add_files_to_server_mediadb(server.col.media, [file_for_server]) 229 | 230 | # Syncing should work. 231 | self.assertEqual(client.sync(), 'OK') 232 | 233 | # A version of the file should be present in both the client's and the 234 | # server's media directory. 235 | self.assertTrue(isfile(join(client.col.media.dir(), "foo.jpg"))) 236 | self.assertEqual(os.listdir(client.col.media.dir()), ['foo.jpg']) 237 | self.assertTrue(isfile(join(server.col.media.dir(), "foo.jpg"))) 238 | self.assertEqual(os.listdir(server.col.media.dir()), ['foo.jpg']) 239 | self.assertEqual(client.sync(), 'noChanges') 240 | 241 | # Both files should have the contents of the server's version. 242 | _checksum = client.col.media._checksum 243 | self.assertEqual(_checksum(join(client.col.media.dir(), "foo.jpg")), 244 | _checksum(file_for_server)) 245 | self.assertEqual(_checksum(join(server.col.media.dir(), "foo.jpg")), 246 | _checksum(file_for_server)) 247 | 248 | def test_sync_add_and_delete_on_client(self): 249 | """ 250 | Adds a file on the client. After syncing, the client and server should 251 | both have the file. Then removes the file from the client's directory 252 | and marks it as deleted in its database. After syncing again, the 253 | server should have removed its version of the file from its media dir 254 | and marked it as deleted in its db. 255 | """ 256 | join = os.path.join 257 | isfile = os.path.isfile 258 | client = self.client_syncer 259 | server = helpers.server_utils.get_syncer_for_hkey(self.server_app, 260 | self.hkey, 261 | 'media') 262 | 263 | # Create a test file. 264 | temp_file_path = helpers.file_utils.create_named_file("foo.jpg", "hello") 265 | 266 | # Add the test file to client's media collection. 267 | helpers.server_utils.add_files_to_client_mediadb(client.col.media, 268 | [temp_file_path], 269 | update_db=True) 270 | 271 | # Syncing client should work. 272 | self.assertEqual(client.sync(), 'OK') 273 | 274 | # The same file should be present in both client's and the server's 275 | # media directory. 276 | self.assertTrue(filecmp.cmp(join(client.col.media.dir(), "foo.jpg"), 277 | join(server.col.media.dir(), "foo.jpg"))) 278 | 279 | # Syncing client again should do nothing. 280 | self.assertEqual(client.sync(), 'noChanges') 281 | 282 | # Remove files from client's media dir and write changes to its db. 283 | os.remove(join(client.col.media.dir(), "foo.jpg")) 284 | 285 | # TODO: client.col.media.findChanges() doesn't work here - why? 286 | client.col.media._logChanges() 287 | self.assertEqual(client.col.media.syncInfo("foo.jpg"), (None, 1)) 288 | self.assertFalse(isfile(join(client.col.media.dir(), "foo.jpg"))) 289 | 290 | # Syncing client again should work. 291 | self.assertEqual(client.sync(), 'OK') 292 | 293 | # server should have picked up the removal from client. 294 | self.assertEqual(server.col.media.syncInfo("foo.jpg"), (None, 0)) 295 | self.assertFalse(isfile(join(server.col.media.dir(), "foo.jpg"))) 296 | 297 | # Syncing client again should do nothing. 298 | self.assertEqual(client.sync(), 'noChanges') 299 | 300 | def test_sync_compare_database_to_expected(self): 301 | """ 302 | Adds a test image file to the client's media directory. After syncing, 303 | the server's database should, except for timestamps, be identical to a 304 | database containing the expected data. 305 | """ 306 | client = self.client_syncer 307 | 308 | # Add a test image file to the client's media collection but don't 309 | # update its media db since the desktop client updates that, using 310 | # findChanges(), only during syncs. 311 | support_file = helpers.file_utils.get_asset_path('blue.jpg') 312 | self.assertTrue(os.path.isfile(support_file)) 313 | helpers.server_utils.add_files_to_client_mediadb(client.col.media, 314 | [support_file], 315 | update_db=False) 316 | 317 | # Syncing should work. 318 | self.assertEqual(client.sync(), "OK") 319 | 320 | # Create temporary db file with expected results. 321 | chksum = client.col.media._checksum(support_file) 322 | sql = (""" 323 | CREATE TABLE meta (dirMod int, lastUsn int); 324 | 325 | INSERT INTO `meta` (dirMod, lastUsn) VALUES (123456789,1); 326 | 327 | CREATE TABLE media ( 328 | fname text not null primary key, 329 | csum text, 330 | mtime int not null, 331 | dirty int not null 332 | ); 333 | 334 | INSERT INTO `media` (fname, csum, mtime, dirty) VALUES ( 335 | 'blue.jpg', 336 | '%s', 337 | 1441483037, 338 | 0 339 | ); 340 | 341 | CREATE INDEX idx_media_dirty on media (dirty); 342 | """ % chksum) 343 | 344 | _, dbpath = tempfile.mkstemp(suffix=".anki2") 345 | helpers.db_utils.from_sql(dbpath, sql) 346 | 347 | # Except for timestamps, the client's db after sync should be identical 348 | # to the expected data. 349 | self.assertFalse(self.media_dbs_differ( 350 | client.col.media.db._path, 351 | dbpath 352 | )) 353 | os.unlink(dbpath) 354 | 355 | def test_sync_mediaChanges(self): 356 | client = self.client_syncer 357 | client2 = self.create_client_syncer(self.colutils.create_empty_col(), self.hkey, self.server_test_app) 358 | server = helpers.server_utils.get_syncer_for_hkey(self.server_app, self.hkey, 'media') 359 | self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], []) 360 | 361 | helpers.server_utils.add_files_to_client_mediadb(client.col.media, [ 362 | helpers.file_utils.create_named_file("a", "lastUsn a"), 363 | helpers.file_utils.create_named_file("b", "lastUsn b"), 364 | helpers.file_utils.create_named_file("c", "lastUsn c"), 365 | ], update_db=True) 366 | self.assertEqual(client.sync(), "OK") 367 | self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], []) 368 | 369 | self.assertEqual(client2.sync(), "OK") 370 | os.remove(os.path.join(client2.col.media.dir(), "c")) 371 | client2.col.media._logChanges() 372 | self.assertEqual(client2.sync(), "OK") 373 | self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], [['c', 4, None]]) 374 | self.assertEqual(client.sync(), "OK") 375 | self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], []) 376 | 377 | helpers.server_utils.add_files_to_client_mediadb(client.col.media, [ 378 | helpers.file_utils.create_named_file("d", "lastUsn d"), 379 | ], update_db=True) 380 | client.col.media._logChanges() 381 | self.assertEqual(client.sync(), "OK") 382 | 383 | self.assertEqual(server.mediaChanges(lastUsn=client2.col.media.lastUsn())['data'], [['d', 5, self.file_checksum(os.path.join(server.col.media.dir(), "d"))]]) 384 | 385 | self.assertEqual(client2.sync(), "OK") 386 | self.assertEqual(server.mediaChanges(lastUsn=client2.col.media.lastUsn())['data'], []) 387 | 388 | dpath = os.path.join(client.col.media.dir(), "d") 389 | with open(dpath, "a") as f: 390 | f.write("\nsome change") 391 | # files with the same mtime and name are considered equivalent by anki.media.MediaManager._changes 392 | os.utime(dpath, (315529200, 315529200)) 393 | client.col.media._logChanges() 394 | self.assertEqual(client.sync(), "OK") 395 | self.assertEqual(server.mediaChanges(lastUsn=client2.col.media.lastUsn())['data'], [['d', 6, self.file_checksum(os.path.join(server.col.media.dir(), "d"))]]) 396 | self.assertEqual(client2.sync(), "OK") 397 | self.assertEqual(server.mediaChanges(lastUsn=client2.col.media.lastUsn())['data'], []) 398 | 399 | def test_sync_rename(self): 400 | """ 401 | Adds 3 media files to the client's media directory, syncs and then 402 | renames them and syncs again. After syncing, both the client and the 403 | server should only have the renamed files. 404 | """ 405 | client = self.client_syncer 406 | client2 = self.create_client_syncer(self.colutils.create_empty_col(), self.hkey, self.server_test_app) 407 | server = helpers.server_utils.get_syncer_for_hkey(self.server_app, self.hkey, 'media') 408 | self.assertEqual(server.mediaChanges(lastUsn=client.col.media.lastUsn())['data'], []) 409 | 410 | helpers.server_utils.add_files_to_client_mediadb(client.col.media, [ 411 | helpers.file_utils.create_named_file("a.wav", "lastUsn a"), 412 | helpers.file_utils.create_named_file("b.wav", "lastUsn b"), 413 | helpers.file_utils.create_named_file("c.wav", "lastUsn c"), 414 | ], update_db=True) 415 | self.assertEqual(client.sync(), "OK") 416 | 417 | for fname in os.listdir(client.col.media.dir()): 418 | os.rename( 419 | os.path.join(client.col.media.dir(), fname), 420 | os.path.join(client.col.media.dir(), fname[:1] + ".mp3") 421 | ) 422 | client.col.media._logChanges() 423 | self.assertEqual(client.sync(), "OK") 424 | self.assertEqual( 425 | set(os.listdir(server.col.media.dir())), 426 | {"a.mp3", "b.mp3", "c.mp3"}, 427 | ) 428 | self.assertEqual( 429 | set(os.listdir(client.col.media.dir())), 430 | set(os.listdir(server.col.media.dir())), 431 | ) 432 | self.assertEqual( 433 | list(client.col.media.db.execute("SELECT fname, csum FROM media ORDER BY fname")), 434 | list(server.col.media.db.execute("SELECT fname, csum FROM media ORDER BY fname")), 435 | ) 436 | -------------------------------------------------------------------------------- /ankisyncd/sync_app.py: -------------------------------------------------------------------------------- 1 | # ankisyncd - A personal Anki sync server 2 | # Copyright (C) 2013 David Snopek 3 | # 4 | # This program is free software: you can redistribute it and/or modify 5 | # it under the terms of the GNU Affero General Public License as 6 | # published by the Free Software Foundation, either version 3 of the 7 | # License, or (at your option) any later version. 8 | # 9 | # This program is distributed in the hope that it will be useful, 10 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | # GNU Affero General Public License for more details. 13 | # 14 | # You should have received a copy of the GNU Affero General Public License 15 | # along with this program. If not, see . 16 | 17 | import gzip 18 | import hashlib 19 | import io 20 | import json 21 | import logging 22 | import os 23 | import random 24 | import re 25 | import string 26 | import sys 27 | import time 28 | import unicodedata 29 | import zipfile 30 | from configparser import ConfigParser 31 | from sqlite3 import dbapi2 as sqlite 32 | 33 | from webob import Response 34 | from webob.dec import wsgify 35 | from webob.exc import * 36 | 37 | import anki.db 38 | import anki.sync 39 | import anki.utils 40 | from anki.consts import SYNC_VER, SYNC_ZIP_SIZE, SYNC_ZIP_COUNT 41 | from anki.consts import REM_CARD, REM_NOTE 42 | 43 | from ankisyncd.users import get_user_manager 44 | from ankisyncd.sessions import get_session_manager 45 | from ankisyncd.full_sync import get_full_sync_manager 46 | 47 | logger = logging.getLogger("ankisyncd") 48 | 49 | 50 | class SyncCollectionHandler(anki.sync.Syncer): 51 | operations = ['meta', 'applyChanges', 'start', 'applyGraves', 'chunk', 'applyChunk', 'sanityCheck2', 'finish'] 52 | 53 | def __init__(self, col): 54 | # So that 'server' (the 3rd argument) can't get set 55 | anki.sync.Syncer.__init__(self, col) 56 | 57 | @staticmethod 58 | def _old_client(cv): 59 | if not cv: 60 | return False 61 | 62 | note = {"alpha": 0, "beta": 0, "rc": 0} 63 | client, version, platform = cv.split(',') 64 | 65 | for name in note.keys(): 66 | if name in version: 67 | vs = version.split(name) 68 | version = vs[0] 69 | note[name] = int(vs[-1]) 70 | 71 | # convert the version string, ignoring non-numeric suffixes like in beta versions of Anki 72 | version_nosuffix = re.sub(r'[^0-9.].*$', '', version) 73 | version_int = [int(x) for x in version_nosuffix.split('.')] 74 | 75 | if client == 'ankidesktop': 76 | return version_int < [2, 0, 27] 77 | elif client == 'ankidroid': 78 | if version_int == [2, 3]: 79 | if note["alpha"]: 80 | return note["alpha"] < 4 81 | else: 82 | return version_int < [2, 2, 3] 83 | else: # unknown client, assume current version 84 | return False 85 | 86 | def meta(self, v=None, cv=None): 87 | if self._old_client(cv): 88 | return Response(status=501) # client needs upgrade 89 | if v > SYNC_VER: 90 | return {"cont": False, "msg": "Your client is using unsupported sync protocol ({}, supported version: {})".format(v, SYNC_VER)} 91 | if v < 9 and self.col.schedVer() >= 2: 92 | return {"cont": False, "msg": "Your client doesn't support the v{} scheduler.".format(self.col.schedVer())} 93 | 94 | # Make sure the media database is open! 95 | if self.col.media.db is None: 96 | self.col.media.connect() 97 | 98 | return { 99 | 'scm': self.col.scm, 100 | 'ts': anki.utils.intTime(), 101 | 'mod': self.col.mod, 102 | 'usn': self.col._usn, 103 | 'musn': self.col.media.lastUsn(), 104 | 'msg': '', 105 | 'cont': True, 106 | } 107 | 108 | def usnLim(self): 109 | return "usn >= %d" % self.minUsn 110 | 111 | # ankidesktop >=2.1rc2 sends graves in applyGraves, but still expects 112 | # server-side deletions to be returned by start 113 | def start(self, minUsn, lnewer, graves={"cards": [], "notes": [], "decks": []}, offset=None): 114 | if offset is not None: 115 | raise NotImplementedError('You are using the experimental V2 scheduler, which is not supported by the server.') 116 | self.maxUsn = self.col._usn 117 | self.minUsn = minUsn 118 | self.lnewer = not lnewer 119 | lgraves = self.removed() 120 | self.remove(graves) 121 | return lgraves 122 | 123 | def applyGraves(self, chunk): 124 | self.remove(chunk) 125 | 126 | def applyChanges(self, changes): 127 | self.rchg = changes 128 | lchg = self.changes() 129 | # merge our side before returning 130 | self.mergeChanges(lchg, self.rchg) 131 | return lchg 132 | 133 | def sanityCheck2(self, client): 134 | server = self.sanityCheck() 135 | if client != server: 136 | return dict(status="bad", c=client, s=server) 137 | return dict(status="ok") 138 | 139 | def finish(self, mod=None): 140 | return anki.sync.Syncer.finish(self, anki.utils.intTime(1000)) 141 | 142 | # This function had to be put here in its entirety because Syncer.removed() 143 | # doesn't use self.usnLim() (which we override in this class) in queries. 144 | # "usn=-1" has been replaced with "usn >= ?", self.minUsn by hand. 145 | def removed(self): 146 | cards = [] 147 | notes = [] 148 | decks = [] 149 | 150 | curs = self.col.db.execute( 151 | "select oid, type from graves where usn >= ?", self.minUsn) 152 | 153 | for oid, type in curs: 154 | if type == REM_CARD: 155 | cards.append(oid) 156 | elif type == REM_NOTE: 157 | notes.append(oid) 158 | else: 159 | decks.append(oid) 160 | 161 | return dict(cards=cards, notes=notes, decks=decks) 162 | 163 | def getModels(self): 164 | return [m for m in self.col.models.all() if m['usn'] >= self.minUsn] 165 | 166 | def getDecks(self): 167 | return [ 168 | [g for g in self.col.decks.all() if g['usn'] >= self.minUsn], 169 | [g for g in self.col.decks.allConf() if g['usn'] >= self.minUsn] 170 | ] 171 | 172 | def getTags(self): 173 | return [t for t, usn in self.col.tags.allItems() 174 | if usn >= self.minUsn] 175 | 176 | class SyncMediaHandler: 177 | operations = ['begin', 'mediaChanges', 'mediaSanity', 'uploadChanges', 'downloadFiles'] 178 | 179 | def __init__(self, col): 180 | self.col = col 181 | 182 | def begin(self, skey): 183 | return { 184 | 'data': { 185 | 'sk': skey, 186 | 'usn': self.col.media.lastUsn(), 187 | }, 188 | 'err': '', 189 | } 190 | 191 | def uploadChanges(self, data): 192 | """ 193 | The zip file contains files the client hasn't synced with the server 194 | yet ('dirty'), and info on files it has deleted from its own media dir. 195 | """ 196 | 197 | with zipfile.ZipFile(io.BytesIO(data), "r") as z: 198 | self._check_zip_data(z) 199 | processed_count = self._adopt_media_changes_from_zip(z) 200 | 201 | return { 202 | 'data': [processed_count, self.col.media.lastUsn()], 203 | 'err': '', 204 | } 205 | 206 | @staticmethod 207 | def _check_zip_data(zip_file): 208 | max_zip_size = 100*1024*1024 209 | max_meta_file_size = 100000 210 | 211 | meta_file_size = zip_file.getinfo("_meta").file_size 212 | sum_file_sizes = sum(info.file_size for info in zip_file.infolist()) 213 | 214 | if meta_file_size > max_meta_file_size: 215 | raise ValueError("Zip file's metadata file is larger than %s " 216 | "Bytes." % max_meta_file_size) 217 | elif sum_file_sizes > max_zip_size: 218 | raise ValueError("Zip file contents are larger than %s Bytes." % 219 | max_zip_size) 220 | 221 | def _adopt_media_changes_from_zip(self, zip_file): 222 | """ 223 | Adds and removes files to/from the database and media directory 224 | according to the data in zip file zipData. 225 | """ 226 | 227 | # Get meta info first. 228 | meta = json.loads(zip_file.read("_meta").decode()) 229 | 230 | # Remove media files that were removed on the client. 231 | media_to_remove = [] 232 | for normname, ordinal in meta: 233 | if ordinal == '': 234 | media_to_remove.append(self._normalize_filename(normname)) 235 | 236 | # Add media files that were added on the client. 237 | media_to_add = [] 238 | usn = self.col.media.lastUsn() 239 | oldUsn = usn 240 | for i in zip_file.infolist(): 241 | if i.filename == "_meta": # Ignore previously retrieved metadata. 242 | continue 243 | 244 | file_data = zip_file.read(i) 245 | csum = anki.utils.checksum(file_data) 246 | filename = self._normalize_filename(meta[int(i.filename)][0]) 247 | file_path = os.path.join(self.col.media.dir(), filename) 248 | 249 | # Save file to media directory. 250 | with open(file_path, 'wb') as f: 251 | f.write(file_data) 252 | 253 | usn += 1 254 | media_to_add.append((filename, usn, csum)) 255 | 256 | # We count all files we are to remove, even if we don't have them in 257 | # our media directory and our db doesn't know about them. 258 | processed_count = len(media_to_remove) + len(media_to_add) 259 | 260 | assert len(meta) == processed_count # sanity check 261 | 262 | if media_to_remove: 263 | self._remove_media_files(media_to_remove) 264 | 265 | if media_to_add: 266 | self.col.media.db.executemany( 267 | "INSERT OR REPLACE INTO media VALUES (?,?,?)", media_to_add) 268 | self.col.media.db.commit() 269 | 270 | assert self.col.media.lastUsn() == oldUsn + processed_count # TODO: move to some unit test 271 | return processed_count 272 | 273 | @staticmethod 274 | def _normalize_filename(filename): 275 | """ 276 | Performs unicode normalization for file names. Logic taken from Anki's 277 | MediaManager.addFilesFromZip(). 278 | """ 279 | 280 | # Normalize name for platform. 281 | if anki.utils.isMac: # global 282 | filename = unicodedata.normalize("NFD", filename) 283 | else: 284 | filename = unicodedata.normalize("NFC", filename) 285 | 286 | return filename 287 | 288 | def _remove_media_files(self, filenames): 289 | """ 290 | Marks all files in list filenames as deleted and removes them from the 291 | media directory. 292 | """ 293 | logger.debug('Removing %d files from media dir.' % len(filenames)) 294 | for filename in filenames: 295 | try: 296 | self.col.media.syncDelete(filename) 297 | self.col.media.db.commit() 298 | except OSError as err: 299 | logger.error("Error when removing file '%s' from media dir: " 300 | "%s" % (filename, str(err))) 301 | 302 | def downloadFiles(self, files): 303 | flist = {} 304 | cnt = 0 305 | sz = 0 306 | f = io.BytesIO() 307 | 308 | with zipfile.ZipFile(f, "w", compression=zipfile.ZIP_DEFLATED) as z: 309 | for fname in files: 310 | z.write(os.path.join(self.col.media.dir(), fname), str(cnt)) 311 | flist[str(cnt)] = fname 312 | sz += os.path.getsize(os.path.join(self.col.media.dir(), fname)) 313 | if sz > SYNC_ZIP_SIZE or cnt > SYNC_ZIP_COUNT: 314 | break 315 | cnt += 1 316 | 317 | z.writestr("_meta", json.dumps(flist)) 318 | 319 | return f.getvalue() 320 | 321 | def mediaChanges(self, lastUsn): 322 | result = [] 323 | server_lastUsn = self.col.media.lastUsn() 324 | fname = csum = None 325 | 326 | if lastUsn < server_lastUsn or lastUsn == 0: 327 | for fname,usn,csum, in self.col.media.db.execute("select fname,usn,csum from media order by usn desc limit ?", server_lastUsn - lastUsn): 328 | result.append([fname, usn, csum]) 329 | 330 | # anki assumes server_lastUsn == result[-1][1] 331 | # ref: anki/sync.py:720 (commit cca3fcb2418880d0430a5c5c2e6b81ba260065b7) 332 | result.reverse() 333 | 334 | return {'data': result, 'err': ''} 335 | 336 | def mediaSanity(self, local=None): 337 | if self.col.media.mediaCount() == local: 338 | result = "OK" 339 | else: 340 | result = "FAILED" 341 | 342 | return {'data': result, 'err': ''} 343 | 344 | class SyncUserSession: 345 | def __init__(self, name, path, collection_manager, setup_new_collection=None): 346 | self.skey = self._generate_session_key() 347 | self.name = name 348 | self.path = path 349 | self.collection_manager = collection_manager 350 | self.setup_new_collection = setup_new_collection 351 | self.version = None 352 | self.client_version = None 353 | self.created = time.time() 354 | self.collection_handler = None 355 | self.media_handler = None 356 | 357 | # make sure the user path exists 358 | if not os.path.exists(path): 359 | os.mkdir(path) 360 | 361 | def _generate_session_key(self): 362 | return anki.utils.checksum(str(random.random()))[:8] 363 | 364 | def get_collection_path(self): 365 | return os.path.realpath(os.path.join(self.path, 'collection.anki2')) 366 | 367 | def get_thread(self): 368 | return self.collection_manager.get_collection(self.get_collection_path(), self.setup_new_collection) 369 | 370 | def get_handler_for_operation(self, operation, col): 371 | if operation in SyncCollectionHandler.operations: 372 | attr, handler_class = 'collection_handler', SyncCollectionHandler 373 | elif operation in SyncMediaHandler.operations: 374 | attr, handler_class = 'media_handler', SyncMediaHandler 375 | else: 376 | raise Exception("no handler for {}".format(operation)) 377 | 378 | if getattr(self, attr) is None: 379 | setattr(self, attr, handler_class(col)) 380 | handler = getattr(self, attr) 381 | # The col object may actually be new now! This happens when we close a collection 382 | # for inactivity and then later re-open it (creating a new Collection object). 383 | handler.col = col 384 | return handler 385 | 386 | class SyncApp: 387 | valid_urls = SyncCollectionHandler.operations + SyncMediaHandler.operations + ['hostKey', 'upload', 'download'] 388 | 389 | def __init__(self, config): 390 | from ankisyncd.thread import get_collection_manager 391 | 392 | self.data_root = os.path.abspath(config['data_root']) 393 | self.base_url = config['base_url'] 394 | self.base_media_url = config['base_media_url'] 395 | self.setup_new_collection = None 396 | 397 | self.prehooks = {} 398 | self.posthooks = {} 399 | 400 | self.user_manager = get_user_manager(config) 401 | self.session_manager = get_session_manager(config) 402 | self.full_sync_manager = get_full_sync_manager(config) 403 | self.collection_manager = get_collection_manager(config) 404 | 405 | # make sure the base_url has a trailing slash 406 | if not self.base_url.endswith('/'): 407 | self.base_url += '/' 408 | if not self.base_media_url.endswith('/'): 409 | self.base_media_url += '/' 410 | 411 | # backwards compat 412 | @property 413 | def hook_pre_sync(self): 414 | return self.prehooks.get("start") 415 | 416 | @hook_pre_sync.setter 417 | def hook_pre_sync(self, value): 418 | self.prehooks['start'] = value 419 | 420 | @property 421 | def hook_post_sync(self): 422 | return self.posthooks.get("finish") 423 | 424 | @hook_post_sync.setter 425 | def hook_post_sync(self, value): 426 | self.posthooks['finish'] = value 427 | 428 | @property 429 | def hook_upload(self): 430 | return self.prehooks.get("upload") 431 | 432 | @hook_upload.setter 433 | def hook_upload(self, value): 434 | self.prehooks['upload'] = value 435 | 436 | @property 437 | def hook_download(self): 438 | return self.posthooks.get("download") 439 | 440 | @hook_download.setter 441 | def hook_download(self, value): 442 | self.posthooks['download'] = value 443 | 444 | def generateHostKey(self, username): 445 | """Generates a new host key to be used by the given username to identify their session. 446 | This values is random.""" 447 | 448 | import hashlib, time, random, string 449 | chars = string.ascii_letters + string.digits 450 | val = ':'.join([username, str(int(time.time())), ''.join(random.choice(chars) for x in range(8))]).encode() 451 | return hashlib.md5(val).hexdigest() 452 | 453 | def create_session(self, username, user_path): 454 | return SyncUserSession(username, user_path, self.collection_manager, self.setup_new_collection) 455 | 456 | def _decode_data(self, data, compression=0): 457 | if compression: 458 | with gzip.GzipFile(mode="rb", fileobj=io.BytesIO(data)) as gz: 459 | data = gz.read() 460 | 461 | try: 462 | data = json.loads(data.decode()) 463 | except (ValueError, UnicodeDecodeError): 464 | data = {'data': data} 465 | 466 | return data 467 | 468 | def operation_hostKey(self, username, password): 469 | if not self.user_manager.authenticate(username, password): 470 | return 471 | 472 | dirname = self.user_manager.userdir(username) 473 | if dirname is None: 474 | return 475 | 476 | hkey = self.generateHostKey(username) 477 | user_path = os.path.join(self.data_root, dirname) 478 | session = self.create_session(username, user_path) 479 | self.session_manager.save(hkey, session) 480 | 481 | return {'key': hkey} 482 | 483 | def operation_upload(self, col, data, session): 484 | # Verify integrity of the received database file before replacing our 485 | # existing db. 486 | 487 | return self.full_sync_manager.upload(col, data, session) 488 | 489 | def operation_download(self, col, session): 490 | # returns user data (not media) as a sqlite3 database for replacing their 491 | # local copy in Anki 492 | return self.full_sync_manager.download(col, session) 493 | 494 | @wsgify 495 | def __call__(self, req): 496 | # Get and verify the session 497 | try: 498 | hkey = req.POST['k'] 499 | except KeyError: 500 | hkey = None 501 | 502 | session = self.session_manager.load(hkey, self.create_session) 503 | 504 | if session is None: 505 | try: 506 | skey = req.POST['sk'] 507 | session = self.session_manager.load_from_skey(skey, self.create_session) 508 | except KeyError: 509 | skey = None 510 | 511 | try: 512 | compression = int(req.POST['c']) 513 | except KeyError: 514 | compression = 0 515 | 516 | try: 517 | data = req.POST['data'].file.read() 518 | data = self._decode_data(data, compression) 519 | except KeyError: 520 | data = {} 521 | 522 | if req.path.startswith(self.base_url): 523 | url = req.path[len(self.base_url):] 524 | if url not in self.valid_urls: 525 | raise HTTPNotFound() 526 | 527 | if url == 'hostKey': 528 | result = self.operation_hostKey(data.get("u"), data.get("p")) 529 | if result: 530 | return json.dumps(result) 531 | else: 532 | # TODO: do I have to pass 'null' for the client to receive None? 533 | raise HTTPForbidden('null') 534 | 535 | if session is None: 536 | raise HTTPForbidden() 537 | 538 | if url in SyncCollectionHandler.operations + SyncMediaHandler.operations: 539 | # 'meta' passes the SYNC_VER but it isn't used in the handler 540 | if url == 'meta': 541 | if session.skey == None and 's' in req.POST: 542 | session.skey = req.POST['s'] 543 | if 'v' in data: 544 | session.version = data['v'] 545 | if 'cv' in data: 546 | session.client_version = data['cv'] 547 | 548 | self.session_manager.save(hkey, session) 549 | session = self.session_manager.load(hkey, self.create_session) 550 | 551 | thread = session.get_thread() 552 | 553 | if url in self.prehooks: 554 | thread.execute(self.prehooks[url], [session]) 555 | 556 | result = self._execute_handler_method_in_thread(url, data, session) 557 | 558 | # If it's a complex data type, we convert it to JSON 559 | if type(result) not in (str, bytes, Response): 560 | result = json.dumps(result) 561 | 562 | if url in self.posthooks: 563 | thread.execute(self.posthooks[url], [session]) 564 | 565 | return result 566 | 567 | elif url == 'upload': 568 | thread = session.get_thread() 569 | if url in self.prehooks: 570 | thread.execute(self.prehooks[url], [session]) 571 | result = thread.execute(self.operation_upload, [data['data'], session]) 572 | if url in self.posthooks: 573 | thread.execute(self.posthooks[url], [session]) 574 | return result 575 | 576 | elif url == 'download': 577 | thread = session.get_thread() 578 | if url in self.prehooks: 579 | thread.execute(self.prehooks[url], [session]) 580 | result = thread.execute(self.operation_download, [session]) 581 | if url in self.posthooks: 582 | thread.execute(self.posthooks[url], [session]) 583 | return result 584 | 585 | # This was one of our operations but it didn't get handled... Oops! 586 | raise HTTPInternalServerError() 587 | 588 | # media sync 589 | elif req.path.startswith(self.base_media_url): 590 | if session is None: 591 | raise HTTPForbidden() 592 | 593 | url = req.path[len(self.base_media_url):] 594 | 595 | if url not in self.valid_urls: 596 | raise HTTPNotFound() 597 | 598 | if url == "begin": 599 | data['skey'] = session.skey 600 | 601 | result = self._execute_handler_method_in_thread(url, data, session) 602 | 603 | # If it's a complex data type, we convert it to JSON 604 | if type(result) not in (str, bytes): 605 | result = json.dumps(result) 606 | 607 | return result 608 | 609 | return "Anki Sync Server" 610 | 611 | @staticmethod 612 | def _execute_handler_method_in_thread(method_name, keyword_args, session): 613 | """ 614 | Gets and runs the handler method specified by method_name inside the 615 | thread for session. The handler method will access the collection as 616 | self.col. 617 | """ 618 | 619 | def run_func(col, **keyword_args): 620 | # Retrieve the correct handler method. 621 | handler = session.get_handler_for_operation(method_name, col) 622 | handler_method = getattr(handler, method_name) 623 | 624 | res = handler_method(**keyword_args) 625 | 626 | col.save() 627 | return res 628 | 629 | run_func.__name__ = method_name # More useful debugging messages. 630 | 631 | # Send the closure to the thread for execution. 632 | thread = session.get_thread() 633 | result = thread.execute(run_func, kw=keyword_args) 634 | 635 | return result 636 | 637 | 638 | def make_app(global_conf, **local_conf): 639 | return SyncApp(**local_conf) 640 | 641 | def main(): 642 | logging.basicConfig(level=logging.INFO, format="[%(asctime)s]:%(levelname)s:%(name)s:%(message)s") 643 | import ankisyncd 644 | logger.info("ankisyncd {} ({})".format(ankisyncd._get_version(), ankisyncd._homepage)) 645 | from wsgiref.simple_server import make_server, WSGIRequestHandler 646 | from ankisyncd.thread import shutdown 647 | import ankisyncd.config 648 | 649 | class RequestHandler(WSGIRequestHandler): 650 | logger = logging.getLogger("ankisyncd.http") 651 | 652 | def log_error(self, format, *args): 653 | self.logger.error("%s %s", self.address_string(), format%args) 654 | 655 | def log_message(self, format, *args): 656 | self.logger.info("%s %s", self.address_string(), format%args) 657 | 658 | if len(sys.argv) > 1: 659 | # backwards compat 660 | config = ankisyncd.config.load(sys.argv[1]) 661 | else: 662 | config = ankisyncd.config.load() 663 | 664 | ankiserver = SyncApp(config) 665 | httpd = make_server(config['host'], int(config['port']), ankiserver, handler_class=RequestHandler) 666 | 667 | try: 668 | logger.info("Serving HTTP on {} port {}...".format(*httpd.server_address)) 669 | httpd.serve_forever() 670 | except KeyboardInterrupt: 671 | logger.info("Exiting...") 672 | finally: 673 | shutdown() 674 | 675 | if __name__ == '__main__': main() 676 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | 2 | GNU AFFERO GENERAL PUBLIC LICENSE 3 | Version 3, 19 November 2007 4 | 5 | Copyright (C) 2007 Free Software Foundation, Inc. 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The GNU Affero General Public License is a free, copyleft license for 12 | software and other kinds of works, specifically designed to ensure 13 | cooperation with the community in the case of network server software. 14 | 15 | The licenses for most software and other practical works are designed 16 | to take away your freedom to share and change the works. By contrast, 17 | our General Public Licenses are intended to guarantee your freedom to 18 | share and change all versions of a program--to make sure it remains free 19 | software for all its users. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | them if you wish), that you receive source code or can get it if you 25 | want it, that you can change the software or use pieces of it in new 26 | free programs, and that you know you can do these things. 27 | 28 | Developers that use our General Public Licenses protect your rights 29 | with two steps: (1) assert copyright on the software, and (2) offer 30 | you this License which gives you legal permission to copy, distribute 31 | and/or modify the software. 32 | 33 | A secondary benefit of defending all users' freedom is that 34 | improvements made in alternate versions of the program, if they 35 | receive widespread use, become available for other developers to 36 | incorporate. Many developers of free software are heartened and 37 | encouraged by the resulting cooperation. However, in the case of 38 | software used on network servers, this result may fail to come about. 39 | The GNU General Public License permits making a modified version and 40 | letting the public access it on a server without ever releasing its 41 | source code to the public. 42 | 43 | The GNU Affero General Public License is designed specifically to 44 | ensure that, in such cases, the modified source code becomes available 45 | to the community. It requires the operator of a network server to 46 | provide the source code of the modified version running there to the 47 | users of that server. Therefore, public use of a modified version, on 48 | a publicly accessible server, gives the public access to the source 49 | code of the modified version. 50 | 51 | An older license, called the Affero General Public License and 52 | published by Affero, was designed to accomplish similar goals. This is 53 | a different license, not a version of the Affero GPL, but Affero has 54 | released a new version of the Affero GPL which permits relicensing under 55 | this license. 56 | 57 | The precise terms and conditions for copying, distribution and 58 | modification follow. 59 | 60 | TERMS AND CONDITIONS 61 | 62 | 0. Definitions. 63 | 64 | "This License" refers to version 3 of the GNU Affero General Public License. 65 | 66 | "Copyright" also means copyright-like laws that apply to other kinds of 67 | works, such as semiconductor masks. 68 | 69 | "The Program" refers to any copyrightable work licensed under this 70 | License. Each licensee is addressed as "you". "Licensees" and 71 | "recipients" may be individuals or organizations. 72 | 73 | To "modify" a work means to copy from or adapt all or part of the work 74 | in a fashion requiring copyright permission, other than the making of an 75 | exact copy. The resulting work is called a "modified version" of the 76 | earlier work or a work "based on" the earlier work. 77 | 78 | A "covered work" means either the unmodified Program or a work based 79 | on the Program. 80 | 81 | To "propagate" a work means to do anything with it that, without 82 | permission, would make you directly or secondarily liable for 83 | infringement under applicable copyright law, except executing it on a 84 | computer or modifying a private copy. Propagation includes copying, 85 | distribution (with or without modification), making available to the 86 | public, and in some countries other activities as well. 87 | 88 | To "convey" a work means any kind of propagation that enables other 89 | parties to make or receive copies. Mere interaction with a user through 90 | a computer network, with no transfer of a copy, is not conveying. 91 | 92 | An interactive user interface displays "Appropriate Legal Notices" 93 | to the extent that it includes a convenient and prominently visible 94 | feature that (1) displays an appropriate copyright notice, and (2) 95 | tells the user that there is no warranty for the work (except to the 96 | extent that warranties are provided), that licensees may convey the 97 | work under this License, and how to view a copy of this License. If 98 | the interface presents a list of user commands or options, such as a 99 | menu, a prominent item in the list meets this criterion. 100 | 101 | 1. Source Code. 102 | 103 | The "source code" for a work means the preferred form of the work 104 | for making modifications to it. "Object code" means any non-source 105 | form of a work. 106 | 107 | A "Standard Interface" means an interface that either is an official 108 | standard defined by a recognized standards body, or, in the case of 109 | interfaces specified for a particular programming language, one that 110 | is widely used among developers working in that language. 111 | 112 | The "System Libraries" of an executable work include anything, other 113 | than the work as a whole, that (a) is included in the normal form of 114 | packaging a Major Component, but which is not part of that Major 115 | Component, and (b) serves only to enable use of the work with that 116 | Major Component, or to implement a Standard Interface for which an 117 | implementation is available to the public in source code form. A 118 | "Major Component", in this context, means a major essential component 119 | (kernel, window system, and so on) of the specific operating system 120 | (if any) on which the executable work runs, or a compiler used to 121 | produce the work, or an object code interpreter used to run it. 122 | 123 | The "Corresponding Source" for a work in object code form means all 124 | the source code needed to generate, install, and (for an executable 125 | work) run the object code and to modify the work, including scripts to 126 | control those activities. However, it does not include the work's 127 | System Libraries, or general-purpose tools or generally available free 128 | programs which are used unmodified in performing those activities but 129 | which are not part of the work. For example, Corresponding Source 130 | includes interface definition files associated with source files for 131 | the work, and the source code for shared libraries and dynamically 132 | linked subprograms that the work is specifically designed to require, 133 | such as by intimate data communication or control flow between those 134 | subprograms and other parts of the work. 135 | 136 | The Corresponding Source need not include anything that users 137 | can regenerate automatically from other parts of the Corresponding 138 | Source. 139 | 140 | The Corresponding Source for a work in source code form is that 141 | same work. 142 | 143 | 2. Basic Permissions. 144 | 145 | All rights granted under this License are granted for the term of 146 | copyright on the Program, and are irrevocable provided the stated 147 | conditions are met. This License explicitly affirms your unlimited 148 | permission to run the unmodified Program. The output from running a 149 | covered work is covered by this License only if the output, given its 150 | content, constitutes a covered work. This License acknowledges your 151 | rights of fair use or other equivalent, as provided by copyright law. 152 | 153 | You may make, run and propagate covered works that you do not 154 | convey, without conditions so long as your license otherwise remains 155 | in force. You may convey covered works to others for the sole purpose 156 | of having them make modifications exclusively for you, or provide you 157 | with facilities for running those works, provided that you comply with 158 | the terms of this License in conveying all material for which you do 159 | not control copyright. Those thus making or running the covered works 160 | for you must do so exclusively on your behalf, under your direction 161 | and control, on terms that prohibit them from making any copies of 162 | your copyrighted material outside their relationship with you. 163 | 164 | Conveying under any other circumstances is permitted solely under 165 | the conditions stated below. Sublicensing is not allowed; section 10 166 | makes it unnecessary. 167 | 168 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 169 | 170 | No covered work shall be deemed part of an effective technological 171 | measure under any applicable law fulfilling obligations under article 172 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or 173 | similar laws prohibiting or restricting circumvention of such 174 | measures. 175 | 176 | When you convey a covered work, you waive any legal power to forbid 177 | circumvention of technological measures to the extent such circumvention 178 | is effected by exercising rights under this License with respect to 179 | the covered work, and you disclaim any intention to limit operation or 180 | modification of the work as a means of enforcing, against the work's 181 | users, your or third parties' legal rights to forbid circumvention of 182 | technological measures. 183 | 184 | 4. Conveying Verbatim Copies. 185 | 186 | You may convey verbatim copies of the Program's source code as you 187 | receive it, in any medium, provided that you conspicuously and 188 | appropriately publish on each copy an appropriate copyright notice; 189 | keep intact all notices stating that this License and any 190 | non-permissive terms added in accord with section 7 apply to the code; 191 | keep intact all notices of the absence of any warranty; and give all 192 | recipients a copy of this License along with the Program. 193 | 194 | You may charge any price or no price for each copy that you convey, 195 | and you may offer support or warranty protection for a fee. 196 | 197 | 5. Conveying Modified Source Versions. 198 | 199 | You may convey a work based on the Program, or the modifications to 200 | produce it from the Program, in the form of source code under the 201 | terms of section 4, provided that you also meet all of these conditions: 202 | 203 | a) The work must carry prominent notices stating that you modified 204 | it, and giving a relevant date. 205 | 206 | b) The work must carry prominent notices stating that it is 207 | released under this License and any conditions added under section 208 | 7. This requirement modifies the requirement in section 4 to 209 | "keep intact all notices". 210 | 211 | c) You must license the entire work, as a whole, under this 212 | License to anyone who comes into possession of a copy. This 213 | License will therefore apply, along with any applicable section 7 214 | additional terms, to the whole of the work, and all its parts, 215 | regardless of how they are packaged. This License gives no 216 | permission to license the work in any other way, but it does not 217 | invalidate such permission if you have separately received it. 218 | 219 | d) If the work has interactive user interfaces, each must display 220 | Appropriate Legal Notices; however, if the Program has interactive 221 | interfaces that do not display Appropriate Legal Notices, your 222 | work need not make them do so. 223 | 224 | A compilation of a covered work with other separate and independent 225 | works, which are not by their nature extensions of the covered work, 226 | and which are not combined with it such as to form a larger program, 227 | in or on a volume of a storage or distribution medium, is called an 228 | "aggregate" if the compilation and its resulting copyright are not 229 | used to limit the access or legal rights of the compilation's users 230 | beyond what the individual works permit. Inclusion of a covered work 231 | in an aggregate does not cause this License to apply to the other 232 | parts of the aggregate. 233 | 234 | 6. Conveying Non-Source Forms. 235 | 236 | You may convey a covered work in object code form under the terms 237 | of sections 4 and 5, provided that you also convey the 238 | machine-readable Corresponding Source under the terms of this License, 239 | in one of these ways: 240 | 241 | a) Convey the object code in, or embodied in, a physical product 242 | (including a physical distribution medium), accompanied by the 243 | Corresponding Source fixed on a durable physical medium 244 | customarily used for software interchange. 245 | 246 | b) Convey the object code in, or embodied in, a physical product 247 | (including a physical distribution medium), accompanied by a 248 | written offer, valid for at least three years and valid for as 249 | long as you offer spare parts or customer support for that product 250 | model, to give anyone who possesses the object code either (1) a 251 | copy of the Corresponding Source for all the software in the 252 | product that is covered by this License, on a durable physical 253 | medium customarily used for software interchange, for a price no 254 | more than your reasonable cost of physically performing this 255 | conveying of source, or (2) access to copy the 256 | Corresponding Source from a network server at no charge. 257 | 258 | c) Convey individual copies of the object code with a copy of the 259 | written offer to provide the Corresponding Source. This 260 | alternative is allowed only occasionally and noncommercially, and 261 | only if you received the object code with such an offer, in accord 262 | with subsection 6b. 263 | 264 | d) Convey the object code by offering access from a designated 265 | place (gratis or for a charge), and offer equivalent access to the 266 | Corresponding Source in the same way through the same place at no 267 | further charge. You need not require recipients to copy the 268 | Corresponding Source along with the object code. If the place to 269 | copy the object code is a network server, the Corresponding Source 270 | may be on a different server (operated by you or a third party) 271 | that supports equivalent copying facilities, provided you maintain 272 | clear directions next to the object code saying where to find the 273 | Corresponding Source. Regardless of what server hosts the 274 | Corresponding Source, you remain obligated to ensure that it is 275 | available for as long as needed to satisfy these requirements. 276 | 277 | e) Convey the object code using peer-to-peer transmission, provided 278 | you inform other peers where the object code and Corresponding 279 | Source of the work are being offered to the general public at no 280 | charge under subsection 6d. 281 | 282 | A separable portion of the object code, whose source code is excluded 283 | from the Corresponding Source as a System Library, need not be 284 | included in conveying the object code work. 285 | 286 | A "User Product" is either (1) a "consumer product", which means any 287 | tangible personal property which is normally used for personal, family, 288 | or household purposes, or (2) anything designed or sold for incorporation 289 | into a dwelling. In determining whether a product is a consumer product, 290 | doubtful cases shall be resolved in favor of coverage. For a particular 291 | product received by a particular user, "normally used" refers to a 292 | typical or common use of that class of product, regardless of the status 293 | of the particular user or of the way in which the particular user 294 | actually uses, or expects or is expected to use, the product. A product 295 | is a consumer product regardless of whether the product has substantial 296 | commercial, industrial or non-consumer uses, unless such uses represent 297 | the only significant mode of use of the product. 298 | 299 | "Installation Information" for a User Product means any methods, 300 | procedures, authorization keys, or other information required to install 301 | and execute modified versions of a covered work in that User Product from 302 | a modified version of its Corresponding Source. The information must 303 | suffice to ensure that the continued functioning of the modified object 304 | code is in no case prevented or interfered with solely because 305 | modification has been made. 306 | 307 | If you convey an object code work under this section in, or with, or 308 | specifically for use in, a User Product, and the conveying occurs as 309 | part of a transaction in which the right of possession and use of the 310 | User Product is transferred to the recipient in perpetuity or for a 311 | fixed term (regardless of how the transaction is characterized), the 312 | Corresponding Source conveyed under this section must be accompanied 313 | by the Installation Information. But this requirement does not apply 314 | if neither you nor any third party retains the ability to install 315 | modified object code on the User Product (for example, the work has 316 | been installed in ROM). 317 | 318 | The requirement to provide Installation Information does not include a 319 | requirement to continue to provide support service, warranty, or updates 320 | for a work that has been modified or installed by the recipient, or for 321 | the User Product in which it has been modified or installed. Access to a 322 | network may be denied when the modification itself materially and 323 | adversely affects the operation of the network or violates the rules and 324 | protocols for communication across the network. 325 | 326 | Corresponding Source conveyed, and Installation Information provided, 327 | in accord with this section must be in a format that is publicly 328 | documented (and with an implementation available to the public in 329 | source code form), and must require no special password or key for 330 | unpacking, reading or copying. 331 | 332 | 7. Additional Terms. 333 | 334 | "Additional permissions" are terms that supplement the terms of this 335 | License by making exceptions from one or more of its conditions. 336 | Additional permissions that are applicable to the entire Program shall 337 | be treated as though they were included in this License, to the extent 338 | that they are valid under applicable law. If additional permissions 339 | apply only to part of the Program, that part may be used separately 340 | under those permissions, but the entire Program remains governed by 341 | this License without regard to the additional permissions. 342 | 343 | When you convey a copy of a covered work, you may at your option 344 | remove any additional permissions from that copy, or from any part of 345 | it. (Additional permissions may be written to require their own 346 | removal in certain cases when you modify the work.) You may place 347 | additional permissions on material, added by you to a covered work, 348 | for which you have or can give appropriate copyright permission. 349 | 350 | Notwithstanding any other provision of this License, for material you 351 | add to a covered work, you may (if authorized by the copyright holders of 352 | that material) supplement the terms of this License with terms: 353 | 354 | a) Disclaiming warranty or limiting liability differently from the 355 | terms of sections 15 and 16 of this License; or 356 | 357 | b) Requiring preservation of specified reasonable legal notices or 358 | author attributions in that material or in the Appropriate Legal 359 | Notices displayed by works containing it; or 360 | 361 | c) Prohibiting misrepresentation of the origin of that material, or 362 | requiring that modified versions of such material be marked in 363 | reasonable ways as different from the original version; or 364 | 365 | d) Limiting the use for publicity purposes of names of licensors or 366 | authors of the material; or 367 | 368 | e) Declining to grant rights under trademark law for use of some 369 | trade names, trademarks, or service marks; or 370 | 371 | f) Requiring indemnification of licensors and authors of that 372 | material by anyone who conveys the material (or modified versions of 373 | it) with contractual assumptions of liability to the recipient, for 374 | any liability that these contractual assumptions directly impose on 375 | those licensors and authors. 376 | 377 | All other non-permissive additional terms are considered "further 378 | restrictions" within the meaning of section 10. If the Program as you 379 | received it, or any part of it, contains a notice stating that it is 380 | governed by this License along with a term that is a further 381 | restriction, you may remove that term. If a license document contains 382 | a further restriction but permits relicensing or conveying under this 383 | License, you may add to a covered work material governed by the terms 384 | of that license document, provided that the further restriction does 385 | not survive such relicensing or conveying. 386 | 387 | If you add terms to a covered work in accord with this section, you 388 | must place, in the relevant source files, a statement of the 389 | additional terms that apply to those files, or a notice indicating 390 | where to find the applicable terms. 391 | 392 | Additional terms, permissive or non-permissive, may be stated in the 393 | form of a separately written license, or stated as exceptions; 394 | the above requirements apply either way. 395 | 396 | 8. Termination. 397 | 398 | You may not propagate or modify a covered work except as expressly 399 | provided under this License. Any attempt otherwise to propagate or 400 | modify it is void, and will automatically terminate your rights under 401 | this License (including any patent licenses granted under the third 402 | paragraph of section 11). 403 | 404 | However, if you cease all violation of this License, then your 405 | license from a particular copyright holder is reinstated (a) 406 | provisionally, unless and until the copyright holder explicitly and 407 | finally terminates your license, and (b) permanently, if the copyright 408 | holder fails to notify you of the violation by some reasonable means 409 | prior to 60 days after the cessation. 410 | 411 | Moreover, your license from a particular copyright holder is 412 | reinstated permanently if the copyright holder notifies you of the 413 | violation by some reasonable means, this is the first time you have 414 | received notice of violation of this License (for any work) from that 415 | copyright holder, and you cure the violation prior to 30 days after 416 | your receipt of the notice. 417 | 418 | Termination of your rights under this section does not terminate the 419 | licenses of parties who have received copies or rights from you under 420 | this License. If your rights have been terminated and not permanently 421 | reinstated, you do not qualify to receive new licenses for the same 422 | material under section 10. 423 | 424 | 9. Acceptance Not Required for Having Copies. 425 | 426 | You are not required to accept this License in order to receive or 427 | run a copy of the Program. Ancillary propagation of a covered work 428 | occurring solely as a consequence of using peer-to-peer transmission 429 | to receive a copy likewise does not require acceptance. However, 430 | nothing other than this License grants you permission to propagate or 431 | modify any covered work. These actions infringe copyright if you do 432 | not accept this License. Therefore, by modifying or propagating a 433 | covered work, you indicate your acceptance of this License to do so. 434 | 435 | 10. Automatic Licensing of Downstream Recipients. 436 | 437 | Each time you convey a covered work, the recipient automatically 438 | receives a license from the original licensors, to run, modify and 439 | propagate that work, subject to this License. You are not responsible 440 | for enforcing compliance by third parties with this License. 441 | 442 | An "entity transaction" is a transaction transferring control of an 443 | organization, or substantially all assets of one, or subdividing an 444 | organization, or merging organizations. If propagation of a covered 445 | work results from an entity transaction, each party to that 446 | transaction who receives a copy of the work also receives whatever 447 | licenses to the work the party's predecessor in interest had or could 448 | give under the previous paragraph, plus a right to possession of the 449 | Corresponding Source of the work from the predecessor in interest, if 450 | the predecessor has it or can get it with reasonable efforts. 451 | 452 | You may not impose any further restrictions on the exercise of the 453 | rights granted or affirmed under this License. For example, you may 454 | not impose a license fee, royalty, or other charge for exercise of 455 | rights granted under this License, and you may not initiate litigation 456 | (including a cross-claim or counterclaim in a lawsuit) alleging that 457 | any patent claim is infringed by making, using, selling, offering for 458 | sale, or importing the Program or any portion of it. 459 | 460 | 11. Patents. 461 | 462 | A "contributor" is a copyright holder who authorizes use under this 463 | License of the Program or a work on which the Program is based. The 464 | work thus licensed is called the contributor's "contributor version". 465 | 466 | A contributor's "essential patent claims" are all patent claims 467 | owned or controlled by the contributor, whether already acquired or 468 | hereafter acquired, that would be infringed by some manner, permitted 469 | by this License, of making, using, or selling its contributor version, 470 | but do not include claims that would be infringed only as a 471 | consequence of further modification of the contributor version. For 472 | purposes of this definition, "control" includes the right to grant 473 | patent sublicenses in a manner consistent with the requirements of 474 | this License. 475 | 476 | Each contributor grants you a non-exclusive, worldwide, royalty-free 477 | patent license under the contributor's essential patent claims, to 478 | make, use, sell, offer for sale, import and otherwise run, modify and 479 | propagate the contents of its contributor version. 480 | 481 | In the following three paragraphs, a "patent license" is any express 482 | agreement or commitment, however denominated, not to enforce a patent 483 | (such as an express permission to practice a patent or covenant not to 484 | sue for patent infringement). To "grant" such a patent license to a 485 | party means to make such an agreement or commitment not to enforce a 486 | patent against the party. 487 | 488 | If you convey a covered work, knowingly relying on a patent license, 489 | and the Corresponding Source of the work is not available for anyone 490 | to copy, free of charge and under the terms of this License, through a 491 | publicly available network server or other readily accessible means, 492 | then you must either (1) cause the Corresponding Source to be so 493 | available, or (2) arrange to deprive yourself of the benefit of the 494 | patent license for this particular work, or (3) arrange, in a manner 495 | consistent with the requirements of this License, to extend the patent 496 | license to downstream recipients. "Knowingly relying" means you have 497 | actual knowledge that, but for the patent license, your conveying the 498 | covered work in a country, or your recipient's use of the covered work 499 | in a country, would infringe one or more identifiable patents in that 500 | country that you have reason to believe are valid. 501 | 502 | If, pursuant to or in connection with a single transaction or 503 | arrangement, you convey, or propagate by procuring conveyance of, a 504 | covered work, and grant a patent license to some of the parties 505 | receiving the covered work authorizing them to use, propagate, modify 506 | or convey a specific copy of the covered work, then the patent license 507 | you grant is automatically extended to all recipients of the covered 508 | work and works based on it. 509 | 510 | A patent license is "discriminatory" if it does not include within 511 | the scope of its coverage, prohibits the exercise of, or is 512 | conditioned on the non-exercise of one or more of the rights that are 513 | specifically granted under this License. You may not convey a covered 514 | work if you are a party to an arrangement with a third party that is 515 | in the business of distributing software, under which you make payment 516 | to the third party based on the extent of your activity of conveying 517 | the work, and under which the third party grants, to any of the 518 | parties who would receive the covered work from you, a discriminatory 519 | patent license (a) in connection with copies of the covered work 520 | conveyed by you (or copies made from those copies), or (b) primarily 521 | for and in connection with specific products or compilations that 522 | contain the covered work, unless you entered into that arrangement, 523 | or that patent license was granted, prior to 28 March 2007. 524 | 525 | Nothing in this License shall be construed as excluding or limiting 526 | any implied license or other defenses to infringement that may 527 | otherwise be available to you under applicable patent law. 528 | 529 | 12. No Surrender of Others' Freedom. 530 | 531 | If conditions are imposed on you (whether by court order, agreement or 532 | otherwise) that contradict the conditions of this License, they do not 533 | excuse you from the conditions of this License. If you cannot convey a 534 | covered work so as to satisfy simultaneously your obligations under this 535 | License and any other pertinent obligations, then as a consequence you may 536 | not convey it at all. For example, if you agree to terms that obligate you 537 | to collect a royalty for further conveying from those to whom you convey 538 | the Program, the only way you could satisfy both those terms and this 539 | License would be to refrain entirely from conveying the Program. 540 | 541 | 13. Remote Network Interaction; Use with the GNU General Public License. 542 | 543 | Notwithstanding any other provision of this License, if you modify the 544 | Program, your modified version must prominently offer all users 545 | interacting with it remotely through a computer network (if your version 546 | supports such interaction) an opportunity to receive the Corresponding 547 | Source of your version by providing access to the Corresponding Source 548 | from a network server at no charge, through some standard or customary 549 | means of facilitating copying of software. This Corresponding Source 550 | shall include the Corresponding Source for any work covered by version 3 551 | of the GNU General Public License that is incorporated pursuant to the 552 | following paragraph. 553 | 554 | Notwithstanding any other provision of this License, you have 555 | permission to link or combine any covered work with a work licensed 556 | under version 3 of the GNU General Public License into a single 557 | combined work, and to convey the resulting work. The terms of this 558 | License will continue to apply to the part which is the covered work, 559 | but the work with which it is combined will remain governed by version 560 | 3 of the GNU General Public License. 561 | 562 | 14. Revised Versions of this License. 563 | 564 | The Free Software Foundation may publish revised and/or new versions of 565 | the GNU Affero General Public License from time to time. Such new versions 566 | will be similar in spirit to the present version, but may differ in detail to 567 | address new problems or concerns. 568 | 569 | Each version is given a distinguishing version number. If the 570 | Program specifies that a certain numbered version of the GNU Affero General 571 | Public License "or any later version" applies to it, you have the 572 | option of following the terms and conditions either of that numbered 573 | version or of any later version published by the Free Software 574 | Foundation. If the Program does not specify a version number of the 575 | GNU Affero General Public License, you may choose any version ever published 576 | by the Free Software Foundation. 577 | 578 | If the Program specifies that a proxy can decide which future 579 | versions of the GNU Affero General Public License can be used, that proxy's 580 | public statement of acceptance of a version permanently authorizes you 581 | to choose that version for the Program. 582 | 583 | Later license versions may give you additional or different 584 | permissions. However, no additional obligations are imposed on any 585 | author or copyright holder as a result of your choosing to follow a 586 | later version. 587 | 588 | 15. Disclaimer of Warranty. 589 | 590 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 591 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 592 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY 593 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 594 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 595 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 596 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 597 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 598 | 599 | 16. Limitation of Liability. 600 | 601 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 602 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 603 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY 604 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE 605 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF 606 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD 607 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), 608 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF 609 | SUCH DAMAGES. 610 | 611 | 17. Interpretation of Sections 15 and 16. 612 | 613 | If the disclaimer of warranty and limitation of liability provided 614 | above cannot be given local legal effect according to their terms, 615 | reviewing courts shall apply local law that most closely approximates 616 | an absolute waiver of all civil liability in connection with the 617 | Program, unless a warranty or assumption of liability accompanies a 618 | copy of the Program in return for a fee. 619 | 620 | END OF TERMS AND CONDITIONS 621 | 622 | How to Apply These Terms to Your New Programs 623 | 624 | If you develop a new program, and you want it to be of the greatest 625 | possible use to the public, the best way to achieve this is to make it 626 | free software which everyone can redistribute and change under these terms. 627 | 628 | To do so, attach the following notices to the program. It is safest 629 | to attach them to the start of each source file to most effectively 630 | state the exclusion of warranty; and each file should have at least 631 | the "copyright" line and a pointer to where the full notice is found. 632 | 633 | 634 | Copyright (C) 635 | 636 | This program is free software: you can redistribute it and/or modify 637 | it under the terms of the GNU Affero General Public License as published by 638 | the Free Software Foundation, either version 3 of the License, or 639 | (at your option) any later version. 640 | 641 | This program is distributed in the hope that it will be useful, 642 | but WITHOUT ANY WARRANTY; without even the implied warranty of 643 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 644 | GNU Affero General Public License for more details. 645 | 646 | You should have received a copy of the GNU Affero General Public License 647 | along with this program. If not, see . 648 | 649 | Also add information on how to contact you by electronic and paper mail. 650 | 651 | If your software can interact with users remotely through a computer 652 | network, you should also make sure that it provides a way for users to 653 | get its source. For example, if your program is a web application, its 654 | interface could display a "Source" link that leads users to an archive 655 | of the code. There are many ways you could offer source, and different 656 | solutions will be better for different programs; see section 13 for the 657 | specific requirements. 658 | 659 | You should also get your employer (if you work as a programmer) or school, 660 | if any, to sign a "copyright disclaimer" for the program, if necessary. 661 | For more information on this, and how to apply and follow the GNU AGPL, see 662 | . 663 | --------------------------------------------------------------------------------