'
25 |
26 | return wrap
27 |
28 |
29 | def get_flaskapis():
30 | app = Flask(import_name='zotero-local')
31 | sql_api_map = get_sql_api_map()
32 |
33 | def res_wrap(func):
34 | @wraps(func)
35 | def inner(*args):
36 | return jsonify({
37 | 'code': 200,
38 | 'result': func(),
39 | 'msg': ''
40 | })
41 |
42 | return inner
43 |
44 | api_list = [
45 |
46 | ]
47 |
48 | for name, func in sql_api_map.items():
49 | wrap_func = res_wrap(func)
50 | path = f'/db/{name}'
51 | _ = app.route(path)(wrap_func)
52 |
53 | api_list.append(_create_url_link(name, path))
54 |
55 | prefs_api_map = get_prefs_api_map()
56 | for name, func in prefs_api_map.items():
57 | wrap_func = res_wrap(func)
58 | path = f'/pref/{name}'
59 | _ = app.route(path)(wrap_func)
60 | api_list.append(_create_url_link(name, path))
61 |
62 | app.route('/docs')(_doc_route(api_list))
63 | return app
64 |
--------------------------------------------------------------------------------
/pyzolocal/beans/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/beans/__init__.py
--------------------------------------------------------------------------------
/pyzolocal/beans/enum.py:
--------------------------------------------------------------------------------
1 | from enum import Enum, unique
2 |
3 |
4 | @unique
5 | class fileds(Enum):
6 | title = 1
7 | abstractNote = 2
8 | artworkMedium = 3
9 | medium = 4
10 | artworkSize = 5
11 | date = 6
12 | language = 7
13 | shortTitle = 8
14 | archive = 9
15 | archiveLocation = 10
16 | libraryCatalog = 11
17 | callNumber = 12
18 | url = 13
19 | accessDate = 14
20 | rights = 15
21 | extra = 16
22 | audioRecordingFormat = 17
23 | seriesTitle = 18
24 | volume = 19
25 | numberOfVolumes = 20
26 | place = 21
27 | label = 22
28 | publisher = 23
29 | runningTime = 24
30 | ISBN = 25
31 | billNumber = 26
32 | number = 27
33 | code = 28
34 | codeVolume = 29
35 | section = 30
36 | codePages = 31
37 | pages = 32
38 | legislativeBody = 33
39 | session = 34
40 | history = 35
41 | blogTitle = 36
42 | publicationTitle = 37
43 | websiteType = 38
44 | type = 39
45 | series = 40
46 | seriesNumber = 41
47 | edition = 42
48 | numPages = 43
49 | bookTitle = 44
50 | caseName = 45
51 | court = 46
52 | dateDecided = 47
53 | docketNumber = 48
54 | reporter = 49
55 | reporterVolume = 50
56 | firstPage = 51
57 | versionNumber = 52
58 | system = 53
59 | company = 54
60 | programmingLanguage = 55
61 | proceedingsTitle = 56
62 | conferenceName = 57
63 | DOI = 58
64 | dictionaryTitle = 59
65 | subject = 60
66 | encyclopediaTitle = 61
67 | distributor = 62
68 | genre = 63
69 | videoRecordingFormat = 64
70 | forumTitle = 65
71 | postType = 66
72 | committee = 67
73 | documentNumber = 68
74 | interviewMedium = 69
75 | issue = 70
76 | seriesText = 71
77 | journalAbbreviation = 72
78 | ISSN = 73
79 | letterType = 74
80 | manuscriptType = 75
81 | mapType = 76
82 | scale = 77
83 | country = 78
84 | assignee = 79
85 | issuingAuthority = 80
86 | patentNumber = 81
87 | filingDate = 82
88 | applicationNumber = 83
89 | priorityNumbers = 84
90 | issueDate = 85
91 | references = 86
92 | legalStatus = 87
93 | episodeNumber = 88
94 | audioFileType = 89
95 | presentationType = 90
96 | meetingName = 91
97 | programTitle = 92
98 | network = 93
99 | reportNumber = 94
100 | reportType = 95
101 | institution = 96
102 | nameOfAct = 97
103 | codeNumber = 98
104 | publicLawNumber = 99
105 | dateEnacted = 100
106 | thesisType = 101
107 | university = 102
108 | studio = 103
109 | websiteTitle = 104
110 |
111 |
112 | @unique
113 | class itemTypes(Enum):
114 | artwork = 1
115 | attachment = 2
116 | audioRecording = 3
117 | bill = 4
118 | blogPost = 5
119 | book = 6
120 | bookSection = 7
121 | case = 8
122 | computerProgram = 9
123 | conferencePaper = 10
124 | dictionaryEntry = 11
125 | document = 12
126 | email = 13
127 | encyclopediaArticle = 14
128 | film = 15
129 | forumPost = 16
130 | hearing = 17
131 | instantMessage = 18
132 | interview = 19
133 | journalArticle = 20
134 | letter = 21
135 | magazineArticle = 22
136 | manuscript = 23
137 | map = 24
138 | newspaperArticle = 25
139 | note = 26
140 | patent = 27
141 | podcast = 28
142 | presentation = 29
143 | radioBroadcast = 30
144 | report = 31
145 | statute = 32
146 | thesis = 33
147 | tvBroadcast = 34
148 | videoRecording = 35
149 | webpage = 36
150 |
151 |
152 | class commonPrefs:
153 | dataDir = "extensions.zotero.dataDir"
154 | attachmentDir = "extensions.zotero.baseAttachmentPath"
155 |
156 |
--------------------------------------------------------------------------------
/pyzolocal/beans/struct.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/beans/struct.py
--------------------------------------------------------------------------------
/pyzolocal/beans/types.py:
--------------------------------------------------------------------------------
1 | from dataclasses import dataclass, field
2 | import os
3 | from .enum import fileds, itemTypes
4 | from typing import Optional, List
5 |
6 |
7 | @dataclass()
8 | class Tag:
9 | tagId: int
10 | name: str
11 | type: Optional[int] = 0
12 |
13 |
14 | @dataclass()
15 | class ItemData():
16 | filed: fileds
17 | valueID: int
18 | value: str
19 |
20 | @dataclass()
21 | class Creator:
22 | creatorID: int = -1
23 | firstName: str = ""
24 | lastName: str = ""
25 | fieldMode: int = 0
26 |
27 | @dataclass()
28 | class Item():
29 | """
30 | one row
31 | """
32 | itemID: int = -1
33 | itemType: itemTypes = None
34 | key: str = None
35 | authors: List[Creator] = None
36 | itemDatas: List[ItemData] = None
37 |
38 |
39 | @dataclass()
40 | class Collection:
41 | collectionID: int
42 | collectionName: str
43 |
44 |
45 | @dataclass()
46 | class Attachment:
47 | itemID: int
48 | key: str
49 | contentType: str
50 | relpath: str
51 |
52 | @property
53 | def abspath(self):
54 | from ..prefs.common import dataDir
55 | return os.path.join(dataDir(), self.relpath)
56 |
57 | @property
58 | def is_attachment_url(self):
59 | if self.relpath is not None:
60 | return 'attachments:' in self.relpath
61 | return self.relpath is None
62 |
63 |
64 |
--------------------------------------------------------------------------------
/pyzolocal/cli/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/cli/__init__.py
--------------------------------------------------------------------------------
/pyzolocal/cli/zolocal.py:
--------------------------------------------------------------------------------
1 | from fire import Fire
2 |
3 | from pyzolocal.prefs.base import (
4 | prefs_root, prefjs_fn, profile_root,
5 | is_mac, is_win, is_linux
6 | )
7 | from pyzolocal.prefs.common import dataDir
8 | from pyzolocal.files.base import storage_root
9 | from pyzolocal import __version__
10 |
11 |
12 | def main(*args, **kwargs):
13 | if len(args) == 0 and len(kwargs) == 0:
14 | version_display = (f'''pyzolocal version {__version__}
15 | |Win: {is_win()}|MAC: {is_mac()}|Linue: {is_linux()}|
16 | zotero.profile_directory = {profile_root()}
17 | zotero.dataDir = {dataDir()}
18 | ''')
19 | print(version_display)
20 |
21 |
22 | Fire(main)
23 | exit(0)
24 |
--------------------------------------------------------------------------------
/pyzolocal/files/__init__.py:
--------------------------------------------------------------------------------
1 | # file search
2 | from .base import index
3 | from .gets import search_content
4 |
--------------------------------------------------------------------------------
/pyzolocal/files/base.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | import os
3 | from itertools import chain
4 | from ..prefs.common import dataDir
5 |
6 | from whoosh.fields import *
7 | from whoosh.index import create_in
8 | from whoosh.index import open_dir
9 | from whoosh.query import *
10 |
11 | import re
12 |
13 |
14 | def index_root():
15 | root = os.path.join(dataDir(), 'index')
16 | if not os.path.exists(root):
17 | os.makedirs(root, exist_ok=True)
18 | return root
19 |
20 |
21 | def storage_root():
22 | root = os.path.join(dataDir(), 'storage')
23 | return root
24 |
25 |
26 | def hash_dir(*fs):
27 | from hashlib import md5
28 | hl = md5()
29 | for f in fs:
30 | hl.update(f.encode('utf-8'))
31 | for f in fs:
32 | hl.update(str(os.stat(f).st_atime + os.stat(f).st_mtime + os.stat(f).st_ctime).encode())
33 | return hl.hexdigest()
34 |
35 |
36 | def index():
37 | schema = Schema(title=TEXT(stored=True, spelling_prefix=True),
38 | itemID=NUMERIC(stored=True),
39 | contentType=KEYWORD(stored=True),
40 | key=ID(stored=True),
41 | content=TEXT(stored=True, spelling_prefix=True), path=TEXT(stored=True))
42 | indexdir = index_root()
43 | datadir = dataDir()
44 |
45 | if not os.path.exists(indexdir):
46 | os.mkdir(indexdir)
47 | ix = create_in(indexdir, schema)
48 | else:
49 | ix = open_dir(indexdir)
50 |
51 | writer = ix.writer()
52 | from ..sqls.base import create_conn
53 | from ..sqls.gets import get_attachments
54 |
55 | with create_conn() as conn:
56 | attachs = get_attachments(conn)
57 |
58 | for attach in attachs:
59 | _attach_f = os.path.join(datadir, attach.relpath)
60 | _attach_dir = os.path.dirname(_attach_f)
61 | _cache_f = os.path.join(_attach_dir, '.zotero-ft-cache')
62 |
63 | if not os.path.exists(_cache_f) or not os.path.exists(_attach_f):
64 | continue
65 |
66 | _dir_hash = hash_dir(_cache_f, _attach_f)
67 | _hash_cache_f = os.path.join(_attach_dir, '.cache')
68 |
69 | if os.path.exists(_hash_cache_f):
70 | with open(_hash_cache_f, 'r') as r:
71 | _old_hash = r.read().strip()
72 | if _dir_hash == _old_hash:
73 | continue
74 | pass
75 |
76 | with open(_hash_cache_f, 'w') as w:
77 | w.write(hash_dir(_cache_f, _attach_f))
78 |
79 | with open(_cache_f, 'r', encoding='utf-8', errors='ignore') as r:
80 | content = r.read()
81 |
82 | _pre, _ = os.path.splitext(os.path.basename(_attach_f))
83 | title = _pre.split(' - ')[-1]
84 |
85 | writer.add_document(title=title, content=content,
86 | itemID=attach.itemID,
87 | key=attach.key, path=attach.relpath)
88 |
89 | writer.commit()
90 | return ix.doc_count()
91 |
--------------------------------------------------------------------------------
/pyzolocal/files/gets.py:
--------------------------------------------------------------------------------
1 | from .base import index_root
2 |
3 | from itertools import chain
4 |
5 | from whoosh.index import open_dir
6 | from whoosh.query import *
7 | import re
8 |
9 | match_highlight = re.compile('
([^<]+)<\/b>')
10 |
11 | __all__ = ['search_content']
12 |
13 |
14 | def search_content(*content: str, limit=10) -> dict:
15 | content = [i.lower() for i in content]
16 | ix = open_dir(index_root())
17 | searcher = ix.searcher()
18 |
19 | items = list(chain(*[i.split(' ') for i in content]))
20 | items = [Term('title', item) for item in items] + [Term('content', item) for item in items]
21 |
22 | quary = Or(items)
23 |
24 | results = searcher.search(q=quary, limit=limit)
25 |
26 | ress = []
27 |
28 | size = min(len(results), limit)
29 | for i in range(size):
30 | res = results[i].fields()
31 | path = res['path']
32 | title = results[i].highlights('title').split('...')[0]
33 | if len(title.strip()) == 0:
34 | title = res['title']
35 | results.fragmenter.surround = 75
36 | contents = results[i].highlights('content').replace('\n', ' ').split('...')
37 |
38 | ress.append({'title': title,
39 | 'path': path,
40 | 'contents': contents})
41 |
42 | return {
43 | 'base': {
44 | 'hit_count': size,
45 | 'result_count': len(results),
46 | 'doc_count': ix.doc_count()
47 | },
48 | 'result': ress
49 | }
50 |
--------------------------------------------------------------------------------
/pyzolocal/gui/__init__.py:
--------------------------------------------------------------------------------
1 | # 开一个 gui
--------------------------------------------------------------------------------
/pyzolocal/prefs/__init__.py:
--------------------------------------------------------------------------------
1 | from . import gets
2 | from .base import prefs_root, profile_root, prefjs_fn
3 |
--------------------------------------------------------------------------------
/pyzolocal/prefs/base.py:
--------------------------------------------------------------------------------
1 | import platform
2 | import json
3 | import configparser
4 | import os
5 | from functools import lru_cache
6 |
7 |
8 | def is_win():
9 | return 'windows' in platform.platform().lower()
10 |
11 |
12 | def is_mac():
13 | # TODO
14 | raise NotImplementedError()
15 |
16 |
17 | def is_linux():
18 | raise NotImplementedError()
19 | # return 'linux' in platform.platform().lower()
20 |
21 |
22 | def read_profile_path(fn):
23 | """
24 | load profile path from profiles.ini
25 | :param fn:
26 | :return:
27 | """
28 | config = configparser.ConfigParser()
29 | config.read(fn)
30 | if len(config.sections()) == 2:
31 | prof_key = config.sections()[-1]
32 | else:
33 | if config['General']['startwithlastprofile'] == '1':
34 | prof_key = config.sections()[-1]
35 | else:
36 | raise NotImplementedError()
37 | return config[prof_key]['Path']
38 |
39 |
40 | def profile_root():
41 | user_root = os.path.expanduser('~')
42 | if is_win():
43 | zotero_root = os.path.join(user_root, r'AppData\Roaming\Zotero\Zotero')
44 | elif is_linux():
45 | zotero_root = os.path.join(user_root, '.zotero/zotero')
46 | elif is_mac():
47 | zotero_root = os.path.join(user_root, 'Library/Application Support/Zotero')
48 | else:
49 | raise NotImplementedError()
50 | return zotero_root
51 |
52 |
53 | def prefs_root():
54 | user_root = os.path.expanduser('~')
55 | zotero_root = profile_root()
56 | if is_win():
57 | prof_relroot = read_profile_path(os.path.join(zotero_root, 'profiles.ini'))
58 | return os.path.join(zotero_root, prof_relroot)
59 | elif is_linux():
60 | zotero_root = os.path.join(user_root, '.zotero/zotero')
61 | elif is_mac():
62 | zotero_root = os.path.join(user_root, 'Library/Application Support/Zotero')
63 | else:
64 | raise NotImplementedError()
65 |
66 |
67 | @lru_cache(1)
68 | def prefjs_fn():
69 | return os.path.join(prefs_root(), 'prefs.js')
70 |
71 |
72 | def try_parse_pref(value: str):
73 | if value is None:
74 | return value
75 |
76 | if value.startswith('"'):
77 | value = value.strip('"')
78 | if value[0] in "[{":
79 | value = value.replace('\\', '')
80 | try:
81 | value = json.loads(value)
82 | except:
83 | pass
84 | return value
85 | else:
86 | if value in {'false', 'true'} or value.isnumeric():
87 | return eval(value.title())
88 |
89 | return value
90 |
--------------------------------------------------------------------------------
/pyzolocal/prefs/common.py:
--------------------------------------------------------------------------------
1 | from ..beans.enum import commonPrefs
2 | import os
3 | from .gets import get_and_parse_user_pref
4 | from .base import prefjs_fn
5 | from functools import lru_cache
6 | import warnings
7 |
8 | _datadir = None
9 |
10 |
11 | def setDataDir(path: str):
12 | """
13 | call setDataDir(None) to clear cache
14 | :param path:
15 | :return:
16 | """
17 | global _datadir
18 | _datadir = path
19 |
20 |
21 | @lru_cache()
22 | def dataDir(path=None):
23 | """
24 |
25 | :param path:
26 | :return:
27 | """
28 | global _datadir
29 | if path is not None:
30 | _datadir = path
31 | if _datadir is not None:
32 | return _datadir
33 |
34 | res = get_and_parse_user_pref(commonPrefs.dataDir, None)
35 | if res is not None and not os.path.exists(res):
36 | warnings.warn(f"""
37 | Cannot find zotero dataDir writed in prefs.js, check your setting in {prefjs_fn()}
38 | or report this issue if located path is wrong.
39 |
40 | You can set data root manually by call `prefs.common.setDataDir()`
41 | """)
42 |
43 | if res is None or not os.path.exists(res):
44 | res = os.path.join(os.path.expanduser('~'), 'Zotero')
45 |
46 | if not os.path.exists(res):
47 | warnings.warn('cannot find default zotero dataDir, '
48 | 'will return inexist default dataDir. '
49 | 'You can set data root manually by call `prefs.common.setDataDir()`')
50 | return res
51 |
52 |
53 | KEY_STORAGE = 'storage'
54 |
--------------------------------------------------------------------------------
/pyzolocal/prefs/gets.py:
--------------------------------------------------------------------------------
1 | # This file used to read Zotero Profile
2 | # see https://www.zotero.org/support/kb/profile_directory
3 | from typing import Dict
4 | from .base import prefjs_fn, try_parse_pref
5 | import re
6 |
7 | match_user_pref = re.compile("""user_pref\("([^"]+)", (.+)\);""")
8 |
9 | __all__ = [
10 | 'get_and_parse_user_prefs',
11 | 'get_and_parse_user_pref',
12 | 'search_user_prefs'
13 | ]
14 |
15 |
16 | # user_pref("extensions.zotero.export.quickCopy.setting", "export=9cb70025-a888-4a29-a210-93ec52da40d4");
17 | def _get_user_prefs() -> Dict[str, str]:
18 | """
19 | saved in /prefs.js, get raw value
20 | :return:
21 | """
22 | prof = {}
23 | with open(prefjs_fn(), 'r') as r:
24 | for line in r:
25 | res = re.search(match_user_pref, line)
26 | if res is not None:
27 | prof[res.group(1)] = res.group(2)
28 | return prof
29 |
30 |
31 | def search_user_prefs(keylike: str) -> dict:
32 | user_prefs = _get_user_prefs()
33 | keylike = keylike.replace('.', '').lower()
34 | return {
35 | k: try_parse_pref(user_prefs[k])
36 | for k in user_prefs if keylike in k.lower()
37 | }
38 |
39 |
40 | def get_user_pref(key, default=None) -> str:
41 | """
42 |
43 | :return:
44 | """
45 | return _get_user_prefs().get(key, default)
46 |
47 |
48 | def get_and_parse_user_pref(key: str, default=""):
49 | value = _get_user_prefs().get(key, default) # type:str
50 | return try_parse_pref(value)
51 |
52 |
53 | def get_and_parse_user_prefs():
54 | prefs = _get_user_prefs()
55 | return {k: try_parse_pref(v) for k, v in prefs.items()}
56 |
--------------------------------------------------------------------------------
/pyzolocal/repair/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/repair/__init__.py
--------------------------------------------------------------------------------
/pyzolocal/repair/db.py:
--------------------------------------------------------------------------------
1 | import os
2 | from ..prefs.common import dataDir, KEY_STORAGE
3 | from ..sqls import gets, base
4 | from ..beans.enum import itemTypes
5 |
6 |
7 | def insert_disappear_dir_in_db():
8 | """"""
9 | storage_root = os.path.join(dataDir(), KEY_STORAGE)
10 | storage_dirs = set(os.listdir(storage_root))
11 | db_keys = [attach.key for attach in gets.get_attachments()]
12 |
13 | sql = """
14 | INSERT INTO items (itemTypeID, libraryID, key)
15 | VALUES ({}, {}, {});
16 | """
17 | # TODO execte sql
18 | res = []
19 | for key in storage_dirs:
20 | if not os.path.isdir(os.path.join(storage_root, key)):
21 | continue
22 | if key not in db_keys:
23 | res.append(os.path.join(storage_root, key))
24 |
--------------------------------------------------------------------------------
/pyzolocal/repair/file.py:
--------------------------------------------------------------------------------
1 | """
2 | 修复命名错误，删除多余的，报告缺失的
3 | """
4 | import os
5 | import shutil
6 | from ..prefs.common import dataDir, KEY_STORAGE
7 | from ..sqls import gets
8 |
9 |
10 | def rename_like_file():
11 | for basename, (disdir, dir_like) in get_disappear().items():
12 | if len(dir_like) > 0:
13 | tgt = os.path.join(disdir, basename)
14 | src = os.path.join(disdir, dir_like[0])
15 | os.rename(src, tgt)
16 | print(src, 'to', tgt)
17 |
18 |
19 | def delete_dir_not_in_db(rm=False):
20 | """
21 | :param rm: remove dir, if False, will only return dir list, or
22 | all directory(attachment) not in database will be deleted
23 | :return:
24 |
25 | see repair.db.insert_disappear_dir_in_db() for reverse operation
26 | """
27 | storage_root = os.path.join(dataDir(), KEY_STORAGE)
28 | storage_dirs = set(os.listdir(storage_root))
29 | db_keys = [attach.key for attach in gets.get_attachments()]
30 |
31 | res = []
32 | for key in storage_dirs:
33 | if not os.path.isdir(os.path.join(storage_root, key)):
34 | continue
35 | if key not in db_keys:
36 | res.append(os.path.join(storage_root, key))
37 | if rm:
38 | for dir in res:
39 | shutil.rmtree(dir)
40 | return res
41 |
42 |
43 | def get_disappear():
44 | """
45 | 获取 attachment 缺失的列表，疑似列表
46 | :return:
47 | """
48 | res = {}
49 |
50 | for attach in gets.get_attachments():
51 | if attach.is_attachment_url:
52 | continue
53 |
54 | if not os.path.exists(attach.abspath):
55 | disfn = os.path.basename(attach.abspath)
56 |
57 | disdir = os.path.dirname(attach.abspath)
58 | fnext = os.path.splitext(attach.abspath)[-1]
59 |
60 | if os.path.exists(disdir):
61 | dir_has = [i for i in os.listdir(disdir) if i.endswith(fnext)]
62 | else:
63 | dir_has = []
64 |
65 | res[disfn] = [
66 | disdir,
67 | dir_has
68 | ]
69 | return res
70 |
--------------------------------------------------------------------------------
/pyzolocal/sqls/__init__.py:
--------------------------------------------------------------------------------
1 | from . import gets
2 | from .base import create_conn
3 |
--------------------------------------------------------------------------------
/pyzolocal/sqls/base.py:
--------------------------------------------------------------------------------
1 | import zlib
2 | import shutil
3 | import os
4 | from sqlite3 import Connection
5 | from ..prefs.common import dataDir
6 | from sqlite3 import connect
7 |
8 |
9 | def exec_fetchall(sql):
10 | conn = create_conn()
11 | cursor = conn.cursor()
12 | cursor.execute(sql)
13 | values = cursor.fetchall()
14 |
15 | return cursor, values
16 |
17 |
18 | def pako_inflate(data):
19 | """
20 | see
21 | https://github.com/zotero/zotero/blob/2adf0e6d3cab50959f0307fa2d663a9c82ea60bc/chrome/content/zotero/xpcom/schema.js#L365
22 | and
23 | https://stackoverflow.com/questions/46351275/using-pako-deflate-with-python
24 | :param data:
25 | :return:
26 | """
27 | decompress = zlib.decompressobj(15)
28 | decompressed_data = decompress.decompress(data)
29 | decompressed_data += decompress.flush()
30 | return decompressed_data
31 |
32 |
33 | def create_conn() -> Connection:
34 | """
35 | try create sqlite connection
36 | :return:
37 | """
38 | data_dir = dataDir()
39 | sqlite_fn = os.path.join(data_dir, 'zotero.sqlite')
40 | shutil.copy(sqlite_fn, os.path.join(data_dir, 'zotero.wrap.sqlite.bak'))
41 | sqlite_fn = os.path.join(data_dir, 'zotero.wrap.sqlite.bak')
42 |
43 | assert os.path.exists(sqlite_fn)
44 | return connect(sqlite_fn)
45 |
--------------------------------------------------------------------------------
/pyzolocal/sqls/deletes.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/sqls/deletes.py
--------------------------------------------------------------------------------
/pyzolocal/sqls/gets.py:
--------------------------------------------------------------------------------
1 | import os
2 | import json
3 | from ..sqls.base import exec_fetchall, pako_inflate
4 | from typing import List, Dict
5 | from ..beans.enum import fileds, itemTypes
6 | from ..beans import types as t
7 |
8 | from collections import defaultdict
9 |
10 |
11 | def get_settings() -> dict:
12 | sql = """
13 | select value from settings
14 | """
15 | _, values = exec_fetchall(sql)
16 |
17 | return json.loads(pako_inflate(values[0][0]))
18 |
19 | def get_creator(creatorID:int=-1) -> t.Creator:
20 | sql = f'select * from creators where creatorID={creatorID}'
21 | cursor, values = exec_fetchall(sql)
22 | values = values[0]
23 | return t.Creator(creatorID=values[0],firstName=values[1], lastName=values[2], fieldMode=values[3])
24 |
25 |
26 | def get_creators() -> List[t.Creator]:
27 | sql = 'select creatorID from creators'
28 | cursor, values = exec_fetchall(sql)
29 | if len(values) > 0:
30 | return [get_creator(value[0]) for value in values]
31 | else:
32 | return []
33 |
34 |
35 | def get_tags() -> List[t.Tag]:
36 | sql = 'select * from tags'
37 | cursor, values = exec_fetchall(sql)
38 |
39 | return [t.Tag(tagId=i[0], name=i[1])
40 | for i in values]
41 |
42 |
43 | def get_collections() -> List[t.Collection]:
44 | sql = """
45 | select * from collections
46 | """
47 |
48 | cursor, values = exec_fetchall(sql)
49 | return [t.Collection(collectionID=i[0],
50 | collectionName=i[1]) for i in values]
51 |
52 |
53 | def get_itemids(include_delete=False) -> List[int]:
54 | if include_delete:
55 | sql = f"""
56 | select itemID from items
57 | """
58 | else:
59 | sql = f"""
60 | select itemID from items
61 | where itemID not in (select itemID from deletedItems)
62 | """
63 |
64 | cursor, values = exec_fetchall(sql)
65 | return [i[0] for i in values]
66 |
67 |
68 | def get_item_creators(itemID:int=-1) -> List[t.Creator]:
69 | if itemID == -1 : return []
70 | sql = f'select * from itemCreators where itemID={itemID}'
71 | cursor, values = exec_fetchall(sql)
72 | if len(values) != 0:
73 | return [get_creator(value[1]) for value in values]
74 | else:
75 | return []
76 |
77 |
78 |
79 | def get_items_info() -> List[t.Item]:
80 | sql = f"""
81 | select * from
82 | (select * from itemData)
83 | left join itemDataValues
84 | using (valueID)
85 | where itemID not in (select itemID from deletedItems)
86 | """
87 |
88 | cursor, values = exec_fetchall(sql)
89 |
90 | if len(values) == 0:
91 | return []
92 |
93 | item_value_map_ = defaultdict(list)
94 | for value in values:
95 | item_value_map_[value[0]].append(value)
96 |
97 | res = []
98 | for item_id, val in item_value_map_.items():
99 | item_authors = get_item_creators(item_id)
100 | item_datas = [t.ItemData(fileds(i[1]), i[2], i[3]) for i in val]
101 | item = t.Item(itemID=item_id,
102 | key=get_item_key_by_itemid(item_id),
103 | itemDatas=item_datas,
104 | authors=item_authors)
105 | res.append(item)
106 | return res
107 |
108 |
109 | def get_item_info_by_itemid(itemID: int) -> t.Item:
110 | sql = f"""
111 | select * from
112 | (select * from itemData where itemID={itemID})
113 | inner join itemDataValues using (valueID)
114 | """
115 |
116 | cursor, values = exec_fetchall(sql)
117 | if len(values) == 0:
118 | return t.Item(-1)
119 |
120 | item_id = values[0][0]
121 |
122 | item_datas = [t.ItemData(fileds(i[1]), i[2], i[3]) for i in values]
123 | item_authors = get_item_creators(item_id)
124 | return t.Item(itemID=item_id,
125 | key=get_item_key_by_itemid(itemID),
126 | itemDatas=item_datas,
127 | authors=item_authors)
128 |
129 |
130 | def get_attachments(type=-1) -> List[t.Attachment]:
131 | """
132 | all attached files
133 | :param conn:
134 | :return:
135 | """
136 | if type == -1:
137 | sql = """
138 | select itemID,key,contentType,path from (
139 | itemAttachments inner join items using (itemID)
140 | )
141 | """
142 | else:
143 | raise NotImplementedError()
144 |
145 | cursor, values = exec_fetchall(sql)
146 |
147 | res = []
148 | for (itemID, key, contentType, path) in values:
149 | if path is not None:
150 | relpath = os.path.join('storage', key, path.replace('storage:', ''))
151 | else:
152 | relpath = None
153 | file = t.Attachment(itemID=itemID, key=key,
154 | contentType=contentType, relpath=relpath)
155 | res.append(file)
156 | return res
157 |
158 |
159 | def get_attachments_by_parentid(parentItemID: int) -> List[t.Attachment]:
160 | """
161 | get attached file from parent
162 | :param conn:
163 | :param parentItemID:
164 | :return:
165 | """
166 | sql = f"""
167 | select itemID,contentType,path from itemAttachments
168 | where itemID in (select itemID from itemAttachments where parentItemID={parentItemID})
169 | """
170 |
171 | cursor, values = exec_fetchall(sql)
172 | if len(values) == 0:
173 | return []
174 |
175 | item_value_map_ = {}
176 | for value in values:
177 | item_value_map_[value[0]] = value
178 |
179 | res = []
180 | # item_id_type_map = get_items_type_by_itemids(*list(item_value_map_.keys()))
181 | item_id_key_map = get_items_key_by_itemid(*list(item_value_map_.keys()))
182 |
183 | for item_id, val in item_value_map_.items():
184 | path = val[2] # type:str
185 | key = item_id_key_map[item_id]
186 | relpath = os.path.join('storage', key, path.replace('storage:', ''))
187 | item = t.Attachment(itemID=item_id,
188 | key=key,
189 | contentType=val[1],
190 | relpath=relpath)
191 | res.append(item)
192 | return res
193 |
194 |
195 | def get_item_attachments_by_parentid(parentItemID: int) -> List[t.Item]:
196 | """
197 | get attached item from parent
198 | :param conn:
199 | :param parentItemID:
200 | :return:
201 | """
202 | sql = f"""
203 | select * from
204 | (select * from itemData)
205 | inner join itemDataValues
206 | using (valueID)
207 | where itemID in (select itemID from itemAttachments where parentItemID={parentItemID})
208 | """
209 |
210 | cursor, values = exec_fetchall(sql)
211 | if len(values) == 0:
212 | return []
213 |
214 | item_value_map_ = defaultdict(list)
215 | for value in values:
216 | item_value_map_[value[0]].append(value)
217 |
218 | res = []
219 | item_id_type_map = get_items_type_by_itemids(*list(item_value_map_.keys()))
220 | item_id_key_map = get_items_key_by_itemid(*list(item_value_map_.keys()))
221 |
222 | for item_id, val in item_value_map_.items():
223 | item_datas = [t.ItemData(fileds(i[1]), i[2], i[3]) for i in val]
224 | item_authors = get_item_creators(item_id)
225 | item = t.Item(itemID=item_id,
226 | key=item_id_key_map[item_id],
227 | itemType=itemTypes(item_id_type_map[item_id]),
228 | itemDatas=item_datas,
229 | authors=item_authors)
230 | res.append(item)
231 | return res
232 |
233 |
234 | def get_items_info_from_tag_by_tagid(tagID: int) -> List[t.Item]:
235 | sql = f"""
236 | select * from
237 | (select * from itemData)
238 | inner join itemDataValues using (valueID)
239 | inner join (select itemID from itemTags where tagID={tagID}) using (itemID)
240 | """
241 | cursor, values = exec_fetchall(sql)
242 |
243 | if len(values) == 0:
244 | return []
245 |
246 | item_value_map_ = defaultdict(list)
247 | for value in values:
248 | item_value_map_[value[0]].append(value)
249 |
250 | res = []
251 | item_id_type_map = get_items_type_by_itemids(*list(item_value_map_.keys()))
252 | item_id_key_map = get_items_key_by_itemid(*list(item_value_map_.keys()))
253 |
254 | for item_id, val in item_value_map_.items():
255 | item_datas = [t.ItemData(fileds(i[1]), i[2], i[3]) for i in val]
256 | item = t.Item(itemID=item_id,
257 | key=item_id_key_map[item_id],
258 | itemType=itemTypes(item_id_type_map[item_id]),
259 | itemDatas=item_datas)
260 | res.append(item)
261 | return res
262 |
263 |
264 | def get_items_info_from_coll_by_collid(collID: int) -> List[t.Item]:
265 | sql = f"""
266 | select * from
267 | (select * from itemData)
268 | inner join itemDataValues using (valueID)
269 | inner join (select itemID from collectionItems where collectionID={collID}) using (itemID)
270 | """
271 |
272 | cursor, values = exec_fetchall(sql)
273 |
274 | if len(values) == 0:
275 | return []
276 |
277 | item_value_map_ = defaultdict(list)
278 | for value in values:
279 | item_value_map_[value[0]].append(value)
280 |
281 | res = []
282 | item_id_type_map = get_items_type_by_itemids(*list(item_value_map_.keys()))
283 | item_id_key_map = get_items_key_by_itemid(*list(item_value_map_.keys()))
284 |
285 | for item_id, val in item_value_map_.items():
286 | item_datas = [t.ItemData(fileds(i[1]), i[2], i[3]) for i in val]
287 | item_authors = get_item_creators(item_id)
288 | item = t.Item(itemID=item_id,
289 | key=item_id_key_map[item_id],
290 | itemType=itemTypes(item_id_type_map[item_id]),
291 | itemDatas=item_datas,
292 | authors=item_authors)
293 | res.append(item)
294 | return res
295 |
296 |
297 | def get_items_key_by_itemid(*itemID: int) -> Dict[int, str]:
298 | itemID_ = ','.join(f'{i}' for i in itemID)
299 | sql = f"""
300 | select itemID,key from items where itemID in ({itemID_})
301 | """
302 | cursor, values = exec_fetchall(sql)
303 |
304 | items_type = {i[0]: i[1] for i in values}
305 |
306 | return items_type
307 |
308 |
309 | def get_item_key_by_itemid(itemID: int) -> str:
310 | return get_items_key_by_itemid(itemID)[itemID]
311 |
312 |
313 | def get_items_type_by_itemids(*itemID: int) -> Dict[int, itemTypes]:
314 | itemID_ = ','.join(f'{i}' for i in itemID)
315 | sql = f"""
316 | select itemID,itemTypeID from items where itemID in ({itemID_})
317 | """
318 | cursor, values = exec_fetchall(sql)
319 |
320 | items_type = {i[0]: itemTypes(i[0]) for i in values}
321 |
322 | return items_type
323 |
324 |
325 | def get_item_type_by_itemid(itemID: int) -> itemTypes:
326 | return get_item_type_by_itemid(itemID)[0]
327 |
328 |
329 | def get_item_tags_by_itemid(itemID: int) -> List[t.Tag]:
330 | sql = f"""
331 | select * from
332 | (select * from itemTags where itemID={itemID})
333 | inner join tags using (tagID)
334 | """
335 | cursor, values = exec_fetchall(sql)
336 | return [t.Tag(i[0], i[1]) for i in values]
337 |
--------------------------------------------------------------------------------
/pyzolocal/sqls/post.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/sqls/post.py
--------------------------------------------------------------------------------
/pyzolocal/sqls/puts.py:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/pyzolocal/sync/__init__.py:
--------------------------------------------------------------------------------
1 | # used to sync something
--------------------------------------------------------------------------------
/pyzolocal/sync/base.py:
--------------------------------------------------------------------------------
1 | import zipfile
2 | import os
3 | from ..prefs.base import profile_root
4 |
5 |
6 | def compress(dirname, zipfilename):
7 | filelist = []
8 | if os.path.isfile(dirname):
9 | filelist.append(dirname)
10 | else:
11 | for root, dirs, files in os.walk(dirname):
12 | for name in files:
13 | filelist.append(os.path.join(root, name))
14 |
15 | zf = zipfile.ZipFile(zipfilename, "w", zipfile.ZIP_DEFLATED)
16 | for tar in filelist:
17 | arcname = tar[len(dirname):]
18 | zf.write(tar, arcname)
19 | zf.close()
20 |
21 |
22 | def decompress(zipfilename, unziptodir):
23 | if not os.path.exists(unziptodir):
24 | os.makedirs(unziptodir, exist_ok=True)
25 |
26 | zfobj = zipfile.ZipFile(zipfilename)
27 | for name in zfobj.namelist():
28 | name = name.replace('\\', '/')
29 |
30 | if name.endswith('/'):
31 | os.mkdir(os.path.join(unziptodir, name))
32 | else:
33 | ext_filename = os.path.join(unziptodir, name)
34 | ext_dir = os.path.dirname(ext_filename)
35 | if not os.path.exists(ext_dir): os.makedirs(ext_dir, exist_ok=True)
36 | # TODO verify before overwrite
37 | with open(ext_filename, 'wb') as outfile:
38 | outfile.write(zfobj.read(name))
39 |
40 |
41 | def bundle(target_dir='./'):
42 | compress(profile_root(), os.path.join(target_dir, 'Profiles.zip'))
43 |
44 |
45 | def dump(source_fn, target_fn):
46 | decompress(source_fn, target_fn)
47 |
--------------------------------------------------------------------------------
/pyzolocal/sync/bypy.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/sync/bypy.py
--------------------------------------------------------------------------------
/pyzolocal/sync/ftp.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/sync/ftp.py
--------------------------------------------------------------------------------
/pyzolocal/sync/gists.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/sync/gists.py
--------------------------------------------------------------------------------
/pyzolocal/wrap/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/sailist/pyzotero-local/e8deac75a7f3b8edb97197ce1f3304652f7e097e/pyzolocal/wrap/__init__.py
--------------------------------------------------------------------------------
/pyzolocal/wrap/zotero.py:
--------------------------------------------------------------------------------
1 | class Zotero():
2 | pass
3 |
4 |
5 | api_list = """
6 | see https://pyzotero.readthedocs.io/en/latest/
7 |
8 | top
9 | everything
10 | key_info
11 | count_items
12 | publications
13 | trash
14 | deleted
15 | item
16 | children
17 | collection_items
18 | collection_items_top
19 | get_subset
20 |
21 |
22 | file
23 | dump
24 |
25 | collections
26 | collections_top
27 | collection
28 | collections_sub
29 | all_collections
30 |
31 | tags
32 | item_tags
33 |
34 | item_versions
35 | collection_versions
36 |
37 | new_fulltext
38 |
39 | fulltext_item
40 |
41 | set_fulltext
42 | follow
43 | iterfollow
44 | makeiter
45 |
46 | The follow(), everything() and makeiter() methods are only valid for methods which can return multiple library items. For instance, you cannot use follow() after an item() call. The generator methods will raise a StopIteration error when all available items retrievable by your chosen API call have been exhausted.
47 |
48 |
49 |
50 | num_items
51 | num_collectionitems
52 |
53 | last_modified_version
54 | """
55 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | fire
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup, find_packages
2 | from pyzolocal import __version__
3 |
4 | setup(
5 | name='zolocal',
6 | version=__version__,
7 | description='A Python tool kit for interacting with the locally hosted Zotero database.',
8 | url='https://github.com/sailist/pyzotero-local',
9 | author='Haozhe Yang',
10 | author_email='yyanghaozhe@outlook.com',
11 | license='GNU General Public License v3',
12 | include_package_data=True,
13 | install_requires=[
14 | 'fire',
15 | ],
16 | classifiers=[
17 | 'Development Status :: 2 - Pre-Alpha',
18 | 'Intended Audience :: Developers',
19 | 'Topic :: Software Development :: Build Tools',
20 | 'License :: OSI Approved :: GNU General Public License v3 (GPLv3)',
21 | 'Programming Language :: Python :: 3',
22 | ],
23 | keywords='zotero sqlite',
24 | packages=find_packages(),
25 | entry_points={
26 | 'console_scripts': [
27 | 'zotero = pyzolocal.cli.zolocal:main'
28 | ]
29 | },
30 | )
31 |
--------------------------------------------------------------------------------