├── .gitignore ├── FakeRedis.py ├── README.MD ├── app.py ├── cache.py ├── config.py ├── onedrive.py ├── process.py ├── requirements.txt ├── templates └── list.html └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | .idea 3 | config.json 4 | **/.DS_Store 5 | __pycache__ 6 | -------------------------------------------------------------------------------- /FakeRedis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- encoding: utf-8 -*- 3 | # Author: MoeClub.org 4 | 5 | # Fake Redis, Cache in memory. 6 | # Support type: bytes, int, str, list, dict 7 | 8 | from threading import Thread 9 | import base64 10 | import pickle 11 | import time 12 | 13 | 14 | class Cache: 15 | def __init__(self, interval=1): 16 | self._interval_time = int(interval) 17 | self._cache = {} 18 | self._valid() 19 | 20 | def _time(self): 21 | return int(time.time()) 22 | 23 | def _interval(self, interval=None): 24 | try: 25 | assert interval 26 | interval = int(interval) 27 | except: 28 | interval = self._interval_time 29 | time.sleep(interval) 30 | 31 | def get(self, item, obj='value'): 32 | try: 33 | assert isinstance(item, str) 34 | assert item and item in self._cache 35 | if obj == 'value': 36 | value = base64.b64decode(str(self._cache[item][obj]).encode('utf-8')) 37 | if not self._cache[item]['bytes']: 38 | value = pickle.loads(value) 39 | else: 40 | value = self._cache[item][obj] 41 | return value 42 | except Exception as error: 43 | print(error) 44 | return None 45 | 46 | def exists(self, item): 47 | try: 48 | assert isinstance(item, str) 49 | assert item and item in self._cache 50 | return 1 51 | except: 52 | return 0 53 | 54 | def set(self, item, item_value, expire=7, refresh=True): 55 | try: 56 | if refresh and item in self._cache: 57 | self.delete(item) 58 | self._cache[item] = {} 59 | if not isinstance(item_value, bytes): 60 | self._cache[item]['bytes'] = 0 61 | item_value = pickle.dumps(item_value) 62 | else: 63 | self._cache[item]['bytes'] = 1 64 | self._cache[item]['value'] = base64.b64encode(item_value).decode('utf-8') 65 | self._cache[item]['ttl'] = str(9999999999) if expire == 0 else str(int(self._time()) + int(expire)) 66 | return "OK" 67 | except Exception as error: 68 | print(error) 69 | return 0 70 | 71 | def delete(self, item): 72 | try: 73 | if isinstance(item, dict): 74 | for item_cache in item: 75 | assert item_cache and item_cache in self._cache 76 | self._cache.pop(item_cache) 77 | elif isinstance(item, str): 78 | assert item and item in self._cache 79 | self._cache.pop(item) 80 | else: 81 | raise Exception 82 | except: 83 | return False 84 | 85 | def flush(self): 86 | self._cache = {} 87 | 88 | def _ttl(self): 89 | while True: 90 | try: 91 | Now = self._time() 92 | for item in self._cache: 93 | try: 94 | if int(self._cache[item]['ttl']) - int(self._interval_time) < Now: 95 | self.delete(item) 96 | except: 97 | break 98 | self._interval() 99 | except: 100 | continue 101 | 102 | def _valid(self): 103 | Task = Thread(target=self._ttl) 104 | Task.setDaemon(True) 105 | Task.start() 106 | 107 | 108 | if __name__ == '__main__': 109 | c = Cache() 110 | c.set('item', 0) 111 | print(c.get('item')) 112 | 113 | 114 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | # OneList 2 | 3 | ## Installation 4 | 5 | ```bash 6 | # apt install python3-pip git redis-server 7 | 8 | git clone https://github.com/0oVicero0/OneList.git 9 | cd OneList 10 | 11 | pip3 install -r requirements.txt 12 | # Get refresh_token --> Setup config 13 | # The in config.json does not need "<" and ">". 14 | 15 | gunicorn app:app -b 127.0.0.1:5000 -D 16 | ``` 17 | 18 | ## Configuration 19 | 20 | Create a config file named `config.json` 21 | 22 | ```json 23 | { 24 | "token": "", 25 | "location_path": "/", 26 | "start_directory": "/", 27 | "threads": 3, 28 | "diff_seconds": 480, 29 | "refresh_seconds": 720, 30 | "metadata_cached_seconds": 768, 31 | "structure_cached_seconds": 840 32 | } 33 | ``` 34 | ## Get refresh_token 35 | 36 | ### Method 1 (Auto, Recommend): 37 | #### Get refresh_token 38 | 39 | > https://login.microsoftonline.com/common/oauth2/authorize?response_type=code&client_id=ea2b36f6-b8ad-40be-bc0f-e5e4a4a7d4fa&redirect_uri=https://api.moeclub.org/onedrive-login 40 | 41 | ### Method 2 (Manual): 42 | 43 | #### Get auth_token in url 44 | 45 | > https://login.microsoftonline.com/common/oauth2/authorize?response_type=code&client_id=ea2b36f6-b8ad-40be-bc0f-e5e4a4a7d4fa&redirect_uri=http://localhost/onedrive-login 46 | 47 | #### Get refresh_token 48 | 49 | ```bash 50 | code="" 51 | wget --no-check-certificate --post-data="client_id=ea2b36f6-b8ad-40be-bc0f-e5e4a4a7d4fa&client_secret=h27zG8pr8BNsLU0JbBh5AOznNS5Of5Y540l/koc7048=&grant_type=authorization_code&resource=https://api.office.com/discovery/&redirect_uri=http://localhost/onedrive-login&code=$code" 'https://login.microsoftonline.com/common/oauth2/token' -qO- 52 | ``` 53 | 54 | ## Demo 55 | > https://moeclub.org/onedrive 56 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- encoding: utf-8 -*- 3 | # Author: MoeClub.org, sxyazi 4 | 5 | from process import od 6 | from config import config 7 | from utils import path_format 8 | from flask import Flask, abort, redirect, render_template, Blueprint 9 | 10 | bp = Blueprint('main', __name__, url_prefix=config.location_path) 11 | 12 | 13 | # Views 14 | @bp.route('/favicon.ico') 15 | def favicon(): 16 | return abort(404) 17 | 18 | 19 | @bp.route('/', defaults={'path': '/'}) 20 | @bp.route('/') 21 | def catch_all(path): 22 | info = od.list_items_with_cache( 23 | path_format(config.start_directory + '/' + path)) 24 | 25 | if info.is_file: # download 26 | return redirect(info.files[0]['download_url']) 27 | 28 | return render_template('list.html', info=info, path=path_format(path).strip('/')) 29 | 30 | 31 | # Filters 32 | @bp.app_template_filter('date_format') 33 | def date_format(str, format='%Y/%m/%d %H:%M:%S'): 34 | from dateutil import tz 35 | from datetime import datetime 36 | 37 | dt = datetime.strptime(str, "%Y-%m-%dT%H:%M:%SZ") 38 | return dt.replace(tzinfo=tz.tzutc()).astimezone(tz.gettz('Asia/Shanghai')).strftime(format) 39 | 40 | 41 | @bp.app_template_filter('file_size') 42 | def file_size(size): 43 | unit = ( 44 | ('B', 2**0), 45 | ('KB', 2**10), 46 | ('MB', 2**20), 47 | ('GB', 2**30), 48 | ('TB', 2**40), 49 | ('PB', 2**50), 50 | ('EB', 2**60), 51 | ('ZB', 2**70), 52 | ('YB', 2**80) 53 | ) 54 | 55 | for k, v in unit: 56 | if size <= v * 1024: 57 | return '%s %s' % (round(size/v, 2), k) 58 | return 'unknown' 59 | 60 | 61 | app = Flask(__name__) 62 | app.register_blueprint(bp) 63 | 64 | if __name__ == '__main__': 65 | app.run(host='127.0.0.1', port='5000', debug=True) 66 | -------------------------------------------------------------------------------- /cache.py: -------------------------------------------------------------------------------- 1 | # Author: MoeClub.org, sxyazi 2 | 3 | import pickle 4 | import hashlib 5 | try: 6 | import redis 7 | r = redis.Redis(host='127.0.0.1', port=6379, db=0) 8 | # r = redis.Redis(host='Redis_URL', port=PORT, db=0, password='Password', ssl=True, ssl_ca_certs='ca.pem') 9 | _ = r.client_list() 10 | except: 11 | import FakeRedis 12 | r = FakeRedis.Cache() 13 | 14 | 15 | class Cache: 16 | CACHED_SECONDS = 768 17 | 18 | @classmethod 19 | def get(cls, path): 20 | if cls.has(path): 21 | return pickle.loads(r.get(cls._get_key(path))) 22 | return False 23 | 24 | @classmethod 25 | def has(cls, path): 26 | return r.exists(cls._get_key(path)) 27 | 28 | @classmethod 29 | def set(cls, path, entity, expire=CACHED_SECONDS): 30 | return r.set(cls._get_key(path), pickle.dumps(entity), expire) 31 | 32 | @classmethod 33 | def rem(cls, path): 34 | return r.delete(cls._get_key(path)) 35 | 36 | @staticmethod 37 | def _get_key(path): 38 | return 'onelist:' + hashlib.md5(path.encode()).hexdigest() 39 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | 4 | class Config: 5 | def __init__(self, opts={}): 6 | self._options = opts 7 | 8 | def __str__(self): 9 | return json.dumps(self._options, indent=2) 10 | 11 | def __getattr__(self, name): 12 | if name in self._options: 13 | if isinstance(self._options[name], dict): 14 | return Config(self._options[name]) 15 | else: 16 | return self._options[name] 17 | 18 | return None 19 | 20 | 21 | def _parse_config(): 22 | try: 23 | with open('config.json', 'r', encoding='utf-8') as f: 24 | return json.loads(f.read()) 25 | except: 26 | return {} 27 | 28 | 29 | config = Config(_parse_config()) 30 | -------------------------------------------------------------------------------- /onedrive.py: -------------------------------------------------------------------------------- 1 | # Author: MoeClub.org, sxyazi 2 | 3 | import json 4 | import pickle 5 | import hashlib 6 | 7 | from cache import Cache 8 | from utils import path_format 9 | from config import config 10 | from urllib import request, parse 11 | 12 | 13 | class _ItemInfo: 14 | def __init__(self): 15 | self.files = [] 16 | self.folders = [] 17 | self.is_file = False 18 | 19 | 20 | class OneDrive(): 21 | _request_headers = {'User-Agent': 'ISV|MoeClub|OneList/1.0', 22 | 'Accept': 'application/json; odata.metadata=none'} 23 | 24 | def __init__(self): 25 | self.api_url = '' 26 | self.resource_id = '' 27 | 28 | self.expires_on = '' 29 | self.access_token = '' 30 | self.refresh_token = config.token 31 | 32 | try: 33 | self.redirect_uri = config.redirect_uri 34 | assert '://' in self.redirect_uri 35 | except: 36 | self.redirect_uri = 'http://localhost/onedrive-login' 37 | 38 | def get_access(self, resource='https://api.office.com/discovery/'): 39 | res = self._http_request('https://login.microsoftonline.com/common/oauth2/token', method='POST', data={ 40 | 'client_id': 'ea2b36f6-b8ad-40be-bc0f-e5e4a4a7d4fa', 41 | 'client_secret': 'h27zG8pr8BNsLU0JbBh5AOznNS5Of5Y540l/koc7048=', 42 | 'redirect_uri': self.redirect_uri, 43 | 'refresh_token': self.refresh_token, 44 | 'grant_type': 'refresh_token', 45 | 'resource': resource 46 | }) 47 | 48 | self.expires_on = res['expires_on'] 49 | self.access_token = res['access_token'] 50 | self.refresh_token = res['refresh_token'] 51 | 52 | if not self.access_token: 53 | print('Unauthorized') 54 | exit(1) 55 | 56 | def get_resource(self): 57 | res = self._http_request( 58 | 'https://api.office.com/discovery/v2.0/me/services') 59 | 60 | for item in res['value']: 61 | if item['serviceApiVersion'] == 'v2.0': 62 | self.api_url = item['serviceEndpointUri'] 63 | self.resource_id = item['serviceResourceId'] 64 | 65 | if not self.api_url: 66 | raise Exception('Failed to get api url') 67 | 68 | def list_items(self, path='/'): 69 | url = '%s/drive/root:%s/?expand=children(select=name,size,file,folder,parentReference,lastModifiedDateTime)' % ( 70 | self.api_url, parse.quote(path_format(path))) 71 | res = self._http_request(url) 72 | 73 | info = _ItemInfo() 74 | self._append_item(info, res) 75 | 76 | if 'children' in res: 77 | for children in res['children']: 78 | self._append_item(info, children) 79 | 80 | if info.files and not info.folders: 81 | info.is_file = True 82 | return info 83 | 84 | def list_all_items(self, path='/'): 85 | ret = _ItemInfo() 86 | tasks = [{'full_path': path}] 87 | 88 | while len(tasks) > 0: 89 | c = tasks.pop(0) 90 | 91 | tmp = self.list_items(c['full_path']) 92 | tasks += tmp.folders[1:] 93 | 94 | ret.files += tmp.files 95 | ret.folders += tmp.folders[1:] 96 | 97 | if ret.files and not ret.folders: 98 | ret.is_file = True 99 | return ret 100 | 101 | def list_items_with_cache(self, path='/', flash=False): 102 | path = path_format(path) 103 | key = ('tmp:' + path) if flash else path 104 | 105 | if not Cache.has(key): 106 | if flash: 107 | Cache.set(key, self.list_items(path), 10) 108 | else: 109 | print('missing: %s' % path) 110 | 111 | info = self.list_items(path) 112 | if info.is_file: 113 | Cache.set(key, info, config.metadata_cached_seconds) 114 | else: 115 | Cache.set(key, info, config.structure_cached_seconds) 116 | 117 | return Cache.get(key) 118 | 119 | def _http_request(self, url, method='GET', data={}): 120 | headers = self._request_headers.copy() 121 | if self.access_token: 122 | headers['Authorization'] = "Bearer " + self.access_token 123 | 124 | data = parse.urlencode(data).encode('utf-8') 125 | res = json.loads(request.urlopen(request.Request( 126 | url, method=method, data=data, headers=headers)).read().decode('utf-8')) 127 | 128 | if 'error' in res: 129 | raise Exception(res['error']['message']) 130 | return res 131 | 132 | def _append_item(self, info, item): 133 | if 'path' not in item['parentReference']: 134 | path = item['name'] = '/' 135 | else: 136 | path = item['parentReference']['path'][12:] or '/' 137 | 138 | dic = { 139 | 'name': item['name'], 140 | 'size': item['size'], 141 | 'hash': self._get_item_hash(item), 142 | 'folder': path, 143 | 'full_path': path_format(path + '/' + item['name']), 144 | 'updated_at': item['lastModifiedDateTime'] 145 | } 146 | if '@content.downloadUrl' in item: 147 | dic['download_url'] = item['@content.downloadUrl'] 148 | 149 | if 'file' in item: 150 | info.files.append(dic) 151 | else: 152 | info.folders.append(dic) 153 | 154 | def _get_item_hash(self, item): 155 | dic = { 156 | 'name': item['name'], 157 | 'size': item['size'], 158 | 'parentReference': item['parentReference'], 159 | 'lastModifiedDateTime': item['lastModifiedDateTime'] 160 | } 161 | 162 | if 'file' in item: 163 | dic['file'] = item['file'] 164 | else: 165 | dic['folder'] = item['folder'] 166 | 167 | return hashlib.md5(pickle.dumps(dic)).hexdigest() 168 | -------------------------------------------------------------------------------- /process.py: -------------------------------------------------------------------------------- 1 | import time 2 | import schedule 3 | import threading 4 | from cache import Cache 5 | from config import config 6 | from onedrive import OneDrive 7 | from utils import path_format 8 | 9 | 10 | od = OneDrive() 11 | 12 | 13 | class Process: 14 | tasks = [] 15 | 16 | @staticmethod 17 | def runner(): 18 | while True: 19 | schedule.run_pending() 20 | time.sleep(1) 21 | 22 | @staticmethod 23 | def refresh_token(): 24 | od.get_access() 25 | od.get_resource() 26 | od.get_access(od.resource_id) 27 | 28 | @classmethod 29 | def refresh_difference(cls): 30 | cls.tasks.append({'full_path': config.start_directory}) 31 | 32 | @classmethod 33 | def worker(cls): 34 | while True: 35 | if len(cls.tasks) < 1: 36 | time.sleep(.1) 37 | continue 38 | 39 | c = cls.tasks.pop(0) 40 | info = od.list_items_with_cache(c['full_path'], True) 41 | 42 | for f in info.files: 43 | p = f['full_path'] 44 | 45 | if not Cache.has(p): 46 | continue 47 | 48 | file = Cache.get(p).files[0] 49 | if file['hash'] != f['hash']: 50 | print('expired file: %s' % p) 51 | Cache.rem(p) 52 | 53 | for f in info.folders: 54 | p = f['full_path'] 55 | 56 | if not Cache.has(p): 57 | print('no cached: %s' % p) 58 | new = od.list_items_with_cache(p, True) 59 | 60 | cls.cache_all(new) 61 | cls.tasks += new.folders[1:] 62 | continue 63 | 64 | folder = Cache.get(p).folders[0] 65 | if folder['hash'] != f['hash']: 66 | print('expired folder: %s' % p) 67 | new = od.list_items_with_cache(p, True) 68 | 69 | cls.cache_all(new) 70 | cls.tasks += new.folders[1:] 71 | 72 | @staticmethod 73 | def cache_all(info): 74 | for f in info.folders: 75 | Cache.set(f['full_path'], od.list_items_with_cache( 76 | f['full_path'], True), config.structure_cached_seconds) 77 | 78 | 79 | Process.refresh_token() 80 | Process.refresh_difference() 81 | 82 | schedule.every(config.refresh_seconds).seconds.do(Process.refresh_token) 83 | schedule.every(config.diff_seconds).seconds.do(Process.refresh_difference) 84 | 85 | threading.Thread(target=Process.runner).start() 86 | for _ in range(config.threads): 87 | threading.Thread(target=Process.worker).start() 88 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==1.0.2 2 | redis==3.1.0 3 | schedule==0.6.0 4 | gunicorn==19.9.0 5 | python-dateutil==2.8.0 -------------------------------------------------------------------------------- /templates/list.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | OneList 6 | 107 | 108 | 109 | 110 |

111 | OneList 112 |

113 | 114 |
115 |
116 |
117 | {% if path%} 118 | 119 | 120 | 121 | {% endif %} 122 |

{{ path or '/' }}

123 |
124 |
125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | {% for v in info.folders[1:] %} 133 | 134 | 137 | 138 | 139 | 140 | {% endfor %} 141 | 142 | {% for v in info.files %} 143 | 144 | 147 | 148 | 149 | 150 | {% endfor %} 151 |
文件修改时间大小
135 | {{ v.name }} 136 | {{ v.updated_at | date_format }}{{ v.size | file_size }}
145 | {{ v.name }} 146 | {{ v.updated_at | date_format }}{{ v.size | file_size }}
152 |
153 |
154 |
155 | 156 | 196 | 197 | 198 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import hashlib 3 | from datetime import datetime 4 | 5 | 6 | def path_format(path): 7 | while '//' in path: 8 | path = path.replace('//', '/') 9 | 10 | return '/' + path.strip('/') 11 | --------------------------------------------------------------------------------