├── main ├── __init__.py ├── routing.py ├── asgi.py ├── wsgi.py ├── urls.py └── settings.py ├── .dockerignore ├── spreadsheetui ├── migrations │ ├── __init__.py │ ├── 0003_auto_20200522_1102.py │ ├── 0004_job_blocking_job.py │ ├── 0006_auto_20220527_0745.py │ ├── 0005_auto_20200607_0957.py │ ├── 0002_auto_20200522_0752.py │ └── 0001_initial.py ├── __init__.py ├── admin.py ├── tests.py ├── apps.py ├── exceptions.py ├── urls.py ├── management │ └── commands │ │ ├── import_config.py │ │ ├── execute_jobs.py │ │ └── update_torrents.py ├── models.py ├── views.py └── tasks.py ├── MANIFEST.in ├── spreadsheetui.png ├── .isort.cfg ├── requirements.txt ├── manage.py ├── docker-compose.yml ├── Dockerfile ├── LICENSE ├── config.toml.example ├── CHANGELOG.rst ├── setup.py ├── .gitignore ├── README.rst ├── wait-for-it.sh └── twisted └── plugins └── spreadsheetui_plugin.py /main/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .env* 2 | dist 3 | -------------------------------------------------------------------------------- /spreadsheetui/migrations/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include spreadsheetui/static * -------------------------------------------------------------------------------- /spreadsheetui/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "1.2.0" 2 | -------------------------------------------------------------------------------- /spreadsheetui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JohnDoee/spreadsheetui/HEAD/spreadsheetui.png -------------------------------------------------------------------------------- /spreadsheetui/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | 3 | # Register your models here. 4 | -------------------------------------------------------------------------------- /spreadsheetui/tests.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | 3 | # Create your tests here. 4 | -------------------------------------------------------------------------------- /spreadsheetui/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | 3 | 4 | class SpreadsheetuiConfig(AppConfig): 5 | name = "spreadsheetui" 6 | -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- 1 | [settings] 2 | multi_line_output=3 3 | include_trailing_comma=True 4 | force_grid_wrap=0 5 | use_parentheses=True 6 | line_length=88 7 | -------------------------------------------------------------------------------- /main/routing.py: -------------------------------------------------------------------------------- 1 | from channels.routing import ProtocolTypeRouter 2 | 3 | application = ProtocolTypeRouter( 4 | { 5 | # Empty for now (http->django views is added by default) 6 | } 7 | ) 8 | -------------------------------------------------------------------------------- /spreadsheetui/exceptions.py: -------------------------------------------------------------------------------- 1 | class SpreadsheetUiException(Exception): 2 | """Base exception""" 3 | 4 | 5 | class FailedToUpdateException(SpreadsheetUiException): 6 | """Failed to update torrent client""" 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | APScheduler~=3.6.3 2 | Django~=3.0.6 3 | django-environ~=0.4.5 4 | django-filter~=2.2.0 5 | djangorestframework~=3.11.0 6 | jsonfield~=3.1.0 7 | timeoutthreadpoolexecutor~=1.0.2 8 | toml~=0.10.0 9 | loguru~=0.5.0 10 | libtc~=1.2.2 -------------------------------------------------------------------------------- /spreadsheetui/urls.py: -------------------------------------------------------------------------------- 1 | from rest_framework import routers 2 | 3 | from . import views 4 | 5 | router = routers.DefaultRouter() 6 | router.register("torrents", views.TorrentViewSet) 7 | router.register("torrentclients", views.TorrentClientViewSet) 8 | router.register("jobs", views.JobViewSet) 9 | urlpatterns = router.urls 10 | -------------------------------------------------------------------------------- /main/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for main project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /main/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for main project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "main.settings") 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /spreadsheetui/management/commands/import_config.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from spreadsheetui.tasks import import_config 4 | 5 | 6 | class Command(BaseCommand): 7 | help = "Import a torrent config" 8 | 9 | def add_arguments(self, parser): 10 | parser.add_argument("config_path", type=str) 11 | 12 | def handle(self, *args, **options): 13 | import_config(options["config_path"]) 14 | self.stdout.write(self.style.SUCCESS("Config imported")) 15 | -------------------------------------------------------------------------------- /spreadsheetui/migrations/0003_auto_20200522_1102.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2020-05-22 11:02 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("spreadsheetui", "0002_auto_20200522_0752"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="job", 15 | name="source_client", 16 | ), 17 | migrations.AddField( 18 | model_name="job", 19 | name="can_execute", 20 | field=models.BooleanField(default=False), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /spreadsheetui/management/commands/execute_jobs.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from django.core.management.base import BaseCommand 4 | 5 | from spreadsheetui.tasks import execute_jobs 6 | 7 | 8 | class Command(BaseCommand): 9 | help = "Execute all available jobs" 10 | 11 | def add_arguments(self, parser): 12 | parser.add_argument("--loop", dest="loop", action="store_true") 13 | 14 | def handle(self, *args, **options): 15 | if options["loop"]: 16 | while True: 17 | self.stdout.write(self.style.SUCCESS("Executing jobs")) 18 | execute_jobs() 19 | time.sleep(5) 20 | else: 21 | execute_jobs() 22 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'main.settings') 9 | try: 10 | from django.core.management import execute_from_command_line 11 | except ImportError as exc: 12 | raise ImportError( 13 | "Couldn't import Django. Are you sure it's installed and " 14 | "available on your PYTHONPATH environment variable? Did you " 15 | "forget to activate a virtual environment?" 16 | ) from exc 17 | execute_from_command_line(sys.argv) 18 | 19 | 20 | if __name__ == '__main__': 21 | main() 22 | -------------------------------------------------------------------------------- /spreadsheetui/migrations/0004_job_blocking_job.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2020-05-22 11:50 2 | 3 | import django.db.models.deletion 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ("spreadsheetui", "0003_auto_20200522_1102"), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name="job", 16 | name="blocking_job", 17 | field=models.ForeignKey( 18 | null=True, 19 | on_delete=django.db.models.deletion.SET_NULL, 20 | related_name="+", 21 | to="spreadsheetui.Job", 22 | ), 23 | ), 24 | ] 25 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | backend: 5 | image: johndoee/spreadsheetui:latest 6 | restart: unless-stopped 7 | command: ["/wait-for-it.sh", "db:5432", "--", "twistd", "-n", "spreadsheetui"] 8 | networks: 9 | - spreadsheetui 10 | environment: 11 | - DATABASE_URL=postgres://spreadsheetui:ZGUtHsekgfQt6vQc6@db/spreadsheetui 12 | volumes: 13 | - ./config.toml:/spreadsheetui/config.toml 14 | logging: 15 | driver: "json-file" 16 | options: 17 | max-file: "5" 18 | max-size: "10m" 19 | 20 | db: 21 | image: postgres:12 22 | restart: unless-stopped 23 | networks: 24 | - spreadsheetui 25 | environment: 26 | - POSTGRES_PASSWORD=ZGUtHsekgfQt6vQc6 27 | - POSTGRES_USER=spreadsheetui 28 | 29 | networks: 30 | spreadsheetui: -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | ENV PYTHONUNBUFFERED 1 4 | 5 | COPY requirements.txt / 6 | RUN pip install -r /requirements.txt 7 | RUN pip install psycopg2-binary==2.8.6 8 | 9 | RUN mkdir /code 10 | COPY setup.py /code/ 11 | COPY MANIFEST.in /code/ 12 | COPY README.rst /code/ 13 | COPY main/ /code/main/ 14 | COPY spreadsheetui/ /code/spreadsheetui/ 15 | COPY twisted/ /code/twisted/ 16 | 17 | COPY --from=johndoee/spreadsheetui-webinterface:latest /dist/ /code/spreadsheetui/static/ 18 | 19 | WORKDIR /code 20 | RUN python setup.py sdist 21 | RUN cp /code/dist/*.tar.gz / 22 | RUN pip install . 23 | 24 | WORKDIR / 25 | 26 | COPY wait-for-it.sh /wait-for-it.sh 27 | 28 | RUN rm -r /code 29 | RUN mkdir /spreadsheetui 30 | 31 | EXPOSE 18816 32 | 33 | VOLUME ["/spreadsheetui"] 34 | WORKDIR /spreadsheetui 35 | 36 | CMD ["twistd", "-n", "spreadsheetui"] 37 | -------------------------------------------------------------------------------- /main/urls.py: -------------------------------------------------------------------------------- 1 | """main URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/3.0/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.contrib import admin 17 | from django.urls import include, path 18 | 19 | urlpatterns = [ 20 | path("admin/", admin.site.urls), 21 | path("api/", include("spreadsheetui.urls")), 22 | ] 23 | -------------------------------------------------------------------------------- /spreadsheetui/management/commands/update_torrents.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand 2 | 3 | from spreadsheetui.tasks import loop_update_torrents, update_torrents 4 | 5 | 6 | class Command(BaseCommand): 7 | help = "Update all active torrents" 8 | 9 | def add_arguments(self, parser): 10 | parser.add_argument("--loop", dest="loop", action="store_true") 11 | parser.add_argument("--partial", dest="partial", action="store_true") 12 | parser.add_argument("clients", nargs="*", type=str) 13 | 14 | def handle(self, *args, **options): 15 | if options["loop"]: 16 | self.stdout.write(self.style.SUCCESS("Starting loop")) 17 | loop_update_torrents() 18 | else: 19 | self.stdout.write(self.style.SUCCESS("Updating torrents")) 20 | update_torrents(options["clients"], options["partial"]) 21 | self.stdout.write(self.style.SUCCESS("Torrents updated")) 22 | -------------------------------------------------------------------------------- /spreadsheetui/migrations/0006_auto_20220527_0745.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.14 on 2022-05-27 07:45 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("spreadsheetui", "0005_auto_20200607_0957"), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name="torrentclient", 15 | name="client_type", 16 | field=models.CharField( 17 | blank=True, 18 | choices=[ 19 | ("deluge", "Deluge"), 20 | ("rtorrent", "rtorrent"), 21 | ("transmission", "Transmission"), 22 | ("fakeclient", "FakeClient"), 23 | ("qbittorrent", "qBittorrent"), 24 | ("liltorrent", "LilTorrent"), 25 | ], 26 | max_length=30, 27 | null=True, 28 | ), 29 | ), 30 | ] 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2020 Anders Jensen 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /config.toml.example: -------------------------------------------------------------------------------- 1 | [spreadsheetui] 2 | endpoint_description = "tcp:18816" 3 | username = "username" 4 | password = "password" 5 | 6 | [django] 7 | secret_key = "a-secret-key" 8 | database_url = "sqlite:///sqlite.db" 9 | 10 | [clients] 11 | 12 | [clients.deluge] 13 | display_name = "A Deluge" 14 | client_type = "deluge" 15 | host = "127.0.0.1" 16 | port = 58846 17 | username = "localclient" 18 | password = "secretpassword" 19 | session_path = "~/.config/deluge/" 20 | 21 | [clients.the-transmission] 22 | display_name = "Some transmission" 23 | client_type = "transmission" 24 | url = "http://127.0.0.1:9091/transmission/rpc" 25 | session_path = "~/.config/transmission-daemon/" 26 | 27 | [clients.another-transmission] 28 | display_name = "Horse transmission" 29 | client_type = "transmission" 30 | url = "http://127.0.0.1:9092/transmission/rpc" 31 | session_path = "~/.config/transmission-daemon2/" 32 | 33 | [clients.rtorrent] 34 | display_name = "rtorrent" 35 | client_type = "rtorrent" 36 | url = "scgi://127.0.0.1:5000" 37 | session_path = "~/.rtorrent/" 38 | 39 | [clients.another-qbittorrent] 40 | display_name = "qBittorrent 1" 41 | client_type = "qbittorrent" 42 | url = "http://localhost:8080/" 43 | username = "admin" 44 | password = "adminadmin" 45 | session_path = "~/.config/qbittorrent/" -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Changelog 3 | ================================ 4 | 5 | Version 1.2.0 (28-05-2022) 6 | -------------------------------- 7 | 8 | * Change: Updated libtc to latest version and use same config parsing as autotorrent 9 | * Change: Limiting data fetched to improve performance, especially when scrolling. 10 | * Change: Updated webinterface library versions 11 | * Change: Removed channels and other asgi trash 12 | * Change: Timezone now local timezone instead of UTC 13 | 14 | * Bugfix: Units are now correct (not TiB while showing TB) 15 | 16 | Version 1.1.0 (07-06-2020) 17 | -------------------------------- 18 | 19 | * Added: Fake client for testing 20 | * Added: Filtering on torrent client and status 21 | * Added: Support for template change 22 | * Added: Jobs and jobmanager 23 | 24 | * Change: Turned top summary into a bottom summary for current result 25 | * Change: Using better date for rtorrent timestamp 26 | * Change: Moved torrent clients out of the project and into its own library 27 | * Change: Moved docker to postgres 28 | * Change: Removed torrent clients into libtc 29 | * Change: Improved readme 30 | 31 | * Bugfix: rtorrent completion date corrected 32 | * Bugfix: Making sure ratio is updated 33 | 34 | 35 | Version 1.0.1 (11-05-2020) 36 | -------------------------------- 37 | 38 | * Change: Added database lock 39 | * Change: Made partial updates only actually modified torrents 40 | * Change: Made torrent client loops run independent of each other 41 | 42 | * Bugfix: getting an updated rtorrent view 43 | 44 | Version 1.0.0 (10-05-2020) 45 | -------------------------------- 46 | 47 | * Initial release -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from setuptools import find_packages, setup 4 | 5 | 6 | def readme(): 7 | with open("README.rst") as f: 8 | return f.read() 9 | 10 | 11 | setup( 12 | name="spreadsheetui", 13 | version="1.2.0", 14 | url="https://github.com/JohnDoee/spreadsheetui", 15 | author="Anders Jensen", 16 | author_email="jd@tridentstream.org", 17 | description="Multiclient bittorrent webui", 18 | long_description=readme(), 19 | long_description_content_type="text/x-rst", 20 | license="MIT", 21 | packages=find_packages() + ["twisted.plugins"], 22 | package_data={ 23 | "": [ 24 | "spreadsheetui/static/*", 25 | "spreadsheetui/static/*/*", 26 | "spreadsheetui/static/*/*/*", 27 | "spreadsheetui/static/*/*/*/*", 28 | ], 29 | }, 30 | include_package_data=True, 31 | install_requires=[ 32 | "APScheduler~=3.9.1", 33 | "Django~=3.0.6", 34 | "django-environ~=0.4.5", 35 | "django-filter~=2.2.0", 36 | "djangorestframework~=3.11.0", 37 | "jsonfield~=3.1.0", 38 | "timeoutthreadpoolexecutor~=1.0.2", 39 | "toml~=0.10.0", 40 | "Twisted~=22.4.0", 41 | "loguru~=0.5.0", 42 | "libtc~=1.2.2", 43 | ], 44 | classifiers=[ 45 | "Development Status :: 3 - Alpha", 46 | "Environment :: Web Environment", 47 | "Framework :: Twisted", 48 | "Intended Audience :: End Users/Desktop", 49 | "License :: OSI Approved :: MIT License", 50 | "Operating System :: OS Independent", 51 | "Programming Language :: Python", 52 | "Programming Language :: Python :: 3.6", 53 | "Programming Language :: Python :: 3.7", 54 | "Topic :: Internet :: WWW/HTTP", 55 | ], 56 | ) 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # Local 132 | dropin.cache 133 | .env* 134 | spreadsheetui/static/ 135 | -------------------------------------------------------------------------------- /spreadsheetui/migrations/0005_auto_20200607_0957.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2020-06-07 09:57 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ("spreadsheetui", "0004_job_blocking_job"), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name="job", 15 | name="blocking_job", 16 | ), 17 | migrations.AlterField( 18 | model_name="job", 19 | name="action", 20 | field=models.CharField( 21 | choices=[ 22 | ("start", "Start"), 23 | ("stop", "Stop"), 24 | ("remove", "Remove"), 25 | ("move", "Move"), 26 | ], 27 | max_length=20, 28 | ), 29 | ), 30 | migrations.AlterField( 31 | model_name="torrent", 32 | name="added", 33 | field=models.DateTimeField(db_index=True), 34 | ), 35 | migrations.AlterField( 36 | model_name="torrent", 37 | name="download_rate", 38 | field=models.BigIntegerField(db_index=True), 39 | ), 40 | migrations.AlterField( 41 | model_name="torrent", 42 | name="size", 43 | field=models.BigIntegerField(db_index=True), 44 | ), 45 | migrations.AlterField( 46 | model_name="torrent", 47 | name="state", 48 | field=models.CharField(db_index=True, max_length=20), 49 | ), 50 | migrations.AlterField( 51 | model_name="torrent", 52 | name="tracker", 53 | field=models.CharField(db_index=True, max_length=200), 54 | ), 55 | migrations.AlterField( 56 | model_name="torrent", 57 | name="upload_rate", 58 | field=models.BigIntegerField(db_index=True), 59 | ), 60 | migrations.AlterField( 61 | model_name="torrent", 62 | name="uploaded", 63 | field=models.BigIntegerField(db_index=True), 64 | ), 65 | migrations.AlterField( 66 | model_name="torrentclient", 67 | name="client_type", 68 | field=models.CharField( 69 | choices=[ 70 | ("deluge", "Deluge"), 71 | ("rtorrent", "rtorrent"), 72 | ("transmission", "Transmission"), 73 | ("fakeclient", "FakeClient"), 74 | ("qbittorrent", "qBittorrent"), 75 | ("liltorrent", "LilTorrent"), 76 | ], 77 | max_length=30, 78 | ), 79 | ), 80 | ] 81 | -------------------------------------------------------------------------------- /spreadsheetui/migrations/0002_auto_20200522_0752.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2020-05-22 07:52 2 | 3 | import django.db.models.deletion 4 | import jsonfield.fields 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ("spreadsheetui", "0001_initial"), 12 | ] 13 | 14 | operations = [ 15 | migrations.AlterField( 16 | model_name="torrentclient", 17 | name="client_type", 18 | field=models.CharField( 19 | choices=[ 20 | ("deluge", "Deluge"), 21 | ("rtorrent", "rtorrent"), 22 | ("transmission", "Transmission"), 23 | ("fakeclient", "Fake Client"), 24 | ], 25 | max_length=30, 26 | ), 27 | ), 28 | migrations.CreateModel( 29 | name="Job", 30 | fields=[ 31 | ( 32 | "id", 33 | models.AutoField( 34 | auto_created=True, 35 | primary_key=True, 36 | serialize=False, 37 | verbose_name="ID", 38 | ), 39 | ), 40 | ( 41 | "action", 42 | models.CharField( 43 | choices=[("start", "Start"), ("stop", "Stop")], max_length=20 44 | ), 45 | ), 46 | ("config", jsonfield.fields.JSONField(blank=True, default={})), 47 | ("execute_start_time", models.DateTimeField(null=True)), 48 | ("created", models.DateTimeField(auto_now_add=True)), 49 | ( 50 | "source_client", 51 | models.ForeignKey( 52 | null=True, 53 | on_delete=django.db.models.deletion.CASCADE, 54 | related_name="+", 55 | to="spreadsheetui.TorrentClient", 56 | ), 57 | ), 58 | ( 59 | "target_client", 60 | models.ForeignKey( 61 | null=True, 62 | on_delete=django.db.models.deletion.CASCADE, 63 | related_name="+", 64 | to="spreadsheetui.TorrentClient", 65 | ), 66 | ), 67 | ( 68 | "torrent", 69 | models.ForeignKey( 70 | null=True, 71 | on_delete=django.db.models.deletion.CASCADE, 72 | related_name="+", 73 | to="spreadsheetui.Torrent", 74 | ), 75 | ), 76 | ], 77 | ), 78 | ] 79 | -------------------------------------------------------------------------------- /spreadsheetui/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2020-05-10 12:49 2 | 3 | import django.db.models.deletion 4 | import jsonfield.fields 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | initial = True 11 | 12 | dependencies = [] 13 | 14 | operations = [ 15 | migrations.CreateModel( 16 | name="TorrentClient", 17 | fields=[ 18 | ( 19 | "id", 20 | models.AutoField( 21 | auto_created=True, 22 | primary_key=True, 23 | serialize=False, 24 | verbose_name="ID", 25 | ), 26 | ), 27 | ("name", models.CharField(max_length=100, unique=True)), 28 | ("display_name", models.CharField(max_length=100)), 29 | ( 30 | "client_type", 31 | models.CharField( 32 | choices=[ 33 | ("deluge", "Deluge"), 34 | ("rtorrent", "rtorrent"), 35 | ("transmission", "Transmission"), 36 | ], 37 | max_length=30, 38 | ), 39 | ), 40 | ("config", jsonfield.fields.JSONField()), 41 | ("enabled", models.BooleanField(default=True)), 42 | ], 43 | ), 44 | migrations.CreateModel( 45 | name="Torrent", 46 | fields=[ 47 | ( 48 | "id", 49 | models.AutoField( 50 | auto_created=True, 51 | primary_key=True, 52 | serialize=False, 53 | verbose_name="ID", 54 | ), 55 | ), 56 | ("infohash", models.CharField(max_length=40)), 57 | ("name", models.CharField(max_length=1000)), 58 | ("size", models.BigIntegerField()), 59 | ("state", models.CharField(max_length=20)), 60 | ("progress", models.DecimalField(decimal_places=2, max_digits=5)), 61 | ("uploaded", models.BigIntegerField()), 62 | ("tracker", models.CharField(max_length=200)), 63 | ("added", models.DateTimeField()), 64 | ("upload_rate", models.BigIntegerField()), 65 | ("download_rate", models.BigIntegerField()), 66 | ("label", models.CharField(blank=True, default="", max_length=1000)), 67 | ( 68 | "ratio", 69 | models.DecimalField(decimal_places=3, default=0.0, max_digits=8), 70 | ), 71 | ( 72 | "torrent_client", 73 | models.ForeignKey( 74 | on_delete=django.db.models.deletion.CASCADE, 75 | to="spreadsheetui.TorrentClient", 76 | ), 77 | ), 78 | ], 79 | options={ 80 | "unique_together": {("torrent_client", "infohash")}, 81 | }, 82 | ), 83 | ] 84 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ================================ 2 | Spreadsheet UI 3 | ================================ 4 | 5 | This is a web frontend that can combine multiple torrent clients into one. 6 | Designed for the power-user who wants an overview and manage existing torrents. 7 | 8 | The interface is based on ag-grid to provide stellar performance even with a large amount of torrents. 9 | As a plus it will also look a bit like a spreadsheet. 10 | 11 | .. image:: spreadsheetui.png 12 | 13 | Requirements 14 | -------------------------------- 15 | 16 | * Python 3.7 or higher 17 | 18 | 19 | Installation 20 | -------------------------------- 21 | 22 | Linux 23 | ```````````````````````````````` 24 | 25 | .. code-block:: bash 26 | 27 | # Create a folder to put it all into 28 | mkdir spreadsheetui 29 | cd spreadsheetui 30 | 31 | # Create a python virtual environment and install spreadsheetui into it 32 | python3 -m venv env 33 | env/bin/pip install spreadsheetui 34 | 35 | # Download an example config file, remember to modify it 36 | curl -L -o config.toml https://github.com/JohnDoee/spreadsheetui/raw/master/config.toml.example 37 | 38 | # Start the UI 39 | env/bin/twistd spreadsheetui 40 | 41 | 42 | Docker 43 | ```````````````````````````````` 44 | 45 | .. code-block:: bash 46 | 47 | # Create a folder to put it all into 48 | mkdir spreadsheetui 49 | cd spreadsheetui 50 | 51 | # Download an example config file, remember to modify it 52 | curl -L -o config.toml https://github.com/JohnDoee/spreadsheetui/raw/master/config.toml.example 53 | 54 | curl -L -o docker-compose.yml https://github.com/JohnDoee/spreadsheetui/raw/master/docker-compose.yml 55 | docker-compose up -d 56 | 57 | 58 | Configuration 59 | -------------------------------- 60 | 61 | Edit config.toml to fit your needs. 62 | 63 | Remember to change username, password. The secret_key should also be changed, anything random will do. 64 | 65 | You can add as many clients as you want, see the provided examples for syntax. 66 | 67 | When you are done and have started Spreadsheet UI, it is accessible on port 18816 68 | 69 | If you need to use a proxy layer to access your client (e.g. with Docker) `check out liltorrent `_. 70 | 71 | Moving torrents 72 | -------------------------------- 73 | 74 | For torrent moving to work, the `session_path` must be correct, `see more about session_path here `_ 75 | 76 | Executing jobs 77 | -------------------------------- 78 | 79 | Jobs in SpreadsheetUI works by using a jobqueue. This means jobs are not executed before you trigger them so you can 80 | make sure all your scheduled jobs are as you want then to be. 81 | 82 | How to run a job 83 | 84 | * Select some torrents 85 | * Right-click and select the action you want (e.g. stop) 86 | * Right-click and click "Goto Jobqueue" 87 | * Verify your scheduled jobs 88 | * Right-click and click "Execute queued jobs" 89 | 90 | Your jobs will now slowly be executed in order. 91 | 92 | To go back to the dashboard: Right-click and click "Goto Dashboard" 93 | 94 | The changes might be slow to show up in your torrent list. 95 | If you want to force a full update, right-click and click "Run full update" 96 | 97 | Features 98 | -------------------------------- 99 | 100 | Clients: 101 | 102 | * rtorrent 103 | * Deluge 104 | * Transmission 105 | * qBittorrent 106 | 107 | Methods: 108 | 109 | * List all torrents 110 | * Start / stop torrents 111 | * Move torrents between clients 112 | 113 | Logo / icon 114 | -------------------------------- 115 | 116 | spreadsheet by Adrien Coquet from the Noun Project 117 | 118 | License 119 | --------------------------------- 120 | 121 | MIT -------------------------------------------------------------------------------- /main/settings.py: -------------------------------------------------------------------------------- 1 | import threading 2 | 3 | import environ 4 | 5 | env = environ.Env(DEBUG=(bool, False)) 6 | # reading .env file 7 | environ.Env.read_env(env.str("ENV_PATH", ".env")) 8 | 9 | 10 | # Quick-start development settings - unsuitable for production 11 | # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ 12 | 13 | # SECURITY WARNING: keep the secret key used in production secret! 14 | SECRET_KEY = env("SECRET_KEY") 15 | 16 | # SECURITY WARNING: don't run with debug turned on in production! 17 | DEBUG = env("DEBUG") 18 | 19 | ALLOWED_HOSTS = ["*"] 20 | 21 | 22 | # Application definition 23 | 24 | INSTALLED_APPS = [ 25 | "django.contrib.admin", 26 | "django.contrib.auth", 27 | "django.contrib.contenttypes", 28 | "django.contrib.sessions", 29 | "django.contrib.messages", 30 | "django.contrib.staticfiles", 31 | "rest_framework", 32 | "django_filters", 33 | "spreadsheetui", 34 | ] 35 | 36 | MIDDLEWARE = [ 37 | "django.middleware.security.SecurityMiddleware", 38 | "django.contrib.sessions.middleware.SessionMiddleware", 39 | "django.middleware.common.CommonMiddleware", 40 | "django.middleware.csrf.CsrfViewMiddleware", 41 | "django.contrib.auth.middleware.AuthenticationMiddleware", 42 | "django.contrib.messages.middleware.MessageMiddleware", 43 | "django.middleware.clickjacking.XFrameOptionsMiddleware", 44 | ] 45 | 46 | ROOT_URLCONF = "main.urls" 47 | 48 | TEMPLATES = [ 49 | { 50 | "BACKEND": "django.template.backends.django.DjangoTemplates", 51 | "DIRS": [], 52 | "APP_DIRS": True, 53 | "OPTIONS": { 54 | "context_processors": [ 55 | "django.template.context_processors.debug", 56 | "django.template.context_processors.request", 57 | "django.contrib.auth.context_processors.auth", 58 | "django.contrib.messages.context_processors.messages", 59 | ], 60 | }, 61 | }, 62 | ] 63 | 64 | WSGI_APPLICATION = "main.wsgi.application" 65 | 66 | 67 | # Database 68 | # https://docs.djangoproject.com/en/3.0/ref/settings/#databases 69 | 70 | DATABASES = { 71 | "default": env.db(), 72 | } 73 | 74 | 75 | # Password validation 76 | # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators 77 | 78 | AUTH_PASSWORD_VALIDATORS = [ 79 | { 80 | "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", 81 | }, 82 | { 83 | "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", 84 | }, 85 | { 86 | "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", 87 | }, 88 | { 89 | "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", 90 | }, 91 | ] 92 | 93 | 94 | # Internationalization 95 | # https://docs.djangoproject.com/en/3.0/topics/i18n/ 96 | 97 | LANGUAGE_CODE = "en-us" 98 | 99 | TIME_ZONE = "UTC" 100 | 101 | USE_I18N = True 102 | 103 | USE_L10N = True 104 | 105 | USE_TZ = True 106 | 107 | 108 | # Static files (CSS, JavaScript, Images) 109 | # https://docs.djangoproject.com/en/3.0/howto/static-files/ 110 | 111 | STATIC_URL = "/static/" 112 | STATIC_ROOT = "./static/" 113 | 114 | ASGI_APPLICATION = "main.routing.application" 115 | 116 | REST_FRAMEWORK = { 117 | "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", 118 | "PAGE_SIZE": 100, 119 | "DEFAULT_FILTER_BACKENDS": ("django_filters.rest_framework.DjangoFilterBackend",), 120 | "DEFAULT_AUTHENTICATION_CLASSES": [ 121 | "rest_framework.authentication.SessionAuthentication", 122 | ], 123 | } 124 | 125 | TORRENT_UPDATE_PARTIAL_DELAY = 4 126 | TORRENT_UPDATE_FULL_DELAY = 300 127 | DATABASE_LOCK = threading.Lock() 128 | SCHEDULER_SERVICE = None 129 | EXECUTE_JOBS_SERVICE = None 130 | TORRENT_CLIENT_UPDATERS = [] 131 | -------------------------------------------------------------------------------- /spreadsheetui/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from jsonfield import JSONField 3 | from libtc import TORRENT_CLIENT_MAPPING, move_torrent, parse_libtc_url 4 | 5 | 6 | class TorrentClient(models.Model): 7 | name = models.CharField(max_length=100, unique=True) 8 | display_name = models.CharField(max_length=100) 9 | 10 | client_type = models.CharField( 11 | max_length=30, 12 | choices=( 13 | (c.identifier, c.display_name) for c in TORRENT_CLIENT_MAPPING.values() 14 | ), 15 | null=True, 16 | blank=True, 17 | ) 18 | config = JSONField() 19 | enabled = models.BooleanField(default=True) 20 | 21 | def get_client(self): 22 | client_url = self.config.pop("client_url", None) 23 | if client_url: 24 | return parse_libtc_url(client_url) 25 | else: 26 | client_cls = TORRENT_CLIENT_MAPPING[self.client_type] 27 | return client_cls(**self.config) 28 | 29 | def __repr__(self): 30 | return f"TorrentClient(name={self.name!r}, client_type={self.client_type!r}, enabled={self.enabled!r})" 31 | 32 | 33 | class Torrent(models.Model): 34 | torrent_client = models.ForeignKey(TorrentClient, on_delete=models.CASCADE) 35 | infohash = models.CharField(max_length=40) 36 | name = models.CharField(max_length=1000) 37 | size = models.BigIntegerField(db_index=True) 38 | state = models.CharField(max_length=20, db_index=True) 39 | progress = models.DecimalField(max_digits=5, decimal_places=2) 40 | uploaded = models.BigIntegerField(db_index=True) 41 | tracker = models.CharField(max_length=200, db_index=True) 42 | added = models.DateTimeField(db_index=True) 43 | upload_rate = models.BigIntegerField(db_index=True) 44 | download_rate = models.BigIntegerField(db_index=True) 45 | label = models.CharField(max_length=1000, default="", blank=True) 46 | ratio = models.DecimalField(max_digits=8, decimal_places=3, default=0.0) 47 | 48 | class Meta: 49 | unique_together = (("torrent_client", "infohash"),) 50 | 51 | 52 | class Job(models.Model): 53 | ACTION_START = "start" 54 | ACTION_STOP = "stop" 55 | ACTION_REMOVE = "remove" 56 | ACTION_MOVE = "move" 57 | action = models.CharField( 58 | max_length=20, 59 | choices=( 60 | (ACTION_START, "Start"), 61 | (ACTION_STOP, "Stop"), 62 | (ACTION_REMOVE, "Remove"), 63 | (ACTION_MOVE, "Move"), 64 | ), 65 | ) 66 | torrent = models.ForeignKey( 67 | Torrent, on_delete=models.CASCADE, null=True, related_name="+" 68 | ) 69 | target_client = models.ForeignKey( 70 | TorrentClient, on_delete=models.CASCADE, null=True, related_name="+" 71 | ) 72 | config = JSONField(default={}, blank=True) 73 | 74 | can_execute = models.BooleanField(default=False) 75 | execute_start_time = models.DateTimeField(null=True) 76 | created = models.DateTimeField(auto_now_add=True) 77 | 78 | def execute(self): 79 | if self.torrent.torrent_client == self.target_client: 80 | return 81 | 82 | if self.action == self.ACTION_START: 83 | client = self.torrent.torrent_client.get_client() 84 | client.start(self.torrent.infohash) 85 | elif self.action == self.ACTION_STOP: 86 | client = self.torrent.torrent_client.get_client() 87 | client.stop(self.torrent.infohash) 88 | elif self.action == self.ACTION_REMOVE: 89 | client = self.torrent.torrent_client.get_client() 90 | client.remove(self.torrent.infohash) 91 | elif self.action == self.ACTION_MOVE: 92 | client = self.torrent.torrent_client.get_client() 93 | target_client = self.target_client.get_client() 94 | fast_resume = self.torrent.progress == 100.0 95 | move_torrent( 96 | self.torrent.infohash, client, target_client, fast_resume=fast_resume 97 | ) 98 | -------------------------------------------------------------------------------- /wait-for-it.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Use this script to test if a given TCP host/port are available 3 | 4 | cmdname=$(basename $0) 5 | 6 | echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } 7 | 8 | usage() 9 | { 10 | cat << USAGE >&2 11 | Usage: 12 | $cmdname host:port [-s] [-t timeout] [-- command args] 13 | -h HOST | --host=HOST Host or IP under test 14 | -p PORT | --port=PORT TCP port under test 15 | Alternatively, you specify the host and port as host:port 16 | -s | --strict Only execute subcommand if the test succeeds 17 | -q | --quiet Don't output any status messages 18 | -t TIMEOUT | --timeout=TIMEOUT 19 | Timeout in seconds, zero for no timeout 20 | -- COMMAND ARGS Execute command with args after the test finishes 21 | USAGE 22 | exit 1 23 | } 24 | 25 | wait_for() 26 | { 27 | if [[ $TIMEOUT -gt 0 ]]; then 28 | echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" 29 | else 30 | echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" 31 | fi 32 | start_ts=$(date +%s) 33 | while : 34 | do 35 | if [[ $ISBUSY -eq 1 ]]; then 36 | nc -z $HOST $PORT 37 | result=$? 38 | else 39 | (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 40 | result=$? 41 | fi 42 | if [[ $result -eq 0 ]]; then 43 | end_ts=$(date +%s) 44 | echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" 45 | break 46 | fi 47 | sleep 1 48 | done 49 | return $result 50 | } 51 | 52 | wait_for_wrapper() 53 | { 54 | # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 55 | if [[ $QUIET -eq 1 ]]; then 56 | timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & 57 | else 58 | timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & 59 | fi 60 | PID=$! 61 | trap "kill -INT -$PID" INT 62 | wait $PID 63 | RESULT=$? 64 | if [[ $RESULT -ne 0 ]]; then 65 | echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" 66 | fi 67 | return $RESULT 68 | } 69 | 70 | # process arguments 71 | while [[ $# -gt 0 ]] 72 | do 73 | case "$1" in 74 | *:* ) 75 | hostport=(${1//:/ }) 76 | HOST=${hostport[0]} 77 | PORT=${hostport[1]} 78 | shift 1 79 | ;; 80 | --child) 81 | CHILD=1 82 | shift 1 83 | ;; 84 | -q | --quiet) 85 | QUIET=1 86 | shift 1 87 | ;; 88 | -s | --strict) 89 | STRICT=1 90 | shift 1 91 | ;; 92 | -h) 93 | HOST="$2" 94 | if [[ $HOST == "" ]]; then break; fi 95 | shift 2 96 | ;; 97 | --host=*) 98 | HOST="${1#*=}" 99 | shift 1 100 | ;; 101 | -p) 102 | PORT="$2" 103 | if [[ $PORT == "" ]]; then break; fi 104 | shift 2 105 | ;; 106 | --port=*) 107 | PORT="${1#*=}" 108 | shift 1 109 | ;; 110 | -t) 111 | TIMEOUT="$2" 112 | if [[ $TIMEOUT == "" ]]; then break; fi 113 | shift 2 114 | ;; 115 | --timeout=*) 116 | TIMEOUT="${1#*=}" 117 | shift 1 118 | ;; 119 | --) 120 | shift 121 | CLI=("$@") 122 | break 123 | ;; 124 | --help) 125 | usage 126 | ;; 127 | *) 128 | echoerr "Unknown argument: $1" 129 | usage 130 | ;; 131 | esac 132 | done 133 | 134 | if [[ "$HOST" == "" || "$PORT" == "" ]]; then 135 | echoerr "Error: you need to provide a host and port to test." 136 | usage 137 | fi 138 | 139 | TIMEOUT=${TIMEOUT:-15} 140 | STRICT=${STRICT:-0} 141 | CHILD=${CHILD:-0} 142 | QUIET=${QUIET:-0} 143 | 144 | # check to see if timeout is from busybox? 145 | # check to see if timeout is from busybox? 146 | TIMEOUT_PATH=$(realpath $(which timeout)) 147 | if [[ $TIMEOUT_PATH =~ "busybox" ]]; then 148 | ISBUSY=1 149 | BUSYTIMEFLAG="-t" 150 | else 151 | ISBUSY=0 152 | BUSYTIMEFLAG="" 153 | fi 154 | 155 | if [[ $CHILD -gt 0 ]]; then 156 | wait_for 157 | RESULT=$? 158 | exit $RESULT 159 | else 160 | if [[ $TIMEOUT -gt 0 ]]; then 161 | wait_for_wrapper 162 | RESULT=$? 163 | else 164 | wait_for 165 | RESULT=$? 166 | fi 167 | fi 168 | 169 | if [[ $CLI != "" ]]; then 170 | if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then 171 | echoerr "$cmdname: strict mode, refusing to execute subprocess" 172 | exit $RESULT 173 | fi 174 | exec "${CLI[@]}" 175 | else 176 | exit $RESULT 177 | fi 178 | -------------------------------------------------------------------------------- /spreadsheetui/views.py: -------------------------------------------------------------------------------- 1 | from django.conf import settings 2 | from django.db.models import Count, Sum 3 | from django_filters import rest_framework as filters 4 | from rest_framework import serializers, viewsets 5 | from rest_framework.decorators import action 6 | from rest_framework.response import Response 7 | 8 | from .models import Job, Torrent, TorrentClient 9 | 10 | 11 | class TorrentSerializer(serializers.ModelSerializer): 12 | torrent_client = serializers.CharField(source="torrent_client.display_name") 13 | torrent_client_id = serializers.IntegerField(source="torrent_client.id") 14 | 15 | class Meta: 16 | model = Torrent 17 | fields = ( 18 | "id", 19 | "infohash", 20 | "name", 21 | "torrent_client", 22 | "torrent_client_id", 23 | "size", 24 | "state", 25 | "progress", 26 | "uploaded", 27 | "upload_rate", 28 | "download_rate", 29 | "tracker", 30 | "added", 31 | "ratio", 32 | "label", 33 | ) 34 | 35 | 36 | class CharInFilter(filters.BaseInFilter, filters.CharFilter): 37 | pass 38 | 39 | 40 | class TorrentFilter(filters.FilterSet): 41 | o = filters.OrderingFilter( 42 | fields=( 43 | ("id", "id"), 44 | ("name", "name"), 45 | ("uploaded", "uploaded"), 46 | ("ratio", "ratio"), 47 | ("size", "size"), 48 | ("upload_rate", "upload_rate"), 49 | ("download_rate", "download_rate"), 50 | ("added", "added"), 51 | ), 52 | ) 53 | torrent_client__in = CharInFilter( 54 | field_name="torrent_client__name", lookup_expr="in" 55 | ) 56 | state__in = CharInFilter(field_name="state", lookup_expr="in") 57 | 58 | class Meta: 59 | model = Torrent 60 | fields = {"name": ["exact", "icontains"]} 61 | 62 | 63 | class TorrentViewSet(viewsets.ReadOnlyModelViewSet): 64 | queryset = Torrent.objects.filter(torrent_client__enabled=True).prefetch_related( 65 | "torrent_client" 66 | ) 67 | serializer_class = TorrentSerializer 68 | filterset_class = TorrentFilter 69 | 70 | @action(detail=False, methods=["post"]) 71 | def schedule_full_update(self, request): 72 | for updater in settings.TORRENT_CLIENT_UPDATERS: 73 | updater.schedule_full_update() 74 | return Response({"status": "success", "message": "Full update scheduled"}) 75 | 76 | @action(detail=False, methods=["get"]) 77 | def stats(self, request): 78 | return Response( 79 | self.queryset.aggregate( 80 | total_torrents=Count("id"), 81 | total_size=Sum("size"), 82 | total_uploaded=Sum("uploaded"), 83 | total_upload_rate=Sum("upload_rate"), 84 | total_download_rate=Sum("download_rate"), 85 | ) 86 | ) 87 | 88 | @action(detail=False, methods=["get"]) 89 | def aggregated(self, request): 90 | f = self.filterset_class(request.query_params, queryset=self.get_queryset()) 91 | result = dict( 92 | f.qs.aggregate( 93 | name=Count("id"), 94 | size=Sum("size"), 95 | uploaded=Sum("uploaded"), 96 | upload_rate=Sum("upload_rate"), 97 | download_rate=Sum("download_rate"), 98 | ) 99 | ) 100 | return Response(result) 101 | 102 | 103 | class TorrentClientSerializer(serializers.ModelSerializer): 104 | class Meta: 105 | model = TorrentClient 106 | fields = ( 107 | "id", 108 | "name", 109 | "display_name", 110 | ) 111 | 112 | 113 | class TorrentClientViewSet(viewsets.ReadOnlyModelViewSet): 114 | queryset = TorrentClient.objects.filter(enabled=True) 115 | serializer_class = TorrentClientSerializer 116 | 117 | 118 | class JobSerializer(serializers.ModelSerializer): 119 | torrent = serializers.CharField(source="torrent.name") 120 | source_client = serializers.CharField( 121 | source="torrent.torrent_client.display_name", allow_null=True 122 | ) 123 | target_client = serializers.CharField( 124 | source="target_client.display_name", allow_null=True 125 | ) 126 | 127 | class Meta: 128 | model = Job 129 | fields = ( 130 | "id", 131 | "action", 132 | "torrent", 133 | "source_client", 134 | "target_client", 135 | "can_execute", 136 | "execute_start_time", 137 | ) 138 | 139 | 140 | class AddJobSerializer(serializers.ModelSerializer): 141 | class Meta: 142 | model = Job 143 | fields = ("action", "torrent", "target_client", "config") 144 | 145 | 146 | class JobViewSet(viewsets.ReadOnlyModelViewSet): 147 | queryset = Job.objects.all().prefetch_related( 148 | "target_client", "torrent", "torrent__torrent_client" 149 | ) 150 | serializer_class = JobSerializer 151 | 152 | @action(detail=False, methods=["POST"]) 153 | def submit_actions(self, request): 154 | serializers = AddJobSerializer(data=request.data, many=True) 155 | serializers.is_valid(raise_exception=True) 156 | serializers.save() 157 | return Response({"status": "success", "message": "Jobs queued"}) 158 | 159 | @action(detail=False, methods=["POST"]) 160 | def wipe_all_actions(self, request): 161 | Job.objects.all().delete() 162 | return Response({"status": "success", "message": "All jobs wiped"}) 163 | 164 | @action(detail=False, methods=["POST"]) 165 | def execute_all_jobs(self, request): 166 | Job.objects.all().update(can_execute=True) 167 | if settings.SCHEDULER_SERVICE: 168 | settings.SCHEDULER_SERVICE.scheduler.add_job( 169 | settings.EXECUTE_JOBS_SERVICE.cycle, 170 | "interval", 171 | id="execute_jobs_job", 172 | seconds=1, 173 | ) 174 | 175 | return Response( 176 | {"status": "success", "message": "All jobs queued for execution"} 177 | ) 178 | -------------------------------------------------------------------------------- /spreadsheetui/tasks.py: -------------------------------------------------------------------------------- 1 | import concurrent.futures 2 | import time 3 | 4 | import toml 5 | from django.conf import settings 6 | from django.db.models import Q 7 | from django.utils.timezone import now 8 | from loguru import logger 9 | 10 | from .exceptions import FailedToUpdateException 11 | from .models import Job, Torrent, TorrentClient 12 | 13 | 14 | def update_torrents(clients=None, partial_update=False): 15 | keys = [ 16 | "name", 17 | "size", 18 | "state", 19 | "progress", 20 | "uploaded", 21 | "tracker", 22 | "added", 23 | "upload_rate", 24 | "download_rate", 25 | "label", 26 | ] 27 | torrent_clients = TorrentClient.objects.filter(enabled=True) 28 | if clients: 29 | torrent_clients = torrent_clients.filter(name__in=clients) 30 | 31 | with concurrent.futures.ThreadPoolExecutor(max_workers=6) as executor: 32 | client_lists = {} 33 | for torrent_client in torrent_clients: 34 | client = torrent_client.get_client() 35 | if partial_update: 36 | method = client.list_active 37 | else: 38 | method = client.list 39 | client_lists[executor.submit(method)] = torrent_client 40 | 41 | for future in concurrent.futures.as_completed(client_lists): 42 | torrent_client = client_lists[future] 43 | try: 44 | torrent_datas = ( 45 | future.result() 46 | ) # TODO: handle exceptions this can throw 47 | except FailedToUpdateException: 48 | logger.warning(f"Failed to update {torrent_client}") 49 | continue 50 | existing_torrents = { 51 | t.infohash: t 52 | for t in Torrent.objects.filter(torrent_client=torrent_client) 53 | } 54 | seen_torrents = set() 55 | new_torrents = [] 56 | update_torrents = [] 57 | modified_torrent_fields = set() 58 | for torrent_data in torrent_datas: 59 | seen_torrents.add(torrent_data.infohash) 60 | if torrent_data.infohash in existing_torrents: 61 | torrent = existing_torrents[torrent_data.infohash] 62 | modified_torrent = False 63 | for key in keys: 64 | old_value = getattr(torrent, key) 65 | new_value = getattr(torrent_data, key) 66 | if old_value != new_value: 67 | modified_torrent = True 68 | modified_torrent_fields.add(key) 69 | setattr(torrent, key, new_value) 70 | if modified_torrent: 71 | update_torrents.append(torrent) 72 | else: 73 | torrent = Torrent( 74 | torrent_client=torrent_client, 75 | infohash=torrent_data.infohash, 76 | ) 77 | for key in keys: 78 | setattr(torrent, key, getattr(torrent_data, key)) 79 | new_torrents.append(torrent) 80 | 81 | torrent.ratio = torrent.uploaded / torrent.size 82 | 83 | with settings.DATABASE_LOCK: 84 | if partial_update: 85 | logger.debug(f"{torrent_client!r} Setting speeds to zero") 86 | Torrent.objects.filter(torrent_client=torrent_client).filter( 87 | Q(upload_rate__gt=0) | Q(download_rate__gt=0) 88 | ).exclude(pk__in=[t.pk for t in update_torrents]).update( 89 | upload_rate=0, download_rate=0 90 | ) 91 | 92 | logger.debug( 93 | f"{torrent_client!r} Creating {len(new_torrents)} torrents" 94 | ) 95 | Torrent.objects.bulk_create(new_torrents) 96 | 97 | if update_torrents: 98 | logger.debug( 99 | f"{torrent_client!r} Updating {len(update_torrents)} torrents with fields {modified_torrent_fields!r}" 100 | ) 101 | if "uploaded" in modified_torrent_fields: 102 | modified_torrent_fields.add("ratio") 103 | Torrent.objects.bulk_update( 104 | update_torrents, 105 | modified_torrent_fields, 106 | ) 107 | else: 108 | logger.debug(f"{torrent_client!r} No torrents to update") 109 | 110 | if not partial_update: 111 | to_delete = set(existing_torrents.keys()) - seen_torrents 112 | logger.debug( 113 | f"{torrent_client!r} Deleting {len(to_delete)} torrents" 114 | ) 115 | Torrent.objects.filter( 116 | torrent_client=torrent_client, infohash__in=to_delete 117 | ).delete() 118 | 119 | 120 | def import_config(config): 121 | seen_clients = [] 122 | for name, client_config in config["clients"].items(): 123 | seen_clients.append(name) 124 | display_name = client_config.pop("display_name", name) 125 | client_type = client_config.pop("client_type", None) 126 | TorrentClient.objects.update_or_create( 127 | name=name, 128 | defaults={ 129 | "display_name": display_name, 130 | "client_type": client_type, 131 | "config": client_config, 132 | "enabled": True, 133 | }, 134 | ) 135 | 136 | TorrentClient.objects.exclude(name__in=seen_clients).update(enabled=False) 137 | 138 | 139 | def loop_update_torrents(): 140 | last_partial, last_full = 0, 0 141 | while True: 142 | if last_full + settings.TORRENT_UPDATE_FULL_DELAY < time.monotonic(): 143 | logger.debug("Running a full update") 144 | update_torrents(partial_update=False) 145 | last_partial, last_full = time.monotonic(), time.monotonic() 146 | elif last_partial + settings.TORRENT_UPDATE_PARTIAL_DELAY < time.monotonic(): 147 | logger.debug("Running a partial update") 148 | update_torrents(partial_update=True) 149 | last_partial = time.monotonic() 150 | time.sleep(1) 151 | 152 | 153 | def execute_jobs(): 154 | while True: 155 | jobs = Job.objects.filter( 156 | can_execute=True, execute_start_time__isnull=True 157 | ).order_by("id") 158 | if not jobs: 159 | break 160 | job = jobs[0] 161 | job.execute_start_time = now() 162 | job.save(update_fields=["execute_start_time"]) 163 | logger.debug(f"Starting job {job}") 164 | 165 | job.execute() 166 | 167 | try: 168 | pass 169 | except KeyboardInterrupt: 170 | raise 171 | except: 172 | logger.exception(f"Failed to execute job {job}") 173 | 174 | logger.debug(f"Finished job {job}") 175 | job.delete() 176 | -------------------------------------------------------------------------------- /twisted/plugins/spreadsheetui_plugin.py: -------------------------------------------------------------------------------- 1 | from timeoutthreadpoolexecutor import TimeoutThreadPoolExecutor # isort:skip 2 | from concurrent.futures import thread # isort:skip 3 | 4 | thread.ThreadPoolExecutor = TimeoutThreadPoolExecutor # isort:skip 5 | 6 | import asyncio # isort:skip 7 | from twisted.internet import asyncioreactor 8 | asyncioreactor.install() 9 | 10 | 11 | import os 12 | import sys 13 | import threading 14 | import time 15 | from pathlib import Path 16 | 17 | import django 18 | import toml 19 | from apscheduler.schedulers.asyncio import AsyncIOScheduler 20 | from django.core.wsgi import get_wsgi_application 21 | 22 | from loguru import logger 23 | from twisted.application import service, strports 24 | from twisted.application.service import IServiceMaker 25 | from twisted.cred.checkers import ICredentialsChecker 26 | from twisted.cred.credentials import IUsernamePassword 27 | from twisted.cred.error import UnauthorizedLogin 28 | from twisted.cred.portal import IRealm, Portal 29 | from twisted.internet import defer, endpoints, threads, reactor 30 | from twisted.plugin import IPlugin 31 | from twisted.python import usage 32 | from twisted.web import resource, server, static 33 | from twisted.web.guard import BasicCredentialFactory, HTTPAuthSessionWrapper 34 | from twisted.web.resource import IResource 35 | from twisted.web.wsgi import WSGIResource 36 | from zope.interface import implementer 37 | 38 | os.environ["DJANGO_SETTINGS_MODULE"] = "main.settings" 39 | 40 | 41 | class Options(usage.Options): 42 | optFlags = [] 43 | 44 | optParameters = [ 45 | [ 46 | "config", 47 | "c", 48 | "config.toml", 49 | "Path to config file", 50 | ], 51 | ] 52 | 53 | 54 | class Root(resource.Resource): 55 | def __init__(self, wsgi_resource, index_resource): 56 | resource.Resource.__init__(self) 57 | self.wsgi_resource = wsgi_resource 58 | self.index_resource = index_resource 59 | 60 | def getChild(self, path, request): 61 | if not request.postpath: 62 | return self.index_resource.getChild(path, request) 63 | path0 = request.prepath.pop(0) 64 | request.postpath.insert(0, path0) 65 | return self.wsgi_resource 66 | 67 | 68 | class ASGIService(service.Service): 69 | def __init__(self, site, resource, description): 70 | self.site = site 71 | self.resource = resource 72 | self.description = description 73 | 74 | def startService(self): 75 | self.endpoint = endpoints.serverFromString(reactor, self.description) 76 | self.endpoint.listen(self.site) 77 | 78 | def stopService(self): 79 | self.resource.stop() 80 | 81 | 82 | class SchedulerService(service.Service): 83 | def __init__(self): 84 | self.scheduler = AsyncIOScheduler() 85 | 86 | def startService(self): 87 | self.scheduler.start() 88 | 89 | def stopService(self): 90 | self.scheduler.shutdown() 91 | 92 | 93 | class UpdateTorrentClient: 94 | def __init__(self, torrent_client, partial_update_delay, full_update_delay): 95 | self.last_partial = 0 96 | self.last_full = 0 97 | self.torrent_client = torrent_client 98 | self.partial_update_delay = partial_update_delay 99 | self.full_update_delay = full_update_delay 100 | self.lock = threading.Lock() 101 | 102 | def schedule_full_update(self): 103 | self.last_full = 0 104 | 105 | def cycle(self): 106 | from spreadsheetui.tasks import update_torrents 107 | 108 | if not self.lock.acquire(blocking=False): 109 | return 110 | try: 111 | if self.last_full + self.full_update_delay < time.monotonic(): 112 | logger.info(f"{self.torrent_client} Running a full update") 113 | update_torrents([self.torrent_client], partial_update=False) 114 | self.last_partial, self.last_full = time.monotonic(), time.monotonic() 115 | elif self.last_partial + self.partial_update_delay < time.monotonic(): 116 | logger.info(f"{self.torrent_client} Running a partial update") 117 | update_torrents([self.torrent_client], partial_update=True) 118 | self.last_partial = time.monotonic() 119 | finally: 120 | self.lock.release() 121 | 122 | 123 | class ExecuteJobs: 124 | def __init__(self): 125 | self.lock = threading.Lock() 126 | 127 | def cycle(self): 128 | from django.conf import settings 129 | 130 | from spreadsheetui.tasks import execute_jobs 131 | 132 | settings.SCHEDULER_SERVICE.scheduler.remove_job("execute_jobs_job") 133 | 134 | if not self.lock.acquire(blocking=False): 135 | return 136 | try: 137 | execute_jobs() 138 | finally: 139 | self.lock.release() 140 | 141 | 142 | class File(static.File): 143 | def directoryListing(self): 144 | return self.forbidden 145 | 146 | 147 | @implementer(IRealm) 148 | class SpreadsheetRealm: 149 | def __init__(self, resource): 150 | self._resource = resource 151 | 152 | def requestAvatar(self, avatarId, mind, *interfaces): 153 | if IResource in interfaces: 154 | return (IResource, self._resource, lambda: None) 155 | raise NotImplementedError() 156 | 157 | 158 | @implementer(ICredentialsChecker) 159 | class PasswordDictCredentialChecker: 160 | credentialInterfaces = (IUsernamePassword,) 161 | 162 | def __init__(self, passwords): 163 | self.passwords = passwords 164 | 165 | def requestAvatarId(self, credentials): 166 | matched = self.passwords.get(credentials.username, None) 167 | if matched and matched == credentials.password: 168 | return defer.succeed(credentials.username) 169 | else: 170 | return defer.fail(UnauthorizedLogin("Invalid username or password")) 171 | 172 | 173 | def wrap_with_auth(resource, passwords, realm="Auth"): 174 | """ 175 | @param resource: resource to protect 176 | @param passwords: a dict-like object mapping usernames to passwords 177 | """ 178 | portal = Portal( 179 | SpreadsheetRealm(resource), [PasswordDictCredentialChecker(passwords)] 180 | ) 181 | credentialFactory = BasicCredentialFactory(realm) 182 | return HTTPAuthSessionWrapper(portal, [credentialFactory]) 183 | 184 | 185 | @implementer(IServiceMaker, IPlugin) 186 | class ServiceMaker(object): 187 | tapname = "spreadsheetui" 188 | description = "Spreadsheet UI" 189 | options = Options 190 | 191 | def makeService(self, options): 192 | config = toml.load(options["config"]) 193 | if not os.environ.get("DATABASE_URL"): 194 | os.environ["DATABASE_URL"] = config["django"]["database_url"] 195 | os.environ["SECRET_KEY"] = config["django"]["secret_key"] 196 | logger.remove(0) 197 | logger.add(sys.stdout, level="INFO") 198 | 199 | multi = service.MultiService() 200 | 201 | scheduler_service = SchedulerService() 202 | multi.addService(scheduler_service) 203 | 204 | django.setup() 205 | application = get_wsgi_application() 206 | from django.conf import settings 207 | 208 | settings.SCHEDULER_SERVICE = scheduler_service 209 | settings.EXECUTE_JOBS_SERVICE = ExecuteJobs() 210 | 211 | from django.core import management 212 | 213 | management.call_command("migrate") 214 | management.call_command("collectstatic", "--no-input") 215 | from spreadsheetui.tasks import import_config 216 | 217 | import_config(config) 218 | 219 | wsgiresource = WSGIResource(reactor, reactor.getThreadPool(), application) 220 | root = Root(wsgiresource, File(Path(settings.STATIC_ROOT) / "spreadsheetui")) 221 | root.putChild( 222 | settings.STATIC_URL.strip("/").encode("utf-8"), 223 | File(settings.STATIC_ROOT.encode("utf-8")), 224 | ) 225 | root.putChild( 226 | b"assets", 227 | File(Path(settings.STATIC_ROOT) / "spreadsheetui" / "assets"), 228 | ) 229 | site = server.Site( 230 | wrap_with_auth( 231 | root, 232 | { 233 | config["spreadsheetui"]["username"] 234 | .encode("utf-8"): config["spreadsheetui"]["password"] 235 | .encode("utf-8") 236 | }, 237 | ) 238 | ) 239 | 240 | multi.addService( 241 | strports.service(config["spreadsheetui"]["endpoint_description"], site) 242 | ) 243 | 244 | def twisted_started(): 245 | def cleanup_thread(): 246 | from django.db import close_old_connections 247 | 248 | close_old_connections() 249 | 250 | scheduler_service.scheduler.add_job(cleanup_thread, "interval", minutes=15) 251 | 252 | def initiate_torrent_clients(): 253 | from spreadsheetui.models import TorrentClient 254 | 255 | settings.TORRENT_CLIENT_UPDATERS = [] 256 | 257 | for torrent_client in TorrentClient.objects.filter(enabled=True): 258 | utc = UpdateTorrentClient( 259 | torrent_client.name, 260 | settings.TORRENT_UPDATE_PARTIAL_DELAY, 261 | settings.TORRENT_UPDATE_FULL_DELAY, 262 | ) 263 | settings.TORRENT_CLIENT_UPDATERS.append(utc) 264 | 265 | scheduler_service.scheduler.add_job( 266 | utc.cycle, "interval", seconds=2, max_instances=2 267 | ) 268 | 269 | threads.deferToThread(initiate_torrent_clients) 270 | 271 | reactor.callLater(0, twisted_started) 272 | 273 | return multi 274 | 275 | 276 | spreadsheetui = ServiceMaker() 277 | --------------------------------------------------------------------------------