├── .github ├── ISSUE_TEMPLATE │ └── issue.md └── workflows │ ├── Run_Tests.yml │ └── workflow.yml ├── .gitignore ├── .gitlab-ci.yml ├── Dockerfile ├── LICENSE ├── MangaTagger.py ├── MangaTaggerLib ├── MangaTaggerLib.py ├── __init__.py ├── _version.py ├── api.py ├── database.py ├── errors.py ├── models.py ├── task_queue.py └── utils.py ├── README.md ├── images └── manga_tagger_logo_cropped.png ├── requirements.txt ├── root └── etc │ ├── cont-init.d │ └── 30-config │ └── services.d │ └── mangatagger │ └── run ├── settings.json └── tests ├── __init__.py ├── data ├── 3D Kanojo Real Girl │ └── data.json ├── BLEACH │ └── data.json ├── Hurejasik │ └── data.json └── Naruto │ └── data.json ├── database.py ├── test_integration.py └── test_manga.py /.github/ISSUE_TEMPLATE/issue.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Issue 3 | about: Log bugs found while using Manga Tagger 4 | title: "[BUG]" 5 | labels: bug 6 | assignees: Inpacchi 7 | 8 | --- 9 | 10 | **OS**: What operating system are you running Manga Tagger on? 11 | 12 | **Manga Tagger Version:** What version of Manga Tagger are you running? 13 | 14 | **Description of the issue:** Give a a clear and concise description of what the issue is. 15 | 16 | **Manga Series:** What manga series (singular or plural) were being tagged when the issue was encountered? 17 | 18 | **Screenshots (If applicable):** Add screenshots to help explain your problem. 19 | 20 | **Log File**: Please make sure JSON logging is enabled and upload the JSON log file. 21 | -------------------------------------------------------------------------------- /.github/workflows/Run_Tests.yml: -------------------------------------------------------------------------------- 1 | name: Python tests 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | build: 9 | # env: 10 | # DISPLAY: ":99.0" 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | python-version: [ '3.8','3.9', ] 15 | name: Python ${{ matrix.python-version }} sample 16 | steps: 17 | - uses: actions/checkout@v3 18 | - uses: actions/setup-python@v3 19 | with: 20 | python-version: ${{ matrix.python-version }} 21 | cache: 'pip' 22 | - run: pip install -r requirements.txt 23 | # - run: sudo apt install xvfb 24 | # - name: Start xvfb 25 | # run: | 26 | # Xvfb :99 -screen 0 1920x1080x24 &disown 27 | - name: Run the tests 28 | run: python3 -m unittest discover -s tests -t . 29 | -------------------------------------------------------------------------------- /.github/workflows/workflow.yml: -------------------------------------------------------------------------------- 1 | # This workflow file will install Python dependencies, 2 | # create a desktop, joystick, and test the application's GUI on multiple versions of Python 3 | name: Python tests & Build 4 | 5 | on: 6 | - push 7 | # - pull_request 8 | env: 9 | IMAGE_NAME: "banhcanh/manga-tagger" 10 | jobs: 11 | build: 12 | # env: 13 | # DISPLAY: ":99.0" 14 | runs-on: ubuntu-latest 15 | strategy: 16 | matrix: 17 | python-version: [ '3.8','3.9', ] 18 | name: Python ${{ matrix.python-version }} sample 19 | steps: 20 | - uses: actions/checkout@v3 21 | - uses: actions/setup-python@v3 22 | with: 23 | python-version: ${{ matrix.python-version }} 24 | cache: 'pip' 25 | - run: pip install -r requirements.txt 26 | - name: Run the tests 27 | run: python3 -m unittest discover -s tests -t . 28 | # sonarcloud: 29 | # name: SonarCloud 30 | # runs-on: ubuntu-latest 31 | # steps: 32 | # - uses: actions/checkout@v2 33 | # with: 34 | # fetch-depth: 0 # Shallow clones should be disabled for a better relevancy of analysis 35 | # - name: SonarCloud Scan 36 | # uses: SonarSource/sonarcloud-github-action@master 37 | # env: 38 | # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any 39 | # SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} 40 | docker_develop: 41 | name: Nightly Build 42 | needs: [ build ] 43 | runs-on: ubuntu-latest 44 | if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} # Only run in develop push 45 | steps: 46 | - name: Set up QEMU 47 | uses: docker/setup-qemu-action@v2 48 | - name: Set up Docker Buildx 49 | uses: docker/setup-buildx-action@v2 50 | - name: Login to DockerHub 51 | uses: docker/login-action@v2 52 | with: 53 | username: ${{ secrets.DOCKERHUB_USERNAME }} 54 | password: ${{ secrets.DOCKERHUB_TOKEN }} 55 | - name: Build and push 56 | uses: docker/build-push-action@v3 57 | with: 58 | push: true 59 | tags: ${{ env.IMAGE_NAME }}:nightly 60 | docker_stable: 61 | name: Stable Build 62 | needs: [ build ] 63 | runs-on: ubuntu-latest 64 | if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} # Only run in master push 65 | steps: 66 | - name: Set up QEMU 67 | uses: docker/setup-qemu-action@v2 68 | - name: Set up Docker Buildx 69 | uses: docker/setup-buildx-action@v2 70 | - name: Login to DockerHub 71 | uses: docker/login-action@v2 72 | with: 73 | username: ${{ secrets.DOCKERHUB_USERNAME }} 74 | password: ${{ secrets.DOCKERHUB_TOKEN }} 75 | - name: Build and push 76 | uses: docker/build-push-action@v3 77 | with: 78 | push: true 79 | tags: ${{ env.IMAGE_NAME }}:latest 80 | docker_test: 81 | name: Test Build 82 | needs: [ build ] 83 | runs-on: ubuntu-latest 84 | if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/test' }} # Only run in test push 85 | steps: 86 | - name: Set up QEMU 87 | uses: docker/setup-qemu-action@v2 88 | - name: Set up Docker Buildx 89 | uses: docker/setup-buildx-action@v2 90 | - name: Login to DockerHub 91 | uses: docker/login-action@v2 92 | with: 93 | username: ${{ secrets.DOCKERHUB_USERNAME }} 94 | password: ${{ secrets.DOCKERHUB_TOKEN }} 95 | - name: Build and push 96 | uses: docker/build-push-action@v3 97 | with: 98 | push: true 99 | tags: ${{ env.IMAGE_NAME }}:test -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # IntelliJ 132 | .idea/ 133 | 134 | # Manga Tagger 135 | library/ 136 | downloads/ 137 | wiki/ 138 | data/ 139 | logs/ 140 | cover/ 141 | manga/ 142 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - test 3 | - build 4 | 5 | MT-filename-parser: 6 | stage: test 7 | image: alpine:latest 8 | script: 9 | #"Installing dependencies..." 10 | - apk add python3 py3-pip py3-numpy py3-multidict py3-yarl py3-psutil py3-watchdog py3-requests py3-tz build-base jpeg-dev zlib-dev python3-dev 11 | - pip3 install pymongo python_json_logger image BeautifulSoup4 12 | #"Testing filename parser and renamer..." 13 | - python3 -m unittest discover -s tests -p 'test_manga.py' 14 | 15 | MT-MetadataXMLConstruct: 16 | stage: test 17 | image: alpine:latest 18 | script: 19 | #"Installing dependencies..." 20 | - apk add python3 py3-pip py3-numpy py3-multidict py3-yarl py3-psutil py3-watchdog py3-requests py3-tz build-base jpeg-dev zlib-dev python3-dev 21 | - pip3 install pymongo python_json_logger image BeautifulSoup4 22 | #"Testing metadata tagger and xml construction" 23 | - python3 -m unittest discover -s tests -p 'test_integration.py' 24 | 25 | build: 26 | stage: build 27 | image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" 28 | variables: 29 | DOCKER_TLS_CERTDIR: "" 30 | services: 31 | - docker:19.03.12-dind 32 | script: 33 | - | 34 | if [[ -z "$CI_COMMIT_TAG" ]]; then 35 | export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG} 36 | export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA} 37 | else 38 | export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE} 39 | export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG} 40 | fi 41 | - /build/build.sh 42 | rules: 43 | - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' 44 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ghcr.io/linuxserver/baseimage-alpine:3.13 2 | 3 | LABEL \ 4 | maintainer="TKVictor-Hang@outlook.fr" 5 | 6 | ### Set default Timezone, overwrite default MangaTagger settings for the container ### 7 | ENV \ 8 | TZ="Europe/Paris" \ 9 | MANGA_TAGGER_DEBUG_MODE=false \ 10 | MANGA_TAGGER_DATA_DIR="/config/data" \ 11 | MANGA_TAGGER_IMAGE_COVER=true \ 12 | MANGA_TAGGER_IMAGE_DIR="/config/cover" \ 13 | MANGA_TAGGER_ADULT_RESULT=false \ 14 | MANGA_TAGGER_DOWNLOAD_DIR="/downloads" \ 15 | MANGA_TAGGER_LIBRARY_DIR="/manga" \ 16 | MANGA_TAGGER_LOGGING_DIR="/config/logs" \ 17 | MANGA_TAGGER_DRY_RUN=false \ 18 | MANGA_TAGGER_DB_INSERT=false \ 19 | MANGA_TAGGER_RENAME_FILE=false \ 20 | MANGA_TAGGER_WRITE_COMICINFO=false \ 21 | MANGA_TAGGER_THREADS=8 \ 22 | MANGA_TAGGER_MAX_QUEUE_SIZE=0 \ 23 | MANGA_TAGGER_DB_NAME=manga_tagger \ 24 | MANGA_TAGGER_DB_HOST_ADDRESS=mangatagger-db \ 25 | MANGA_TAGGER_DB_PORT=27017 \ 26 | MANGA_TAGGER_DB_USERNAME=manga_tagger \ 27 | MANGA_TAGGER_DB_PASSWORD=Manga4LYFE \ 28 | MANGA_TAGGER_DB_AUTH_SOURCE=admin \ 29 | MANGA_TAGGER_DB_SELECTION_TIMEOUT=10000 \ 30 | MANGA_TAGGER_LOGGING_LEVEL=info \ 31 | MANGA_TAGGER_LOGGING_CONSOLE=true \ 32 | MANGA_TAGGER_LOGGING_FILE=true \ 33 | MANGA_TAGGER_LOGGING_JSON=false \ 34 | MANGA_TAGGER_LOGGING_TCP=false \ 35 | MANGA_TAGGER_LOGGING_JSONTCP=false 36 | 37 | ### Upgrade ### 38 | RUN \ 39 | apk update && apk upgrade 40 | 41 | ### Manga Tagger ### 42 | COPY . /app/Manga-Tagger 43 | 44 | RUN \ 45 | echo "Installing Manga-Tagger" 46 | 47 | COPY root/ / 48 | 49 | ### Dependencies ### 50 | RUN \ 51 | echo "Install dependencies" && \ 52 | echo "@testing http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \ 53 | apk add --no-cache --update \ 54 | python3 py3-pip py3-numpy py3-multidict py3-yarl \ 55 | py3-psutil py3-watchdog py3-requests py3-tz \ 56 | build-base jpeg-dev zlib-dev python3-dev && \ 57 | pip3 install --no-cache-dir -r /app/Manga-Tagger/requirements.txt && \ 58 | mkdir /manga && \ 59 | mkdir /downloads 60 | 61 | VOLUME /config 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Yovarni Yearwood 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MangaTagger.py: -------------------------------------------------------------------------------- 1 | from MangaTaggerLib import MangaTaggerLib 2 | 3 | if __name__ == '__main__': 4 | MangaTaggerLib.main() 5 | -------------------------------------------------------------------------------- /MangaTaggerLib/MangaTaggerLib.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | import re 4 | import requests 5 | import unicodedata 6 | import shutil 7 | import json 8 | 9 | from datetime import datetime 10 | from os import path 11 | from pathlib import Path 12 | from PIL import Image 13 | from requests.exceptions import ConnectionError 14 | from xml.etree.ElementTree import SubElement, Element, Comment, tostring 15 | from xml.dom.minidom import parseString 16 | from zipfile import ZipFile 17 | from bs4 import BeautifulSoup 18 | 19 | from MangaTaggerLib._version import __version__ 20 | from MangaTaggerLib.api import AniList, AniListRateLimit 21 | from MangaTaggerLib.database import MetadataTable, ProcFilesTable, ProcSeriesTable 22 | from MangaTaggerLib.errors import FileAlreadyProcessedError, FileUpdateNotRequiredError, UnparsableFilenameError, \ 23 | MangaNotFoundError, MangaMatchedException 24 | from MangaTaggerLib.models import Metadata 25 | from MangaTaggerLib.task_queue import QueueWorker 26 | from MangaTaggerLib.utils import AppSettings, compare 27 | 28 | # Global Variable Declaration 29 | LOG = logging.getLogger('MangaTaggerLib.MangaTaggerLib') 30 | 31 | CURRENTLY_PENDING_RENAME = set() 32 | 33 | 34 | def main(): 35 | AppSettings.load() 36 | 37 | LOG.info(f'Starting Manga Tagger - Version {__version__}') 38 | LOG.debug('RUNNING WITH DEBUG LOG') 39 | 40 | if AppSettings.mode_settings is not None: 41 | LOG.info('DRY RUN MODE ENABLED') 42 | LOG.info(f"MetadataTable Insertion: {AppSettings.mode_settings['database_insert']}") 43 | LOG.info(f"Renaming Files: {AppSettings.mode_settings['rename_file']}") 44 | LOG.info(f"Writing Comicinfo.xml: {AppSettings.mode_settings['write_comicinfo']}") 45 | 46 | QueueWorker.run() 47 | 48 | 49 | def process_manga_chapter(file_path: Path, event_id): 50 | filename = file_path.name 51 | directory_path = file_path.parent 52 | directory_name = file_path.parent.name 53 | 54 | logging_info = { 55 | 'event_id': event_id, 56 | 'manga_title': directory_name, 57 | "original_filename": filename 58 | } 59 | 60 | LOG.info(f'Now processing "{file_path}"...', extra=logging_info) 61 | 62 | LOG.debug(f'filename: {filename}') 63 | LOG.debug(f'directory_path: {directory_path}') 64 | LOG.debug(f'directory_name: {directory_name}') 65 | 66 | manga_details = filename_parser(filename, logging_info) 67 | 68 | metadata_tagger(file_path, manga_details[0], manga_details[1], manga_details[2], logging_info, manga_details[3]) 69 | 70 | # Remove manga directory if empty 71 | try: 72 | LOG.info(f'Deleting {directory_path}...') 73 | if directory_path != AppSettings.download_dir: 74 | LOG.info(f'Deleting {directory_path}...') 75 | directory_path.rmdir() 76 | except OSError as e: 77 | LOG.info("Error: %s : %s" % (directory_path, e.strerror)) 78 | 79 | 80 | def filename_parser(filename, logging_info): 81 | LOG.info(f'Attempting to rename "{filename}"...', extra=logging_info) 82 | 83 | # Parse the manga title and chapter name/number (this depends on where the manga is downloaded from) 84 | try: 85 | if filename.find('-.-') == -1: 86 | raise UnparsableFilenameError(filename, '-.-') 87 | 88 | filename = filename.split(' -.- ') 89 | LOG.info(f'Filename was successfully parsed as {filename}.', extra=logging_info) 90 | except UnparsableFilenameError as ufe: 91 | LOG.exception(ufe, extra=logging_info) 92 | return None 93 | 94 | manga_title: str = filename[0] 95 | chapter_title: str = path.splitext(filename[1].lower())[0] 96 | manga_title = manga_title.strip() 97 | LOG.debug(f'manga_title: {manga_title}') 98 | LOG.debug(f'chapter: {chapter_title}') 99 | 100 | format = "MANGA" 101 | volume = re.findall(r"(?i)(?:Vol|v|volume)(?:\s|\.)?(?:\s|\.)?([0-9]+(?:\.[0-9]+)?)", chapter_title) 102 | if volume: 103 | volume = volume[0] 104 | else: 105 | volume = None 106 | 107 | chapter = re.findall( 108 | r"(?i)(?:(?:ch|chapter|c)(?:\s|\.)?(?:\s|\.)?(?:([0-9]+(?:\.[0-9]+)?)+(?:-([0-9]+(?:\.[0-9]+)?))?))", 109 | chapter_title) 110 | if chapter: 111 | chapter = f"{chapter[0][0]}" 112 | else: 113 | chapter = None 114 | # If "chapter" is in the chapter substring 115 | try: 116 | if not hasNumbers(chapter_title): 117 | if "oneshot" in chapter_title.lower(): 118 | format = "ONE_SHOT" 119 | chapter_title = "chap000" 120 | 121 | if "prologue" in chapter_title.lower(): 122 | chapter_title = chapter_title.replace(' ', '') 123 | chapter_title = re.sub('^\D*', '', chapter_title) 124 | chapter_title = "chap000." + chapter_title 125 | 126 | chapter_title = chapter_title.replace(' ', '') 127 | chapter_title = re.sub('\(\d*\)$', '', chapter_title) 128 | # Remove (1) (2) .. because it's often redundant and mess with parsing 129 | chapter_title = re.sub('\D*$', '', chapter_title) 130 | # Removed space and any character at the end of the chapter_title that are not number. Usually that's the name of the chapter. 131 | 132 | # Match "Chapter5" "GAME005" "Page/005" "ACT-50" "#505" "V05.5CHAP5.5" without the chapter number, we removed spaces above 133 | chapter_title_pattern = "[^\d\.]\D*\d*[.,]?\d*[^\d\.]\D*" 134 | 135 | if re.match(chapter_title_pattern, chapter_title): 136 | p = re.compile(chapter_title_pattern) 137 | prog = p.match(chapter_title) 138 | chapter_title_name = prog.group(0) 139 | delimiter = chapter_title_name 140 | delimiter_index = len(chapter_title_name) 141 | else: 142 | raise UnparsableFilenameError(filename, 'ch/chapter') 143 | except UnparsableFilenameError as ufe: 144 | LOG.exception(ufe, extra=logging_info) 145 | return None 146 | 147 | LOG.debug(f'delimiter: {delimiter}') 148 | LOG.debug(f'delimiter_index: {delimiter_index}') 149 | 150 | i = chapter_title.index(delimiter) + delimiter_index 151 | LOG.debug(f'Iterator i: {i}') 152 | LOG.debug(f'Length: {len(chapter_title)}') 153 | 154 | chapter_number = '' 155 | while i < len(chapter_title): 156 | substring = chapter_title[i] 157 | LOG.debug(f'substring: {substring}') 158 | 159 | if substring.isdigit() or substring == '.': 160 | chapter_number += chapter_title[i] 161 | i += 1 162 | 163 | LOG.debug(f'chapter_number: {chapter_number}') 164 | LOG.debug(f'Iterator i: {i}') 165 | else: 166 | break 167 | 168 | if chapter_number.find('.') == -1: 169 | chapter_number = chapter_number.zfill(3) 170 | else: 171 | chapter_number = chapter_number.zfill(5) 172 | 173 | LOG.debug(f'chapter_number: {chapter_number}') 174 | 175 | logging_info['chapter_number'] = chapter_number 176 | if chapter is not None: 177 | return manga_title, chapter, format, volume 178 | else: 179 | return manga_title, chapter_number, format, volume 180 | 181 | 182 | def rename_action(current_file_path: Path, new_file_path: Path, manga_title, chapter_number, logging_info): 183 | chapter_number = chapter_number.replace('.', '-') 184 | results = ProcFilesTable.search(manga_title, chapter_number) 185 | LOG.debug(f'Results: {results}') 186 | 187 | # If the series OR the chapter has not been processed 188 | if results is None: 189 | LOG.info(f'"{manga_title}" chapter {chapter_number} has not been processed before. ' 190 | f'Proceeding with file rename...', extra=logging_info) 191 | shutil.move(current_file_path, new_file_path) 192 | LOG.info(f'"{new_file_path.name.strip(".cbz")}" has been renamed.', extra=logging_info) 193 | ProcFilesTable.insert_record(current_file_path, new_file_path, manga_title, chapter_number, 194 | logging_info) 195 | else: 196 | versions = ['v2', 'v3', 'v4', 'v5'] 197 | 198 | existing_old_filename = results['old_filename'] 199 | existing_current_filename = results['new_filename'] 200 | 201 | # If currently processing file has the same name as an existing file 202 | if existing_current_filename == new_file_path.name: 203 | # If currently processing file has a version in it's filename 204 | if any(version in current_file_path.name.lower() for version in versions): 205 | # If the version is newer than the existing file 206 | if compare_versions(existing_old_filename, current_file_path.name): 207 | LOG.info(f'Newer version of "{manga_title}" chapter {chapter_number} has been found. Deleting ' 208 | f'existing file and proceeding with file rename...', extra=logging_info) 209 | new_file_path.unlink() 210 | LOG.info(f'"{new_file_path.name}" has been deleted! Proceeding to rename new file...', 211 | extra=logging_info) 212 | shutil.move(current_file_path, new_file_path) 213 | LOG.info(f'"{new_file_path.name.strip(".cbz")}" has been renamed.', extra=logging_info) 214 | ProcFilesTable.update_record(results, current_file_path, new_file_path, logging_info) 215 | else: 216 | LOG.warning(f'"{current_file_path.name}" was not renamed due being the exact same as the ' 217 | f'existing chapter; file currently being processed will be deleted', 218 | extra=logging_info) 219 | current_file_path.unlink() 220 | raise FileUpdateNotRequiredError(current_file_path.name) 221 | # If the current file doesn't have a version in it's filename, but the existing file does 222 | elif any(version in existing_old_filename.lower() for version in versions): 223 | LOG.warning(f'"{current_file_path.name}" was not renamed due to not being an updated version ' 224 | f'of the existing chapter; file currently being processed will be deleted', 225 | extra=logging_info) 226 | current_file_path.unlink() 227 | raise FileUpdateNotRequiredError(current_file_path.name) 228 | # If all else fails 229 | else: 230 | LOG.warning(f'No changes have been found for "{existing_current_filename}"; file currently being ' 231 | f'processed will be deleted', extra=logging_info) 232 | current_file_path.unlink() 233 | raise FileAlreadyProcessedError(current_file_path.name) 234 | 235 | LOG.info(f'"{new_file_path.name}" will be unlocked for any pending processes.', extra=logging_info) 236 | CURRENTLY_PENDING_RENAME.remove(new_file_path) 237 | 238 | 239 | def compare_versions(old_filename: str, new_filename: str): 240 | old_version = 0 241 | new_version = 0 242 | 243 | LOG.debug('Preprocessing') 244 | LOG.debug(f'Old Version: {old_version}') 245 | LOG.debug(f'New Version: {new_version}') 246 | 247 | if 'v2' in old_filename.lower(): 248 | old_version = 2 249 | elif 'v3' in old_filename.lower(): 250 | old_version = 3 251 | elif 'v4' in old_filename.lower(): 252 | old_version = 4 253 | elif 'v5' in old_filename.lower(): 254 | old_version = 5 255 | 256 | if 'v2' in new_filename.lower(): 257 | new_version = 2 258 | elif 'v3' in new_filename.lower(): 259 | new_version = 3 260 | elif 'v4' in new_filename.lower(): 261 | new_version = 4 262 | elif 'v5' in new_filename.lower(): 263 | new_version = 5 264 | 265 | LOG.debug('Postprocessing') 266 | LOG.debug(f'Old Version: {old_version}') 267 | LOG.debug(f'New Version: {new_version}') 268 | 269 | if new_version > old_version: 270 | return True 271 | else: 272 | return False 273 | 274 | 275 | def metadata_tagger(file_path, manga_title, manga_chapter_number, format, logging_info, volume): 276 | manga_search = None 277 | db_exists = True 278 | retries = 0 279 | isadult = False 280 | anilist_id = None 281 | 282 | if AppSettings.adult_result: 283 | isadult = True 284 | 285 | if Path(f'{AppSettings.data_dir}/exceptions.json').exists(): 286 | with open(f'{AppSettings.data_dir}/exceptions.json', 'r') as exceptions_json: 287 | exceptions = json.load(exceptions_json) 288 | if manga_title in exceptions: 289 | LOG.info('Manga_title found in exceptions.json, using manga specific configuration...', extra=logging_info) 290 | if exceptions[manga_title]['format'] == "MANGA" or exceptions[manga_title]['format'] == "ONE_SHOT": 291 | format = exceptions[manga_title]['format'] 292 | if exceptions[manga_title]['adult'] is True or exceptions[manga_title]['adult'] is False: 293 | isadult = exceptions[manga_title]['adult'] 294 | if "anilist_id" in exceptions[manga_title]: 295 | anilist_id = exceptions[manga_title]['anilist_id'] 296 | if "anilist_title" in exceptions[manga_title]: 297 | manga_title = exceptions[manga_title]['anilist_title'] 298 | 299 | LOG.info(f'Table search value is "{manga_title}"', extra=logging_info) 300 | while manga_search is None: 301 | if retries == 0: 302 | if anilist_id is not None: 303 | LOG.info('Searching manga_metadata for anilist id.', extra=logging_info) 304 | manga_search = MetadataTable.search_by_search_id(anilist_id) 305 | else: 306 | LOG.info('Searching manga_metadata for manga title by search value...', extra=logging_info) 307 | manga_search = MetadataTable.search_by_search_value(manga_title) 308 | retries = 1 309 | else: # The manga is not in the database, so ping the API and create the database 310 | LOG.info('Manga was not found in the database; resorting to Anilist API.', extra=logging_info) 311 | try: 312 | if anilist_id: 313 | LOG.info('Searching based on id given in exception file. ') 314 | manga_search = AniList.search_for_manga_title_by_id(anilist_id, logging_info) 315 | elif isadult: # enable adult result in Anilist 316 | LOG.info('Adult result enabled') 317 | manga_search = AniList.search_for_manga_title_by_manga_title_with_adult(manga_title, format, logging_info) 318 | else: 319 | manga_search = AniList.search_for_manga_title_by_manga_title(manga_title, format, logging_info) 320 | except AniListRateLimit as e: 321 | LOG.warning(e, extra=logging_info) 322 | LOG.warning('Manga Tagger has unintentionally breached the API limits on Anilist. Waiting 60s to clear ' 323 | 'all rate limiting limits...') 324 | time.sleep(60) 325 | if anilist_id: 326 | LOG.info('Searching based on id given in exception file: ') 327 | manga_search = AniList.search_for_manga_title_by_id(anilist_id, logging_info) 328 | elif isadult: # enable adult result in Anilist 329 | LOG.info('Adult result enabled') 330 | manga_search = AniList.search_for_manga_title_by_manga_title_with_adult(manga_title, format, logging_info) 331 | else: 332 | manga_search = AniList.search_for_manga_title_by_manga_title(manga_title, format, logging_info) 333 | if manga_search is None: 334 | raise MangaNotFoundError(manga_title) 335 | db_exists = False 336 | 337 | if db_exists: 338 | series_title = MetadataTable.search_series_title(manga_title) 339 | series_title_legal = slugify(series_title) 340 | manga_library_dir = Path(AppSettings.library_dir, series_title_legal) 341 | try: 342 | if volume is not None: 343 | new_filename = f"{series_title_legal} Vol.{volume} {manga_chapter_number}.cbz" 344 | else: 345 | new_filename = f"{series_title_legal} {manga_chapter_number}.cbz" 346 | LOG.debug(f'new_filename: {new_filename}') 347 | except TypeError: 348 | LOG.warning(f'Manga Tagger was unable to process "{file_path}"', extra=logging_info) 349 | return None 350 | new_file_path = Path(manga_library_dir, new_filename) 351 | 352 | if AppSettings.mode_settings is None or AppSettings.mode_settings['rename_file']: 353 | if not manga_library_dir.exists(): 354 | LOG.info( 355 | f'A directory for "{series_title}" in "{AppSettings.library_dir}" does not exist; creating now.') 356 | manga_library_dir.mkdir() 357 | try: 358 | # Multithreading Optimization 359 | if new_file_path in CURRENTLY_PENDING_RENAME: 360 | LOG.info(f'A file is currently being renamed under the filename "{new_filename}". Locking ' 361 | f'{file_path} from further processing until this rename action is complete...', 362 | extra=logging_info) 363 | 364 | while new_file_path in CURRENTLY_PENDING_RENAME: 365 | time.sleep(1) 366 | 367 | LOG.info(f'The file being renamed to "{new_file_path}" has been completed. Unlocking ' 368 | f'"{new_filename}" for file rename processing.', extra=logging_info) 369 | else: 370 | LOG.info(f'No files currently currently being processed under the filename ' 371 | f'"{new_filename}". Locking new filename for processing...', extra=logging_info) 372 | CURRENTLY_PENDING_RENAME.add(new_file_path) 373 | 374 | rename_action(file_path, new_file_path, series_title, manga_chapter_number, logging_info) 375 | except (FileExistsError, FileUpdateNotRequiredError, FileAlreadyProcessedError) as e: 376 | LOG.exception(e, extra=logging_info) 377 | CURRENTLY_PENDING_RENAME.remove(new_file_path) 378 | return 379 | 380 | if manga_title in ProcSeriesTable.processed_series: 381 | LOG.info(f'Found an entry in manga_metadata for "{manga_title}".', extra=logging_info) 382 | else: 383 | LOG.info(f'Found an entry in manga_metadata for "{manga_title}"; unlocking series for processing.', 384 | extra=logging_info) 385 | ProcSeriesTable.processed_series.add(manga_title) 386 | 387 | if AppSettings.image: 388 | if not Path(f'{AppSettings.image_dir}/{series_title}_cover.jpg').exists(): 389 | LOG.info(f'Image directory configured but cover not found. Send request to Anilist for necessary data.',extra=logging_info) 390 | manga_id = MetadataTable.search_by_series_title(series_title)['_id'] 391 | anilist_details = AniList.search_details_by_series_id(manga_id, format, logging_info) 392 | LOG.info('Downloading series cover image...', extra=logging_info) 393 | download_cover_image(series_title, anilist_details['coverImage']['extraLarge']) 394 | else: 395 | LOG.info('Series cover image already exist, not downloading.', extra=logging_info) 396 | else: 397 | LOG.info('Image flag not set, not downloading series cover image.', extra=logging_info) 398 | 399 | 400 | 401 | manga_metadata = Metadata(series_title, logging_info, details=manga_search) 402 | logging_info['metadata'] = manga_metadata.__dict__ 403 | else: 404 | 405 | anilist_titles = construct_anilist_titles(manga_search['title']) 406 | logging_info['anilist_titles'] = anilist_titles 407 | 408 | if not anilist_titles == 'None' or anilist_titles is not None: 409 | manga_found = True 410 | 411 | series_title = anilist_titles.get('romaji') 412 | series_title_legal = slugify(series_title) 413 | LOG.info(f'Manga title found for "{manga_title}" found as "{series_title}".', extra=logging_info) 414 | 415 | series_id = manga_search['id'] 416 | anilist_details = AniList.search_details_by_series_id(series_id, format, logging_info) 417 | LOG.debug(f'anilist_details: {anilist_details}') 418 | 419 | try: 420 | if volume is not None: 421 | new_filename = f"{series_title_legal} Vol.{volume} {manga_chapter_number}.cbz" 422 | else: 423 | new_filename = f"{series_title_legal} {manga_chapter_number}.cbz" 424 | LOG.debug(f'new_filename: {new_filename}') 425 | except TypeError: 426 | LOG.warning(f'Manga Tagger was unable to process "{file_path}"', extra=logging_info) 427 | return None 428 | 429 | manga_library_dir = Path(AppSettings.library_dir, series_title_legal) 430 | LOG.debug(f'Manga Library Directory: {manga_library_dir}') 431 | 432 | new_file_path = Path(manga_library_dir, new_filename) 433 | LOG.debug(f'new_file_path: {new_file_path}') 434 | 435 | LOG.info(f'Checking for current and previously processed files with filename "{new_filename}"...', 436 | extra=logging_info) 437 | 438 | if AppSettings.mode_settings is None or AppSettings.mode_settings['rename_file']: 439 | if not manga_library_dir.exists(): 440 | LOG.info( 441 | f'A directory for "{series_title}" in "{AppSettings.library_dir}" does not exist; creating now.') 442 | manga_library_dir.mkdir() 443 | try: 444 | # Multithreading Optimization 445 | if new_file_path in CURRENTLY_PENDING_RENAME: 446 | LOG.info(f'A file is currently being renamed under the filename "{new_filename}". Locking ' 447 | f'{file_path} from further processing until this rename action is complete...', 448 | extra=logging_info) 449 | 450 | while new_file_path in CURRENTLY_PENDING_RENAME: 451 | time.sleep(1) 452 | 453 | LOG.info(f'The file being renamed to "{new_file_path}" has been completed. Unlocking ' 454 | f'"{new_filename}" for file rename processing.', extra=logging_info) 455 | else: 456 | LOG.info(f'No files currently currently being processed under the filename ' 457 | f'"{new_filename}". Locking new filename for processing...', extra=logging_info) 458 | CURRENTLY_PENDING_RENAME.add(new_file_path) 459 | 460 | rename_action(file_path, new_file_path, series_title, manga_chapter_number, logging_info) 461 | 462 | except (FileExistsError, FileUpdateNotRequiredError, FileAlreadyProcessedError) as e: 463 | LOG.exception(e, extra=logging_info) 464 | CURRENTLY_PENDING_RENAME.remove(new_file_path) 465 | return 466 | 467 | manga_metadata = Metadata(series_title, logging_info, anilist_details) 468 | logging_info['metadata'] = manga_metadata.__dict__ 469 | 470 | if series_title in ProcSeriesTable.processed_series: 471 | LOG.info( 472 | f'Found an entry in manga_metadata for "{series_title}". Filename was probably not perfectly named according to MAL. Not adding metadata to MetadataTable.', 473 | extra=logging_info) 474 | else: 475 | if AppSettings.mode_settings is None or ('database_insert' in AppSettings.mode_settings.keys() 476 | and AppSettings.mode_settings['database_insert']): 477 | MetadataTable.insert(manga_metadata, logging_info) 478 | LOG.info(f'Retrieved metadata for "{series_title}" from the Anilist and MyAnimeList APIs; ' 479 | f'now unlocking series for processing!', extra=logging_info) 480 | ProcSeriesTable.processed_series.add(series_title) 481 | 482 | if AppSettings.mode_settings is None or ('write_comicinfo' in AppSettings.mode_settings.keys() 483 | and AppSettings.mode_settings['write_comicinfo']): 484 | if AppSettings.image: 485 | if not Path(f'{AppSettings.image_dir}/{series_title}_cover.jpg').exists(): 486 | LOG.info(f'Image directory configured but cover not found. Downloading series cover image...', extra=logging_info) 487 | download_cover_image(series_title, anilist_details['coverImage']['extraLarge']) 488 | else: 489 | LOG.info('Series cover image already exist, not downloading.', extra=logging_info) 490 | comicinfo_xml = construct_comicinfo_xml(manga_metadata, manga_chapter_number, logging_info, volume) 491 | reconstruct_manga_chapter(series_title, comicinfo_xml, new_file_path, logging_info) 492 | if AppSettings.image and (not AppSettings.image_first or (AppSettings.image_first and int(float(manga_chapter_number))==1)): 493 | add_cover_to_manga_chapter(series_title, new_file_path, logging_info) 494 | 495 | LOG.info(f'Processing on "{new_file_path}" has finished.', extra=logging_info) 496 | return manga_metadata 497 | 498 | 499 | def construct_anilist_titles(anilist_details): 500 | anilist_titles = {} 501 | 502 | if anilist_details['romaji'] is not None: 503 | anilist_titles['romaji'] = anilist_details['romaji'] 504 | 505 | if anilist_details['english'] is not None: 506 | anilist_titles['english'] = anilist_details['english'] 507 | 508 | if anilist_details['native'] is not None: 509 | anilist_titles['native'] = anilist_details['native'] 510 | 511 | return anilist_titles 512 | 513 | 514 | def construct_comicinfo_xml(metadata: Metadata, chapter_number, logging_info, volume_number): 515 | LOG.info(f'Constructing comicinfo object for "{metadata.series_title}", chapter {chapter_number}...', 516 | extra=logging_info) 517 | 518 | comicinfo = Element('ComicInfo') 519 | 520 | application_tag = Comment('Generated by Manga Tagger, an Endless Galaxy Studios project') 521 | comicinfo.append(application_tag) 522 | 523 | series = SubElement(comicinfo, 'Series') 524 | series.text = metadata.series_title 525 | 526 | if metadata.series_title_eng is not None and compare(metadata.series_title, 527 | metadata.series_title_eng) != 1 and metadata.series_title_eng != "": 528 | localized_series = SubElement(comicinfo, 'LocalizedSeries') 529 | localized_series.text = metadata.series_title_eng 530 | 531 | number = SubElement(comicinfo, 'Number') 532 | number.text = f'{chapter_number}' 533 | if volume_number is not None: 534 | volume = SubElement(comicinfo, 'Volume') 535 | volume.text = f'{volume_number}' 536 | 537 | if metadata.volumes is not None: 538 | count = SubElement(comicinfo,"Count") 539 | count.text = f'{metadata.volumes}' 540 | 541 | summary = SubElement(comicinfo, 'Summary') 542 | soup = BeautifulSoup(metadata.description, "html.parser") 543 | summary.text = soup.get_text() 544 | 545 | publish_date = datetime.strptime(metadata.publish_date, '%Y-%m-%d').date() 546 | year = SubElement(comicinfo, 'Year') 547 | year.text = f'{publish_date.year}' 548 | 549 | month = SubElement(comicinfo, 'Month') 550 | month.text = f'{publish_date.month}' 551 | 552 | writer = SubElement(comicinfo, 'Writer') 553 | writer.text = next(iter(metadata.staff['story'])) 554 | 555 | penciller = SubElement(comicinfo, 'Penciller') 556 | penciller.text = next(iter(metadata.staff['art'])) 557 | 558 | inker = SubElement(comicinfo, 'Inker') 559 | inker.text = next(iter(metadata.staff['art'])) 560 | 561 | colorist = SubElement(comicinfo, 'Colorist') 562 | colorist.text = next(iter(metadata.staff['art'])) 563 | 564 | letterer = SubElement(comicinfo, 'Letterer') 565 | letterer.text = next(iter(metadata.staff['art'])) 566 | 567 | cover_artist = SubElement(comicinfo, 'CoverArtist') 568 | cover_artist.text = next(iter(metadata.staff['art'])) 569 | 570 | # publisher = SubElement(comicinfo, 'Publisher') 571 | # publisher.text = next(iter(metadata.serializations)) 572 | 573 | genre = SubElement(comicinfo, 'Genre') 574 | for mg in metadata.genres: 575 | if genre.text is not None: 576 | genre.text += f',{mg}' 577 | else: 578 | genre.text = f'{mg}' 579 | 580 | web = SubElement(comicinfo, 'Web') 581 | web.text = metadata.anilist_url 582 | 583 | language = SubElement(comicinfo, 'LanguageISO') 584 | language.text = 'en' 585 | 586 | manga = SubElement(comicinfo, 'Manga') 587 | manga.text = 'Yes' 588 | 589 | notes = SubElement(comicinfo, 'Notes') 590 | notes.text = f'Scraped metadata from AniList on {metadata.scrape_date}' 591 | 592 | comicinfo.set('xmlns:xsd', 'http://www.w3.org/2001/XMLSchema') 593 | comicinfo.set('xmlns:xsi', 'http://www.w3.org/2001/XMLSchema-instance') 594 | 595 | LOG.info(f'Finished creating ComicInfo object for "{metadata.series_title}", chapter {chapter_number}.', 596 | extra=logging_info) 597 | return parseString(tostring(comicinfo)).toprettyxml(indent=" ") 598 | 599 | 600 | def reconstruct_manga_chapter(manga_title, comicinfo_xml, manga_file_path, logging_info): 601 | try: 602 | with ZipFile(manga_file_path, 'a') as zipfile: 603 | zipfile.writestr('ComicInfo.xml', comicinfo_xml) 604 | except Exception as e: 605 | LOG.exception(e, extra=logging_info) 606 | LOG.warning('Manga Tagger is unfamiliar with this error. Please log an issue for investigation.', 607 | extra=logging_info) 608 | return 609 | LOG.info(f'ComicInfo.xml has been created and appended to "{manga_file_path}".', extra=logging_info) 610 | 611 | def add_cover_to_manga_chapter(manga_title, manga_file_path, logging_info): 612 | try: 613 | with ZipFile(manga_file_path, 'a') as zipfile: 614 | if AppSettings.image and Path(f'{AppSettings.image_dir}/{manga_title}_cover.jpg').exists(): 615 | zipfile.write(f'{AppSettings.image_dir}/{manga_title}_cover.jpg', '000_cover.jpg') 616 | except Exception as e: 617 | LOG.exception(e, extra=logging_info) 618 | LOG.warning('Manga Tagger is unfamiliar with this error. Please log an issue for investigation.', 619 | extra=logging_info) 620 | return 621 | LOG.info(f'Cover Image has been added to "{manga_file_path}".', extra=logging_info) 622 | 623 | def download_cover_image(manga_title, image_url): 624 | image = requests.get(image_url) 625 | with open(f'{AppSettings.image_dir}/{manga_title}_cover.jpg', 'wb') as image_file: 626 | image_file.write(image.content) 627 | 628 | 629 | def slugify(value, allow_unicode=False): 630 | """ 631 | Taken from https://github.com/django/django/blob/master/django/utils/text.py 632 | Convert to ASCII if 'allow_unicode' is False. Convert spaces or repeated 633 | dashes to single dashes. Remove characters that aren't alphanumerics, 634 | underscores, or hyphens. Convert to lowercase. Also strip leading and 635 | trailing whitespace, dashes, and underscores. 636 | """ 637 | value = str(value) 638 | if allow_unicode: 639 | value = unicodedata.normalize('NFKC', value) 640 | else: 641 | value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') 642 | value = re.sub(r'[^\w\s-]', '', value) 643 | value = value.strip() 644 | return value 645 | 646 | 647 | def hasNumbers(inputString): 648 | return bool(re.search(r'\d', inputString)) 649 | -------------------------------------------------------------------------------- /MangaTaggerLib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MangaManagerORG/Manga-Tagger/132379ad6e8c9057ed8c079d16838b2e74f5e420/MangaTaggerLib/__init__.py -------------------------------------------------------------------------------- /MangaTaggerLib/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = '1.2.3-alpha' 2 | -------------------------------------------------------------------------------- /MangaTaggerLib/api.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import requests 3 | import time 4 | from datetime import datetime 5 | from typing import Optional, Dict, Mapping, Union, Any 6 | 7 | class AniList: 8 | _log = None 9 | 10 | @classmethod 11 | def initialize(cls): 12 | cls._log = logging.getLogger(f'{cls.__module__}.{cls.__name__}') 13 | 14 | @classmethod 15 | def _post(cls, query, variables, logging_info): 16 | try: 17 | response = requests.post('https://graphql.anilist.co', json={'query': query, 'variables': variables}) 18 | if response.status_code == 429: # Anilist rate-limit code 19 | raise AniListRateLimit() 20 | except Exception as e: 21 | cls._log.exception(e, extra=logging_info) 22 | cls._log.warning('Manga Tagger is unfamiliar with this error. Please log an issue for investigation.', 23 | extra=logging_info) 24 | return None 25 | 26 | cls._log.debug(f'Query: {query}') 27 | cls._log.debug(f'Variables: {variables}') 28 | cls._log.debug(f'Response JSON: {response.json()}') 29 | try: 30 | return response.json()['data']['Media'] 31 | except TypeError: 32 | return None 33 | 34 | @classmethod 35 | def search_for_manga_title_by_id(cls, manga_id, logging_info): 36 | query = ''' 37 | query search_for_manga_title_by_id ($manga_id: Int) { 38 | Media (id: $manga_id, type: MANGA) { 39 | id 40 | title { 41 | romaji 42 | english 43 | native 44 | } 45 | synonyms 46 | } 47 | } 48 | ''' 49 | 50 | variables = { 51 | 'manga_id': manga_id, 52 | } 53 | 54 | return cls._post(query, variables, logging_info) 55 | 56 | @classmethod 57 | def search_for_manga_title_by_manga_title(cls, manga_title, format, logging_info): 58 | query = ''' 59 | query search_manga_by_manga_title ($manga_title: String, $format: MediaFormat) { 60 | Media (search: $manga_title, type: MANGA, format: $format, isAdult: false) { 61 | id 62 | title { 63 | romaji 64 | english 65 | native 66 | } 67 | synonyms 68 | } 69 | } 70 | ''' 71 | 72 | variables = { 73 | 'manga_title': manga_title, 74 | 'format': format 75 | } 76 | 77 | return cls._post(query, variables, logging_info) 78 | 79 | @classmethod 80 | def search_for_manga_title_by_manga_title_with_adult(cls, manga_title, format, logging_info): 81 | query = ''' 82 | query search_manga_by_manga_title ($manga_title: String, $format: MediaFormat) { 83 | Media (search: $manga_title, type: MANGA, format: $format) { 84 | id 85 | title { 86 | romaji 87 | english 88 | native 89 | } 90 | synonyms 91 | } 92 | } 93 | ''' 94 | 95 | variables = { 96 | 'manga_title': manga_title, 97 | 'format': format 98 | } 99 | 100 | return cls._post(query, variables, logging_info) 101 | 102 | @classmethod 103 | def search_details_by_series_id(cls, series_id, format, logging_info): 104 | query = ''' 105 | query search_details_by_series_id ($series_id: Int, $format: MediaFormat) { 106 | Media (id: $series_id, type: MANGA, format: $format) { 107 | id 108 | status 109 | volumes 110 | siteUrl 111 | title { 112 | romaji 113 | english 114 | native 115 | } 116 | type 117 | genres 118 | synonyms 119 | startDate { 120 | day 121 | month 122 | year 123 | } 124 | coverImage { 125 | extraLarge 126 | } 127 | staff { 128 | edges { 129 | node{ 130 | name { 131 | first 132 | last 133 | full 134 | alternative 135 | } 136 | siteUrl 137 | } 138 | role 139 | } 140 | } 141 | description 142 | } 143 | } 144 | ''' 145 | 146 | variables = { 147 | 'series_id': series_id, 148 | 'format': format 149 | } 150 | 151 | return cls._post(query, variables, logging_info) 152 | 153 | 154 | class AniListRateLimit(Exception): 155 | """ 156 | Exception raised when AniList rate-limit is breached. 157 | """ -------------------------------------------------------------------------------- /MangaTaggerLib/database.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | from datetime import datetime 4 | from pathlib import Path 5 | from queue import Queue 6 | 7 | from bson.errors import InvalidDocument 8 | from pymongo import MongoClient 9 | from pymongo.errors import ServerSelectionTimeoutError, DuplicateKeyError 10 | 11 | 12 | class Database: 13 | database_name = None 14 | host_address = None 15 | port = None 16 | username = None 17 | password = None 18 | auth_source = None 19 | server_selection_timeout_ms = None 20 | 21 | _client = None 22 | _database = None 23 | _log = None 24 | 25 | @classmethod 26 | def initialize(cls): 27 | cls._log = logging.getLogger(f'{cls.__module__}.{cls.__name__}') 28 | 29 | if cls.auth_source is None: 30 | cls._client = MongoClient(cls.host_address, 31 | cls.port, 32 | username=cls.username, 33 | password=cls.password, 34 | serverSelectionTimeoutMS=cls.server_selection_timeout_ms) 35 | else: 36 | cls._client = MongoClient(cls.host_address, 37 | cls.port, 38 | username=cls.username, 39 | password=cls.password, 40 | authSource=cls.auth_source, 41 | serverSelectionTimeoutMS=cls.server_selection_timeout_ms) 42 | 43 | try: 44 | cls._log.info('Establishing database connection...') 45 | cls._client.is_mongos 46 | except ServerSelectionTimeoutError as sste: 47 | cls._log.exception(sste) 48 | cls._log.critical('Manga Tagger cannot run without a database connection. Please check the' 49 | 'configuration in settings.json and try again.') 50 | sys.exit(1) 51 | 52 | cls._database = cls._client[cls.database_name] 53 | 54 | MetadataTable.initialize() 55 | ProcFilesTable.initialize() 56 | ProcSeriesTable.initialize() 57 | TaskQueueTable.initialize() 58 | 59 | cls._log.info('Database connection established!') 60 | cls._log.debug(f'{cls.__name__} class has been initialized') 61 | 62 | @classmethod 63 | def load_database_tables(cls): 64 | ProcSeriesTable.load() 65 | 66 | @classmethod 67 | def save_database_tables(cls): 68 | ProcSeriesTable.save() 69 | 70 | @classmethod 71 | def close_connection(cls): 72 | cls._log.info('Closing database connection...') 73 | cls._client.close() 74 | 75 | @classmethod 76 | def print_debug_settings(cls): 77 | cls._log.debug(f'Database Name: {Database.database_name}') 78 | cls._log.debug(f'Host Address: {Database.host_address}') 79 | cls._log.debug(f'Port: {Database.port}') 80 | cls._log.debug(f'Username: {Database.username}') 81 | cls._log.debug(f'Password: {Database.password}') 82 | cls._log.debug(f'Authentication Source: {Database.auth_source}') 83 | cls._log.debug(f'Server Selection Timeout (ms): {Database.server_selection_timeout_ms}') 84 | 85 | @classmethod 86 | def insert(cls, data, logging_info=None): 87 | try: 88 | cls._log.info('Attempting to insert record into the database...', extra=logging_info) 89 | 90 | if type(data) is dict: 91 | cls._database.insert_one(data) 92 | else: 93 | cls._database.insert_one(data.__dict__) 94 | except (DuplicateKeyError, InvalidDocument) as e: 95 | cls._log.exception(e, extra=logging_info) 96 | return 97 | except Exception as e: 98 | cls._log.exception(e, extra=logging_info) 99 | cls._log.warning('Manga Tagger is unfamiliar with this error. Please log an issue for investigation.', 100 | extra=logging_info) 101 | return 102 | 103 | cls._log.info('Insertion was successful!', extra=logging_info) 104 | 105 | @classmethod 106 | def update(cls, search_filter, data, logging_info): 107 | try: 108 | cls._log.info('Attempting to update record in the database...', extra=logging_info) 109 | cls._database.update_one(search_filter, data) 110 | except Exception as e: 111 | cls._log.exception(e, extra=logging_info) 112 | cls._log.warning('Manga Tagger is unfamiliar with this error. Please log an issue for investigation.', 113 | extra=logging_info) 114 | return 115 | 116 | cls._log.info('Update was successful!', extra=logging_info) 117 | 118 | @classmethod 119 | def delete_all(cls, logging_info): 120 | try: 121 | cls._log.info('Attempting to delete all records in the database...', extra=logging_info) 122 | cls._database.delete_many({}) 123 | except Exception as e: 124 | cls._log.exception(e, extra=logging_info) 125 | cls._log.warning('Manga Tagger is unfamiliar with this error. Please log an issue for investigation.', 126 | extra=logging_info) 127 | return 128 | 129 | cls._log.info('Deletion was successful!', extra=logging_info) 130 | 131 | 132 | class MetadataTable(Database): 133 | @classmethod 134 | def initialize(cls): 135 | cls._log = logging.getLogger(f'{cls.__module__}.{cls.__name__}') 136 | cls._database = super()._database['manga_metadata'] 137 | cls._log.debug(f'{cls.__name__} class has been initialized') 138 | 139 | @classmethod 140 | def search_by_search_id(cls, manga_id): 141 | cls._log.debug(f'Searching manga_metadata cls by key "_id" using value "{manga_id}"') 142 | return cls._database.find_one({ 143 | '_id': manga_id 144 | }) 145 | 146 | @classmethod 147 | def search_by_search_value(cls, manga_title): 148 | cls._log.debug(f'Searching manga_metadata cls by key "search_value" using value "{manga_title}"') 149 | return cls._database.find_one({'$or': [ 150 | {'search_value': manga_title}, 151 | {'series_title': manga_title}, 152 | {'series_title_eng': manga_title}, 153 | {'series_title_jap': manga_title}, 154 | {'synonyms': manga_title} 155 | ]}) 156 | 157 | @classmethod 158 | def search_id_by_search_value(cls, manga_title): 159 | cls._log.debug(f'Searching "series_id" using value "{manga_title}"') 160 | cursor = cls._database.find_one({"search_value": manga_title}, {"_id": 1}) 161 | return cursor['_id'] 162 | 163 | @classmethod 164 | def search_series_title(cls, manga_title): 165 | cls._log.debug(f'Searching "series_title" using value "{manga_title}"') 166 | return cls._database.find_one({"$or": [ 167 | {'search_value': manga_title}, 168 | {'series_title': manga_title}, 169 | {'series_title_eng': manga_title}, 170 | {'series_title_jap': manga_title}, 171 | {'synonyms': manga_title} 172 | ]}, {'series_title': 1})['series_title'] 173 | 174 | 175 | class ProcFilesTable(Database): 176 | @classmethod 177 | def initialize(cls): 178 | cls._log = logging.getLogger(f'{cls.__module__}.{cls.__name__}') 179 | cls._database = super()._database['processed_files'] 180 | cls._log.debug(f'{cls.__name__} class has been initialized') 181 | 182 | @classmethod 183 | def search(cls, manga_title, chapter_number): 184 | cls._log.debug(f'Searching processed_files cls by keys "series_title" and "chapter_number" ' 185 | f'using values "{manga_title}" and {chapter_number}') 186 | return cls._database.find_one({ 187 | 'series_title': manga_title, 188 | 'chapter_number': chapter_number 189 | }) 190 | 191 | @classmethod 192 | def insert_record(cls, old_file_path: Path, new_file_path: Path, manga_title, chapter, logging_info): 193 | record = { 194 | "series_title": manga_title, 195 | "chapter_number": chapter, 196 | "old_filename": old_file_path.name, 197 | "new_filename": new_file_path.name, 198 | "process_date": datetime.now().date().strftime('%Y-%m-%d @ %I:%M:%S %p') 199 | } 200 | 201 | cls._log.debug(f'Record: {record}') 202 | 203 | logging_info['inserted_processed_record'] = record 204 | cls._database.insert(record, logging_info) 205 | 206 | @classmethod 207 | def update_record(cls, results, old_file_path: Path, new_file_path: Path, logging_info): 208 | record = { 209 | "$set": { 210 | "old_filename": old_file_path.name, 211 | "update_date": datetime.now().date().strftime('%Y-%m-%d @ %I:%M:%S %p') 212 | } 213 | } 214 | cls._log.debug(f'Record: {record}') 215 | 216 | logging_info['updated_processed_record'] = record 217 | cls._database.update(results, record, logging_info) 218 | 219 | 220 | class ProcSeriesTable(Database): 221 | processed_series = set() 222 | 223 | @classmethod 224 | def initialize(cls): 225 | cls._log = logging.getLogger(f'{cls.__module__}.{cls.__name__}') 226 | cls._database = super()._database['processed_series'] 227 | cls._id = None 228 | cls._last_save_time = None 229 | cls._log.debug(f'{cls.__name__} class has been initialized') 230 | 231 | @classmethod 232 | def save(cls): 233 | cls._log.info('Saving processed series...') 234 | cls._database.delete_one({ 235 | '_id': cls._id 236 | }) 237 | super(ProcSeriesTable, cls).insert(dict.fromkeys(cls.processed_series, True)) 238 | 239 | @classmethod 240 | def load(cls): 241 | cls._log.info('Loading processed series...') 242 | results = cls._database.find_one() 243 | if results is not None: 244 | cls._id = results.pop('_id') 245 | cls.processed_series = set(results.keys()) 246 | 247 | @classmethod 248 | def save_while_running(cls): 249 | if cls._last_save_time is not None: 250 | last_save_delta = (datetime.now() - cls._last_save_time).total_seconds() 251 | 252 | # Save every hour 253 | if last_save_delta > 3600: 254 | cls._last_save_time = datetime.now() 255 | cls.save() 256 | 257 | 258 | class TaskQueueTable(Database): 259 | @classmethod 260 | def initialize(cls): 261 | cls._log = logging.getLogger(f'{cls.__module__}.{cls.__name__}') 262 | cls._database = super()._database['task_queue'] 263 | cls.queue = Queue() 264 | cls._log.debug(f'{cls.__name__} class has been initialized') 265 | 266 | @classmethod 267 | def load(cls, task_list: dict): 268 | cls._log.info('Loading task queue...') 269 | results = cls._database.find() 270 | 271 | if results is not None: 272 | for result in results: 273 | task_list[result['manga_chapter']] = result 274 | 275 | @classmethod 276 | def save(cls, queue): 277 | if not queue.empty(): 278 | cls._log.info('Saving task queue...') 279 | while not queue.empty(): 280 | event = queue.get() 281 | super(TaskQueueTable, cls).insert(event.dictionary()) 282 | 283 | @classmethod 284 | def delete_all(cls): 285 | super(TaskQueueTable, cls).delete_all(None) 286 | -------------------------------------------------------------------------------- /MangaTaggerLib/errors.py: -------------------------------------------------------------------------------- 1 | """Exceptions raised by Manga Tagger""" 2 | 3 | class MangaNotFoundError(Exception): 4 | """ 5 | Exception raised when the manga cannot be found in the results from MyAnimeList (Jikan). 6 | """ 7 | def __init__(self, manga_title): 8 | super().__init__(f'"{manga_title}" was not found in the returned results from Anilist ' 9 | f'This may be due to a difference in manga series titles, or may be something else entirely. ' 10 | f'Please open an issue for investigation.') 11 | 12 | class MetadataNotCompleteError(Exception): 13 | """ 14 | Exception raised when not enough data is given to create a Metadata object. 15 | """ 16 | 17 | def __init__(self, current_directory): 18 | super().__init__('Tried to create Metadata object, but was not given the proper data to interpret') 19 | 20 | 21 | class UnparsableFilenameError(Exception): 22 | """ 23 | Exception raised when the chapter filename is unparsable; specifically when 'chapter' is not found in the filename. 24 | 25 | Attributes: 26 | filename - Name of the file 27 | delimiter_key - The sequence of characters not found that triggered the error 28 | """ 29 | 30 | def __init__(self, filename, delimiter_key): 31 | super().__init__(f'Unable to parse filename "{filename}" due to delimiter "{delimiter_key}" missing') 32 | 33 | 34 | class FileAlreadyProcessedError(FileExistsError): 35 | """ 36 | Exception raised when the chapter currently being processed has already been processed and the filename does not 37 | indicate a different version from what was previously processed. 38 | 39 | Attributes: 40 | filename - Name of the file 41 | """ 42 | def __init__(self, filename): 43 | super().__init__(f'"{filename}" has already been processed by Manga Tagger; skipping') 44 | 45 | 46 | class FileUpdateNotRequiredError(FileExistsError): 47 | """ 48 | Exception raised when the chapter currently being processed has been found to be older than the chapter that was 49 | previously processed. 50 | 51 | Attributes: 52 | filename - Name of the file 53 | """ 54 | def __init__(self, filename): 55 | super().__init__(f'"{filename}" is older than or the same as the current existing chapter; skipping') 56 | 57 | 58 | class MangaMatchedException(Exception): 59 | """ 60 | Exception raised to bypass try...except when comparing manga titles. 61 | """ -------------------------------------------------------------------------------- /MangaTaggerLib/models.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime 3 | from pytz import timezone 4 | 5 | from MangaTaggerLib.errors import MetadataNotCompleteError 6 | from MangaTaggerLib.utils import AppSettings, compare 7 | 8 | 9 | class Metadata: 10 | _log = None 11 | 12 | @classmethod 13 | def fully_qualified_class_name(cls): 14 | return f'{cls.__module__}.{cls.__name__}' 15 | 16 | def __init__(self, manga_title, logging_info, anilist_details=None, details=None): 17 | Metadata._log = logging.getLogger(self.fully_qualified_class_name()) 18 | 19 | self.search_value = manga_title 20 | Metadata._log.info(f'Creating Metadata model for series "{manga_title}"...', extra=logging_info) 21 | 22 | if anilist_details: # If details are grabbed from Anilist APIs 23 | self._construct_api_metadata(anilist_details, logging_info) 24 | elif details: # If details were stored in the database 25 | self._construct_database_metadata(details) 26 | else: 27 | Metadata._log.exception(MetadataNotCompleteError, extra=logging_info) 28 | Metadata._log.debug(f'{self.search_value} Metadata Model: {self.__dict__.__str__()}') 29 | 30 | logging_info['metadata'] = self.__dict__ 31 | Metadata._log.info('Successfully created Metadata model.', extra=logging_info) 32 | 33 | def _construct_api_metadata(self, anilist_details, logging_info): 34 | self._id = anilist_details['id'] 35 | self.series_title = anilist_details['title']['romaji'] 36 | 37 | if anilist_details['title']['english'] == 'None' or anilist_details['title']['english'] is None: 38 | self.series_title_eng = None 39 | else: 40 | self.series_title_eng = anilist_details['title']['english'] 41 | 42 | if anilist_details['title']['native'] == 'None' or anilist_details['title']['native'] is None: 43 | self.series_title_jap = None 44 | else: 45 | self.series_title_jap = anilist_details['title']['native'] 46 | 47 | self.status = anilist_details['status'] 48 | if anilist_details.get('volumes'): 49 | self.volumes = anilist_details.get('volumes') 50 | else: 51 | self.volumes = None 52 | 53 | self.type = anilist_details['type'] 54 | self.description = anilist_details['description'] 55 | self.anilist_url = anilist_details['siteUrl'] 56 | self.publish_date = None 57 | self.genres = [] 58 | self.synonyms = [] 59 | self.staff = {} 60 | 61 | self._construct_publish_date(anilist_details['startDate']) 62 | self._parse_genres(anilist_details['genres'], logging_info) 63 | self._parse_synonyms(anilist_details.get('synonyms'), logging_info) 64 | self._parse_staff(anilist_details['staff']['edges'], logging_info) 65 | 66 | self.scrape_date = timezone(AppSettings.timezone).localize(datetime.now()).strftime('%Y-%m-%d %I:%M %p %Z') 67 | 68 | def _construct_database_metadata(self, details): 69 | self._id = details['_id'] 70 | self.series_title = details['series_title'] 71 | self.series_title_eng = details['series_title_eng'] 72 | self.series_title_jap = details['series_title_jap'] 73 | self.status = details['status'] 74 | self.volumes = details.get("volumes") 75 | self.type = details['type'] 76 | self.description = details['description'] 77 | self.anilist_url = details['anilist_url'] 78 | self.publish_date = details['publish_date'] 79 | self.genres = details['genres'] 80 | self.synonyms = details['synonyms'] 81 | self.staff = details['staff'] 82 | self.publish_date = details['publish_date'] 83 | self.scrape_date = details['scrape_date'] 84 | 85 | def _construct_publish_date(self, date): 86 | if date['month'] == 'None' or date['day'] == 'None' or date['day'] is None or date['month'] is None: 87 | yearstr = str(date.get('year')) 88 | datestr = yearstr + "-" + "01" + "-" "01" 89 | else: 90 | datestr = str(date.get('year')) + "-" + str(date.get('month')) + "-" + str(date.get('day')) 91 | self.publish_date = datetime.strptime(datestr, '%Y-%m-%d').strftime('%Y-%m-%d') 92 | Metadata._log.debug(f'Publish date constructed: {self.publish_date}') 93 | 94 | def _parse_genres(self, genres, logging_info): 95 | Metadata._log.info('Parsing genres...', extra=logging_info) 96 | Metadata._log.debug(f'Genre found: {", ".join(genres)}') 97 | self.genres.extend(genres) 98 | 99 | def _parse_synonyms(self, synonyms, logging_info): 100 | Metadata._log.info('Parsing Synonyms...', extra=logging_info) 101 | Metadata._log.debug(f'Synonyms found: {synonyms}') 102 | self.synonyms.extend(synonyms or []) 103 | 104 | def _parse_staff(self, anilist_staff, logging_info): 105 | Metadata._log.info('Parsing staff roles...', extra=logging_info) 106 | 107 | roles = [] 108 | 109 | self.staff = { 110 | 'story': {}, 111 | 'art': {} 112 | } 113 | 114 | for a_staff in anilist_staff: 115 | Metadata._log.debug(f'Staff Member (Anilist): {a_staff}') 116 | 117 | anilist_staff_name = '' 118 | 119 | if a_staff['node']['name']['last'] is not None: 120 | anilist_staff_name = a_staff['node']['name']['last'] 121 | 122 | if a_staff['node']['name']['first'] is not None: 123 | anilist_staff_name += ', ' + a_staff['node']['name']['first'] 124 | 125 | names_to_compare = [anilist_staff_name] 126 | if '' not in a_staff['node']['name']['alternative']: 127 | for name in a_staff['node']['name']['alternative']: 128 | names_to_compare.append(name) 129 | 130 | role = a_staff['role'].lower() 131 | if 'story & art' in role: 132 | roles.append('story') 133 | roles.append('art') 134 | self._add_anilist_staff_member('story', a_staff) 135 | self._add_anilist_staff_member('art', a_staff) 136 | elif 'story' in role: 137 | roles.append('story') 138 | self._add_anilist_staff_member('story', a_staff) 139 | elif 'art' in role: 140 | roles.append('art') 141 | self._add_anilist_staff_member('art', a_staff) 142 | else: 143 | Metadata._log.warning(f'Expected role not found for staff member "{a_staff}"; instead' 144 | f' found "{role}"', extra=logging_info) 145 | 146 | # Validate expected roles for staff members 147 | role_set = ['story', 'art'] 148 | 149 | if set(roles) != set(role_set): 150 | 151 | Metadata._log.warning(f'Not all expected roles are present for series "{self.search_value}"; ' 152 | f'double check ID "{self._id}"', extra=logging_info) 153 | 154 | def _add_anilist_staff_member(self, role, a_staff): 155 | self.staff[role][a_staff['node']['name']['full']] = { 156 | 'first_name': a_staff['node']['name']['first'], 157 | 'last_name': a_staff['node']['name']['last'], 158 | 'anilist_url': a_staff['node']['siteUrl'], 159 | } 160 | 161 | def _parse_serializations(self, serializations, logging_info): 162 | Metadata._log.info('Parsing serializations...', extra=logging_info) 163 | for serialization in serializations: 164 | Metadata._log.debug(serialization) 165 | self.serializations[serialization['name'].strip('.')] = { 166 | 'mal_id': serialization['mal_id'], 167 | 'url': serialization['url'] 168 | } 169 | 170 | def test_value(self): 171 | return { 172 | 'series_title': self.series_title, 173 | 'series_title_eng': self.series_title_eng, 174 | 'series_title_jap': self.series_title_jap, 175 | 'status': self.status, 176 | 'volumes':self.volumes, 177 | # 'mal_url': self.mal_url, 178 | 'anilist_url': self.anilist_url, 179 | 'publish_date': self.publish_date, 180 | 'genres': self.genres, 181 | 'synonyms': self.synonyms, 182 | 'staff': self.staff, 183 | # 'serializations': self.serializations 184 | } 185 | -------------------------------------------------------------------------------- /MangaTaggerLib/task_queue.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import time 3 | import uuid 4 | from enum import Enum 5 | from pathlib import Path 6 | from queue import Queue 7 | from threading import Thread 8 | from typing import List 9 | 10 | from watchdog.events import PatternMatchingEventHandler 11 | from watchdog.observers import Observer 12 | from watchdog.observers.polling import PollingObserver 13 | 14 | from MangaTaggerLib import MangaTaggerLib 15 | from MangaTaggerLib.database import TaskQueueTable 16 | 17 | 18 | class QueueEventOrigin(Enum): 19 | WATCHDOG = 1 20 | FROM_DB = 2 21 | SCAN = 3 22 | 23 | 24 | class QueueEvent: 25 | def __init__(self, event, origin=QueueEventOrigin.WATCHDOG): 26 | if origin == QueueEventOrigin.WATCHDOG: 27 | self.event_type = event.event_type 28 | self.src_path = Path(event.src_path) 29 | try: 30 | self.dest_path = Path(event.dest_path) 31 | except AttributeError: 32 | pass 33 | elif origin == QueueEventOrigin.FROM_DB: 34 | self.event_type = event['event_type'] 35 | self.src_path = Path(event['src_path']) 36 | try: 37 | self.dest_path = Path(event['dest_path']) 38 | except KeyError: 39 | pass 40 | elif origin == QueueEventOrigin.SCAN: 41 | self.event_type = 'existing' 42 | self.src_path = event 43 | 44 | def __str__(self): 45 | if self.event_type in ('created', 'existing'): 46 | return f'File {self.event_type} event at {self.src_path.absolute()}' 47 | elif self.event_type == 'modified': 48 | return f'File {self.event_type} event at {self.dest_path.absolute()}' 49 | 50 | def dictionary(self): 51 | ret_dict = { 52 | 'event_type': self.event_type, 53 | 'src_path': str(self.src_path.absolute()), 54 | 'manga_chapter': str(self.src_path.name.strip('.cbz')) 55 | } 56 | 57 | try: 58 | ret_dict['dest_path'] = str(self.dest_path.absolute()) 59 | except AttributeError: 60 | pass 61 | 62 | return ret_dict 63 | 64 | 65 | class QueueWorker: 66 | _queue: Queue = None 67 | _observer: Observer = None 68 | _log: logging = None 69 | _worker_list: List[Thread] = None 70 | _running: bool = False 71 | _debug_mode = False 72 | 73 | max_queue_size = None 74 | threads = None 75 | is_library_network_path = False 76 | download_dir: Path = None 77 | task_list = {} 78 | 79 | @classmethod 80 | def initialize(cls): 81 | cls._log = logging.getLogger(f'{cls.__module__}.{cls.__name__}') 82 | cls._queue = Queue(maxsize=cls.max_queue_size) 83 | cls._worker_list = [] 84 | cls._running = True 85 | 86 | for i in range(cls.threads): 87 | if not cls._debug_mode: 88 | worker = Thread(target=cls.process, name=f'MTT-{i}', daemon=True) 89 | else: 90 | worker = Thread(target=cls.dummy_process, name=f'MTT-{i}', daemon=True) 91 | cls._log.debug(f'Worker thread {worker.name} has been initialized') 92 | cls._worker_list.append(worker) 93 | 94 | if cls.is_library_network_path: 95 | cls._observer = PollingObserver() 96 | else: 97 | cls._observer = Observer() 98 | 99 | cls._observer.schedule(SeriesHandler(cls._queue), cls.download_dir, True) 100 | 101 | @classmethod 102 | def load_task_queue(cls): 103 | TaskQueueTable.load(cls.task_list) 104 | 105 | for task in cls.task_list.values(): 106 | event = QueueEvent(task, QueueEventOrigin.FROM_DB) 107 | cls._log.info(f'{event} has been added to the task queue') 108 | cls._queue.put(event) 109 | 110 | TaskQueueTable.delete_all() 111 | 112 | @classmethod 113 | def save_task_queue(cls): 114 | TaskQueueTable.save(cls._queue) 115 | with cls._queue.mutex: 116 | cls._queue.queue.clear() 117 | 118 | @classmethod 119 | def add_to_task_queue(cls, manga_chapter): 120 | event = QueueEvent(manga_chapter, QueueEventOrigin.SCAN) 121 | cls._log.info(f'{event} has been added to the task queue') 122 | cls._queue.put(event) 123 | 124 | @classmethod 125 | def exit(cls): 126 | # Stop worker threads from picking new items from the queue in process() 127 | cls._log.info('Stopping processing...') 128 | cls._running = False 129 | 130 | # Stop watchdog from adding new events to the queue 131 | cls._log.debug('Stopping watchdog...') 132 | cls._observer.stop() 133 | cls._observer.join() 134 | 135 | # Save and empty task queue 136 | cls.save_task_queue() 137 | 138 | # Finish current running jobs and stop worker threads 139 | cls._log.info('Stopping worker threads...') 140 | for worker in cls._worker_list: 141 | worker.join() 142 | cls._log.debug(f'Worker thread {worker.name} has been shut down') 143 | 144 | @classmethod 145 | def run(cls): 146 | for worker in cls._worker_list: 147 | worker.start() 148 | 149 | cls._observer.start() 150 | 151 | cls._log.info(f'Watching "{cls.download_dir}" for new downloads') 152 | 153 | while cls._running: 154 | time.sleep(1) 155 | 156 | @classmethod 157 | def dummy_process(cls): 158 | pass 159 | 160 | @classmethod 161 | def process(cls): 162 | while cls._running: 163 | if not cls._queue.empty(): 164 | event = cls._queue.get() 165 | 166 | if event.event_type in ('created', 'existing'): 167 | cls._log.info(f'Pulling "file {event.event_type}" event from the queue for "{event.src_path}"') 168 | path = Path(event.src_path) 169 | elif event.event_type == 'moved': 170 | cls._log.info(f'Pulling "file {event.event_type}" event from the queue for "{event.dest_path}"') 171 | path = Path(event.dest_path) 172 | else: 173 | cls._log.error('Event was passed, but Manga Tagger does not know how to handle it. Please open an ' 174 | 'issue for further investigation.') 175 | cls._queue.task_done() 176 | return 177 | 178 | current_size = -1 179 | try: 180 | destination_size = path.stat().st_size 181 | while current_size != destination_size: 182 | current_size = destination_size 183 | time.sleep(1) 184 | except FileNotFoundError as fnfe: 185 | cls._log.exception(fnfe) 186 | 187 | try: 188 | MangaTaggerLib.process_manga_chapter(path, uuid.uuid1()) 189 | except Exception as e: 190 | cls._log.exception(e) 191 | cls._log.warning('Manga Tagger is unfamiliar with this error. Please log an issue for ' 192 | 'investigation.') 193 | 194 | cls._queue.task_done() 195 | time.sleep(1) 196 | 197 | class SeriesHandler(PatternMatchingEventHandler): 198 | _log = None 199 | 200 | @classmethod 201 | def class_name(cls): 202 | return cls.__name__ 203 | 204 | @classmethod 205 | def fully_qualified_class_name(cls): 206 | return f'{cls.__module__}.{cls.__name__}' 207 | 208 | def __init__(self, queue): 209 | self._log = logging.getLogger(self.fully_qualified_class_name()) 210 | super().__init__(patterns=['*.cbz']) 211 | self.queue = queue 212 | self._log.debug(f'{self.class_name()} class has been initialized') 213 | 214 | def on_created(self, event): 215 | self._log.debug(f'Event Type: {event.event_type}') 216 | self._log.debug(f'Event Path: {event.src_path}') 217 | 218 | self.queue.put(QueueEvent(event, QueueEventOrigin.WATCHDOG)) 219 | self._log.info(f'Creation event for "{event.src_path}" will be added to the queue') 220 | 221 | def on_moved(self, event): 222 | self._log.debug(f'Event Type: {event.event_type}') 223 | self._log.debug(f'Event Source Path: {event.src_path}') 224 | self._log.debug(f'Event Destination Path: {event.dest_path}') 225 | 226 | if Path(event.src_path) == Path(event.dest_path) and '-.-' in event.dest_path: 227 | self.queue.put(QueueEvent(event, QueueEventOrigin.WATCHDOG)) 228 | self._log.info(f'Moved event for "{event.dest_path}" will be added to the queue') 229 | -------------------------------------------------------------------------------- /MangaTaggerLib/utils.py: -------------------------------------------------------------------------------- 1 | import atexit 2 | import json 3 | import logging 4 | import subprocess 5 | import sys 6 | import os 7 | 8 | from logging.handlers import RotatingFileHandler, SocketHandler 9 | from pathlib import Path 10 | 11 | from pythonjsonlogger import jsonlogger 12 | 13 | from MangaTaggerLib.database import Database 14 | from MangaTaggerLib.task_queue import QueueWorker 15 | from MangaTaggerLib.api import AniList 16 | 17 | 18 | class AppSettings: 19 | mode_settings = None 20 | timezone = None 21 | version = None 22 | image = False 23 | image_first = False 24 | adult_result = False 25 | download_dir = None 26 | image_dir = None 27 | library_dir = None 28 | data_dir = None 29 | is_network_path = None 30 | 31 | processed_series = None 32 | 33 | _log = None 34 | 35 | @classmethod 36 | def load(cls): 37 | settings_location = Path(Path.cwd(), 'settings.json') 38 | if Path(settings_location).exists(): 39 | with open(settings_location, 'r') as settings_json: 40 | settings = json.load(settings_json) 41 | else: 42 | with open(settings_location, 'w+') as settings_json: 43 | settings = cls._create_settings() 44 | json.dump(settings, settings_json, indent=4) 45 | 46 | with open(settings_location, 'r+') as settings_json: 47 | if os.getenv("MANGA_TAGGER_DB_NAME") is not None: 48 | settings['database']['database_name'] = os.getenv("MANGA_TAGGER_DB_NAME") 49 | if os.getenv("MANGA_TAGGER_DB_HOST_ADDRESS") is not None: 50 | settings['database']['host_address'] = os.getenv("MANGA_TAGGER_DB_HOST_ADDRESS") 51 | if os.getenv("MANGA_TAGGER_DB_PORT") is not None: 52 | settings['database']['port'] = int(os.getenv("MANGA_TAGGER_DB_PORT")) 53 | if os.getenv("MANGA_TAGGER_DB_USERNAME") is not None: 54 | settings['database']['username'] = os.getenv("MANGA_TAGGER_DB_USERNAME") 55 | if os.getenv("MANGA_TAGGER_DB_PASSWORD") is not None: 56 | settings['database']['password'] = os.getenv("MANGA_TAGGER_DB_PASSWORD") 57 | if os.getenv("MANGA_TAGGER_DB_AUTH_SOURCE") is not None: 58 | settings['database']['auth_source'] = os.getenv("MANGA_TAGGER_DB_AUTH_SOURCE") 59 | if os.getenv("MANGA_TAGGER_DB_SELECTION_TIMEOUT") is not None: 60 | settings['database']['server_selection_timeout_ms'] = int(os.getenv("MANGA_TAGGER_DB_SELECTION_TIMEOUT")) 61 | 62 | if os.getenv("MANGA_TAGGER_DOWNLOAD_DIR") is not None: 63 | settings['application']['library']['download_dir'] = os.getenv("MANGA_TAGGER_DOWNLOAD_DIR") 64 | 65 | if os.getenv("MANGA_TAGGER_DATA_DIR") is not None: 66 | settings['application']['data_dir'] = os.getenv("MANGA_TAGGER_DATA_DIR") 67 | 68 | if os.getenv('TZ') is not None: 69 | settings['application']['timezone'] = os.getenv("TZ") 70 | 71 | if os.getenv("MANGA_TAGGER_DRY_RUN") is not None: 72 | if os.getenv("MANGA_TAGGER_DRY_RUN").lower() == 'true': 73 | settings['application']['dry_run']['enabled'] = True 74 | elif os.getenv("MANGA_TAGGER_DRY_RUN").lower() == 'false': 75 | settings['application']['dry_run']['enabled'] = False 76 | if os.getenv("MANGA_TAGGER_DB_INSERT") is not None: 77 | if os.getenv("MANGA_TAGGER_DB_INSERT").lower() == 'true': 78 | settings['application']['dry_run']['database_insert'] = True 79 | elif os.getenv("MANGA_TAGGER_DB_INSERT").lower() == 'false': 80 | settings['application']['dry_run']['database_insert'] = False 81 | if os.getenv("MANGA_TAGGER_RENAME_FILE") is not None: 82 | if os.getenv("MANGA_TAGGER_RENAME_FILE").lower() == 'true': 83 | settings['application']['dry_run']['rename_file'] = True 84 | elif os.getenv("MANGA_TAGGER_RENAME_FILE").lower() == 'false': 85 | settings['application']['dry_run']['rename_file'] = False 86 | if os.getenv("MANGA_TAGGER_WRITE_COMICINFO") is not None: 87 | if os.getenv("MANGA_TAGGER_WRITE_COMICINFO").lower() == 'true': 88 | settings['application']['dry_run']['write_comicinfo'] = True 89 | elif os.getenv("MANGA_TAGGER_WRITE_COMICINFO").lower() == 'false': 90 | settings['application']['dry_run']['write_comicinfo'] = False 91 | 92 | if os.getenv("MANGA_TAGGER_THREADS") is not None: 93 | settings['application']['multithreading']['threads'] = int(os.getenv("MANGA_TAGGER_THREADS")) 94 | if os.getenv("MANGA_TAGGER_MAX_QUEUE_SIZE") is not None: 95 | settings['application']['multithreading']['max_queue_size'] = int(os.getenv("MANGA_TAGGER_MAX_QUEUE_SIZE")) 96 | 97 | if os.getenv("MANGA_TAGGER_DEBUG_MODE") is not None: 98 | if os.getenv("MANGA_TAGGER_DEBUG_MODE").lower() == 'true': 99 | settings['application']['debug_mode'] = True 100 | elif os.getenv("MANGA_TAGGER_DEBUG_MODE").lower() == 'false': 101 | settings['application']['debug_mode'] = False 102 | 103 | if os.getenv("MANGA_TAGGER_IMAGE_COVER") is not None: 104 | if os.getenv("MANGA_TAGGER_IMAGE_COVER").lower() == 'true': 105 | settings['application']['image']['enabled'] = True 106 | elif os.getenv("MANGA_TAGGER_IMAGE_COVER").lower() == 'first': 107 | settings['application']['image']['enabled'] = True 108 | settings['application']['image']['first'] = True 109 | elif os.getenv("MANGA_TAGGER_IMAGE_COVER").lower() == 'false': 110 | settings['application']['image']['enabled'] = False 111 | if os.getenv("MANGA_TAGGER_IMAGE_DIR") is not None: 112 | settings['application']['image']['image_dir'] = os.getenv("MANGA_TAGGER_IMAGE_DIR") 113 | 114 | if os.getenv("MANGA_TAGGER_ADULT_RESULT") is not None: 115 | if os.getenv("MANGA_TAGGER_ADULT_RESULT").lower() == 'true': 116 | settings['application']['adult_result'] = True 117 | elif os.getenv("MANGA_TAGGER_ADULT_RESULT").lower() == 'false': 118 | settings['application']['adult_result'] = False 119 | 120 | if os.getenv("MANGA_TAGGER_LIBRARY_DIR") is not None: 121 | settings['application']['library']['dir'] = os.getenv("MANGA_TAGGER_LIBRARY_DIR") 122 | 123 | if os.getenv("MANGA_TAGGER_LOGGING_LEVEL") is not None: 124 | settings['logger']['logging_level'] = os.getenv("MANGA_TAGGER_LOGGING_LEVEL") 125 | if os.getenv("MANGA_TAGGER_LOGGING_DIR") is not None: 126 | settings['logger']['log_dir'] = os.getenv("MANGA_TAGGER_LOGGING_DIR") 127 | if os.getenv("MANGA_TAGGER_LOGGING_CONSOLE") is not None: 128 | if os.getenv("MANGA_TAGGER_LOGGING_CONSOLE").lower() == 'true': 129 | settings['logger']['console']['enabled'] = True 130 | elif os.getenv("MANGA_TAGGER_LOGGING_CONSOLE").lower() == 'false': 131 | settings['logger']['console']['enabled'] = False 132 | if os.getenv("MANGA_TAGGER_LOGGING_FILE") is not None: 133 | if os.getenv("MANGA_TAGGER_LOGGING_FILE").lower() == 'true': 134 | settings['logger']['file']['enabled'] = True 135 | elif os.getenv("MANGA_TAGGER_LOGGING_FILE").lower() == 'false': 136 | settings['logger']['file']['enabled'] = False 137 | if os.getenv("MANGA_TAGGER_LOGGING_JSON") is not None: 138 | if os.getenv("MANGA_TAGGER_LOGGING_JSON").lower() == 'true': 139 | settings['logger']['json']['enabled'] = True 140 | elif os.getenv("MANGA_TAGGER_LOGGING_JSON").lower() == 'false': 141 | settings['logger']['json']['enabled'] = False 142 | if os.getenv("MANGA_TAGGER_LOGGING_TCP") is not None: 143 | if os.getenv("MANGA_TAGGER_LOGGING_TCP").lower() == 'true': 144 | settings['logger']['tcp']['enabled'] = True 145 | elif os.getenv("MANGA_TAGGER_LOGGING_TCP").lower() == 'false': 146 | settings['logger']['tcp']['enabled'] = False 147 | if os.getenv("MANGA_TAGGER_LOGGING_JSONTCP") is not None: 148 | if os.getenv("MANGA_TAGGER_LOGGING_JSONTCP").lower() == 'true': 149 | settings['logger']['json_tcp']['enabled'] = True 150 | elif os.getenv("MANGA_TAGGER_LOGGING_JSONTCP").lower() == 'false': 151 | settings['logger']['json_tcp']['enabled'] = False 152 | 153 | with open(settings_location, 'w+') as settings_json: 154 | json.dump(settings, settings_json, indent=4) 155 | 156 | cls._initialize_logger(settings['logger']) 157 | cls._log = logging.getLogger(f'{cls.__module__}.{cls.__name__}') 158 | 159 | # Database Configuration 160 | cls._log.debug('Now setting database configuration...') 161 | 162 | Database.database_name = settings['database']['database_name'] 163 | Database.host_address = settings['database']['host_address'] 164 | Database.port = settings['database']['port'] 165 | Database.username = settings['database']['username'] 166 | Database.password = settings['database']['password'] 167 | Database.auth_source = settings['database']['auth_source'] 168 | Database.server_selection_timeout_ms = settings['database']['server_selection_timeout_ms'] 169 | 170 | cls._log.debug('Database settings configured!') 171 | Database.initialize() 172 | Database.print_debug_settings() 173 | 174 | # Download Directory Configuration 175 | # Set the download directory 176 | if settings['application']['library']['download_dir'] is not None: 177 | cls.download_dir = Path(settings['application']['library']['download_dir']) 178 | if not cls.download_dir.exists(): 179 | cls._log.info(f'Library directory "{AppSettings.library_dir}" does not exist; creating now.') 180 | cls.download_dir.mkdir() 181 | QueueWorker.download_dir = cls.download_dir 182 | cls._log.info(f'Download directory has been set as "{QueueWorker.download_dir}"') 183 | else: 184 | cls._log.critical('Manga Tagger cannot function without a download directory for moving processed ' 185 | 'files into. Configure one in the "settings.json" and try again.') 186 | sys.exit(1) 187 | 188 | # Set Application Timezone 189 | cls.timezone = settings['application']['timezone'] 190 | if os.getenv('TZ') is not None: 191 | cls.timezone = os.getenv("TZ") 192 | cls._log.debug(f'Timezone: {cls.timezone}') 193 | 194 | # Dry Run Mode Configuration 195 | # No logging here due to being handled at the INFO level in MangaTaggerLib 196 | if settings['application']['dry_run']['enabled']: 197 | cls.mode_settings = {'database_insert': settings['application']['dry_run']['database_insert'], 198 | 'rename_file': settings['application']['dry_run']['rename_file'], 199 | 'write_comicinfo': settings['application']['dry_run']['write_comicinfo']} 200 | 201 | # Multithreading Configuration 202 | if settings['application']['multithreading']['threads'] <= 0: 203 | QueueWorker.threads = 1 204 | else: 205 | QueueWorker.threads = settings['application']['multithreading']['threads'] 206 | 207 | cls._log.debug(f'Threads: {QueueWorker.threads}') 208 | 209 | if settings['application']['multithreading']['max_queue_size'] < 0: 210 | QueueWorker.max_queue_size = 0 211 | else: 212 | QueueWorker.max_queue_size = settings['application']['multithreading']['max_queue_size'] 213 | 214 | cls._log.debug(f'Max Queue Size: {QueueWorker.max_queue_size}') 215 | 216 | # Debug Mode - Prevent application from processing files 217 | if settings['application']['debug_mode']: 218 | QueueWorker._debug_mode = True 219 | 220 | cls._log.debug(f'Debug Mode: {QueueWorker._debug_mode}') 221 | 222 | # Image Directory 223 | if settings['application']['image']['enabled']: 224 | cls.image = True 225 | if settings['application']['image'].get('first'): 226 | cls.image_first = True 227 | if settings['application']['image']['image_dir'] is not None: 228 | if settings['application']['image']['image_dir'] is not None: 229 | cls.image_dir = settings['application']['image']['image_dir'] 230 | if not Path(cls.image_dir).exists(): 231 | cls._log.info(f'Image directory "{cls.image_dir}" does not exist; creating now.') 232 | Path(cls.image_dir).mkdir() 233 | cls._log.debug(f'Image Directory: {cls.image_dir}') 234 | else: 235 | cls._log.critical('Image cover is enabled but cannot function without an image directory for moving downloaded cover images ' 236 | 'files into. Configure one in the "settings.json" and try again.') 237 | sys.exit(1) 238 | else: 239 | cls._log.debug(f'Image cover not enabled') 240 | 241 | # Data Dir 242 | if settings['application']['data_dir'] is not None: 243 | cls.data_dir = settings['application']['data_dir'] 244 | cls._log.debug(f'Data Directory: {cls.library_dir}') 245 | if not Path(cls.data_dir).exists(): 246 | cls._log.info(f'Data directory "{AppSettings.library_dir}" does not exist; creating now.') 247 | Path(cls.data_dir).mkdir() 248 | 249 | # Enable or disable adult result 250 | if settings['application']['adult_result']: 251 | cls.adult_result = True 252 | cls._log.info('Adult result enabled.') 253 | 254 | # Manga Library Configuration 255 | if settings['application']['library']['dir'] is not None: 256 | cls.library_dir = settings['application']['library']['dir'] 257 | cls._log.debug(f'Library Directory: {cls.library_dir}') 258 | 259 | cls.is_network_path = settings['application']['library']['is_network_path'] 260 | 261 | if not Path(cls.library_dir).exists(): 262 | cls._log.info(f'Library directory "{AppSettings.library_dir}" does not exist; creating now.') 263 | Path(cls.library_dir).mkdir() 264 | else: 265 | cls._log.critical('Manga Tagger cannot function without a library directory for moving processed ' 266 | 'files into. Configure one in the "settings.json" and try again.') 267 | sys.exit(1) 268 | 269 | # Load necessary database tables 270 | Database.load_database_tables() 271 | 272 | # Initialize QueueWorker and load task queue 273 | QueueWorker.initialize() 274 | QueueWorker.load_task_queue() 275 | 276 | # Scan download directory for downloads not already in database upon loading 277 | cls._scan_download_dir() 278 | 279 | # Initialize API 280 | AniList.initialize() 281 | 282 | # Register function to be run prior to application termination 283 | atexit.register(cls._exit_handler) 284 | cls._log.debug(f'{cls.__name__} class has been initialized') 285 | 286 | @classmethod 287 | def _initialize_logger(cls, settings): 288 | logger = logging.getLogger('MangaTaggerLib') 289 | logging_level = settings['logging_level'] 290 | log_dir = settings['log_dir'] 291 | 292 | if logging_level.lower() == 'info': 293 | logging_level = logging.INFO 294 | elif logging_level.lower() == 'debug': 295 | logging_level = logging.DEBUG 296 | else: 297 | logger.critical('Logging level not of expected values "info" or "debug". Double check the configuration' 298 | 'in settings.json and try again.') 299 | sys.exit(1) 300 | 301 | logger.setLevel(logging_level) 302 | 303 | # Create log directory and allow the application access to it 304 | if not Path(log_dir).exists(): 305 | Path(log_dir).mkdir() 306 | 307 | # Console Logging 308 | if settings['console']['enabled']: 309 | log_handler = logging.StreamHandler() 310 | log_handler.setFormatter(logging.Formatter(settings['console']['log_format'])) 311 | logger.addHandler(log_handler) 312 | 313 | # File Logging 314 | if settings['file']['enabled']: 315 | log_handler = cls._create_rotating_file_handler(log_dir, 'log', settings, 'utf-8') 316 | log_handler.setFormatter(logging.Formatter(settings['file']['log_format'])) 317 | logger.addHandler(log_handler) 318 | 319 | # JSON Logging 320 | if settings['json']['enabled']: 321 | log_handler = cls._create_rotating_file_handler(log_dir, 'json', settings) 322 | log_handler.setFormatter(jsonlogger.JsonFormatter(settings['json']['log_format'])) 323 | logger.addHandler(log_handler) 324 | 325 | # Check TCP and JSON TCP for port conflicts before creating the handlers 326 | if settings['tcp']['enabled'] and settings['json_tcp']['enabled']: 327 | if settings['tcp']['port'] == settings['json_tcp']['port']: 328 | logger.critical('TCP and JSON TCP logging are both enabled, but their port numbers are the same. ' 329 | 'Either change the port value or disable one of the handlers in settings.json ' 330 | 'and try again.') 331 | sys.exit(1) 332 | 333 | # TCP Logging 334 | if settings['tcp']['enabled']: 335 | log_handler = SocketHandler(settings['tcp']['host'], settings['tcp']['port']) 336 | log_handler.setFormatter(logging.Formatter(settings['tcp']['log_format'])) 337 | logger.addHandler(log_handler) 338 | 339 | # JSON TCP Logging 340 | if settings['json_tcp']['enabled']: 341 | log_handler = SocketHandler(settings['json_tcp']['host'], settings['json_tcp']['port']) 342 | log_handler.setFormatter(jsonlogger.JsonFormatter(settings['json_tcp']['log_format'])) 343 | logger.addHandler(log_handler) 344 | 345 | @staticmethod 346 | def _create_rotating_file_handler(log_dir, extension, settings, encoder=None): 347 | return RotatingFileHandler(Path(log_dir, f'MangaTagger.{extension}'), 348 | maxBytes=settings['max_size'], 349 | backupCount=settings['backup_count'], 350 | encoding=encoder) 351 | 352 | @classmethod 353 | def _exit_handler(cls): 354 | cls._log.info('Initiating shutdown procedures...') 355 | 356 | # Stop worker threads 357 | QueueWorker.exit() 358 | 359 | # Save necessary database tables 360 | Database.save_database_tables() 361 | 362 | # Close MongoDB connection 363 | Database.close_connection() 364 | 365 | cls._log.info('Now exiting Manga Tagger') 366 | 367 | @classmethod 368 | def _create_settings(cls): 369 | 370 | return { 371 | "application": { 372 | "debug_mode": False, 373 | "timezone": "Europe/Paris", 374 | "data_dir": "data", 375 | "image": { 376 | "enabled" : True, 377 | "image_dir" : "cover" 378 | }, 379 | "adult_result" : False, 380 | "library": { 381 | "dir": "manga", 382 | "is_network_path": False, 383 | "download_dir": "downloads" 384 | }, 385 | "dry_run": { 386 | "enabled": False, 387 | "rename_file": False, 388 | "database_insert": False, 389 | "write_comicinfo": False 390 | }, 391 | "multithreading": { 392 | "threads": 8, 393 | "max_queue_size": 0 394 | } 395 | }, 396 | "database": { 397 | "database_name": "manga_tagger", 398 | "host_address": "localhost", 399 | "port": 27017, 400 | "username": "manga_tagger", 401 | "password": "Manga4LYFE", 402 | "auth_source": "admin", 403 | "server_selection_timeout_ms": 1 404 | }, 405 | "logger": { 406 | "logging_level": "info", 407 | "log_dir": "logs", 408 | "max_size": 10485760, 409 | "backup_count": 5, 410 | "console": { 411 | "enabled": True, 412 | "log_format": "%(asctime)s | %(threadName)s %(thread)d | %(name)s | %(levelname)s - %(message)s" 413 | }, 414 | "file": { 415 | "enabled": True, 416 | "log_format": "%(asctime)s | %(threadName)s %(thread)d | %(name)s | %(levelname)s - %(message)s" 417 | }, 418 | "json": { 419 | "enabled": False, 420 | "log_format": "%(threadName)s %(thread)d %(asctime)s %(name)s %(levelname)s %(message)s" 421 | }, 422 | "tcp": { 423 | "enabled": False, 424 | "host": "localhost", 425 | "port": 1798, 426 | "log_format": "%(threadName)s %(thread)d | %(asctime)s | %(name)s | %(levelname)s - %(message)s" 427 | }, 428 | "json_tcp": { 429 | "enabled": False, 430 | "host": "localhost", 431 | "port": 1799, 432 | "log_format": "%(threadName)s %(thread)d %(asctime)s %(name)s %(levelname)s %(message)s" 433 | } 434 | } 435 | } 436 | 437 | @classmethod 438 | def _scan_download_dir(cls): 439 | for directory in QueueWorker.download_dir.iterdir(): 440 | for manga_chapter in directory.glob('*.cbz'): 441 | if manga_chapter.name.strip('.cbz') not in QueueWorker.task_list.keys(): 442 | QueueWorker.add_to_task_queue(manga_chapter) 443 | 444 | def levenshtein_distance_no_numpy(s1, s2): 445 | """ 446 | Calculates the Levenshtein distance between two strings without using NumPy. 447 | 448 | Args: 449 | s1 (str): The first string. 450 | s2 (str): The second string. 451 | 452 | Returns: 453 | int: The Levenshtein distance between the two strings. 454 | """ 455 | 456 | rows = len(s1) + 1 457 | cols = len(s2) + 1 458 | distance = [[0 for _ in range(cols)] for _ in range(rows)] 459 | 460 | for i in range(1, rows): 461 | for j in range(1, cols): 462 | if s1[i - 1] == s2[j - 1]: 463 | distance[i][j] = distance[i - 1][j - 1] 464 | else: 465 | distance[i][j] = min(distance[i - 1][j] + 1, distance[i][j - 1] + 1, distance[i - 1][j - 1] + 1) 466 | 467 | return distance[rows - 1][cols - 1] 468 | def compare(s1, s2): 469 | s1 = s1.lower().strip('/[^a-zA-Z ]/g", ') 470 | s2 = s2.lower().strip('/[^a-zA-Z ]/g", ') 471 | 472 | rows = len(s1) + 1 473 | cols = len(s2) + 1 474 | distance = levenshtein_distance_no_numpy(s1, s2) 475 | 476 | for i in range(1, rows): 477 | distance[i][0] = i 478 | 479 | for i in range(1, cols): 480 | distance[0][i] = i 481 | 482 | for col in range(1, cols): 483 | for row in range(1, rows): 484 | if s1[row - 1] == s2[col - 1]: 485 | cost = 0 486 | else: 487 | cost = 2 488 | 489 | distance[row][col] = min(distance[row - 1][col] + 1, 490 | distance[row][col - 1] + 1, 491 | distance[row - 1][col - 1] + cost) 492 | 493 | return ((len(s1) + len(s2)) - distance[row][col]) / (len(s1) + len(s2)) 494 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Manga Tagger 2 | --- 3 | 4 | [![mt-hub-img]][mt-hub-lnk] 5 | [![Python tests](https://github.com/Banh-Canh/Manga-Tagger/actions/workflows/Run_Tests.yml/badge.svg)](https://github.com/Banh-Canh/Manga-Tagger/actions/workflows/Run_Tests.yml) 6 | ## Descriptions 7 | 8 | This fork doesn't require FMD2. Running MangaTagger.py will make it watch the directory configured in the settings.json. 9 | 10 | Intended to be used in a docker container: 11 | https://hub.docker.com/r/banhcanh/manga-tagger 12 | 13 | input Files still have to be named like this (they can be in their own `%MANGA%` directory, or not) : `%MANGA% -.- %CHAPTER%.cbz` 14 | 15 | ## Features: 16 | 17 | * Does not require FMD2 18 | * Only scrapes metadata from [Anilist](https://anilist.co/) 19 | * Support for Manga/Manhwa/Manhua 20 | * Download cover image for each chapter 21 | * Slightly increased filename parsing capability 22 | * Docker image available 23 | * Manga specific configuration 24 | 25 | More infos: 26 | https://github.com/Inpacchi/Manga-Tagger 27 | 28 | [Check the wiki for install and usage instructions](https://github.com/Banh-Canh/Manga-Tagger/wiki) 29 | --- 30 | 31 | ## Aditional info 32 | 33 | I recommend using this with my FMD2 docker image: https://hub.docker.com/r/banhcanh/docker-fmd2 34 | 35 | **Note**: 36 | - Environnement Variables overwrite the settings.json. In docker, it is only possible to configure with environnement variables. 37 | - Enabling adult result may give wrong manga match. Make sure the input manga title is as accurate as possible if enabling this or it may confuse Anilist's search. 38 | 39 | ## License 40 | [MIT](https://choosealicense.com/licenses/mit/) 41 | 42 | 43 | [mt-hub-img]: https://img.shields.io/docker/pulls/banhcanh/manga-tagger.svg 44 | [mt-hub-lnk]: https://hub.docker.com/r/banhcanh/manga-tagger 45 | -------------------------------------------------------------------------------- /images/manga_tagger_logo_cropped.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MangaManagerORG/Manga-Tagger/132379ad6e8c9057ed8c079d16838b2e74f5e420/images/manga_tagger_logo_cropped.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pymongo==3.11.0 2 | watchdog==2.1.9 3 | requests==2.28.1 4 | python_json_logger==2.0.1 5 | pytz==2020.1 6 | pillow 7 | BeautifulSoup4==4.9.3 8 | psutil 9 | -------------------------------------------------------------------------------- /root/etc/cont-init.d/30-config: -------------------------------------------------------------------------------- 1 | #!/usr/bin/with-contenv bash 2 | 3 | # permissions 4 | chown -R abc:abc \ 5 | /config 6 | chown -R abc:abc \ 7 | /downloads 8 | chown -R abc:abc \ 9 | /app/Manga-Tagger 10 | -------------------------------------------------------------------------------- /root/etc/services.d/mangatagger/run: -------------------------------------------------------------------------------- 1 | #!/usr/bin/with-contenv bash 2 | 3 | if [ -n "${UMASK_SET}" ] && [ -z "${UMASK}" ]; then 4 | echo -e "You are using a legacy method of defining umask\nplease update your environment variable from UMASK_SET to UMASK\nto keep the functionality after July 2021" 5 | umask ${UMASK_SET} 6 | fi 7 | 8 | cd /app/Manga-Tagger 9 | 10 | exec \ 11 | s6-setuidgid abc python3 /app/Manga-Tagger/MangaTagger.py 12 | -------------------------------------------------------------------------------- /settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "application": { 3 | "debug_mode": false, 4 | "timezone": "Europe/Paris", 5 | "data_dir": "data", 6 | "image": { 7 | "enabled": true, 8 | "image_dir": "cover" 9 | }, 10 | "adult_result": false, 11 | "library": { 12 | "dir": "manga", 13 | "is_network_path": false, 14 | "download_dir": "downloads" 15 | }, 16 | "dry_run": { 17 | "enabled": false, 18 | "rename_file": false, 19 | "database_insert": false, 20 | "write_comicinfo": false 21 | }, 22 | "multithreading": { 23 | "threads": 8, 24 | "max_queue_size": 0 25 | } 26 | }, 27 | "database": { 28 | "database_name": "manga_tagger", 29 | "host_address": "localhost", 30 | "port": 27017, 31 | "username": "manga_tagger", 32 | "password": "Manga4LYFE", 33 | "auth_source": "admin", 34 | "server_selection_timeout_ms": 1 35 | }, 36 | "logger": { 37 | "logging_level": "info", 38 | "log_dir": "logs", 39 | "max_size": 10485760, 40 | "backup_count": 5, 41 | "console": { 42 | "enabled": true, 43 | "log_format": "%(asctime)s | %(threadName)s %(thread)d | %(name)s | %(levelname)s - %(message)s" 44 | }, 45 | "file": { 46 | "enabled": true, 47 | "log_format": "%(asctime)s | %(threadName)s %(thread)d | %(name)s | %(levelname)s - %(message)s" 48 | }, 49 | "json": { 50 | "enabled": false, 51 | "log_format": "%(threadName)s %(thread)d %(asctime)s %(name)s %(levelname)s %(message)s" 52 | }, 53 | "tcp": { 54 | "enabled": false, 55 | "host": "localhost", 56 | "port": 1798, 57 | "log_format": "%(threadName)s %(thread)d | %(asctime)s | %(name)s | %(levelname)s - %(message)s" 58 | }, 59 | "json_tcp": { 60 | "enabled": false, 61 | "host": "localhost", 62 | "port": 1799, 63 | "log_format": "%(threadName)s %(thread)d %(asctime)s %(name)s %(levelname)s %(message)s" 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MangaManagerORG/Manga-Tagger/132379ad6e8c9057ed8c079d16838b2e74f5e420/tests/__init__.py -------------------------------------------------------------------------------- /tests/data/3D Kanojo Real Girl/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 80767, 3 | "status": "FINISHED", 4 | "synonyms": ["3D Girlfriend"], 5 | "volumes": 13, 6 | "siteUrl": "https://anilist.co/manga/80767", 7 | "title": { 8 | "romaji": "3D Kanojo: Real Girl", 9 | "english": "Real Girl", 10 | "native": "３Ｄ彼女　リアルガール" 11 | }, 12 | "type": "MANGA", 13 | "genres": ["Comedy", "Drama", "Romance", "Slice of Life"], 14 | "startDate": { 15 | "day": 23, 16 | "month": 7, 17 | "year": 2011 18 | }, 19 | "coverImage": { 20 | "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/bx80767-U74GsMO3XpQe.jpg" 21 | }, 22 | "staff": { 23 | "edges": [ 24 | { 25 | "node": { 26 | "name": { 27 | "first": "Mao", 28 | "last": "Nanami", 29 | "full": "Mao Nanami", 30 | "alternative": [ 31 | "" 32 | ] 33 | }, 34 | "siteUrl": "https://anilist.co/staff/107286" 35 | }, 36 | "role": "Story & Art" 37 | } 38 | ] 39 | }, 40 | "description": "Tsutsui Hikari is an otaku, and he mostly avoids social life. Hikari has only one friend at school, who is also a social misfit, and he is mocked brutally by most of his classmates for being creepy and weird. One day, he ends up having to clean the school pool with Igarashi Iroha, who appears to be pretty much everything he hates in real-life girls. She skips school, has a blunt manner, doesn't have female friends, and seems the sort to be promiscuous. However, she is friendly to Hikari, and even stands up to the people who make fun of him. Hikari's bitterness and trust issues lead him to say pretty harsh things to Iroha, but she never dismisses him as creepy. After a while, it starts to look like Iroha may become his first real-life, 3D girlfriend! Will he be able to handle it?

\n(Source: tethysdust)" 41 | } 42 | -------------------------------------------------------------------------------- /tests/data/BLEACH/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 30012, 3 | "status": "FINISHED", 4 | "synonyms": ["ブリーチ", "بلیچ", "سفید کننده"], 5 | "volumes": 74, 6 | "siteUrl": "https://anilist.co/manga/30012", 7 | "title": { 8 | "romaji": "BLEACH", 9 | "english": "Bleach", 10 | "native": "BLEACH" 11 | }, 12 | "type": "MANGA", 13 | "genres": [ 14 | "Action", 15 | "Adventure", 16 | "Supernatural" 17 | ], 18 | "startDate": { 19 | "day": 7, 20 | "month": 8, 21 | "year": 2001 22 | }, 23 | "coverImage": { 24 | "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/bx30012-z7U138mUaPdN.png" 25 | }, 26 | "staff": { 27 | "edges": [ 28 | { 29 | "node": { 30 | "name": { 31 | "first": "Tite", 32 | "last": "Kubo", 33 | "full": "Tite Kubo", 34 | "alternative": [ 35 | "Taito Kubo", 36 | "Noriaki Kubo (\u4e45\u4fdd\u5ba3\u7ae0)" 37 | ] 38 | }, 39 | "siteUrl": "https://anilist.co/staff/96880" 40 | }, 41 | "role": "Story & Art" 42 | }, 43 | { 44 | "node": { 45 | "name": { 46 | "first": "Ricardo", 47 | "last": "Cruz", 48 | "full": "Ricardo Cruz", 49 | "alternative": [] 50 | }, 51 | "siteUrl": "https://anilist.co/staff/104809" 52 | }, 53 | "role": "Translator (Portuguese: vols 1-24)" 54 | }, 55 | { 56 | "node": { 57 | "name": { 58 | "first": "Drik", 59 | "last": "Sada", 60 | "full": "Drik Sada", 61 | "alternative": [ 62 | "Adriana Kazue Sada" 63 | ] 64 | }, 65 | "siteUrl": "https://anilist.co/staff/220675" 66 | }, 67 | "role": "Translator (Portuguese: vols 25-52, 71-74)" 68 | }, 69 | { 70 | "node": { 71 | "name": { 72 | "first": "Christine", 73 | "last": "Dashiell", 74 | "full": "Christine Dashiell", 75 | "alternative": [ 76 | "Christine Schilling" 77 | ] 78 | }, 79 | "siteUrl": "https://anilist.co/staff/219441" 80 | }, 81 | "role": "Translator (English: vol 49)" 82 | }, 83 | { 84 | "node": { 85 | "name": { 86 | "first": "Joe", 87 | "last": "Yamazaki", 88 | "full": "Joe Yamazaki", 89 | "alternative": [] 90 | }, 91 | "siteUrl": "https://anilist.co/staff/218469" 92 | }, 93 | "role": "Translator (English)" 94 | }, 95 | { 96 | "node": { 97 | "name": { 98 | "first": "Mark", 99 | "last": "McMurray", 100 | "full": "Mark McMurray", 101 | "alternative": [] 102 | }, 103 | "siteUrl": "https://anilist.co/staff/224874" 104 | }, 105 | "role": "Touch-up Art & Lettering (English: vols 16, 20)" 106 | }, 107 | { 108 | "node": { 109 | "name": { 110 | "first": "Mark", 111 | "last": "McMurray", 112 | "full": "Mark McMurray", 113 | "alternative": [] 114 | }, 115 | "siteUrl": "https://anilist.co/staff/224874" 116 | }, 117 | "role": "Lettering (English: vols: 16, 17, 19, 20, 22, 23, 24)" 118 | }, 119 | { 120 | "node": { 121 | "name": { 122 | "first": "Evan", 123 | "last": "Waldinger", 124 | "full": "Evan Waldinger", 125 | "alternative": [] 126 | }, 127 | "siteUrl": "https://anilist.co/staff/227608" 128 | }, 129 | "role": "Touch-up Art & Lettering (English: vol 21)" 130 | }, 131 | { 132 | "node": { 133 | "name": { 134 | "first": "Dave", 135 | "last": "Lanphear", 136 | "full": "Dave Lanphear", 137 | "alternative": [] 138 | }, 139 | "siteUrl": "https://anilist.co/staff/227606" 140 | }, 141 | "role": "Touch-up Art & Lettering (English: vols 3, 5, 6, 7, 8)" 142 | }, 143 | { 144 | "node": { 145 | "name": { 146 | "first": "Andy", 147 | "last": "Ristaino", 148 | "full": "Andy Ristaino", 149 | "alternative": [] 150 | }, 151 | "siteUrl": "https://anilist.co/staff/227607" 152 | }, 153 | "role": "Touch-up Art & Lettering (English)" 154 | }, 155 | { 156 | "node": { 157 | "name": { 158 | "first": "Marta", 159 | "last": "Gallego", 160 | "full": "Marta Gallego", 161 | "alternative": [] 162 | }, 163 | "siteUrl": "https://anilist.co/staff/225332" 164 | }, 165 | "role": "Translator (Spanish)" 166 | }, 167 | { 168 | "node": { 169 | "name": { 170 | "first": "Marc", 171 | "last": "Bernab\u00e9", 172 | "full": "Marc Bernab\u00e9", 173 | "alternative": [] 174 | }, 175 | "siteUrl": "https://anilist.co/staff/214718" 176 | }, 177 | "role": "Translator (Spanish)" 178 | }, 179 | { 180 | "node": { 181 | "name": { 182 | "first": "Ver\u00f3nica", 183 | "last": "Calafell", 184 | "full": "Ver\u00f3nica Calafell", 185 | "alternative": [ 186 | "Ver\u00f3nica Calafell Callejo" 187 | ] 188 | }, 189 | "siteUrl": "https://anilist.co/staff/225627" 190 | }, 191 | "role": "Translator (Spanish)" 192 | }, 193 | { 194 | "node": { 195 | "name": { 196 | "first": "Simona", 197 | "last": "Stanzani", 198 | "full": "Simona Stanzani", 199 | "alternative": [] 200 | }, 201 | "siteUrl": "https://anilist.co/staff/225407" 202 | }, 203 | "role": "Translator (Italian)" 204 | }, 205 | { 206 | "node": { 207 | "name": { 208 | "first": "Rafael", 209 | "last": "Morata", 210 | "full": "Rafael Morata", 211 | "alternative": [] 212 | }, 213 | "siteUrl": "https://anilist.co/staff/227609" 214 | }, 215 | "role": "Translator (Spanish)" 216 | }, 217 | { 218 | "node": { 219 | "name": { 220 | "first": "Daniel", 221 | "last": "B\u00fcchner", 222 | "full": "Daniel B\u00fcchner", 223 | "alternative": [] 224 | }, 225 | "siteUrl": "https://anilist.co/staff/227610" 226 | }, 227 | "role": "Translator (German)" 228 | }, 229 | { 230 | "node": { 231 | "name": { 232 | "first": "Kentarou", 233 | "last": "Kurimoto", 234 | "full": "Kentarou Kurimoto", 235 | "alternative": [] 236 | }, 237 | "siteUrl": "https://anilist.co/staff/119236" 238 | }, 239 | "role": "Assistant" 240 | }, 241 | { 242 | "node": { 243 | "name": { 244 | "first": "Shou", 245 | "last": "Aimoto", 246 | "full": "Shou Aimoto", 247 | "alternative": [] 248 | }, 249 | "siteUrl": "https://anilist.co/staff/98753" 250 | }, 251 | "role": "Assistant" 252 | }, 253 | { 254 | "node": { 255 | "name": { 256 | "first": "Hitoshi", 257 | "last": "Imoto", 258 | "full": "Hitoshi Imoto", 259 | "alternative": [] 260 | }, 261 | "siteUrl": "https://anilist.co/staff/126703" 262 | }, 263 | "role": "Assistant" 264 | }, 265 | { 266 | "node": { 267 | "name": { 268 | "first": "Anne-Sophie", 269 | "last": "Thevenon", 270 | "full": "Anne-Sophie Thevenon", 271 | "alternative": [ 272 | "Anne-Sophie Th\u00e9venon" 273 | ] 274 | }, 275 | "siteUrl": "https://anilist.co/staff/236835" 276 | }, 277 | "role": "Translator (French)" 278 | }, 279 | { 280 | "node": { 281 | "name": { 282 | "first": "Shou", 283 | "last": "Shiromoto", 284 | "full": "Shou Shiromoto", 285 | "alternative": [] 286 | }, 287 | "siteUrl": "https://anilist.co/staff/249345" 288 | }, 289 | "role": "Assistant" 290 | }, 291 | { 292 | "node": { 293 | "name": { 294 | "first": "Tayfun", 295 | "last": "Nitahara Haks\u00f6yliyen", 296 | "full": "Tayfun Nitahara Haks\u00f6yliyen", 297 | "alternative": [ 298 | "Tayfun Nitahara Haksoyliyen" 299 | ] 300 | }, 301 | "siteUrl": "https://anilist.co/staff/275233" 302 | }, 303 | "role": "Translator (Turkish: vols 1-17)" 304 | } 305 | ] 306 | }, 307 | "description": "Ichigo Kurosaki has always been able to see ghosts, but this ability doesn't change his life nearly as much as his close encounter with Rukia Kuchiki, a Soul Reaper and member of the mysterious Soul Society. While fighting a Hollow, an evil spirit that preys on humans who display psychic energy, Rukia attempts to lend Ichigo some of her powers so that he can save his family; but much to her surprise, Ichigo absorbs every last drop of her energy. Now a full-fledged Soul Reaper himself, Ichigo quickly learns that the world he inhabits is one full of dangerous spirits and, along with Rukia\u2014 who is slowly regaining her powers\u2014 it's Ichigo's job to protect the innocent from Hollows and help the spirits themselves find peace.

\n(Source: Anime News Network)

\nNote: Chapter count includes the 12-chapter \u201cTurn Back The Pendulum\u201d side story and 8 extra chapters." 308 | } 309 | -------------------------------------------------------------------------------- /tests/data/Hurejasik/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 86964, 3 | "status": "FINISHED", 4 | "synonyms": ["BASTARD", "不肖子", "Ублюдок", "Bâtard", "Bastardo", "バスタード"], 5 | "volumes": 5, 6 | "siteUrl": "https://anilist.co/manga/86964", 7 | "title": { 8 | "romaji": "Hurejasik", 9 | "english": "Bastard", 10 | "native": "후레자식" 11 | }, 12 | "type": "MANGA", 13 | "genres": [ 14 | "Drama", 15 | "Horror", 16 | "Mystery", 17 | "Psychological", 18 | "Romance", 19 | "Thriller" 20 | ], 21 | "startDate": { 22 | "day": 4, 23 | "month": 7, 24 | "year": 2014 25 | }, 26 | "coverImage": { 27 | "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/nx86964-r7S3IbJNr4SD.jpg" 28 | }, 29 | "staff": { 30 | "edges": [ 31 | { 32 | "node": { 33 | "name": { 34 | "first": "Yeong-Chan", 35 | "last": "Hwang", 36 | "full": "Yeong-Chan Hwang", 37 | "alternative": [ 38 | "" 39 | ] 40 | }, 41 | "siteUrl": "https://anilist.co/staff/119717" 42 | }, 43 | "role": "Art" 44 | }, 45 | { 46 | "node": { 47 | "name": { 48 | "first": "Carnby", 49 | "last": "Kim", 50 | "full": "Carnby Kim", 51 | "alternative": [ 52 | "김칸비" 53 | ] 54 | }, 55 | "siteUrl": "https://anilist.co/staff/119718" 56 | }, 57 | "role": "Story" 58 | } 59 | ] 60 | }, 61 | "description": "There is nowhere that Seon Jin can find solace. At school, he is ruthlessly bullied due to his unsettlingly quiet nature and weak appearance. However, this is not the source of Jin's insurmountable terror: the thing that he fears more than anything else is his own father.

\nTo most, Jin's father is a successful businessman, good samaritan, and doting parent. But that is merely a facade; in truth, he is a deranged serial killer—and Jin is his unwilling accomplice. For years, they have been carrying out this ruse with the police being none the wiser. However, when his father takes an interest in the pretty transfer student Yoon Kyun, Jin must make a decision—be the coward who sends her to the gallows like all the rest, or be the bastard of a son who defies his wicked parent.

\n(Source: MAL Rewrite)

\nNote: Includes the prologue." 62 | } 63 | -------------------------------------------------------------------------------- /tests/data/Naruto/data.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 30011, 3 | "status": "FINISHED", 4 | "synonyms": ["נארוטו", "Наруто"], 5 | "volumes": 72, 6 | "siteUrl": "https://anilist.co/manga/30011", 7 | "title": { 8 | "romaji": "NARUTO", 9 | "english": "Naruto", 10 | "native": "NARUTO -ナルト-" 11 | }, 12 | "type": "MANGA", 13 | "genres": [ 14 | "Action", 15 | "Adventure" 16 | ], 17 | "startDate": { 18 | "day": 21, 19 | "month": 9, 20 | "year": 1999 21 | }, 22 | "coverImage": { 23 | "extraLarge": "https://s4.anilist.co/file/anilistcdn/media/manga/cover/large/nx30011-9yUF1dXWgDOx.jpg" 24 | }, 25 | "staff": { 26 | "edges": [ 27 | { 28 | "node": { 29 | "name": { 30 | "first": "Masashi", 31 | "last": "Kishimoto", 32 | "full": "Masashi Kishimoto", 33 | "alternative": [ 34 | "" 35 | ] 36 | }, 37 | "siteUrl": "https://anilist.co/staff/96879" 38 | }, 39 | "role": "Story & Art" 40 | }, 41 | { 42 | "node": { 43 | "name": { 44 | "first": "Kazuhiro", 45 | "last": "Takahashi", 46 | "full": "Kazuhiro Takahashi", 47 | "alternative": [ 48 | "" 49 | ] 50 | }, 51 | "siteUrl": "https://anilist.co/staff/137325" 52 | }, 53 | "role": "Assistant (beta)" 54 | }, 55 | { 56 | "node": { 57 | "name": { 58 | "first": "Osamu", 59 | "last": "Kazisa", 60 | "full": "Osamu Kazisa", 61 | "alternative": [ 62 | "" 63 | ] 64 | }, 65 | "siteUrl": "https://anilist.co/staff/137326" 66 | }, 67 | "role": "Assistant (beta)" 68 | }, 69 | { 70 | "node": { 71 | "name": { 72 | "first": "Mikio", 73 | "last": "Ikemoto", 74 | "full": "Mikio Ikemoto", 75 | "alternative": [ 76 | "" 77 | ] 78 | }, 79 | "siteUrl": "https://anilist.co/staff/137327" 80 | }, 81 | "role": "Assistant (mob)" 82 | } 83 | ] 84 | }, 85 | "description": "Before Naruto's birth, a great demon fox had attacked the Hidden Leaf Village. A man known as the 4th Hokage sealed the demon inside the newly born Naruto, causing him to unknowingly grow up detested by his fellow villagers. Despite his lack of talent in many areas of ninjutsu, Naruto strives for only one goal: to gain the title of Hokage, the strongest ninja in his village. Desiring the respect he never received, Naruto works towards his dream with fellow friends Sasuke and Sakura and mentor Kakashi as they go through many trials and battles that come with being a ninja. " 86 | } 87 | -------------------------------------------------------------------------------- /tests/database.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | 4 | class ProcFilesTable: 5 | @classmethod 6 | def search_return_results(cls, manga_title, chapter_number): 7 | return { 8 | "chapter_number": "001", 9 | "new_filename": "Absolute Boyfriend 001.cbz", 10 | "old_filename": "Absolute Boyfriend -.- Absolute Boyfriend 01 Lover Shop.cbz", 11 | "process_date": datetime.now().strftime('%Y-%m-%d'), 12 | "series_title": "Absolute Boyfriend" 13 | } 14 | 15 | @classmethod 16 | def search_return_results_version(cls, manga_title, chapter_number): 17 | return { 18 | "chapter_number": "001", 19 | "new_filename": "Absolute Boyfriend 001.cbz", 20 | "old_filename": "Absolute Boyfriend -.- Absolute Boyfriend 01 Lover Shop v2.cbz", 21 | "process_date": datetime.now().strftime('%Y-%m-%d'), 22 | "series_title": "Absolute Boyfriend" 23 | } 24 | 25 | @classmethod 26 | def search_return_no_results(cls, manga_title, chapter_number): 27 | return None 28 | 29 | 30 | class MetadataTable: 31 | @classmethod 32 | def search_return_no_results(cls, manga_title): 33 | return None 34 | -------------------------------------------------------------------------------- /tests/test_integration.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import os 4 | import unittest 5 | from pathlib import Path 6 | from unittest.mock import patch 7 | 8 | from MangaTaggerLib.api import AniList 9 | from MangaTaggerLib.MangaTaggerLib import metadata_tagger, construct_comicinfo_xml 10 | from MangaTaggerLib.models import Metadata 11 | from tests.database import MetadataTable as MetadataTableTest 12 | 13 | 14 | # noinspection DuplicatedCode 15 | class TestMetadata(unittest.TestCase): 16 | @classmethod 17 | def setUpClass(cls) -> None: 18 | cls.data_dir = 'tests/data' 19 | cls.data_file = 'data.json' 20 | 21 | def setUp(self) -> None: 22 | logging.disable(logging.CRITICAL) 23 | AniList.initialize() 24 | self.maxDiff = None 25 | patch1 = patch('MangaTaggerLib.models.AppSettings') 26 | self.models_AppSettings = patch1.start() 27 | self.addCleanup(patch1.stop) 28 | self.models_AppSettings.timezone = 'America/New_York' 29 | 30 | patch2 = patch('MangaTaggerLib.MangaTaggerLib.MetadataTable') 31 | self.MetadataTable = patch2.start() 32 | self.addCleanup(patch2.stop) 33 | self.MetadataTable.search_by_search_value = MetadataTableTest.search_return_no_results 34 | self.MetadataTable.search_by_series_title = MetadataTableTest.search_return_no_results 35 | self.MetadataTable.search_by_series_title_eng = MetadataTableTest.search_return_no_results 36 | 37 | patch3 = patch('MangaTaggerLib.MangaTaggerLib.AppSettings') 38 | self.MangaTaggerLib_AppSettings = patch3.start() 39 | self.addCleanup(patch3.stop) 40 | 41 | def test_comicinfo_xml_creation_case_1(self): 42 | title = 'BLEACH' 43 | 44 | self.MangaTaggerLib_AppSettings.mode_settings = {} 45 | 46 | with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: 47 | anilist_details = json.load(data) 48 | 49 | manga_metadata = Metadata(title, {}, anilist_details) 50 | 51 | self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {}, None)) 52 | 53 | def test_comicinfo_xml_creation_case_2(self): 54 | title = 'Naruto' 55 | 56 | self.MangaTaggerLib_AppSettings.mode_settings = {} 57 | 58 | with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: 59 | anilist_details = json.load(data) 60 | 61 | manga_metadata = Metadata(title, {}, anilist_details) 62 | 63 | self.assertTrue(construct_comicinfo_xml(manga_metadata, '001', {}, None)) 64 | 65 | def test_metadata_case_1(self): 66 | title = 'BLEACH' 67 | 68 | self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} 69 | self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} 70 | 71 | with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: 72 | anilist_details = json.load(data) 73 | 74 | expected_manga_metadata = Metadata(title, {}, anilist_details) 75 | actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {}, None) 76 | 77 | self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) 78 | 79 | def test_metadata_case_2(self): 80 | title = 'Naruto' 81 | 82 | self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} 83 | self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} 84 | 85 | with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: 86 | anilist_details = json.load(data) 87 | 88 | expected_manga_metadata = Metadata(title, {}, anilist_details) 89 | actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "MANGA", {}, None) 90 | 91 | self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) 92 | 93 | def test_metadata_case_3(self): 94 | title = '3D Kanojo Real Girl' 95 | downloaded_title = '3D Kanojo' 96 | 97 | self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} 98 | self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} 99 | self.MangaTaggerLib_AppSettings.adult_result = False 100 | 101 | with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: 102 | anilist_details = json.load(data) 103 | 104 | expected_manga_metadata = Metadata(title, {}, anilist_details) 105 | actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {}, None) 106 | 107 | self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) 108 | 109 | def test_metadata_case_4(self): 110 | title = 'Hurejasik' 111 | downloaded_title = 'Bastard' 112 | 113 | self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} 114 | self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} 115 | 116 | with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: 117 | anilist_details = json.load(data) 118 | 119 | expected_manga_metadata = Metadata(title, {}, anilist_details) 120 | actual_manga_metadata = metadata_tagger("NOWHERE", downloaded_title, '001', "MANGA", {}, None) 121 | 122 | self.assertEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) 123 | 124 | def test_metadata_case_5(self): 125 | title = 'Naruto' 126 | 127 | self.MangaTaggerLib_AppSettings.mode_settings = {'write_comicinfo': False} 128 | self.MangaTaggerLib_AppSettings.mode_settings = {'rename_file': False} 129 | 130 | with open(Path(self.data_dir, title, self.data_file), encoding='utf-8') as data: 131 | anilist_details = json.load(data) 132 | 133 | expected_manga_metadata = Metadata(title, {}, anilist_details) 134 | actual_manga_metadata = metadata_tagger("NOWHERE", title, '001', "ONE_SHOT", {}, None) 135 | 136 | self.assertNotEqual(expected_manga_metadata.test_value(), actual_manga_metadata.test_value()) 137 | -------------------------------------------------------------------------------- /tests/test_manga.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import logging 3 | import shutil 4 | from pathlib import Path 5 | from typing import List 6 | from unittest.mock import patch 7 | 8 | from MangaTaggerLib.MangaTaggerLib import filename_parser, rename_action 9 | from MangaTaggerLib.errors import FileAlreadyProcessedError, FileUpdateNotRequiredError 10 | from tests.database import ProcFilesTable as ProcFilesTableTest 11 | 12 | class FilenameParserTestCase(unittest.TestCase): 13 | 14 | def test_filename_parser_manga(self): 15 | filename = "Naruto -.- Chap 0.cbz" 16 | directory_name = "Naruto" 17 | logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } 18 | expected_result = ("Naruto", "000", "MANGA",None) 19 | result = filename_parser(filename, logging_info) 20 | self.assertEqual(expected_result, result) 21 | 22 | def test_filename_parser_decnumber(self): 23 | filename = "Naruto -.- Chap 15.5.cbz" 24 | directory_name = "Naruto" 25 | logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } 26 | expected_result = ("Naruto", "015.5", "MANGA",None) 27 | result = filename_parser(filename, logging_info) 28 | self.assertEqual(expected_result, result) 29 | 30 | def test_filename_parser_oneshot(self): 31 | ## Naruto Oneshot for the same author 32 | filename = "Naruto -.- Oneshot.cbz" 33 | directory_name = "Naruto" 34 | logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } 35 | expected_result = ("Naruto", "000", "ONE_SHOT",None) 36 | result = filename_parser(filename, logging_info) 37 | self.assertEqual(expected_result, result) 38 | 39 | def test_filename_parser_prologue(self): 40 | ## Berserk Prologue 1-16 goes back to chapter 1 once the prologue ends. 41 | filename = "Berserk -.- Prologue 5.cbz" 42 | directory_name = "Berserk" 43 | logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } 44 | expected_result = ("Berserk", "000.5", "MANGA",None) 45 | result = filename_parser(filename, logging_info) 46 | self.assertEqual(expected_result, result) 47 | 48 | def test_filename_parser_ignore_fluff(self): 49 | ## Ignore Volume, chapter name and (part) 50 | filename = "One Piece -.- Volume 50 Episode 156 A Chapter Name (15).cbz" 51 | directory_name = "Naruto" 52 | logging_info = { 'event_id': 0, 'manga_title': directory_name, "original_filename": filename } 53 | expected_result = ("One Piece", "156", "MANGA","50") 54 | result = filename_parser(filename, logging_info) 55 | self.assertEqual(expected_result, result) 56 | 57 | def test_filename_parser_ignore_fluff_2(self): 58 | ## Ignore Volume, chapter name and (part) 59 | filename = "Kuma Kuma Kuma Bear -.- Ch. 064 - Kuma-san and the Shop's Opening Day 2.cbz" 60 | directory_name = "Kuma Kuma Kuma Bear" 61 | logging_info = {'event_id': 0, 'manga_title': directory_name, "original_filename": filename} 62 | expected_result = ("Kuma Kuma Kuma Bear", "064", "MANGA", None) 63 | result = filename_parser(filename, logging_info) 64 | self.assertEqual(expected_result, result) 65 | class TestMangaRenameAction(unittest.TestCase): 66 | download_dir = Path('tests/downloads') 67 | library_dir = Path('tests/library') 68 | current_file = None 69 | new_file = None 70 | 71 | @classmethod 72 | def setUpClass(cls) -> None: 73 | logging.disable(logging.CRITICAL) 74 | cls.current_file = Path(cls.download_dir, 'Absolute Boyfriend -.- Absolute Boyfriend 01 Lover Shop.cbz') 75 | cls.new_file = Path(cls.library_dir, 'Absolute Boyfriend 001.cbz') 76 | 77 | def setUp(self) -> None: 78 | self.download_dir.mkdir() 79 | self.library_dir.mkdir() 80 | 81 | def tearDown(self) -> None: 82 | shutil.rmtree(self.download_dir) 83 | shutil.rmtree(self.library_dir) 84 | 85 | @patch('MangaTaggerLib.MangaTaggerLib.ProcFilesTable') 86 | @patch('MangaTaggerLib.MangaTaggerLib.CURRENTLY_PENDING_RENAME', new_callable=list) 87 | def test_rename_action_initial(self, CURRENTLY_PENDING_RENAME: List, ProcFilesTable): 88 | """ 89 | Tests for initial file rename when no results are returned from the database. Test should execute without error. 90 | """ 91 | self.current_file.touch() 92 | ProcFilesTable.search = ProcFilesTableTest.search_return_no_results 93 | 94 | CURRENTLY_PENDING_RENAME.append(self.new_file) 95 | 96 | self.assertFalse(rename_action(self.current_file, self.new_file, 'Absolute Boyfriend', '01', {})) 97 | 98 | @patch('MangaTaggerLib.MangaTaggerLib.ProcFilesTable') 99 | def test_rename_action_duplicate(self, ProcFilesTable): 100 | """ 101 | Tests for duplicate file rename when results are returned from the database. Test should assert 102 | FileAlreadyProcessedError. 103 | """ 104 | self.current_file.touch() 105 | ProcFilesTable.search = ProcFilesTableTest.search_return_results 106 | 107 | with self.assertRaises(FileAlreadyProcessedError): 108 | rename_action(self.current_file, self.new_file, 'Absolute Boyfriend', '01', {}) 109 | 110 | @patch('MangaTaggerLib.MangaTaggerLib.ProcFilesTable') 111 | def test_rename_action_downgrade(self, ProcFilesTable): 112 | """ 113 | Tests for version in file rename when results are returned from the database. Since the current file is a 114 | lower version than the existing file, test should assert FileUpdateNotRequiredError. 115 | """ 116 | self.current_file.touch() 117 | ProcFilesTable.search = ProcFilesTableTest.search_return_results_version 118 | 119 | with self.assertRaises(FileUpdateNotRequiredError): 120 | rename_action(self.current_file, self.new_file, 'Absolute Boyfriend', '01', {}) 121 | 122 | @patch('MangaTaggerLib.MangaTaggerLib.ProcFilesTable') 123 | def test_rename_action_version_duplicate(self, ProcFilesTable): 124 | """ 125 | Tests for version and duplicate file rename when results are returned from the database. Since the current 126 | version is the same as the one in the database, test should assert FileUpdateNotRequiredError. 127 | """ 128 | self.current_file = Path(self.download_dir, 'Absolute Boyfriend -.- Absolute Boyfriend 01 Lover Shop v2.cbz') 129 | self.current_file.touch() 130 | 131 | ProcFilesTable.search = ProcFilesTableTest.search_return_results_version 132 | 133 | with self.assertRaises(FileUpdateNotRequiredError): 134 | rename_action(self.current_file, self.new_file, 'Absolute Boyfriend', '01', {}) 135 | 136 | @patch('MangaTaggerLib.MangaTaggerLib.ProcFilesTable') 137 | @patch('MangaTaggerLib.MangaTaggerLib.CURRENTLY_PENDING_RENAME', new_callable=list) 138 | def test_rename_action_upgrade(self, CURRENTLY_PENDING_RENAME: List, ProcFilesTable): 139 | """ 140 | Tests for version in file rename when results are returned from the database. Since the current file is a 141 | higher version than the exisitng file, test should execute without error. 142 | """ 143 | self.current_file = Path(self.download_dir, 'Absolute Boyfriend -.- Absolute Boyfriend 01 Lover Shop v3.cbz') 144 | self.current_file.touch() 145 | 146 | self.new_file.touch() 147 | CURRENTLY_PENDING_RENAME.append(self.new_file) 148 | 149 | ProcFilesTable.search = ProcFilesTableTest.search_return_results_version 150 | 151 | self.assertFalse(rename_action(self.current_file, self.new_file, 'Absolute Boyfriend', '01', {})) 152 | 153 | 154 | 155 | --------------------------------------------------------------------------------