├── .envrc ├── docs ├── authors.rst ├── history.rst ├── readme.rst ├── contributing.rst ├── usage.rst ├── index.rst ├── Makefile ├── make.bat ├── installation.rst └── conf.py ├── tests ├── __init__.py └── test_rest_uploader.py ├── requirements.txt ├── .pyup.yml ├── requirements_dev.txt ├── rest_uploader ├── __init__.py ├── api_token.py ├── email_processor.py ├── cli.py └── rest_uploader.py ├── AUTHORS.rst ├── MANIFEST.in ├── tox.ini ├── .vscode ├── launch.json └── settings.json ├── pyproject.toml ├── setup.cfg ├── rest_uploader.service ├── LICENSE ├── .gitignore ├── setup.py ├── Makefile ├── HISTORY.md ├── README.md └── CONTRIBUTING.rst /.envrc: -------------------------------------------------------------------------------- 1 | layout python 2 | -------------------------------------------------------------------------------- /docs/authors.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../AUTHORS.rst 2 | -------------------------------------------------------------------------------- /docs/history.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../HISTORY.rst 2 | -------------------------------------------------------------------------------- /docs/readme.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../README.rst 2 | -------------------------------------------------------------------------------- /docs/contributing.rst: -------------------------------------------------------------------------------- 1 | .. include:: ../CONTRIBUTING.rst 2 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Unit test package for rest_uploader.""" 4 | -------------------------------------------------------------------------------- /docs/usage.rst: -------------------------------------------------------------------------------- 1 | ===== 2 | Usage 3 | ===== 4 | 5 | To use rest_uploader in a project:: 6 | 7 | import rest_uploader 8 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | watchdog>=3.0.0 2 | requests>=2.31.0 3 | tabulate>=0.9.0 4 | click>=8.1.3 5 | img_processor>=0.20.0 6 | -------------------------------------------------------------------------------- /.pyup.yml: -------------------------------------------------------------------------------- 1 | # autogenerated pyup.io config file 2 | # see https://pyup.io/docs/configuration/ for all available options 3 | 4 | schedule: every week 5 | update: false 6 | -------------------------------------------------------------------------------- /requirements_dev.txt: -------------------------------------------------------------------------------- 1 | pip>=23.1.2 2 | bumpversion>=0.6.0 3 | wheel>=0.38.4 4 | flake8>=6.0.0 5 | tox>=4.6.3 6 | coverage>=7.2.7 7 | twine>=4.0.2 8 | keyring>=24.0.0 9 | imbox>=0.9.8 -------------------------------------------------------------------------------- /rest_uploader/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Top-level package for rest_uploader.""" 4 | 5 | __author__ = """Justin Keller""" 6 | __email__ = "kellerjustin@protonmail.com" 7 | __version__ = "__version__ = '1.22.0'" 8 | -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Credits 3 | ======= 4 | 5 | Development Lead 6 | ---------------- 7 | 8 | * Justin Keller 9 | 10 | Contributors 11 | ------------ 12 | 13 | None yet. Why not be the first? 14 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include AUTHORS.rst 2 | include CONTRIBUTING.rst 3 | include HISTORY.md 4 | include LICENSE 5 | include README.md 6 | 7 | recursive-include tests * 8 | recursive-exclude * __pycache__ 9 | recursive-exclude * *.py[co] 10 | 11 | recursive-include docs *.rst conf.py Makefile make.bat *.jpg *.png *.gif 12 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = py27, py34, py35, py36, flake8 3 | 4 | [travis] 5 | python = 6 | 3.6: py36 7 | 3.5: py35 8 | 3.4: py34 9 | 2.7: py27 10 | 11 | [testenv:flake8] 12 | basepython = python 13 | deps = flake8 14 | commands = flake8 rest_uploader 15 | 16 | [testenv] 17 | setenv = 18 | PYTHONPATH = {toxinidir} 19 | 20 | commands = python setup.py test 21 | 22 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to rest_uploader's documentation! 2 | ====================================== 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: Contents: 7 | 8 | readme 9 | installation 10 | usage 11 | modules 12 | contributing 13 | authors 14 | history 15 | 16 | Indices and tables 17 | ================== 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "name": "Python: Module", 9 | "type": "python", 10 | "request": "launch", 11 | "module": "rest_uploader.cli", 12 | "args": [ 13 | ".", 14 | ], 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.linting.enabled": true, 3 | "python.linting.pylintEnabled": false, 4 | "python.linting.pycodestyleEnabled": false, 5 | "python.linting.prospectorEnabled": false, 6 | "python.linting.flake8Enabled": true, 7 | "python.formatting.provider": "black", 8 | "python.formatting.blackArgs": [ 9 | "--line-length", 10 | "120" 11 | ], 12 | 13 | "python.linting.flake8Args": [ 14 | "--max-line-length=120", 15 | "--ignore=E402", 16 | "--ignore=W503" 17 | ] 18 | } -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = rest_uploader 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "rest_uploader" 7 | version = "1.22.0" 8 | authors = [ 9 | { name="Justin Keller", email="kellerjustin@protonmail.com" }, 10 | ] 11 | description = "REST API File Uploader for Joplin" 12 | readme = "README.md" 13 | license = { file="LICENSE" } 14 | requires-python = ">=3.7" 15 | classifiers = [ 16 | "Programming Language :: Python :: 3", 17 | "License :: OSI Approved :: MIT License", 18 | "Operating System :: OS Independent", 19 | ] 20 | 21 | [project.urls] 22 | "Homepage" = "https://github.com/cerealkella/rest-uploader" 23 | "Bug Tracker" = "https://github.com/cerealkella/rest-uploader/issues" 24 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 1.22.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | search = version="{current_version}" 8 | replace = version="{new_version}" 9 | 10 | [bumpversion:file:rest_uploader/__init__.py] 11 | search = "__version__ = '{current_version}'" 12 | replace = "__version__ = '{new_version}'" 13 | 14 | [bumpversion:file:pyproject.toml] 15 | search = version = "{current_version}" 16 | replace = version = "{new_version}" 17 | 18 | [bdist_wheel] 19 | universal = 1 20 | 21 | [flake8] 22 | exclude = docs 23 | 24 | [aliases] 25 | 26 | [metadata] 27 | name = rest-uploader 28 | version = attr: rest-uploader.__version__ 29 | long_description = file: README.md 30 | long_description_content_type = text/markdown 31 | -------------------------------------------------------------------------------- /rest_uploader.service: -------------------------------------------------------------------------------- 1 | # For Arch/Manjaro Systems, place this file here: $HOME/.config/systemd/user/rest_uploader.Service 2 | # Adjust accordingly for other systems, but this service works great as a user service 3 | # to start: systemctl --user start rest_uploader.service 4 | # to enable: systemctl --user enable rest_uploader.service 5 | [Service] 6 | Description=rest-uploader script for Joplin 7 | After=network.target 8 | 9 | [Service] 10 | Type=simple 11 | WorkingDirectory=%h 12 | ExecStart=/home/yourusername/.virtualenvs/rest_uploader/bin/python -m rest_uploader.cli -d inbox /home/yourusername/joplin_upload --moveto /home/yourusername/joplin_upload/archive 13 | CPUSchedulingPolicy=idle 14 | IOSchedulingClass=3 15 | Restart=on-failure 16 | 17 | [Install] 18 | WantedBy=default.target 19 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=python -msphinx 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | set SPHINXPROJ=rest_uploader 13 | 14 | if "%1" == "" goto help 15 | 16 | %SPHINXBUILD% >NUL 2>NUL 17 | if errorlevel 9009 ( 18 | echo. 19 | echo.The Sphinx module was not found. Make sure you have Sphinx installed, 20 | echo.then set the SPHINXBUILD environment variable to point to the full 21 | echo.path of the 'sphinx-build' executable. Alternatively you may add the 22 | echo.Sphinx directory to PATH. 23 | echo. 24 | echo.If you don't have Sphinx installed, grab it from 25 | echo.http://sphinx-doc.org/ 26 | exit /b 1 27 | ) 28 | 29 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 30 | goto end 31 | 32 | :help 33 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 34 | 35 | :end 36 | popd 37 | -------------------------------------------------------------------------------- /tests/test_rest_uploader.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """Tests for `rest_uploader` package.""" 5 | 6 | 7 | import unittest 8 | from click.testing import CliRunner 9 | 10 | from rest_uploader import rest_uploader 11 | from rest_uploader import cli 12 | 13 | 14 | class TestRest_uploader(unittest.TestCase): 15 | """Tests for `rest_uploader` package.""" 16 | 17 | def setUp(self): 18 | """Set up test fixtures, if any.""" 19 | 20 | def tearDown(self): 21 | """Tear down test fixtures, if any.""" 22 | 23 | def test_000_something(self): 24 | """Test something.""" 25 | 26 | def test_command_line_interface(self): 27 | """Test the CLI.""" 28 | runner = CliRunner() 29 | result = runner.invoke(cli.main) 30 | assert result.exit_code == 0 31 | assert "rest_uploader.cli.main" in result.output 32 | help_result = runner.invoke(cli.main, ["--help"]) 33 | assert help_result.exit_code == 0 34 | assert "--help Show this message and exit." in help_result.output 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019, Justin Keller 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /docs/installation.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Installation 5 | ============ 6 | 7 | 8 | Stable release 9 | -------------- 10 | 11 | To install rest_uploader, run this command in your terminal: 12 | 13 | .. code-block:: console 14 | 15 | $ pip install rest_uploader 16 | 17 | This is the preferred method to install rest_uploader, as it will always install the most recent stable release. 18 | 19 | If you don't have `pip`_ installed, this `Python installation guide`_ can guide 20 | you through the process. 21 | 22 | .. _pip: https://pip.pypa.io 23 | .. _Python installation guide: http://docs.python-guide.org/en/latest/starting/installation/ 24 | 25 | 26 | From sources 27 | ------------ 28 | 29 | The sources for rest_uploader can be downloaded from the `Github repo`_. 30 | 31 | You can either clone the public repository: 32 | 33 | .. code-block:: console 34 | 35 | $ git clone git://github.com/kellerjustin/rest_uploader 36 | 37 | Or download the `tarball`_: 38 | 39 | .. code-block:: console 40 | 41 | $ curl -OL https://github.com/kellerjustin/rest_uploader/tarball/master 42 | 43 | Once you have a copy of the source, you can install it with: 44 | 45 | .. code-block:: console 46 | 47 | $ python setup.py install 48 | 49 | 50 | .. _Github repo: https://github.com/kellerjustin/rest_uploader 51 | .. _tarball: https://github.com/kellerjustin/rest_uploader/tarball/master 52 | -------------------------------------------------------------------------------- /rest_uploader/api_token.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import pathlib 3 | 4 | 5 | def get_os_datadir() -> pathlib.Path: 6 | """ 7 | Returns a parent directory path 8 | where persistent application data can be stored. 9 | 10 | # linux: ~/.local/share 11 | # macOS: ~/Library/Application Support 12 | # windows: C:/Users//AppData/Roaming 13 | """ 14 | home = pathlib.Path.home() 15 | 16 | if sys.platform == "win32": 17 | return home / "AppData/Roaming" 18 | elif sys.platform == "linux": 19 | return home / ".local/share" 20 | elif sys.platform == "darwin": 21 | return home / "Library/Application Support" 22 | 23 | 24 | def get_my_datadir() -> pathlib.Path: 25 | """Find datadir, create it if it doesn't exist""" 26 | my_datadir = get_os_datadir() / __package__ 27 | try: 28 | my_datadir.mkdir(parents=True) 29 | except FileExistsError: 30 | pass 31 | return my_datadir 32 | 33 | 34 | def get_token() -> str: 35 | """gets token from .api_token.txt file. If it doesn't exist, 36 | prompt for the user to paste it in. 37 | 38 | Returns: 39 | str: token text 40 | """ 41 | token = "" 42 | token_file = get_my_datadir() / ".api_token.txt" 43 | if token_file.exists(): 44 | with open(token_file, "r") as f: 45 | token = f.readline().rstrip() 46 | else: 47 | token = input("Paste your Joplin API Token:") 48 | with open(token_file, "w") as f: 49 | f.write(token.rstrip()) 50 | return token 51 | 52 | 53 | def get_token_suffix(): 54 | return "?token=" + get_token() 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # token 107 | .api_token.txt 108 | 109 | # direnv 110 | .direnv/ 111 | 112 | # email_processing 113 | rest_uploader/creds.toml 114 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """The setup script.""" 5 | 6 | from setuptools import setup, find_packages 7 | from pathlib import Path 8 | 9 | # Build the long description using README & HISTORY 10 | readme = (Path(__file__).parent / "README.md").read_text() 11 | history = (Path(__file__).parent / "HISTORY.md").read_text() 12 | long_description = readme + history 13 | 14 | requirements = [ 15 | "watchdog>=3.0.0", 16 | "requests>=2.31.0", 17 | "tabulate>=0.9.0", 18 | "click>=8.1.3", 19 | "img_processor>=0.20.0", 20 | ] 21 | 22 | setup_requirements = [] 23 | 24 | test_requirements = [] 25 | 26 | setup( 27 | author="Justin Keller", 28 | author_email="kellerjustin@protonmail.com", 29 | classifiers=[ 30 | "Development Status :: 2 - Pre-Alpha", 31 | "Intended Audience :: Developers", 32 | "License :: OSI Approved :: MIT License", 33 | "Natural Language :: English", 34 | "Programming Language :: Python :: 2", 35 | "Programming Language :: Python :: 2.7", 36 | "Programming Language :: Python :: 3", 37 | "Programming Language :: Python :: 3.4", 38 | "Programming Language :: Python :: 3.5", 39 | "Programming Language :: Python :: 3.6", 40 | "Programming Language :: Python :: 3.7", 41 | ], 42 | description="REST API Uploader", 43 | entry_points={"console_scripts": ["rest_uploader=rest_uploader.cli:main"]}, 44 | install_requires=requirements, 45 | license="MIT license", 46 | long_description=long_description, 47 | long_description_content_type="text/markdown", 48 | include_package_data=True, 49 | keywords=["rest_uploader", "joplin", "rest-uploader"], 50 | name="rest_uploader", 51 | packages=find_packages(include=["rest_uploader"]), 52 | setup_requires=setup_requirements, 53 | test_suite="tests", 54 | tests_require=test_requirements, 55 | url="https://github.com/kellerjustin/rest-uploader", 56 | version="1.22.0", 57 | zip_safe=False, 58 | ) 59 | -------------------------------------------------------------------------------- /rest_uploader/email_processor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import traceback 4 | from imbox import Imbox 5 | from imaplib import IMAP4 6 | 7 | 8 | HOST="email.server" 9 | USERNAME="user" 10 | PASSWORD="pass" 11 | DOWNLOAD_FOLDER="." 12 | POLLING_INTERVAL=10 13 | 14 | def connect_to_mailbox(host=HOST, username=USERNAME, password=PASSWORD): 15 | mail = Imbox( 16 | HOST, 17 | username=USERNAME, 18 | password=PASSWORD, 19 | ssl=True, 20 | ssl_context=None, 21 | starttls=False, 22 | ) 23 | return mail 24 | 25 | 26 | def process_attachments(message): 27 | attachments = [] 28 | for idx, attachment in enumerate(message.attachments): 29 | try: 30 | att_fn = attachment.get("filename") 31 | download_path = f"{DOWNLOAD_FOLDER}/{idx}_{att_fn}" 32 | with open(download_path, "wb") as fp: 33 | fp.write(attachment.get("content").read()) 34 | attachments.append(download_path) 35 | except: 36 | print(traceback.print_exc()) 37 | return attachments 38 | 39 | 40 | def process_unread_messages(mail): 41 | try: 42 | messages = mail.messages(unread=True) # defaults to inbox 43 | 44 | for (uid, message) in messages: 45 | print(uid) 46 | sent_from = message.sent_from[0]["name"] 47 | subject = message.subject 48 | print(f"Processing email {subject} from {sent_from}") 49 | 50 | attachments = process_attachments(message) 51 | 52 | mail.mark_seen(uid) # mark message as read 53 | return 0 54 | except (ConnectionResetError, IMAP4.abort) as e: 55 | print(e) 56 | print("Connection Reset, waiting for five minutes before retrying...") 57 | time.sleep(300) # Wait 5 minutes before trying again 58 | return -1 59 | 60 | mail = connect_to_mailbox() 61 | 62 | while True: 63 | t = time.localtime() 64 | current_time = time.strftime("%H:%M:%S", t) 65 | print(f"{current_time} - Checking for new messages...") 66 | if process_unread_messages(mail) < 0: 67 | # Connection Reset, recreate mail object 68 | del mail 69 | mail = connect_to_mailbox() 70 | else: 71 | time.sleep(POLLING_INTERVAL) 72 | 73 | mail.logout() 74 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean clean-test clean-pyc clean-build docs help 2 | .DEFAULT_GOAL := help 3 | 4 | define BROWSER_PYSCRIPT 5 | import os, webbrowser, sys 6 | 7 | try: 8 | from urllib import pathname2url 9 | except: 10 | from urllib.request import pathname2url 11 | 12 | webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1]))) 13 | endef 14 | export BROWSER_PYSCRIPT 15 | 16 | define PRINT_HELP_PYSCRIPT 17 | import re, sys 18 | 19 | for line in sys.stdin: 20 | match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line) 21 | if match: 22 | target, help = match.groups() 23 | print("%-20s %s" % (target, help)) 24 | endef 25 | export PRINT_HELP_PYSCRIPT 26 | 27 | BROWSER := python -c "$$BROWSER_PYSCRIPT" 28 | 29 | help: 30 | @python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST) 31 | 32 | clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts 33 | 34 | clean-build: ## remove build artifacts 35 | rm -fr build/ 36 | rm -fr dist/ 37 | rm -fr .eggs/ 38 | find . -name '*.egg-info' -exec rm -fr {} + 39 | find . -name '*.egg' -exec rm -f {} + 40 | 41 | clean-pyc: ## remove Python file artifacts 42 | find . -name '*.pyc' -exec rm -f {} + 43 | find . -name '*.pyo' -exec rm -f {} + 44 | find . -name '*~' -exec rm -f {} + 45 | find . -name '__pycache__' -exec rm -fr {} + 46 | 47 | clean-test: ## remove test and coverage artifacts 48 | rm -fr .tox/ 49 | rm -f .coverage 50 | rm -fr htmlcov/ 51 | rm -fr .pytest_cache 52 | 53 | lint: ## check style with flake8 54 | flake8 rest_uploader tests 55 | 56 | test: ## run tests quickly with the default Python 57 | python setup.py test 58 | 59 | test-all: ## run tests on every Python version with tox 60 | tox 61 | 62 | coverage: ## check code coverage quickly with the default Python 63 | coverage run --source rest_uploader setup.py test 64 | coverage report -m 65 | coverage html 66 | $(BROWSER) htmlcov/index.html 67 | 68 | docs: ## generate Sphinx HTML documentation, including API docs 69 | rm -f docs/rest_uploader.rst 70 | rm -f docs/modules.rst 71 | sphinx-apidoc -o docs/ rest_uploader 72 | $(MAKE) -C docs clean 73 | $(MAKE) -C docs html 74 | $(BROWSER) docs/_build/html/index.html 75 | 76 | servedocs: docs ## compile the docs watching for changes 77 | watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . 78 | 79 | release: dist ## package and upload a release 80 | twine upload dist/* 81 | 82 | dist: clean ## builds source and wheel package 83 | python setup.py sdist 84 | python setup.py bdist_wheel 85 | ls -l dist 86 | 87 | install: clean ## install the package to the active Python's site-packages 88 | python setup.py install 89 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | ## HISTORY 2 | --- 3 | 4 | ## 0.1.0 - 2019-04-14 5 | 6 | * First super-unpolished and barely usable release on PyPI. 7 | 8 | 9 | ## 0.3.0 - 2019-06-26 10 | 11 | * First real release on PyPI. 12 | 13 | 14 | ## 0.4.0 - 2019-11-12 15 | 16 | * Fixed multipage OCR on PDFs 17 | * Added Language as a command line option 18 | 19 | 20 | ## 0.5.0 - 2019-11-14 21 | * Removed unidecode function on OCR, no longer needed 22 | 23 | 24 | ## 0.6.0 - 2019-11-14 25 | 26 | * Cleaned up PDF reading function 27 | * Syncronized versioning 28 | 29 | 30 | ## 0.7.0 - 2019-11-14 31 | 32 | * Nothing of consequence 33 | 34 | 35 | ## 0.8.0 - 2019-11-14 36 | 37 | * Added --version flag on cli 38 | 39 | 40 | ## 1.2.0 - 2019-11-14 41 | 42 | * Resolved failed builds related to bumpversion and 43 | manual versioning out of sync 44 | 45 | 46 | ## 1.3.0 - 2019-12-03 47 | 48 | * Added special processing to turn CSVs into markdown tables 49 | 50 | 51 | ## 1.4.0 - 2020-02-19 52 | 53 | * Ignore .part files (KDE Bug) 54 | * Added logic to automatically assign tags to newly created notes 55 | * Added click option to turn off automatic tag generation 56 | 57 | 58 | ## 1.5.0 - 2020-02-19 59 | 60 | * Embarrassingly dumb bug fix for autotag argument 61 | 62 | 63 | ## 1.7.0 - 2020-05-11 64 | 65 | * Added command line options for server, port, destination notebook 66 | * Added some error handling to quit when Joplin is closed or there is no valid notebook detected 67 | * Removed the settings.py file - everything handled via command line switches 68 | * More verbose command line notifications 69 | * Not leaving temp file location setting to user, handling this by detecting OS on startup 70 | 71 | 72 | ## 1.9.0 - 2020-05-12 73 | 74 | * Immediately realized closing application if Joplin isn't running is a bad idea in the case of startup scripts, etc - fixed logic to wait 75 | * Made use of the tempfile library which made life easier 76 | 77 | 78 | ## 1.10.0 - 2020-05-12 79 | 80 | * Fixed minor but annoying typo on a print statement 81 | 82 | 83 | ## 1.11.0 - 2020-09-23 84 | 85 | * Refactoring the imaging module out of rest_uploader and into its own img_processor module for reusability 86 | * Added logic and command line switches (-r, -m) for autorotation and moving/"sweeping" files after uploading. Run rest-uploader --help 87 | 88 | 89 | ## 1.12.0 - 2020-10-08 90 | 91 | * Added additional error handling and better output 92 | 93 | 94 | ## 1.13.0 - 2020-12-11 95 | 96 | * Fixed breaking changes from Joplin API in version 1.4.19 97 | * rest_uploader as of version 1.13.0 will not work with versions of Joplin prior to 1.4.19 98 | 99 | 100 | ## 1.14.0 - 2022-07-25 101 | 102 | * Moves Joplin API file to userspace 103 | 104 | 105 | ## 1.15.0 - 2022-07-25 106 | 107 | * Fixes documentation problems 108 | 109 | 110 | ## 1.16.0 - 2022-07-25 111 | 112 | * More fixes to documentation. 113 | 114 | 115 | ## 1.17.0 - 2022-07-25 116 | 117 | * Removed a couple redundant files 118 | 119 | ## 1.20.0 - 2023-10-05 120 | 121 | * Enabled PDF processing of .PDF (uppercase) extensions 122 | 123 | 124 | ## 1.21.0 - 2024-04-10 125 | 126 | * Enabled improved handling of HTML files 127 | 128 | ## 1.22.0 - 2024-04-11 129 | 130 | * Adds a function to check for duplicate resources 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rest-uploader 2 | REST API File Uploader for Joplin 3 | 4 | [API Reference](https://joplinapp.org/api/references/rest_api/) 5 | 6 | Joplin client will need Webclipper service enabled. 7 | * Go to Tools -> Web Clipper Options -> Enable Web Clipper Service 8 | 9 | ### History 10 | 2018-09-24 11 | This program was created to upload files from a folder specified in the 12 | PATH variable to Joplin. The following resource was helpful in figuring out 13 | the logic for Watchdog: 14 | https://stackoverflow.com/questions/18599339/python-watchdog-monitoring-file-for-changes 15 | 16 | Tested with the following extensions: 17 | .md 18 | .txt 19 | .pdf 20 | .png 21 | .jpg 22 | .url 23 | .csv 24 | 25 | Caveat: 26 | Uploader only triggered upon new file creation, not modification 27 | 28 | ### Tesseract 29 | In order for OCR text recognition to work, you'll need to download and 30 | install Tesseract. Windows systems may require that you add a path variable 31 | in order for it to work. 32 | 33 | ### Poppler 34 | Poppler is required for PDF processing. 35 | For Windows, download x86 binary and add to environment path. 36 | https://anaconda.org/conda-forge/poppler 37 | 38 | ### Running application 39 | Launch using the executable rest_uploader, specify the monitoring path 40 | as an argument. You'll need your Joplin API key the first time you 41 | launch. If your API key changes or gets pasted incorrectly, delete the .api_token.txt file that gets stored to the package. 42 | 43 | ` rest_uploader /path/to/directory ` 44 | 45 | To launch as python module: 46 | 47 | ` python -m rest_uploader.cli /path/to/directory ` 48 | 49 | ### Languages other than English 50 | Version 0.4.0 added language support via Tesseract (check Tesseract docs). To enable, use the cli -l or --language argument. 51 | The following example will OCR text using the German dictionary: 52 | ` rest_uploader /path/to/directory --language ger ` 53 | 54 | ### Additional Command Line Options added in version 1.8.0 55 | By default notes will be dropped into a notebook called "inbox". 56 | Specify a different upload notebook by specifying a destination from the 57 | command line using -d or --destination 58 | 59 | rest_uploader will fail if the specified notebook does not exist 60 | I am guessing if you have two notebooks with the same name, it will dump 61 | new notes in the first one it finds. 62 | 63 | Default server = 127.0.0.1 64 | To specify a different server, use the -s or --server cli option 65 | 66 | Default port = 41184 67 | To specify a different port, use the -p or --port cli option 68 | 69 | Default autotagging behavior = yes 70 | To turn off autotagging, use the -t or --autotag option 71 | Example: 72 | ` rest_uploader /path/to/directory -t no ` 73 | 74 | It doesn't matter if the options are before or after the path. Path is mandatory, though. 75 | The following example will upload newly created notes to the Taxes Notebook, and OCR in German: 76 | ` rest_uploader -d "Taxes" -l ger /path/to/directory ` 77 | 78 | ### Additional Command Line Options added in version 1.11.0 79 | Added autorotation switch to turn off if this behavior is not wanted. By default, after performing OCR, the application will rotate the image according to how the OCR was able to detect text. Turn this off using the -r command line switch: 80 | ` rest_uploader -r no /path/to/directory ` 81 | 82 | Added moveto option to specify a directory in which to place processed files. Use the -o option to specify a moveto directory By default this is off but in the background it uses the operating system's temp directory as a placeholder for the option to indicate "off" since the option requires a valid path. Do not attempt to use the OS temp dir as a moveto directory. Example of using the option: 83 | ` rest_uploader /path/to/directory -o /path/to/moveto/directory ` 84 | -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | .. highlight:: shell 2 | 3 | ============ 4 | Contributing 5 | ============ 6 | 7 | Contributions are welcome, and they are greatly appreciated! Every little bit 8 | helps, and credit will always be given. 9 | 10 | You can contribute in many ways: 11 | 12 | Types of Contributions 13 | ---------------------- 14 | 15 | Report Bugs 16 | ~~~~~~~~~~~ 17 | 18 | Report bugs at https://github.com/kellerjustin/rest_uploader/issues. 19 | 20 | If you are reporting a bug, please include: 21 | 22 | * Your operating system name and version. 23 | * Any details about your local setup that might be helpful in troubleshooting. 24 | * Detailed steps to reproduce the bug. 25 | 26 | Fix Bugs 27 | ~~~~~~~~ 28 | 29 | Look through the GitHub issues for bugs. Anything tagged with "bug" and "help 30 | wanted" is open to whoever wants to implement it. 31 | 32 | Implement Features 33 | ~~~~~~~~~~~~~~~~~~ 34 | 35 | Look through the GitHub issues for features. Anything tagged with "enhancement" 36 | and "help wanted" is open to whoever wants to implement it. 37 | 38 | Write Documentation 39 | ~~~~~~~~~~~~~~~~~~~ 40 | 41 | rest_uploader could always use more documentation, whether as part of the 42 | official rest_uploader docs, in docstrings, or even on the web in blog posts, 43 | articles, and such. 44 | 45 | Submit Feedback 46 | ~~~~~~~~~~~~~~~ 47 | 48 | The best way to send feedback is to file an issue at https://github.com/kellerjustin/rest_uploader/issues. 49 | 50 | If you are proposing a feature: 51 | 52 | * Explain in detail how it would work. 53 | * Keep the scope as narrow as possible, to make it easier to implement. 54 | * Remember that this is a volunteer-driven project, and that contributions 55 | are welcome :) 56 | 57 | Get Started! 58 | ------------ 59 | 60 | Ready to contribute? Here's how to set up `rest_uploader` for local development. 61 | 62 | 1. Fork the `rest_uploader` repo on GitHub. 63 | 2. Clone your fork locally:: 64 | 65 | $ git clone git@github.com:your_name_here/rest_uploader.git 66 | 67 | 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: 68 | 69 | $ mkvirtualenv rest_uploader 70 | $ cd rest_uploader/ 71 | $ python setup.py develop 72 | 73 | 4. Create a branch for local development:: 74 | 75 | $ git checkout -b name-of-your-bugfix-or-feature 76 | 77 | Now you can make your changes locally. 78 | 79 | 5. When you're done making changes, check that your changes pass flake8 and the 80 | tests, including testing other Python versions with tox:: 81 | 82 | $ flake8 rest_uploader tests 83 | $ python setup.py test or py.test 84 | $ tox 85 | 86 | To get flake8 and tox, just pip install them into your virtualenv. 87 | 88 | 6. Commit your changes and push your branch to GitHub:: 89 | 90 | $ git add . 91 | $ git commit -m "Your detailed description of your changes." 92 | $ git push origin name-of-your-bugfix-or-feature 93 | 94 | 7. Submit a pull request through the GitHub website. 95 | 96 | Pull Request Guidelines 97 | ----------------------- 98 | 99 | Before you submit a pull request, check that it meets these guidelines: 100 | 101 | 1. The pull request should include tests. 102 | 2. If the pull request adds functionality, the docs should be updated. Put 103 | your new functionality into a function with a docstring, and add the 104 | feature to the list in README.rst. 105 | 3. The pull request should work for Python 2.7, 3.4, 3.5 and 3.6, and for PyPy. Check 106 | https://travis-ci.org/kellerjustin/rest_uploader/pull_requests 107 | and make sure that the tests pass for all supported Python versions. 108 | 109 | Tips 110 | ---- 111 | 112 | To run a subset of tests:: 113 | 114 | 115 | $ python -m unittest tests.test_rest_uploader 116 | 117 | Deploying 118 | --------- 119 | 120 | A reminder for the maintainers on how to deploy. 121 | Make sure all your changes are committed (including an entry in HISTORY.rst). 122 | Then run:: 123 | 124 | $ bumpversion patch # possible: major / minor / patch 125 | $ git push 126 | $ git push --tags 127 | 128 | Travis will then deploy to PyPI if tests pass. 129 | -------------------------------------------------------------------------------- /rest_uploader/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Console script for rest_uploader.""" 4 | import sys 5 | import click 6 | import tempfile 7 | from .rest_uploader import ( 8 | watcher, 9 | set_autotag, 10 | set_notebook_id, 11 | set_working_directory, 12 | set_endpoint, 13 | set_token, 14 | set_language, 15 | set_autorotation, 16 | set_moveto, 17 | ) 18 | from . import __version__ 19 | 20 | 21 | def parse_argument(arg): 22 | """Helper function for wild arguments""" 23 | if arg in ["No", "N", "NO", "OFF", "off", "n", "no"]: 24 | arg = "no" 25 | else: 26 | arg = "yes" 27 | return arg 28 | 29 | 30 | @click.command() 31 | @click.argument( 32 | "path", 33 | type=click.Path( 34 | exists=True, 35 | file_okay=False, 36 | dir_okay=True, 37 | writable=False, 38 | readable=True, 39 | resolve_path=True, 40 | ), 41 | ) 42 | @click.option( 43 | "-s", 44 | "--server", 45 | "server", 46 | default="127.0.0.1", 47 | help="""Specify the server to which the application""" 48 | """ should connect. Default = "127.0.0.1" """, 49 | ) 50 | @click.option( 51 | "-p", 52 | "--port", 53 | "port", 54 | default="41184", 55 | help="""Specify the port to which the application should connect.""" 56 | """ Default = 41184 """, 57 | ) 58 | @click.option( 59 | "-l", 60 | "--language", 61 | "language", 62 | default="eng", 63 | help="""Specify OCR Language. Refer to Tesseract's documentation found here: 64 | https://github.com/tesseract-ocr/tesseract/wiki""", 65 | ) 66 | @click.option( 67 | "-t", 68 | "--autotag", 69 | "autotag", 70 | default="yes", 71 | help="""Specify whether or not to automatically tag notes based on""" 72 | """ OCR'd text. Default = 'yes', specify 'no' if this behavior is""" 73 | """ not desired""", 74 | ) 75 | @click.option( 76 | "-d", 77 | "--destination", 78 | "destination", 79 | default="inbox", 80 | help="""Specify the notebook in which to place newly created notes.""" 81 | """ Specified notebook must exist or program will exit.""" 82 | """ Default = "inbox". """, 83 | ) 84 | @click.option( 85 | "-r", 86 | "--autorotation", 87 | "autorotation", 88 | default="yes", 89 | help="""Specify whether to rotate images.""" 90 | """ Default = yes (autorotation on, specify 'no' to disable). """, 91 | ) 92 | @click.option( 93 | "-o", 94 | "--moveto", 95 | "moveto", 96 | default=tempfile.gettempdir(), 97 | type=click.Path( 98 | exists=True, 99 | file_okay=False, 100 | dir_okay=True, 101 | writable=False, 102 | readable=True, 103 | resolve_path=True, 104 | ), 105 | ) 106 | @click.version_option(version=__version__) 107 | def main( 108 | path=None, 109 | server="server", 110 | port="port", 111 | language="eng", 112 | autotag="yes", 113 | destination="inbox", 114 | autorotation="yes", 115 | moveto="", 116 | ): 117 | """Console script for rest_uploader. 118 | Define file path to monitor, e.g. 119 | rest_uploader /home/user/Docouments/scans 120 | """ 121 | click.echo("Launching Application " "rest_uploader.cli.main") 122 | set_working_directory() 123 | set_endpoint(server, port) 124 | set_token() 125 | # set_temp_path() # Do I need to do this here? 126 | notebook_id = set_notebook_id(destination.strip()) 127 | if notebook_id == "err": 128 | click.echo("Joplin may not be running, please ensure it is open.") 129 | click.echo(" will check again when processing a file.") 130 | elif notebook_id == "": 131 | click.echo(f"Invalid Notebook, check to see if {destination.strip()} exists.") 132 | click.echo(f"Please specify a valid notebook. Quitting application.") 133 | return 0 134 | else: 135 | click.echo(f"Found Notebook ID: {notebook_id}") 136 | set_language(language) 137 | autotag = parse_argument(autotag) 138 | set_autotag(parse_argument(autotag)) 139 | autorotation = parse_argument(autorotation) 140 | set_autorotation(autorotation) 141 | moveto = set_moveto(moveto) 142 | click.echo("Language: " + language) 143 | click.echo("Automatically Tag Notes? " + autotag) 144 | click.echo("Destination Notebook: " + destination) 145 | click.echo("Autorotation: " + autorotation) 146 | if moveto == "": 147 | click.echo("Files will remain in the monitoring directory") 148 | else: 149 | click.echo("File move to location: " + moveto) 150 | watcher(path=path) 151 | return 0 152 | 153 | 154 | if __name__ == "__main__": 155 | sys.exit(main()) # pragma: no cover 156 | -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # rest_uploader documentation build configuration file, created by 5 | # sphinx-quickstart on Fri Jun 9 13:47:02 2017. 6 | # 7 | # This file is execfile()d with the current directory set to its 8 | # containing dir. 9 | # 10 | # Note that not all possible configuration values are present in this 11 | # autogenerated file. 12 | # 13 | # All configuration values have a default; values that are commented out 14 | # serve to show the default. 15 | 16 | # If extensions (or modules to document with autodoc) are in another 17 | # directory, add these directories to sys.path here. If the directory is 18 | # relative to the documentation root, use os.path.abspath to make it 19 | # absolute, like shown here. 20 | # 21 | import os 22 | import sys 23 | 24 | sys.path.insert(0, os.path.abspath("..")) 25 | 26 | import rest_uploader 27 | 28 | # -- General configuration --------------------------------------------- 29 | 30 | # If your documentation needs a minimal Sphinx version, state it here. 31 | # 32 | # needs_sphinx = '1.0' 33 | 34 | # Add any Sphinx extension module names here, as strings. They can be 35 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. 36 | extensions = ["sphinx.ext.autodoc", "sphinx.ext.viewcode"] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ["_templates"] 40 | 41 | # The suffix(es) of source filenames. 42 | # You can specify multiple suffix as a list of string: 43 | # 44 | # source_suffix = ['.rst', '.md'] 45 | source_suffix = ".rst" 46 | 47 | # The master toctree document. 48 | master_doc = "index" 49 | 50 | # General information about the project. 51 | project = "rest_uploader" 52 | copyright = "2019, Justin Keller" 53 | author = "Justin Keller" 54 | 55 | # The version info for the project you're documenting, acts as replacement 56 | # for |version| and |release|, also used in various other places throughout 57 | # the built documents. 58 | # 59 | # The short X.Y version. 60 | version = rest_uploader.__version__ 61 | # The full version, including alpha/beta/rc tags. 62 | release = rest_uploader.__version__ 63 | 64 | # The language for content autogenerated by Sphinx. Refer to documentation 65 | # for a list of supported languages. 66 | # 67 | # This is also used if you do content translation via gettext catalogs. 68 | # Usually you set "language" from the command line for these cases. 69 | language = None 70 | 71 | # List of patterns, relative to source directory, that match files and 72 | # directories to ignore when looking for source files. 73 | # This patterns also effect to html_static_path and html_extra_path 74 | exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] 75 | 76 | # The name of the Pygments (syntax highlighting) style to use. 77 | pygments_style = "sphinx" 78 | 79 | # If true, `todo` and `todoList` produce output, else they produce nothing. 80 | todo_include_todos = False 81 | 82 | 83 | # -- Options for HTML output ------------------------------------------- 84 | 85 | # The theme to use for HTML and HTML Help pages. See the documentation for 86 | # a list of builtin themes. 87 | # 88 | html_theme = "alabaster" 89 | 90 | # Theme options are theme-specific and customize the look and feel of a 91 | # theme further. For a list of options available for each theme, see the 92 | # documentation. 93 | # 94 | # html_theme_options = {} 95 | 96 | # Add any paths that contain custom static files (such as style sheets) here, 97 | # relative to this directory. They are copied after the builtin static files, 98 | # so a file named "default.css" will overwrite the builtin "default.css". 99 | html_static_path = ["_static"] 100 | 101 | 102 | # -- Options for HTMLHelp output --------------------------------------- 103 | 104 | # Output file base name for HTML help builder. 105 | htmlhelp_basename = "rest_uploaderdoc" 106 | 107 | 108 | # -- Options for LaTeX output ------------------------------------------ 109 | 110 | latex_elements = { 111 | # The paper size ('letterpaper' or 'a4paper'). 112 | # 113 | # 'papersize': 'letterpaper', 114 | # The font size ('10pt', '11pt' or '12pt'). 115 | # 116 | # 'pointsize': '10pt', 117 | # Additional stuff for the LaTeX preamble. 118 | # 119 | # 'preamble': '', 120 | # Latex figure (float) alignment 121 | # 122 | # 'figure_align': 'htbp', 123 | } 124 | 125 | # Grouping the document tree into LaTeX files. List of tuples 126 | # (source start file, target name, title, author, documentclass 127 | # [howto, manual, or own class]). 128 | latex_documents = [ 129 | ( 130 | master_doc, 131 | "rest_uploader.tex", 132 | "rest_uploader Documentation", 133 | "Justin Keller", 134 | "manual", 135 | ) 136 | ] 137 | 138 | 139 | # -- Options for manual page output ------------------------------------ 140 | 141 | # One entry per manual page. List of tuples 142 | # (source start file, name, description, authors, manual section). 143 | man_pages = [(master_doc, "rest_uploader", "rest_uploader Documentation", [author], 1)] 144 | 145 | 146 | # -- Options for Texinfo output ---------------------------------------- 147 | 148 | # Grouping the document tree into Texinfo files. List of tuples 149 | # (source start file, target name, title, author, 150 | # dir menu entry, description, category) 151 | texinfo_documents = [ 152 | ( 153 | master_doc, 154 | "rest_uploader", 155 | "rest_uploader Documentation", 156 | author, 157 | "rest_uploader", 158 | "One line description of project.", 159 | "Miscellaneous", 160 | ) 161 | ] 162 | -------------------------------------------------------------------------------- /rest_uploader/rest_uploader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Main module. Launch by running python -m rest_uploader.cli""" 4 | 5 | import csv 6 | import json 7 | import mimetypes 8 | import os 9 | import platform 10 | import shutil 11 | import tempfile 12 | import time 13 | from pathlib import Path 14 | 15 | import requests 16 | from img_processor import ImageProcessor 17 | from tabulate import tabulate 18 | from watchdog.events import FileSystemEventHandler 19 | from watchdog.observers.polling import PollingObserver 20 | 21 | from .api_token import get_token_suffix 22 | 23 | 24 | class MyHandler(FileSystemEventHandler): 25 | def _event_handler(self, path): 26 | filename, ext = os.path.splitext(path) 27 | if ext not in (".tmp", ".part", ".crdownload") and ext[:2] not in (".~"): 28 | filesize = self.valid_file(ext, path) 29 | if filesize > 10000000: 30 | print(f"Filesize = {filesize}. Too big for Joplin, skipping upload") 31 | return False 32 | else: 33 | i = 1 34 | max_retries = 5 35 | while i <= max_retries: 36 | if i > 1: 37 | print(f"Retrying file upload {i} of {max_retries}...") 38 | if upload(path, filesize) < 0: 39 | time.sleep(5) 40 | else: 41 | return True 42 | print(f"Tried {max_retries} times but failed to upload file {path}") 43 | return False 44 | else: 45 | print("Detected temp file. Temp files are ignored.") 46 | 47 | def valid_file(self, ext, path): 48 | """Ensure file is completely written before processing""" 49 | size_past = -1 50 | while True: 51 | size_now = os.path.getsize(path) 52 | if size_now == size_past: 53 | print(f"File xfer complete. Size={size_now}") 54 | return size_now 55 | else: 56 | size_past = os.path.getsize(path) 57 | print(f"File transferring...{size_now}") 58 | time.sleep(1) 59 | return -1 60 | 61 | def on_created(self, event): 62 | print(event.event_type + " -- " + event.src_path) 63 | self._event_handler(event.src_path) 64 | 65 | def on_moved(self, event): 66 | print(event.event_type + " -- " + event.dest_path) 67 | self._event_handler(event.dest_path) 68 | 69 | 70 | def set_working_directory(): 71 | """Set working directory""" 72 | if os.getcwd() != os.chdir(os.path.dirname(os.path.realpath(__file__))): 73 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 74 | 75 | 76 | def set_language(language): 77 | global LANGUAGE 78 | LANGUAGE = language 79 | 80 | 81 | def set_token(): 82 | global TOKEN 83 | TOKEN = get_token_suffix() 84 | 85 | 86 | def set_autotag(autotag): 87 | global AUTOTAG 88 | AUTOTAG = True 89 | if autotag == "no": 90 | AUTOTAG = False 91 | 92 | 93 | def set_endpoint(server, port): 94 | global ENDPOINT 95 | ENDPOINT = f"http://{server}:{port}" 96 | print(f"Endpoint: {ENDPOINT}") 97 | 98 | 99 | def set_autorotation(autorotation): 100 | global AUTOROTATION 101 | AUTOROTATION = True 102 | if autorotation == "no": 103 | AUTOROTATION = False 104 | 105 | 106 | def set_moveto(moveto): 107 | global MOVETO 108 | if moveto == tempfile.gettempdir(): 109 | moveto = "" 110 | MOVETO = moveto 111 | return MOVETO 112 | 113 | 114 | def initialize_notebook(notebook_name): 115 | global NOTEBOOK_NAME 116 | NOTEBOOK_NAME = notebook_name 117 | global NOTEBOOK_ID 118 | NOTEBOOK_ID = "" 119 | return NOTEBOOK_NAME 120 | 121 | 122 | def set_notebook_id(notebook_name=None): 123 | """Find the ID of the destination folder 124 | adapted logic from jhf2442 on Joplin forum 125 | https://discourse.joplin.cozic.net/t/import-txt-files/692 126 | """ 127 | global NOTEBOOK_NAME 128 | global NOTEBOOK_ID 129 | if notebook_name is not None: 130 | NOTEBOOK_NAME = initialize_notebook(notebook_name) 131 | try: 132 | res = requests.get(ENDPOINT + "/folders" + TOKEN) 133 | folders = res.json()["items"] 134 | for folder in folders: 135 | if folder.get("title") == NOTEBOOK_NAME: 136 | NOTEBOOK_ID = folder.get("id") 137 | if NOTEBOOK_ID == "": 138 | for folder in folders: 139 | if "children" in folder: 140 | for child in folder.get("children"): 141 | if child.get("title") == NOTEBOOK_NAME: 142 | NOTEBOOK_ID = child.get("id") 143 | return NOTEBOOK_ID 144 | except requests.ConnectionError as exception: 145 | print("Connection Error - Is Joplin Running?") 146 | return exception 147 | 148 | 149 | def read_text_note(filename): 150 | with open(filename, "r") as myfile: 151 | text = myfile.read() 152 | print(text) 153 | return text 154 | 155 | 156 | def read_csv(filename): 157 | return csv.DictReader(open(filename)) 158 | 159 | 160 | def apply_tags(text_to_match, note_id): 161 | """Rudimentary Tag match using OCR'd text""" 162 | res = requests.get(ENDPOINT + "/tags" + TOKEN) 163 | tags = res.json()["items"] 164 | counter = 0 165 | for tag in tags: 166 | if tag.get("title").lower() in text_to_match.lower(): 167 | counter += 1 168 | tag_id = tag.get("id") 169 | requests.post( 170 | ENDPOINT + f"/tags/{tag_id}/notes" + TOKEN, 171 | data=f'{{"id": "{note_id}"}}', 172 | ) 173 | print(f"Matched {counter} tag(s) for note {note_id}") 174 | return counter 175 | 176 | 177 | def check_for_duplicates(filename, filesize): 178 | http_notes_request = f"""{ENDPOINT}/search?query="{filename}"{TOKEN.replace("?", "&")}""" 179 | notes = requests.get(http_notes_request).json()["items"] 180 | notify = "" 181 | for note in notes: 182 | http_resources_request = f"""{ENDPOINT}/notes/{note["id"]}/resources?fields=id,title,size{TOKEN.replace("?", "&")}""" 183 | resources = requests.get(http_resources_request).json()["items"] 184 | for resource in resources: 185 | if filesize == int(resource["size"]): 186 | notify += f"""## Found identical resource: [{note["title"]}](:/{note["id"]})\n""" 187 | print(notify) 188 | return notify 189 | 190 | 191 | def create_resource(filename): 192 | if NOTEBOOK_ID == "": 193 | set_notebook_id() 194 | basefile = os.path.basename(filename) 195 | title = os.path.splitext(basefile)[0] 196 | files = { 197 | "data": (json.dumps(filename), open(filename, "rb")), 198 | "props": (None, f'{{"title":"{title}", "filename":"{basefile}"}}'), 199 | } 200 | response = requests.post(ENDPOINT + "/resources" + TOKEN, files=files) 201 | return response.json() 202 | 203 | 204 | def delete_resource(resource_id): 205 | apitext = ENDPOINT + "/resources/" + resource_id + TOKEN 206 | response = requests.delete(apitext) 207 | return response 208 | 209 | 210 | def get_resource(resource_id): 211 | apitext = ENDPOINT + "/resources/" + resource_id + TOKEN 212 | response = requests.get(apitext) 213 | return response 214 | 215 | 216 | def set_json_string(title, NOTEBOOK_ID, body, img=None): 217 | if img is None: 218 | return '{{ "title": {}, "parent_id": "{}", "body": {} }}'.format( 219 | json.dumps(title), NOTEBOOK_ID, json.dumps(body) 220 | ) 221 | else: 222 | return '{{ "title": "{}", "parent_id": "{}", "body": {}, "image_data_url": "{}" }}'.format( 223 | title, NOTEBOOK_ID, json.dumps(body), img 224 | ) 225 | 226 | 227 | def upload(filename, filesize): 228 | """Get the default Notebook ID and process the passed in file""" 229 | basefile = os.path.basename(filename) 230 | title, ext = os.path.splitext(basefile) 231 | body = f"{basefile} uploaded from {platform.node()}\n" 232 | body += check_for_duplicates(title, filesize) 233 | datatype = mimetypes.guess_type(filename)[0] 234 | if datatype is None: 235 | # avoid subscript exception if datatype is None 236 | if ext in (".url", ".lnk"): 237 | datatype = "text/plain" 238 | else: 239 | datatype = "" 240 | elif datatype in ("text/plain", "text/html"): 241 | body += read_text_note(filename) 242 | values = set_json_string(title, NOTEBOOK_ID, body) 243 | if datatype == "text/html": 244 | values = values.replace('"body":', '"markup_language": 2, "body":') 245 | elif datatype == "text/csv": 246 | table = read_csv(filename) 247 | body += tabulate(table, headers="keys", numalign="right", tablefmt="pipe") 248 | values = set_json_string(title, NOTEBOOK_ID, body) 249 | elif datatype[:5] == "image": 250 | img_processor = ImageProcessor(LANGUAGE) 251 | body += "\n\n" 262 | img = img_processor.encode_image(filename, datatype) 263 | del img_processor 264 | values = set_json_string(title, NOTEBOOK_ID, body, img) 265 | else: 266 | response = create_resource(filename) 267 | body += f"[{basefile}](:/{response['id']})" 268 | values = set_json_string(title, NOTEBOOK_ID, body) 269 | if response["file_extension"].lower() == "pdf": 270 | img_processor = ImageProcessor(LANGUAGE) 271 | if img_processor.pdf_valid(filename): 272 | # Special handling for PDFs 273 | body += "\n\n" 276 | previewfile = img_processor.PREVIEWFILE 277 | if not os.path.exists(previewfile): 278 | previewfile = img_processor.pdf_page_to_image(filename) 279 | img = img_processor.encode_image(previewfile, "image/png") 280 | os.remove(previewfile) 281 | values = set_json_string(title, NOTEBOOK_ID, body, img) 282 | response = requests.post(ENDPOINT + "/notes" + TOKEN, data=values) 283 | if response.status_code == 200: 284 | if AUTOTAG: 285 | apply_tags(body, response.json().get("id")) 286 | print(f"Placed note into notebook {NOTEBOOK_ID}: {NOTEBOOK_NAME}") 287 | if os.path.isdir(MOVETO): 288 | moveto_filename = os.path.join(MOVETO, basefile) 289 | print(moveto_filename) 290 | if os.path.exists(moveto_filename): 291 | print(f"{basefile} exists in moveto dir, not moving!") 292 | else: 293 | try: 294 | # Give it a few seconds to release file lock 295 | time.sleep(3) 296 | shutil.move(filename, MOVETO) 297 | except IOError: 298 | print(f"File Locked-unable to move {filename}") 299 | return 0 300 | else: 301 | print("ERROR! NOTE NOT CREATED") 302 | print("Something went wrong corrupt file or note > 10MB?") 303 | return -1 304 | 305 | 306 | def watcher(path=None): 307 | if path is None: 308 | path = str(Path.home()) 309 | event_handler = MyHandler() 310 | print("POLLING OBSERVER!") 311 | print(f"Monitoring directory: {path}") 312 | observer = PollingObserver() 313 | observer.schedule(event_handler, path=path, recursive=False) 314 | observer.start() 315 | 316 | try: 317 | while True: 318 | time.sleep(1) 319 | except KeyboardInterrupt: 320 | observer.stop() 321 | observer.join() 322 | 323 | 324 | if __name__ == "__main__": 325 | watcher() 326 | --------------------------------------------------------------------------------