├── .gitattributes
├── .github
└── workflows
│ ├── black.yml
│ ├── build-docker-on-push-master.yml
│ ├── build-docker-on-release.yml
│ ├── build-docker-specific-version.yml
│ ├── codecov.yml
│ ├── deploy-docs.yml
│ └── pypi-auto-deploy.yml
├── .gitignore
├── API.md
├── LICENSE
├── MANIFEST.in
├── README.md
├── SECURITY.md
├── archivy
├── __init__.py
├── api.py
├── cli.py
├── click_web
│ ├── __init__.py
│ ├── exceptions.py
│ ├── resources
│ │ ├── __init__.py
│ │ ├── cmd_exec.py
│ │ ├── cmd_form.py
│ │ ├── index.py
│ │ └── input_fields.py
│ └── web_click_types.py
├── config.py
├── data.py
├── forms.py
├── helpers.py
├── models.py
├── routes.py
├── search.py
├── static
│ ├── HIGHLIGHTJS-LICENSE
│ ├── accessibility.css
│ ├── archivy.svg
│ ├── delete.png
│ ├── editor.css
│ ├── editor.js
│ ├── editor_dark.css
│ ├── highlight.js
│ ├── logo.png
│ ├── main.css
│ ├── main_dark.css
│ ├── markdown.css
│ ├── markdown_dark.css
│ ├── math.css
│ ├── math.js
│ ├── monokai.css
│ ├── mvp.css
│ ├── open_form.js
│ ├── parser.js
│ ├── post_and_read.js
│ └── profile.svg
├── tags.py
└── templates
│ ├── base.html
│ ├── bookmarklet.html
│ ├── click_web
│ ├── command_form.html
│ ├── form_macros.html
│ └── show_tree.html
│ ├── config.html
│ ├── dataobjs
│ ├── new.html
│ └── show.html
│ ├── home.html
│ ├── markdown-parser.html
│ ├── tags
│ ├── all.html
│ └── show.html
│ └── users
│ ├── edit.html
│ └── login.html
├── conftest.py
├── docs
├── CONTRIBUTING.md
├── config.md
├── difference.md
├── editing.md
├── example-plugin
│ ├── README.md
│ ├── archivy_extra_metadata
│ │ └── __init__.py
│ └── setup.py
├── img
│ ├── local-edit.png
│ ├── logo-2.png
│ └── logo.png
├── index.md
├── install.md
├── overrides
│ └── main.html
├── plugins.md
├── reference
│ ├── architecture.md
│ ├── filesystem_layer.md
│ ├── helpers.md
│ ├── hooks.md
│ ├── models.md
│ ├── search.md
│ ├── web_api.md
│ └── web_inputs.md
├── requirements.txt
├── setup-search.md
├── usage.md
└── whats_next.md
├── mkdocs.yml
├── plugins.md
├── requirements-tests.txt
├── requirements.txt
├── setup.py
└── tests
├── __init__.py
├── functional
├── test_plugins.py
└── test_routes.py
├── integration
└── test_api.py
├── test_click_web.py
├── test_plugin
├── plugin.py
└── setup.py
├── test_request_parsing.py
└── unit
├── test_advanced_conf.py
├── test_cli.py
├── test_click_web.py
└── test_models.py
/.gitattributes:
--------------------------------------------------------------------------------
1 | archivy/static/*.js linguist-vendored
2 | archivy/static/math.css linguist-vendored
3 | archivy/static/monokai.css linguist-vendored
4 |
--------------------------------------------------------------------------------
/.github/workflows/black.yml:
--------------------------------------------------------------------------------
1 | name: Black Code Quality
2 |
3 | on:
4 | push:
5 | branches:
6 | - 'master'
7 | paths:
8 | - 'archivy/**'
9 | - '.github/workflows/black.yml'
10 | - 'conftest.py'
11 | - 'tests/**'
12 |
13 | pull_request:
14 | paths:
15 | - 'archivy/**'
16 | - '.github/workflows/black.yml'
17 | - 'conftest.py'
18 | - 'tests/**'
19 |
20 |
21 | jobs:
22 | quality:
23 | runs-on: ubuntu-latest
24 | steps:
25 | - uses: actions/checkout@v1
26 | - name: Set up Python 3.8
27 | uses: actions/setup-python@v1
28 | with:
29 | python-version: 3.8
30 | - name: Install dependencies
31 | run: |
32 | python -m pip install --upgrade pip
33 | python -m pip install wheel
34 | pip install -r requirements.txt
35 | - name: Lint with black
36 | run: |
37 | pip install black
38 | black --check .
39 |
40 |
--------------------------------------------------------------------------------
/.github/workflows/build-docker-on-push-master.yml:
--------------------------------------------------------------------------------
1 | name: Building, Testing, and Pushing Archivy Container Image On Push and PRs To Master Branch
2 |
3 | on:
4 | workflow_dispatch:
5 | push:
6 | branches:
7 | - 'master'
8 | paths:
9 | - 'archivy/**'
10 | - '.github/workflows/build-docker-on-push-master.yml'
11 |
12 | jobs:
13 | dockerBuildPush:
14 | name: Build and push image with release version tag
15 | runs-on: ubuntu-latest
16 | steps:
17 | - name: Checkout files from repository
18 | uses: actions/checkout@v2
19 | with:
20 | repository: archivy/archivy-docker
21 |
22 | - name: Setting variables
23 | if: success()
24 | run: |
25 | date -u +'%Y-%m-%dT%H:%M:%SZ' > TIMESTAMP
26 | echo "${GITHUB_SHA}" | cut -c1-8 > SHORT_SHA
27 | echo "source" > IMAGE_TAG
28 | echo "uzayg" > DOCKER_USERNAME
29 | echo "docker.io/uzayg/archivy" > DOCKER_IMAGE
30 |
31 | - name: Set up Docker Buildx
32 | if: success()
33 | uses: docker/setup-buildx-action@v1.0.2
34 | with:
35 | install: true
36 | version: latest
37 |
38 | - name: Docker login
39 | if: success()
40 | env:
41 | DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
42 | run: |
43 | echo "${DOCKER_PASSWORD}" | docker login --username "$( cat DOCKER_USERNAME )" --password-stdin docker.io
44 |
45 | # Build and push images with the tags
46 | # source
47 | # hash - Commit hash(first 8 characters)
48 | - name: Build and push with Docker Buildx
49 | if: success()
50 | run: |
51 | docker build \
52 | --output type=image,name="$(cat DOCKER_IMAGE)",push=true \
53 | --build-arg BUILD_DATE="$( cat TIMESTAMP )" --build-arg VCS_REF="$( cat SHORT_SHA )" \
54 | --tag "$( cat DOCKER_IMAGE ):$( cat IMAGE_TAG )" \
55 | --file ./Dockerfile.source .
56 |
--------------------------------------------------------------------------------
/.github/workflows/build-docker-on-release.yml:
--------------------------------------------------------------------------------
1 | name: Building and Pushing Archivy Container Image On Version Release
2 |
3 | on:
4 | release:
5 | types: [published]
6 |
7 | jobs:
8 | dockerBuildPush:
9 | name: Build and push image with release version tag
10 | runs-on: ubuntu-latest
11 | steps:
12 | - name: Checkout files from repository
13 | uses: actions/checkout@v2
14 | with:
15 | repository: archivy/archivy-docker
16 |
17 | - name: Setting variables
18 | if: success()
19 | run: |
20 | date -u +'%Y-%m-%dT%H:%M:%SZ' > TIMESTAMP
21 | echo "$GITHUB_SHA" | cut -c1-8 > SHORT_SHA
22 | echo "$GITHUB_REF" | cut -d / -f 3 > VERSION
23 | echo "uzayg" > DOCKER_USERNAME
24 | echo "docker.io/uzayg/archivy" > DOCKER_IMAGE
25 |
26 | - name: Set up Docker Buildx
27 | if: success()
28 | uses: docker/setup-buildx-action@v1.0.2
29 | with:
30 | install: true
31 | version: latest
32 |
33 | - name: Docker login
34 | if: success()
35 | env:
36 | DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
37 | run: |
38 | echo "${DOCKER_PASSWORD}" | docker login --username "$( cat DOCKER_USERNAME )" --password-stdin docker.io
39 |
40 | - name: Build and push with Docker Buildx
41 | if: success()
42 | run: |
43 | docker build \
44 | --output type=image,name="$( cat DOCKER_IMAGE )",push=true \
45 | --build-arg VERSION="$( cat VERSION )" --build-arg BUILD_DATE="$( cat TIMESTAMP )" \
46 | --build-arg VCS_REF="$( cat SHORT_SHA )" \
47 | --tag "$( cat DOCKER_IMAGE ):$( cat VERSION )" \
48 | --tag "$( cat DOCKER_IMAGE ):latest" \
49 | --file ./Dockerfile . \
50 | && docker build \
51 | --output type=image,name="$( cat DOCKER_IMAGE )",push=true \
52 | --build-arg VERSION="$( cat VERSION )" --build-arg BUILD_DATE="$( cat TIMESTAMP )" \
53 | --build-arg VCS_REF="$( cat SHORT_SHA )" \
54 | --tag "$( cat DOCKER_IMAGE ):$( cat VERSION )-lite" \
55 | --tag "$( cat DOCKER_IMAGE ):latest-lite" \
56 | --file ./Dockerfile .
57 |
--------------------------------------------------------------------------------
/.github/workflows/build-docker-specific-version.yml:
--------------------------------------------------------------------------------
1 |
2 | name: Building and Pushing Archivy Container to DockerHub on manual workflow dispatch
3 |
4 | on:
5 | workflow_dispatch:
6 | inputs:
7 | version:
8 | description: Version to deploy to Dockerhub
9 | required: true
10 |
11 | jobs:
12 | dockerBuildPush:
13 | name: Build and push image with release version tag
14 | runs-on: ubuntu-latest
15 | steps:
16 | - name: Checkout files from repository
17 | uses: actions/checkout@v2
18 | with:
19 | repository: archivy/archivy-docker
20 |
21 | - name: Setting variables
22 | if: success()
23 | run: |
24 | date -u +'%Y-%m-%dT%H:%M:%SZ' > TIMESTAMP
25 | echo "${{ github.event.inputs.version }}" > VERSION
26 | echo "uzayg" > DOCKER_USERNAME
27 | echo "docker.io/uzayg/archivy" > DOCKER_IMAGE
28 |
29 | - name: Set up Docker Buildx
30 | if: success()
31 | uses: docker/setup-buildx-action@v1.0.2
32 | with:
33 | install: true
34 | version: latest
35 |
36 | - name: Docker login
37 | if: success()
38 | env:
39 | DOCKER_PASSWORD: ${{ secrets.DOCKER_ACCESS_TOKEN }}
40 | run: |
41 | echo "${DOCKER_PASSWORD}" | docker login --username "$( cat DOCKER_USERNAME )" --password-stdin docker.io
42 |
43 | - name: Build and push with Docker Buildx
44 | if: success()
45 | run: |
46 | docker build \
47 | --output type=image,name="$( cat DOCKER_IMAGE )",push=true \
48 | --build-arg VERSION="$( cat VERSION )" --build-arg BUILD_DATE="$( cat TIMESTAMP )" \
49 | --tag "$( cat DOCKER_IMAGE ):$( cat VERSION )" \
50 | --file ./Dockerfile . \
51 | && docker build \
52 | --output type=image,name="$( cat DOCKER_IMAGE )",push=true \
53 | --build-arg VERSION="$( cat VERSION )" --build-arg BUILD_DATE="$( cat TIMESTAMP )" \
54 | --tag "$( cat DOCKER_IMAGE ):$( cat VERSION )-lite" \
55 | --file ./Dockerfile-light .
56 |
--------------------------------------------------------------------------------
/.github/workflows/codecov.yml:
--------------------------------------------------------------------------------
1 | name: Testing and automated code coverage
2 |
3 | on:
4 | push:
5 | branches:
6 | - 'master'
7 | paths:
8 | - 'archivy/**'
9 | - 'tests/**'
10 | - '.github/workflows/codecov.yml'
11 | - 'requirements.txt'
12 | - 'requirements-tests.txt'
13 | pull_request:
14 | paths:
15 | - 'archivy/**'
16 | - 'tests/**'
17 | - '.github/workflows/codecov.yml'
18 | - 'requirements.txt'
19 | - 'requirements-tests.txt'
20 |
21 | jobs:
22 | run:
23 | runs-on: ubuntu-latest
24 | env:
25 | PYTHON: '3.9'
26 | steps:
27 | - uses: actions/checkout@master
28 | - name: Setup Python
29 | uses: actions/setup-python@master
30 | with:
31 | python-version: 3.9
32 | - name: Generate coverage report
33 | run: |
34 | sudo apt-get install ripgrep
35 | pip install -r requirements-tests.txt
36 | pip install .
37 | pip install tests/test_plugin
38 | pytest --cov=./ --cov-report=xml
39 | - name: Upload coverage to Codecov
40 | uses: codecov/codecov-action@v1
41 | with:
42 | file: ./coverage.xml
43 | files: ./coverage1.xml,./coverage2.xml
44 | directory: ./coverage/reports/
45 | env_vars: PYTHON
46 | fail_ci_if_error: true
47 |
--------------------------------------------------------------------------------
/.github/workflows/deploy-docs.yml:
--------------------------------------------------------------------------------
1 | name: Deploy Docs to archivy.github.io
2 | on:
3 | push:
4 | branches:
5 | - master
6 |
7 |
8 | jobs:
9 | deploy:
10 | name: Deploy website
11 | runs-on: ubuntu-latest
12 | steps:
13 | - uses: actions/checkout@v2
14 | with:
15 | path: main
16 | - name: Set up Python 3.9 and Pandoc
17 | uses: actions/setup-python@v1
18 | with:
19 | python-version: 3.9
20 | - name: Install doc deps
21 | run: >-
22 | python -m pip install -r main/docs/requirements.txt
23 | - name: Install archivy
24 | run: >-
25 | python -m pip install main/
26 | - name: Clone archivy docs website
27 | uses: actions/checkout@v2
28 | with:
29 | repository: archivy/archivy.github.io
30 | ssh-key: ${{ secrets.ARCHIVY_ACCESS_KEY }}
31 | path: docs-website
32 | - name: Deploy docs
33 | run: >-
34 | cd docs-website && mkdocs gh-deploy --config-file ../main/mkdocs.yml --remote-branch main
35 |
--------------------------------------------------------------------------------
/.github/workflows/pypi-auto-deploy.yml:
--------------------------------------------------------------------------------
1 | name: Publish Python 🐍 distributions 📦 to PyPI and TestPyPI and create Github Release
2 |
3 | on:
4 | push:
5 | # remove branchname otherwise github actions will trigger event regardless of tag.
6 | tags: v[0-9]+.[0-9]+*
7 |
8 | jobs:
9 | builds-n-publish:
10 | name: Build and publish Python 🐍 distributions 📦 to PyPI and TestPyPI + github release
11 | runs-on: ubuntu-latest
12 | steps:
13 | - uses: actions/checkout@master
14 | - name: Set up Python 3.9
15 | uses: actions/setup-python@v1
16 | with:
17 | python-version: 3.9
18 | - name: Install setuptools
19 | run: >-
20 | python -m pip install --user --upgrade setuptools wheel
21 | - name: Build a binary wheel and source tarball
22 | run: >-
23 | python setup.py sdist bdist_wheel
24 | - name: Publish distribution 📦 to PyPI
25 | uses: pypa/gh-action-pypi-publish@master
26 | with:
27 | user: __token__
28 | password: ${{ secrets.pypi_password }}
29 | - name: Upload to release
30 | uses: marvinpinto/action-automatic-releases@latest
31 | with:
32 | repo_token: ${{ secrets.RELEASE_TOKEN }}
33 | prerelease: false
34 | draft: true
35 | files: dist/*
36 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Source for the following rules: https://raw.githubusercontent.com/github/gitignore/master/Python.gitignore
2 | # Byte-compiled / optimized / DLL files
3 | .flaskenv
4 | __pycache__/
5 | *.py[cod]
6 | *$py.class
7 |
8 | # C extensions
9 | *.so
10 |
11 | # Distribution / packaging
12 | .Python
13 | build/
14 | develop-eggs/
15 | dist/
16 | downloads/
17 | eggs/
18 | .eggs/
19 | lib/
20 | lib64/
21 | parts/
22 | sdist/
23 | var/
24 | wheels/
25 | *.egg-info/
26 | .installed.cfg
27 | *.egg
28 | MANIFEST
29 |
30 | # PyInstaller
31 | # Usually these files are written by a python script from a template
32 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
33 | *.manifest
34 | *.spec
35 |
36 | # Installer logs
37 | pip-log.txt
38 | pip-delete-this-directory.txt
39 |
40 | # Unit test / coverage reports
41 | htmlcov/
42 | .tox/
43 | .nox/
44 | .coverage
45 | .coverage.*
46 | .cache
47 | nosetests.xml
48 | coverage.xml
49 | *.cover
50 | .hypothesis/
51 | .pytest_cache/
52 |
53 | # Translations
54 | *.mo
55 | *.pot
56 |
57 | # Flask stuff:
58 | instance/
59 | .webassets-cache
60 |
61 | # Scrapy stuff:
62 | .scrapy
63 |
64 | # Sphinx documentation
65 | docs/_build/
66 |
67 | # pyenv
68 | .python-version
69 |
70 | # Environments
71 | .env
72 | .venv
73 | env/
74 | venv/*
75 | ENV/
76 | env.bak/
77 | venv.bak/
78 | db.json
79 | data/
80 | .vim/
81 | .vscode/
82 |
--------------------------------------------------------------------------------
/API.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | # API documentation
4 |
5 | The archivy api allows you to interact with your archivy instance through HTTP. This allows a complete, programmatic access to the archivy's functionality.
6 |
7 | All these requests must be made by an authenticated user. For example, using the python `requests` module:
8 |
9 | ```python
10 | import requests
11 | # we create a new session that will allow us to login once
12 | s = requests.session()
13 |
14 | INSTANCE_URL = <your instance url>
15 | s.post(f"{INSTANCE_URL}/api/login", auth=(<username>, <password>))
16 |
17 | # once you've logged in - you can make authenticated requests to the api, like:
18 | resp = s.get(f"{INSTANCE_URL}/api/dataobjs").content)
19 | ```
20 |
21 | ## API spec
22 | This is an api specification for the routes you might find useful in your scripts. The url prefix for all the requests is `/api`.
23 |
24 | ### General routes
25 |
26 | | Route name | Parameters | Description |
27 | | ------------- | ------------------------------------------------------------ | ---------------------------------------------------- |
28 | | POST `/login` | [HTTP Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication): username and password | Logs you in with your archivy username and password |
29 | | GET `/search` | `query`: search query | Fetches elasticsearch results for your search terms. |
30 |
31 |
32 |
33 | ### Folders
34 |
35 | | Route name | Parameters | Description |
36 | | ------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ |
37 | | POST `/folders/new` | `path`: path of new directory For example, if you want to create the directory `trees` in the existing directory `nature`, `path = "nature/trees"` | Allows you to create new directories |
38 | | DELETE `/folders/delete` | `path`: path of directory to delete. For example, if you want to delete the `trees` dir in `nature`, `path = natures/trees` | Deletes existing directories. Also works if the directories contain data, which will be deleted with it. |
39 |
40 | ### Dataobjs
41 |
42 | | Route name | Parameters | Description |
43 | | --------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ |
44 | | POST `/notes` | `title`, `content`, `desc`, `tags`: array of tags to associate with the note, `path`: string with the relative dir in which the note should be stored. | Creates a new note in the knowledge base. The only required parameter is the title of the note. |
45 | | POST `/bookmarks` | `url`, `desc`, `tags`: array of tags to associate with the bookmark, `path`: string with the relative dir in which the note should be stored. | Stores a new bookmark. Only required parameter is `url`. |
46 | | GET `/dataobjs` | | Returns an array of all dataobjs with their title, id, contents, url, path etc... This request is resource-heavy so we might need to consider not sending the large contents. |
47 | | GET `/dataobjs/id` | | Returns data for **one** dataobj, specified by his id. |
48 | | DELETE `/dataobjs/id` | | Deletes specified dataobj. |
49 |
50 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 Uzay-G
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | include LICENSE
2 | include README.md
3 | include requirements.txt
4 | recursive-include archivy/static *
5 | recursive-include archivy/templates *
6 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 
2 |
3 | # Archivy
4 |
5 | Archivy is a self-hostable knowledge repository that allows you to learn and retain information in your own personal and extensible wiki.
6 |
7 | Features:
8 |
9 | - If you add bookmarks, their web-pages contents' will be saved to ensure that you will **always** have access to it, following the idea of [digital preservation](https://jeffhuang.com/designed_to_last/). Archivy is also easily integrated with other services and your online accounts.
10 | - Knowledge base organization with bidirectional links between notes, and embedded tags.
11 | - Everything is a file! For ease of access and editing, all the content is stored in extended markdown files with yaml front matter. This format supports footnotes, LaTeX math rendering, syntax highlighting and more.
12 | - Extensible plugin system and API for power users to take control of their knowledge process
13 | - [syncing options](https://github.com/archivy/archivy-git)
14 | - Powerful and advanced search.
15 | - Image upload
16 |
17 |
18 | [demo video](https://www.uzpg.me/assets/images/archivy.mov)
19 |
20 | [Roadmap](https://github.com/archivy/archivy/issues/74#issuecomment-764828063)
21 |
22 | Upcoming:
23 |
24 | - Annotations
25 | - Multi User System with permission setup.
26 |
27 | ## Quickstart
28 |
29 |
30 | Install archivy with `pip install archivy`. Other installations methods are listed [here](https://archivy.github.io/install), including Docker.
31 |
32 | Run the `archivy init` command to setup you installation.
33 |
34 | Then run this and enter a password to create a new user:
35 |
36 | ```bash
37 | $ archivy create-admin <username>
38 | ```
39 |
40 | Finally, execute `archivy run` to serve the app. You can open it at https://localhost:5000 and login with the credentials you entered before.
41 |
42 | You can then use archivy to create notes, bookmarks and then organize and store information.
43 |
44 | See the [official docs](https://archivy.github.io) for information on other installation methods.
45 |
46 | ## Community
47 |
48 | Archivy is dedicated to building **open and quality knowledge base software** through collaboration and community discussion.
49 |
50 | To get news and updates on Archivy and its development, you can [watch the archivy repository](https://github.com/archivy/archivy) or follow [@uzpg_ on Twitter](https://twitter.com/uzpg_).
51 |
52 | You can interact with us through the [issue board](https://github.com/archivy/archivy/issues) and the more casual [discord server](https://discord.gg/uQsqyxB).
53 |
54 | Note: If you're interested in the applications of AI to knowledge management, we're also working on this with [Espial](https://github.com/Uzay-G/espial).
55 |
--------------------------------------------------------------------------------
/SECURITY.md:
--------------------------------------------------------------------------------
1 | If you think you've found a security issue in Archivy, please contact halcyon@disroot.org before disclosing anything publicly.
2 |
--------------------------------------------------------------------------------
/archivy/__init__.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from pathlib import Path
3 | from shutil import which
4 |
5 | from elasticsearch.exceptions import RequestError
6 | from flask import Flask
7 | from flask_compress import Compress
8 | from flask_login import LoginManager
9 | from flask_wtf.csrf import CSRFProtect
10 |
11 | from archivy import helpers
12 | from archivy.api import api_bp
13 | from archivy.models import User
14 | from archivy.config import Config
15 | from archivy.helpers import load_config, get_elastic_client
16 |
17 | app = Flask(__name__)
18 | app.logger.setLevel(logging.INFO)
19 | config = Config()
20 | try:
21 | # if it exists, load user config
22 | config.override(load_config(config.INTERNAL_DIR))
23 | except FileNotFoundError:
24 | pass
25 |
26 | app.config.from_object(config)
27 | (Path(app.config["USER_DIR"]) / "data").mkdir(parents=True, exist_ok=True)
28 | (Path(app.config["USER_DIR"]) / "images").mkdir(parents=True, exist_ok=True)
29 |
30 | with app.app_context():
31 | app.config["RG_INSTALLED"] = which("rg") != None
32 | app.config["HOOKS"] = helpers.load_hooks()
33 | app.config["SCRAPING_PATTERNS"] = helpers.load_scraper()
34 | if app.config["SEARCH_CONF"]["enabled"]:
35 | with app.app_context():
36 | search_engines = ["elasticsearch", "ripgrep"]
37 | es = None
38 | if (
39 | "engine" not in app.config["SEARCH_CONF"]
40 | or app.config["SEARCH_CONF"]["engine"] not in search_engines
41 | ):
42 | # try to guess desired search engine if present
43 | app.logger.warning(
44 | "Search is enabled but engine option is invalid or absent. Archivy will"
45 | " try to guess preferred search engine."
46 | )
47 | app.config["SEARCH_CONF"]["engine"] = "none"
48 |
49 | es = get_elastic_client(error_if_invalid=False)
50 | if es:
51 | app.config["SEARCH_CONF"]["engine"] = "elasticsearch"
52 | else:
53 | if which("rg"):
54 | app.config["SEARCH_CONF"]["engine"] = "ripgrep"
55 | engine = app.config["SEARCH_CONF"]["engine"]
56 | if engine == "none":
57 | app.logger.warning("No working search engine found. Disabling search.")
58 | app.config["SEARCH_CONF"]["enabled"] = 0
59 | else:
60 | app.logger.info(f"Running {engine} installation found.")
61 |
62 | if app.config["SEARCH_CONF"]["engine"] == "elasticsearch":
63 | es = es or get_elastic_client()
64 | try:
65 | es.indices.create(
66 | index=app.config["SEARCH_CONF"]["index_name"],
67 | body=app.config["SEARCH_CONF"]["es_processing_conf"],
68 | )
69 | except RequestError:
70 | app.logger.info("Elasticsearch index already created")
71 | if app.config["SEARCH_CONF"]["engine"] == "ripgrep" and not which("rg"):
72 | app.logger.info("Ripgrep not found on system. Disabling search.")
73 | app.config["SEARCH_CONF"]["enabled"] = 0
74 |
75 |
76 | # login routes / setup
77 | login_manager = LoginManager()
78 | login_manager.login_view = "login"
79 | login_manager.init_app(app)
80 | app.register_blueprint(api_bp, url_prefix="/api")
81 | csrf = CSRFProtect(app)
82 | csrf.exempt(api_bp)
83 |
84 | # compress files
85 | Compress(app)
86 |
87 |
88 | @login_manager.user_loader
89 | def load_user(user_id):
90 | db = helpers.get_db()
91 | res = db.get(doc_id=int(user_id))
92 | if res and res["type"] == "user":
93 | return User.from_db(res)
94 | return None
95 |
96 |
97 | app.jinja_env.add_extension("jinja2.ext.do")
98 |
99 |
100 | @app.template_filter("pluralize")
101 | def pluralize(number, singular="", plural="s"):
102 | if number == 1:
103 | return singular
104 | else:
105 | return plural
106 |
107 |
108 | from archivy import routes # noqa:
109 |
--------------------------------------------------------------------------------
/archivy/api.py:
--------------------------------------------------------------------------------
1 | from flask import Response, jsonify, request, Blueprint, current_app
2 | from werkzeug.security import check_password_hash
3 | from flask_login import login_user
4 | from tinydb import Query
5 |
6 | from archivy import data, tags
7 | from archivy.search import search
8 | from archivy.models import DataObj, User
9 | from archivy.helpers import get_db
10 |
11 |
12 | api_bp = Blueprint("api", __name__)
13 |
14 |
15 | @api_bp.route("/login", methods=["POST"])
16 | def login():
17 | """
18 | Logs in the API client using
19 | [HTTP Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication).
20 | Pass in the username and password of your account.
21 | """
22 | db = get_db()
23 | user = db.search(Query().username == request.authorization["username"])
24 | if user and check_password_hash(
25 | user[0]["hashed_password"], request.authorization["password"]
26 | ):
27 | # user is verified so we can log him in from the db
28 | user = User.from_db(user[0])
29 | login_user(user, remember=True)
30 | return Response(status=200)
31 | return Response(status=401)
32 |
33 |
34 | @api_bp.route("/bookmarks", methods=["POST"])
35 | def create_bookmark():
36 | """
37 | Creates a new bookmark
38 |
39 | **Parameters:**
40 |
41 | All parameters are sent through the JSON body.
42 | - **url** (required)
43 | - **tags**
44 | - **path**
45 | """
46 | json_data = request.get_json()
47 | bookmark = DataObj(
48 | url=json_data["url"],
49 | tags=json_data.get("tags", []),
50 | path=json_data.get("path", current_app.config["DEFAULT_BOOKMARKS_DIR"]),
51 | type="bookmark",
52 | )
53 | bookmark.process_bookmark_url()
54 | bookmark_id = bookmark.insert()
55 | if bookmark_id:
56 | return jsonify(
57 | bookmark_id=bookmark_id,
58 | )
59 | return Response(status=400)
60 |
61 |
62 | @api_bp.route("/notes", methods=["POST"])
63 | def create_note():
64 | """
65 | Creates a new note.
66 |
67 | **Parameters:**
68 |
69 | All parameters are sent through the JSON body.
70 | - **title** (required)
71 | - **content** (required)
72 | - **tags**
73 | - **path**
74 | """
75 | json_data = request.get_json()
76 | note = DataObj(
77 | title=json_data["title"],
78 | content=json_data["content"],
79 | path=json_data.get("path", ""),
80 | tags=json_data.get("tags", []),
81 | type="note",
82 | )
83 |
84 | note_id = note.insert()
85 | if note_id:
86 | return jsonify(note_id=note_id)
87 | return Response(status=400)
88 |
89 |
90 | @api_bp.route("/dataobjs/<int:dataobj_id>")
91 | def get_dataobj(dataobj_id):
92 | """Returns dataobj of given id"""
93 | dataobj = data.get_item(dataobj_id)
94 |
95 | return (
96 | jsonify(
97 | dataobj_id=dataobj_id,
98 | title=dataobj["title"],
99 | content=dataobj.content,
100 | md_path=dataobj["fullpath"],
101 | )
102 | if dataobj
103 | else Response(status=404)
104 | )
105 |
106 |
107 | @api_bp.route("/dataobjs/<int:dataobj_id>", methods=["DELETE"])
108 | def delete_dataobj(dataobj_id):
109 | """Deletes object of given id"""
110 | if not data.get_item(dataobj_id):
111 | return Response(status=404)
112 | data.delete_item(dataobj_id)
113 | return Response(status=204)
114 |
115 |
116 | @api_bp.route("/dataobjs/<int:dataobj_id>", methods=["PUT"])
117 | def update_dataobj(dataobj_id):
118 | """
119 | Updates object of given id.
120 |
121 | Paramter in JSON body:
122 |
123 | - **content**: markdown text of new dataobj.
124 | """
125 | if request.json.get("content"):
126 | try:
127 | data.update_item_md(dataobj_id, request.json.get("content"))
128 | return Response(status=200)
129 | except BaseException:
130 | return Response(status=404)
131 | return Response("Must provide content parameter", status=401)
132 |
133 |
134 | @api_bp.route("/dataobjs/frontmatter/<int:dataobj_id>", methods=["PUT"])
135 | def update_dataobj_frontmatter(dataobj_id):
136 | """
137 | Updates frontmatter of object of given id.
138 |
139 | Paramter in JSON body:
140 |
141 | - **title**: the new title of the dataobj.
142 | """
143 |
144 | new_frontmatter = {
145 | "title": request.json.get("title"),
146 | }
147 |
148 | try:
149 | data.update_item_frontmatter(dataobj_id, new_frontmatter)
150 | return Response(status=200)
151 | except BaseException:
152 | return Response(status=404)
153 |
154 |
155 | @api_bp.route("/dataobjs", methods=["GET"])
156 | def get_dataobjs():
157 | """Gets all dataobjs"""
158 | cur_dir = data.get_items(structured=False, json_format=True)
159 | return jsonify(cur_dir)
160 |
161 |
162 | @api_bp.route("/tags/add_to_index", methods=["PUT"])
163 | def add_tag_to_index():
164 | """Add a tag to the database."""
165 | tag = request.json.get("tag", False)
166 | if tag and type(tag) is str and tags.is_tag_format(tag):
167 | if tags.add_tag_to_index(tag):
168 | return Response(status=200)
169 | else:
170 | return Response(status=404)
171 |
172 | return Response("Must provide valid tag name.", status=401)
173 |
174 |
175 | @api_bp.route("/dataobj/local_edit/<dataobj_id>", methods=["GET"])
176 | def local_edit(dataobj_id):
177 | dataobj = data.get_item(dataobj_id)
178 | if dataobj:
179 | data.open_file(dataobj["fullpath"])
180 | return Response(status=200)
181 | return Response(status=404)
182 |
183 |
184 | @api_bp.route("/folders/new", methods=["POST"])
185 | def create_folder():
186 | """
187 | Creates new directory
188 |
189 | Parameter in JSON body:
190 | - **path** (required) - path of newdir
191 | """
192 | directory = request.json.get("path")
193 | try:
194 | sanitized_name = data.create_dir(directory)
195 | if not sanitized_name:
196 | return Response("Invalid dirname", status=400)
197 | except FileExistsError:
198 | return Response("Directory already exists", status=400)
199 | return Response(sanitized_name, status=200)
200 |
201 |
202 | @api_bp.route("/folders/delete", methods=["DELETE"])
203 | def delete_folder():
204 | """
205 | Deletes directory.
206 |
207 | Parameter in JSON body:
208 | - **path** of dir to delete
209 | """
210 | directory = request.json.get("path")
211 | if directory == "":
212 | return Response("Cannot delete root dir", status=401)
213 | if data.delete_dir(directory):
214 | return Response("Successfully deleted", status=200)
215 | return Response("Could not delete directory", status=400)
216 |
217 |
218 | @api_bp.route("/search", methods=["GET"])
219 | def search_endpoint():
220 | """
221 | Searches the instance.
222 |
223 | Request URL Parameter:
224 | - **query**
225 | """
226 | if not current_app.config["SEARCH_CONF"]["enabled"]:
227 | return Response("Search is disabled", status=401)
228 | query = request.args.get("query")
229 | search_results = search(query)
230 | return jsonify(search_results)
231 |
232 |
233 | @api_bp.route("/images", methods=["POST"])
234 | def image_upload():
235 | CONTENT_TYPES = ["image/jpeg", "image/png", "image/gif"]
236 | if "image" not in request.files:
237 | return jsonify({"error": "400"}), 400
238 | image = request.files["image"]
239 | if (
240 | data.valid_image_filename(image.filename)
241 | and image.headers["Content-Type"].strip() in CONTENT_TYPES
242 | ):
243 | saved_to = data.save_image(image)
244 | return jsonify({"data": {"filePath": f"/images/{saved_to}"}}), 200
245 | return jsonify({"error": "415"}), 415
246 |
--------------------------------------------------------------------------------
/archivy/cli.py:
--------------------------------------------------------------------------------
1 | from pathlib import Path
2 | from os import environ
3 | from pkg_resources import iter_entry_points
4 |
5 | import click
6 | from click_plugins import with_plugins
7 | from flask.cli import FlaskGroup, load_dotenv, shell_command
8 |
9 | from archivy import app
10 | from archivy.config import Config
11 | from archivy.click_web import create_click_web_app
12 | from archivy.data import open_file, format_file, unformat_file
13 | from archivy.helpers import load_config, write_config, create_plugin_dir
14 | from archivy.models import User, DataObj
15 |
16 |
17 | def create_app():
18 | return app
19 |
20 |
21 | @with_plugins(iter_entry_points("archivy.plugins"))
22 | @click.group(cls=FlaskGroup, create_app=create_app)
23 | def cli():
24 | pass
25 |
26 |
27 | @cli.command("init", short_help="Initialise your archivy application")
28 | @click.pass_context
29 | def init(ctx):
30 | try:
31 | load_config()
32 | click.confirm(
33 | "Config already found. Do you wish to reset it? "
34 | "Otherwise run `archivy config`",
35 | abort=True,
36 | )
37 | except FileNotFoundError:
38 | pass
39 |
40 | config = Config()
41 |
42 | click.echo("This is the archivy installation initialization wizard.")
43 | data_dir = click.prompt(
44 | "Enter the full path of the " "directory where you'd like us to store data.",
45 | type=str,
46 | default=str(Path(".").resolve()),
47 | )
48 |
49 | desires_search = click.confirm(
50 | "Would you like to enable search on your knowledge base contents?"
51 | )
52 |
53 | if desires_search:
54 | search_engine = click.prompt(
55 | "Then go to https://archivy.github.io/setup-search/ to see the different backends you can use for search and how you can configure them.",
56 | type=click.Choice(["elasticsearch", "ripgrep", "cancel"]),
57 | show_choices=True,
58 | )
59 | if search_engine != "cancel":
60 | config.SEARCH_CONF["enabled"] = 1
61 | config.SEARCH_CONF["engine"] = search_engine
62 |
63 | create_new_user = click.confirm("Would you like to create a new admin user?")
64 | if create_new_user:
65 | username = click.prompt("Username")
66 | password = click.prompt("Password", hide_input=True, confirmation_prompt=True)
67 | if not ctx.invoke(create_admin, username=username, password=password):
68 | return
69 |
70 | config.HOST = click.prompt(
71 | "Host [localhost (127.0.0.1)]",
72 | type=str,
73 | default="127.0.0.1",
74 | show_default=False,
75 | )
76 |
77 | config.override({"USER_DIR": data_dir})
78 | app.config["USER_DIR"] = data_dir
79 |
80 | # create data dir
81 | (Path(data_dir) / "data").mkdir(exist_ok=True, parents=True)
82 |
83 | write_config(vars(config))
84 | click.echo(
85 | "Config successfully created at "
86 | + str((Path(app.config["INTERNAL_DIR"]) / "config.yml").resolve())
87 | )
88 |
89 |
90 | @cli.command("config", short_help="Open archivy config.")
91 | def config():
92 | open_file(str(Path(app.config["INTERNAL_DIR"]) / "config.yml"))
93 |
94 |
95 | @cli.command("hooks", short_help="Creates hook file if it is not setup and opens it.")
96 | def hooks():
97 | hook_path = Path(app.config["USER_DIR"]) / "hooks.py"
98 | if not hook_path.exists():
99 | with hook_path.open("w") as f:
100 | f.write(
101 | "from archivy.config import BaseHooks\n"
102 | "class Hooks(BaseHooks):\n"
103 | " # see available hooks at https://archivy.github.io/reference/hooks/\n"
104 | " def on_dataobj_create(self, dataobj): # for example\n"
105 | " pass"
106 | )
107 | open_file(hook_path)
108 |
109 |
110 | @cli.command("run", short_help="Runs archivy web application")
111 | def run():
112 | click.echo("Running archivy...")
113 | load_dotenv()
114 | environ["FLASK_RUN_FROM_CLI"] = "false"
115 | app_with_cli = create_click_web_app(click, cli, app)
116 | app_with_cli.run(host=app.config["HOST"], port=app.config["PORT"])
117 |
118 |
119 | @cli.command(short_help="Creates a new admin user")
120 | @click.argument("username")
121 | @click.password_option()
122 | def create_admin(username, password):
123 | if len(password) < 8:
124 | click.echo("Password length too short")
125 | return False
126 | else:
127 | user = User(username=username, password=password, is_admin=True)
128 | if user.insert():
129 | click.echo(f"User {username} successfully created.")
130 | return True
131 | else:
132 | click.echo("User with given username already exists.")
133 | return False
134 |
135 |
136 | @cli.command(short_help="Format normal markdown files for archivy.")
137 | @click.argument("filenames", type=click.Path(exists=True), nargs=-1)
138 | def format(filenames):
139 | for path in filenames:
140 | format_file(path)
141 |
142 |
143 | @cli.command(short_help="Convert archivy-formatted files back to normal markdown.")
144 | @click.argument("filenames", type=click.Path(exists=True), nargs=-1)
145 | @click.argument("output_dir", type=click.Path(exists=True, file_okay=False))
146 | def unformat(filenames, output_dir):
147 | for path in filenames:
148 | unformat_file(path, output_dir)
149 |
150 |
151 | @cli.command(short_help="Sync content to Elasticsearch")
152 | def index():
153 | data_dir = Path(app.config["USER_DIR"]) / "data"
154 |
155 | if not app.config["SEARCH_CONF"]["enabled"]:
156 | click.echo("Search must be enabled for this command.")
157 | return
158 |
159 | for filename in data_dir.rglob("*.md"):
160 | cur_file = open(filename)
161 | dataobj = DataObj.from_md(cur_file.read())
162 | cur_file.close()
163 |
164 | if dataobj.index():
165 | click.echo(f"Indexed {dataobj.title}...")
166 | else:
167 | click.echo(f"Failed to index {dataobj.title}")
168 |
169 |
170 | @cli.command(
171 | short_help="Helper command to auto-generate plugin directory with structure"
172 | )
173 | @click.argument("name")
174 | def plugin_new(name):
175 | if create_plugin_dir(name):
176 | click.echo(f"Successfully Created the plugin directory structure at '{name}'/.")
177 | else:
178 | click.echo(f"Directory with name '{name}' already exists.")
179 |
180 |
181 | if __name__ == "__main__":
182 | cli()
183 |
--------------------------------------------------------------------------------
/archivy/click_web/__init__.py:
--------------------------------------------------------------------------------
1 | import tempfile
2 | from pathlib import Path
3 |
4 | import click
5 | import jinja2
6 | from flask import Blueprint
7 |
8 | from archivy.click_web.resources import cmd_exec, cmd_form, index
9 |
10 | jinja_env = jinja2.Environment(extensions=["jinja2.ext.do"])
11 |
12 | "The full path to the click script file to execute."
13 | script_file = None
14 | "The click root command to serve"
15 | click_root_cmd = None
16 |
17 |
18 | def _get_output_folder():
19 | _output_folder = Path(tempfile.gettempdir()) / "click-web"
20 | if not _output_folder.exists():
21 | _output_folder.mkdir()
22 | return _output_folder
23 |
24 |
25 | "Where to place result files for download"
26 | OUTPUT_FOLDER = str(_get_output_folder())
27 |
28 | _flask_app = None
29 | logger = None
30 |
31 |
32 | def create_click_web_app(module, command: click.BaseCommand, flask_app):
33 | """
34 | Create a Flask app that wraps a click command. (Call once)
35 |
36 | :param module: the module that contains the click command,
37 | needed to get the path to the script.
38 | :param command: The actual click root command, needed
39 | to be able to read the command tree and arguments
40 | in order to generate the index page and the html forms
41 | """
42 | global _flask_app, logger
43 | assert _flask_app is None, "Flask App already created."
44 |
45 | _register(module, command)
46 |
47 | _flask_app = flask_app
48 |
49 | # add the "do" extension needed by our jinja templates
50 | _flask_app.jinja_env.add_extension("jinja2.ext.do")
51 |
52 | _flask_app.add_url_rule("/plugins", "cli_index", index.index)
53 | _flask_app.add_url_rule(
54 | "/cli/<path:command_path>", "command", cmd_form.get_form_for
55 | )
56 | _flask_app.add_url_rule(
57 | "/cli/<path:command_path>", "command_execute", cmd_exec.exec, methods=["POST"]
58 | )
59 |
60 | _flask_app.logger.info(f"OUTPUT_FOLDER: {OUTPUT_FOLDER}")
61 | results_blueprint = Blueprint(
62 | "results",
63 | __name__,
64 | static_url_path="/static/results",
65 | static_folder=OUTPUT_FOLDER,
66 | )
67 | _flask_app.register_blueprint(results_blueprint)
68 |
69 | logger = _flask_app.logger
70 |
71 | return _flask_app
72 |
73 |
74 | def _register(module, command: click.BaseCommand):
75 | """
76 | :param module: the module that contains the command, needed to get the path to the script.
77 | :param command: The actual click root command,
78 | needed to be able to read the command tree and arguments
79 | in order to generate the index page and the html forms
80 | """
81 | global click_root_cmd, script_file
82 | script_file = str(Path(module.__file__).absolute())
83 | click_root_cmd = command
84 |
--------------------------------------------------------------------------------
/archivy/click_web/exceptions.py:
--------------------------------------------------------------------------------
1 | class ClickWebException(Exception):
2 | pass
3 |
4 |
5 | class CommandNotFound(ClickWebException):
6 | pass
7 |
--------------------------------------------------------------------------------
/archivy/click_web/resources/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/archivy/click_web/resources/__init__.py
--------------------------------------------------------------------------------
/archivy/click_web/resources/cmd_form.py:
--------------------------------------------------------------------------------
1 | from html import escape
2 | from typing import List, Tuple
3 |
4 | import click
5 | from flask import abort, render_template
6 |
7 | from archivy import click_web
8 | from archivy.click_web.exceptions import CommandNotFound
9 | from archivy.click_web.resources.input_fields import get_input_field
10 |
11 |
12 | def get_form_for(command_path: str):
13 | try:
14 | ctx_and_commands = _get_commands_by_path(command_path)
15 | except CommandNotFound as err:
16 | return abort(404, str(err))
17 |
18 | levels = _generate_form_data(ctx_and_commands)
19 | return render_template(
20 | "click_web/command_form.html",
21 | levels=levels,
22 | command=levels[-1]["command"],
23 | command_path=command_path,
24 | )
25 |
26 |
27 | def _get_commands_by_path(command_path: str) -> Tuple[click.Context, click.Command]:
28 | """
29 | Take a (slash separated) string and generate (context, command) for each level.
30 | :param command_path: "some_group/a_command"
31 | :return: Return a list from root to leaf commands.
32 | """
33 | command_path = "cli/" + command_path
34 | command_path_items = command_path.split("/")
35 | command_name, *command_path_items = command_path_items
36 | command = click_web.click_root_cmd
37 | if command.name != command_name:
38 | raise CommandNotFound(
39 | "Failed to find root command {}. There is a root commande named: {}".format(
40 | command_name, command.name
41 | )
42 | )
43 | result = []
44 | with click.Context(command, info_name=command, parent=None) as ctx:
45 | result.append((ctx, command))
46 | # dig down the path parts to find the leaf command
47 | parent_command = command
48 | for command_name in command_path_items:
49 | command = parent_command.get_command(ctx, command_name)
50 | if command:
51 | # create sub context for command
52 | ctx = click.Context(command, info_name=command, parent=ctx)
53 | parent_command = command
54 | else:
55 | raise CommandNotFound(
56 | """Failed to find command for path "{}".
57 | Command "{}" not found. Must be one of {}""".format(
58 | command_path, command_name, parent_command.list_commands(ctx)
59 | )
60 | )
61 | result.append((ctx, command))
62 | return result
63 |
64 |
65 | def _generate_form_data(ctx_and_commands: List[Tuple[click.Context, click.Command]]):
66 | """
67 | Construct a list of contexts and commands generate a
68 | python data structure for rendering jinja form
69 | :return: a list of dicts
70 | """
71 | levels = []
72 | for command_index, (ctx, command) in enumerate(ctx_and_commands):
73 | # force help option off, no need in web.
74 | command.add_help_option = False
75 | command.html_help = _process_help(command.help)
76 |
77 | input_fields = [
78 | get_input_field(ctx, param, command_index, param_index)
79 | for param_index, param in enumerate(command.get_params(ctx))
80 | ]
81 | levels.append({"command": command, "fields": input_fields})
82 |
83 | return levels
84 |
85 |
86 | def _process_help(help_text):
87 | """
88 | Convert click command help into html to be presented to browser.
89 | Respects the '\b' char used by click to mark pre-formatted blocks.
90 | Also escapes html reserved characters in the help text.
91 |
92 | :param help_text: str
93 | :return: A html formatted help string.
94 | """
95 | help = []
96 | in_pre = False
97 | html_help = ""
98 | if not help_text:
99 | return html_help
100 |
101 | line_iter = iter(help_text.splitlines())
102 | while True:
103 | try:
104 | line = next(line_iter)
105 | if in_pre and not line.strip():
106 | # end of code block
107 | in_pre = False
108 | html_help += "\n".join(help)
109 | help = []
110 | help.append("</pre>")
111 | continue
112 | elif line.strip() == "\b":
113 | # start of code block
114 | in_pre = True
115 | html_help += "<br>\n".join(help)
116 | help = []
117 | help.append("<pre>")
118 | continue
119 | help.append(escape(line))
120 | except StopIteration:
121 | break
122 |
123 | html_help += "\n".join(help) if in_pre else "<br>\n".join(help)
124 | return html_help
125 |
--------------------------------------------------------------------------------
/archivy/click_web/resources/index.py:
--------------------------------------------------------------------------------
1 | from collections import OrderedDict
2 |
3 | import click
4 | from flask import render_template
5 |
6 | from archivy import click_web
7 |
8 |
9 | def index():
10 | with click.Context(
11 | click_web.click_root_cmd, info_name=click_web.click_root_cmd.name, parent=None
12 | ) as ctx:
13 | return render_template(
14 | "click_web/show_tree.html",
15 | ctx=ctx,
16 | tree=_click_to_tree(ctx, click_web.click_root_cmd),
17 | title="Plugins",
18 | )
19 |
20 |
21 | def _click_to_tree(ctx: click.Context, node: click.BaseCommand, ancestors=[]):
22 | """
23 | Convert a click root command to a tree of dicts and lists
24 | :return: a json like tree
25 | """
26 | res_childs = []
27 | res = OrderedDict()
28 | res["is_group"] = isinstance(node, click.core.MultiCommand)
29 | omitted = ["shell", "run", "routes", "create-admin"]
30 | if res["is_group"]:
31 | # a group, recurse for e very child
32 | subcommand_names = set(click.Group.list_commands(node, ctx))
33 | children = [
34 | node.get_command(ctx, key) for key in subcommand_names if key not in omitted
35 | ]
36 | # Sort so commands comes before groups
37 | children = sorted(
38 | children, key=lambda c: isinstance(c, click.core.MultiCommand)
39 | )
40 | for child in children:
41 | res_childs.append(
42 | _click_to_tree(
43 | ctx,
44 | child,
45 | ancestors[:]
46 | + [
47 | node,
48 | ],
49 | )
50 | )
51 |
52 | res["name"] = node.name
53 |
54 | # Do not include any preformatted block (\b) for the short help.
55 | res["short_help"] = node.get_short_help_str().split("\b")[0]
56 | res["help"] = node.help
57 | path_parts = ancestors + [node]
58 | res["path"] = "/" + "/".join(p.name for p in path_parts)
59 | if res_childs:
60 | res["childs"] = res_childs
61 | return res
62 |
--------------------------------------------------------------------------------
/archivy/click_web/web_click_types.py:
--------------------------------------------------------------------------------
1 | """
2 | Extra click types that could be useful in a web context as
3 | they have corresponding HTML form input type.
4 | The custom web click types need only be imported
5 | into the main script, not the app.py that flask runs.
6 |
7 | Example usage in your click command:
8 | \b
9 | from click_web.web_click_types import EMAIL_TYPE
10 | @cli.command()
11 | @click.option("--the_email", type=EMAIL_TYPE)
12 | def email(the_email):
13 | click.echo(f"{the_email} is a valid email syntax.")
14 |
15 | """
16 | import re
17 |
18 | import click
19 |
20 |
21 | class EmailParamType(click.ParamType):
22 | name = "email"
23 | EMAIL_REGEX = re.compile(r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
24 |
25 | def convert(self, value, param, ctx):
26 | if self.EMAIL_REGEX.match(value):
27 | return value
28 | else:
29 | self.fail(f"{value} is not a valid email", param, ctx)
30 |
31 |
32 | class PasswordParamType(click.ParamType):
33 | name = "password"
34 |
35 | def convert(self, value, param, ctx):
36 | return value
37 |
38 |
39 | EMAIL_TYPE = EmailParamType()
40 | PASSWORD_TYPE = PasswordParamType()
41 |
--------------------------------------------------------------------------------
/archivy/config.py:
--------------------------------------------------------------------------------
1 | import os
2 | import appdirs
3 |
4 |
5 | class Config(object):
6 | """Configuration object for the application"""
7 |
8 | def __init__(self):
9 | self.SECRET_KEY = os.urandom(32)
10 | self.PORT = 5000
11 | self.HOST = "127.0.0.1"
12 |
13 | overridden_internal_dir = os.environ.get("ARCHIVY_INTERNAL_DIR_PATH")
14 | self.INTERNAL_DIR = overridden_internal_dir or appdirs.user_data_dir("archivy")
15 |
16 | self.USER_DIR = self.INTERNAL_DIR
17 | self.DEFAULT_BOOKMARKS_DIR = ""
18 | self.SITE_TITLE = "Archivy"
19 | os.makedirs(self.INTERNAL_DIR, exist_ok=True)
20 |
21 | self.PANDOC_HIGHLIGHT_THEME = "pygments"
22 | self.SCRAPING_CONF = {"save_images": False}
23 |
24 | self.THEME_CONF = {
25 | "use_theme_dark": False,
26 | "use_custom_css": False,
27 | "custom_css_file": "",
28 | }
29 | self.DATAOBJ_JS_EXTENSION = ""
30 | self.EDITOR_CONF = {
31 | "autosave": False,
32 | "settings": {
33 | "html": False,
34 | "xhtmlOut": False,
35 | "breaks": False,
36 | "linkify": True,
37 | "typographer": False,
38 | },
39 | "plugins": {
40 | "markdownitFootnote": {},
41 | "markdownitMark": {},
42 | "markdownItAnchor": {"permalink": True, "permalinkSymbol": "¶"},
43 | "markdownItTocDoneRight": {},
44 | },
45 | "spellcheck": False,
46 | "toolbar_icons": [
47 | "bold",
48 | "italic",
49 | "link",
50 | "upload-image",
51 | "heading",
52 | "code",
53 | "strikethrough",
54 | "quote",
55 | "table",
56 | ], # https://github.com/Ionaru/easy-markdown-editor#toolbar-icons
57 | }
58 |
59 | self.SEARCH_CONF = {
60 | "enabled": 0,
61 | "url": "http://localhost:9200",
62 | "index_name": "dataobj",
63 | "engine": "",
64 | "es_user": "",
65 | "es_password": "",
66 | "es_processing_conf": {
67 | "settings": {
68 | "highlight": {"max_analyzed_offset": 100000000},
69 | "analysis": {
70 | "analyzer": {
71 | "rebuilt_standard": {
72 | "stopwords": "_english_",
73 | "tokenizer": "standard",
74 | "filter": ["lowercase", "kstem", "trim", "unique"],
75 | }
76 | }
77 | },
78 | },
79 | "mappings": {
80 | "properties": {
81 | "title": {
82 | "type": "text",
83 | "analyzer": "rebuilt_standard",
84 | "term_vector": "with_positions_offsets",
85 | },
86 | "tags": {"type": "text", "analyzer": "rebuilt_standard"},
87 | "body": {"type": "text", "analyzer": "rebuilt_standard"},
88 | }
89 | },
90 | },
91 | }
92 |
93 | def override(self, user_conf: dict, nested_dict=None):
94 | """
95 | This function enables an override of the default configuration with user values.
96 |
97 | Acts smartly so as to only set options already set in the default config.
98 |
99 | - user_conf: current (nested) dictionary of user config key/values
100 | - nested_dict: reference to the current object that should be modified.
101 | If none it's just a reference to the current Config itself, otherwise it's a nested dict of the Config
102 | """
103 | for k, v in user_conf.items():
104 | if (nested_dict and not k in nested_dict) or (
105 | not nested_dict and not hasattr(self, k)
106 | ):
107 | # check the key is indeed defined in our defaults
108 | continue
109 | curr_default_val = nested_dict[k] if nested_dict else getattr(self, k)
110 | if type(v) is dict:
111 | # pass on override to sub configuration dictionary if the current type of value being traversed is dict
112 | self.override(v, curr_default_val)
113 | else:
114 | # otherwise just set
115 | if type(curr_default_val) == list and type(v) == str:
116 | v = v.split(", ")
117 | if nested_dict:
118 | nested_dict[k] = v
119 | else:
120 | setattr(self, k, v)
121 |
122 |
123 | class BaseHooks:
124 | """
125 | Class of methods users can inherit to configure and extend archivy with hooks.
126 |
127 |
128 | ## Usage:
129 |
130 | Archivy checks for the presence of a `hooks.py` file in the
131 | user directory that stores the `data/` directory with your notes and bookmarks.
132 | This location is usually set during `archivy init`.
133 |
134 | Example `hooks.py` file:
135 |
136 | ```python
137 | from archivy.config import BaseHooks
138 |
139 | class Hooks(BaseHooks):
140 | def on_edit(self, dataobj):
141 | print(f"Edit made to {dataobj.title}")
142 |
143 | def before_dataobj_create(self, dataobj):
144 | from random import randint
145 | dataobj.content += f"\\nThis note's random number is {randint(1, 10)}"
146 |
147 | # ...
148 | ```
149 |
150 | If you have ideas for any other hooks you'd find useful if they were supported,
151 | please open an [issue](https://github.com/archivy/archivy/issues).
152 | """
153 |
154 | def on_dataobj_create(self, dataobj):
155 | """Hook for dataobj creation."""
156 |
157 | def before_dataobj_create(self, dataobj):
158 | """Hook called immediately before dataobj creation."""
159 |
160 | def on_user_create(self, user):
161 | """Hook called after a new user is created."""
162 |
163 | def on_edit(self, dataobj):
164 | """Hook called whenever a user edits through the web interface or the API."""
165 |
--------------------------------------------------------------------------------
/archivy/forms.py:
--------------------------------------------------------------------------------
1 | from flask_wtf import FlaskForm
2 | from wtforms import (
3 | StringField,
4 | SubmitField,
5 | SelectField,
6 | PasswordField,
7 | HiddenField,
8 | BooleanField,
9 | FormField,
10 | )
11 | from wtforms.fields.html5 import IntegerField
12 | from wtforms.validators import DataRequired, URL
13 | from archivy.config import Config
14 |
15 |
16 | class NewBookmarkForm(FlaskForm):
17 | url = StringField("url", validators=[DataRequired(), URL()])
18 | tags = StringField("tags")
19 | path = SelectField("Folder")
20 | submit = SubmitField("Save")
21 |
22 |
23 | class NewNoteForm(FlaskForm):
24 | title = StringField("title", validators=[DataRequired()])
25 | tags = StringField("tags")
26 | path = SelectField("Folder")
27 | submit = SubmitField("Save")
28 |
29 |
30 | class NewFolderForm(FlaskForm):
31 | parent_dir = HiddenField()
32 | new_dir = StringField("New folder", validators=[DataRequired()])
33 | submit = SubmitField("Create sub directory")
34 |
35 |
36 | class MoveItemForm(FlaskForm):
37 | path = SelectField("Move to")
38 | submit = SubmitField("✓")
39 |
40 |
41 | class RenameDirectoryForm(FlaskForm):
42 | new_name = StringField("New name", validators=[DataRequired()])
43 | current_path = HiddenField()
44 | submit = SubmitField("Rename current folder")
45 |
46 |
47 | class DeleteDataForm(FlaskForm):
48 | submit = SubmitField("Delete")
49 |
50 |
51 | class DeleteFolderForm(FlaskForm):
52 | dir_name = HiddenField(validators=[DataRequired()])
53 |
54 |
55 | class UserForm(FlaskForm):
56 | username = StringField("username")
57 | password = PasswordField("password")
58 | submit = SubmitField("Submit")
59 |
60 |
61 | class TitleForm(FlaskForm):
62 | title = StringField("title")
63 | submit = SubmitField("✓")
64 |
65 |
66 | def config_form(current_conf, sub=0, allowed=vars(Config())):
67 | """
68 | This function defines a Config form that loads default configuration values and creates inputs for each field option
69 |
70 | - current_conf: object of current configuration that we will use to set the defaults
71 | - sub: this function is recursive and can return a sub form to represent the nesting of the config.
72 | Sub is a boolean value indicating whether the current form is nested.
73 |
74 | - allowed represents a dictionary of the keys that are allowed in our current level of nesting.
75 | It's fetched from the default config.
76 | """
77 |
78 | class ConfigForm(FlaskForm):
79 | pass
80 |
81 | def process_conf_value(name, val):
82 | """
83 | Create and set different form fields.
84 |
85 | """
86 | val_type = type(val)
87 | if not name in allowed:
88 | return
89 | if val_type is dict:
90 | sub_form = config_form(val, 1, allowed[name])
91 | setattr(ConfigForm, name, FormField(sub_form))
92 | elif val_type is int:
93 | setattr(ConfigForm, name, IntegerField(name, default=val))
94 | elif val_type is str:
95 | setattr(ConfigForm, name, StringField(name, default=val))
96 | elif val_type is bool:
97 | setattr(ConfigForm, name, BooleanField(name, default=val))
98 | elif val_type is list:
99 | setattr(ConfigForm, name, StringField(name, default=", ".join(val)))
100 |
101 | for key, val in current_conf.items():
102 | process_conf_value(key, val)
103 | if sub:
104 | return ConfigForm
105 | else:
106 | ConfigForm.submit = SubmitField("Submit")
107 | return ConfigForm()
108 |
--------------------------------------------------------------------------------
/archivy/helpers.py:
--------------------------------------------------------------------------------
1 | from pathlib import Path
2 | import sys
3 | import os
4 |
5 | import elasticsearch
6 | import yaml
7 | from elasticsearch import Elasticsearch
8 | from flask import current_app, g, request
9 | from tinydb import TinyDB, Query, operations
10 | from urllib.parse import urlparse, urljoin
11 |
12 | from archivy.config import BaseHooks, Config
13 |
14 |
15 | def load_config(path=""):
16 | """Loads `config.yml` file safely and deserializes it to a python dict."""
17 | path = path or current_app.config["INTERNAL_DIR"]
18 | with (Path(path) / "config.yml").open() as f:
19 | return yaml.load(f.read(), Loader=yaml.SafeLoader)
20 |
21 |
22 | def config_diff(curr_key, curr_val, parent_dict, defaults):
23 | """
24 | This function diffs the user config with the defaults to only save what is actually different.
25 |
26 | Returns 1 if the current element or its nested elements are different and have been preserved.
27 | """
28 | if type(curr_val) is dict:
29 | # the any call here diffs all nested children of the current dict and returns whether any have modifications
30 | if not any(
31 | [
32 | config_diff(k, v, curr_val, defaults[curr_key])
33 | for k, v in list(curr_val.items())
34 | ]
35 | ):
36 | parent_dict.pop(curr_key)
37 | return 0
38 | else:
39 | if defaults[curr_key] == curr_val:
40 | parent_dict.pop(curr_key)
41 | return 0
42 | return 1
43 |
44 |
45 | def write_config(config: dict):
46 | """
47 | Writes a new config dict to a `config.yml` file that will override defaults.
48 | Compares user config with defaults to only save changes.
49 | """
50 | defaults = vars(Config())
51 | for k, v in list(config.items()):
52 | if k != "SECRET_KEY":
53 | config_diff(k, v, config, defaults)
54 | with (Path(current_app.config["INTERNAL_DIR"]) / "config.yml").open("w") as f:
55 | yaml.dump(config, f)
56 |
57 |
58 | def load_hooks():
59 | try:
60 | user_hooks = (Path(current_app.config["USER_DIR"]) / "hooks.py").open()
61 | except FileNotFoundError:
62 | return BaseHooks()
63 |
64 | user_locals = {}
65 | exec(user_hooks.read(), globals(), user_locals)
66 | user_hooks.close()
67 | return user_locals.get("Hooks", BaseHooks)()
68 |
69 |
70 | def load_scraper():
71 | try:
72 | user_scraping = (Path(current_app.config["USER_DIR"]) / "scraping.py").open()
73 | except FileNotFoundError:
74 | return {}
75 | user_locals = {}
76 | exec(user_scraping.read(), globals(), user_locals)
77 | user_scraping.close()
78 | return user_locals.get("PATTERNS", {})
79 |
80 |
81 | def get_db(force_reconnect=False):
82 | """
83 | Returns the database object that you can use to
84 | store data persistently
85 | """
86 | if "db" not in g or force_reconnect:
87 | g.db = TinyDB(str(Path(current_app.config["INTERNAL_DIR"]) / "db.json"))
88 |
89 | return g.db
90 |
91 |
92 | def get_max_id():
93 | """Returns the current maximum id of dataobjs in the database."""
94 | db = get_db()
95 | max_id = db.search(Query().name == "max_id")
96 | if not max_id:
97 | db.insert({"name": "max_id", "val": 0})
98 | return 0
99 | return max_id[0]["val"]
100 |
101 |
102 | def set_max_id(val):
103 | """Sets a new max_id"""
104 | db = get_db()
105 | db.update(operations.set("val", val), Query().name == "max_id")
106 |
107 |
108 | def test_es_connection(es):
109 | """Tests health and presence of connection to elasticsearch."""
110 | try:
111 | health = es.cluster.health()
112 | except elasticsearch.exceptions.ConnectionError:
113 | current_app.logger.error(
114 | "Elasticsearch does not seem to be running on "
115 | f"{current_app.config['SEARCH_CONF']['url']}. Please start "
116 | "it, for example with: sudo service elasticsearch restart"
117 | )
118 | current_app.logger.error(
119 | "You can disable Elasticsearch by modifying the `enabled` variable "
120 | f"in {str(Path(current_app.config['INTERNAL_DIR']) / 'config.yml')}"
121 | )
122 | sys.exit(1)
123 |
124 | if health["status"] not in ("yellow", "green"):
125 | current_app.logger.warning(
126 | "Elasticsearch reports that it is not working "
127 | "properly. Search might not work. You can disable "
128 | "Elasticsearch by setting ELASTICSEARCH_ENABLED to 0."
129 | )
130 |
131 |
132 | def get_elastic_client(error_if_invalid=True):
133 | """Returns the elasticsearch client you can use to search and insert / delete data"""
134 | if (
135 | not current_app.config["SEARCH_CONF"]["enabled"]
136 | or current_app.config["SEARCH_CONF"]["engine"] != "elasticsearch"
137 | ) and error_if_invalid:
138 | return None
139 |
140 | auth_setup = (
141 | current_app.config["SEARCH_CONF"]["es_user"]
142 | and current_app.config["SEARCH_CONF"]["es_password"]
143 | )
144 | if auth_setup:
145 | es = Elasticsearch(
146 | current_app.config["SEARCH_CONF"]["url"],
147 | http_auth=(
148 | current_app.config["SEARCH_CONF"]["es_user"],
149 | current_app.config["SEARCH_CONF"]["es_password"],
150 | ),
151 | )
152 | else:
153 | es = Elasticsearch(current_app.config["SEARCH_CONF"]["url"])
154 | if error_if_invalid:
155 | test_es_connection(es)
156 | else:
157 | try:
158 | es.cluster.health()
159 | except elasticsearch.exceptions.ConnectionError:
160 | return False
161 | return es
162 |
163 |
164 | def create_plugin_dir(name):
165 | """Creates a sample plugin directory"""
166 | raw_name = name.replace("archivy_", "").replace("archivy-", "")
167 | try:
168 | os.makedirs(f"{name}/{name}")
169 |
170 | # Creates requirements.txt.
171 | with open(f"{name}/requirements.txt", "w") as fp:
172 | fp.writelines(["archivy", "\nclick"])
173 |
174 | # Creates an empty readme file to be filled
175 | with open(f"{name}/README.md", "w+") as fp:
176 | fp.writelines(
177 | [
178 | f"# {name}",
179 | "\n\n## Install",
180 | "\n\nYou need to have `archivy` already installed.",
181 | f"\n\nRun `pip install archivy_{name}`",
182 | "\n\n## Usage",
183 | ]
184 | )
185 |
186 | # Creates a setup.py file
187 | with open(f"{name}/setup.py", "w") as setup_f:
188 | setup_f.writelines(
189 | [
190 | "from setuptools import setup, find_packages",
191 | '\n\nwith open("README.md", "r") as fh:',
192 | "\n\tlong_description = fh.read()",
193 | '\n\nwith open("requirements.txt", encoding="utf-8") as f:',
194 | '\n\tall_reqs = f.read().split("\\n")',
195 | "\n\tinstall_requires = [x.strip() for x in all_reqs]",
196 | "\n\n#Fill in the details below for distribution purposes"
197 | f'\nsetup(\n\tname="{name}",',
198 | '\n\tversion="0.0.1",',
199 | '\n\tauthor="",',
200 | '\n\tauthor_email="",',
201 | '\n\tdescription="",',
202 | "\n\tlong_description=long_description,",
203 | '\n\tlong_description_content_type="text/markdown",',
204 | '\n\tclassifiers=["Programming Language :: Python :: 3"],'
205 | "\n\tpackages=find_packages(),",
206 | "\n\tinstall_requires=install_requires,",
207 | f'\n\tentry_points="""\n\t\t[archivy.plugins]'
208 | f'\n\t\t{raw_name}={name}:{raw_name}"""\n)',
209 | ]
210 | )
211 |
212 | # Creating a basic __init__.py file where the main function of the plugin goes
213 | with open(f"{name}/{name}/__init__.py", "w") as fp:
214 | fp.writelines(
215 | [
216 | "import archivy",
217 | "\nimport click",
218 | "\n\n# Fill in the functionality for the commands (see https://archivy.github.io/plugins/)",
219 | "\n@click.group()",
220 | f"\ndef {raw_name}():",
221 | "\n\tpass",
222 | f"\n\n@{raw_name}.command()",
223 | "\ndef command1():",
224 | "\n\tpass",
225 | f"\n\n@{raw_name}.command()",
226 | "\ndef command2():",
227 | "\n\tpass",
228 | ]
229 | )
230 |
231 | return True
232 | except FileExistsError:
233 | return False
234 |
235 |
236 | def is_safe_redirect_url(target):
237 | host_url = urlparse(request.host_url)
238 | redirect_url = urlparse(urljoin(request.host_url, target))
239 | return (
240 | redirect_url.scheme in ("http", "https")
241 | and host_url.netloc == redirect_url.netloc
242 | )
243 |
--------------------------------------------------------------------------------
/archivy/search.py:
--------------------------------------------------------------------------------
1 | from pathlib import Path
2 | from shutil import which
3 | from subprocess import run, PIPE
4 | import json
5 |
6 | from flask import current_app
7 |
8 | from archivy.helpers import get_elastic_client
9 |
10 |
11 | # Example command ["rg", RG_MISC_ARGS, RG_FILETYPE, RG_REGEX_ARG, query, str(get_data_dir())]
12 | # rg -il -t md -e query files
13 | # -i -> case insensitive
14 | # -l -> only output filenames
15 | # -t -> file type
16 | # -e -> regexp
17 | RG_MISC_ARGS = "-it"
18 | RG_REGEX_ARG = "-e"
19 | RG_FILETYPE = "md"
20 |
21 |
22 | def add_to_index(model):
23 | """
24 | Adds dataobj to given index. If object of given id already exists, it will be updated.
25 |
26 | Params:
27 |
28 | - **index** - String of the ES Index. Archivy uses `dataobj` by default.
29 | - **model** - Instance of `archivy.models.Dataobj`, the object you want to index.
30 | """
31 | es = get_elastic_client()
32 | if not es:
33 | return
34 | payload = {}
35 | for field in model.__searchable__:
36 | payload[field] = getattr(model, field)
37 | es.index(
38 | index=current_app.config["SEARCH_CONF"]["index_name"], id=model.id, body=payload
39 | )
40 | return True
41 |
42 |
43 | def remove_from_index(dataobj_id):
44 | """Removes object of given id"""
45 | es = get_elastic_client()
46 | if not es:
47 | return
48 | es.delete(index=current_app.config["SEARCH_CONF"]["index_name"], id=dataobj_id)
49 |
50 |
51 | def query_es_index(query, strict=False):
52 | """
53 | Returns search results for your given query
54 |
55 | Specify strict=True if you want only exact result (in case you're using ES.
56 | """
57 | es = get_elastic_client()
58 | if not es:
59 | return []
60 | search = es.search(
61 | index=current_app.config["SEARCH_CONF"]["index_name"],
62 | body={
63 | "query": {
64 | "multi_match": {
65 | "query": query,
66 | "fields": ["*"],
67 | "analyzer": "rebuilt_standard",
68 | }
69 | },
70 | "highlight": {
71 | "fragment_size": 0,
72 | "fields": {
73 | "content": {
74 | "pre_tags": "",
75 | "post_tags": "",
76 | }
77 | },
78 | },
79 | },
80 | )
81 |
82 | hits = []
83 | for hit in search["hits"]["hits"]:
84 | formatted_hit = {"id": hit["_id"], "title": hit["_source"]["title"]}
85 | if "highlight" in hit:
86 | formatted_hit["matches"] = hit["highlight"]["content"]
87 | reformatted_match = " ".join(formatted_hit["matches"])
88 | if strict and not (query in reformatted_match):
89 | continue
90 | hits.append(formatted_hit)
91 | return hits
92 |
93 |
94 | def parse_ripgrep_line(line):
95 | """Parses a line of ripgrep JSON output"""
96 | hit = json.loads(line)
97 | data = {}
98 | if hit["type"] == "begin":
99 | curr_file = (
100 | Path(hit["data"]["path"]["text"]).parts[-1].replace(".md", "").split("-")
101 | ) # parse target note data from path
102 | curr_id = int(curr_file[0])
103 | title = curr_file[-1].replace("_", " ")
104 | data = {"title": title, "matches": [], "id": curr_id}
105 | elif hit["type"] == "match":
106 | data = hit["data"]["lines"]["text"].strip()
107 | else:
108 | return None # only process begin and match events, we don't care about endings
109 | return (data, hit["type"])
110 |
111 |
112 | def query_ripgrep(query):
113 | """
114 | Uses ripgrep to search data with a simpler setup than ES.
115 | Returns a list of dicts with detailed matches.
116 | """
117 |
118 | from archivy.data import get_data_dir
119 |
120 | if not which("rg"):
121 | return []
122 |
123 | rg_cmd = ["rg", RG_MISC_ARGS, RG_FILETYPE, "--json", query, str(get_data_dir())]
124 | rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=60)
125 | output = rg.stdout.decode().splitlines()
126 | hits = []
127 | for line in output:
128 | parsed = parse_ripgrep_line(line)
129 | if not parsed:
130 | continue
131 | if parsed[1] == "begin":
132 | hits.append(parsed[0])
133 | if parsed[1] == "match":
134 | if not (parsed[0].startswith("tags: [") or parsed[0].startswith("title:")):
135 | hits[-1]["matches"].append(parsed[0])
136 | return sorted(
137 | hits, key=lambda x: len(x["matches"]), reverse=True
138 | ) # sort by number of matches
139 |
140 |
141 | def search_frontmatter_tags(tag=None):
142 | """
143 | Returns a list of dataobj ids that have the given tag.
144 | """
145 | from archivy.data import get_data_dir
146 |
147 | if not which("rg"):
148 | return []
149 | META_PATTERN = r"(^|\n)tags:(\n- [_a-zA-ZÀ-ÖØ-öø-ÿ0-9]+)+"
150 | hits = []
151 | rg_cmd = [
152 | "rg",
153 | "-Uo",
154 | RG_MISC_ARGS,
155 | RG_FILETYPE,
156 | "--json",
157 | RG_REGEX_ARG,
158 | META_PATTERN,
159 | str(get_data_dir()),
160 | ]
161 | rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=60)
162 | output = rg.stdout.decode().splitlines()
163 | for line in output:
164 | parsed = parse_ripgrep_line(line)
165 | if not parsed: # the event doesn't interest us
166 | continue
167 | if parsed[1] == "begin":
168 | hits.append(parsed[0]) # append current hit data
169 | if parsed[1] == "match":
170 | sanitized = parsed[0].replace("- ", "").split("\n")[2:]
171 | hits[-1]["tags"] = hits[-1].get("tags", []) + sanitized # get tags
172 | if tag:
173 | hits = list(filter(lambda x: tag in x["tags"], hits))
174 | return hits
175 |
176 |
177 | def query_ripgrep_tags():
178 | """
179 | Uses ripgrep to search for tags.
180 | Mandatory reference: https://xkcd.com/1171/
181 | """
182 |
183 | EMB_PATTERN = r"(^|\n| )#([-_a-zA-ZÀ-ÖØ-öø-ÿ0-9]+)#"
184 | from archivy.data import get_data_dir
185 |
186 | if not which("rg"):
187 | return []
188 |
189 | # embedded tags
190 | # io: case insensitive
191 | rg_cmd = ["rg", "-Uio", RG_FILETYPE, RG_REGEX_ARG, EMB_PATTERN, str(get_data_dir())]
192 | rg = run(rg_cmd, stdout=PIPE, stderr=PIPE, timeout=60)
193 | hits = set()
194 | for line in rg.stdout.splitlines():
195 | tag = Path(line.decode()).parts[-1].split(":")[-1]
196 | tag = tag.replace("#", "").lstrip()
197 | hits.add(tag)
198 | # metadata tags
199 | for item in search_frontmatter_tags():
200 | for tag in item["tags"]:
201 | hits.add(tag)
202 | return hits
203 |
204 |
205 | def search(query, strict=False):
206 | """
207 | Wrapper to search methods for different engines.
208 |
209 | If using ES, specify strict=True if you only want results that strictly match the query, without parsing / tokenization.
210 | """
211 | if current_app.config["SEARCH_CONF"]["engine"] == "elasticsearch":
212 | return query_es_index(query, strict=strict)
213 | elif current_app.config["SEARCH_CONF"]["engine"] == "ripgrep" or which("rg"):
214 | return query_ripgrep(query)
215 |
--------------------------------------------------------------------------------
/archivy/static/HIGHLIGHTJS-LICENSE:
--------------------------------------------------------------------------------
1 | BSD 3-Clause License
2 |
3 | Copyright (c) 2006, Ivan Sagalaev.
4 | All rights reserved.
5 |
6 | Redistribution and use in source and binary forms, with or without
7 | modification, are permitted provided that the following conditions are met:
8 |
9 | * Redistributions of source code must retain the above copyright notice, this
10 | list of conditions and the following disclaimer.
11 |
12 | * Redistributions in binary form must reproduce the above copyright notice,
13 | this list of conditions and the following disclaimer in the documentation
14 | and/or other materials provided with the distribution.
15 |
16 | * Neither the name of the copyright holder nor the names of its
17 | contributors may be used to endorse or promote products derived from
18 | this software without specific prior written permission.
19 |
20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30 |
--------------------------------------------------------------------------------
/archivy/static/accessibility.css:
--------------------------------------------------------------------------------
1 | .sr-only {
2 | position: absolute;
3 | left: -10000px;
4 | top: auto;
5 | width: 1px;
6 | height: 1px;
7 | overflow: hidden;
8 | }
9 |
--------------------------------------------------------------------------------
/archivy/static/archivy.svg:
--------------------------------------------------------------------------------
1 | <svg id="svg" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="400" height="400" viewBox="0, 0, 400,400"><g id="svgg"><path id="path0" d="M184.820 33.521 L 169.625 48.704 215.670 94.815 C 259.477 138.685,263.490 142.764,266.111 146.084 C 266.722 146.858,267.472 147.793,267.778 148.161 C 272.630 154.015,278.372 165.138,281.442 174.630 C 286.675 190.810,286.300 212.739,280.526 228.057 C 280.009 229.431,279.429 231.056,279.239 231.667 C 278.098 235.337,271.460 247.218,268.514 250.861 C 268.321 251.100,267.118 252.584,265.841 254.159 C 264.563 255.733,255.978 264.566,246.761 273.788 L 230.003 290.556 245.194 305.741 L 260.384 320.926 279.005 302.222 C 298.049 283.094,300.833 279.981,306.663 271.296 C 310.824 265.100,314.859 257.751,317.465 251.626 C 317.802 250.834,318.427 249.395,318.854 248.430 C 319.280 247.465,319.630 246.456,319.630 246.189 C 319.630 245.922,319.773 245.545,319.949 245.352 C 320.459 244.790,323.025 236.740,324.281 231.759 C 325.159 228.279,325.892 224.920,326.119 223.341 C 326.250 222.429,326.408 221.595,326.471 221.489 C 326.533 221.383,326.692 220.357,326.824 219.209 C 326.955 218.061,327.140 216.728,327.234 216.246 C 328.604 209.211,328.162 187.092,326.477 178.333 C 324.648 168.828,323.773 165.418,321.327 158.277 C 316.129 143.102,308.494 129.636,297.841 116.852 C 296.313 115.019,273.677 92.103,247.539 65.928 L 200.015 18.337 184.820 33.521 M114.353 104.082 C 104.297 114.233,103.266 115.315,101.006 118.089 C 100.351 118.893,99.315 120.164,98.704 120.913 C 96.953 123.058,91.582 130.791,90.849 132.222 C 90.484 132.935,90.072 133.604,89.935 133.709 C 89.405 134.111,84.366 143.693,82.734 147.402 C 62.859 192.561,69.860 243.830,101.102 281.914 C 104.420 285.959,138.750 320.741,139.424 320.741 C 139.915 320.741,169.630 291.057,169.630 290.566 C 169.630 290.370,161.740 282.287,152.096 272.605 C 132.851 253.281,130.401 250.421,125.230 241.231 C 119.940 231.831,116.191 219.923,114.992 208.704 C 113.972 199.171,114.944 187.322,117.544 177.593 C 119.171 171.505,125.315 156.991,126.240 157.053 C 126.373 157.062,133.359 163.854,141.765 172.146 L 157.048 187.222 156.474 189.444 C 155.650 192.633,155.460 201.591,156.131 205.600 C 160.025 228.853,183.508 245.337,205.926 240.554 C 206.435 240.446,207.435 240.260,208.148 240.141 C 208.861 240.023,209.736 239.770,210.093 239.579 C 210.449 239.389,210.741 239.315,210.741 239.415 C 210.741 240.224,219.553 235.954,223.333 233.312 C 253.396 212.305,243.000 162.729,207.222 156.481 C 206.407 156.339,205.324 156.139,204.815 156.038 C 201.157 155.311,192.223 155.726,188.148 156.813 C 187.800 156.905,175.808 145.191,156.020 125.427 L 124.448 93.894 114.353 104.082 M202.603 188.773 C 210.617 192.319,210.976 203.825,203.197 207.839 C 194.647 212.251,184.916 203.215,188.773 194.444 C 191.222 188.874,197.246 186.404,202.603 188.773 M184.726 335.837 L 169.444 351.107 184.631 366.295 L 199.817 381.483 215.094 366.203 L 230.370 350.923 215.189 335.745 L 200.008 320.567 184.726 335.837 " stroke="none" fill="#5cbc74" fill-rule="evenodd"></path><path id="path1" d="M179.259 58.704 C 184.655 64.102,189.152 68.519,189.254 68.519 C 189.356 68.519,185.025 64.102,179.630 58.704 C 174.234 53.306,169.737 48.889,169.635 48.889 C 169.533 48.889,173.864 53.306,179.259 58.704 M225.370 104.815 C 245.129 124.574,261.378 140.741,261.480 140.741 C 261.582 140.741,245.499 124.574,225.741 104.815 C 205.982 85.056,189.733 68.889,189.631 68.889 C 189.529 68.889,205.612 85.056,225.370 104.815 M149.634 180.185 C 153.604 184.259,156.928 187.544,157.021 187.485 C 157.114 187.426,153.866 184.093,149.803 180.078 L 142.417 172.778 149.634 180.185 M284.926 194.444 C 284.926 195.565,284.995 196.023,285.080 195.463 C 285.165 194.903,285.165 193.986,285.080 193.426 C 284.995 192.866,284.926 193.324,284.926 194.444 M155.296 198.519 C 155.296 199.639,155.366 200.097,155.450 199.537 C 155.535 198.977,155.535 198.060,155.450 197.500 C 155.366 196.940,155.296 197.398,155.296 198.519 M284.926 205.185 C 284.926 206.306,284.995 206.764,285.080 206.204 C 285.165 205.644,285.165 204.727,285.080 204.167 C 284.995 203.606,284.926 204.065,284.926 205.185 M284.530 209.630 C 284.530 210.343,284.607 210.634,284.700 210.278 C 284.793 209.921,284.793 209.338,284.700 208.981 C 284.607 208.625,284.530 208.917,284.530 209.630 M166.667 227.523 C 166.667 227.587,167.208 228.129,167.870 228.727 L 169.074 229.815 167.986 228.611 C 166.972 227.489,166.667 227.237,166.667 227.523 M244.815 305.741 C 253.063 313.991,259.895 320.741,259.997 320.741 C 260.098 320.741,253.433 313.991,245.185 305.741 C 236.937 297.491,230.105 290.741,230.003 290.741 C 229.902 290.741,236.567 297.491,244.815 305.741 " stroke="none" fill="#5cbc7c" fill-rule="evenodd"></path><path id="path2" d="M275.924 305.463 L 260.185 321.296 276.019 305.557 C 284.727 296.901,291.852 289.776,291.852 289.724 C 291.852 289.463,290.553 290.746,275.924 305.463 M214.998 366.389 L 199.815 381.667 215.093 366.483 C 223.495 358.132,230.370 351.257,230.370 351.205 C 230.370 350.945,229.111 352.189,214.998 366.389 " stroke="none" fill="#60bc74" fill-rule="evenodd"></path><path id="path3" d="M275.924 305.463 L 260.185 321.296 276.019 305.557 C 284.727 296.901,291.852 289.776,291.852 289.724 C 291.852 289.463,290.553 290.746,275.924 305.463 M214.998 366.389 L 199.815 381.667 215.093 366.483 C 223.495 358.132,230.370 351.257,230.370 351.205 C 230.370 350.945,229.111 352.189,214.998 366.389 " stroke="none" fill="#60bc74" fill-rule="evenodd"></path><path id="path4" d="M275.924 305.463 L 260.185 321.296 276.019 305.557 C 284.727 296.901,291.852 289.776,291.852 289.724 C 291.852 289.463,290.553 290.746,275.924 305.463 M214.998 366.389 L 199.815 381.667 215.093 366.483 C 223.495 358.132,230.370 351.257,230.370 351.205 C 230.370 350.945,229.111 352.189,214.998 366.389 " stroke="none" fill="#60bc74" fill-rule="evenodd"></path></g></svg>
--------------------------------------------------------------------------------
/archivy/static/delete.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/archivy/static/delete.png
--------------------------------------------------------------------------------
/archivy/static/editor_dark.css:
--------------------------------------------------------------------------------
1 | .EasyMDEContainer .CodeMirror {
2 | color: var(--mid-light-grey);
3 | background: var(--dark-grey);
4 | border: 1px solid var(--mid-grey);
5 | }
6 |
7 | .EasyMDEContainer .CodeMirror-fullscreen {
8 | background: var(--dark-grey);
9 | }
10 |
11 | .EasyMDEContainer .CodeMirror-focused .CodeMirror-selected, .CodeMirror-selectedtext, .CodeMirror-line::selection, .CodeMirror-line > span::selection, .CodeMirror-line > span > span::selection {
12 | background: #3297fd;
13 | color: white;
14 | }
15 |
16 | .editor-toolbar {
17 | background-color: var(--mid-dark-grey);
18 | border-top: 1px solid var(--mid-grey);
19 | border-left: 1px solid var(--mid-grey);
20 | border-right: 1px solid var(--mid-grey);
21 | }
22 |
23 | .editor-toolbar button {
24 | color: var(--mid-light-grey);
25 | }
26 |
27 | .editor-toolbar.fullscreen {
28 | background: var(--dark-grey);
29 | }
30 |
31 | .editor-preview {
32 | background: var(--dark-grey);
33 | }
34 |
35 | .editor-preview-side {
36 | border: 1px solid var(--mid-grey);
37 | }
38 |
--------------------------------------------------------------------------------
/archivy/static/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/archivy/static/logo.png
--------------------------------------------------------------------------------
/archivy/static/main_dark.css:
--------------------------------------------------------------------------------
1 | :root {
2 | --light-green: #f6efa6;
3 | --mid-green: #48BB78;
4 | --rose: #ffe3e6;
5 | --mid-rose: #f97583;
6 | --dark-rose: #e5534b;
7 | --grape: #7848bb;
8 | --dark-grey: #22272e;
9 | --mid-dark-grey: #373e47;
10 | --mid-grey: #636e7b;
11 | --mid-light-grey: #909dab;
12 | --light-grey: #cdd9e5;
13 | --hyperlink-color: #539bf5;
14 | }
15 |
16 |
17 | body {
18 | color: var(--mid-light-grey);
19 | background: var(--dark-grey);
20 | }
21 |
22 |
23 | a {
24 | color: var(--hyperlink-color);
25 | }
26 |
27 | mark {
28 | background-color: var(--light-green);
29 | }
30 |
31 | /* default form stylings */
32 | input, select {
33 | color: var(--mid-light-grey);
34 | background-color: var(--mid-dark-grey);
35 | border-bottom: 3px var(--mid-grey) solid;
36 | }
37 |
38 | input:focus:not([type="submit"]), input:hover:not([type="submit"]), select:focus, select:hover {
39 | border-color: var(--mid-green);
40 | }
41 |
42 | .folder-cont {
43 | border-left: 2px var(--mid-green) solid;
44 | }
45 | .folder-cont h3 {
46 | color: var(--mid-light-grey);
47 | }
48 | .folder-cont a {
49 | color: var(--mid-light-grey);
50 | }
51 | .sidebar {
52 | border-right: 5px solid var(--mid-dark-grey);
53 | }
54 |
55 | #main-links .btn {
56 | color: var(--mid-light-grey);
57 | background-color: var(--mid-dark-grey);
58 | border-color: var(--mid-grey);
59 | }
60 |
61 | #main-links .btn:hover, .btn.hover, [open] > .btn {
62 | background-color: var(--mid-grey);
63 | transition-duration: 0.1s;
64 | }
65 |
66 | #current-path a {
67 | color: var(--hyperlink-color);
68 | }
69 | #files ul li {
70 | border-top: 1px var(--mid-grey) solid;
71 | }
72 |
73 | #files ul a {
74 | color: var(--mid-light-grey);
75 | }
76 | #post-btns button {
77 | color: var(--mid-light-grey);
78 | background-color: var(--mid-dark-grey);
79 | border-color: var(--mid-grey);
80 | }
81 |
82 | #post-btns .btn:hover, .btn.hover, [open] > .btn {
83 | background-color: var(--mid-grey);
84 | transition-duration: 0.1s;
85 | }
86 | .post-title-form-sucess {
87 | color: var(--mid-green);
88 | }
89 |
90 | .post-tags span {
91 | color: var(--mid-dark-grey);
92 | background: var(--light-green);
93 | }
94 |
95 | /*Partly inspired by Primer -> https://github.com/primer/css*/
96 | .Header {
97 | color: var(--mid-light-grey);
98 | background-color: var(--mid-dark-grey) !important;
99 | }
100 |
101 | .Header-link {
102 | color: var(--light-grey);
103 | }
104 | .flash {
105 | color: var(--mid-dark-grey);
106 | }
107 |
108 | .flash-error {
109 | background-color: var(--rose);
110 | border-color: var(--dark-rose);
111 | }
112 |
113 | .flash-success {
114 | color: var(--mid-light-grey);
115 | background-color: var(--mid-dark-grey);
116 | border-color: var(--mid-grey);
117 | }
118 |
119 |
120 | .btn {
121 | color: var(--mid-light-grey);
122 | background-color: var(--mid-dark-grey);
123 | border-color: var(--mid-grey);
124 | box-shadow: 0 1px 0 var(--dark-grey),inset 0 1px 0 var(--mid-dark-grey);
125 | }
126 |
127 | .btn:hover, .btn.hover, [open] > .btn {
128 | background-color: var(--mid-grey);
129 | }
130 |
131 | #link-form {
132 | border: 1px var(--light-grey) solid;
133 | background-color: var(--light-grey);
134 | }
135 |
--------------------------------------------------------------------------------
/archivy/static/markdown.css:
--------------------------------------------------------------------------------
1 | /* GitHub's markdown rendering stylesheet from https://github.com/primer/css */
2 |
3 | .markdown-body kbd {
4 | display: inline-block;
5 | padding: 3px 5px;
6 | font: 11px "SFMono-Regular",Consolas,"Liberation Mono",Menlo,monospace;
7 | line-height: 10px;
8 | color: #444d56;
9 | vertical-align: middle;
10 | background-color: #fafbfc;
11 | border: solid 1px #d1d5da;
12 | border-bottom-color: #d1d5da;
13 | border-radius: 6px;
14 | box-shadow: inset 0 -1px 0 #d1d5da;
15 | }
16 | .markdown-body::before {
17 | display: table;
18 | content: "";
19 | }
20 | .markdown-body::after {
21 | display: table;
22 | clear: both;
23 | content: "";
24 | }
25 | .markdown-body > *:first-child {
26 | margin-top: 0 !important;
27 | }
28 | .markdown-body > *:last-child {
29 | margin-bottom: 0 !important;
30 | }
31 | .markdown-body a:not([href]) {
32 | color: inherit;
33 | text-decoration: none;
34 | }
35 | .markdown-body .absent {
36 | color: #cb2431;
37 | }
38 | .markdown-body .anchor {
39 | float: left;
40 | padding-right: 4px;
41 | margin-left: -20px;
42 | line-height: 1;
43 | }
44 | .markdown-body .anchor:focus {
45 | outline: none;
46 | }
47 | blockquote
48 | .markdown-body details,
49 | .markdown-body dl,
50 | .markdown-body ol,
51 | .markdown-body p,
52 | .markdown-body pre,
53 | .markdown-body table,
54 | .markdown-body ul {
55 | margin-top: 0;
56 | margin-bottom: 16px;
57 | }
58 | .markdown-body h1,
59 | .markdown-body h2,
60 | .markdown-body h3,
61 | .markdown-body h4,
62 | .markdown-body h5,
63 | .markdown-body h6 {
64 | margin-top: 24px;
65 | margin-bottom: 16px;
66 | font-weight: 600;
67 | line-height: 1.25;
68 | }
69 | .markdown-body h1 .octicon-link,
70 | .markdown-body h2 .octicon-link,
71 | .markdown-body h3 .octicon-link,
72 | .markdown-body h4 .octicon-link,
73 | .markdown-body h5 .octicon-link,
74 | .markdown-body h6 .octicon-link {
75 | color: #1b1f23;
76 | vertical-align: middle;
77 | visibility: hidden;
78 | }
79 | .markdown-body h1:hover .anchor,
80 | .markdown-body h2:hover .anchor,
81 | .markdown-body h3:hover .anchor,
82 | .markdown-body h4:hover .anchor,
83 | .markdown-body h5:hover .anchor,
84 | .markdown-body h6:hover .anchor {
85 | text-decoration: none;
86 | }
87 | .markdown-body h1:hover .anchor .octicon-link,
88 | .markdown-body h2:hover .anchor .octicon-link,
89 | .markdown-body h3:hover .anchor .octicon-link,
90 | .markdown-body h4:hover .anchor .octicon-link,
91 | .markdown-body h5:hover .anchor .octicon-link,
92 | .markdown-body h6:hover .anchor .octicon-link {
93 | visibility: visible;
94 | }
95 | .markdown-body h1 code,
96 | .markdown-body h1 tt,
97 | .markdown-body h2 code,
98 | .markdown-body h2 tt,
99 | .markdown-body h3 code,
100 | .markdown-body h3 tt,
101 | .markdown-body h4 code,
102 | .markdown-body h4 tt,
103 | .markdown-body h5 code,
104 | .markdown-body h5 tt,
105 | .markdown-body h6 code,
106 | .markdown-body h6 tt {
107 | font-size: inherit;
108 | }
109 | .markdown-body h1 {
110 | padding-bottom: 0.3em;
111 | font-size: 2em;
112 | border-bottom: 1px solid #eaecef;
113 | }
114 | .markdown-body h2 {
115 | padding-bottom: 0.3em;
116 | font-size: 1.5em;
117 | border-bottom: 1px solid #eaecef;
118 | }
119 | .markdown-body h3 {
120 | font-size: 1.25em;
121 | }
122 | .markdown-body h4 {
123 | font-size: 1em;
124 | }
125 | .markdown-body h5 {
126 | font-size: 0.875em;
127 | }
128 | .markdown-body h6 {
129 | font-size: 0.85em;
130 | color: #6a737d;
131 | }
132 | .markdown-body ol,
133 | .markdown-body ul {
134 | padding-left: 2em;
135 | }
136 | .markdown-body ol.no-list,
137 | .markdown-body ul.no-list {
138 | padding: 0;
139 | list-style-type: none;
140 | }
141 | .markdown-body ol ol,
142 | .markdown-body ol ul,
143 | .markdown-body ul ol,
144 | .markdown-body ul ul {
145 | margin-top: 0;
146 | margin-bottom: 0;
147 | }
148 | .markdown-body li {
149 | word-wrap: break-all;
150 | }
151 | .markdown-body li > p {
152 | margin-top: 16px;
153 | }
154 | .markdown-body li+li {
155 | margin-top: 0.25em;
156 | }
157 | .markdown-body dl {
158 | padding: 0;
159 | }
160 | .markdown-body dl dt {
161 | padding: 0;
162 | margin-top: 16px;
163 | font-size: 1em;
164 | font-style: italic;
165 | font-weight: 600;
166 | }
167 | .markdown-body dl dd {
168 | padding: 0 16px;
169 | margin-bottom: 16px;
170 | }
171 | .markdown-body table {
172 | display: block;
173 | width: 100%;
174 | width: -webkit-max-content;
175 | width: -moz-max-content;
176 | width: max-content;
177 | max-width: 100%;
178 | overflow: auto;
179 | }
180 | .markdown-body table th {
181 | font-weight: 600;
182 | }
183 | .markdown-body table td,
184 | .markdown-body table th {
185 | padding: 6px 13px;
186 | border: 1px solid #dfe2e5;
187 | }
188 | .markdown-body table tr {
189 | background-color: #fff;
190 | border-top: 1px solid #c6cbd1;
191 | }
192 | .markdown-body table tr:nth-child(2n) {
193 | background-color: #f6f8fa;
194 | }
195 | .markdown-body table img {
196 | background-color: transparent;
197 | }
198 | .markdown-body img {
199 | max-width: 100%;
200 | box-sizing: content-box;
201 | background-color: #fff;
202 | }
203 | .markdown-body img[align=right] {
204 | padding-left: 20px;
205 | }
206 | .markdown-body img[align=left] {
207 | padding-right: 20px;
208 | }
209 | .markdown-body .emoji {
210 | max-width: none;
211 | vertical-align: text-top;
212 | background-color: transparent;
213 | }
214 | .markdown-body span.frame {
215 | display: block;
216 | overflow: hidden;
217 | }
218 | .markdown-body span.frame > span {
219 | display: block;
220 | float: left;
221 | width: auto;
222 | padding: 7px;
223 | margin: 13px 0 0;
224 | overflow: hidden;
225 | border: 1px solid #dfe2e5;
226 | }
227 | .markdown-body span.frame span img {
228 | display: block;
229 | float: left;
230 | }
231 | .markdown-body span.frame span span {
232 | display: block;
233 | padding: 5px 0 0;
234 | clear: both;
235 | color: #24292e;
236 | }
237 | .markdown-body span.align-center {
238 | display: block;
239 | overflow: hidden;
240 | clear: both;
241 | }
242 | .markdown-body span.align-center > span {
243 | display: block;
244 | margin: 13px auto 0;
245 | overflow: hidden;
246 | text-align: center;
247 | }
248 | .markdown-body span.align-center span img {
249 | margin: 0 auto;
250 | text-align: center;
251 | }
252 | .markdown-body span.align-right {
253 | display: block;
254 | overflow: hidden;
255 | clear: both;
256 | }
257 | .markdown-body span.align-right > span {
258 | display: block;
259 | margin: 13px 0 0;
260 | overflow: hidden;
261 | text-align: right;
262 | }
263 | .markdown-body span.align-right span img {
264 | margin: 0;
265 | text-align: right;
266 | }
267 | .markdown-body span.float-left {
268 | display: block;
269 | float: left;
270 | margin-right: 13px;
271 | overflow: hidden;
272 | }
273 | .markdown-body span.float-left span {
274 | margin: 13px 0 0;
275 | }
276 | .markdown-body span.float-right {
277 | display: block;
278 | float: right;
279 | margin-left: 13px;
280 | overflow: hidden;
281 | }
282 | .markdown-body span.float-right > span {
283 | display: block;
284 | margin: 13px auto 0;
285 | overflow: hidden;
286 | text-align: right;
287 | }
288 | .markdown-body code,
289 | .markdown-body tt {
290 | padding: 0.2em 0.4em;
291 | margin: 0;
292 | font-size: 85%;
293 | background-color: rgba(27,31,35,0.05);
294 | border-radius: 6px;
295 | }
296 | .markdown-body code br,
297 | .markdown-body tt br {
298 | display: none;
299 | }
300 | .markdown-body del code {
301 | text-decoration: inherit;
302 | }
303 | .markdown-body pre {
304 | word-wrap: normal;
305 | }
306 | .markdown-body pre > code {
307 | padding: 0;
308 | margin: 0;
309 | font-size: 100%;
310 | word-break: normal;
311 | white-space: pre;
312 | border: 0;
313 | background: transparent;
314 | }
315 | .markdown-body .highlight {
316 | margin-bottom: 16px;
317 | }
318 | .markdown-body .highlight pre {
319 | margin-bottom: 0;
320 | word-break: normal;
321 | }
322 | .markdown-body .highlight pre,
323 | .markdown-body pre {
324 | padding: 16px;
325 | overflow: auto;
326 | font-size: 85%;
327 | line-height: 1.45;
328 | border-radius: 6px;
329 | }
330 | .markdown-body pre code,
331 | .markdown-body pre tt {
332 | display: inline;
333 | max-width: auto;
334 | padding: 0;
335 | margin: 0;
336 | overflow: visible;
337 | line-height: inherit;
338 | word-wrap: normal;
339 | background-color: transparent;
340 | border: 0;
341 | }
342 | .markdown-body .csv-data td,
343 | .markdown-body .csv-data th {
344 | padding: 5px;
345 | overflow: hidden;
346 | font-size: 12px;
347 | line-height: 1;
348 | text-align: left;
349 | white-space: nowrap;
350 | }
351 | .markdown-body .csv-data .blob-num {
352 | padding: 10px 8px 9px;
353 | text-align: right;
354 | background: #fff;
355 | border: 0;
356 | }
357 | .markdown-body .csv-data tr {
358 | border-top: 0;
359 | }
360 | .markdown-body .csv-data th {
361 | font-weight: 600;
362 | background: #f6f8fa;
363 | border-top: 0;
364 | }
365 |
--------------------------------------------------------------------------------
/archivy/static/markdown_dark.css:
--------------------------------------------------------------------------------
1 | .markdown-body h1 {
2 | padding-bottom: 0.3em;
3 | font-size: 2em;
4 | border-bottom: 1px solid var(--mid-grey);
5 | }
6 |
--------------------------------------------------------------------------------
/archivy/static/monokai.css:
--------------------------------------------------------------------------------
1 | /*
2 |
3 | Monokai Sublime style. Derived from Monokai by noformnocontent http://nn.mit-license.org/
4 |
5 | */
6 |
7 | .hljs {
8 | display: block;
9 | overflow-x: auto;
10 | padding: 0.5em;
11 | background: #23241f;
12 | }
13 |
14 | .hljs,
15 | .hljs-tag,
16 | .hljs-subst {
17 | color: #f8f8f2;
18 | }
19 |
20 | .hljs-strong,
21 | .hljs-emphasis {
22 | color: #a8a8a2;
23 | }
24 |
25 | .hljs-bullet,
26 | .hljs-quote,
27 | .hljs-number,
28 | .hljs-regexp,
29 | .hljs-literal,
30 | .hljs-link {
31 | color: #ae81ff;
32 | }
33 |
34 | .hljs-code,
35 | .hljs-title,
36 | .hljs-section,
37 | .hljs-selector-class {
38 | color: #a6e22e;
39 | }
40 |
41 | .hljs-strong {
42 | font-weight: bold;
43 | }
44 |
45 | .hljs-emphasis {
46 | font-style: italic;
47 | }
48 |
49 | .hljs-keyword,
50 | .hljs-selector-tag,
51 | .hljs-name,
52 | .hljs-attr {
53 | color: #f92672;
54 | }
55 |
56 | .hljs-symbol,
57 | .hljs-attribute {
58 | color: #66d9ef;
59 | }
60 |
61 | .hljs-params,
62 | .hljs-class .hljs-title {
63 | color: #f8f8f2;
64 | }
65 |
66 | .hljs-string,
67 | .hljs-type,
68 | .hljs-built_in,
69 | .hljs-builtin-name,
70 | .hljs-selector-id,
71 | .hljs-selector-attr,
72 | .hljs-selector-pseudo,
73 | .hljs-addition,
74 | .hljs-variable,
75 | .hljs-template-variable {
76 | color: #e6db74;
77 | }
78 |
79 | .hljs-comment,
80 | .hljs-deletion,
81 | .hljs-meta {
82 | color: #75715e;
83 | }
84 |
--------------------------------------------------------------------------------
/archivy/static/mvp.css:
--------------------------------------------------------------------------------
1 | /* MVP.css v1.6.2 - https://github.com/andybrewer/mvp */
2 |
3 | :root {
4 | --border-radius: 5px;
5 | --box-shadow: 2px 2px 10px;
6 | --color: #118bee;
7 | --color-accent: #118bee15;
8 | --color-bg: #fff;
9 | --color-bg-secondary: #e9e9e9;
10 | --color-secondary: #920de9;
11 | --color-secondary-accent: #920de90b;
12 | --color-shadow: #f4f4f4;
13 | --color-text: #000;
14 | --color-text-secondary: #999;
15 | --font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen-Sans, Ubuntu, Cantarell, "Helvetica Neue", sans-serif;
16 | --hover-brightness: 1.2;
17 | --justify-important: center;
18 | --justify-normal: left;
19 | --line-height: 1.5;
20 | --width-card: 285px;
21 | --width-card-medium: 460px;
22 | --width-card-wide: 800px;
23 | --width-content: 1080px;
24 | }
25 |
26 | /*
27 | @media (prefers-color-scheme: dark) {
28 | :root {
29 | --color: #0097fc;
30 | --color-accent: #0097fc4f;
31 | --color-bg: #333;
32 | --color-bg-secondary: #555;
33 | --color-secondary: #e20de9;
34 | --color-secondary-accent: #e20de94f;
35 | --color-shadow: #bbbbbb20;
36 | --color-text: #f7f7f7;
37 | --color-text-secondary: #aaa;
38 | }
39 | }
40 | */
41 |
42 | /* Layout */
43 | article aside {
44 | background: var(--color-secondary-accent);
45 | border-left: 4px solid var(--color-secondary);
46 | padding: 0.01rem 0.8rem;
47 | }
48 |
49 | body {
50 | background: var(--color-bg);
51 | color: var(--color-text);
52 | font-family: var(--font-family);
53 | line-height: var(--line-height);
54 | margin: 0;
55 | overflow-x: hidden;
56 | padding: 1rem 0;
57 | }
58 |
59 | footer,
60 | header,
61 | main {
62 | margin: 0 auto;
63 | max-width: var(--width-content);
64 | padding: 2rem 1rem;
65 | }
66 |
67 | hr {
68 | background-color: var(--color-bg-secondary);
69 | border: none;
70 | height: 1px;
71 | margin: 4rem 0;
72 | }
73 |
74 | section {
75 | display: flex;
76 | flex-wrap: wrap;
77 | justify-content: var(--justify-important);
78 | }
79 |
80 | section aside {
81 | border: 1px solid var(--color-bg-secondary);
82 | border-radius: var(--border-radius);
83 | box-shadow: var(--box-shadow) var(--color-shadow);
84 | margin: 1rem;
85 | padding: 1.25rem;
86 | width: var(--width-card);
87 | }
88 |
89 | section aside:hover {
90 | box-shadow: var(--box-shadow) var(--color-bg-secondary);
91 | }
92 |
93 | section aside img {
94 | max-width: 100%;
95 | }
96 |
97 | [hidden] {
98 | display: none;
99 | }
100 |
101 | /* Headers */
102 | article header,
103 | div header,
104 | main header {
105 | padding-top: 0;
106 | }
107 |
108 | header {
109 | text-align: var(--justify-important);
110 | }
111 |
112 | header a b,
113 | header a em,
114 | header a i,
115 | header a strong {
116 | margin-left: 0.5rem;
117 | margin-right: 0.5rem;
118 | }
119 |
120 | header nav img {
121 | margin: 1rem 0;
122 | }
123 |
124 | section header {
125 | padding-top: 0;
126 | width: 100%;
127 | }
128 |
129 | /* Nav */
130 | nav {
131 | align-items: center;
132 | display: flex;
133 | font-weight: bold;
134 | justify-content: space-between;
135 | }
136 |
137 | nav ul {
138 | list-style: none;
139 | padding: 0;
140 | }
141 |
142 | nav ul li {
143 | display: inline-block;
144 | margin: 0 0.5rem;
145 | position: relative;
146 | text-align: left;
147 | }
148 |
149 | /* Nav Dropdown */
150 | nav ul li:hover ul {
151 | display: block;
152 | }
153 |
154 | nav ul li ul {
155 | background: var(--color-bg);
156 | border: 1px solid var(--color-bg-secondary);
157 | border-radius: var(--border-radius);
158 | box-shadow: var(--box-shadow) var(--color-shadow);
159 | display: none;
160 | height: auto;
161 | left: -2px;
162 | padding: .5rem 1rem;
163 | position: absolute;
164 | top: 1.7rem;
165 | white-space: nowrap;
166 | width: auto;
167 | }
168 |
169 | nav ul li ul li,
170 | nav ul li ul li a {
171 | display: block;
172 | }
173 |
174 | /* Typography */
175 | code,
176 | samp {
177 | background-color: var(--color-accent);
178 | border-radius: var(--border-radius);
179 | display: inline-block;
180 | margin: 0 0.1rem;
181 | padding: 0 0.5rem;
182 | }
183 |
184 | details {
185 | margin: 1.3rem 0;
186 | }
187 |
188 | details summary {
189 | font-weight: bold;
190 | cursor: pointer;
191 | }
192 |
193 | h1,
194 | h2,
195 | h3,
196 | h4,
197 | h5,
198 | h6 {
199 | line-height: var(--line-height);
200 | }
201 |
202 | mark {
203 | padding: 0.1rem;
204 | }
205 |
206 | ol li,
207 | ul li {
208 | padding: 0.2rem 0;
209 | }
210 |
211 | p {
212 | margin: 0.75rem 0;
213 | padding: 0;
214 | }
215 |
216 | pre {
217 | margin: 1rem 0;
218 | max-width: var(--width-card-wide);
219 | padding: 1rem 0;
220 | }
221 |
222 | pre code,
223 | pre samp {
224 | display: block;
225 | max-width: var(--width-card-wide);
226 | padding: 0.5rem 2rem;
227 | white-space: pre-wrap;
228 | }
229 |
230 | small {
231 | color: var(--color-text-secondary);
232 | }
233 |
234 | sup {
235 | border-radius: var(--border-radius);
236 | color: var(--color-bg);
237 | font-size: xx-small;
238 | font-weight: bold;
239 | margin: 0.2rem;
240 | padding: 0.2rem 0.3rem;
241 | position: relative;
242 | top: -2px;
243 | }
244 |
245 | /* Links */
246 | a {
247 | color: var(--color-secondary);
248 | display: inline-block;
249 | font-weight: bold;
250 | text-decoration: none;
251 | }
252 |
253 | a:hover {
254 | filter: brightness(var(--hover-brightness));
255 | text-decoration: underline;
256 | }
257 |
258 | button {
259 | border-radius: var(--border-radius);
260 | display: inline-block;
261 | font-size: medium;
262 | font-weight: bold;
263 | line-height: var(--line-height);
264 | margin: 0.5rem 0;
265 | padding: 1rem 2rem;
266 | }
267 |
268 | button {
269 | font-family: var(--font-family);
270 | }
271 |
272 | button:hover {
273 | cursor: pointer;
274 | filter: brightness(var(--hover-brightness));
275 | }
276 |
277 | /* Images */
278 | figure {
279 | margin: 0;
280 | padding: 0;
281 | }
282 |
283 | figure img {
284 | max-width: 100%;
285 | }
286 |
287 | figure figcaption {
288 | color: var(--color-text-secondary);
289 | }
290 |
291 | /* Forms */
292 |
293 | button:disabled,
294 | input:disabled {
295 | background: var(--color-bg-secondary);
296 | border-color: var(--color-bg-secondary);
297 | color: var(--color-text-secondary);
298 | cursor: not-allowed;
299 | }
300 |
301 | button[disabled]:hover {
302 | filter: none;
303 | }
304 |
305 | form {
306 | border: 1px solid var(--color-bg-secondary);
307 | border-radius: var(--border-radius);
308 | box-shadow: var(--box-shadow) var(--color-shadow);
309 | display: block;
310 | max-width: var(--width-card-wide);
311 | min-width: var(--width-card);
312 | padding: 1.5rem;
313 | text-align: var(--justify-normal);
314 | }
315 |
316 | form header {
317 | margin: 1.5rem 0;
318 | padding: 1.5rem 0;
319 | }
320 |
321 | input,
322 | label,
323 | select,
324 | textarea {
325 | display: block;
326 | font-size: inherit;
327 | max-width: var(--width-card-wide);
328 | }
329 |
330 | input[type="checkbox"],
331 | input[type="radio"] {
332 | display: inline-block;
333 | }
334 |
335 | input[type="checkbox"]+label,
336 | input[type="radio"]+label {
337 | display: inline-block;
338 | font-weight: normal;
339 | position: relative;
340 | top: 1px;
341 | }
342 |
343 | input,
344 | select,
345 | textarea {
346 | border: 1px solid var(--color-bg-secondary);
347 | border-radius: var(--border-radius);
348 | margin-bottom: 1rem;
349 | padding: 0.4rem 0.8rem;
350 | }
351 |
352 | input[readonly],
353 | textarea[readonly] {
354 | background-color: var(--color-bg-secondary);
355 | }
356 |
357 | label {
358 | font-weight: bold;
359 | margin-bottom: 0.2rem;
360 | }
361 |
362 | /* Tables */
363 | table {
364 | border: 1px solid var(--color-bg-secondary);
365 | border-radius: var(--border-radius);
366 | border-spacing: 0;
367 | display: inline-block;
368 | max-width: 100%;
369 | overflow-x: auto;
370 | padding: 0;
371 | white-space: nowrap;
372 | }
373 |
374 | table td,
375 | table th,
376 | table tr {
377 | padding: 0.4rem 0.8rem;
378 | text-align: var(--justify-important);
379 | }
380 |
381 | table thead {
382 | background-color: var(--color);
383 | border-collapse: collapse;
384 | border-radius: var(--border-radius);
385 | color: var(--color-bg);
386 | margin: 0;
387 | padding: 0;
388 | }
389 |
390 | table thead th:first-child {
391 | border-top-left-radius: var(--border-radius);
392 | }
393 |
394 | table thead th:last-child {
395 | border-top-right-radius: var(--border-radius);
396 | }
397 |
398 | table thead th:first-child,
399 | table tr td:first-child {
400 | text-align: var(--justify-normal);
401 | }
402 |
403 | table tr:nth-child(even) {
404 | background-color: var(--color-accent);
405 | }
406 |
407 | /* Quotes */
408 | blockquote {
409 | display: block;
410 | line-height: var(--line-height);
411 | max-width: var(--width-card-medium);
412 | text-align: var(--justify-important);
413 | border-left: 5px solid #ff4301;
414 | padding: 10px;
415 | margin: 10px 0px;
416 | color: #555555;
417 | }
418 | blockquote footer {
419 | color: var(--color-text-secondary);
420 | display: block;
421 | font-size: small;
422 | line-height: var(--line-height);
423 | padding: 1.5rem 0;
424 | }
425 |
--------------------------------------------------------------------------------
/archivy/static/open_form.js:
--------------------------------------------------------------------------------
1 | function openCommand(cmdUrl, updateState, menuItem) {
2 | if (REQUEST_RUNNING) {
3 | let leave = confirm("Command is still running. Leave anyway?");
4 | if (leave) {
5 | // hack as this is global on page and if we leave the other command cannot run.
6 | // TODO: fetch will continue to run.
7 | // Do a real abort of fetch: https://developer.mozilla.org/en-US/docs/Web/API/AbortController/abort
8 | REQUEST_RUNNING = false;
9 | } else {
10 | return;
11 | }
12 | }
13 |
14 | let formDiv = document.getElementById('form-div');
15 | // if (updateState) {
16 | //
17 | // history.pushState({'cmdUrl': cmdUrl}, null, cmdUrl);
18 | // }
19 | fetch(cmdUrl)
20 | .then(function(response) {
21 | selectMenuItem(menuItem);
22 | return response.text();
23 | })
24 | .then(function(theFormHtml) {
25 | formDiv.innerHTML = theFormHtml;
26 | formDiv.scrollIntoView();
27 | });
28 | }
29 |
30 | function selectMenuItem(menuItem) {
31 | var x = document.getElementsByClassName("command-selected");
32 | var i;
33 | for (i = 0; i < x.length; i++) {
34 | x[i].classList.remove('command-selected');
35 | }
36 | menuItem.classList.add('command-selected');
37 | }
38 |
--------------------------------------------------------------------------------
/archivy/static/post_and_read.js:
--------------------------------------------------------------------------------
1 | let REQUEST_RUNNING = false;
2 |
3 | function postAndRead(commandUrl) {
4 | if (REQUEST_RUNNING) {
5 | return false;
6 | }
7 |
8 | try {
9 | TextDecoder
10 | } catch (e) {
11 | console.error(e);
12 | // Browser missing Text decoder (Edge?)
13 | // post form the normal way.
14 | return true;
15 |
16 | }
17 |
18 | input_form = document.getElementById("inputform");
19 | let submit_btn = document.getElementById("submit_btn");
20 |
21 |
22 | try {
23 | REQUEST_RUNNING = true;
24 | submit_btn.disabled = true;
25 | let runner = new ExecuteAndProcessOutput(input_form, commandUrl);
26 | runner.run();
27 | } catch (e) {
28 | console.error(e);
29 |
30 | } finally {
31 | // if we executed anything never post form
32 | // as we do not know if form already was submitted.
33 | return false;
34 |
35 | }
36 | }
37 |
38 | class ExecuteAndProcessOutput {
39 | constructor(form, commandPath) {
40 | this.form = form;
41 | this.commandUrl = commandPath;
42 | this.decoder = new TextDecoder('utf-8');
43 | this.output_header_div = document.getElementById("output-header")
44 | this.output_div = document.getElementById("output")
45 | this.output_footer_div = document.getElementById("output-footer")
46 | // clear old content
47 | this.output_header_div.innerHTML = '';
48 | this.output_div.innerHTML = '';
49 | this.output_footer_div.innerHTML = '';
50 | // show script output
51 | this.output_header_div.hidden = false;
52 | this.output_div.hidden = false;
53 | this.output_footer_div.hidden = false;
54 | }
55 |
56 | run() {
57 | let submit_btn = document.getElementById("submit_btn");
58 | this.post(this.commandUrl)
59 | .then(response => {
60 | this.form.disabled = true;
61 | if (response.body === undefined) {
62 | firefoxFallback(response)
63 | return;
64 | } else {
65 | let reader = response.body.getReader();
66 | return this.processStreamReader(reader);
67 | }
68 | })
69 | .then(_ => {
70 | REQUEST_RUNNING = false
71 | submit_btn.disabled = false;
72 | })
73 | .catch(error => {
74 | console.error(error);
75 | REQUEST_RUNNING = false;
76 | submit_btn.disabled = false;
77 | }
78 | );
79 | }
80 |
81 | firefoxFallback(response) {
82 | console.log('Firefox < 65 body streams are experimental and not enabled by default.');
83 | console.warn('Falling back to reading full response.');
84 | response.text()
85 | .then(text => this.output_div.innerHTML = text);
86 |
87 | submit_btn.disabled = false;
88 |
89 | }
90 |
91 | post() {
92 | console.log("Posting to " + this.commandUrl);
93 | return fetch(this.commandUrl, {
94 | method: "POST",
95 | body: new FormData(this.form),
96 | // for fetch streaming only accept plain text, we wont handle html
97 | headers: {Accept: 'text/plain'}
98 | });
99 | }
100 |
101 | async processStreamReader(reader) {
102 | while (true) {
103 | const result = await reader.read();
104 | let chunk = this.decoder.decode(result.value);
105 | let insert_func = this.output_div.insertAdjacentText;
106 | let elem = this.output_div;
107 |
108 | // Split the read chunk into sections if needed.
109 | // Below implementation is not perfect as it expects the CLICK_WEB section markers to be
110 | // complete and not in separate chunks. However it seems to work fine
111 | // as long as the generating server yields the CLICK_WEB section in one string as they should be
112 | // quite small.
113 | chunk = chunk.replace("<br>", "\n")
114 | if (chunk.includes('<!-- CLICK_WEB')) {
115 | // there are one or more click web special sections, use regexp split chunk into parts
116 | // and process them individually
117 | let parts = chunk.split(/(<!-- CLICK_WEB [A-Z]+ [A-Z]+ -->)/);
118 | for (let part of parts) {
119 | [elem, insert_func] = this.getInsertFunc(part, elem, insert_func);
120 | if (part.startsWith('<!-- ')) {
121 | // no not display end section comments.
122 | continue;
123 | } else {
124 | insert_func.call(elem, 'beforeend', part);
125 | }
126 | }
127 | } else {
128 | insert_func.call(elem, 'beforeend', chunk);
129 | }
130 |
131 | if (result.done) {
132 | break
133 | }
134 | }
135 | }
136 |
137 | getInsertFunc(part, current_elem, current_func) {
138 | // If we enter new section modify output method accordingly.
139 | if (part.includes(' START ')) {
140 | if (part.includes(' HEADER ')) {
141 | return [this.output_header_div, this.output_header_div.insertAdjacentHTML];
142 | } else if (part.includes(' FOOTER ')) {
143 | return [this.output_footer_div, this.output_footer_div.insertAdjacentHTML];
144 | } else {
145 | throw new Error("Unknown part:" + part);
146 | }
147 | } else if (part.includes(' END ')) {
148 | // plain text again
149 | return [this.output_div, this.output_div.insertAdjacentText];
150 | } else {
151 | // no change
152 | return [current_elem, current_func];
153 | }
154 | }
155 | }
156 |
157 |
--------------------------------------------------------------------------------
/archivy/static/profile.svg:
--------------------------------------------------------------------------------
1 | <svg width="25" height="41" viewBox="0 0 25 41" fill="none" xmlns="http://www.w3.org/2000/svg">
2 | <circle cx="13" cy="7" r="7" fill="black"/>
3 | <path d="M25 28.5C25 35.4036 19.4036 41 12.5 41C5.59644 41 0 35.4036 0 28.5C0 21.5964 5.59644 16 12.5 16C19.4036 16 25 21.5964 25 28.5Z" fill="black"/>
4 | </svg>
5 |
--------------------------------------------------------------------------------
/archivy/tags.py:
--------------------------------------------------------------------------------
1 | import re
2 |
3 | from flask import current_app
4 | from archivy import helpers, data
5 | from tinydb import Query, operations
6 | from archivy.search import query_ripgrep_tags
7 |
8 |
9 | def is_tag_format(tag_name):
10 | return re.match("^[a-zA-Z0-9_-]+quot;, tag_name)
11 |
12 |
13 | def get_all_tags(force=False):
14 | db = helpers.get_db()
15 | list_query = db.search(Query().name == "tag_list")
16 |
17 | # If the "tag_list" doesn't exist in the database: create it.
18 | newly_created = list_query == []
19 | if newly_created:
20 | db.insert({"name": "tag_list", "val": []})
21 |
22 | # Then update it if needed
23 | tags = []
24 | if newly_created or force:
25 | tags = list(query_ripgrep_tags())
26 | db.update(operations.set("val", tags), Query().name == "tag_list")
27 | else:
28 | tags = list_query[0]["val"]
29 | return tags
30 |
31 |
32 | def add_tag_to_index(tag_name):
33 | all_tags = get_all_tags()
34 | if tag_name not in all_tags:
35 | all_tags.append(tag_name)
36 | db = helpers.get_db()
37 | db.update(operations.set("val", all_tags), Query().name == "tag_list")
38 | return True
39 |
--------------------------------------------------------------------------------
/archivy/templates/bookmarklet.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 | {% block content %}
3 |
4 | <h2>Bookmarklet</h2>
5 |
6 | Using the bookmarklet allows you to quickly add bookmarks to archivy.
7 |
8 | All you need to do is drage the link below to your browser toolbar. Then, when you find an interesting link, just click the button on your toolbar named "Add to archivy".
9 |
10 | <h3><a href="javascript: (async function() {
11 | document.body.insertAdjacentHTML('beforeend', `
12 | <form style='display: none' method='post' target='_blank' id='archivy_bookmarklet' action='{{ request.host_url | safe }}/save_from_bookmarklet'>
13 | <input type='hidden' name='url'>
14 | <input type='text' name='html'>
15 | </form>
16 | `);
17 | const form = document.getElementById('archivy_bookmarklet');
18 | form.querySelector('input[name=url]').value = window.location.href;
19 | form.querySelector('input[name=html]').value = document.documentElement.outerHTML;
20 | form.submit();})();">
21 | Add to {{ config.SITE_TITLE }}
22 | </a></h3>
23 |
24 | {% endblock %}
25 |
--------------------------------------------------------------------------------
/archivy/templates/click_web/command_form.html:
--------------------------------------------------------------------------------
1 | {% import 'click_web/form_macros.html' as macros %}
2 | <h2 class="command-title-parents">{{ levels[1:-1]| join(' - ', attribute='command.name')|title }}</h2>
3 | <h3 class="command-title">{{ command.name|title }}</h3>
4 | <div class="command-help">{{ command.html_help }}</div>
5 |
6 | <form id="inputform"
7 | method="post"
8 | action="{{ request.url }}"
9 | onsubmit="return postAndRead('{{ request.url }}');"
10 | class="pure-form pure-form-aligned"
11 | enctype="multipart/form-data">
12 | {% set command_list = [] %}
13 | {% for level in levels %}
14 | {% for field in level.fields %}
15 | <div class="parameter {{ field.param }} {{ field.type }}">
16 | <div class="pure-control-group">
17 | {% if field.nargs == -1 %}
18 | {{ macros.add_variadic_field_input(field) }}
19 | {% else %}
20 | <label for="{{ field.name }}">
21 | {{ field.human_readable_name|capitalize }}
22 | </label>
23 | {% for arg in range(field.nargs) %}
24 | {{ macros.add_field_input(field) }}
25 | {% endfor %}
26 |
27 | {% endif %}
28 | </div>
29 | </div>
30 | {% endfor %}
31 | {% endfor %}
32 | <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
33 | <button type="submit" id="submit_btn" class="btn btn-primary m-2">Run</button>
34 | </form>
35 |
36 | <div id="output-header" hidden="true"></div>
37 | <div class="script-output" id="output" hidden="true"></div>
38 | <div id="output-footer" hidden="true"></div>
39 |
--------------------------------------------------------------------------------
/archivy/templates/click_web/form_macros.html:
--------------------------------------------------------------------------------
1 | {% macro add_field_input(field) -%}
2 | {% if field.type == 'option' %}
3 | {{ add_option_field(field) }}
4 | {% else %}
5 | {% if field.type == "checkbox" and field.checked %}
6 | {#
7 | Hack to work with flags that are default on.
8 | As checkbox is only sent down when checked we duplicate
9 | hidden field that is always sent but with empty value.
10 | https://stackoverflow.com/a/1992745/1306577 #}
11 | <input name="{{ field.name }}" type='hidden' value='{{ field.off_flag }}'>
12 | {% endif %}
13 | <input name="{{ field.name }}"
14 | id="{{ field.name }}"
15 | class="form-control input-lg {% if field.type != 'checkbox' %}d-block{% endif %} mt-2 mb-2"
16 | placeholder="{{ field.desc |default('', true)|capitalize}}"
17 | type="{{ field.type }}"
18 | {% if field.step %} step="{{ field.step }}"{% endif %}
19 | {% if field.accept %} accept="{{ field.accept }}"{% endif %}
20 | value="{{ field.value|default('', true) }}"
21 | {{ field.checked }}
22 | {{ 'required' if field.required else '' }}
23 |
24 |
25 | >
26 | {% endif %}
27 |
28 | {%- endmacro %}
29 |
30 | {% macro add_option_field(field) -%}
31 | <select name="{{ field.name }}">
32 | {% for option in field.options %}
33 | <option value="{{ option }}"
34 | {{ 'selected' if field.default == option else '' }}>
35 | {{ option }}
36 | </option>
37 | {% endfor %}
38 | </select>
39 | {%- endmacro %}
40 |
41 |
42 | {% macro add_variadic_field_input(field) -%}
43 | <label for="{{ field.name }}">
44 | {{ field.human_readable_name|capitalize }}
45 | </label>
46 | {% if field.type == 'option' or field.type == "checkbox" %}
47 | VARIARDIC OPTIONS OR CHECKBOXES ARE NOT SUPPORTED
48 | {% else %}
49 | <textarea class="d-block mt-2 mb-2" name="{{ field.name }}" id="{{ field.name }}" cols="40" rows="5"></textarea>
50 | {% endif %}
51 | <div class="help" style="display: inline-block">
52 | (Each line will be passed as separate argument)
53 | {{ field.desc|default('', true)|capitalize }}
54 | </div>
55 |
56 | {%- endmacro %}
57 |
--------------------------------------------------------------------------------
/archivy/templates/click_web/show_tree.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 |
3 | {% block content %}
4 | <script src="{{ url_for('static', filename='open_form.js') }}"></script>
5 | <script src="{{ url_for('static', filename='post_and_read.js') }}"></script>
6 | <div>
7 | <div class="command-tree" id="files">
8 | <h1>Plugins</h1>
9 | <ul>
10 | {%- for command in tree.childs recursive %}
11 | <li class="file" title="{{ command.help|replace('\b', '')|escape }}">
12 | {% if command.is_group %}
13 | <p>{{ command.name|title }} {% if command.short_help %} | {{ command.short_help }} {% endif %}</p>
14 | {% else %}
15 | <a title="{{ command.help|replace('\b', '')|escape }}"
16 | href="#" onclick="openCommand('{{ command.path }}', true, this);">{{ command.name }}</a>
17 | {% endif %}
18 | {%- if command.childs -%}
19 | <ul>{{ loop(command.childs) }}</ul>
20 | {%- endif %}</li>
21 | {%- endfor %}
22 | </ul>
23 |
24 | </div>
25 | </div>
26 |
27 | <div class="p-4">
28 | <div id="form-div">
29 | <p>Select a command to run above.</p>
30 | </div>
31 | </div>
32 |
33 | {% endblock %}
34 |
--------------------------------------------------------------------------------
/archivy/templates/config.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 | {% macro render_form(form, depth) %}
3 | {% for field in form %}
4 | {% if field.type == "FormField"%}
5 | <div>
6 | {% if field.data | length > 1 %}
7 | <h{{ [4, depth + 1] | min}} id="{{ field.id }}">{{ field.label.text }}</h{{ [4, depth + 1] | min}}>
8 | {% endif %}
9 | {{ render_form(field, depth + 1) }}
10 | </div>
11 | {% elif not field.type in ["CSRFTokenField", "FormField", "SubmitField"] %}
12 | <div>
13 | {{ field.label }}
14 | {{ field() }}
15 | </div>
16 | {% elif field.type in ["CSRFTokenField", "SubmitField"] %}
17 | {{ field() }}
18 | {% endif %}
19 | {% endfor %}
20 | {%- endmacro -%}
21 | {% macro render_toc(form) %}
22 | <ul>
23 | {% for field in form %}
24 | {% if field.type == "FormField" and field.data | length > 1 %}
25 | <li>
26 | <a href="#{{ field.id }}">{{ field.label.text }}</a>
27 | {{ render_toc(field) }}
28 | </li>
29 | {% endif %}
30 | {% endfor %}
31 | </ul>
32 | {%- endmacro -%}
33 | {% block content %}
34 | <h1>Config</h1>
35 | <p>Check out <a href="https://archivy.github.io/config" target="_blank" rel="noopener">the documentation</a> for more info on these options.</p>
36 | {{ render_toc(conf) }}
37 | <form action="/config" method="POST">
38 | {{ render_form(conf, 1) }}
39 | </form>
40 |
41 | {% endblock %}
42 |
--------------------------------------------------------------------------------
/archivy/templates/dataobjs/new.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 |
3 | {% block content %}
4 | <form action="" method="post" novalidate>
5 | <h1>{{ title }}</h1>
6 | {{ form.hidden_tag() }}
7 | {% for field in form %}
8 | {% if field.name != "csrf_token" and field.name != "submit" %}
9 | <p>
10 | {% if field.name == "path" %}
11 | {{ field.label }}
12 | {% endif %}
13 | {{ field(placeholder=field.name, **{"aria-describedby":"username-input-validation"}) }}
14 |
15 | {% for error in field.errors %}
16 | <div class="error-wrapper">
17 | <p class="note-error" id="validation-{{ field.name }}">{{ error }}</p>
18 | </div>
19 | {% endfor %}
20 | </p>
21 | {% endif %}
22 | {% endfor %}
23 | <p>{{ form.submit() }}</p>
24 | </form>
25 | {% endblock %}
26 |
--------------------------------------------------------------------------------
/archivy/templates/home.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 | {% block content %}
3 |
4 | {% set search_enabled = config['SEARCH_CONF']['enabled'] %}
5 | {% if search_enabled %}
6 | <input type="text" id="searchBar" placeholder="Search">
7 | <ul id="searchHits"></ul>
8 | {% endif %}
9 |
10 | <div id="files">
11 |
12 | {% if not view_only %}
13 | <div class="d-flex" id="main-links">
14 | <div style="width: 100%;" class="d-flex">
15 | <a class="btn" href="/notes/new?path={{current_path}}">
16 | New Note
17 | <svg class="octicon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill-rule="evenodd" d="M0 3.75C0 2.784.784 2 1.75 2h12.5c.966 0 1.75.784 1.75 1.75v8.5A1.75 1.75 0 0114.25 14H1.75A1.75 1.75 0 010 12.25v-8.5zm1.75-.25a.25.25 0 00-.25.25v8.5c0 .138.112.25.25.25h12.5a.25.25 0 00.25-.25v-8.5a.25.25 0 00-.25-.25H1.75zM3.5 6.25a.75.75 0 01.75-.75h7a.75.75 0 010 1.5h-7a.75.75 0 01-.75-.75zm.75 2.25a.75.75 0 000 1.5h4a.75.75 0 000-1.5h-4z"></path></svg>
18 | </a>
19 | <a class="btn" href="/bookmarks/new?path={{current_path}}">
20 | New Bookmark
21 | <svg class="octicon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill-rule="evenodd" d="M4.75 2.5a.25.25 0 00-.25.25v9.91l3.023-2.489a.75.75 0 01.954 0l3.023 2.49V2.75a.25.25 0 00-.25-.25h-6.5zM3 2.75C3 1.784 3.784 1 4.75 1h6.5c.966 0 1.75.784 1.75 1.75v11.5a.75.75 0 01-1.227.579L8 11.722l-3.773 3.107A.75.75 0 013 14.25V2.75z"></path></svg>
22 | </a>
23 | <form action="/folders/delete" method="post" onsubmit="return confirm('Delete this folder and its items permanently?')" style="margin: 0px;">
24 | {{ delete_form.csrf_token }}
25 | {{ delete_form.dir_name(value=current_path) }}
26 | <button class="btn btn-delete">
27 | Delete folder
28 | <svg class="octicon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill-rule="evenodd" d="M6.5 1.75a.25.25 0 01.25-.25h2.5a.25.25 0 01.25.25V3h-3V1.75zm4.5 0V3h2.25a.75.75 0 010 1.5H2.75a.75.75 0 010-1.5H5V1.75C5 .784 5.784 0 6.75 0h2.5C10.216 0 11 .784 11 1.75zM4.496 6.675a.75.75 0 10-1.492.15l.66 6.6A1.75 1.75 0 005.405 15h5.19c.9 0 1.652-.681 1.741-1.576l.66-6.6a.75.75 0 00-1.492-.149l-.66 6.6a.25.25 0 01-.249.225h-5.19a.25.25 0 01-.249-.225l-.66-6.6z"></path></svg>
29 | </button>
30 | </form>
31 | </div>
32 | <div class="d-flex">
33 | <form action="/folders/create" method="post" class="d-flex">
34 | {{ new_folder_form.csrf_token }}
35 | {{ new_folder_form.parent_dir(value=current_path) }}
36 | {{ new_folder_form.new_dir(placeholder="New folder name") }}
37 | {{ new_folder_form.submit() }}
38 | </form>
39 | {% if current_path != "" %}
40 | <form action="/folders/rename" method="post" class="d-flex">
41 | {{ rename_form.csrf_token }}
42 | {{ rename_form.current_path(value=current_path) }}
43 | {{ rename_form.new_name(placeholder="Name") }}
44 | {{ rename_form.submit() }}
45 | </form>
46 | {% endif %}
47 | </div>
48 | </div>
49 | {% endif %}
50 |
51 | <h3>Recently modified</h3>
52 | <ul>
53 | {% for d in most_recent %}
54 | <li style="display: flex; justify-content: space-between"><a href="/dataobj/{{ d['id'] }}">{{ d['title'] }}</a><p style="color: #696969; margin: 0">Modified at {{ d['modified_at'] }}</p></li>
55 | {% endfor %}
56 | </ul>
57 | <ul>
58 |
59 | <div class="folder-tags">
60 | {% for tag in tag_cloud %}
61 | <a href="/tags/{{ tag }}">
62 | <span class="post-tag">{{ tag }}</span>
63 | </a>
64 | {% endfor %}
65 | </div>
66 |
67 | {% for subdir in dir.child_dirs.values() | sort(attribute="name") %}
68 | <li class="dir">
69 | <svg class="octicon ml-2 mr-2" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill-rule="evenodd" d="M1.75 1A1.75 1.75 0 000 2.75v10.5C0 14.216.784 15 1.75 15h12.5A1.75 1.75 0 0016 13.25v-8.5A1.75 1.75 0 0014.25 3h-6.5a.25.25 0 01-.2-.1l-.9-1.2c-.33-.44-.85-.7-1.4-.7h-3.5z"></path></svg>
70 | <a href="/?path={{current_path}}{{subdir.name}}/">{{ subdir.name }}</a>
71 | </li>
72 | {% endfor %}
73 | {% for file in dir.child_files | sort(attribute="title") %}
74 | <li class="file">
75 | <svg class="octicon" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path fill-rule="evenodd" d="M3.75 1.5a.25.25 0 00-.25.25v11.5c0 .138.112.25.25.25h8.5a.25.25 0 00.25-.25V6H9.75A1.75 1.75 0 018 4.25V1.5H3.75zm5.75.56v2.19c0 .138.112.25.25.25h2.19L9.5 2.06zM2 1.75C2 .784 2.784 0 3.75 0h5.086c.464 0 .909.184 1.237.513l3.414 3.414c.329.328.513.773.513 1.237v8.086A1.75 1.75 0 0112.25 15h-8.5A1.75 1.75 0 012 13.25V1.75z"></path></svg>
76 | <a href="/dataobj/{{file.id}}">{{ file.title }}</a>
77 | </li>
78 | {% endfor %}
79 | </ul>
80 | </div>
81 |
82 | {% if search_enabled %}
83 | {% include "markdown-parser.html" %}
84 | {% endif %}
85 | <script>
86 | // search functionality
87 | {% if search_enabled %}
88 | function sanitize_match_html(rendered, q)
89 | {
90 | let stripped_tags = new RegExp("(?:<img.*>|<mark>|</mark>|<bold>|<bold>)", "ig"); // remove unnecessary tags
91 | let replace_headings = new RegExp("<h[0-9][^>]*>(.*)</h[0-9]>", "ig"); // replace headings with <p>
92 | let highlight_regex = new RegExp(`(?![^<>]*>)(${q})`, "ig"); // replace query in html text, not attributes, with highlights
93 | let output = rendered.replace(stripped_tags, "").replace(replace_headings, "<p>$1</p>").replace(highlight_regex, "<mark>$1</mark>");
94 | if (!output.match("(?![^<>]*>)\\w+")) { output = ""; } // check there is actual text, not html, in the output
95 | return output;
96 | }
97 | function appendHit(hit, q) {
98 | let hitsDiv = document.getElementById("searchHits");
99 | let hitLi = document.createElement("li");
100 | let a = document.createElement("a");
101 | a.href = `/dataobj/${hit["id"]}`, a.textContent = hit["title"];
102 | hitLi.append(a);
103 | let body = document.createElement("div");
104 | if ("matches" in hit)
105 | {
106 | let max_matches = 4, rendered_matches = "<p>Matches:</p>", n_rendered_matches = 0;
107 | hit["matches"].every((match) => {
108 | let curr_match_output = sanitize_match_html(window.parser.customRender(match), q);
109 | if (curr_match_output != "") { // only render if there is text remaining after sanitization
110 | rendered_matches += "<hr>" + curr_match_output;
111 | n_rendered_matches++;
112 | }
113 | if (n_rendered_matches == max_matches) {
114 | rendered_matches += "...";
115 | return false; // stop looping
116 | }
117 | return true; // keep going
118 | })
119 | body.innerHTML += rendered_matches;
120 | }
121 | hitLi.append(body);
122 | hitsDiv.appendChild(hitLi);
123 | }
124 |
125 | let input = document.getElementById("searchBar");
126 | input.addEventListener('input', async function(e) {
127 | let query = input.value;
128 | if (input.value !== "")
129 | {
130 | let searchQuery = await fetch(`${SCRIPT_ROOT}/search?query=${encodeURIComponent(input.value)}`, {
131 | "method": "GET"
132 | });
133 | if (searchQuery.ok && query == input.value) {
134 | let data = await searchQuery.json(), i = 0;
135 | document.getElementById("searchHits").innerHTML = "";
136 | data.forEach(function(hit)
137 | {
138 | if (query !== input.value) return;
139 | appendHit(hit, query);
140 | })
141 |
142 | }
143 | }
144 | else {
145 | document.getElementById("searchHits").innerHTML = "";
146 | }
147 | });
148 | {% endif %}
149 | </script>
150 | {% endblock %}
151 |
--------------------------------------------------------------------------------
/archivy/templates/markdown-parser.html:
--------------------------------------------------------------------------------
1 | <link rel="stylesheet" href="/static/math.css">
2 | <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex/dist/katex.min.css">
3 | <link rel="stylesheet" href="/static/monokai.css">
4 | <script src="/static/math.js">
5 | </script>
6 | <script src="/static/highlight.js"></script>
7 | <script src="/static/parser.js"></script>
8 | <script>
9 | window.parser = window.markdownit({
10 | {# The lower is needed to make JS understand Python bools #}
11 | {% for name, value in config.EDITOR_CONF.settings.items() %}
12 | {{ name }}: {{ value | lower }},
13 | {% endfor %}
14 | highlight: function (str, lang) {
15 | if (lang && hljs.getLanguage(lang)) {
16 | try {
17 | return '<pre class="hljs"><code>' +
18 | hljs.highlight(lang, str, true).value +
19 | '</code></pre>';
20 | } catch (__) {}
21 | }
22 | }
23 | })
24 | .use(texmath.use(katex), {
25 | engine: katex,
26 | delimiters:'dollars',
27 | katexOptions: { macros: {"\\RR": "\\mathbb{R}"} }
28 | })
29 | {% for plugin_name, plugin_params in config.EDITOR_CONF.plugins.items() %}
30 | .use(window.{{plugin_name}}{% if plugin_params | length %}, {{plugin_params|tojson}}{% endif %})
31 | {% endfor %}
32 | window.parser.customRender = function(content) {
33 | let tag_regex = new RegExp("(^|\\n| )#([-_a-zA-ZÀ-ÖØ-öø-ÿ0-9]+)#", "g");
34 | let note_link_regex = /\[\[(.+)\|([0-9]+)\]\]/g;
35 | content = content.replace(tag_regex, "$1[#$2](/tags/$2)");
36 | content = content.replace(note_link_regex, "[[[$1]]](/dataobj/$2)");
37 | return window.parser.render(content);
38 | }
39 | </script>
40 |
--------------------------------------------------------------------------------
/archivy/templates/tags/all.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 |
3 | {% block content %}
4 | <h2>All tags ({{ tags | length }})</h2>
5 | <ul class="post-tags all-tags">
6 | {% for tag in tags %}
7 | <a href="/tags/{{ tag }}"><span class="post-tag">#{{ tag }}</span></a>
8 | {% endfor %}
9 | </ul>
10 | {% endblock %}
11 |
--------------------------------------------------------------------------------
/archivy/templates/tags/show.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 |
3 | {% block content %}
4 | {% include "markdown-parser.html" %}
5 | <h2>{{ tag_name }} ({{ search_result | length }})</h2>
6 | <ul>
7 | {% for res in search_result %}
8 | <li><a href="/dataobj/{{ res['id'] }}">{{ res['title'] }}</li></a>
9 | {% if "matches" in res %}
10 | {% for match in res["matches"] %}
11 | <div class="markdown-result">{{ match }}</div>
12 | {% endfor %}
13 | {% endif %}
14 | {% endfor %}
15 | </ul>
16 | <script>
17 | document.querySelectorAll(".markdown-result").forEach((match) => {
18 | match.innerHTML = window.parser.customRender(`> ${match.innerHTML.trim()}`)
19 | })
20 | </script>
21 |
22 |
23 | {% endblock %}
24 |
--------------------------------------------------------------------------------
/archivy/templates/users/edit.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 | {% block content %}
3 | <br>
4 | <div id="user-cont">
5 | <form id="user-form" action="" method="post">
6 | <h1>Edit Profile</h1>
7 | {{ form.hidden_tag() }}
8 | {% for error in form.username.errors %}
9 | <div class="flash flash-error">{{ error }}</div>
10 | {% endfor %}
11 | {{ form.username(placeholder="Username") }}
12 |
13 | {% for error in form.password.errors %}
14 | <span style="color: red;">{{ error }}</span>
15 | {% endfor %}
16 | {{ form.password(placeholder="New Password") }}
17 |
18 | {{ form.submit(class="btn") }}
19 |
20 | <a class="btn" href="/logout">
21 | Logout
22 | </a>
23 | </form>
24 | </div>
25 | {% endblock %}
26 |
--------------------------------------------------------------------------------
/archivy/templates/users/login.html:
--------------------------------------------------------------------------------
1 | {% extends "base.html" %}
2 | {% block content %}
3 | <br>
4 | <style>
5 | .content {
6 | margin: auto;
7 | }
8 | </style>
9 | <div id="login-cont">
10 | <img src="https://archivy.github.io/img/logo-2.png" width="100" height="100">
11 | <form id="login-form" action="" method="post">
12 | <h1>Sign in</h1>
13 | {{ form.hidden_tag() }}
14 | {% for error in form.username.errors %}
15 | <div class="flash flash-error">{{ error }}</div>
16 | {% endfor %}
17 | {{ form.username(placeholder="Username") }}
18 |
19 |
20 | {% for error in form.password.errors %}
21 | <span style="color: red;">{{ error }}</span>
22 | {% endfor %}
23 | {{ form.password(placeholder="Password") }}
24 |
25 | {{ form.submit(class="btn") }}
26 | </form>
27 | </div>
28 |
29 | {% endblock %}
30 |
--------------------------------------------------------------------------------
/conftest.py:
--------------------------------------------------------------------------------
1 | import shutil
2 | import tempfile
3 | from pathlib import Path
4 |
5 | import click
6 | import pytest
7 | import responses
8 |
9 | from archivy import app, cli
10 | from archivy.click_web import create_click_web_app, _flask_app
11 | from archivy.helpers import get_db, load_hooks
12 | from archivy.models import DataObj, User
13 |
14 | _app = None
15 |
16 |
17 | @pytest.fixture
18 | def test_app():
19 | """Instantiate the app for each test with its own temporary data directory
20 |
21 | Each test using this fixture will use its own db.json and its own data
22 | directory, and then delete them.
23 | """
24 | # create a temporary file to isolate the database for each test
25 | global _app
26 | if _app is None:
27 | _app = create_click_web_app(cli, cli.cli, app)
28 | app_dir = Path(tempfile.mkdtemp())
29 | _app.config["INTERNAL_DIR"] = str(app_dir)
30 | _app.config["USER_DIR"] = str(app_dir)
31 | (app_dir / "data").mkdir()
32 | (app_dir / "images").mkdir()
33 |
34 | _app.config["TESTING"] = True
35 | _app.config["WTF_CSRF_ENABLED"] = False
36 | _app.config["SCRAPING_CONF"]["save_images"] = False
37 | # This setups a TinyDB instance, using the `app_dir` temporary
38 | # directory defined above
39 | # Required so that `flask.current_app` can be called in data.py and
40 | # models.py
41 | # See https://flask.palletsprojects.com/en/1.1.x/appcontext/ for more
42 | # information.
43 | with _app.app_context():
44 | _ = get_db()
45 | user = {"username": "halcyon", "password": "password"}
46 |
47 | User(**user).insert()
48 | yield _app
49 |
50 | # close and remove the temporary database
51 | shutil.rmtree(app_dir)
52 |
53 |
54 | @pytest.fixture
55 | def client(test_app):
56 | """HTTP client for calling a test instance of the app"""
57 | with test_app.test_client() as client:
58 | client.post("/login", data={"username": "halcyon", "password": "password"})
59 | yield client
60 |
61 |
62 | @pytest.fixture
63 | def mocked_responses():
64 | """
65 | Setup mock responses using the `responses` python package.
66 |
67 | Using https://pypi.org/project/responses/, this fixture will mock out
68 | HTTP calls made by the requests library.
69 |
70 | For example,
71 | >>> mocked_responses.add(responses.GET, "http://example.org",
72 | json={'key': 'val'}
73 | )
74 | >>> r = requests.get("http://example.org")
75 | >>> print(r.json())
76 | {'key': 'val'}
77 | """
78 | with responses.RequestsMock() as rsps:
79 | # this ensure that all requests calls are mocked out
80 | rsps.assert_all_requests_are_fired = False
81 | yield rsps
82 |
83 |
84 | @pytest.fixture
85 | def note_fixture(test_app):
86 | note_dict = {
87 | "type": "note",
88 | "title": "Test Note",
89 | "tags": ["testing", "archivy"],
90 | "path": "",
91 | }
92 |
93 | with test_app.app_context():
94 | note = DataObj(**note_dict)
95 | note.insert()
96 | return note
97 |
98 |
99 | @pytest.fixture
100 | def bookmark_fixture(test_app, mocked_responses):
101 | mocked_responses.add(
102 | responses.GET,
103 | "https://example.com/",
104 | body="""<html>
105 | <head><title>Example</title></head><body><p>
106 | Lorem ipsum dolor sit amet, consectetur adipiscing elit
107 | <script>console.log("this should be sanitized")</script>
108 | <img src="/images/image1.png">
109 | <a href="/testing-absolute-url">link</a>
110 | <a href"/empty-link"></a>
111 | #embedded-tag# #tag2#
112 | </p></body></html>
113 | """,
114 | )
115 |
116 | datapoints = {
117 | "type": "bookmark",
118 | "title": "Test Bookmark",
119 | "tags": ["testing", "archivy"],
120 | "path": "",
121 | "url": "https://example.com/",
122 | }
123 |
124 | with test_app.app_context():
125 | bookmark = DataObj(**datapoints)
126 | bookmark.process_bookmark_url()
127 | bookmark.insert()
128 | return bookmark
129 |
130 |
131 | @pytest.fixture
132 | def different_bookmark_fixture(test_app, mocked_responses):
133 | mocked_responses.add(
134 | responses.GET,
135 | "https://example2.com/",
136 | body="""<html>
137 | <head><title>Example</title></head><body><p>asdsad<div class="nested">aaa</div></body></html>
138 | """,
139 | )
140 |
141 | datapoints = {
142 | "type": "bookmark",
143 | "title": "Test Bookmark2",
144 | "url": "https://example2.com/",
145 | }
146 |
147 | with test_app.app_context():
148 | bookmark = DataObj(**datapoints)
149 | bookmark.process_bookmark_url()
150 | bookmark.insert()
151 | return bookmark
152 |
153 |
154 | @pytest.fixture()
155 | def user_fixture(test_app):
156 | user = {"username": "__username__", "password": "__password__"}
157 |
158 | user = User(**user)
159 | user.insert()
160 | return user
161 |
162 |
163 | @pytest.fixture()
164 | def pocket_fixture(test_app, mocked_responses):
165 | """Sets up pocket key and mocked responses for testing pocket sync
166 |
167 | When using this fixture, all calls to https://getpocket.com/v3/get will
168 | succeed and return a single article whose url is https://example.com.
169 | """
170 | with test_app.app_context():
171 | db = get_db()
172 |
173 | mocked_responses.add(
174 | responses.POST,
175 | "https://getpocket.com/v3/oauth/authorize",
176 | json={
177 | "access_token": "5678defg-5678-defg-5678-defg56",
178 | "username": "test_user",
179 | },
180 | )
181 |
182 | # fake /get response from pocket API
183 | mocked_responses.add(
184 | responses.POST,
185 | "https://getpocket.com/v3/get",
186 | json={
187 | "status": 1,
188 | "complete": 1,
189 | "list": {
190 | "3088163616": {
191 | "given_url": "https://example.com",
192 | "status": "0",
193 | "resolved_url": "https://example.com",
194 | "excerpt": "Lorem ipsum",
195 | "is_article": "1",
196 | },
197 | },
198 | },
199 | )
200 |
201 | pocket_key = {
202 | "type": "pocket_key",
203 | "consumer_key": "1234-abcd1234abcd1234abcd1234",
204 | "code": "dcba4321-dcba-4321-dcba-4321dc",
205 | }
206 | db.insert(pocket_key)
207 | return pocket_key
208 |
209 |
210 | @pytest.fixture()
211 | def click_cli():
212 | yield cli.cli
213 |
214 |
215 | @pytest.fixture()
216 | def ctx(click_cli):
217 | with click.Context(click_cli, info_name=click_cli, parent=None) as ctx:
218 | yield ctx
219 |
220 |
221 | @pytest.fixture()
222 | def cli_runner():
223 | yield click.testing.CliRunner()
224 |
--------------------------------------------------------------------------------
/docs/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 |
2 | This is a short guide on the things you should know if you'd like to contribute to Archivy.
3 |
4 | ## Setting up a dev environment.
5 |
6 | - Fork the [archivy repo](https://github.com/archivy/archivy) and then clone the fork on your local machine.
7 | - Create a virtual environment by running `python -m venv venv/`. This will hold all archivy dependencies.
8 | - Run `source venv/bin/activate` to activate this new environment.
9 | - Run `pip install -r requirements.txt` to download all dependencies.
10 |
11 | ## Running the dev server.
12 | ```
13 | # after sourcing the virtualenv
14 |
15 | $ export FLASK_APP=archivy/__init__.py
16 | $ export FLASK_ENV=development
17 | $ flask run
18 | ```
19 |
20 | ## Running cli commands
21 | ```
22 | # after sourcing the virtualenv
23 |
24 | $ python -m archivy.cli --help
25 | ```
26 |
27 |
28 | If you'd like to work on an [existing issue](https://github.com/archivy/archivy/issues), please comment on the github thread for the issue to notify that you're working on it, and then create a new branch with a suitable name.
29 |
30 | For example, if you'd like to work on something about "Improving the UI", you'd call it `improve_ui`. Once you're done with your changes, you can open a pull request and we'll review them.
31 |
32 | **Do not** begin working on a new feature without first discussing it and opening an issue, as we might not agree with your vision.
33 |
34 | If your feature is more isolated and specific, it can also be interesting to develop a [plugin](plugins.md) for it, in which case we can help you with any questions related to plugin development, and would be happy to list your plugin on [awesome-archivy](https://github.com/archivy/awesome-archivy).
35 |
36 | Thanks for contributing!
37 |
--------------------------------------------------------------------------------
/docs/config.md:
--------------------------------------------------------------------------------
1 | Once you've [initialized](install.md) your archivy install, an archivy config is automatically generated. You can edit it through the archivy interface by clicking on the gear icon on the top right of the navbar, or by running `archivy config` in your terminal.
2 |
3 | Here's an overview of the different values you can set and modify.
4 |
5 |
6 | ### General
7 |
8 | | Variable | Default | Description |
9 | |-------------------------|-----------------------------|---------------------------------------|
10 | | `USER_DIR` | System-dependent, see below. It is recommended to set this through `archivy init` | Directory in which markdown data will be saved |
11 | | `INTERNAL_DIR` | System-dependent, see below | Directory where archivy internals will be stored (config, db...)
12 | | `PORT` | 5000 | Port on which archivy will run |
13 | | `HOST` | 127.0.0.1 | Host on which the app will run. |
14 | | `DEFAULT_BOOKMARKS_DIR` | empty string (represents the root directory) | any subdirectory of the `data/` directory with your notes.
15 | | `SITE_TITLE` | Archivy | String value to be displayed in page title and headings. |
16 |
17 | ### Scraping
18 |
19 | An in-progress configuration object to customize how you'd like bookmarking / scraping to work. The options are children of the `SCRAPING_CONF` object, like so:
20 |
21 | ```yaml
22 | SCRAPING_CONF:
23 | save_images:
24 | ...
25 | ```
26 |
27 | | Variable | Default | Description |
28 | |-------------------------|-----------------------------|---------------------------------------|
29 | | `save_images` | False | If true, whenever you save a bookmark, every linked image will also be downloaded locally. |
30 |
31 | If you want to configure the scraping progress more, you can also create a `scraping.py` file in the root of your user directory. This file allows you to override the default bookmarking behavior for certain websites / links, which you can match with regex.
32 |
33 | Once it matches a link, you can either pass your own custom function for parsing, or simply pass a string, which corresponds to the CSS selector for the part of the page you want archivy to scrape. If there are several matches, only the first will be treated.
34 |
35 | Example that processes youtube videos:
36 |
37 | ```python
38 | def process_videos(data):
39 | url = data.url
40 | # modify whatever you want / set the metadata
41 | data.title = "Video - " + url
42 | data.tags = ["video"]
43 | data.content = "..."
44 |
45 | # declare your patterns in this PATTERNS variable
46 | PATTERNS = {
47 | "*youtube.com/*": process_videos
48 | }
49 | ```
50 |
51 | With this example, whenever you create a bookmark of a youtube video, instead of going through the default archival, your function will be called on the data.
52 |
53 |
54 | Example that tells archivy only to scrape the main body of Wikipedia pages:
55 |
56 | ```python
57 | PATTERNS = {
58 | "https://*.wikipedia.org/wiki/*": "#bodyContent"
59 | }
60 | ```
61 |
62 | Example patterns:
63 |
64 | - `*wikipedia*` (`*` matches everything)
65 | - `https://duckduckg?.com*` (? matches a single character)
66 | - `https://www.[nl][ya]times.com` ([] matches any character inside the brackets. Here it'll match nytimes or latimes, for example. Use ![] to match any character **not** inside the brackets)
67 |
68 | ### Theming
69 |
70 | Configure the way your Archivy install looks.
71 |
72 | These configuration options are children of the `THEME_CONF` object, like this:
73 |
74 | ```yaml
75 | THEME_CONF:
76 | use_theme_dark:
77 | use_custom_css:
78 | custom_css_file:
79 | ```
80 |
81 | | Variable | Default | Description |
82 | |------|-------|----|
83 | | `use_theme_dark` | false | Whether or not to load the dark version of the default theme CSS. |
84 | | `use_custom_css` | false | Whether or not to load custom css from `custom_css_file` |
85 | | `custom_css_file` | "" | Name of file to load in the `css/` subdirectory of your user directory (the one with your data or hooks). Create `css/` if it doesn't exist. |
86 |
87 |
88 |
89 | ### Search
90 |
91 | See [Setup Search](setup-search.md) for more information.
92 |
93 | All of these options are children of the `SEARCH_CONF` object, like this in the `config.yml`:
94 |
95 | ```yaml
96 | SEARCH_CONF:
97 | enabled:
98 | url:
99 | # ...
100 | ```
101 | To use search, you first have to enable it either through the `archivy init` setup script or by modifying the `enabled` variable (see below).
102 |
103 | Variables marked `ES only` in their description are only relevant when using the Elasticsearch engine.
104 |
105 | | Variable | Default | Description |
106 | |-------------------------|--------------------------------|---------------------------------------|
107 | | `enabled` | 1 | |
108 | | `engine` | empty string | search engine you'd like to use. One of `["ripgrep", ["elasticsearch"]`|
109 | | `url` | http://localhost:9200 | **[ES only]** Url to the elasticsearch server |
110 | | `es_user` and `es_password` | None | If you're using authentication, for example with a cloud-hosted ES install, you can specify a user and password |
111 | | `es_processing_conf` | Long dict of ES config options | **[ES only]** Configuration of Elasticsearch [analyzer](https://www.elastic.co/guide/en/elasticsearch/reference/current/analysis.html), [mappings](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html) and general settings. |
112 |
113 |
114 | `INTERNAL_DIR` and `USER_DIR` by default will be set by the
115 | [appdirs](https://pypi.org/project/appdirs/) python library:
116 |
117 | On Linux systems, it follows the [XDG
118 | specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html):
119 | `~/.local/share/archivy`
120 |
121 |
122 | ### Editor configuration
123 |
124 | To enable auto save in the editor, set `EDITOR_CONF -> autosave` to True.
125 |
126 | Archivy uses the [markdown-it](https://github.com/markdown-it/markdown-it) parser for its editor. This parser can be configured to change the output according to your needs. The default values of `EDITOR_CONF` are given below. Refer to the [markdown-it docs](https://github.com/markdown-it/markdown-it#init-with-presets-and-options) for a full list of possible options.
127 |
128 | ```yaml
129 | EDITOR_CONF:
130 | autosave: False
131 | settings:
132 | linkify: true
133 | html: false
134 | xhtmlOut: false
135 | breaks: true
136 | typographer: false
137 | plugins: ...
138 | toolbar_icons: ["bold", "italic", "link", "upload-image", "heading", "code", "strikethrough", "quote", "table"] # see https://github.com/Ionaru/easy-markdown-editor#toolbar-icons for more options
139 | ```
140 |
141 | Archivy uses several markdown plugins to enhance its functionality:
142 |
143 | - [markdown-it-anchor](https://github.com/valeriangalliat/markdown-it-anchor)
144 | - [markdown-it-toc-done-right](https://github.com/nagaozen/markdown-it-toc-done-right)
145 | - [markdown-it-mark](https://github.com/markdown-it/markdown-it-mark)
146 | - [markdown-it-footnote](https://github.com/markdown-it/markdown-it-footnote)
147 | - [markdown-it-texmath](https://github.com/goessner/markdown-it-texmath)
148 |
149 | Some of these plugins (see below) can be configured and modified. Refer to their homepages above to see what you can change. They are set up with the following configuration by default:
150 |
151 | ```yaml
152 | EDITOR_CONF:
153 | plugins:
154 | markdownitFootnote: {}
155 | markdownitMark: {}
156 | markdownItAnchor:
157 | permalink: True
158 | permalinkSymbol: '¶'
159 | markdownItTocDoneRight: {}
160 | ```
161 |
--------------------------------------------------------------------------------
/docs/difference.md:
--------------------------------------------------------------------------------
1 |
2 | There are many great tools out there to create your knowledge base.
3 |
4 |
5 | So why should you use Archivy?
6 |
7 | Here are the ingredients that make Archivy stand out (of course many tools have other interesting components / focuses, and you should pick the one that resonates with what you want):
8 |
9 | - **Focus on scripting and extensibility**: When I began developing archivy, I had plans for developing **in the app's core** additional features that would allow you to sync up to your digital presence, for exemple a Reddit extension that would download your upvoted posts, etc... I quickly realised that this would be a bad way to go, as many people often want maybe **one** feature, but not a ton of useless included extensions. That's when I decided to instead build a flexible framework for people to build installable [**plugins**](plugins.md), as this allows a) users only download what they want and b) Archivy can focus on its core and users can build their own extensions for themselves and others.
10 | - **Importance of Digital Preservation**: ^ I mentioned above the idea of a plugin for saving all your upvoted reddit posts. This is just an example of how Archivy is intended to be used not only as a knowledge base, but also a resilient stronghold for the data that used to be solely held by third-party services. This idea of automatically syncing and saving content you've found valuable could be expanded to HN upvoted posts, browser bookmarks, etc... [^1]
11 | - **deployable OR you can just run it on your computer**: The way archivy was engineered makes it possible for you to just run it on your laptop or pc, and still use it without problems. On the other hand, it is also very much a possibility to self-host it and expose it publicly for use from anywhere, which is why archivy has auth.
12 | - Archivy supports several [options for its search](setup-search.md), including Elasticsearch. This tool might be considered overkill for one's knowledge base, but as your knowledge base grows also with content from other data sources and automation, it can become a large amount of data. Elasticsearch provides very high quality search on this information at a swift speed and the other alternative [ripgrep](https://github.com/BurntSushi/ripgrep) is much lighter while still reacting well to large amounts of data.
13 |
14 | [^1]: https://beepb00p.xyz/hpi.html is an intriguing tool on this topic.
15 |
16 | [^2]: See [this thread](https://github.com/archivy/archivy/issues/13) if you have any tools in mind for this.
17 |
--------------------------------------------------------------------------------
/docs/editing.md:
--------------------------------------------------------------------------------
1 | Note: to enable auto save in the editor, see [this](https://archivy.github.io/config/#editor-configuration).
2 |
3 | ## Format
4 |
5 | Archivy files are in the [markdown](https://daringfireball.net/projects/markdown/basics) format following the [commonmark spec](https://spec.commonmark.org/).
6 |
7 | We've also included a few powerful extensions:
8 |
9 | - **Bidirectional links**: You can easily link to a new note in the web editor by typing `[[`: an input box will appear where you can enter the title of the note you want to link to. Otherwise, links between notes are in the format `[[linked note title|linked note id]]`. If the title you wrote doesn't refer to an existing note, you can click enter and Archivy will create a new note.
10 |
11 | - **Embedded tags**: You can directly add tags inside your notes with this `#tag#` syntax (see below). These tags and their groupings can then be viewed by clicking `Tags` on the navigation bar. Starting to type `#` in the web editor will display an input where you can search existing tags.
12 |
13 | ```md
14 | I was going to a #python# conference, when I saw a #lion#.
15 | ```
16 |
17 | - **In-editor bookmarking**: if you'd like to store a local copy of a webpage you're referring to inside an archivy note, simply select the url and click the "bookmark" icon.
18 |
19 | - **LaTeX**: you can render mathematical expressions like this:
20 |
21 | ```md
22 | $
23 | \pi = 3.14
24 | $
25 | ```
26 |
27 | - **Footnotes**:
28 | ```md
29 | What does this describe? [^1]
30 |
31 | [^1]: test foot note.
32 | ```
33 |
34 | - **Tables**:
35 | ```md
36 | | Column 1 | Column 2 |
37 | | -------- | -------- |
38 | | ... | ... |
39 | ```
40 |
41 | - **Code blocks with syntax highlighting**:
42 | ````md
43 | ```python
44 | print("this will be highlighted")
45 | x = 1337
46 | ```
47 | ````
48 |
49 | There are several ways you can edit content in Archivy.
50 |
51 | Whenever you open a note or bookmark, at the bottom of the page you'll find a few buttons that allow you to edit it.
52 |
53 | ## Editing through the web interface
54 |
55 | You can edit through the web app, by clicking "Toggle web editor" at the bottom. This is the recommended way because the Archivy web editor provides useful functionality. You can save your work, link to other notes with the "Link to a note button", and archive webpages referenced in your note, all inside the editor!
56 |
57 | ## Locally
58 |
59 | You can do a **local edit**. This option is only viable if running archivy on your own computer. This will open the concerned file with the default app set to edit markdown.
60 |
61 | For example like this:
62 |
63 | 
64 |
--------------------------------------------------------------------------------
/docs/example-plugin/README.md:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/docs/example-plugin/README.md
--------------------------------------------------------------------------------
/docs/example-plugin/archivy_extra_metadata/__init__.py:
--------------------------------------------------------------------------------
1 | import click
2 | from tinydb import Query
3 | from archivy.helpers import get_db
4 | from archivy import app
5 |
6 |
7 | @click.group()
8 | def extra_metadata():
9 | """`archivy_extra_metadata` plugin to add metadata to your notes."""
10 | pass
11 |
12 |
13 | @extra_metadata.command()
14 | @click.option("--author", required=True)
15 | @click.option("--location", required=True)
16 | def setup(author, location):
17 | """Save metadata values."""
18 | with app.app_context():
19 | # save data in db
20 | get_db().insert({"type": "metadata", "author": author, "location": location})
21 | click.echo("Metadata saved!")
22 |
23 |
24 | def add_metadata(dataobj):
25 | with app.app_context():
26 | metadata = get_db().search(Query().type == "metadata")[0]
27 | dataobj.content += f"Made by {metadata['author']} in {metadata['location']}."
28 |
--------------------------------------------------------------------------------
/docs/example-plugin/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup, find_packages
2 |
3 | with open("README.md", "r") as fh:
4 | long_description = fh.read()
5 |
6 | setup(
7 | name="archivy_extra_metadata",
8 | version="0.1.0",
9 | author="Uzay-G",
10 | description=(
11 | "Archivy extension to add some metadata at the end of your notes / bookmarks."
12 | ),
13 | long_description=long_description,
14 | long_description_content_type="text/markdown",
15 | classifiers=[
16 | "Programming Language :: Python :: 3",
17 | ],
18 | packages=find_packages(),
19 | entry_points="""
20 | [archivy.plugins]
21 | extra-metadata=archivy_extra_metadata:extra_metadata
22 | """,
23 | )
24 |
--------------------------------------------------------------------------------
/docs/img/local-edit.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/docs/img/local-edit.png
--------------------------------------------------------------------------------
/docs/img/logo-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/docs/img/logo-2.png
--------------------------------------------------------------------------------
/docs/img/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/docs/img/logo.png
--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
1 | 
2 |
3 | # Archivy
4 |
5 | Archivy is a self-hostable knowledge repository that allows you to learn and retain information in your own personal and extensible wiki.
6 |
7 | Features:
8 |
9 | - If you add bookmarks, their web-pages contents' will be saved to ensure that you will **always** have access to it, following the idea of [digital preservation](https://jeffhuang.com/designed_to_last/). Archivy is also easily integrated with other services and your online accounts.
10 | - Knowledge base organization with bidirectional links between notes, and embedded tags.
11 | - Everything is a file! For ease of access and editing, all the content is stored in extended markdown files with yaml front matter. This format supports footnotes, LaTeX math rendering, syntax highlighting and more.
12 | - Extensible plugin system and API for power users to take control of their knowledge process
13 | - [syncing options](https://github.com/archivy/archivy-git)
14 | - Powerful and advanced search.
15 | - Image upload
16 |
17 |
18 | <video src="https://www.uzpg.me/assets/images/archivy.mov" style="width: 100%" controls>
19 | </video>
20 |
21 | [Roadmap](https://github.com/archivy/archivy/issues/74#issuecomment-764828063)
22 |
23 | Upcoming:
24 |
25 | - Annotations
26 | - Multi User System with permission setup.
27 |
28 | ## Quickstart
29 |
30 |
31 | Install archivy with `pip install archivy`. Other installations methods are listed [here](https://archivy.github.io/install)
32 |
33 | Then run this and enter a password to create a new user:
34 |
35 | ```bash
36 | $ archivy create-admin <username>
37 | ```
38 |
39 | Finally, execute `archivy run` to serve the app. You can open it at https://localhost:5000 and login with the credentials you entered before.
40 |
41 | You can then use archivy to create notes, bookmarks and then organize and store information.
42 |
43 | See [the docs](install.md) for information on other installation methods.
44 |
45 | ## Community
46 |
47 | Archivy is dedicated at building **open and quality knowledge base software** through collaboration and community discussion.
48 |
49 | To get news and updates on Archivy and its development, you can [watch the archivy repository](https://github.com/archivy/archivy) or follow [@uzpg_ on Twitter](https://twitter.com/uzpg_).
50 |
51 | You can interact with us through the [issue board](https://github.com/archivy/archivy/issues) and the more casual [discord server](https://discord.gg/uQsqyxB).
52 |
53 | If you'd like to support the project and its development, you can also [sponsor](https://github.com/sponsors/Uzay-G/) the Archivy maintainer.
54 |
55 |
56 | Note: If you're interested in the applications of AI to knowledge management, we're also working on this with [Espial](https://github.com/Uzay-G/espial).
57 |
58 | ## License
59 |
60 | This project is licensed under the MIT License. See [LICENSE](./LICENSE) for more information.
61 | The Archivy Logo is designed by [Roy Quilor](https://www.quilor.com/), licensed under [CC BY-NC 4.0](https://creativecommons.org/licenses/by-nc/4.0)
62 |
63 | [Changelog](https://github.com/archivy/archivy/releases)
64 |
65 |
66 |
--------------------------------------------------------------------------------
/docs/install.md:
--------------------------------------------------------------------------------
1 | ## With pip
2 |
3 | You can easily install archivy with `pip`. (pip is the default package installer for Python, you can use pip to install many packages and apps, see this [link](https://pypi.org/project/pip/) for more information if needed)
4 |
5 |
6 | 1. Make sure your system has Python and pip installed. The Python programming language can also be downloaded from [here](https://www.python.org/downloads/).
7 | 2. Install the python package with `pip install archivy`
8 | 3. It's highly recommended to install [ripgrep](https://github.com/BurntSushi/ripgrep), which Archivy uses for some of it's organization features (note links & tags inside notes).
9 | 3. If you'd like to use search, follow [these docs](setup-search.md) first and then do this part. Run `archivy init` to create a new user and use the setup wizard.
10 | 4. There you go! You should be able to start the app by running `archivy run` in your terminal and then just login.
11 |
12 | ## With docker
13 |
14 | You can also use archivy with Docker.
15 |
16 | See [the docker documentation](https://github.com/archivy/archivy-docker) for instructions on this.
17 |
18 | We might implement an AppImage install. Comment [here](https://github.com/archivy/archivy/issues/44) if you'd like to see that happen.
19 |
20 | ## With Nix
21 | ```ShellSession
22 | $ nix-env -i archivy
23 | ```
24 |
--------------------------------------------------------------------------------
/docs/overrides/main.html:
--------------------------------------------------------------------------------
1 |
2 | {% extends "base.html" %}
3 |
4 | {% block analytics %}
5 | <!-- privacy respectful analytics -->
6 | <script async defer src="https://scripts.simpleanalyticscdn.com/latest.js"></script>
7 | <noscript><img src="https://queue.simpleanalyticscdn.com/noscript.gif" alt=""/></noscript>
8 | {% endblock %}
9 |
--------------------------------------------------------------------------------
/docs/reference/architecture.md:
--------------------------------------------------------------------------------
1 |
2 | This document is a general overview of how the different pieces of archivy interact and what technologies it uses. Reading this will be useful for people looking to access the inner archivy API to write plugins. Read [this post](https://www.uzpg.me/tech/2020/07/21/architecture-md.html) to understand what the function of an `architecture.md` file is.
3 |
4 |
5 | Archivy is:
6 |
7 | - A [Flask](https://flask.palletsprojects.com/) web application.
8 | - A [click](https://click.palletsprojects.com/) backend command line interface.
9 |
10 |
11 | You use the cli to run the app, and you'll probably be using the web application for direct usage of archivy.
12 |
13 |
14 | ## Data Storage
15 |
16 | - `DataObjs` is the term used to denote a note or bookmark that is stored in your knowledge base (abbreviation for Data Object). These are stored in a directory on your filesystem of which you can [configure the location](../config.md). They are organized in markdown files with `yaml` front matter like this:
17 |
18 | ```yaml
19 | ---
20 | date: 08-31-20
21 | desc: ''
22 | id: 100
23 | path: ''
24 | tags: []
25 | title: ...
26 | type: note
27 | ---
28 |
29 | ...
30 | ```
31 |
32 | Archivy uses the [python-frontmatter](https://python-frontmatter.readthedocs.io/en/latest/) package to handle the parsing of these files. They can be organized into user-specified sub-directories. Check out [the reference](filesystem_layer.md) to see the methods archivy uses for this.
33 |
34 | - Another storage method Archivy uses is [TinyDB](https://tinydb.readthedocs.io/en/stable/). This is a small, simple document-oriented database archivy gives you access to for persistent data you might want to store in archivy plugins. Use [`helpers.get_db`](/reference/helpers/#archivy.helpers.get_db) to call the database.
35 |
36 | ## Search
37 | Archivy supports two search engines:
38 |
39 | 1. [Elasticsearch](https://www.elastic.co/) - an incredibly powerful solution that is however harder to install.
40 | 2. [ripgrep](https://github.com/BurntSushi/ripgrep), much more lightweight but also less powerful.
41 |
42 | These allow archivy to to index and provide full-text search on their knowledge bases.
43 |
44 |
45 | Elasticsearch requires configuration to have higher quality search results. You can check out the top-notch config archivy already uses by default [here](https://github.com/archivy/archivy/blob/master/archivy/config.py).
46 |
47 | Check out the [helper methods](search.md) archivy exposes for ES.
48 |
49 | ## Auth
50 |
51 | Archivy uses [flask-login](https://flask-login.readthedocs.io/en/latest/) for auth. All endpoints require to be authenticated. You can create an admin user with the `create-admin` command.
52 |
53 | In our roadmap we plan to extend our permission framework to have a multi-user system, and define configuration for the permissions of non-logged in users. In general we want to make things more flexible on the auth side.
54 |
55 |
56 | ## How bookmarks work
57 |
58 | One of the core features of archivy is being able to save webpages locally. The way this works is the conversion of the html of the page you specify to a simple, markdown file.
59 |
60 | We might want to extend this to also be able to save PDF, EPUB and other formats. You can find the reference for this part [here](models.md).
61 |
62 | Further down the road, it'd be nice to add background processing and not only download the webpage, but also save the essential assets it loads for a more complete process. This feature of preserving web content aligns with the mission against [link rot](https://en.wikipedia.org/wiki/Link_rot) [^1].
63 |
64 | ## Plugins
65 |
66 | Plugins in archivy function as standalone python packages. The phenomenal [click-plugins](https://github.com/click-contrib/click-plugins) package allows us to do this by basically adding commands to the cli.
67 |
68 | So you create a python package where you specify commands to extend your pre-existing archivy cli. Then these added commands will be able to be used through the cli. But what makes plugins interesting is that you can actually also use the plugins through the web interface, without having access to the system running archivy. We use an adaptation of the [click-web](https://github.com/fredrik-corneliusson/click-web) to convert your cli commands to interactive web forms.
69 |
70 |
71 | [^1]: See [this manifesto](https://jeffhuang.com/designed_to_last/) to learn more about this phenomenon.
72 |
--------------------------------------------------------------------------------
/docs/reference/filesystem_layer.md:
--------------------------------------------------------------------------------
1 | This module holds the methods used to access, modify, and delete components of the filesystem where `Dataobjs` are stored in Archivy.
2 |
3 | ::: archivy.data
4 |
--------------------------------------------------------------------------------
/docs/reference/helpers.md:
--------------------------------------------------------------------------------
1 |
2 | This is a series of helper functions that could be useful for you.
3 |
4 | Notably, the `get_db` and `get_elastic_client` could help with writing an archivy plugin.
5 |
6 | ::: archivy.helpers
7 |
--------------------------------------------------------------------------------
/docs/reference/hooks.md:
--------------------------------------------------------------------------------
1 |
2 | ::: archivy.config.BaseHooks
3 |
--------------------------------------------------------------------------------
/docs/reference/models.md:
--------------------------------------------------------------------------------
1 | Internal API for the models Archivy uses in the backend that could be useful for writing plugins.
2 |
3 | ::: archivy.models
4 | selection:
5 | filters:
6 | - "!__"
7 |
--------------------------------------------------------------------------------
/docs/reference/search.md:
--------------------------------------------------------------------------------
1 |
2 | These are a few methods to interface between archivy and the elasticsearch instance.
3 |
4 | ::: archivy.search
5 |
--------------------------------------------------------------------------------
/docs/reference/web_api.md:
--------------------------------------------------------------------------------
1 |
2 | The Archivy HTTP API allows you to run small scripts in any language that will interact with your archivy instance through HTTP.
3 |
4 | All calls must be first logged in with the [login](/reference/web_api/#archivy.api.login) endpoint.
5 |
6 | ## Small example in Python
7 |
8 | This code uses the `requests` module to interact with the API:
9 |
10 | ```python
11 | import requests
12 | # we create a new session that will allow us to login once
13 | s = requests.session()
14 |
15 | INSTANCE_URL = <your instance url>
16 | s.post(f"{INSTANCE_URL}/api/login", auth=(<username>, <password>))
17 |
18 | # once you've logged in - you can make authenticated requests to the api, like:
19 | resp = s.get(f"{INSTANCE_URL}/api/dataobjs").content)
20 | ```
21 |
22 |
23 | ## Reference
24 |
25 | ::: archivy.api
26 |
--------------------------------------------------------------------------------
/docs/reference/web_inputs.md:
--------------------------------------------------------------------------------
1 | When developing plugins, you may want to use custom HTML input types on the frontend, like `email` or `password`.
2 |
3 | Archivy currently allows you use these two types in your click options.
4 |
5 | For example:
6 |
7 | ```python
8 |
9 | from archivy.click_web.web_click_types import EMAIL_TYPE, PASSWORD_TYPE
10 | @cli.command()
11 | @click.option("--the_email", type=EMAIL_TYPE) # this will validate the email format on the frontend and backend
12 | @click.option("--password", type=PASSWORD_TYPE) # type='password' on the HTML frontend.
13 | def login(the_email, password):
14 | ...
15 | ```
16 |
--------------------------------------------------------------------------------
/docs/requirements.txt:
--------------------------------------------------------------------------------
1 | mkdocs-material
2 | mkdocstrings[python-legacy]
3 | mkdocs==1.2.4
4 | pymdown-extensions
5 |
--------------------------------------------------------------------------------
/docs/setup-search.md:
--------------------------------------------------------------------------------
1 | Archivy supports two search engines:
2 |
3 | 1. [Elasticsearch](https://www.elastic.co/) - an incredibly powerful solution that is however harder to install.
4 | 2. [ripgrep](https://github.com/BurntSushi/ripgrep), much more lightweight but also less powerful.
5 |
6 | These allow archivy to index and provide full-text search on their knowledge bases.
7 |
8 | You can select these in the `archivy init` script, and the [the config docs](config.md) also has more information on this.
9 |
10 | ## Elasticsearch
11 |
12 | Elasticsearch, is a complex and extendable search engine that returns high-quality results.
13 |
14 | Instructions to install and run the service which needs to be running when you use Elasticsearch can be found [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html).
15 |
16 |
17 | Append these two lines to your [elasticsearch.yml config file](https://www.elastic.co/guide/en/elasticsearch/reference/current/settings.html):
18 |
19 | ```yaml
20 | http.cors.enabled: true
21 | http.cors.allow-origin: "http://localhost:5000"
22 | ```
23 |
24 | Then, when you run `archivy init` simply specify you have enabled ES to integrate it with archivy.
25 |
26 | You will now have full-text search on your knowledge base!
27 |
28 | If you're adding ES to an existing knowledge base, use `archivy index` to sync any changes.
29 |
30 | Elasticsearch can be a hefty dependency, so if you have any ideas for something more light-weight that could be used as an alternative, please share on [this thread](https://github.com/archivy/archivy/issues/13).
31 |
32 | ## Ripgrep
33 |
34 | [Ripgrep](https://github.com/BurntSushi/ripgrep), on the other hand, is much more lightweight and small, but is also a bit limited in its functionality.
35 |
36 | Follow [these instructions](https://github.com/BurntSushi/ripgrep#installation) to install ripgrep.
37 |
38 | Then simply specify you want to use it during the `archivy init` script (or edit the config to add it).
39 |
40 |
--------------------------------------------------------------------------------
/docs/usage.md:
--------------------------------------------------------------------------------
1 | Archivy comes with a simple command line interface that you use on the backend to run archivy:
2 |
3 | ```
4 | Usage: archivy [OPTIONS] COMMAND [ARGS]...
5 |
6 | Options:
7 | --version Show the flask version
8 | --help Show this message and exit.
9 |
10 | Commands:
11 | config Open archivy config.
12 | create-admin Creates a new admin user
13 | format Format normal markdown files for archivy.
14 | index Sync content to Elasticsearch
15 | init Initialise your archivy application
16 | run Runs archivy web application
17 | shell Run a shell in the app context.
18 | unformat Convert archivy-formatted files back to normal markdown.
19 | ```
20 |
21 | Make sure you've configured Archivy by running `archivy init`, as outlined in [install](install.md).
22 |
23 | If you'd like to add users, you can simply create new admin users with the `create-admin` command. Only give credentials to trusted people.
24 |
25 | If you have normal md files you'd like to migrate to archivy, move your files into your archivy data directory and then run `archivy format <filenames>` to make them conform to [archivy's formatting](/reference/architecture/#data-storage). Run `archivy unformat` to convert the other way around.
26 |
27 | You can sync changes to files to the Elasticsearch index by running `archivy index` or by simply using the web editor which updates ES when you push a change.
28 |
29 | The `config` command allows you to play around with [configuration](config.md) and use `shell` if you'd like to play around with the archivy python API.
30 |
31 | You can then use archivy to create notes, bookmarks and to organize and store information.
32 |
33 | The [web api](reference/web_api.md) is also useful to extend archivy, or [plugins](plugins.md).
34 |
35 | These have been recently introduced, but you can check the existing plugins that you can install onto your instance [here](https://github.com/archivy/awesome-archivy).
36 |
--------------------------------------------------------------------------------
/docs/whats_next.md:
--------------------------------------------------------------------------------
1 |
2 | Now that you have a working installation and you know how to use, where can you go from here?
3 |
4 | If you'd like to write a plugin for archivy with standalone functionality you'd like to see, check out the [plugin tutorial](plugins.md). You can also use our [Web API](reference/web_api.md)!
5 |
6 | If you notice **any** bugs or problems, or if you have any features you'd like to see, **please** open up an issue on [GitHub](https://github.com/archivy/archivy).
7 |
8 | You can also directly talk to us on [the archivy discord server](https://discord.gg/uQsqyxB)!
9 |
10 |
--------------------------------------------------------------------------------
/mkdocs.yml:
--------------------------------------------------------------------------------
1 | site_name: Archivy
2 | theme:
3 | name: material
4 | logo: img/logo-2.png
5 | favicon: img/logo-2.png
6 | features:
7 | - navigation.tabs
8 | custom_dir: docs/overrides
9 |
10 |
11 | plugins:
12 | - mkdocstrings
13 | - search
14 |
15 |
16 | markdown_extensions:
17 | - pymdownx.highlight
18 | - pymdownx.superfences
19 | - pymdownx.tasklist:
20 | clickable_checkbox: true
21 | - footnotes
22 |
23 | nav:
24 | - Home:
25 | - "index.md"
26 | - What makes Archivy different: "difference.md"
27 | - Architecture: "reference/architecture.md"
28 | - Contributing: "CONTRIBUTING.md"
29 | - Getting Started:
30 | - Installing Archivy: "install.md"
31 | - Usage: "usage.md"
32 | - Search: "setup-search.md"
33 | - Config: "config.md"
34 | - Editing: "editing.md"
35 | - What's Next: "whats_next.md"
36 | - Plugins:
37 | - Index: "plugins.md"
38 | - API Reference:
39 | - Architecture: "reference/architecture.md"
40 | - Web API: "reference/web_api.md"
41 | - Models for User and DataObj: "reference/models.md"
42 | - Dataobj Filesystem Layer: "reference/filesystem_layer.md"
43 | - Helpers: "reference/helpers.md"
44 | - Hooks: "reference/hooks.md"
45 | - Search: "reference/search.md"
46 | - Web Inputs For Plugins: "reference/web_inputs.md"
47 |
48 | repo_url: https://github.com/archivy/archivy
49 | repo_name: archivy
50 |
--------------------------------------------------------------------------------
/plugins.md:
--------------------------------------------------------------------------------
1 | # Plugins
2 |
3 | Plugins are a newly introduced method to add extensions to the archivy cli and web interface. It relies on the extremely useful [click-plugins](https://github.com/click-contrib/click-plugins) package that is loaded through pip and the [click-web](https://github.com/click-contrib/click-plugins) which has been modified and whose can be found in `archivy/click_web/`, some of the tests, templates and static files.
4 |
5 |
6 | To help you understand the way the plugin system works, we're going to build our own plugin that allows users to sync content from `pocket`. We'll even deploy it to Pypi so that other people can install it.
7 |
8 |
9 | Prerequisites: A python and pip installation with archivy.
10 |
11 | ## Step 1: Defining what our archivy extension does
12 |
13 | `archivy` source used to have a built-in feature that allowed you to sync up to the bookmarks of your pocket account. We removed this and prefer to replace it with a standalone plugin, so let's build it!
14 |
15 | What it will do is allow a user to download their bookmarks from pocket, without redownloading content that already exists.
16 |
17 | ## Step 2: Setting up the project
18 |
19 | Make a new directory named `archivy_pocket` wherever you want on your system and create a new [`setup.py`](https://stackoverflow.com/questions/1471994/what-is-setup-py) file that will define the characteristics of our package.
20 |
21 | This is what it looks like:
22 |
23 | ```python
24 |
25 | from setuptools import setup, find_packages
26 |
27 | with open("README.md", "r") as fh:
28 | long_description = fh.read()
29 |
30 | setup(
31 | name='archivy_pocket',
32 | version='0.1.0',
33 | author="Uzay-G",
34 | author_email="halcyon@disroot.org",
35 | description=(
36 | "Archivy extension to sync content to your pocket account."
37 | ),
38 | long_description=long_description,
39 | long_description_content_type="text/markdown",
40 | classifiers=[
41 | "Programming Language :: Python :: 3",
42 | ],
43 | packages=find_packages(),
44 | entry_points='''
45 | [archivy.plugins]
46 | pocket=archivy_pocket:pocket
47 | '''
48 | )
49 | ```
50 |
51 | Let's walk through what this is doing. We are setting up our package and we define a few characteristics of the package. We specify some metadata you can adapt to your own package. We then load our package by using the `find_packages` function. The `entry_points` part is the most important. The `[archivy.plugins]` tells archivy that this package will extend our CLI and then we define the command we want to add. In our case, people will call the extension using `archivy pocket`. We will actually be creating a group so that people will call subcommands like this: `archivy pocket <subcommand>`. You can do things either way.
52 |
53 | Create a `README.md` file that you can keep empty for now but should haCve information on your project.
54 |
55 | ## Step 3: Writing the core code of our plugin
56 |
57 | Create an `archivy_web` directory inside the current directory where `setup.py` is stored. Create an `__init__.py` file in that directory where we'll store our main project code. For larger projects, it's better to separate concerns but that'll be good for now.
58 |
59 | This is what the skeleton of our code looks will look like.
60 |
61 | ```python
62 | import click # the package that manages the cli
63 |
64 | @click.group()
65 | def pocket():
66 | pass
67 |
68 | @pocket.command()
69 | def command1():
70 | ...
71 |
72 | @pocket.command()
73 | def command2():
74 | ...
75 | ```
76 |
77 | With this structure, you'll be able to call `archivy pocket command1` and `archivy pocket command2`. Read the [click docs](https://click.palletsprojects.com/en/7.x/options/) to learn about how to build more intricate
78 |
79 | Let's get into actually writing a command that interacts with the archivy codebase.
80 |
81 | The code below does a few things:
82 |
83 | - It imports the archivy `app` that basically is the interface for the webserver and many essential Flask features (flask is the web framework archivy uses).
84 | - It imports the `get_db` function that allows us to access and modify the db.
85 | - We define our pocket group.
86 | - We create a new command from that group, what's important here is the `with app.app_context` part. We need to run our code inside the archivy `app_context` to be able to call some of the archivy methods. If you call archivy methods in your plugins, it might fail if you don't include this part.
87 | - Then we just have our command code.
88 |
89 |
90 | ```python
91 | import click
92 | from archivy.extensions import get_db
93 | from archivy import app
94 | from archivy.data import get_items
95 |
96 | @click.group()
97 | def pocket():
98 | pass
99 |
100 | @pocket.command()
101 | @click.argument("api_key")
102 | def auth(api_key):
103 | with app.app_context():
104 | db = get_db()
105 | # ...
106 | ```
107 |
108 |
109 | We also added some other commands, but we'll skip them for brevity and you can check out the source code [here](https://github.com/archivy/archivy_pocket).
110 |
111 | Now you just need to do `pip install .` in the main directory and you'll have access to the commands. Check it out by running `archivy --help`.
112 |
113 | ## Step 4: Publishing our package to Pypi
114 |
115 | [Pypi](https://pypi.org) is the Python package repository. Publishing our package to it will allow other users to easily install our code onto their own archivy instance.
116 |
117 | This is a short overview. Check out [this website](https://packaging.python.org/) for more info.
118 |
119 | This section is inspired by [this](https://packaging.python.org/tutorials/packaging-projects/#installing-your-newly-uploaded-package).
120 |
121 |
122 | Make sure the required utilities are installed:
123 |
124 | ```python
125 | python3 -m pip install --user --upgrade setuptools wheel
126 | ```
127 |
128 | Now run this command in the main dir to build the source:
129 |
130 | ```python
131 | python3 setup.py sdist bdist_wheel
132 | ```
133 |
134 | Now you need to create an account on [Pypi](https://pypi.org). Then go [here](https://pypi.org/manage/account/#api-tokens) and create a new API token; don’t limit its scope to a particular project, since you are creating a new project.
135 |
136 |
137 | Once you've saved your token, install `twine`, the program that will take care of the upload:
138 |
139 | ```python
140 | python3 -m pip install --user --upgrade twine
141 | ```
142 |
143 | And you can finally upload your code! The username you should enter is `__token__` and then the password is your API token.
144 |
145 | ```python
146 | python3 -m twine upload dist/*
147 | ```
148 |
149 |
--------------------------------------------------------------------------------
/requirements-tests.txt:
--------------------------------------------------------------------------------
1 | -r requirements.txt
2 |
3 | pytest==6.0.2
4 | pytest-cov
5 | # for mocking out requests HTTP calls
6 | responses==0.12.0
7 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | # Automatically generated by https://github.com/damnever/pigar.
2 |
3 | # archivy/__init__.py: 7
4 | # archivy/check_changes.py: 5
5 | # archivy/data.py: 9
6 | # archivy/extensions.py: 6
7 | # archivy/models.py: 13
8 | # archivy/routes.py: 7
9 | Flask == 2.3.2
10 | werkzeug == 2.3.3
11 | jinja2 == 3.1.2
12 |
13 | # archivy/forms.py: 2
14 | Flask_WTF == 1.1.1
15 |
16 | # archivy/forms.py: 4
17 | WTForms == 2.3.1
18 |
19 | # archivy/config.py: 2
20 | appdirs == 1.4.4
21 |
22 | # archivy/models.py: 10
23 | attrs == 20.2.0
24 |
25 | # archivy/models.py: 11
26 | beautifulsoup4 >= 4.8.2
27 |
28 |
29 | # archivy/__init__.py: 6
30 | # archivy/extensions.py: 4,5
31 | elasticsearch == 7.7.1
32 |
33 |
34 | # archivy/run.py: 1
35 | python_dotenv == 0.13.0
36 |
37 | # archivy/data.py: 8
38 | # archivy/models.py: 5
39 |
40 | python_frontmatter == 0.5.0
41 |
42 | # archivy/models.py: 7
43 | # archivy/routes.py: 4
44 | requests == 2.30.0
45 |
46 | # archivy/extensions.py: 7
47 | # archivy/routes.py: 5
48 | tinydb == 4.1.1
49 |
50 | # archivy/models.py: 8
51 | validators == 0.15.0
52 |
53 | # archivy/__init__.py: 9
54 | # archivy/models.py: 13
55 | # archivy/routes.py: 10
56 | flask-login == 0.6.2
57 |
58 | click_plugins
59 | html2text
60 | flask_compress
61 | readability-lxml
62 |
63 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | import setuptools
2 |
3 | with open("README.md", "r") as fh:
4 | long_description = fh.read()
5 |
6 | with open("requirements.txt", encoding="utf-8") as f:
7 | all_reqs = f.read().split("\n")
8 | install_requires = [
9 | x.strip()
10 | for x in all_reqs
11 | if not x.startswith("#") and not x.startswith("-e git")
12 | ]
13 |
14 | setuptools.setup(
15 | name="archivy",
16 | version="1.7.7",
17 | author="Uzay-G",
18 | author_email="halcyon@disroot.org",
19 | description=(
20 | "Minimalist knowledge base focused on digital preservation"
21 | " and building your second brain."
22 | ),
23 | long_description=long_description,
24 | long_description_content_type="text/markdown",
25 | url="https://github.com/archivy/archivy",
26 | packages=setuptools.find_packages(),
27 | classifiers=[
28 | "Programming Language :: Python :: 3",
29 | "License :: OSI Approved :: MIT License",
30 | ],
31 | entry_points={
32 | "console_scripts": [
33 | "archivy = archivy.cli:cli",
34 | ]
35 | },
36 | include_package_data=True,
37 | zip_safe=False,
38 | install_requires=install_requires,
39 | python_requires=">=3.6",
40 | )
41 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/tests/__init__.py
--------------------------------------------------------------------------------
/tests/functional/test_plugins.py:
--------------------------------------------------------------------------------
1 | import re
2 |
3 | from flask.testing import FlaskClient
4 | from flask import request
5 | from flask_login import current_user
6 |
7 | from responses import RequestsMock, GET
8 | from archivy.helpers import get_max_id, get_db
9 |
10 |
11 | def test_plugin_index(test_app, client: FlaskClient):
12 | resp = client.get("/plugins")
13 | assert resp.status_code == 200
14 | assert b"Plugins" in resp.data
15 |
16 | # random number is one of the seeded plugins we added for testing
17 | assert b"random-number" in resp.data
18 |
19 |
20 | def test_get_command_form(test_app, client: FlaskClient):
21 | cmd_path = "/cli/test-plugin/random-number"
22 | resp = client.get(cmd_path)
23 |
24 | command_input = b"2.0.argument.text.1.text.upper-bound"
25 | command_name = b"Random-Number"
26 |
27 | assert command_input in resp.data
28 | assert command_name in resp.data
29 |
30 |
31 | def test_exec_random_command(test_app, client: FlaskClient):
32 | cmd_path = "/cli/test-plugin/random-number"
33 | upper_bound = 10
34 | cmd_data = {"2.0.argument.text.1.text.upper-bound": upper_bound}
35 |
36 | resp = client.post(cmd_path, data=cmd_data)
37 |
38 | assert resp.status_code == 200
39 | # check for random number <= 10 in response
40 | assert re.match("(0*(?:[1-9]?|10))", str(resp.data))
41 |
42 |
43 | def test_exec_command_app_context(
44 | test_app, note_fixture, bookmark_fixture, client: FlaskClient
45 | ):
46 | cmd_path = "/cli/test-plugin/get-random-dataobj-title"
47 | resp = client.post(cmd_path)
48 |
49 | assert resp.status_code == 200
50 | assert b"Test Note" or b"Example" in resp.data
51 |
--------------------------------------------------------------------------------
/tests/test_click_web.py:
--------------------------------------------------------------------------------
1 | import pprint
2 | from pathlib import Path
3 |
4 | import click
5 | import pytest
6 |
7 | from archivy import click_web, cli
8 | from archivy.click_web.resources import cmd_form, input_fields
9 |
10 |
11 | @pytest.mark.parametrize(
12 | "param, command_index, expected",
13 | [
14 | (
15 | click.Argument(
16 | [
17 | "an_argument",
18 | ]
19 | ),
20 | 0,
21 | {
22 | "checked": "",
23 | "click_type": "text",
24 | "help": "",
25 | "human_readable_name": "AN ARGUMENT",
26 | "name": "0.0.argument.text.1.text.an-argument",
27 | "nargs": 1,
28 | "param": "argument",
29 | "required": True,
30 | "type": "text",
31 | "value": None,
32 | },
33 | ),
34 | (
35 | click.Argument(
36 | [
37 | "an_argument",
38 | ],
39 | nargs=2,
40 | ),
41 | 1,
42 | {
43 | "checked": "",
44 | "click_type": "text",
45 | "help": "",
46 | "human_readable_name": "AN ARGUMENT",
47 | "name": "1.0.argument.text.2.text.an-argument",
48 | "nargs": 2,
49 | "param": "argument",
50 | "required": True,
51 | "type": "text",
52 | "value": None,
53 | },
54 | ),
55 | (
56 | click.Option(
57 | [
58 | "--an_option",
59 | ]
60 | ),
61 | 0,
62 | {
63 | "checked": "",
64 | "click_type": "text",
65 | "desc": None,
66 | "help": ("--an_option TEXT", ""),
67 | "human_readable_name": "an option",
68 | "name": "0.0.option.text.1.text.--an-option",
69 | "nargs": 1,
70 | "param": "option",
71 | "required": False,
72 | "type": "text",
73 | "value": "",
74 | },
75 | ),
76 | (
77 | click.Option(
78 | [
79 | "--an_option",
80 | ],
81 | nargs=2,
82 | ),
83 | 1,
84 | {
85 | "checked": "",
86 | "click_type": "text",
87 | "desc": None,
88 | "help": ("--an_option TEXT...", ""),
89 | "human_readable_name": "an option",
90 | "name": "1.0.option.text.2.text.--an-option",
91 | "nargs": 2,
92 | "param": "option",
93 | "required": False,
94 | "type": "text",
95 | "value": "",
96 | },
97 | ),
98 | (
99 | click.Option(
100 | [
101 | "--flag/--no-flag",
102 | ],
103 | default=True,
104 | help="help",
105 | ),
106 | 3,
107 | {
108 | "checked": 'checked="checked"',
109 | "click_type": "bool_flag",
110 | "desc": "help",
111 | "help": ("--flag / --no-flag", "help"),
112 | "human_readable_name": "flag",
113 | "name": "3.0.flag.bool_flag.1.checkbox.--flag",
114 | "nargs": 1,
115 | "off_flag": "--no-flag",
116 | "on_flag": "--flag",
117 | "param": "option",
118 | "required": False,
119 | "type": "checkbox",
120 | "value": "--flag",
121 | },
122 | ),
123 | ],
124 | )
125 | def test_get_input_field(ctx, click_cli, param, expected, command_index):
126 | res = input_fields.get_input_field(ctx, param, command_index, 0)
127 | pprint.pprint(res)
128 | assert res == expected
129 |
130 |
131 | @pytest.mark.parametrize(
132 | "param, command_index, expected",
133 | [
134 | (
135 | click.Argument(
136 | [
137 | "an_argument",
138 | ],
139 | nargs=-1,
140 | ),
141 | 0,
142 | {
143 | "checked": "",
144 | "click_type": "text",
145 | "help": "",
146 | "human_readable_name": "AN ARGUMENT",
147 | "name": "0.0.argument.text.-1.text.an-argument",
148 | "nargs": -1,
149 | "param": "argument",
150 | "required": False,
151 | "type": "text",
152 | "value": None,
153 | },
154 | ),
155 | ],
156 | )
157 | def test_variadic_arguments(ctx, click_cli, param, expected, command_index):
158 | res = input_fields.get_input_field(ctx, param, command_index, 0)
159 | pprint.pprint(res)
160 | assert res == expected
161 |
162 |
163 | @pytest.mark.parametrize(
164 | "param, command_index, expected",
165 | [
166 | (
167 | click.Argument(
168 | [
169 | "a_file_argument",
170 | ],
171 | type=click.File("rb"),
172 | ),
173 | 0,
174 | {
175 | "checked": "",
176 | "click_type": "file[rb]",
177 | "help": "",
178 | "human_readable_name": "A FILE ARGUMENT",
179 | "name": "0.0.argument.file[rb].1.file.a-file-argument",
180 | "nargs": 1,
181 | "param": "argument",
182 | "required": True,
183 | "type": "file",
184 | "value": None,
185 | },
186 | ),
187 | (
188 | click.Option(
189 | [
190 | "--a_file_option",
191 | ],
192 | type=click.File("rb"),
193 | ),
194 | 0,
195 | {
196 | "checked": "",
197 | "click_type": "file[rb]",
198 | "desc": None,
199 | "help": ("--a_file_option FILENAME", ""),
200 | "human_readable_name": "a file option",
201 | "name": "0.0.option.file[rb].1.file.--a-file-option",
202 | "nargs": 1,
203 | "param": "option",
204 | "required": False,
205 | "type": "file",
206 | "value": "",
207 | },
208 | ),
209 | ],
210 | )
211 | def test_get_file_input_field(ctx, click_cli, param, expected, command_index):
212 | res = input_fields.get_input_field(ctx, param, command_index, 0)
213 | pprint.pprint(res)
214 | assert res == expected
215 |
216 |
217 | @pytest.mark.parametrize(
218 | "param, command_index, expected",
219 | [
220 | (
221 | click.Argument(
222 | [
223 | "a_file_argument",
224 | ],
225 | type=click.File("wb"),
226 | ),
227 | 0,
228 | {
229 | "checked": "",
230 | "click_type": "file[wb]",
231 | "help": "",
232 | "human_readable_name": "A FILE ARGUMENT",
233 | "name": "0.0.argument.file[wb].1.hidden.a-file-argument",
234 | "nargs": 1,
235 | "param": "argument",
236 | "required": True,
237 | "type": "hidden",
238 | "value": None,
239 | },
240 | ),
241 | (
242 | click.Option(
243 | [
244 | "--a_file_option",
245 | ],
246 | type=click.File("wb"),
247 | ),
248 | 0,
249 | {
250 | "checked": "",
251 | "click_type": "file[wb]",
252 | "desc": None,
253 | "help": ("--a_file_option FILENAME", ""),
254 | "human_readable_name": "a file option",
255 | "name": "0.0.option.file[wb].1.text.--a-file-option",
256 | "nargs": 1,
257 | "param": "option",
258 | "required": False,
259 | "type": "text",
260 | "value": "",
261 | },
262 | ),
263 | ],
264 | )
265 | def test_get_output_file_input_field(ctx, click_cli, param, expected, command_index):
266 | res = input_fields.get_input_field(ctx, param, command_index, 0)
267 | pprint.pprint(res)
268 | assert res == expected
269 |
--------------------------------------------------------------------------------
/tests/test_plugin/plugin.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python3
2 | from random import randint
3 |
4 | import click
5 |
6 | from archivy import app
7 | from archivy.helpers import get_max_id
8 | from archivy.data import get_items
9 |
10 |
11 | @click.group()
12 | def test_plugin():
13 | pass
14 |
15 |
16 | @test_plugin.command()
17 | @click.argument("upper_bound")
18 | def random_number(upper_bound):
19 | click.echo(randint(1, int(upper_bound)))
20 |
21 |
22 | @test_plugin.command()
23 | def get_random_dataobj_title():
24 | with app.app_context():
25 | dataobjs = get_items(structured=False)
26 | click.echo(dataobjs[randint(0, len(dataobjs))]["title"])
27 |
--------------------------------------------------------------------------------
/tests/test_plugin/setup.py:
--------------------------------------------------------------------------------
1 | from setuptools import setup
2 |
3 |
4 | setup(
5 | name="plugins2",
6 | version="0.1dev0",
7 | py_modules=["plugin"],
8 | entry_points="""
9 | [archivy.plugins]
10 | test_plugin=plugin:test_plugin
11 | """,
12 | )
13 |
--------------------------------------------------------------------------------
/tests/test_request_parsing.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/archivy/archivy/bdcdd39ac6cf9f7b3709b984d8be2f0fa898139e/tests/test_request_parsing.py
--------------------------------------------------------------------------------
/tests/unit/test_advanced_conf.py:
--------------------------------------------------------------------------------
1 | from textwrap import dedent
2 |
3 | import pytest
4 | from tinydb import Query
5 |
6 | from archivy.helpers import get_db, load_hooks, load_scraper
7 | from archivy import data
8 |
9 |
10 | @pytest.fixture()
11 | def hooks_cli_runner(test_app, cli_runner, click_cli):
12 | """
13 | Saves hooks to user config directory for tests.
14 |
15 | All of the hooks except `before_dataobj_create` store some form of message in
16 | the db, whose existence is then checked in the tests.
17 | """
18 | hookfile = """\
19 | from archivy.config import BaseHooks
20 | from archivy.helpers import get_db
21 |
22 | class Hooks(BaseHooks):
23 | def on_edit(self, dataobj):
24 | get_db().insert({"type": "edit_message", "content": f"Changes made to content of {dataobj.title}."})
25 | def on_user_create(self, user):
26 | get_db().insert({"type": "user_creation_message", "content": f"New user {user.username} created."})
27 | def on_dataobj_create(self, dataobj):
28 | get_db().insert({"type": "dataobj_creation_message", "content": f"New dataobj on {dataobj.title} with tags: {dataobj.tags}"})
29 | def before_dataobj_create(self, dataobj):
30 | dataobj.content += "Dataobj made for test." """
31 | with cli_runner.isolated_filesystem():
32 | cli_runner.invoke(click_cli, ["init"], input="\nn\nn\n\n")
33 | with open("hooks.py", "w") as f:
34 | f.write(dedent(hookfile))
35 | with test_app.app_context():
36 | test_app.config["HOOKS"] = load_hooks()
37 | yield cli_runner
38 |
39 |
40 | @pytest.fixture()
41 | def custom_scraping_setup(test_app, cli_runner, click_cli):
42 | scraping_file = """\
43 | def test_pattern(data):
44 | data.title = "Overridden note"
45 | data.content = "this note was not processed by default archivy bookmarking, but a user-specified function"
46 | data.tags = ["test"]
47 |
48 | PATTERNS = {
49 | "https://example.com/": test_pattern,
50 | "https://example2.com/": ".nested"
51 | }"""
52 |
53 | with cli_runner.isolated_filesystem():
54 | cli_runner.invoke(click_cli, ["init"], input="\nn\nn\n\n")
55 | with open("scraping.py", "w") as f:
56 | f.write(dedent(scraping_file))
57 | with test_app.app_context():
58 | test_app.config["SCRAPING_PATTERNS"] = load_scraper()
59 | yield cli_runner
60 |
61 |
62 | def test_dataobj_creation_hook(test_app, hooks_cli_runner, note_fixture):
63 | creation_message = get_db().search(Query().type == "dataobj_creation_message")[0]
64 | assert (
65 | creation_message["content"]
66 | == f"New dataobj on {note_fixture.title} with tags: {note_fixture.tags}"
67 | )
68 |
69 |
70 | def test_before_dataobj_creation_hook(
71 | test_app, hooks_cli_runner, note_fixture, bookmark_fixture
72 | ):
73 | # check hook that added content at the end of body succeeded.
74 | message = "Dataobj made for test."
75 | assert message in note_fixture.content
76 | assert message in bookmark_fixture.content
77 |
78 |
79 | def test_dataobj_edit_hook(test_app, hooks_cli_runner, note_fixture, client):
80 | client.put(
81 | f"/api/dataobjs/{note_fixture.id}", json={"content": "Updated note content"}
82 | )
83 |
84 | edit_message = get_db().search(Query().type == "edit_message")[0]
85 | assert (
86 | f"Changes made to content of {note_fixture.title}." == edit_message["content"]
87 | )
88 |
89 |
90 | def test_user_creation_hook(test_app, hooks_cli_runner, user_fixture):
91 | creation_message = get_db().search(Query().type == "user_creation_message")[1]
92 | assert f"New user {user_fixture.username} created." == creation_message["content"]
93 |
94 |
95 | def test_custom_scraping_patterns(
96 | custom_scraping_setup, test_app, bookmark_fixture, different_bookmark_fixture
97 | ):
98 | pattern = "example.com"
99 | assert pattern in bookmark_fixture.url
100 | assert bookmark_fixture.title == "Overridden note"
101 | assert bookmark_fixture.tags == ["test"]
102 | pattern = "example2.com"
103 | assert pattern in different_bookmark_fixture.url
104 | # check that the CSS selector was parsed and other parts of the document were not selected
105 | assert different_bookmark_fixture.content.startswith("aaa")
106 | test_app.config["SCRAPING_PATTERNS"] = {}
107 |
--------------------------------------------------------------------------------
/tests/unit/test_cli.py:
--------------------------------------------------------------------------------
1 | import os
2 | from pathlib import Path
3 | from tempfile import mkdtemp
4 |
5 | from tinydb import Query
6 |
7 | from archivy.cli import cli
8 | from archivy.helpers import get_db
9 | from archivy.models import DataObj
10 | from archivy.data import get_items, create_dir, get_data_dir
11 |
12 |
13 | def test_initialization(test_app, cli_runner, click_cli):
14 | conf_path = os.path.join(test_app.config["USER_DIR"], "config.yml")
15 | try:
16 | # conf shouldn't exist
17 | open(conf_path)
18 | assert False
19 | except FileNotFoundError:
20 | pass
21 | old_data_dir = test_app.config["USER_DIR"]
22 |
23 | with cli_runner.isolated_filesystem():
24 | # create user, localhost, and don't use ES
25 | res = cli_runner.invoke(
26 | click_cli, ["init"], input="\nn\ny\nusername\npassword\npassword\n\n"
27 | )
28 | assert "Config successfully created" in res.output
29 |
30 | # verify user was created
31 | assert len(
32 | get_db().search(Query().type == "user" and Query().username == "username")
33 | )
34 |
35 | # verify dataobj creation works
36 | assert DataObj(type="note", title="Test note").insert()
37 | assert len(get_items(structured=False)) == 1
38 |
39 | conf = open(conf_path).read()
40 |
41 | # assert defaults are saved
42 | assert f"USER_DIR: {test_app.config['USER_DIR']}" in conf
43 | assert "HOST: 127.0.0.1"
44 | # check ES config not saved
45 | assert "ELASTICSEARCH" not in conf
46 |
47 | # check initialization in random directory
48 | # has resulted in change of user dir
49 | assert old_data_dir != test_app.config["USER_DIR"]
50 |
51 |
52 | def test_initialization_with_es(test_app, cli_runner, click_cli):
53 | conf_path = os.path.join(test_app.config["USER_DIR"], "config.yml")
54 | old_data_dir = test_app.config["USER_DIR"]
55 |
56 | with cli_runner.isolated_filesystem():
57 | # use ES, localhost and don't create user
58 | res = cli_runner.invoke(click_cli, ["init"], input="\ny\nelasticsearch\nn\n\n")
59 |
60 | assert "Config successfully created" in res.output
61 | conf = open(conf_path).read()
62 |
63 | # assert ES Config is saved
64 | assert "SEARCH_CONF" in conf
65 | assert "enabled: 1" in conf
66 |
67 | # however, check that defaults are not saved, as they have not been modified:
68 | assert not "es_password" in conf
69 | assert not "EDITOR_CONF" in conf
70 |
71 | # check initialization in random directory
72 | # has resulted in change of user dir
73 | assert old_data_dir != test_app.config["USER_DIR"]
74 |
75 |
76 | def test_initialization_with_ripgrep(test_app, cli_runner, click_cli):
77 | conf_path = os.path.join(test_app.config["USER_DIR"], "config.yml")
78 | old_data_dir = test_app.config["USER_DIR"]
79 |
80 | with cli_runner.isolated_filesystem():
81 | # use ES, localhost and don't create user
82 | res = cli_runner.invoke(click_cli, ["init"], input="\ny\nripgrep\nn\n\n")
83 |
84 | assert "Config successfully created" in res.output
85 | conf = open(conf_path).read()
86 |
87 | # assert search Config is saved
88 | assert "SEARCH_CONF" in conf
89 | assert "enabled: 1" in conf
90 | assert "ripgrep" in conf
91 |
92 |
93 | def test_initialization_in_diff_than_curr_dir(test_app, cli_runner, click_cli):
94 | conf_path = os.path.join(test_app.config["USER_DIR"], "config.yml")
95 | data_dir = mkdtemp()
96 |
97 | with cli_runner.isolated_filesystem():
98 | # input data dir - localhost - don't use ES and don't create user
99 | res = cli_runner.invoke(cli, ["init"], input=f"{data_dir}\nn\nn\n\n")
100 |
101 | assert "Config successfully created" in res.output
102 | conf = open(conf_path).read()
103 |
104 | assert f"USER_DIR: {data_dir}" in conf
105 |
106 | # check initialization in random directory
107 | # has resulted in change of user dir
108 | assert data_dir == test_app.config["USER_DIR"]
109 |
110 | # verify dataobj creation works
111 | assert DataObj(type="note", title="Test note").insert()
112 | assert len(get_items(structured=False)) == 1
113 |
114 |
115 | def test_initialization_custom_host(test_app, cli_runner, click_cli):
116 | conf_path = os.path.join(test_app.config["USER_DIR"], "config.yml")
117 | try:
118 | # conf shouldn't exist
119 | open(conf_path)
120 | assert False
121 | except FileNotFoundError:
122 | pass
123 |
124 | with cli_runner.isolated_filesystem():
125 | # create user, localhost, and don't use ES
126 | res = cli_runner.invoke(click_cli, ["init"], input="\nn\nn\n0.0.0.0")
127 | assert "Host" in res.output
128 | assert "Config successfully created" in res.output
129 |
130 | conf = open(conf_path).read()
131 |
132 | # assert defaults are saved
133 | print(res.output)
134 | assert f"HOST: 0.0.0.0" in conf
135 |
136 |
137 | def test_create_admin(test_app, cli_runner, click_cli):
138 | db = get_db()
139 | nb_users = len(db.search(Query().type == "user"))
140 | cli_runner.invoke(
141 | click_cli, ["create-admin", "__username__"], input="password\npassword"
142 | )
143 |
144 | # need to reconnect to db because it has been modified by different processes
145 | # so the connection needs to be updated for new changes
146 | db = get_db(force_reconnect=True)
147 | assert nb_users + 1 == len(db.search(Query().type == "user"))
148 | assert len(db.search(Query().type == "user" and Query().username == "__username__"))
149 |
150 |
151 | def test_create_admin_small_password_fails(test_app, cli_runner, click_cli):
152 | cli_runner.invoke(click_cli, ["create-admin", "__username__"], input="short\nshort")
153 | db = get_db()
154 | assert not len(
155 | db.search(Query().type == "user" and Query().username == "__username__")
156 | )
157 |
158 |
159 | def test_format_multiple_md_file(test_app, cli_runner, click_cli):
160 | with cli_runner.isolated_filesystem():
161 | files = ["test-note-1.md", "test-note-2.md"]
162 | for filename in files:
163 | with open(filename, "w") as f:
164 | f.write("Unformatted Test Content")
165 |
166 | res = cli_runner.invoke(cli, ["format"] + files)
167 |
168 | for filename in files:
169 | assert f"Formatted and moved {filename}" in res.output
170 |
171 |
172 | def test_format_entire_directory(test_app, cli_runner, click_cli):
173 | with cli_runner.isolated_filesystem():
174 | files = [
175 | "unformatted/test-note-1.md",
176 | "unformatted/test-note-2.md",
177 | "unformatted/nested/test-note-3.md",
178 | ]
179 | os.mkdir("data")
180 | os.mkdir("data/unformatted")
181 | os.mkdir("data/unformatted/nested")
182 |
183 | for filename in files:
184 | with open("data/" + filename, "w") as f:
185 | f.write("Unformatted Test Content")
186 |
187 | # set user_dir to current dir by configuring
188 | res = cli_runner.invoke(cli, ["init"], input="\nn\nn\n\n\n")
189 |
190 | # format directory
191 | res = cli_runner.invoke(cli, ["format", "data/unformatted/"])
192 |
193 | for filename in files:
194 | # assert directory files got moved correctly
195 | assert f"Formatted and moved {filename}" in res.output
196 | assert (os.path.abspath("") + "/data/unformatted") in res.output
197 |
198 |
199 | def test_unformat_multiple_md_file(
200 | test_app, cli_runner, click_cli, bookmark_fixture, note_fixture
201 | ):
202 | out_dir = mkdtemp()
203 | create_dir("")
204 | res = cli_runner.invoke(
205 | cli,
206 | [
207 | "unformat",
208 | str(bookmark_fixture.fullpath),
209 | str(note_fixture.fullpath),
210 | out_dir,
211 | ],
212 | )
213 | assert (
214 | f"Unformatted and moved {bookmark_fixture.fullpath} to {out_dir}/{bookmark_fixture.title}"
215 | in res.output
216 | )
217 | assert (
218 | f"Unformatted and moved {note_fixture.fullpath} to {out_dir}/{note_fixture.title}"
219 | in res.output
220 | )
221 |
222 |
223 | def test_unformat_directory(
224 | test_app, cli_runner, click_cli, bookmark_fixture, note_fixture
225 | ):
226 | out_dir = mkdtemp()
227 |
228 | # create directory to store archivy note
229 | note_dir = "note-dir"
230 | create_dir(note_dir)
231 | nested_note = DataObj(type="note", title="Nested note", path=note_dir)
232 | nested_note.insert()
233 |
234 | # unformat directory
235 | res = cli_runner.invoke(
236 | cli, ["unformat", os.path.join(get_data_dir(), note_dir), out_dir]
237 | )
238 | assert (
239 | f"Unformatted and moved {nested_note.fullpath} to {out_dir}/{note_dir}/{nested_note.title}"
240 | in res.output
241 | )
242 |
243 |
244 | def test_create_plugin_dir(test_app, cli_runner, click_cli):
245 | with cli_runner.isolated_filesystem():
246 | res = cli_runner.invoke(cli, ["plugin-new", "archivy_test_plugin"])
247 | plugin_dir = Path("archivy_test_plugin")
248 | assert plugin_dir.exists()
249 | files = [
250 | "README.md",
251 | "requirements.txt",
252 | "archivy_test_plugin/__init__.py",
253 | "setup.py",
254 | ]
255 | for file in files:
256 | assert (plugin_dir / file).exists()
257 |
--------------------------------------------------------------------------------
/tests/unit/test_models.py:
--------------------------------------------------------------------------------
1 | from pathlib import Path
2 |
3 | import frontmatter
4 |
5 | from archivy.models import DataObj
6 | from archivy.helpers import get_max_id
7 | from responses import GET
8 |
9 | from archivy.models import DataObj
10 |
11 | attributes = ["type", "title", "tags", "path", "id"]
12 |
13 |
14 | def test_new_bookmark(test_app):
15 | bookmark = DataObj(
16 | type="bookmark",
17 | tags=["example"],
18 | url="http://example.org",
19 | )
20 | bookmark.process_bookmark_url()
21 | bookmark_id = bookmark.insert()
22 | assert bookmark_id == 1
23 |
24 |
25 | def test_new_note(test_app, note_fixture):
26 | """
27 | Check that a new note is correctly saved into the filesystem
28 | with the right attributes and the right id.
29 | """
30 |
31 | with test_app.app_context():
32 | max_id = get_max_id()
33 | assert note_fixture.id == max_id
34 |
35 | saved_file = frontmatter.load(note_fixture.fullpath)
36 | for attr in attributes:
37 | assert getattr(note_fixture, attr) == saved_file[attr]
38 |
39 |
40 | def test_bookmark_sanitization(test_app, client, mocked_responses, bookmark_fixture):
41 | """Test that bookmark content is correctly saved as correct Markdown"""
42 |
43 | with test_app.app_context():
44 | assert bookmark_fixture.id == get_max_id()
45 |
46 | saved_file = frontmatter.load(bookmark_fixture.fullpath)
47 | for attr in attributes:
48 | assert getattr(bookmark_fixture, attr) == saved_file[attr]
49 | assert bookmark_fixture.url == saved_file["url"]
50 |
51 | # remove buggy newlines that interfere with checks:
52 | bookmark_fixture.content = bookmark_fixture.content.replace("\n", "")
53 | # test script is sanitized
54 | assert bookmark_fixture.content.find("<script>") == -1
55 | # test relative urls in the HTML are remapped to an absolute urls
56 | assert "example.com/images/image1.png" in bookmark_fixture.content
57 | assert "example.com/testing-absolute-url" in bookmark_fixture.content
58 |
59 | # check empty link has been cleared
60 | assert "[](empty-link)" not in bookmark_fixture.content
61 |
62 |
63 | def test_bookmark_included_images_are_saved(test_app, client, mocked_responses):
64 | mocked_responses.add(
65 | GET, "https://example.com", body="""<html><img src='/image.png'></html>"""
66 | )
67 | mocked_responses.add(
68 | GET, "https://example.com/image.png", body=open("docs/img/logo.png", "rb")
69 | )
70 | test_app.config["SCRAPING_CONF"]["save_images"] = True
71 | bookmark = DataObj(type="bookmark", url="https://example.com")
72 | bookmark.process_bookmark_url()
73 | bookmark.insert()
74 | images_dir = Path(test_app.config["USER_DIR"]) / "images"
75 | assert images_dir.exists()
76 | assert (images_dir / "image.png").exists()
77 |
--------------------------------------------------------------------------------