├── app ├── __init__.py ├── api │ ├── __init__.py │ ├── api_v1 │ │ ├── __init__.py │ │ ├── endpoints │ │ │ ├── __init__.py │ │ │ └── file.py │ │ └── api.py │ ├── api_v2 │ │ ├── __init__.py │ │ ├── endpoints │ │ │ ├── __init__.py │ │ │ ├── video │ │ │ │ ├── __init__.py │ │ │ │ ├── delete.py │ │ │ │ ├── patch_video.py │ │ │ │ ├── list_videos.py │ │ │ │ └── upload.py │ │ │ ├── users.py │ │ │ ├── tags.py │ │ │ ├── like.py │ │ │ └── user_thumbnail.py │ │ └── api.py │ ├── dependencies.py │ ├── services.py │ └── security.py ├── core │ ├── __init__.py │ ├── db.py │ ├── logging_config.py │ └── config.py ├── models │ ├── __init__.py │ └── klepp.py ├── render │ ├── __init__.py │ ├── urls.py │ └── video_player.py ├── schemas │ ├── __init__.py │ └── schemas_v1 │ │ ├── __init__.py │ │ ├── user.py │ │ └── file.py └── main.py ├── klepp.png ├── example.gif ├── klepp_v2.png ├── .dockerignore ├── Procfile ├── .gitignore ├── .env.test ├── CONTRIBUTING.md ├── docker-compose.yaml ├── migrations ├── README ├── script.py.mako ├── versions │ ├── 1849bb6b7e57_add_thumbnail_uri.py │ ├── 39c1fdf365c0_uploaded_at.py │ ├── 55632ac588ff_likes.py │ └── 3ce3da92f858_db_v1.py └── env.py ├── templates ├── 404.html └── video.html ├── mypy.ini ├── .flake8 ├── pyproject.toml ├── .pre-commit-config.yaml ├── alembic.ini └── README.md /app/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/render/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/schemas/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/api_v1/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/api_v2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/schemas/schemas_v1/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/video/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /klepp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klepp-me/klepp-backend/HEAD/klepp.png -------------------------------------------------------------------------------- /example.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klepp-me/klepp-backend/HEAD/example.gif -------------------------------------------------------------------------------- /klepp_v2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/klepp-me/klepp-backend/HEAD/klepp_v2.png -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | env 2 | .dockerignore 3 | Dockerfile 4 | Dockerfile.prod 5 | .venv 6 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: gunicorn --pythonpath app -w 3 -k uvicorn.workers.UvicornWorker app.main:app 2 | -------------------------------------------------------------------------------- /app/schemas/schemas_v1/user.py: -------------------------------------------------------------------------------- 1 | from pydantic import BaseModel 2 | 3 | 4 | class User(BaseModel): 5 | username: str 6 | -------------------------------------------------------------------------------- /app/render/urls.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | 3 | from app.render import video_player 4 | 5 | api_router = APIRouter() 6 | api_router.include_router(video_player.router, tags=['video']) 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .idea/* 3 | env/ 4 | .env 5 | venv/ 6 | .venv/ 7 | build/ 8 | dist/ 9 | *.egg-info/ 10 | notes 11 | .pytest_cache 12 | .coverage 13 | htmlcov/ 14 | coverage.xml 15 | .DS_Store 16 | -------------------------------------------------------------------------------- /app/api/api_v1/api.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | 3 | from app.api.api_v1.endpoints import file 4 | 5 | api_router = APIRouter() 6 | api_router.include_router(file.router, tags=['deprecated_api'], deprecated=True) 7 | -------------------------------------------------------------------------------- /.env.test: -------------------------------------------------------------------------------- 1 | # Any variables you want to overwrite, you put here 2 | SECRET_KEY=my-secret-key-for-fastapi 3 | 4 | AWS_USER_POOL_ID=eu-north-1_... 5 | 6 | AWS_S3_ACCESS_KEY_ID=my-aws-id-with-access-to-bucket 7 | AWS_S3_SECRET_ACCESS_KEY=my-aws-secret-with-access-to-bucket 8 | DATABASE_URL=my-psql-connstr 9 | ENVIRONMENT=dev 10 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing to ACI-watcher-backend 2 | =================================== 3 | 4 | 1. Install poetry running ``pip install poetry`` or ``curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -`` 5 | 6 | 2. Install dependencies by running ``poetry install`` 7 | 8 | 3. Activate the environment 9 | 10 | 4. Install pre-commit (for linting) by running ``pre-commit install`` 11 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | postgres: 5 | container_name: klepp_postgres 6 | environment: 7 | POSTGRES_USER: klepp 8 | POSTGRES_PASSWORD: some-long-password-generate-here 9 | image: postgres:12.7-alpine 10 | ports: 11 | - '127.0.0.1:5556:5432' 12 | restart: always 13 | volumes: 14 | - database:/var/lib/postgresql/data 15 | volumes: 16 | database: 17 | -------------------------------------------------------------------------------- /migrations/README: -------------------------------------------------------------------------------- 1 | Generic single-database configuration with an async dbapi. 2 | This file was generated by `alembic init -t async migrations`. 3 | 4 | **Generate a new migration file** 5 | `alembic revision --autogenerate -m "my_comment"` 6 | 7 | **Upgrade database to reflect migrations** 8 | `alembic upgrade head` (This is done on startup for containers) 9 | 10 | # New models 11 | For new models, add them to `env.py`: 12 | ``` 13 | from app.models.cars import Car # noqa 14 | ``` 15 | -------------------------------------------------------------------------------- /app/core/db.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.ext.asyncio import create_async_engine 2 | from sqlmodel import create_engine 3 | from sqlmodel.sql.expression import Select, SelectOfScalar 4 | 5 | from app.core.config import settings 6 | 7 | ASYNC_ENGINE = create_async_engine(settings.DATABASE_URL, echo=False) # echo can be True/False or 'debug' 8 | 9 | SYNC_ENGINE = create_engine(settings.DATABASE_URL.replace('+asyncpg', ''), echo='debug') 10 | 11 | SelectOfScalar.inherit_cache = True # type: ignore 12 | Select.inherit_cache = True # type: ignore 13 | -------------------------------------------------------------------------------- /templates/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Klepp 404 9 | 12 | 13 | 14 |
15 | 404 video not found 16 |
17 | 18 | 19 | -------------------------------------------------------------------------------- /migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | import sqlmodel 11 | ${imports if imports else ""} 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = ${repr(up_revision)} 15 | down_revision = ${repr(down_revision)} 16 | branch_labels = ${repr(branch_labels)} 17 | depends_on = ${repr(depends_on)} 18 | 19 | 20 | def upgrade(): 21 | ${upgrades if upgrades else "pass"} 22 | 23 | 24 | def downgrade(): 25 | ${downgrades if downgrades else "pass"} 26 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | # Global options 2 | [mypy] 3 | python_version = 3.10 4 | # flake8-mypy expects the two following for sensible formatting 5 | show_column_numbers = True 6 | show_error_context = False 7 | show_error_codes = True 8 | warn_unused_ignores = True 9 | warn_redundant_casts = True 10 | warn_unused_configs = True 11 | warn_unreachable = True 12 | warn_return_any = True 13 | strict = True 14 | disallow_untyped_decorators = True 15 | disallow_any_generics = False 16 | implicit_reexport = False 17 | 18 | 19 | [mypy-tests.*] 20 | ignore_errors = True 21 | 22 | [pydantic-mypy] 23 | init_forbid_extra = True 24 | init_typed = True 25 | warn_required_dynamic_aliases = True 26 | warn_untyped_fields = True 27 | -------------------------------------------------------------------------------- /app/api/api_v2/api.py: -------------------------------------------------------------------------------- 1 | from fastapi import APIRouter 2 | 3 | from app.api.api_v2.endpoints import like, tags, user_thumbnail, users 4 | from app.api.api_v2.endpoints.video import delete, list_videos, patch_video, upload 5 | 6 | api_router = APIRouter() 7 | api_router.include_router(list_videos.router, tags=['video']) 8 | api_router.include_router(upload.router, tags=['video']) 9 | api_router.include_router(delete.router, tags=['video']) 10 | api_router.include_router(patch_video.router, tags=['video']) 11 | api_router.include_router(tags.router, tags=['tags']) 12 | api_router.include_router(user_thumbnail.router, tags=['user']) 13 | api_router.include_router(users.router, tags=['user']) 14 | api_router.include_router(like.router, tags=['video']) 15 | -------------------------------------------------------------------------------- /migrations/versions/1849bb6b7e57_add_thumbnail_uri.py: -------------------------------------------------------------------------------- 1 | """Add thumbnail uri 2 | 3 | Revision ID: 1849bb6b7e57 4 | Revises: 3ce3da92f858 5 | Create Date: 2022-03-16 13:32:43.197168 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | import sqlmodel 11 | 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = '1849bb6b7e57' 15 | down_revision = '3ce3da92f858' 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade(): 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.add_column('video', sa.Column('thumbnail_uri', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) 23 | # ### end Alembic commands ### 24 | 25 | 26 | def downgrade(): 27 | # ### commands auto generated by Alembic - please adjust! ### 28 | op.drop_column('video', 'thumbnail_uri') 29 | # ### end Alembic commands ### 30 | -------------------------------------------------------------------------------- /migrations/versions/39c1fdf365c0_uploaded_at.py: -------------------------------------------------------------------------------- 1 | """uploaded_at 2 | 3 | Revision ID: 39c1fdf365c0 4 | Revises: 55632ac588ff 5 | Create Date: 2022-03-21 17:29:05.685464 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | import sqlmodel 11 | from sqlalchemy.dialects import postgresql 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = '39c1fdf365c0' 15 | down_revision = '55632ac588ff' 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade(): 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.alter_column('video', 'uploaded', server_default=None, new_column_name='uploaded_at') 23 | op.drop_constraint('videolikelink_user_id_key', 'videolikelink', type_='unique') 24 | # ### end Alembic commands ### 25 | 26 | 27 | def downgrade(): 28 | # ### commands auto generated by Alembic - please adjust! ### 29 | op.alter_column('video', 'uploaded_at', server_default=None, new_column_name='uploaded') 30 | op.create_unique_constraint('videolikelink_user_id_key', 'videolikelink', ['user_id']) 31 | # ### end Alembic commands ### 32 | -------------------------------------------------------------------------------- /app/api/dependencies.py: -------------------------------------------------------------------------------- 1 | from typing import AsyncGenerator 2 | 3 | from aiobotocore.client import AioBaseClient 4 | from aiobotocore.session import get_session 5 | from sqlmodel.ext.asyncio.session import AsyncSession 6 | 7 | from app.core.config import settings 8 | from app.core.db import ASYNC_ENGINE 9 | 10 | session = get_session() 11 | 12 | 13 | async def get_boto() -> AioBaseClient: 14 | """ 15 | Create a boto client which can be shared 16 | """ 17 | async with session.create_client( 18 | 's3', 19 | region_name=settings.AWS_REGION, 20 | aws_secret_access_key=settings.AWS_S3_SECRET_ACCESS_KEY, 21 | aws_access_key_id=settings.AWS_S3_ACCESS_KEY_ID, 22 | ) as client: 23 | yield client 24 | 25 | 26 | async def get_db_session() -> AsyncSession: 27 | """ 28 | Return a session to the database 29 | """ 30 | return AsyncSession(ASYNC_ENGINE, expire_on_commit=False) 31 | 32 | 33 | async def yield_db_session() -> AsyncGenerator[AsyncSession, None]: 34 | """ 35 | Yield a session to the database 36 | """ 37 | async with AsyncSession(ASYNC_ENGINE, expire_on_commit=False) as db_session: 38 | yield db_session 39 | -------------------------------------------------------------------------------- /app/core/logging_config.py: -------------------------------------------------------------------------------- 1 | from logging.config import dictConfig 2 | 3 | from asgi_correlation_id import correlation_id_filter 4 | 5 | from app.core.config import settings 6 | 7 | LOGGING: dict = { 8 | 'version': 1, 9 | 'disable_existing_loggers': False, 10 | 'filters': { 11 | 'correlation_id': {'()': correlation_id_filter(8 if settings.ENVIRONMENT == 'dev' else 32)}, 12 | }, 13 | 'formatters': { 14 | 'console': { 15 | 'format': '%(levelname)-8s [%(correlation_id)s] %(name)s:%(lineno)d %(message)s', 16 | }, 17 | }, 18 | 'handlers': { 19 | 'console': { 20 | 'class': 'logging.StreamHandler', 21 | 'filters': ['correlation_id'], 22 | 'formatter': 'console', 23 | }, 24 | }, 25 | 'loggers': { 26 | # third-party packages 27 | 'httpx': {'level': 'DEBUG'}, 28 | 'asgi_correlation_id': {'level': 'WARNING'}, 29 | 'arq': {'level': 'INFO', 'propagate': True}, 30 | }, 31 | 'root': {'handlers': ['console'], 'level': 'DEBUG'}, 32 | } 33 | 34 | 35 | def setup_logging() -> None: 36 | """ 37 | Call this function to setup logging for the app. 38 | """ 39 | dictConfig(LOGGING) 40 | -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from fastapi.middleware.cors import CORSMiddleware 3 | 4 | from app.api.api_v1.api import api_router 5 | from app.api.api_v2.api import api_router as api_v2_router 6 | from app.api.security import cognito_scheme 7 | from app.core.config import settings 8 | from app.core.logging_config import setup_logging 9 | from app.render.urls import api_router as render_router 10 | 11 | app = FastAPI( 12 | title=settings.PROJECT_NAME, 13 | openapi_url=f'{settings.API_V2_STR}/openapi.json', 14 | swagger_ui_oauth2_redirect_url='/oauth2-redirect', 15 | swagger_ui_init_oauth={ 16 | 'usePkceWithAuthorizationCodeGrant': True, 17 | 'clientId': settings.AWS_OPENAPI_CLIENT_ID, 18 | }, 19 | on_startup=[setup_logging, cognito_scheme.openid_config.load_config], 20 | ) 21 | 22 | # Set all CORS enabled origins 23 | if settings.BACKEND_CORS_ORIGINS: 24 | app.add_middleware( 25 | CORSMiddleware, 26 | allow_origin_regex=r'.*', 27 | allow_credentials=True, 28 | allow_methods=['*'], 29 | allow_headers=['*'], 30 | ) 31 | 32 | app.include_router(api_router, prefix=settings.API_V1_STR) 33 | app.include_router(api_v2_router, prefix=settings.API_V2_STR) 34 | app.include_router(render_router) 35 | -------------------------------------------------------------------------------- /migrations/versions/55632ac588ff_likes.py: -------------------------------------------------------------------------------- 1 | """likes 2 | 3 | Revision ID: 55632ac588ff 4 | Revises: 1849bb6b7e57 5 | Create Date: 2022-03-19 21:38:10.317951 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | import sqlmodel 11 | 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = '55632ac588ff' 15 | down_revision = '1849bb6b7e57' 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade(): 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.create_table('videolikelink', 23 | sa.Column('video_path', sqlmodel.sql.sqltypes.AutoString(), nullable=False), 24 | sa.Column('user_id', sqlmodel.sql.sqltypes.GUID(), nullable=False), 25 | sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), 26 | sa.ForeignKeyConstraint(['video_path'], ['video.path'], ), 27 | sa.PrimaryKeyConstraint('video_path', 'user_id'), 28 | sa.UniqueConstraint('user_id') 29 | ) 30 | op.add_column('user', sa.Column('thumbnail_uri', sqlmodel.sql.sqltypes.AutoString(), nullable=True)) 31 | # ### end Alembic commands ### 32 | 33 | 34 | def downgrade(): 35 | # ### commands auto generated by Alembic - please adjust! ### 36 | op.drop_column('user', 'thumbnail_uri') 37 | op.drop_table('videolikelink') 38 | # ### end Alembic commands ### 39 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/users.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from fastapi import APIRouter, Depends, Query 4 | from sqlalchemy import desc, func 5 | from sqlmodel import select 6 | from sqlmodel.ext.asyncio.session import AsyncSession 7 | 8 | from app.api.dependencies import yield_db_session 9 | from app.api.security import cognito_scheme_or_anonymous 10 | from app.models.klepp import ListResponse, User, UserRead 11 | 12 | router = APIRouter() 13 | 14 | 15 | @router.get('/users', response_model=ListResponse[UserRead], dependencies=[Depends(cognito_scheme_or_anonymous)]) 16 | async def get_users( 17 | session: AsyncSession = Depends(yield_db_session), 18 | offset: int = 0, 19 | limit: int = Query(default=100, lte=100), 20 | ) -> dict[str, int | list]: 21 | """ 22 | Get a list of users 23 | """ 24 | # User query 25 | tag_statement = select(User).order_by(desc(User.name)) 26 | # Total count query based on query params, without pagination 27 | count_statement = select(func.count('*')).select_from(tag_statement) # type: ignore 28 | 29 | # Add pagination 30 | tag_statement = tag_statement.offset(offset=offset).limit(limit=limit) 31 | # Do DB requests async 32 | tasks = [ 33 | asyncio.create_task(session.exec(tag_statement)), # type: ignore 34 | asyncio.create_task(session.exec(count_statement)), 35 | ] 36 | results, count = await asyncio.gather(*tasks) 37 | count_number = count.one_or_none() 38 | return {'total_count': count_number, 'response': results.all()} 39 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/tags.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from fastapi import APIRouter, Depends, Query 4 | from sqlalchemy import desc, func 5 | from sqlmodel import select 6 | from sqlmodel.ext.asyncio.session import AsyncSession 7 | 8 | from app.api.dependencies import yield_db_session 9 | from app.api.security import cognito_scheme_or_anonymous 10 | from app.models.klepp import ListResponse, Tag, TagRead 11 | 12 | router = APIRouter() 13 | 14 | 15 | @router.get('/tags', response_model=ListResponse[TagRead], dependencies=[Depends(cognito_scheme_or_anonymous)]) 16 | async def get_all_tags( 17 | session: AsyncSession = Depends(yield_db_session), 18 | offset: int = 0, 19 | limit: int = Query(default=100, lte=100), 20 | ) -> dict[str, int | list]: 21 | """ 22 | Gets possible tags to use 23 | """ 24 | # Video query 25 | tag_statement = select(Tag).order_by(desc(Tag.name)) 26 | # Total count query based on query params, without pagination 27 | count_statement = select(func.count('*')).select_from(tag_statement) # type: ignore 28 | 29 | # Add pagination 30 | tag_statement = tag_statement.offset(offset=offset).limit(limit=limit) 31 | # Do DB requests async 32 | tasks = [ 33 | asyncio.create_task(session.exec(tag_statement)), # type: ignore 34 | asyncio.create_task(session.exec(count_statement)), 35 | ] 36 | results, count = await asyncio.gather(*tasks) 37 | count_number = count.one_or_none() 38 | return {'total_count': count_number, 'response': results.all()} 39 | -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | 4 | ignore= 5 | # Missing type annotations for **args 6 | ANN002 7 | # Missing type annotations for **kwargs 8 | ANN003 9 | # Missing type annotations for self 10 | ANN101 11 | # Missing type annotation for cls in classmethod 12 | ANN102 13 | 14 | # Complains on function calls in argument defaults 15 | B008 16 | 17 | # Docstring at the top of a public module 18 | D100 19 | # Docstring at the top of a public class (method is enough) 20 | D101 21 | # Missing docstring in __init__, nested class 22 | D106 23 | D107 24 | # Missing docstring in public package 25 | D104 26 | # Make docstrings one line if it can fit. 27 | D200 28 | # 1 blank line required between summary line and description 29 | D205 30 | # First line should end with a period - here we have a few cases where the first line is too long, and 31 | # this issue can't be fixed without using noqa notation 32 | D400 33 | # Imperative docstring declarations 34 | D401 35 | 36 | # Whitespace before ':'. Black formats code this way. 37 | E203 38 | # E501: Line length 39 | E501 40 | 41 | # Missing f-string, we ignore this due to URL patterns 42 | FS003 43 | 44 | # Missing type annotations for `**kwargs` 45 | TYP003 46 | # Type annotation for `self` 47 | TYP101 48 | TYP102 # for cls 49 | 50 | # W503 line break before binary operator - conflicts with black 51 | W503 52 | 53 | 54 | exclude = 55 | .git, 56 | .idea, 57 | __pycache__, 58 | tests/*, 59 | venv, 60 | .venv 61 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "klepp" 3 | version = "0.1.0" 4 | description = "API for uploading/deleting S3 files" 5 | authors = [ 6 | "Jonas Krüger Svensson ", 7 | ] 8 | license = "MIT" 9 | 10 | [tool.poetry.dependencies] 11 | python = "3.10.4" 12 | fastapi = ">=0.72.0" 13 | cryptography = ">=35.0.0" 14 | python-jose = {extras = ["cryptography"], version = "^3.3.0"} 15 | uvicorn = {extras = ["standard"], version = "^0.13.4"} 16 | python-multipart = "^0.0.5" 17 | aiobotocore = "^2.1.0" 18 | aioboto3 = "^9.3.1" 19 | httpx = "^0.21.3" 20 | pydantic = "1.9.0" 21 | gunicorn = "^20.1.0" 22 | typed-ast = "^1.5.2" 23 | alembic = "^1.7.6" 24 | sqlmodel = "^0.0.6" 25 | asgi-correlation-id = "^1.1.2" 26 | asyncpg = "^0.25.0" 27 | greenlet = "^1.1.2" 28 | psycopg2 = "^2.9.3" 29 | aiofiles = "^0.8.0" 30 | asyncffmpeg = "^1.2.0" 31 | asynccpu = "^1.2.2" 32 | ffmpeg-python = "^0.2.0" 33 | Jinja2 = "^3.1.1" 34 | 35 | [tool.poetry.dev-dependencies] 36 | pre-commit = "^2.9.3" 37 | black = "^20.8b1" 38 | isort = "^5.8.0" 39 | mypy = ">=0.812" 40 | ipython = "^8.1.1" 41 | 42 | [tool.black] 43 | line-length = 120 44 | skip-string-normalization = true 45 | target-version = ['py37'] 46 | include = '\.pyi?$' 47 | exclude = ''' 48 | ( 49 | (\.eggs|\.git|\.hg|\.mypy_cache|\.tox|\.venv|\venv|\.github|\docs|\tests|\__pycache__) 50 | ) 51 | ''' 52 | 53 | [tool.isort] 54 | profile = "black" 55 | src_paths = ["app"] 56 | combine_as_imports = true 57 | line_length = 120 58 | sections = [ 59 | 'FUTURE', 60 | 'STDLIB', 61 | 'THIRDPARTY', 62 | 'FIRSTPARTY', 63 | 'LOCALFOLDER', 64 | ] 65 | 66 | [build-system] 67 | requires = ["poetry-core>=1.0.0"] 68 | build-backend = "poetry.core.masonry.api" 69 | -------------------------------------------------------------------------------- /app/render/video_player.py: -------------------------------------------------------------------------------- 1 | import typing 2 | 3 | from fastapi import APIRouter, Depends, Request 4 | from fastapi.responses import HTMLResponse 5 | from fastapi.templating import Jinja2Templates 6 | from sqlalchemy import desc 7 | from sqlalchemy.orm import selectinload 8 | from sqlmodel import select 9 | from sqlmodel.ext.asyncio.session import AsyncSession 10 | 11 | from app.api.dependencies import yield_db_session 12 | from app.models.klepp import Video 13 | 14 | if typing.TYPE_CHECKING: 15 | from starlette.templating import _TemplateResponse 16 | 17 | templates = Jinja2Templates(directory='templates') 18 | 19 | router = APIRouter() 20 | 21 | 22 | @router.get('/', response_class=HTMLResponse, include_in_schema=False) 23 | async def render_video_page( 24 | request: Request, path: str, session: AsyncSession = Depends(yield_db_session) 25 | ) -> '_TemplateResponse': 26 | """ 27 | Static site for share.klepp.me?path= 28 | """ 29 | video_statement = ( 30 | select(Video) 31 | .options(selectinload(Video.user)) 32 | .options(selectinload(Video.tags)) 33 | .options(selectinload(Video.likes)) 34 | .order_by(desc(Video.uploaded_at)) 35 | ) 36 | if path: 37 | # Short route, specific path requested. This cannot be a `files/{path}` API due to `/` in video paths. 38 | video_statement = video_statement.where(Video.path == path) 39 | video_response = await session.exec(video_statement) # type: ignore 40 | if found := video_response.one_or_none(): 41 | return templates.TemplateResponse('video.html', {'request': request, 'video_dict': found.dict()}) 42 | return templates.TemplateResponse('404.html', {'request': request}) 43 | -------------------------------------------------------------------------------- /app/api/services.py: -------------------------------------------------------------------------------- 1 | from typing import Callable, Optional 2 | 3 | import ffmpeg 4 | from asynccpu import ProcessTaskPoolExecutor 5 | from asyncffmpeg import FFmpegCoroutineFactory, StreamSpec 6 | from sqlalchemy.orm import selectinload 7 | from sqlmodel import select 8 | from sqlmodel.ext.asyncio.session import AsyncSession 9 | 10 | from app.models.klepp import Video, VideoRead 11 | 12 | 13 | async def generate_user_thumbnail(path: str, name: str) -> StreamSpec: 14 | """ 15 | Cuts first frame and generates a new file 16 | """ 17 | return ffmpeg.input(path).filter('scale', 420, 420, force_original_aspect_ratio='decrease').output(name) 18 | 19 | 20 | async def generate_video_thumbnail(path: str, name: str) -> StreamSpec: 21 | """ 22 | Cuts first frame and generates a new file 23 | """ 24 | return ffmpeg.input(path).filter('scale', 840, -1).output(name, vframes=1) 25 | 26 | 27 | async def await_ffmpeg(function: Callable) -> None: 28 | """ 29 | Make ffmpeg awaitable 30 | """ 31 | ffmpeg_coroutine = FFmpegCoroutineFactory.create() 32 | 33 | with ProcessTaskPoolExecutor(max_workers=3, cancel_tasks_when_shutdown=True) as executor: 34 | await executor.create_process_task(ffmpeg_coroutine.execute, function) 35 | 36 | 37 | async def fetch_one_or_none_video(video_path: str, db_session: AsyncSession) -> Optional[VideoRead]: 38 | """ 39 | Takes a video path and fetches everything about it. 40 | """ 41 | query_video = ( 42 | select(Video) 43 | .where(Video.path == video_path) 44 | .options(selectinload(Video.user)) 45 | .options(selectinload(Video.tags)) 46 | .options(selectinload(Video.likes)) 47 | ) 48 | result = await db_session.exec(query_video) # type: ignore 49 | return result.one_or_none() 50 | -------------------------------------------------------------------------------- /app/core/config.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from pydantic import AnyHttpUrl, BaseSettings, Field, validator 4 | 5 | 6 | class AWS(BaseSettings): 7 | # General 8 | AWS_REGION: str = Field('eu-north-1') 9 | S3_BUCKET_URL: str = Field('gg.klepp.me') 10 | 11 | # Auth 12 | AWS_USER_POOL_ID: str = Field(..., env='AWS_USER_POOL_ID') 13 | AWS_OPENAPI_CLIENT_ID: str = Field(..., env='AWS_OPENAPI_CLIENT_ID') 14 | 15 | # Management 16 | AWS_S3_ACCESS_KEY_ID: str = Field(..., env='AWS_S3_ACCESS_KEY_ID') 17 | AWS_S3_SECRET_ACCESS_KEY: str = Field(..., env='AWS_S3_SECRET_ACCESS_KEY') 18 | 19 | 20 | class Settings(AWS): 21 | PROJECT_NAME: str = 'klepp.me' 22 | API_V1_STR: str = '/api/v1' 23 | API_V2_STR: str = '/api/v2' 24 | 25 | ENVIRONMENT: str = Field('dev', env='ENVIRONMENT') 26 | TESTING: bool = Field(False, env='TESTING') 27 | SECRET_KEY: str = Field(..., env='SECRET_KEY') 28 | DATABASE_URL: str = Field(..., env='AZURE_DATABASE_URL') 29 | 30 | @validator('DATABASE_URL', pre=True) 31 | def name_must_contain_space(cls, value: str) -> str: 32 | """ 33 | Replace Heroku postgres connection string to an async one, and change the prefix 34 | """ 35 | return value.replace('postgres://', 'postgresql+asyncpg://') 36 | 37 | # BACKEND_CORS_ORIGINS is a JSON-formatted list of origins 38 | # e.g: '["http://localhost", "http://localhost:4200", "http://localhost:3000", \ 39 | # "http://localhost:8080", "http://local.dockertoolbox.tiangolo.com"]' 40 | BACKEND_CORS_ORIGINS: List[AnyHttpUrl] = ['http://localhost:3000', 'http://localhost:5555'] # type: ignore 41 | 42 | class Config: # noqa 43 | env_file = '.env' 44 | env_file_encoding = 'utf-8' 45 | case_sensitive = True 46 | 47 | 48 | settings: Settings = Settings() 49 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/video/delete.py: -------------------------------------------------------------------------------- 1 | from aiobotocore.client import AioBaseClient 2 | from fastapi import APIRouter, Depends, HTTPException, status 3 | from pydantic import BaseModel, Field 4 | from sqlalchemy import and_ 5 | from sqlmodel import select 6 | from sqlmodel.ext.asyncio.session import AsyncSession 7 | 8 | from app.api.dependencies import get_boto, yield_db_session 9 | from app.api.security import cognito_signed_in 10 | from app.core.config import settings 11 | from app.models.klepp import User, Video 12 | 13 | router = APIRouter() 14 | 15 | 16 | class DeletedFileResponse(BaseModel): 17 | path: str = Field(...) 18 | 19 | 20 | @router.delete('/files', response_model=DeletedFileResponse) 21 | async def delete_file( 22 | path: DeletedFileResponse, 23 | boto_session: AioBaseClient = Depends(get_boto), 24 | user: User = Depends(cognito_signed_in), 25 | db_session: AsyncSession = Depends(yield_db_session), 26 | ) -> dict[str, str]: 27 | """ 28 | Delete file with filename 29 | """ 30 | video_statement = select(Video).where(and_(Video.path == path.path, Video.user_id == user.id)) 31 | db_result = await db_session.exec(video_statement) # type: ignore 32 | video = db_result.one_or_none() 33 | if not video: 34 | raise HTTPException( 35 | status_code=status.HTTP_404_NOT_FOUND, 36 | detail='File not found. Ensure you own the file, and that the file already exist.', 37 | ) 38 | await boto_session.delete_object(Bucket=settings.S3_BUCKET_URL, Key=video.path) 39 | if video.thumbnail_uri: 40 | await boto_session.delete_object( 41 | Bucket=settings.S3_BUCKET_URL, Key=video.thumbnail_uri.split('https://gg.klepp.me/')[1] 42 | ) 43 | await db_session.delete(video) 44 | await db_session.commit() 45 | return {'path': path.path} 46 | -------------------------------------------------------------------------------- /app/schemas/schemas_v1/file.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from typing import List 3 | from urllib.parse import quote 4 | 5 | from pydantic import BaseModel, Field, root_validator, validator 6 | 7 | from app.core.config import settings 8 | 9 | 10 | class FileName(BaseModel): 11 | file_name: str = Field(..., alias='fileName') 12 | 13 | class Config: 14 | allow_population_by_field_name = True 15 | 16 | 17 | class DeletedFileResponse(FileName): 18 | pass 19 | 20 | 21 | class FileResponse(FileName): 22 | uri: str 23 | datetime: datetime.datetime 24 | username: str 25 | 26 | @root_validator(pre=True) 27 | def extract_uri(cls, value: dict) -> dict: 28 | """ 29 | Extract file_name and create an uri 30 | """ 31 | value['uri'] = f'https://{settings.S3_BUCKET_URL}/{quote(value["file_name"])}' 32 | return value 33 | 34 | 35 | class HideFile(FileName): 36 | @validator('file_name') 37 | def validate_file_name(cls, value: str) -> str: 38 | """ 39 | Validate file path is according to specification 40 | """ 41 | if '/hidden/' in value: 42 | raise ValueError('Must not contain /hidden/') 43 | return value 44 | 45 | 46 | class ShowFile(FileName): 47 | @validator('file_name') 48 | def validate_file_name(cls, value: str) -> str: 49 | """ 50 | Validate file path is according to specification 51 | """ 52 | if '/hidden/' not in value: 53 | raise ValueError('Must not contain /hidden/') 54 | return value 55 | 56 | 57 | class DeleteFile(FileName): 58 | pass 59 | 60 | 61 | class ListFilesResponse(BaseModel): 62 | files: List[FileResponse] = Field(default=[]) 63 | hidden_files: List[FileResponse] = Field(default=[], alias='hiddenFiles') 64 | 65 | class Config: 66 | allow_population_by_field_name = True 67 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: README.md|migrations 2 | repos: 3 | - repo: https://github.com/ambv/black 4 | rev: '22.1.0' 5 | hooks: 6 | - id: black 7 | args: ['--quiet'] 8 | - repo: https://github.com/pre-commit/pre-commit-hooks 9 | rev: v4.1.0 10 | hooks: 11 | - id: check-case-conflict 12 | - id: end-of-file-fixer 13 | - id: trailing-whitespace 14 | - id: check-ast 15 | - id: check-json 16 | - id: check-merge-conflict 17 | - id: detect-private-key 18 | - id: double-quote-string-fixer 19 | - repo: https://gitlab.com/pycqa/flake8 20 | rev: 3.9.2 21 | hooks: 22 | - id: flake8 23 | additional_dependencies: [ 24 | 'flake8-bugbear==22.1.11', # Looks for likely bugs and design problems 25 | 'flake8-comprehensions==3.8.0', # Looks for unnecessary generator functions that can be converted to list comprehensions 26 | 'flake8-deprecated==1.3', # Looks for method deprecations 27 | 'flake8-use-fstring==1.3', # Enforces use of f-strings over .format and %s 28 | 'flake8-print==4.0.0', # Checks for print statements 29 | 'flake8-docstrings==1.6.0', # Verifies that all functions/methods have docstrings 30 | 'flake8-annotations==2.7.0', # Enforces type annotation 31 | ] 32 | args: ['--enable-extensions=G'] 33 | - repo: https://github.com/asottile/pyupgrade 34 | rev: v2.31.1 35 | hooks: 36 | - id: pyupgrade 37 | args: ["--py36-plus"] 38 | - repo: https://github.com/pycqa/isort 39 | rev: 5.10.1 40 | hooks: 41 | - id: isort 42 | - repo: https://github.com/pre-commit/mirrors-mypy 43 | rev: "v0.941" 44 | hooks: 45 | - id: mypy 46 | exclude: "test_*" 47 | additional_dependencies: 48 | [ 49 | fastapi, 50 | pydantic, 51 | starlette, 52 | sqlmodel, 53 | types-aiofiles, 54 | aioboto3 55 | ] 56 | -------------------------------------------------------------------------------- /templates/video.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Klepp.me - {{video_dict.display_name}} 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 42 | 43 | 44 |
45 | 49 |
50 | 51 | 52 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/like.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | from fastapi import APIRouter, Depends, HTTPException, status 4 | from pydantic import BaseModel 5 | from sqlalchemy import and_ 6 | from sqlalchemy.orm import selectinload 7 | from sqlmodel import select 8 | from sqlmodel.ext.asyncio.session import AsyncSession 9 | 10 | from app.api.dependencies import yield_db_session 11 | from app.api.security import cognito_signed_in 12 | from app.api.services import fetch_one_or_none_video 13 | from app.models.klepp import User, Video, VideoRead 14 | 15 | router = APIRouter() 16 | 17 | 18 | class VideoLikeUnlike(BaseModel): 19 | path: str 20 | 21 | 22 | @router.post('/like', response_model=VideoRead, status_code=status.HTTP_201_CREATED) 23 | async def add_like( 24 | path: VideoLikeUnlike, 25 | user: User = Depends(cognito_signed_in), 26 | db_session: AsyncSession = Depends(yield_db_session), 27 | ) -> Any: 28 | """ 29 | Add a like to a video 30 | """ 31 | video_statement = select(Video).where(Video.path == path.path).options(selectinload(Video.likes)) # noqa 32 | result = await db_session.exec(video_statement) # type: ignore 33 | video: Video | None = result.one_or_none() 34 | if not video: 35 | raise HTTPException( 36 | status_code=status.HTTP_404_NOT_FOUND, 37 | detail='Video not found.', 38 | ) 39 | video.likes.append(user) 40 | db_session.add(video) 41 | await db_session.commit() 42 | await db_session.refresh(video) 43 | return await fetch_one_or_none_video(video_path=video.path, db_session=db_session) 44 | 45 | 46 | @router.delete('/like', response_model=VideoRead, status_code=status.HTTP_200_OK) 47 | async def delete_like( 48 | path: VideoLikeUnlike, 49 | user: User = Depends(cognito_signed_in), 50 | db_session: AsyncSession = Depends(yield_db_session), 51 | ) -> Any: 52 | """ 53 | Remove like to a video 54 | """ 55 | video_statement = ( 56 | select(Video) 57 | .where( 58 | and_( 59 | Video.path == path.path, 60 | Video.likes.any(name=user.name), # type: ignore 61 | ) 62 | ) 63 | .options(selectinload(Video.likes)) 64 | ) 65 | result = await db_session.exec(video_statement) # type: ignore 66 | video = result.one_or_none() 67 | if not video: 68 | raise HTTPException( 69 | status_code=status.HTTP_404_NOT_FOUND, 70 | detail='Video not found.', 71 | ) 72 | video.likes.remove(user) 73 | db_session.add(video) 74 | await db_session.commit() 75 | await db_session.refresh(video) 76 | return await fetch_one_or_none_video(video_path=video.path, db_session=db_session) 77 | -------------------------------------------------------------------------------- /migrations/versions/3ce3da92f858_db_v1.py: -------------------------------------------------------------------------------- 1 | """db_v1 2 | 3 | Revision ID: 3ce3da92f858 4 | Revises: 5 | Create Date: 2022-03-14 22:35:04.690521 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | import sqlmodel 11 | 12 | 13 | # revision identifiers, used by Alembic. 14 | revision = '3ce3da92f858' 15 | down_revision = None 16 | branch_labels = None 17 | depends_on = None 18 | 19 | 20 | def upgrade(): 21 | # ### commands auto generated by Alembic - please adjust! ### 22 | op.create_table('tag', 23 | sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), 24 | sa.Column('id', sqlmodel.sql.sqltypes.GUID(), nullable=False), 25 | sa.PrimaryKeyConstraint('id') 26 | ) 27 | op.create_index(op.f('ix_tag_id'), 'tag', ['id'], unique=False) 28 | op.create_table('user', 29 | sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), 30 | sa.Column('id', sqlmodel.sql.sqltypes.GUID(), nullable=False), 31 | sa.PrimaryKeyConstraint('id') 32 | ) 33 | op.create_index(op.f('ix_user_id'), 'user', ['id'], unique=False) 34 | op.create_index(op.f('ix_user_name'), 'user', ['name'], unique=False) 35 | op.create_table('video', 36 | sa.Column('path', sqlmodel.sql.sqltypes.AutoString(), nullable=False), 37 | sa.Column('display_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), 38 | sa.Column('hidden', sa.Boolean(), nullable=True), 39 | sa.Column('uploaded', sa.DateTime(), nullable=True), 40 | sa.Column('uri', sqlmodel.sql.sqltypes.AutoString(), nullable=False), 41 | sa.Column('expire_at', sa.DateTime(), nullable=True), 42 | sa.Column('user_id', sqlmodel.sql.sqltypes.GUID(), nullable=False), 43 | sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), 44 | sa.PrimaryKeyConstraint('path') 45 | ) 46 | op.create_index(op.f('ix_video_display_name'), 'video', ['display_name'], unique=False) 47 | op.create_table('videotaglink', 48 | sa.Column('tag_id', sqlmodel.sql.sqltypes.GUID(), nullable=False), 49 | sa.Column('video_path', sqlmodel.sql.sqltypes.AutoString(), nullable=False), 50 | sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ), 51 | sa.ForeignKeyConstraint(['video_path'], ['video.path'], ), 52 | sa.PrimaryKeyConstraint('tag_id', 'video_path') 53 | ) 54 | # ### end Alembic commands ### 55 | 56 | 57 | def downgrade(): 58 | # ### commands auto generated by Alembic - please adjust! ### 59 | op.drop_table('videotaglink') 60 | op.drop_index(op.f('ix_video_display_name'), table_name='video') 61 | op.drop_table('video') 62 | op.drop_index(op.f('ix_user_name'), table_name='user') 63 | op.drop_index(op.f('ix_user_id'), table_name='user') 64 | op.drop_table('user') 65 | op.drop_index(op.f('ix_tag_id'), table_name='tag') 66 | op.drop_table('tag') 67 | # ### end Alembic commands ### 68 | -------------------------------------------------------------------------------- /migrations/env.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from logging.config import fileConfig 3 | 4 | from alembic import context 5 | from sqlalchemy import engine_from_config, pool 6 | from sqlalchemy.ext.asyncio import AsyncEngine 7 | from sqlmodel import SQLModel 8 | 9 | from app.core.config import settings 10 | 11 | # this is the Alembic Config object, which provides 12 | # access to the values within the .ini file in use. 13 | config = context.config 14 | config.set_main_option('sqlalchemy.url', settings.DATABASE_URL) 15 | # Interpret the config file for Python logging. 16 | # This line sets up loggers basically. 17 | fileConfig(config.config_file_name) 18 | 19 | # add your model's MetaData object here 20 | # for 'autogenerate' support 21 | # from myapp import mymodel 22 | # target_metadata = mymodel.Base.metadata 23 | from app.models.klepp import Video, Tag, VideoTagLink, User, VideoLikeLink 24 | 25 | target_metadata = SQLModel.metadata 26 | 27 | # other values from the config, defined by the needs of env.py, 28 | # can be acquired: 29 | # my_important_option = config.get_main_option("my_important_option") 30 | # ... etc. 31 | 32 | 33 | def run_migrations_offline() -> None: 34 | """Run migrations in 'offline' mode. 35 | 36 | This configures the context with just a URL 37 | and not an Engine, though an Engine is acceptable 38 | here as well. By skipping the Engine creation 39 | we don't even need a DBAPI to be available. 40 | 41 | Calls to context.execute() here emit the given string to the 42 | script output. 43 | 44 | """ 45 | url = config.get_main_option('sqlalchemy.url') 46 | context.configure( 47 | url=url, 48 | target_metadata=target_metadata, 49 | literal_binds=True, 50 | dialect_opts={'paramstyle': 'named'}, 51 | ) 52 | 53 | with context.begin_transaction(): 54 | context.run_migrations() 55 | 56 | 57 | def do_run_migrations(connection): 58 | context.configure(connection=connection, target_metadata=target_metadata) 59 | 60 | with context.begin_transaction(): 61 | context.run_migrations() 62 | 63 | 64 | async def run_migrations_online(): 65 | """Run migrations in 'online' mode. 66 | In this scenario we need to create an Engine 67 | and associate a connection with the context. 68 | """ 69 | print(config.get_main_option('sqlalchemy.url')) 70 | connectable = AsyncEngine( 71 | engine_from_config( 72 | config.get_section(config.config_ini_section), 73 | prefix='sqlalchemy.', 74 | poolclass=pool.NullPool, 75 | future=True, 76 | ) 77 | ) 78 | 79 | async with connectable.connect() as connection: 80 | await connection.run_sync(do_run_migrations) 81 | 82 | 83 | if context.is_offline_mode(): 84 | run_migrations_offline() 85 | else: 86 | asyncio.run(run_migrations_online()) 87 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/video/patch_video.py: -------------------------------------------------------------------------------- 1 | from typing import Any, Optional 2 | 3 | from fastapi import APIRouter, Depends, HTTPException, status 4 | from pydantic import BaseModel, Field 5 | from sqlalchemy import and_ 6 | from sqlalchemy.orm import selectinload 7 | from sqlmodel import select 8 | from sqlmodel.ext.asyncio.session import AsyncSession 9 | 10 | from app.api.dependencies import yield_db_session 11 | from app.api.security import cognito_signed_in 12 | from app.api.services import fetch_one_or_none_video 13 | from app.models.klepp import Tag, TagBase, User, Video, VideoRead 14 | 15 | router = APIRouter() 16 | 17 | 18 | class VideoPatch(BaseModel): 19 | path: str 20 | display_name: Optional[str] = Field(default=None, regex=r'^[\s\w\d_-]*$', min_length=2, max_length=40) 21 | hidden: Optional[bool] = Field(default=None) 22 | tags: Optional[list[TagBase]] = Field(default=None) 23 | 24 | 25 | @router.patch('/files', response_model=VideoRead) 26 | async def patch_video( 27 | video_patch: VideoPatch, 28 | db_session: AsyncSession = Depends(yield_db_session), 29 | user: User = Depends(cognito_signed_in), 30 | ) -> Any: 31 | """ 32 | Partially update a video. 33 | """ 34 | excluded = video_patch.dict(exclude_unset=True) 35 | [excluded.pop(x) for x in [key for key, value in excluded.items() if value is None]] 36 | 37 | query_video = ( 38 | select(Video) 39 | .where(and_(Video.path == video_patch.path, Video.user_id == user.id)) 40 | .options(selectinload(Video.tags)) 41 | ) 42 | db_result = await db_session.exec(query_video) # type: ignore 43 | video = db_result.one_or_none() 44 | if not video: 45 | raise HTTPException( 46 | status_code=status.HTTP_404_NOT_FOUND, 47 | detail='File not found. Ensure you own the file, and that the file already exist.', 48 | ) 49 | if excluded_tags := excluded.get('tags'): 50 | # They want to update tags, fetch available tags first 51 | list_tag = [tag['name'] for tag in excluded_tags] 52 | query_tags = select(Tag).where(Tag.name.in_(list_tag)) # type: ignore 53 | tag_result = await db_session.exec(query_tags) # type: ignore 54 | tags: list[Tag] = tag_result.all() 55 | if len(list_tag) != len(tags): 56 | db_list_tag = [tag.name for tag in tags] 57 | not_found_tags = [f'`{tag}`' for tag in list_tag if tag not in db_list_tag] 58 | raise HTTPException( 59 | status_code=status.HTTP_404_NOT_FOUND, 60 | detail=f'Tag {", ".join(not_found_tags)} not found.', 61 | ) 62 | video.tags = tags 63 | excluded.pop('tags') 64 | 65 | # Patch remaining attributes 66 | for key, value in excluded.items(): 67 | setattr(video, key, value) 68 | 69 | db_session.add(video) 70 | await db_session.commit() 71 | 72 | return await fetch_one_or_none_video(video_path=video_patch.path, db_session=db_session) 73 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/user_thumbnail.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | from typing import Any 4 | from uuid import uuid4 5 | 6 | import aiofiles 7 | from aiobotocore.client import AioBaseClient 8 | from aiofiles import os 9 | from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status 10 | from sqlmodel.ext.asyncio.session import AsyncSession 11 | 12 | from app.api.dependencies import get_boto, yield_db_session 13 | from app.api.security import cognito_signed_in 14 | from app.api.services import await_ffmpeg, generate_user_thumbnail 15 | from app.core.config import settings 16 | from app.models.klepp import User, UserRead 17 | 18 | router = APIRouter() 19 | 20 | 21 | @router.put('/user', response_model=UserRead, status_code=status.HTTP_201_CREATED) 22 | async def user_thumbnail( 23 | file: UploadFile = File(..., description='File to upload'), 24 | user: User = Depends(cognito_signed_in), 25 | db_session: AsyncSession = Depends(yield_db_session), 26 | boto_session: AioBaseClient = Depends(get_boto), 27 | ) -> Any: 28 | """ 29 | Upload a profile thumbnail. 30 | This is behind a CDN, so it might take a little while for it to update 31 | """ 32 | if not file: 33 | raise HTTPException(status_code=400, detail='You must provide a file.') 34 | allowed_formats = ['image/jpg', 'image/jpeg', 'image/png'] 35 | if file.content_type not in allowed_formats: 36 | raise HTTPException(status_code=400, detail=f'Currently only support for {",".join(allowed_formats)}') 37 | 38 | # Save thumbnail 39 | temp_name = uuid4().hex 40 | output_name = f'{temp_name}.png' 41 | async with aiofiles.open(temp_name, 'wb') as img: 42 | while content := await file.read(1024): 43 | await img.write(content) # type: ignore 44 | # Scale it 45 | await await_ffmpeg(functools.partial(generate_user_thumbnail, temp_name, output_name)) 46 | 47 | # Upload thumbnail, delete original, delete old thumbnail in s3 48 | profile_pic_path = f'{user.name}/profile/{output_name}' 49 | async with aiofiles.open(temp_name, 'rb+') as thumbnail_img: 50 | await boto_session.put_object( 51 | Bucket=settings.S3_BUCKET_URL, 52 | Key=profile_pic_path, 53 | Body=await thumbnail_img.read(), 54 | ACL='public-read', 55 | ) 56 | 57 | # Cleanup 58 | remove_original = asyncio.create_task(os.remove(temp_name)) 59 | remove_thumbnail = asyncio.create_task(os.remove(output_name)) 60 | if user.thumbnail_uri: 61 | delete_old_s3_thumbnail = asyncio.create_task( 62 | boto_session.delete_object( 63 | Bucket=settings.S3_BUCKET_URL, Key=user.thumbnail_uri.split('https://gg.klepp.me/')[1] 64 | ) 65 | ) 66 | await asyncio.gather(remove_original, remove_thumbnail, delete_old_s3_thumbnail) 67 | 68 | user.thumbnail_uri = f'https://gg.klepp.me/{profile_pic_path}' 69 | db_session.add(user) 70 | await db_session.commit() 71 | await db_session.refresh(user) 72 | return user.dict() 73 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/video/list_videos.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from typing import Optional 3 | 4 | from fastapi import APIRouter, Depends, Query 5 | from sqlalchemy import and_, desc, func, or_ 6 | from sqlalchemy.orm import selectinload 7 | from sqlmodel import select 8 | from sqlmodel.ext.asyncio.session import AsyncSession 9 | 10 | from app.api.dependencies import yield_db_session 11 | from app.api.security import cognito_scheme_or_anonymous 12 | from app.models.klepp import ListResponse, Video, VideoRead 13 | from app.schemas.schemas_v1.user import User as CognitoUser 14 | 15 | router = APIRouter() 16 | 17 | 18 | @router.get('/files', response_model=ListResponse[VideoRead]) 19 | async def get_all_files( 20 | session: AsyncSession = Depends(yield_db_session), 21 | user: CognitoUser | None = Depends(cognito_scheme_or_anonymous), 22 | username: Optional[str] = None, 23 | name: Optional[str] = None, 24 | hidden: Optional[bool] = None, 25 | tag: list[str] = Query(default=[]), 26 | offset: int = 0, 27 | limit: int = Query(default=100, lte=100), 28 | ) -> dict[str, int | list]: 29 | """ 30 | Get a list of all non-hidden files, unless you're the owner of the file, then you can request 31 | hidden files. 32 | Works both as anonymous user and as a signed-in user. 33 | """ 34 | # Video query 35 | video_statement = ( 36 | select(Video) 37 | .options(selectinload(Video.user)) 38 | .options(selectinload(Video.tags)) 39 | .options(selectinload(Video.likes)) 40 | .order_by(desc(Video.uploaded_at)) 41 | ) 42 | if username: 43 | video_statement = video_statement.where(Video.user.has(name=username)) # type: ignore 44 | if name: 45 | video_statement = video_statement.where(Video.display_name.contains(name)) # type: ignore 46 | 47 | if user and hidden is None: 48 | # Default behavior, include your own hidden videos 49 | video_statement = video_statement.where( 50 | or_( 51 | and_(Video.hidden == True, Video.user.has(name=user.username)), # type:ignore # noqa 52 | Video.hidden == False, 53 | ) 54 | ) 55 | elif user and hidden: 56 | # Only show your own hidden videos 57 | video_statement = video_statement.where( 58 | and_(Video.hidden == True, Video.user.has(name=user.username)), # type:ignore # noqa 59 | ) 60 | else: 61 | # Do not show any hidden videos 62 | video_statement = video_statement.where(Video.hidden == False) # noqa 63 | 64 | if tag: 65 | video_statement = video_statement.where(or_(Video.tags.any(name=t) for t in tag)) # type: ignore 66 | 67 | # Total count query based on query params, without pagination 68 | count_statement = select(func.count('*')).select_from(video_statement) # type: ignore 69 | 70 | # Add pagination 71 | video_statement = video_statement.offset(offset=offset).limit(limit=limit) 72 | # Do DB requests async 73 | tasks = [ 74 | asyncio.create_task(session.exec(video_statement)), # type: ignore 75 | asyncio.create_task(session.exec(count_statement)), 76 | ] 77 | results, count = await asyncio.gather(*tasks) 78 | count_number = count.one_or_none() 79 | return {'total_count': count_number, 'response': results.all()} 80 | -------------------------------------------------------------------------------- /alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # path to migration scripts 5 | script_location = migrations 6 | 7 | # template used to generate migration files 8 | # file_template = %%(rev)s_%%(slug)s 9 | 10 | # sys.path path, will be prepended to sys.path if present. 11 | # defaults to the current working directory. 12 | prepend_sys_path = . 13 | 14 | # timezone to use when rendering the date within the migration file 15 | # as well as the filename. 16 | # If specified, requires the python-dateutil library that can be 17 | # installed by adding `alembic[tz]` to the pip requirements 18 | # string value is passed to dateutil.tz.gettz() 19 | # leave blank for localtime 20 | # timezone = 21 | 22 | # max length of characters to apply to the 23 | # "slug" field 24 | # truncate_slug_length = 40 25 | 26 | # set to 'true' to run the environment during 27 | # the 'revision' command, regardless of autogenerate 28 | # revision_environment = false 29 | 30 | # set to 'true' to allow .pyc and .pyo files without 31 | # a source .py file to be detected as revisions in the 32 | # versions/ directory 33 | # sourceless = false 34 | 35 | # version location specification; This defaults 36 | # to migrations/versions. When using multiple version 37 | # directories, initial revisions must be specified with --version-path. 38 | # The path separator used here should be the separator specified by "version_path_separator" below. 39 | # version_locations = %(here)s/bar:%(here)s/bat:migrations/versions 40 | 41 | # version path separator; As mentioned above, this is the character used to split 42 | # version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. 43 | # If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. 44 | # Valid values for version_path_separator are: 45 | # 46 | # version_path_separator = : 47 | # version_path_separator = ; 48 | # version_path_separator = space 49 | version_path_separator = os # Use os.pathsep. Default configuration used for new projects. 50 | 51 | # the output encoding used when revision files 52 | # are written from script.py.mako 53 | # output_encoding = utf-8 54 | 55 | sqlalchemy.url = driver://user:pass@localhost/dbname 56 | 57 | 58 | [post_write_hooks] 59 | # post_write_hooks defines scripts or Python functions that are run 60 | # on newly generated revision scripts. See the documentation for further 61 | # detail and examples 62 | 63 | # format using "black" - use the console_scripts runner, against the "black" entrypoint 64 | # hooks = black 65 | # black.type = console_scripts 66 | # black.entrypoint = black 67 | # black.options = -l 79 REVISION_SCRIPT_FILENAME 68 | 69 | # Logging configuration 70 | [loggers] 71 | keys = root,sqlalchemy,alembic 72 | 73 | [handlers] 74 | keys = console 75 | 76 | [formatters] 77 | keys = generic 78 | 79 | [logger_root] 80 | level = WARN 81 | handlers = console 82 | qualname = 83 | 84 | [logger_sqlalchemy] 85 | level = WARN 86 | handlers = 87 | qualname = sqlalchemy.engine 88 | 89 | [logger_alembic] 90 | level = INFO 91 | handlers = 92 | qualname = alembic 93 | 94 | [handler_console] 95 | class = StreamHandler 96 | args = (sys.stderr,) 97 | level = NOTSET 98 | formatter = generic 99 | 100 | [formatter_generic] 101 | format = %(levelname)-5.5s [%(name)s] %(message)s 102 | datefmt = %H:%M:%S 103 | -------------------------------------------------------------------------------- /app/models/klepp.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | from datetime import datetime, timedelta 3 | from typing import Generic, List, Optional, TypeVar 4 | 5 | from pydantic.generics import GenericModel 6 | from sqlmodel import Field, Relationship, SQLModel 7 | 8 | ResponseModel = TypeVar('ResponseModel') 9 | 10 | 11 | def generate_expire_at() -> datetime: 12 | """ 13 | Generate util for video expiry date 14 | """ 15 | return datetime.utcnow() + timedelta(weeks=12) 16 | 17 | 18 | class ListResponse(GenericModel, Generic[ResponseModel]): 19 | total_count: int 20 | response: list[ResponseModel] 21 | 22 | 23 | class VideoTagLink(SQLModel, table=True): 24 | tag_id: uuid.UUID = Field(default=None, foreign_key='tag.id', primary_key=True, nullable=False) 25 | video_path: str = Field(default=None, foreign_key='video.path', primary_key=True, nullable=False) 26 | 27 | 28 | class VideoLikeLink(SQLModel, table=True): 29 | video_path: str = Field(foreign_key='video.path', primary_key=True, nullable=False) 30 | user_id: uuid.UUID = Field(foreign_key='user.id', primary_key=True, nullable=False) 31 | 32 | 33 | class UserBase(SQLModel): 34 | name: str = Field(index=True) 35 | thumbnail_uri: Optional[str] = Field(default=None, nullable=True) 36 | 37 | 38 | class User(UserBase, table=True): 39 | """ 40 | We store usernames just to be able to add likes, comments etc. 41 | All authentication is done through Cognito in AWS. 42 | I've decided to not have username as the primary key, since a username could change. 43 | """ 44 | 45 | id: Optional[uuid.UUID] = Field(default_factory=uuid.uuid4, primary_key=True, index=True, nullable=False) 46 | videos: List['Video'] = Relationship(back_populates='user') 47 | liked_videos: List['Video'] = Relationship(back_populates='likes', link_model=VideoLikeLink) 48 | 49 | 50 | class UserRead(UserBase): 51 | pass 52 | 53 | 54 | class TagBase(SQLModel): 55 | name: str = Field(...) 56 | 57 | 58 | class Tag(TagBase, table=True): 59 | id: Optional[uuid.UUID] = Field(default_factory=uuid.uuid4, primary_key=True, index=True, nullable=False) 60 | 61 | videos: List['Video'] = Relationship(back_populates='tags', link_model=VideoTagLink) 62 | 63 | 64 | class TagRead(TagBase): 65 | pass 66 | 67 | 68 | class VideoBase(SQLModel): 69 | path: str = Field(primary_key=True, nullable=False, description='s3 path, primary key') 70 | display_name: str = Field(index=True, description='Display name of the video') 71 | hidden: bool = Field(default=False, description='Whether the file can be seen by anyone on the frontpage') 72 | uploaded_at: datetime = Field(default_factory=datetime.utcnow, description='When the file was uploaded') 73 | uri: str = Field(..., description='Link to the video') 74 | expire_at: Optional[datetime] = Field( 75 | nullable=True, description='When the file is to be deleted', default_factory=generate_expire_at 76 | ) 77 | 78 | 79 | class Video(VideoBase, table=True): 80 | user_id: uuid.UUID = Field(foreign_key='user.id', nullable=False, description='User primary key') 81 | user: User = Relationship(back_populates='videos') 82 | thumbnail_uri: Optional[str] = Field(default=None, nullable=True) 83 | 84 | tags: List[Tag] = Relationship(back_populates='videos', link_model=VideoTagLink) 85 | likes: List[User] = Relationship(back_populates='liked_videos', link_model=VideoLikeLink) 86 | 87 | 88 | class VideoRead(VideoBase): 89 | user: 'UserRead' 90 | tags: List['TagRead'] 91 | thumbnail_uri: Optional[str] = Field(default=None, description='If it exist, we have a thumbnail for the video') 92 | likes: List['UserRead'] 93 | -------------------------------------------------------------------------------- /app/api/api_v2/endpoints/video/upload.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import functools 3 | from typing import Any, Optional 4 | from uuid import uuid4 5 | 6 | import aiofiles 7 | from aiobotocore.client import AioBaseClient 8 | from aiofiles import os 9 | from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status 10 | from sqlmodel.ext.asyncio.session import AsyncSession 11 | 12 | from app.api.dependencies import get_boto, yield_db_session 13 | from app.api.security import cognito_signed_in 14 | from app.api.services import await_ffmpeg, fetch_one_or_none_video, generate_video_thumbnail 15 | from app.core.config import settings 16 | from app.models.klepp import User, Video, VideoRead 17 | 18 | router = APIRouter() 19 | 20 | 21 | async def upload_video(boto_session: AioBaseClient, path: str, temp_video_name: str) -> None: 22 | """ 23 | Upload a stored file to s3 24 | """ 25 | async with aiofiles.open(temp_video_name, 'rb+') as video_file: 26 | await boto_session.put_object( 27 | Bucket=settings.S3_BUCKET_URL, 28 | Key=path, 29 | Body=await video_file.read(), 30 | ACL='public-read', 31 | ) 32 | 33 | 34 | @router.post('/files', response_model=VideoRead, status_code=status.HTTP_201_CREATED) 35 | async def upload_file( 36 | file: UploadFile = File(..., description='File to upload'), 37 | file_name: Optional[str] = Form( 38 | default=None, example='my_file', regex=r'^[\s\w\d_-]*$', min_length=2, max_length=40 39 | ), 40 | boto_session: AioBaseClient = Depends(get_boto), 41 | user: User = Depends(cognito_signed_in), 42 | db_session: AsyncSession = Depends(yield_db_session), 43 | ) -> Any: 44 | """ 45 | Upload a file. 46 | """ 47 | if not file: 48 | raise HTTPException(status_code=400, detail='You must provide a file.') 49 | 50 | if file.content_type != 'video/mp4': 51 | raise HTTPException(status_code=400, detail='Currently only support for video/mp4 files through this API.') 52 | 53 | upload_file_name = f'{file_name}.mp4' if file_name else file.filename 54 | s3_path = f'{user.name}/{upload_file_name}' 55 | video_uri = f'https://gg.klepp.me/{s3_path}' 56 | thumbnail_uri = f'https://gg.klepp.me/{user.name}/{upload_file_name}'.replace('.mp4', '.png') 57 | 58 | exist = await boto_session.list_objects_v2(Bucket=settings.S3_BUCKET_URL, Prefix=s3_path) 59 | if exist.get('Contents'): 60 | raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='Video already exist in s3.') 61 | 62 | # Save video 63 | temp_name = uuid4().hex 64 | temp_vido_name = f'{temp_name}.mp4' 65 | temp_thumbnail_name = f'{temp_name}.png' 66 | async with aiofiles.open(temp_vido_name, 'wb') as video: 67 | while content := await file.read(1024): 68 | await video.write(content) # type: ignore 69 | 70 | # Upload video and generate thumbnail 71 | upload_task = asyncio.create_task( 72 | upload_video(boto_session=boto_session, path=s3_path, temp_video_name=temp_vido_name) 73 | ) 74 | ffmpeg_task = asyncio.create_task( 75 | # create task calling await_ffmpeg 76 | await_ffmpeg( 77 | # passing the `generate_video_thumbnail` function with the arguments temp_vido_name, temp_thumbnail_name 78 | functools.partial(generate_video_thumbnail, temp_vido_name, temp_thumbnail_name) 79 | ) 80 | ) 81 | await asyncio.gather(upload_task, ffmpeg_task) 82 | 83 | # Upload thumbnail and clean up 84 | async with aiofiles.open(temp_thumbnail_name, 'rb+') as thumbnail_img: 85 | await boto_session.put_object( 86 | Bucket=settings.S3_BUCKET_URL, 87 | Key=s3_path.replace('.mp4', '.png'), 88 | Body=await thumbnail_img.read(), 89 | ACL='public-read', 90 | ) 91 | 92 | # Cleanup 93 | remove_video = asyncio.create_task(os.remove(f'{temp_name}.mp4')) 94 | remove_thumbnail = asyncio.create_task(os.remove(f'{temp_name}.png')) 95 | await asyncio.gather(remove_video, remove_thumbnail) 96 | 97 | # Add to DB and fetch it 98 | db_video: Video = Video( 99 | path=s3_path, 100 | display_name=upload_file_name.split('.mp4')[0], 101 | user=user, 102 | user_id=user.id, 103 | uri=video_uri, 104 | thumbnail_uri=thumbnail_uri, 105 | ) 106 | db_session.add(db_video) 107 | await db_session.commit() 108 | return await fetch_one_or_none_video(video_path=db_video.path, db_session=db_session) 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # klepp.me 2 | *A service to share clips among friends, while owning and managing our own data* 3 | 4 | **NB:** v2 rewrite in progress. 5 | 6 | ### What? 7 | 8 | [klepp.me](https://klepp.me) version 1 started off to be a [streamable.com](https://streamable.com/) / [pomf.cat](https://pomf.cat/) clone, 9 | which integrated natively with [`ShareX`](https://getsharex.com/). 10 | Any screenshot or video recorded through this program is automatically be uploaded to `gg.klepp.me`. When the file has been 11 | uploaded, a URL with a link to the resource is automatically stored in your clipboard. 12 | 13 | The v1 APIs of `klepp` only used S3 APIs to list and manage files. Due to object storage having very limited 14 | functionality, we outgrew its functionality within hours. The need for a better list-API with pagination and 15 | extensible sorting on users, tags, likes etc. was immediate. The frontend also had performance issues 16 | due to loading too many videos, so automatic generation and storage of video thumbnails became a necessity. 17 | 18 |
19 | V1 and V2 feature comparison table 20 | 21 | | | **v1** | **v2** | 22 | |----------------------|--------|---------| 23 | | Cognito login | ✅ | ✅️ | 24 | | List Videos | ✅ | ✅️ | 25 | | Sorting | ❌ | ✅️ | 26 | | Pagination | ❌ | ✅️ | 27 | | ShareX support | ✅ | ✅️* | 28 | | Upload Videos | ✅ | ✅️ | 29 | | Delete Videos | ✅ | ✅️ | 30 | | Hide videos | ✅ | ✅️ | 31 | | Like videos | ❌ | ✅️ | 32 | | Tags | ❌ | ✅️ | 33 | | Thumbnail generation | ❌ | ✅️ | 34 | 35 | 36 | *\* ShareX support still there, but these videos will not show up in the list API, since a DB is used instead 37 | of reflecting the S3 bucket. We might add a lambda to create the entry uploaded from ShareX in the database, 38 | but this a low priority now that we have a GUI.* 39 |
40 | 41 | 42 | ### Stack 43 | 44 | * **Storage**: [gg.klepp.me](https://gg.klepp.me) -> AWS Cloudfront CDN -> AWS S3 bucket 45 | * **API**: [api.klepp.me](https://api.klepp.me/docs) -> Hosted on Heroku -> FastAPI -> [validate tokens/Cognito auth](app/api/security.py) -> API View 46 | * **Database**: PostgreSQL using SQLModel/SQLAlchemy 47 | * **Authentication** [auth.klepp.me](https://auth.klepp.me) -> AWS Cognito 48 | * **Frontend**: [klepp.me](https://klepp.me) -> GitHub pages -> React frontend -> Cognito auth -> Requests to the API 49 | 50 | TLS is achieved on all sites using either AWS Certificate Manager, Heroku or GitHub pages. 51 | 52 | Visualized something like this: 53 | ![visualized v2 stack](klepp_v2.png) 54 | 55 |
56 | You can see the v1 visualization here 57 | 58 | ![visualized v1 stack](klepp.png) 59 |
60 | 61 |
62 | ShareX support visualized: 63 | 64 | ![example gif](example.gif) 65 |
66 | 67 | 68 | ## Why? 69 | 70 | I started using `pomf.cat` until I automatically uploaded a screenshot of my desktop with personal information in it through ShareX. 71 | Since `pomf.cat` has no users, there was no way for me to delete this screenshot without mailing the owners of the site hoping they would listen (they did!). 72 | At this point, I knew I could never use pomf.cat or an untrusted service for this purpose again. 73 | For videos, I used `streamable.com`. The size limit is better, but 13 dollar per month (per user!) for 74 | a simple, permanent storage of video clips is *steep*. 75 | 76 | TL;DR: Trust issues to external sites, price, permanent storage, frontend with our clips only. 77 | 78 | 79 | | | **Klepp** | **Streamable** | **Pomf.cat** | 80 | |--------------------------------------------------------|-----------|----------------|--------------| 81 | | Frontend to browse friends videos
(community feel) | ✅ | ❌ | ❌ | 82 | | Browse your own previous videos | ✅ | ✅ | ❌ | 83 | | Permanent storage | ✅ (Cheap) | ✅ (Expensive) | ❌ | 84 | | Own our own data | ✅ | ❌ | ❌ | 85 | 86 | 87 | 88 | ## But klepp..? 89 | Yes, we tend to yell "CLIP!" or "KLEPP!" whenever someone does something we think should be clipped (ShadowPlay) and shared after a game :) 90 | 91 | 92 | ## Cool! 93 | ~~Keep in mind this all of this was done in a few days, without any thought of beautiful code or tests.~~ 94 | This project has evolved and been rewritten to use databases, generate thumbnails, have likes, tags etc. 95 | However, this project is more about learning together with friends, hacking solutions and make things work than writing 96 | beautiful and well tested enterprise code. 97 | 98 | The authentication and token validation is inspired by [FastAPI-Azure-Auth](https://github.com/Intility/fastapi-azure-auth), 99 | a python package written by me. If you'd like to use Cognito authentication in your FastAPI app, I suggest you 100 | look at that package instead of this repository, as I have not ported any tests over to this app. 101 | This API is probably secure, but I wouldn't bet in it. When it comes to security, you either test it or don't trust it, simple as that. 102 | 103 | ## The future 104 | A lot of improvements can be done, but here's a few that I've been thinking about: 105 | 106 | * Generate thumbnails using lambda instead of using the API container 107 | * Pre-sign URLs and upload directly to S3 instead of through the S3 bucket 108 | * Use lambda to generate metadata in the database (using the Klepp-API) 109 | * Infrastructure as Code 110 | * Multi-tenancy/groups to allow multiple communities -------------------------------------------------------------------------------- /app/api/api_v1/endpoints/file.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime, timezone 2 | from typing import Any, Optional 3 | 4 | from aiobotocore.client import AioBaseClient 5 | from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile, status 6 | 7 | from app.api.dependencies import get_boto 8 | from app.api.security import cognito_scheme, cognito_scheme_or_anonymous 9 | from app.core.config import settings 10 | from app.schemas.schemas_v1.file import ( 11 | DeletedFileResponse, 12 | DeleteFile, 13 | FileResponse, 14 | HideFile, 15 | ListFilesResponse, 16 | ShowFile, 17 | ) 18 | from app.schemas.schemas_v1.user import User 19 | 20 | router = APIRouter() 21 | 22 | 23 | @router.post('/hide', response_model=FileResponse) 24 | async def hide_file( 25 | file: HideFile, session: AioBaseClient = Depends(get_boto), user: User = Depends(cognito_scheme) 26 | ) -> Any: 27 | """ 28 | Put file in hidden folder, which is still public, but not listed on the front page. 29 | """ 30 | new_path = file.file_name.replace(user.username, f'{user.username}/hidden') 31 | 32 | exist = await session.list_objects_v2(Bucket=settings.S3_BUCKET_URL, Prefix=file.file_name) 33 | if not exist.get('Contents'): 34 | raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='Could not find the provided file.') 35 | 36 | await session.copy_object( 37 | ACL='public-read', 38 | Bucket=settings.S3_BUCKET_URL, 39 | CopySource={'Bucket': settings.S3_BUCKET_URL, 'Key': file.file_name}, 40 | Key=new_path, 41 | ) 42 | await session.delete_object(Bucket=settings.S3_BUCKET_URL, Key=file.file_name) 43 | 44 | return { 45 | 'file_name': new_path, 46 | 'datetime': datetime.now(timezone.utc).isoformat(' ', 'seconds'), 47 | 'username': user.username, 48 | } 49 | 50 | 51 | @router.post('/show', response_model=FileResponse) 52 | async def show_file( 53 | file: ShowFile, session: AioBaseClient = Depends(get_boto), user: User = Depends(cognito_scheme) 54 | ) -> Any: 55 | """ 56 | Remove the file from the hidden folder, so that it is listed on the front page. 57 | """ 58 | new_path = file.file_name.replace(f'{user.username}/hidden', user.username) 59 | exist = await session.list_objects_v2(Bucket=settings.S3_BUCKET_URL, Prefix=file.file_name) 60 | if not exist.get('Contents'): 61 | raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='Could not find the provided file.') 62 | 63 | await session.copy_object( 64 | ACL='public-read', 65 | Bucket=settings.S3_BUCKET_URL, 66 | CopySource={'Bucket': settings.S3_BUCKET_URL, 'Key': file.file_name}, 67 | Key=new_path, 68 | ) 69 | await session.delete_object(Bucket=settings.S3_BUCKET_URL, Key=file.file_name) 70 | 71 | return { 72 | 'file_name': new_path, 73 | 'username': user.username, 74 | 'datetime': datetime.now(timezone.utc).isoformat(' ', 'seconds'), 75 | } 76 | 77 | 78 | @router.post('/files', response_model=FileResponse, status_code=status.HTTP_201_CREATED) 79 | async def upload_file( 80 | file: UploadFile = File(..., description='File to upload'), 81 | file_name: Optional[str] = Form( 82 | default=None, alias='fileName', example='my_file.mp4', regex=r'^[\s\w\d_-]*$', min_length=2, max_length=40 83 | ), 84 | session: AioBaseClient = Depends(get_boto), 85 | user: User = Depends(cognito_scheme), 86 | ) -> Any: 87 | """ 88 | Upload a file 89 | """ 90 | if not file: 91 | raise HTTPException(status_code=400, detail='You must provide a file.') 92 | 93 | if file.content_type != 'video/mp4': 94 | raise HTTPException(status_code=400, detail='Currently only support for video/mp4 files through this API.') 95 | 96 | file_name = f'{file_name}.mp4' if file_name and not file_name.endswith('.mp4') else file_name 97 | 98 | new_file_name = f'{user.username}/{file_name or file.filename}' 99 | exist = await session.list_objects_v2(Bucket=settings.S3_BUCKET_URL, Prefix=new_file_name) 100 | if exist.get('Contents'): 101 | raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail='File already exist.') 102 | await session.put_object( 103 | Bucket=settings.S3_BUCKET_URL, 104 | Key=new_file_name, 105 | Body=await file.read(), 106 | ACL='public-read', 107 | ) 108 | 109 | return { 110 | 'file_name': new_file_name, 111 | 'username': user.username, 112 | 'datetime': datetime.now(timezone.utc).isoformat(' ', 'seconds'), 113 | } 114 | 115 | 116 | @router.delete('/files', response_model=DeletedFileResponse) 117 | async def delete_file( 118 | file: DeleteFile, session: AioBaseClient = Depends(get_boto), user: User = Depends(cognito_scheme) 119 | ) -> DeletedFileResponse: 120 | """ 121 | Delete file with filename 122 | """ 123 | if not file.file_name.startswith(f'{user.username}/'): 124 | raise HTTPException( 125 | status_code=status.HTTP_403_FORBIDDEN, 126 | detail='You can only delete your own files.', 127 | ) 128 | exist = await session.list_objects_v2(Bucket=settings.S3_BUCKET_URL, Prefix=file.file_name) 129 | if not exist.get('Contents'): 130 | raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail='Could not find the file.') 131 | await session.delete_object(Bucket=settings.S3_BUCKET_URL, Key=file.file_name) 132 | return DeletedFileResponse(file_name=file.file_name) 133 | 134 | 135 | @router.get('/files', response_model=ListFilesResponse) 136 | async def get_all_files( 137 | session: AioBaseClient = Depends(get_boto), user: User | None = Depends(cognito_scheme_or_anonymous) 138 | ) -> dict[str, list[dict]]: 139 | """ 140 | Get a list of all non-hidden files, unless you're the owner of the file. 141 | Works both as anonymous user and as a signed in user. 142 | """ 143 | bucket = await session.list_objects_v2(Bucket=settings.S3_BUCKET_URL) 144 | 145 | if not user: 146 | user = User(username='AnonymousUser') 147 | 148 | file_list_response: dict[str, list[dict]] = {'files': [], 'hidden_files': []} 149 | 150 | for file in bucket['Contents']: 151 | path: str = file['Key'] 152 | if not path.endswith('.mp4'): 153 | continue 154 | split_path = path.split('/') 155 | if len(split_path) <= 1: 156 | # If a path don't contain at least a username, we don't want to list it at all. 157 | continue 158 | path_owner = split_path[0] 159 | if path_owner != user.username and split_path[1] == 'hidden': 160 | continue 161 | 162 | if path_owner == user.username and split_path[1] == 'hidden': 163 | file_list_response['hidden_files'].append( 164 | {'file_name': path, 'datetime': file['LastModified'], 'username': path_owner} 165 | ) 166 | file_list_response['files'].append( 167 | {'file_name': path, 'datetime': file['LastModified'], 'username': path_owner} 168 | ) 169 | 170 | return file_list_response 171 | -------------------------------------------------------------------------------- /app/api/security.py: -------------------------------------------------------------------------------- 1 | """ 2 | Most of this security has been well tested (and stolen) from https://github.com/Intility/fastapi-azure-auth, 3 | which I'm the author of. However, this specific project has been written in a day or two for fun, not for enterprise 4 | security. If you're using this library as inspiration for anything, please keep that in mind. 5 | """ 6 | 7 | import logging 8 | from datetime import datetime, timedelta 9 | from typing import Any 10 | 11 | from fastapi import Depends, HTTPException, status 12 | from fastapi.security import OAuth2AuthorizationCodeBearer, SecurityScopes 13 | from fastapi.security.base import SecurityBase 14 | from httpx import AsyncClient 15 | from jose import ExpiredSignatureError, jwk, jwt 16 | from jose.backends.cryptography_backend import CryptographyRSAKey 17 | from jose.exceptions import JWTClaimsError, JWTError 18 | from sqlmodel import select 19 | from sqlmodel.ext.asyncio.session import AsyncSession 20 | from starlette.requests import Request 21 | 22 | from app.api.dependencies import yield_db_session 23 | from app.core.config import settings 24 | from app.models.klepp import User 25 | from app.schemas.schemas_v1.user import User as CognitoUser 26 | 27 | 28 | class InvalidAuth(HTTPException): 29 | """ 30 | Exception raised when the user is not authorized 31 | """ 32 | 33 | def __init__(self, detail: str) -> None: 34 | super().__init__( 35 | status_code=status.HTTP_401_UNAUTHORIZED, detail=detail, headers={'WWW-Authenticate': 'Bearer'} 36 | ) 37 | 38 | 39 | log = logging.getLogger(__name__) 40 | 41 | 42 | class OpenIdConfig: 43 | def __init__(self) -> None: 44 | self._config_timestamp: datetime | None = None 45 | self.openid_url = ( 46 | f'https://cognito-idp.{settings.AWS_REGION}.amazonaws.com/' 47 | f'{settings.AWS_USER_POOL_ID}/.well-known/openid-configuration' 48 | ) 49 | 50 | self.issuer: str 51 | 52 | async def load_config(self) -> None: 53 | """ 54 | Loads config from the openid endpoint if it's over 24 hours old (or don't exist) 55 | """ 56 | refresh_time = datetime.now() - timedelta(hours=24) 57 | if not self._config_timestamp or self._config_timestamp < refresh_time: 58 | try: 59 | log.debug('Loading Cognito OpenID configuration.') 60 | await self._load_openid_config() 61 | self._config_timestamp = datetime.now() 62 | except Exception as error: 63 | log.exception('Unable to fetch OpenID configuration from Cognito. Error: %s', error) 64 | # We can't fetch an up to date openid-config, so authentication will not work. 65 | if self._config_timestamp: 66 | raise HTTPException( 67 | status_code=status.HTTP_401_UNAUTHORIZED, 68 | detail='Connection to Cognito is down. Unable to fetch provider configuration', 69 | headers={'WWW-Authenticate': 'Bearer'}, 70 | ) 71 | else: 72 | raise RuntimeError(f'Unable to fetch provider information. {error}') 73 | 74 | log.info('Loaded settings from Cognito.') 75 | log.info('Issuer: %s', self.issuer) 76 | 77 | async def _load_openid_config(self) -> None: 78 | """ 79 | Load openid config, fetch signing keys 80 | """ 81 | async with AsyncClient(timeout=10) as client: 82 | log.info('Fetching OpenID Connect config from %s', self.openid_url) 83 | openid_response = await client.get(self.openid_url) 84 | openid_response.raise_for_status() 85 | openid_cfg = openid_response.json() 86 | 87 | self.issuer = openid_cfg['issuer'] 88 | 89 | jwks_uri = openid_cfg['jwks_uri'] 90 | log.info('Fetching jwks from %s', jwks_uri) 91 | jwks_response = await client.get(jwks_uri) 92 | jwks_response.raise_for_status() 93 | self._load_keys(jwks_response.json()['keys']) 94 | 95 | def _load_keys(self, keys: list[dict[str, Any]]) -> None: 96 | """ 97 | Create certificates based on signing keys and store them 98 | """ 99 | self.signing_keys: dict[str, CryptographyRSAKey] = {} 100 | for key in keys: 101 | if key.get('use') == 'sig': # Only care about keys that are used for signatures, not encryption 102 | log.debug('Loading public key from certificate: %s', key) 103 | cert_obj = jwk.construct(key, 'RS256') 104 | if kid := key.get('kid'): 105 | self.signing_keys[kid] = cert_obj.public_key() 106 | 107 | 108 | class CognitoAuthorizationCodeBearerBase(SecurityBase): 109 | def __init__(self, auto_error: bool = True) -> None: 110 | self.auto_error = auto_error 111 | 112 | self.openid_config: OpenIdConfig = OpenIdConfig() 113 | self.oauth = OAuth2AuthorizationCodeBearer( 114 | authorizationUrl='https://auth.klepp.me/oauth2/authorize', 115 | tokenUrl='https://auth.klepp.me/oauth2/token', 116 | scopes={'openid': 'openid'}, 117 | scheme_name='CognitoAuth', 118 | auto_error=True, 119 | ) 120 | self.model = self.oauth.model 121 | self.scheme_name: str = 'Cognito' 122 | 123 | async def __call__(self, request: Request, security_scopes: SecurityScopes) -> CognitoUser | None: 124 | """ 125 | Extends call to also validate the token. 126 | """ 127 | try: 128 | access_token = await self.oauth(request=request) 129 | try: 130 | # Extract header information of the token. 131 | header: dict[str, str] = jwt.get_unverified_header(token=access_token) or {} 132 | claims: dict[str, Any] = jwt.get_unverified_claims(token=access_token) or {} 133 | except Exception as error: 134 | log.warning('Malformed token received. %s. Error: %s', access_token, error, exc_info=True) 135 | raise InvalidAuth(detail='Invalid token format') 136 | 137 | for scope in security_scopes.scopes: 138 | token_scope_string = claims.get('scp', '') 139 | if isinstance(token_scope_string, str): 140 | token_scopes = token_scope_string.split(' ') 141 | if scope not in token_scopes: 142 | raise InvalidAuth('Required scope missing') 143 | else: 144 | raise InvalidAuth('Token contains invalid formatted scopes') 145 | 146 | # Load new config if old 147 | await self.openid_config.load_config() 148 | 149 | # Use the `kid` from the header to find a matching signing key to use 150 | try: 151 | if key := self.openid_config.signing_keys.get(header.get('kid', '')): 152 | # We require and validate all fields in a Cognito token 153 | options = { 154 | 'verify_signature': True, 155 | 'verify_aud': False, 156 | 'verify_iat': True, 157 | 'verify_exp': True, 158 | 'verify_nbf': False, 159 | 'verify_iss': True, 160 | 'verify_sub': True, 161 | 'verify_jti': True, 162 | 'verify_at_hash': True, 163 | 'require_aud': False, 164 | 'require_iat': True, 165 | 'require_exp': True, 166 | 'require_nbf': False, 167 | 'require_iss': True, 168 | 'require_sub': True, 169 | 'require_jti': False, 170 | 'require_at_hash': False, 171 | 'leeway': 0, 172 | } 173 | # Validate token 174 | token = jwt.decode( 175 | access_token, 176 | key=key, # noqa 177 | algorithms=['RS256'], 178 | issuer=self.openid_config.issuer, 179 | options=options, 180 | ) 181 | # Attach the user to the request. Can be accessed through `request.state.user` 182 | user: CognitoUser = CognitoUser(**token) 183 | request.state.user = user 184 | return user 185 | except JWTClaimsError as error: 186 | log.info('Token contains invalid claims. %s', error) 187 | raise InvalidAuth(detail='Token contains invalid claims') 188 | except ExpiredSignatureError as error: 189 | log.info('Token signature has expired. %s', error) 190 | raise InvalidAuth(detail='Token signature has expired') 191 | except JWTError as error: 192 | log.warning('Invalid token. Error: %s', error, exc_info=True) 193 | raise InvalidAuth(detail='Unable to validate token') 194 | except Exception as error: 195 | # Extra failsafe in case of a bug in a future version of the jwt library 196 | log.exception('Unable to process jwt token. Uncaught error: %s', error) 197 | raise InvalidAuth(detail='Unable to process token') 198 | log.warning('Unable to verify token. No signing keys found') 199 | raise InvalidAuth(detail='Unable to verify token, no signing keys found') 200 | except (HTTPException, InvalidAuth): 201 | if not self.auto_error: 202 | return None 203 | raise 204 | 205 | 206 | cognito_scheme = CognitoAuthorizationCodeBearerBase() 207 | cognito_scheme_or_anonymous = CognitoAuthorizationCodeBearerBase(auto_error=False) 208 | 209 | 210 | async def cognito_signed_in( 211 | cognito_user: CognitoUser = Depends(cognito_scheme), 212 | db_session: AsyncSession = Depends(yield_db_session), 213 | ) -> User: 214 | """ 215 | Creates a user in the DB for a signed in Cognito user if it don't exist 216 | """ 217 | select_user = select(User).where(User.name == cognito_user.username) 218 | user_query = await db_session.exec(select_user) # type: ignore 219 | user = user_query.one_or_none() 220 | if not user: 221 | new_user = User(name=cognito_user.username) 222 | db_session.add(new_user) 223 | await db_session.commit() 224 | await db_session.refresh(new_user) 225 | return new_user 226 | return user # type: ignore 227 | --------------------------------------------------------------------------------