├── index.js ├── log └── .gitkeep ├── scratch ├── planning └── .gitkeep ├── results └── .gitkeep ├── steps └── .gitkeep ├── tests └── .gitkeep ├── weights └── .gitkeep ├── vc ├── service │ ├── helper │ │ ├── rife │ │ │ ├── __init__.py │ │ │ ├── requirements.txt │ │ │ └── model │ │ │ │ ├── warplayer.py │ │ │ │ ├── laplacian.py │ │ │ │ └── RIFE_HDv3.py │ │ ├── inpainting │ │ │ └── __init__.py │ │ ├── abme │ │ │ ├── correlation_package │ │ │ │ ├── __init__.py │ │ │ │ ├── pyproject.toml │ │ │ │ ├── setup.py │ │ │ │ ├── nvcc setting.md │ │ │ │ ├── correlation_cuda_kernel.cuh │ │ │ │ └── correlation.py │ │ │ ├── model │ │ │ │ └── __init__.py │ │ │ ├── Upsample.py │ │ │ └── utils.py │ │ ├── midas │ │ │ ├── __init__.py │ │ │ └── midas │ │ │ │ ├── base_model.py │ │ │ │ └── midas_net.py │ │ ├── __init__.py │ │ ├── esrgan │ │ │ ├── __init__.py │ │ │ ├── train.py │ │ │ ├── archs │ │ │ │ ├── __init__.py │ │ │ │ └── discriminator_arch.py │ │ │ ├── models │ │ │ │ └── __init__.py │ │ │ └── data │ │ │ │ └── __init__.py │ │ ├── string.py │ │ ├── hash.py │ │ ├── dimensions.py │ │ ├── tier.py │ │ └── vqgan.py │ ├── provider │ │ ├── __init__.py │ │ └── api.py │ ├── __init__.py │ ├── job.py │ ├── isr.py │ └── file.py ├── q.py ├── r.py ├── app.py ├── event_listener │ ├── __init__.py │ ├── base.py │ └── generation_request.py ├── db.py ├── injector.py ├── migrate.py ├── model │ ├── __init__.py │ ├── base.py │ ├── user.py │ ├── generation_result.py │ └── generation_request.py ├── job │ ├── __init__.py │ └── base.py ├── auth.py ├── value_object │ ├── __init__.py │ ├── generation_progress.py │ └── generation_spec.py ├── main.py ├── uwsgi.ini ├── exception │ ├── third_party_exception.py │ ├── __init__.py │ ├── not_authenticated_exception.py │ ├── not_found_exception.py │ ├── tier_exception.py │ └── vc_exception.py ├── restore.py ├── manager │ ├── __init__.py │ ├── generation_result.py │ ├── user.py │ └── base.py ├── event │ ├── __init__.py │ ├── base.py │ └── generation_request.py ├── command │ ├── base.py │ ├── __init__.py │ ├── trigger.py │ ├── video.py │ ├── esrgan.py │ ├── abme.py │ ├── rife.py │ ├── inpainting.py │ ├── vqgan_clip.py │ ├── abme_steps.py │ └── rife_steps.py ├── controller │ ├── __init__.py │ └── base.py ├── hash.py ├── worker.py ├── cli.py ├── __init__.py ├── services.py ├── api.py └── validator │ └── generation_request.py ├── styles.txt ├── redis.conf.example ├── scss.sh ├── migrations ├── README ├── script.py.mako └── alembic.ini ├── app ├── assets │ ├── help.png │ ├── icon.png │ ├── logo.png │ ├── tezos.png │ ├── colours.png │ ├── github.png │ ├── logo-sm.png │ ├── patreon.png │ ├── twitter.png │ ├── add-image.png │ ├── add-video.png │ ├── news │ │ ├── foss.png │ │ ├── index.png │ │ └── ajmoon.png │ ├── placeholder.png │ ├── watermark-1280.png │ ├── watermark-400.png │ ├── watermark-533.png │ └── watermark-800.png ├── ts │ ├── models │ │ ├── base-model.ts │ │ ├── video-spec.ts │ │ ├── user.ts │ │ ├── generation-result.ts │ │ ├── generation-spec.ts │ │ ├── image-spec.ts │ │ └── generation-request.ts │ ├── elements │ │ ├── info.inc │ │ ├── login.ts │ │ ├── info.ts │ │ ├── generation-request-form │ │ │ ├── add-video-step.ts │ │ │ ├── add-spec.ts │ │ │ ├── image-spec-option.ts │ │ │ └── video-spec-form.ts │ │ ├── generation-requests.ts │ │ ├── chip.ts │ │ ├── base-element.ts │ │ ├── login-form.ts │ │ ├── chipset.ts │ │ ├── generation-request.ts │ │ ├── news │ │ │ └── nav.ts │ │ └── generation-request │ │ │ └── step.ts │ ├── helpers │ │ ├── env.ts │ │ ├── notification.ts │ │ ├── auth.ts │ │ └── status.ts │ ├── managers │ │ ├── user-manager.ts │ │ ├── generation-request-manager.ts │ │ └── base-manager.ts │ └── elements.ts ├── scss │ ├── _config.scss │ ├── _generation_requests.scss │ ├── _login_form.scss │ ├── chipset.scss │ ├── _generation_request.scss │ ├── chip.scss │ ├── _container.scss │ ├── _news.scss │ ├── vc.scss │ ├── generation_request_form │ │ ├── _video_spec_form.scss │ │ ├── _add_spec.scss │ │ ├── _add_video_step.scss │ │ ├── _image_spec_option.scss │ │ └── _image_spec_form.scss │ ├── vc.css.map │ ├── _normalize.scss │ ├── _header.scss │ ├── _info.scss │ ├── _theme.scss │ ├── _footer.scss │ ├── generation_request │ │ ├── _step.scss │ │ └── _details.scss │ ├── _generation_request_form.scss │ └── news │ │ └── _text.scss ├── index.html └── partials │ ├── footer.html │ └── header.html ├── public └── favicon.ico ├── s.sh ├── scripts ├── frontend.sh ├── migrate.sh ├── db.sh ├── aws.start.sh ├── aws.stop.sh ├── backend.sh ├── public.sh ├── dump.sh ├── build.sh ├── restore.sh ├── sh.sh └── run.sh ├── docker.flask.sh ├── restore.sh ├── docker ├── redis │ └── Dockerfile ├── nginx │ ├── Dockerfile │ ├── ssl │ │ ├── vc.local.crt │ │ └── vc.local.key │ └── conf │ │ └── flask.conf └── flask │ └── Dockerfile ├── local-wrapper.sh ├── update.sh ├── .dockerignore ├── zoom3d.py ├── restart.sh ├── dump.sh ├── image-high.sh ├── supervisor.api.conf ├── supervisor.worker.conf ├── sofar.sh ├── video.sh ├── video-high.sh ├── docker.worker.sh ├── zoom.sh ├── image.sh ├── index.sh ├── .bashrc ├── latest.py ├── requirements.worker.txt ├── .env.example ├── .gitignore ├── nginx.unsecure.conf ├── diagnosis.sh ├── LICENSE ├── vqgan ├── LICENSE ├── zoom.sh ├── random.sh ├── video_styler.sh ├── opt_tester.sh └── vqgan.yml ├── debug.html ├── argument.yml ├── package.json ├── Taskfile.yml ├── requirements.api.txt ├── tests.sh ├── docker-compose.yml ├── planning.py ├── isr.py └── nginx.conf /index.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /log/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scratch: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /planning/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /results/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /steps/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /weights/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vc/service/helper/rife/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vc/service/helper/inpainting/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /styles.txt: -------------------------------------------------------------------------------- 1 | high resolution 2 | upscale 3 | 4 | -------------------------------------------------------------------------------- /redis.conf.example: -------------------------------------------------------------------------------- 1 | requirepass password-goes-here 2 | -------------------------------------------------------------------------------- /scss.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | npm run scss 4 | 5 | -------------------------------------------------------------------------------- /vc/q.py: -------------------------------------------------------------------------------- 1 | from flask_rq import RQ 2 | 3 | q = RQ() 4 | -------------------------------------------------------------------------------- /vc/r.py: -------------------------------------------------------------------------------- 1 | from redis import Redis 2 | 3 | r = Redis() 4 | -------------------------------------------------------------------------------- /migrations/README: -------------------------------------------------------------------------------- 1 | Single-database configuration for Flask. 2 | -------------------------------------------------------------------------------- /vc/service/helper/abme/correlation_package/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /vc/app.py: -------------------------------------------------------------------------------- 1 | from vc import create_app 2 | 3 | app = create_app() 4 | -------------------------------------------------------------------------------- /vc/event_listener/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import VcEventListener 2 | -------------------------------------------------------------------------------- /vc/service/helper/midas/__init__.py: -------------------------------------------------------------------------------- 1 | from .run import run_depth 2 | -------------------------------------------------------------------------------- /vc/service/provider/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import ApiProvider 2 | -------------------------------------------------------------------------------- /vc/service/helper/__init__.py: -------------------------------------------------------------------------------- 1 | from .diagnosis import DiagnosisHelper 2 | -------------------------------------------------------------------------------- /vc/db.py: -------------------------------------------------------------------------------- 1 | from flask_sqlalchemy import SQLAlchemy 2 | 3 | db = SQLAlchemy() 4 | -------------------------------------------------------------------------------- /vc/injector.py: -------------------------------------------------------------------------------- 1 | from injector import Injector 2 | 3 | injector = Injector() 4 | -------------------------------------------------------------------------------- /vc/migrate.py: -------------------------------------------------------------------------------- 1 | from flask_migrate import Migrate 2 | 3 | migrate = Migrate() 4 | -------------------------------------------------------------------------------- /vc/model/__init__.py: -------------------------------------------------------------------------------- 1 | # DO NOT IMPORT MODELS HERE! IT CREATES A CIRCULAR IMPORT! 2 | -------------------------------------------------------------------------------- /app/assets/help.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/help.png -------------------------------------------------------------------------------- /app/assets/icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/icon.png -------------------------------------------------------------------------------- /app/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/logo.png -------------------------------------------------------------------------------- /app/assets/tezos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/tezos.png -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/public/favicon.ico -------------------------------------------------------------------------------- /vc/job/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import Job 2 | from .generation import GenerationJob 3 | -------------------------------------------------------------------------------- /app/assets/colours.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/colours.png -------------------------------------------------------------------------------- /app/assets/github.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/github.png -------------------------------------------------------------------------------- /app/assets/logo-sm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/logo-sm.png -------------------------------------------------------------------------------- /app/assets/patreon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/patreon.png -------------------------------------------------------------------------------- /app/assets/twitter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/twitter.png -------------------------------------------------------------------------------- /s.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | sudo supervisorctl -c /etc/supervisor/supervisord.conf $@ 4 | 5 | -------------------------------------------------------------------------------- /app/assets/add-image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/add-image.png -------------------------------------------------------------------------------- /app/assets/add-video.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/add-video.png -------------------------------------------------------------------------------- /app/assets/news/foss.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/news/foss.png -------------------------------------------------------------------------------- /app/assets/news/index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/news/index.png -------------------------------------------------------------------------------- /scripts/frontend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | npm install 4 | npx webpack build --node-env=private 5 | -------------------------------------------------------------------------------- /app/assets/news/ajmoon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/news/ajmoon.png -------------------------------------------------------------------------------- /app/assets/placeholder.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/placeholder.png -------------------------------------------------------------------------------- /vc/auth.py: -------------------------------------------------------------------------------- 1 | from flask_httpauth import HTTPTokenAuth 2 | 3 | auth = HTTPTokenAuth(scheme='Bearer') 4 | -------------------------------------------------------------------------------- /app/assets/watermark-1280.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/watermark-1280.png -------------------------------------------------------------------------------- /app/assets/watermark-400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/watermark-400.png -------------------------------------------------------------------------------- /app/assets/watermark-533.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/watermark-533.png -------------------------------------------------------------------------------- /app/assets/watermark-800.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alex-moon/vc/HEAD/app/assets/watermark-800.png -------------------------------------------------------------------------------- /vc/event_listener/base.py: -------------------------------------------------------------------------------- 1 | class VcEventListener: 2 | def on(self, *args, **kwargs): 3 | pass 4 | -------------------------------------------------------------------------------- /vc/value_object/__init__.py: -------------------------------------------------------------------------------- 1 | from .generation_spec import GenerationSpec, ImageSpec, VideoSpec, VideoStepSpec 2 | -------------------------------------------------------------------------------- /vc/main.py: -------------------------------------------------------------------------------- 1 | from vc import create_app 2 | 3 | if __name__ == '__main__': 4 | create_app().run(host='0.0.0.0') 5 | -------------------------------------------------------------------------------- /docker.flask.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python3 -m venv venv 4 | source venv/bin/activate 5 | uwsgi --ini vc/uwsgi.ini 6 | -------------------------------------------------------------------------------- /vc/uwsgi.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | module = vc:create_app() 3 | 4 | master = true 5 | processes = 1 6 | 7 | http = 0.0.0.0:5000 8 | -------------------------------------------------------------------------------- /app/ts/models/base-model.ts: -------------------------------------------------------------------------------- 1 | export class BaseModel { 2 | id ?: number; 3 | created ?: string; 4 | updated ?: string; 5 | } 6 | -------------------------------------------------------------------------------- /scripts/migrate.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source venv/bin/activate 4 | export FLASK_APP=vc.app:app 5 | flask db migrate 6 | flask db upgrade 7 | -------------------------------------------------------------------------------- /app/ts/models/video-spec.ts: -------------------------------------------------------------------------------- 1 | import {VideoStepSpec} from "./image-spec"; 2 | 3 | export class VideoSpec { 4 | steps: VideoStepSpec[]; 5 | } 6 | -------------------------------------------------------------------------------- /vc/exception/third_party_exception.py: -------------------------------------------------------------------------------- 1 | from .vc_exception import VcException 2 | 3 | 4 | class ThirdPartyException(VcException): 5 | pass 6 | -------------------------------------------------------------------------------- /vc/service/__init__.py: -------------------------------------------------------------------------------- 1 | from .job import JobService 2 | from .provider.api import ApiProvider 3 | from .queue import QueueService, JobSerializer 4 | -------------------------------------------------------------------------------- /vc/service/helper/abme/model/__init__.py: -------------------------------------------------------------------------------- 1 | from .SBMNet import SBMENet 2 | from .ABMNet import ABMRNet 3 | from .SynthesisNet import SynthesisNet 4 | -------------------------------------------------------------------------------- /restore.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source venv/bin/activate 4 | sudo service supervisor stop 5 | python3 -m vc.restore 6 | sudo service supervisor start 7 | -------------------------------------------------------------------------------- /app/scss/_config.scss: -------------------------------------------------------------------------------- 1 | $mobile-width: 600px; 2 | 3 | @mixin mobile { 4 | @media (max-width: #{$mobile-width - 1px}) { 5 | @content; 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /docker/redis/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM redis:6.2.4 2 | COPY redis.conf /usr/local/etc/redis/redis.conf 3 | CMD [ "redis-server", "/usr/local/etc/redis/redis.conf" ] 4 | -------------------------------------------------------------------------------- /vc/service/helper/esrgan/__init__.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | from .archs import * 3 | from .data import * 4 | from .models import * 5 | from .utils import * 6 | -------------------------------------------------------------------------------- /vc/restore.py: -------------------------------------------------------------------------------- 1 | from vc import create_app 2 | from vc.db import db 3 | 4 | if __name__ == '__main__': 5 | create_app() 6 | db.drop_all() 7 | db.create_all() 8 | -------------------------------------------------------------------------------- /vc/job/base.py: -------------------------------------------------------------------------------- 1 | class Job: 2 | # @todo could define id/key? coule define args? latter would be useful! 3 | 4 | def handle(self, *args, **kwargs): 5 | pass 6 | -------------------------------------------------------------------------------- /vc/manager/__init__.py: -------------------------------------------------------------------------------- 1 | from .generation_request import GenerationRequestManager 2 | from .generation_result import GenerationResultManager 3 | from .user import UserManager 4 | -------------------------------------------------------------------------------- /vc/service/helper/rife/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy>=1.16 2 | tqdm>=4.35.0 3 | sk-video>=1.1.10 4 | torch==1.6.0 5 | opencv-python>=4.1.2 6 | moviepy>=1.0.3 7 | torchvision==0.7.0 8 | -------------------------------------------------------------------------------- /scripts/db.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source /opt/vc/.env 4 | 5 | echo "Connecting to $DB_HOST" 6 | 7 | PGPASSWORD=$DB_PASS psql --user=$DB_USER --host=$DB_HOST --port=$DB_PORT $DB_NAME 8 | -------------------------------------------------------------------------------- /app/scss/_generation_requests.scss: -------------------------------------------------------------------------------- 1 | vc-generation-requests { 2 | flex-grow: 1; 3 | background-color: #63575b; 4 | overflow: auto; 5 | 6 | @import "generation_request"; 7 | } 8 | -------------------------------------------------------------------------------- /app/ts/models/user.ts: -------------------------------------------------------------------------------- 1 | import {BaseModel} from "./base-model"; 2 | 3 | export class User extends BaseModel { 4 | name ?: string; 5 | email ?: string; 6 | tier ?: number; 7 | } 8 | -------------------------------------------------------------------------------- /local-wrapper.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | python3 -m venv venv 4 | source venv/bin/activate 5 | export FLASK_ENV=development 6 | export FLASK_APP="vc:create_app()" 7 | flask run --host=0.0.0.0 8 | -------------------------------------------------------------------------------- /vc/event/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import VcEvent, VcEventDispatcher 2 | from .generation_request import ( 3 | GenerationRequestCreatedEvent, 4 | GenerationRequestCancelledEvent, 5 | ) 6 | -------------------------------------------------------------------------------- /app/ts/models/generation-result.ts: -------------------------------------------------------------------------------- 1 | import {BaseModel} from "./base-model"; 2 | 3 | export class GenerationResult extends BaseModel { 4 | url: string; 5 | url_watermarked: string; 6 | } 7 | -------------------------------------------------------------------------------- /update.sh: -------------------------------------------------------------------------------- 1 | ffmpeg -y -i input/input-%04d.png -b:v 8M -c:v h264_nvenc -pix_fmt yuv420p -strict -2 -filter:v minterpolate=mi_mode=mci:mc_mode=aobmc:vsbmc=1:fps=60 results/video-sixth-attempt.mp4 2 | -------------------------------------------------------------------------------- /app/ts/elements/info.inc: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

Help text would go here

5 |
6 |
7 |
8 | -------------------------------------------------------------------------------- /vc/command/base.py: -------------------------------------------------------------------------------- 1 | class BaseCommand: 2 | description: str 3 | args = [] 4 | 5 | def run(self, args): 6 | self.handle(args) 7 | 8 | def handle(self, args): 9 | pass 10 | -------------------------------------------------------------------------------- /vc/controller/__init__.py: -------------------------------------------------------------------------------- 1 | from .generation_request import * 2 | from .user import * 3 | 4 | 5 | def init_app(app): 6 | # DO NOT REMOVE THIS! THIS FILE MUST BE IMPORTED FOR CONTROLLERS TO WORK 7 | pass 8 | -------------------------------------------------------------------------------- /app/ts/models/generation-spec.ts: -------------------------------------------------------------------------------- 1 | import {VideoSpec} from "./video-spec"; 2 | import {ImageSpec} from "./image-spec"; 3 | 4 | export class GenerationSpec { 5 | images: ImageSpec[]; 6 | videos: VideoSpec[]; 7 | } 8 | -------------------------------------------------------------------------------- /vc/service/helper/abme/correlation_package/pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | # Minimum requirements for the build system to execute. 3 | requires = ["setuptools", "wheel", "numpy", "torch"] # PEP 508 specifications. 4 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | venv 2 | venv.mac 3 | node_modules 4 | log 5 | results 6 | mesh 7 | depth 8 | checkpoints 9 | taming-transformers 10 | samples 11 | .git 12 | BoostingMonocularDepth 13 | CLIP 14 | MiDaS 15 | pgsql 16 | -------------------------------------------------------------------------------- /scripts/aws.start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | id=$(aws ec2 describe-instances | jq -r '.[][].Instances[] | select(.Tags[].Key=="Name" and .Tags[].Value=="vc") | .InstanceId') 4 | aws ec2 start-instances --instance-ids $id 5 | 6 | -------------------------------------------------------------------------------- /scripts/aws.stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | id=$(aws ec2 describe-instances | jq -r '.[][].Instances[] | select(.Tags[].Key=="Name" and .Tags[].Value=="vc") | .InstanceId') 4 | aws ec2 stop-instances --instance-ids $id 5 | 6 | -------------------------------------------------------------------------------- /vc/manager/generation_result.py: -------------------------------------------------------------------------------- 1 | from vc.manager.base import Manager 2 | from vc.model.generation_result import GenerationResult 3 | 4 | 5 | class GenerationResultManager(Manager): 6 | model_class = GenerationResult 7 | -------------------------------------------------------------------------------- /app/scss/_login_form.scss: -------------------------------------------------------------------------------- 1 | vc-login-form { 2 | background-color: #d7e6d6; 3 | 4 | vc-login { 5 | display: block; 6 | padding: 16px; 7 | } 8 | 9 | @import "generation_request_form"; 10 | } 11 | -------------------------------------------------------------------------------- /scripts/backend.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker exec -it $(docker ps | grep vc_flask | awk '{ print $1 }') bash -c " 4 | source venv/bin/activate 5 | 6 | pip3 install -r requirements.api.txt 7 | 8 | scripts/migrate.sh 9 | " 10 | -------------------------------------------------------------------------------- /app/scss/chipset.scss: -------------------------------------------------------------------------------- 1 | vc-chipset { 2 | display: block; 3 | 4 | .chipset { 5 | display: flex; 6 | flex-wrap: wrap; 7 | justify-content: flex-start; 8 | 9 | @import "chip"; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /app/ts/helpers/env.ts: -------------------------------------------------------------------------------- 1 | export class EnvHelper { 2 | static get useLocal(): boolean { 3 | return (window as any).env.useLocal; 4 | } 5 | static get host(): string { 6 | return (window as any).env.host; 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /vc/exception/__init__.py: -------------------------------------------------------------------------------- 1 | from .not_found_exception import NotFoundException 2 | from .not_authenticated_exception import NotAuthenticatedException 3 | from .vc_exception import VcException 4 | from .third_party_exception import ThirdPartyException 5 | -------------------------------------------------------------------------------- /vc/hash.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from vc.service.helper.hash import HashHelper 3 | 4 | if len(sys.argv) < 2: 5 | print('Usage: %s ' % sys.argv[0]) 6 | print() 7 | exit(9) 8 | 9 | print(HashHelper.get(sys.argv[1])) 10 | print() 11 | -------------------------------------------------------------------------------- /zoom3d.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | from vc.service.inpainting import InpaintingService, InpaintingOptions 4 | 5 | config = yaml.load(open('argument.yml', 'r')) 6 | args = InpaintingOptions(**config) 7 | service = InpaintingService() 8 | service.handle(args) 9 | -------------------------------------------------------------------------------- /restart.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | git pull origin $(git rev-parse --abbrev-ref HEAD) 4 | if [[ ! -z "$1" ]]; then 5 | ./restore.sh 6 | else 7 | sudo service supervisor restart 8 | fi 9 | npx webpack 10 | tail -f /var/log/supervisor/* /opt/vc/log/* 11 | -------------------------------------------------------------------------------- /dump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | touch backup.sql 4 | mv backup.sql backup.sql.bak 5 | pass=$(cat .env | grep SQLALCHEMY_DATABASE_URI | awk -F':' '{ print $3 }' | awk -F'@' '{ print $1 }') 6 | PGPASSWORD=$pass pg_dump --user=vc --host=127.0.0.1 vc "$@" > backup.sql 7 | 8 | -------------------------------------------------------------------------------- /vc/exception/not_authenticated_exception.py: -------------------------------------------------------------------------------- 1 | from .vc_exception import VcException 2 | 3 | 4 | class NotAuthenticatedException(VcException): 5 | code = 403 6 | 7 | def __init__(self, token: str): 8 | super().__init__("Invalid token: [%s]" % token) 9 | -------------------------------------------------------------------------------- /docker/nginx/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM nginx:latest 2 | 3 | RUN apt-get -y update 4 | RUN apt-get -y install vim less curl gnupg apt-transport-https 5 | 6 | COPY docker/nginx/conf/flask.conf /etc/nginx/nginx.conf 7 | 8 | EXPOSE 80 9 | EXPOSE 443 10 | 11 | WORKDIR /etc/nginx 12 | -------------------------------------------------------------------------------- /scripts/public.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | npm install 4 | npx webpack build --node-env=public 5 | ssh vos " 6 | cp /opt/vc/public/assets/latest.json /opt/vc/latest.bak 7 | " 8 | rsync -av --delete public/ vos:/opt/vc/public/ 9 | ssh vos " 10 | cd /opt/vc/ 11 | python3 latest.py 12 | " 13 | -------------------------------------------------------------------------------- /vc/exception/not_found_exception.py: -------------------------------------------------------------------------------- 1 | from .vc_exception import VcException 2 | 3 | 4 | class NotFoundException(VcException): 5 | code = 404 6 | 7 | def __init__(self, model_class: str, id_: int): 8 | super().__init__("Model not found: [%s] for id [%s]" % (model_class, id_)) 9 | -------------------------------------------------------------------------------- /scripts/dump.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source /opt/vc/.env 4 | 5 | echo "Taking dump from $DB_HOST - hit any key to continue, or CTRL+C to cancel" 6 | read 7 | 8 | mv backup.sql backup.sql.bak 9 | 10 | PGPASSWORD=$DB_PASS pg_dump --user=$DB_USER --host=$DB_HOST --port=$DB_PORT $DB_NAME > backup.sql 11 | -------------------------------------------------------------------------------- /vc/worker.py: -------------------------------------------------------------------------------- 1 | from vc.service.queue import QueueService 2 | 3 | from . import create_app 4 | from .injector import injector 5 | from .job import * 6 | 7 | if __name__ == '__main__': 8 | app = create_app() 9 | queue_service = injector.get(QueueService) 10 | queue_service.get_worker().work() 11 | -------------------------------------------------------------------------------- /image-high.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "$@" ]]; then 4 | echo "Usage: $0 <prompt>" 5 | exit 6 | fi 7 | 8 | source venv/bin/activate 9 | python3 generate.py -p "$@" -s 720 720 10 | slug=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9][^a-z0-9]*/-/g') 11 | mv output.png results/$slug.png 12 | 13 | -------------------------------------------------------------------------------- /supervisor.api.conf: -------------------------------------------------------------------------------- 1 | [program:uwsgi] 2 | user=ubuntu 3 | directory=/opt/vc 4 | command=/opt/vc/venv/bin/uwsgi --ini /opt/vc/vc/uwsgi.ini 5 | autostart=true 6 | autorestart=true 7 | stderr_logfile=/var/log/supervisor/vc.error.log 8 | stdout_logfile=/var/log/supervisor/vc.log 9 | stopsignal=INT 10 | environment=SERVICE=uwsgi 11 | -------------------------------------------------------------------------------- /vc/command/__init__.py: -------------------------------------------------------------------------------- 1 | from .base import BaseCommand 2 | from .inpainting import InpaintingCommand 3 | from .vqgan_clip import VqganClipCommand 4 | from .esrgan import EsrganCommand 5 | from .rife import RifeCommand 6 | from .rife_steps import RifeStepsCommand 7 | from .video import VideoCommand 8 | from .trigger import TriggerCommand 9 | -------------------------------------------------------------------------------- /app/index.html: -------------------------------------------------------------------------------- 1 | <%= require('./partials/header.html?raw') %> 2 | <div class="content"> 3 | <!-- vc-info></vc-info --> 4 | <div class="scrollable"> 5 | <vc-login-form></vc-login-form> 6 | <vc-generation-requests></vc-generation-requests> 7 | </div> 8 | </div> 9 | <%= require('./partials/footer.html?raw') %> 10 | -------------------------------------------------------------------------------- /supervisor.worker.conf: -------------------------------------------------------------------------------- 1 | [program:worker] 2 | user=ubuntu 3 | directory=/opt/vc 4 | command=/opt/vc/venv/bin/python3 -m vc.worker 5 | autostart=true 6 | autorestart=true 7 | stderr_logfile=/var/log/supervisor/vc.worker.error.log 8 | stdout_logfile=/var/log/supervisor/vc.worker.log 9 | stopsignal=INT 10 | environment=SERVICE=worker 11 | -------------------------------------------------------------------------------- /sofar.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ffmpeg -y -i "steps/%04d.png" \ 4 | -b:v 8M -c:v h264_nvenc -pix_fmt yuv420p -strict -2 \ 5 | -filter:v "minterpolate='mi_mode=mci:mc_mode=aobmc:me_mode=bidir:vsbmc=1:fps=60'" \ 6 | sofar.mp4 7 | 8 | ffmpeg -i sofar.mp4 -i app/assets/watermark.png -filter_complex "overlay=0:0" \ 9 | sofar-watermarked.mp4 10 | -------------------------------------------------------------------------------- /video.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "$@" ]]; then 4 | echo "Usage: $0 <prompt>" 5 | exit 6 | fi 7 | 8 | source venv/bin/activate 9 | python3 generate.py -p "$@" -s 400 400 --video 10 | slug=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9][^a-z0-9]*/-/g') 11 | mv output.mp4 results/$slug.mp4 12 | mv steps/500.png results/$slug.png 13 | 14 | -------------------------------------------------------------------------------- /video-high.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "$@" ]]; then 4 | echo "Usage: $0 <prompt>" 5 | exit 6 | fi 7 | 8 | source venv/bin/activate 9 | python3 generate.py -p "$@" -s 720 720 --video 10 | slug=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9][^a-z0-9]*/-/g') 11 | mv output.mp4 results/$slug.mp4 12 | mv steps/500.png results/$slug.png 13 | 14 | -------------------------------------------------------------------------------- /docker.worker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "$(grep 'ROLE.*worker' .env)" ]]; then 4 | echo "Role is not worker, so not starting worker." 5 | sleep 999999999 6 | exit 7 | fi 8 | 9 | Xvfb :0 -screen 0 1024x768x24 -ac +extension GLX +render -noreset & 10 | export DISPLAY=:0 11 | 12 | python3 -m venv venv 13 | source venv/bin/activate 14 | python3 -m vc.worker 15 | -------------------------------------------------------------------------------- /vc/exception/tier_exception.py: -------------------------------------------------------------------------------- 1 | from .vc_exception import VcException 2 | 3 | 4 | class TierException(VcException): 5 | code = 403 6 | 7 | def __init__(self, tier: str, info: str = None): 8 | message = "Minimum tier required: [%s]" % tier 9 | if info is not None: 10 | message += ': %s' % info 11 | super().__init__(message) 12 | -------------------------------------------------------------------------------- /vc/value_object/generation_progress.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | 4 | @dataclass 5 | class GenerationProgress: 6 | steps_total: int 7 | steps_completed: int 8 | name: str 9 | preview: str = None 10 | result: str = None 11 | result_watermarked: str = None 12 | interim: str = None 13 | interim_watermarked: str = None 14 | 15 | -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | red="$(tput setaf 1)" 4 | green="$(tput setaf 2)" 5 | yellow="$(tput setaf 3)" 6 | reset="$(tput sgr0)" 7 | function red() { echo "${red}${@}${reset}"; } 8 | function green() { echo "${green}${@}${reset}"; } 9 | function yellow() { echo "${yellow}${@}${reset}"; } 10 | 11 | green "Building images" 12 | docker-compose build 13 | 14 | green Done 15 | -------------------------------------------------------------------------------- /vc/service/helper/esrgan/train.py: -------------------------------------------------------------------------------- 1 | # flake8: noqa 2 | import os.path as osp 3 | from basicsr.train import train_pipeline 4 | 5 | import vc.service.helper.esrgan.archs 6 | import vc.service.helper.esrgan.data 7 | import vc.service.helper.esrgan.models 8 | 9 | if __name__ == '__main__': 10 | root_path = osp.abspath(osp.join(__file__, osp.pardir, osp.pardir)) 11 | train_pipeline(root_path) 12 | -------------------------------------------------------------------------------- /app/scss/_generation_request.scss: -------------------------------------------------------------------------------- 1 | vc-generation-request { 2 | display: block; 3 | margin: $border-width $border-width 0; 4 | border-radius: $border-radius; 5 | box-sizing: border-box; 6 | 7 | .request { 8 | display: flex; 9 | flex-direction: column; 10 | 11 | @import "generation_request/summary"; 12 | @import "generation_request/details"; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /zoom.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "$@" ]]; then 4 | echo "Usage: $0 <prompt>" 5 | exit 6 | fi 7 | 8 | source venv/bin/activate 9 | slug=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9][^a-z0-9]*/-/g') 10 | 11 | if [[ -f "results/$slug.png" ]]; then 12 | echo "results/$slug.png exists - skipping" 13 | exit 14 | fi 15 | 16 | python3 generate.py -p "$@" -s 400 400 17 | mv output.png results/$slug.png 18 | -------------------------------------------------------------------------------- /vc/service/helper/string.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | 3 | 4 | class StringHelper: 5 | @classmethod 6 | def timedelta(cls, d: timedelta): 7 | s = d.seconds 8 | hours, remainder = divmod(s, 3600) 9 | minutes, seconds = divmod(remainder, 60) 10 | return '{hours}h {minutes}m {seconds}s'.format( 11 | hours=hours, 12 | minutes=minutes, 13 | seconds=seconds 14 | ) 15 | -------------------------------------------------------------------------------- /image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ -z "$@" ]]; then 4 | echo "Usage: $0 <prompt>" 5 | exit 6 | fi 7 | 8 | source venv/bin/activate 9 | slug=$(echo "$@" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9][^a-z0-9]*/-/g') 10 | 11 | if [[ -f "results/$slug.png" ]]; then 12 | echo "results/$slug.png exists - skipping" 13 | exit 14 | fi 15 | 16 | python3 generate.py -in gradient -i 200 -p "$@" -s 400 400 17 | mv output.png results/$slug.png 18 | -------------------------------------------------------------------------------- /app/scss/chip.scss: -------------------------------------------------------------------------------- 1 | vc-chip { 2 | .chip { 3 | margin: 8px 8px 0 0; 4 | font-family: sans-serif; 5 | background-color: #535c69; 6 | color: #f9f8ef; 7 | border-radius: 8px; 8 | padding: 8px; 9 | opacity: 0.8; 10 | width: auto; 11 | display: flex; 12 | align-items: center; 13 | } 14 | .remove { 15 | margin-left: 4px; 16 | color: #f9f8ef !important; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /vc/service/helper/midas/midas/base_model.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | class BaseModel(torch.nn.Module): 5 | def load(self, path): 6 | """Load model from file. 7 | 8 | Args: 9 | path (str): file path 10 | """ 11 | parameters = torch.load(path, map_location=torch.device('cpu')) 12 | 13 | if "optimizer" in parameters: 14 | parameters = parameters["model"] 15 | 16 | self.load_state_dict(parameters) 17 | -------------------------------------------------------------------------------- /app/ts/managers/user-manager.ts: -------------------------------------------------------------------------------- 1 | import {User} from "../models/user"; 2 | import {BaseManager} from "./base-manager"; 3 | 4 | // @todo BaseManager 5 | export class UserManager extends BaseManager<User> { 6 | base_url = '/api/user/me' 7 | 8 | public user: User; 9 | 10 | get(): Promise<User> { 11 | return this.fetch().then((user) => { 12 | user = user as User; 13 | this.user = user; 14 | return user; 15 | }); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /scripts/restore.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | source /opt/vc/.env 4 | 5 | set -x 6 | 7 | dbp() { 8 | PGPASSWORD=$DB_PASS psql --user=$DB_USER --host=$DB_HOST --port=$DB_PORT postgres -c "$@" 9 | } 10 | dbp "SELECT pg_terminate_backend(pid) 11 | FROM pg_stat_activity 12 | WHERE pid <> pg_backend_pid() 13 | AND datname = 'vc'" 14 | dbp "DROP DATABASE vc" 15 | dbp "CREATE DATABASE vc WITH OWNER vc" 16 | 17 | PGPASSWORD=$DB_PASS psql --user=$DB_USER --host=$DB_HOST --port=$DB_PORT $DB_NAME < backup.sql 18 | -------------------------------------------------------------------------------- /index.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | function doit () { 4 | rsync -a vc:/opt/vc/results/ results/ 5 | 6 | echo "window.imagePaths = [" > images.js 7 | 8 | for i in $(ls -1t results/*.{png,mp4}); do 9 | echo "'$i'," >> images.js 10 | done 11 | 12 | echo "];window.handler.drawImages();" >> images.js 13 | 14 | echo "index.sh: last pull:" $(date) 15 | } 16 | 17 | if [[ -z "$1" ]]; then 18 | doit 19 | else 20 | while true; do 21 | doit 22 | sleep 150 23 | clear 24 | done 25 | fi 26 | -------------------------------------------------------------------------------- /vc/exception/vc_exception.py: -------------------------------------------------------------------------------- 1 | class VcException(Exception): 2 | message = "An error occurred" 3 | 4 | def __init__(self, message: str = None): 5 | super().__init__() 6 | if message is not None: 7 | self.message = message 8 | 9 | def __repr__(self): 10 | return self.message 11 | 12 | def __str__(self): 13 | return self.message 14 | 15 | @classmethod 16 | def for_exception(cls, e: Exception): 17 | return cls(getattr(e, 'message', str(e))) 18 | -------------------------------------------------------------------------------- /app/scss/_container.scss: -------------------------------------------------------------------------------- 1 | @import "news"; 2 | 3 | .content { 4 | flex-grow: 1; 5 | overflow: hidden; 6 | display: flex; 7 | 8 | @include mobile { 9 | flex-direction: column; 10 | } 11 | 12 | @import "chipset"; 13 | @import "info"; 14 | 15 | .scrollable { 16 | overflow: auto; 17 | flex-grow: 1; 18 | display: flex; 19 | flex-direction: column; 20 | 21 | @import "login_form"; 22 | @import "generation_requests"; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /vc/event/base.py: -------------------------------------------------------------------------------- 1 | from typing import Callable 2 | from whistle import Event, EventDispatcher 3 | 4 | 5 | class VcEvent(Event): 6 | id: str 7 | 8 | 9 | class VcEventDispatcher: 10 | dispatcher: EventDispatcher 11 | 12 | def __init__(self): 13 | self.dispatcher = EventDispatcher() 14 | 15 | def dispatch(self, event: VcEvent): 16 | self.dispatcher.dispatch(event.id, event) 17 | 18 | def register(self, event_id, fn: Callable): 19 | self.dispatcher.add_listener(event_id, fn) 20 | -------------------------------------------------------------------------------- /vc/service/job.py: -------------------------------------------------------------------------------- 1 | from injector import inject 2 | 3 | from vc.service.queue import QueueService 4 | 5 | 6 | class JobService: 7 | queue_service: QueueService 8 | 9 | @inject 10 | def __init__(self, queue_service: QueueService): 11 | self.queue_service = queue_service 12 | 13 | def enqueue(self, job: str, *args) -> str: 14 | return self.queue_service.enqueue(job + '.handle', args=args) 15 | 16 | def cancel(self, hash: str): 17 | self.queue_service.cancel_and_stop(hash) 18 | -------------------------------------------------------------------------------- /.bashrc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | pyv () { 4 | python -m trace -t $1 | grep $1 5 | } 6 | 7 | venv() { 8 | python3 -m venv venv 9 | source venv/bin/activate 10 | # pip install -r requirements.txt 11 | } 12 | 13 | run() { 14 | export FLASK_ENV=development 15 | export FLASK_APP="vc:create_app()" 16 | flask run --host=0.0.0.0 17 | } 18 | 19 | 20 | # needed for inpainting - see docker/flask/Dockerfile 21 | # Xvfb :0 -screen 0 1024x768x24 -ac +extension GLX +render -noreset & 22 | # export DISPLAY=:0 23 | 24 | venv 25 | -------------------------------------------------------------------------------- /app/ts/models/image-spec.ts: -------------------------------------------------------------------------------- 1 | export class ImageSpec { 2 | texts: string[] = []; 3 | styles: string[] = []; 4 | iterations ?: number; 5 | upscale ?: boolean; 6 | } 7 | 8 | export class VideoStepSpec extends ImageSpec { 9 | init_iterations ?: number; 10 | epochs ?: number; 11 | x_velocity ?: number; 12 | y_velocity ?: number; 13 | z_velocity ?: number; 14 | pan_velocity ?: number; 15 | tilt_velocity ?: number; 16 | roll_velocity ?: number; 17 | interpolate ?: boolean; 18 | transition ?: number; 19 | } 20 | -------------------------------------------------------------------------------- /vc/service/helper/esrgan/archs/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import arch modules for registry 6 | # scan all the files that end with '_arch.py' under the archs folder 7 | arch_folder = osp.dirname(osp.abspath(__file__)) 8 | arch_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(arch_folder) if v.endswith('_arch.py')] 9 | # import all the arch modules 10 | _arch_modules = [importlib.import_module(f'vc.service.helper.esrgan.archs.{file_name}') for file_name in arch_filenames] 11 | -------------------------------------------------------------------------------- /latest.py: -------------------------------------------------------------------------------- 1 | import dotenv 2 | import requests 3 | import shutil 4 | from datetime import datetime 5 | 6 | print("Running latest %s" % datetime.now()) 7 | 8 | response = requests.get( 9 | 'https://vc.ajmoon.com/api/generation-request/', 10 | headers={ 11 | "Authorization": "Bearer %s" % dotenv.get_key('.env', 'APP_KEY') 12 | } 13 | ) 14 | if response.status_code == 200: 15 | print('- got latest, replacing latest.json') 16 | # print(response.text);exit() 17 | with open('/opt/vc/app/assets/latest.json', 'w') as outfile: 18 | outfile.write(response.text) 19 | -------------------------------------------------------------------------------- /docker/flask/Dockerfile: -------------------------------------------------------------------------------- 1 | # FROM pytorch/pytorch:1.9.0-cuda11.1-cudnn8-runtime 2 | FROM ubuntu:20.04 3 | 4 | WORKDIR /opt/vc 5 | 6 | RUN apt update \ 7 | && apt install -y software-properties-common \ 8 | && add-apt-repository -y ppa:deadsnakes/ppa \ 9 | && apt update 10 | 11 | RUN apt install -y \ 12 | python3.9 \ 13 | python3-pip \ 14 | python3-venv \ 15 | vim \ 16 | less \ 17 | git \ 18 | wget \ 19 | curl \ 20 | ffmpeg \ 21 | zip \ 22 | redis-tools 23 | 24 | RUN pip3 install uwsgi 25 | 26 | EXPOSE 5000 27 | 28 | CMD './docker.flask.sh' 29 | -------------------------------------------------------------------------------- /vc/service/helper/esrgan/models/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import model modules for registry 6 | # scan all the files that end with '_model.py' under the model folder 7 | model_folder = osp.dirname(osp.abspath(__file__)) 8 | model_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(model_folder) if v.endswith('_model.py')] 9 | # import all the model modules 10 | _model_modules = [importlib.import_module(f'vc.service.helper.esrgan.models.{file_name}') for file_name in model_filenames] 11 | -------------------------------------------------------------------------------- /vc/service/helper/esrgan/data/__init__.py: -------------------------------------------------------------------------------- 1 | import importlib 2 | from basicsr.utils import scandir 3 | from os import path as osp 4 | 5 | # automatically scan and import dataset modules for registry 6 | # scan all the files that end with '_dataset.py' under the data folder 7 | data_folder = osp.dirname(osp.abspath(__file__)) 8 | dataset_filenames = [osp.splitext(osp.basename(v))[0] for v in scandir(data_folder) if v.endswith('_dataset.py')] 9 | # import all the dataset modules 10 | _dataset_modules = [importlib.import_module(f'vc.service.helper.esrgan.data.{file_name}') for file_name in dataset_filenames] 11 | -------------------------------------------------------------------------------- /migrations/script.py.mako: -------------------------------------------------------------------------------- 1 | """${message} 2 | 3 | Revision ID: ${up_revision} 4 | Revises: ${down_revision | comma,n} 5 | Create Date: ${create_date} 6 | 7 | """ 8 | from alembic import op 9 | import sqlalchemy as sa 10 | ${imports if imports else ""} 11 | 12 | # revision identifiers, used by Alembic. 13 | revision = ${repr(up_revision)} 14 | down_revision = ${repr(down_revision)} 15 | branch_labels = ${repr(branch_labels)} 16 | depends_on = ${repr(depends_on)} 17 | 18 | 19 | def upgrade(): 20 | ${upgrades if upgrades else "pass"} 21 | 22 | 23 | def downgrade(): 24 | ${downgrades if downgrades else "pass"} 25 | -------------------------------------------------------------------------------- /requirements.worker.txt: -------------------------------------------------------------------------------- 1 | basicsr>=1.3.3.11 2 | cynetworkx==2.2rc1.dev20180527181709 3 | cython==0.29.24 4 | facexlib>=0.2.0.3 5 | gfpgan>=0.2.1 6 | hsluv==5.0.2 7 | imageio==2.9.0 8 | imageio-ffmpeg==0.4.4 9 | kiwisolver==1.3.1 10 | kornia==0.5.7 11 | moviepy==1.0.2 12 | networkx==2.3 13 | numpy==1.19.5 14 | opencv-python>=4 15 | pillow>=8.3.2 16 | pygments==2.9.0 17 | pytorch-lightning==1.4.0 18 | pytorch-ranger==0.1.1 19 | scikit-image 20 | tensorboard==2.4.1 21 | tensorboard-plugin-wit==1.8.0 22 | timm==0.4.5 23 | torchmetrics==0.4.1 24 | torch-optimizer==0.1.0 25 | transforms3d==0.3.1 26 | vispy==0.8.1 27 | zipp==3.5.0 28 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | DEBUG=True 2 | DEBUG_FILES= 3 | 4 | DB_HOST=127.0.0.1 5 | DB_PORT=5432 6 | DB_NAME=vc 7 | DB_USER=vc 8 | DB_PASS=vc 9 | SQLALCHEMY_TRACK_MODIFICATIONS=False 10 | 11 | REDIS_HOST=127.0.0.1 12 | REDIS_PORT=6379 13 | REDIS_PASS=vc 14 | 15 | AWS_ACCESS_KEY_ID=aws-access-key-id 16 | AWS_SECRET_ACCESS_KEY=aws-secret-access-key 17 | AWS_BUCKET_NAME=unique-bucket-name 18 | AWS_BUCKET_REGION=eu-west-1 19 | 20 | APP_ADMIN_EMAIL=email@goes.here 21 | APP_ADMIN_TOKEN=api-token-goes-here 22 | APP_KEY=not-sure-what-this-is 23 | 24 | IMAGE_WIDTH_SM=533 25 | IMAGE_HEIGHT_SM=300 26 | IMAGE_WIDTH_LG=1280 27 | IMAGE_HEIGHT_LG=720 28 | 29 | ROLE=worker 30 | -------------------------------------------------------------------------------- /vc/model/base.py: -------------------------------------------------------------------------------- 1 | from sqlalchemy.sql import func 2 | 3 | from vc.db import db 4 | 5 | 6 | class BaseModel: 7 | FIELDS = [] 8 | GOD_FIELDS = [] 9 | 10 | id = db.Column(db.Integer, primary_key=True) 11 | created = db.Column( 12 | db.DateTime, 13 | nullable=False, 14 | server_default=func.now() 15 | ) 16 | updated = db.Column( 17 | db.DateTime, 18 | nullable=False, 19 | server_default=func.now(), 20 | onupdate=func.now() 21 | ) 22 | deleted = db.Column(db.DateTime) 23 | 24 | def __repr__(self): 25 | return '<%s %r>' % (self.__class__.__name__, self.id) 26 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | /app/assets/latest.json 3 | backup.sql 4 | backup.sql.bak 5 | BoostingMonocularDepth 6 | checkpoints 7 | CLIP 8 | depth 9 | .env 10 | .idea 11 | images.js 12 | /input 13 | *.log 14 | log 15 | log_file 16 | mesh 17 | model.pt 18 | *.mp4 19 | node_modules 20 | notes.txt 21 | /planning 22 | /*.png 23 | public/* 24 | __pycache__ 25 | /pytti 26 | results 27 | results.zip 28 | steps 29 | styles.txt 30 | taming 31 | taming-transformers 32 | tests 33 | tests.txt 34 | /tmp 35 | vc/service/helper/abme/correlation_package/ 36 | venv 37 | venv.mac 38 | video 39 | weights 40 | *.xcf 41 | .env.local 42 | .env.live 43 | venv.bak 44 | redis.conf 45 | pgsql/ 46 | -------------------------------------------------------------------------------- /nginx.unsecure.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name _; 4 | 5 | access_log /var/log/nginx/vc.access.log; 6 | error_log /var/log/nginx/vc.error.log; 7 | 8 | location /api/ { 9 | proxy_pass http://127.0.0.1:5000/; 10 | proxy_redirect off; 11 | 12 | proxy_set_header Host $host; 13 | proxy_set_header X-Real-IP $remote_addr; 14 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 15 | proxy_set_header X-Forwarded-Proto $scheme; 16 | } 17 | 18 | location / { 19 | root /opt/vc/public; 20 | index index.html index.htm; 21 | try_files $uri $uri/ /index.html; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /diagnosis.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | while true; do 4 | echo 'Diagnostics as at ' $(date) 5 | echo 'top' 6 | top -bn1 -o +%MEM | head -n10 | tail -n4 | awk '{ printf("%-8s %-8s %-8s\n", $9, $10, $12); }' 7 | echo 8 | echo 'db' 9 | pass=$(cat .env | grep SQLALCHEMY_DATABASE_URI | awk -F':' '{ print $3 }' | awk -F'@' '{ print $1 }') 10 | PGPASSWORD=$pass psql --user=vc --host=127.0.0.1 vc "$@" -A -c " 11 | SELECT id, started, completed, failed, steps_completed, steps_total 12 | FROM generation_request WHERE id = (SELECT MAX(id) FROM generation_request) 13 | " 14 | echo 15 | echo 'logs' 16 | grep "DEBUG.*DEBUG" log/vc.* | tail -n10 17 | sleep 5 18 | clear 19 | done 20 | -------------------------------------------------------------------------------- /vc/command/trigger.py: -------------------------------------------------------------------------------- 1 | from injector import inject 2 | 3 | from vc.command.base import BaseCommand 4 | from vc.manager import GenerationRequestManager 5 | 6 | 7 | class TriggerCommand(BaseCommand): 8 | description = 'Sends a job to the generation queue' 9 | args = [ 10 | { 11 | 'dest': 'id', 12 | 'type': int, 13 | 'help': 'Request id', 14 | } 15 | ] 16 | 17 | manager: GenerationRequestManager 18 | 19 | @inject 20 | def __init__(self, manager: GenerationRequestManager): 21 | self.manager = manager 22 | 23 | def handle(self, args): 24 | print('Triggering %s' % args.id) 25 | self.manager.trigger(args.id) 26 | -------------------------------------------------------------------------------- /app/scss/_news.scss: -------------------------------------------------------------------------------- 1 | @import "theme"; 2 | 3 | .content.news { 4 | @import "news/nav"; 5 | 6 | .scrollable { 7 | background: #f9f8ef; 8 | 9 | .hero { 10 | height: 232px; 11 | width: 100%; 12 | img { 13 | height: 232px; 14 | max-width: 100%; 15 | object-fit: cover; 16 | object-position: top left; 17 | } 18 | } 19 | .help { 20 | height: 100%; 21 | display: flex; 22 | flex-direction: column; 23 | align-items: center; 24 | justify-content: center; 25 | } 26 | @import "news/text"; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /app/ts/models/generation-request.ts: -------------------------------------------------------------------------------- 1 | import {GenerationResult} from "./generation-result"; 2 | import {GenerationSpec} from "./generation-spec"; 3 | import {BaseModel} from "./base-model"; 4 | 5 | export class GenerationRequest extends BaseModel { 6 | spec ?: GenerationSpec; 7 | name ?: string; 8 | preview ?: string; 9 | interim ?: string; 10 | interim_watermarked ?: string; 11 | steps_completed ?: number; 12 | steps_total ?: number; 13 | results ?: GenerationResult[]; 14 | 15 | started ?: string; 16 | completed ?: string; 17 | failed ?: string; 18 | 19 | cancelled ?: string; 20 | retried ?: string; 21 | deleted ?: string; 22 | 23 | published ?: string; 24 | } 25 | -------------------------------------------------------------------------------- /scripts/sh.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # helper functions 4 | function confirm() { 5 | while true; do 6 | read -p "[Y/n] " YESNO 7 | case "$YESNO" in 8 | [Yy]*|"" ) return 0;; 9 | [Nn]* ) return 1;; 10 | * ) printf "Try again hotshot. " ;; 11 | esac 12 | done 13 | } 14 | 15 | red="$(tput setaf 1)" 16 | green="$(tput setaf 2)" 17 | yellow="$(tput setaf 3)" 18 | reset="$(tput sgr0)" 19 | function red() { echo "${red}${@}${reset}"; } 20 | function green() { echo "${green}${@}${reset}"; } 21 | function yellow() { echo "${yellow}${@}${reset}"; } 22 | 23 | service=$1 24 | if [[ -z "$service" ]]; then 25 | service=flask 26 | fi 27 | 28 | docker-compose exec $service bash -i 29 | -------------------------------------------------------------------------------- /scripts/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | red="$(tput setaf 1)" 4 | green="$(tput setaf 2)" 5 | yellow="$(tput setaf 3)" 6 | reset="$(tput sgr0)" 7 | function red() { echo "${red}${@}${reset}"; } 8 | function green() { echo "${green}${@}${reset}"; } 9 | function yellow() { echo "${yellow}${@}${reset}"; } 10 | 11 | service=$1 12 | if [[ -z "$service" ]]; then 13 | service=flask 14 | fi 15 | 16 | green "Stopping docker containers" 17 | docker-compose down 18 | 19 | green "Starting docker containers" 20 | docker-compose up -d 21 | 22 | green "Obtaining shell for $service" 23 | docker-compose exec $service bash -i 24 | 25 | green "Your containers are still running" 26 | echo "Do task sh to obtain another shell" 27 | echo "Do task run to restart containers" 28 | -------------------------------------------------------------------------------- /app/ts/helpers/notification.ts: -------------------------------------------------------------------------------- 1 | import {Unnotify} from "../lib/unnotify"; 2 | 3 | export class Notification { 4 | private un: Unnotify = null; 5 | 6 | public constructor() { 7 | this.un = new Unnotify('right', true); 8 | } 9 | 10 | public error(error: Error, message: string = null, title: string = null) { 11 | let content = error.message; 12 | if (message !== null) { 13 | content += ': ' + message; 14 | } 15 | if (title === null) { 16 | title = 'Something went wrong'; 17 | } 18 | this.show(title, content, 'danger'); 19 | } 20 | 21 | private show(title: string, content: string, type: string) { 22 | this.un.show(title, content, {type}); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /app/scss/vc.scss: -------------------------------------------------------------------------------- 1 | @import "theme"; 2 | 3 | html, body { 4 | width: 100%; 5 | height: 100%; 6 | margin: 0; 7 | padding: 0; 8 | font-family: 'Roboto Slab', serif; 9 | } 10 | 11 | a { 12 | text-decoration: none; 13 | color: #63575b; 14 | } 15 | a:hover { 16 | color: #655f62; 17 | } 18 | a:active { 19 | color: #676666; 20 | } 21 | 22 | .container { 23 | height: 100%; 24 | display: flex; 25 | flex-direction: column; 26 | justify-content: space-between; 27 | 28 | @import "header"; 29 | @import "container"; 30 | @import "footer"; 31 | } 32 | 33 | button.unnotify-button { 34 | color: white; 35 | } 36 | 37 | // @todo why is this suddenly necessary? 38 | .material-icons { 39 | font-family: 'Material Icons', serif !important; 40 | } 41 | -------------------------------------------------------------------------------- /vc/event/generation_request.py: -------------------------------------------------------------------------------- 1 | from vc.event.base import VcEvent 2 | from vc.model.generation_request import GenerationRequest 3 | from vc.model.user import User 4 | 5 | 6 | class GenerationRequestCreatedEvent(VcEvent): 7 | id = 'generation_request.created' 8 | generation_request: GenerationRequest 9 | user: User 10 | 11 | def __init__(self, generation_request: GenerationRequest, user: User=None): 12 | self.generation_request = generation_request 13 | self.user = user 14 | 15 | 16 | class GenerationRequestCancelledEvent(VcEvent): 17 | id = 'generation_request.cancelled' 18 | generation_request: GenerationRequest 19 | user: User 20 | 21 | def __init__(self, generation_request: GenerationRequest, user: User): 22 | self.generation_request = generation_request 23 | self.user = user 24 | -------------------------------------------------------------------------------- /vc/service/helper/abme/Upsample.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | def upsample_kernel2d(w, device): 5 | c = w // 2 6 | kernel = 1 - torch.abs(c - torch.arange(w, dtype=torch.float32, device=device)) / (c + 1) 7 | kernel = kernel.repeat(w).view(w,-1) * kernel.unsqueeze(1) 8 | return kernel.view(1, 1, w, w) 9 | 10 | def Upsample(img, factor): 11 | if factor == 1: 12 | return img 13 | B, C, H, W = img.shape 14 | batch_img = img.view(B*C, 1, H, W) 15 | batch_img = F.pad(batch_img, [0, 1, 0, 1], mode='replicate') 16 | kernel = upsample_kernel2d(factor * 2 - 1, img.device) 17 | upsamp_img = F.conv_transpose2d(batch_img, kernel, stride=factor, padding=(factor-1)) 18 | upsamp_img = upsamp_img[:, :, : -1, :-1] 19 | _, _, H_up, W_up = upsamp_img.shape 20 | return upsamp_img.view(B, C, H_up, W_up) -------------------------------------------------------------------------------- /vc/service/helper/hash.py: -------------------------------------------------------------------------------- 1 | import bcrypt 2 | import os 3 | import hashlib 4 | import base64 5 | 6 | 7 | class HashHelper: 8 | @classmethod 9 | def get(cls, value: str): 10 | salt = cls.get_salt(value) 11 | return bcrypt.hashpw( 12 | value.encode('utf-8'), 13 | salt 14 | ).decode() 15 | 16 | @classmethod 17 | def get_salt(cls, value: str): 18 | return os.getenv('SALT').encode('utf-8') 19 | 20 | @classmethod 21 | def get_salt_not_working(cls, value: str): 22 | secret_key = os.getenv('APP_KEY') 23 | 24 | hmac_obj = hashlib.sha256(secret_key.encode('utf-8')) 25 | hmac_obj.update(value.encode('utf-8')) 26 | hmac_digest = hmac_obj.digest() 27 | 28 | salt_value = base64.b64encode(hmac_digest)[:22] 29 | return ('$2b$12$%s' % salt_value.decode('utf-8')).encode('utf-8') 30 | -------------------------------------------------------------------------------- /app/ts/elements.ts: -------------------------------------------------------------------------------- 1 | export * from './elements/generation-requests' 2 | export * from './elements/generation-request' 3 | export * from './elements/generation-request/details' 4 | export * from './elements/generation-request/step' 5 | export * from './elements/generation-request/summary' 6 | export * from './elements/news/nav' 7 | export * from './elements/login-form' 8 | export * from './elements/login' 9 | export * from './elements/generation-request-form' 10 | export * from './elements/generation-request-form/add-spec' 11 | export * from './elements/generation-request-form/image-spec-form' 12 | export * from './elements/generation-request-form/image-spec-option' 13 | export * from './elements/generation-request-form/video-spec-form' 14 | export * from './elements/generation-request-form/add-video-step' 15 | export * from './elements/chipset' 16 | export * from './elements/chip' 17 | export * from './elements/info' 18 | -------------------------------------------------------------------------------- /app/scss/generation_request_form/_video_spec_form.scss: -------------------------------------------------------------------------------- 1 | @import "../theme"; 2 | 3 | vc-video-spec-form { 4 | .video-spec-form { 5 | background-color: #ebdeca; 6 | padding: 8px; 7 | border-radius: $border-radius; 8 | border: solid $border-width #5c6267; 9 | box-sizing: border-box; 10 | width: 100%; 11 | margin-bottom: 16px; 12 | 13 | button { 14 | width: 32px !important; 15 | height: 32px !important; 16 | font-size: 32px !important; 17 | line-height: 32px !important; 18 | } 19 | 20 | .spec-header { 21 | display: flex; 22 | justify-content: space-between; 23 | align-items: center; 24 | h3 { 25 | margin: 0; 26 | padding: 0; 27 | } 28 | } 29 | 30 | h3 { 31 | margin: 0; 32 | padding: 0; 33 | } 34 | 35 | .steps { 36 | margin: 8px; 37 | @import 'add_video_step'; 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /app/scss/generation_request_form/_add_spec.scss: -------------------------------------------------------------------------------- 1 | @import "../theme"; 2 | 3 | vc-add-spec { 4 | .add-spec { 5 | display: flex; 6 | background-color: #c5d9dd; 7 | padding: 8px; 8 | border-radius: $border-radius; 9 | border: solid $border-width #5c6267; 10 | box-sizing: border-box; 11 | width: 100%; 12 | 13 | .add-image, .add-video { 14 | height: 128px; 15 | background-color: #eaf1f2; 16 | padding: 8px; 17 | border-radius: $border-radius; 18 | border: solid 1px #5c6267; 19 | margin: 8px; 20 | flex-grow: 1; 21 | cursor: pointer; 22 | position: relative; 23 | 24 | &:hover { 25 | background-color: #f0f5f0; 26 | 27 | img { 28 | opacity: 1; 29 | } 30 | } 31 | 32 | img { 33 | position: absolute; 34 | left: calc(50% - 60px); 35 | opacity: 0.3; 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /app/scss/generation_request_form/_add_video_step.scss: -------------------------------------------------------------------------------- 1 | @import "../theme"; 2 | 3 | vc-add-video-step { 4 | .add-video-step { 5 | display: flex; 6 | background-color: #c5d9dd; 7 | padding: 8px; 8 | border-radius: $border-radius; 9 | border: solid $border-width #5c6267; 10 | box-sizing: border-box; 11 | width: 100%; 12 | 13 | .add-image { 14 | height: 128px; 15 | background-color: #eaf1f2; 16 | padding: 8px; 17 | border-radius: $border-radius; 18 | border: solid 1px #5c6267; 19 | margin: 8px; 20 | flex-grow: 1; 21 | cursor: pointer; 22 | position: relative; 23 | 24 | &:hover { 25 | background-color: #f0f5f0; 26 | 27 | img { 28 | opacity: 1; 29 | } 30 | } 31 | 32 | img { 33 | position: absolute; 34 | left: calc(50% - 60px); 35 | opacity: 0.3; 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /vc/service/helper/abme/correlation_package/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import os 3 | import torch 4 | 5 | from setuptools import setup, find_packages 6 | from torch.utils.cpp_extension import BuildExtension, CUDAExtension 7 | 8 | cxx_args = ['-std=c++14'] 9 | 10 | nvcc_args = [ 11 | '-gencode', 'arch=compute_50,code=sm_50', 12 | '-gencode', 'arch=compute_52,code=sm_52', 13 | '-gencode', 'arch=compute_60,code=sm_60', 14 | '-gencode', 'arch=compute_61,code=sm_61', 15 | '-gencode', 'arch=compute_70,code=sm_70', 16 | '-gencode', 'arch=compute_70,code=compute_70' 17 | ] 18 | 19 | setup( 20 | name='correlation_cuda', 21 | ext_modules=[ 22 | CUDAExtension('correlation_cuda', [ 23 | 'correlation_cuda.cc', 24 | 'correlation_cuda_kernel.cu' 25 | ], extra_compile_args={'cxx': cxx_args, 'nvcc': nvcc_args}) 26 | ], 27 | cmdclass={ 28 | 'build_ext': BuildExtension 29 | }) 30 | -------------------------------------------------------------------------------- /app/scss/vc.css.map: -------------------------------------------------------------------------------- 1 | {"version":3,"sourceRoot":"","sources":["vc.scss"],"names":[],"mappings":"AAAA;EACI;EACA;EACA;EACA;EACA;;;AAGJ;EACI;EACA;;;AAEJ;EACI;;;AAEJ;EACI;;;AAGJ;EACI;EACA;EACA;EACA;;AAEA;EACI;EACA;EACA;EACA;EACA;;AAEA;EACI;EACA;EACA;;AAEA;EACI;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;AAIR;EACI;EACA;EACA;EACA;;AAIR;EACI;EACA;EACA;EACA;EACA;;AAEA;EACI;EACA;EACA;;AAGJ;EACI;EACA;EACA;;AAEA;EACI;EACA;EACA;EACA;EACA;;AAEA;EACI;EACA;;AAEA;EACI;EACA;EACA;;AAEA;EACI;EACA;EACA;EACA;EACA;EACA;;AAEA;EACI;EACA;EACA;;AAIR;EACI;EACA;EACA;EACA;EACA;;AAEA;EACI;EACA;EACA;;AAEA;EACI;;AAGJ;EACI;EACA;;AAIR;EACI;EACA;EACA;EACA;EACA;;AAEA;EACI;EACA;EACA;EACA;;AAOpB;EACI;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;AACA;EACI;;AAEJ;EACI;EACA;;AAOpB;EACI;;AAGJ;EACI;EACA;EACA;EACA;;AAGJ;EACI;EACA;;AAGJ;EACI;EACA;;AAGJ;EACI;IACI;;EAGJ;IACI;;EAGJ;IACI;;EAGJ;IACI;IACA;IACA;;EAGJ;IACI;IACA;;EAGJ;IACI;;EAGJ;IACI;IACA;IACA;;;AAIR;EACI;EACA;EACA;EACA;EACA;EACA;;AAEA;EACI;EACA;;AAGJ;EACI;;AAGJ;EACI","file":"vc.css"} -------------------------------------------------------------------------------- /vc/service/helper/dimensions.py: -------------------------------------------------------------------------------- 1 | import dotenv 2 | 3 | 4 | class DimensionsHelper: 5 | IMAGE_WIDTH_SM = 400 6 | IMAGE_HEIGHT_SM = 400 7 | IMAGE_WIDTH_LG = 800 8 | IMAGE_HEIGHT_LG = 800 9 | 10 | @classmethod 11 | def width_small(cls): 12 | return int( 13 | dotenv.get_key('.env', 'IMAGE_WIDTH_SM') 14 | or cls.IMAGE_WIDTH_SM 15 | ) 16 | 17 | @classmethod 18 | def height_small(cls): 19 | return int( 20 | dotenv.get_key('.env', 'IMAGE_HEIGHT_SM') 21 | or cls.IMAGE_HEIGHT_SM 22 | ) 23 | 24 | @classmethod 25 | def width_large(cls): 26 | return int( 27 | dotenv.get_key('.env', 'IMAGE_WIDTH_LG') 28 | or cls.IMAGE_WIDTH_LG 29 | ) 30 | 31 | @classmethod 32 | def height_large(cls): 33 | return int( 34 | dotenv.get_key('.env', 'IMAGE_HEIGHT_LG') 35 | or cls.IMAGE_HEIGHT_LG 36 | ) 37 | -------------------------------------------------------------------------------- /app/scss/_normalize.scss: -------------------------------------------------------------------------------- 1 | @import "config"; 2 | 3 | /** 4 | * 1. Change the font styles in all browsers. 5 | * 2. Remove the margin in Firefox and Safari. 6 | */ 7 | 8 | button, 9 | input, 10 | optgroup, 11 | select, 12 | textarea { 13 | font-family: inherit; /* 1 */ 14 | font-size: 100%; /* 1 */ 15 | line-height: 1.15; /* 1 */ 16 | margin: 0; /* 2 */ 17 | } 18 | 19 | /** 20 | * Show the overflow in IE. 21 | * 1. Show the overflow in Edge. 22 | */ 23 | 24 | button, 25 | input { /* 1 */ 26 | overflow: visible; 27 | } 28 | 29 | /** 30 | * Remove the inheritance of text transform in Edge, Firefox, and IE. 31 | * 1. Remove the inheritance of text transform in Firefox. 32 | */ 33 | 34 | button, 35 | select { /* 1 */ 36 | text-transform: none; 37 | } 38 | 39 | /** 40 | * Correct the inability to style clickable types in iOS and Safari. 41 | */ 42 | 43 | button, 44 | [type="button"], 45 | [type="reset"], 46 | [type="submit"] { 47 | -webkit-appearance: button; 48 | } 49 | -------------------------------------------------------------------------------- /app/ts/elements/login.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement} from 'custom-elements-ts'; 2 | import {BaseElement} from "./base-element"; 3 | 4 | @CustomElement({ 5 | tag: 'vc-login', 6 | shadow: false, 7 | style: ``, 8 | template: ` 9 | <div class="login"> 10 | <form> 11 | <input type="text" placeholder="API key" /> 12 | </form> 13 | </div> 14 | ` 15 | }) 16 | export class Login extends BaseElement { 17 | $root: HTMLElement; 18 | $form: HTMLElement; 19 | $input: HTMLInputElement; 20 | 21 | constructor() { 22 | super(); 23 | } 24 | 25 | connectedCallback() { 26 | this.$root = this.querySelector('.login'); 27 | this.$form = this.$root.querySelector('form'); 28 | this.$form.addEventListener('submit', this.submit.bind(this)); 29 | this.$input = this.$form.querySelector('input'); 30 | } 31 | 32 | submit(event: Event) { 33 | event.preventDefault(); 34 | this.vc.authenticate(this.$input.value); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /vc/service/helper/tier.py: -------------------------------------------------------------------------------- 1 | from vc.model.user import User, UserTier 2 | 3 | 4 | class TierHelper: 5 | INTERIM_STEPS = 60 6 | 7 | @classmethod 8 | def get_user_roles(cls, user): 9 | result = [] 10 | i = user.tier 11 | while i >= 0: 12 | result.append(i) 13 | i -= 1 14 | return result 15 | 16 | @classmethod 17 | def is_tier(cls, user: User, value): 18 | if user is None: 19 | return False 20 | return value in cls.get_user_roles(user) 21 | 22 | @classmethod 23 | def is_god(cls, user: User): 24 | return cls.is_tier(user, UserTier.God) 25 | 26 | @classmethod 27 | def is_artist(cls, user: User): 28 | return cls.is_tier(user, UserTier.Artist) 29 | 30 | @classmethod 31 | def is_coder(cls, user: User): 32 | return cls.is_tier(user, UserTier.Coder) 33 | 34 | @classmethod 35 | def is_supporter(cls, user: User): 36 | return cls.is_tier(user, UserTier.Supporter) 37 | -------------------------------------------------------------------------------- /vc/service/helper/abme/correlation_package/nvcc setting.md: -------------------------------------------------------------------------------- 1 | # How to set the nvcc version (Default:CUDA-11.0) 2 | 3 | ## Pre-requisite 4 | 5 | Check installed `cuda-11.0` path: 6 | ```bash 7 | $ cd /usr/local 8 | $ find . -maxdepth 1 -name 'cuda-11.0' 9 | ``` 10 | - If there is no `cuda-11.0` folder in the directory, install `cuda-11.0` first. 11 | 12 | ## Change environments of the terminal (temporal) 13 | Change your terminal `PATH` and `LD_LIBRARY_PATH`: 14 | ```bash 15 | $ export PATH=/usr/local/cuda-11.0/bin${PATH:+:${PATH}} 16 | $ export LD_LIBRARY_PATH=/usr/local/cuda-11.0/lib64 17 | $ nvcc --version 18 | ``` 19 | ## Change default environments of the terminal 20 | For change default `nvcc` version of your terminal, you should add below two lines in your `~/.bashrc`. 21 | ```bash 22 | $ gedit ~/.bashrc 23 | export PATH=/usr/local/cuda-11.0/bin${PATH:+:${PATH}} 24 | export LD_LIBRARY_PATH=/usr/local/cuda-11.0/lib64 25 | ``` 26 | Save and then open new terminal: 27 | ```bash 28 | $ nvcc --version 29 | ``` 30 | -------------------------------------------------------------------------------- /vc/model/user.py: -------------------------------------------------------------------------------- 1 | from flask_restful import fields 2 | 3 | from vc.db import db 4 | from vc.model.base import BaseModel 5 | 6 | 7 | class UserTier: 8 | Supporter = 0 9 | Coder = 1 10 | Artist = 2 11 | God = 3 12 | 13 | 14 | class User(db.Model, BaseModel): 15 | FIELDS = [ 16 | 'name', 17 | 'email', 18 | ] 19 | GOD_FIELDS = [ 20 | 'tier', 21 | ] 22 | 23 | api_token: str = None 24 | 25 | name = db.Column(db.String) 26 | email = db.Column(db.String, nullable=False) 27 | token = db.Column(db.String, nullable=False) 28 | tier = db.Column(db.Integer, nullable=False, default=0) 29 | 30 | requests = db.relationship('GenerationRequest', backref='user') 31 | 32 | schema = { 33 | 'id': fields.Integer, 34 | 'email': fields.String, 35 | 'name': fields.String, 36 | 'api_token': fields.String, 37 | 'created': fields.DateTime(), 38 | 'updated': fields.DateTime(), 39 | 'deleted': fields.DateTime(), 40 | } 41 | -------------------------------------------------------------------------------- /vc/command/video.py: -------------------------------------------------------------------------------- 1 | from injector import inject 2 | 3 | from vc.command.base import BaseCommand 4 | from vc.service.video import VideoService 5 | 6 | 7 | class VideoCommand(BaseCommand): 8 | description = 'Makes a video from steps files' 9 | args = [ 10 | { 11 | 'dest': 'fps_multiple', 12 | 'type': int, 13 | 'help': 'FPS multiple (1 or 4)', 14 | 'default': 4, 15 | }, 16 | { 17 | 'dest': 'steps_dir', 18 | 'type': str, 19 | 'help': 'Steps dir', 20 | 'default': 'steps', 21 | 'nargs': '?', 22 | } 23 | ] 24 | 25 | video: VideoService 26 | 27 | @inject 28 | def __init__(self, video: VideoService): 29 | self.video = video 30 | 31 | def handle(self, args): 32 | self.video.make_unwatermarked_video( 33 | 'output.mp4', 34 | args.steps_dir, 35 | suffix='sofar', 36 | interpolate=False, 37 | fps_multiple=args.fps_multiple, 38 | ) 39 | -------------------------------------------------------------------------------- /app/scss/generation_request_form/_image_spec_option.scss: -------------------------------------------------------------------------------- 1 | @import "../theme"; 2 | 3 | vc-image-spec-option { 4 | flex: 1 1 14%; 5 | 6 | .image-spec-option { 7 | display: flex; 8 | justify-content: flex-start; 9 | 10 | background: #eaf1f2; 11 | border: solid 1px #5c6267; 12 | border-radius: $border-radius; 13 | position: relative; 14 | margin: 8px 8px 0 0; 15 | padding: 8px; 16 | height: 2em; 17 | 18 | &:hover { 19 | background-color: #f0f5f0; 20 | } 21 | 22 | @include mobile { 23 | width: auto; 24 | margin: 8px 0 0; 25 | } 26 | 27 | label { 28 | align-items: center; 29 | display: flex; 30 | font-weight: bold; 31 | width: 100%; 32 | cursor: pointer; 33 | justify-content: space-between; 34 | } 35 | 36 | input { 37 | margin-left: 8px; 38 | height: 32px; 39 | } 40 | 41 | input[type=number] { 42 | width: 6em; 43 | padding: 0 8px; 44 | } 45 | 46 | input[type=checkbox] { 47 | width: 32px; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /app/ts/elements/info.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement} from 'custom-elements-ts'; 2 | import {BaseElement} from "./base-element"; 3 | 4 | @CustomElement({ 5 | tag: 'vc-info', 6 | shadow: false, 7 | style: ``, 8 | template: require('./info.inc') 9 | }) 10 | export class Info extends BaseElement { 11 | $root: HTMLElement; 12 | $info: HTMLElement; 13 | $close: HTMLButtonElement; 14 | expanded = false; 15 | 16 | constructor() { 17 | super(); 18 | } 19 | 20 | connectedCallback() { 21 | this.$root = this.querySelector('.info-container'); 22 | this.$info = this.$root.querySelector('.info'); 23 | this.$close = this.$info.querySelector('button.close'); 24 | this.$close.addEventListener('click', () => { 25 | this.expand(); 26 | }); 27 | } 28 | 29 | expand() { 30 | this.expanded = !this.expanded; 31 | if (this.expanded) { 32 | this.$root.classList.add('expanded'); 33 | } else { 34 | this.$root.classList.remove('expanded'); 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /migrations/alembic.ini: -------------------------------------------------------------------------------- 1 | # A generic, single database configuration. 2 | 3 | [alembic] 4 | # template used to generate migration files 5 | # file_template = %%(rev)s_%%(slug)s 6 | 7 | # set to 'true' to run the environment during 8 | # the 'revision' command, regardless of autogenerate 9 | # revision_environment = false 10 | 11 | 12 | # Logging configuration 13 | [loggers] 14 | keys = root,sqlalchemy,alembic,flask_migrate 15 | 16 | [handlers] 17 | keys = console 18 | 19 | [formatters] 20 | keys = generic 21 | 22 | [logger_root] 23 | level = WARN 24 | handlers = console 25 | qualname = 26 | 27 | [logger_sqlalchemy] 28 | level = WARN 29 | handlers = 30 | qualname = sqlalchemy.engine 31 | 32 | [logger_alembic] 33 | level = INFO 34 | handlers = 35 | qualname = alembic 36 | 37 | [logger_flask_migrate] 38 | level = INFO 39 | handlers = 40 | qualname = flask_migrate 41 | 42 | [handler_console] 43 | class = StreamHandler 44 | args = (sys.stderr,) 45 | level = NOTSET 46 | formatter = generic 47 | 48 | [formatter_generic] 49 | format = %(levelname)-5.5s [%(name)s] %(message)s 50 | datefmt = %H:%M:%S 51 | -------------------------------------------------------------------------------- /vc/controller/base.py: -------------------------------------------------------------------------------- 1 | from flask_restful import Resource 2 | 3 | from vc.auth import auth 4 | from vc.manager.user import UserManager 5 | from vc.service.helper.tier import TierHelper 6 | 7 | 8 | class BaseController(Resource): 9 | user_manager: UserManager 10 | 11 | def __init__(self, user_manager: UserManager, *args, **kwargs): 12 | super().__init__(*args, **kwargs) 13 | self.user_manager = user_manager 14 | auth.verify_token(self.verify_token) 15 | auth.get_user_roles(TierHelper.get_user_roles) 16 | 17 | def verify_token(self, token): 18 | return self.user_manager.authenticate(token) 19 | 20 | def current_user(self): 21 | return auth.current_user() 22 | 23 | def is_god(self): 24 | return TierHelper.is_god(self.current_user()) 25 | 26 | def is_artist(self): 27 | return TierHelper.is_artist(self.current_user()) 28 | 29 | def is_coder(self): 30 | return TierHelper.is_coder(self.current_user()) 31 | 32 | def is_supporter(self): 33 | return TierHelper.is_supporter(self.current_user()) 34 | -------------------------------------------------------------------------------- /vc/command/esrgan.py: -------------------------------------------------------------------------------- 1 | from injector import inject 2 | 3 | from vc.command.base import BaseCommand 4 | from vc.service.esrgan import EsrganService, EsrganOptions 5 | 6 | 7 | class EsrganCommand(BaseCommand): 8 | description = 'Runs esrgan on an input file' 9 | args = [ 10 | { 11 | 'dest': 'input_file', 12 | 'type': str, 13 | 'help': 'Input file', 14 | 'default': 'output.png', 15 | 'nargs': '?', 16 | }, 17 | { 18 | 'dest': 'output_file', 19 | 'type': str, 20 | 'help': 'Output file', 21 | 'default': 'debug.png', 22 | 'nargs': '?', 23 | }, 24 | ] 25 | 26 | esrgan: EsrganService 27 | 28 | @inject 29 | def __init__(self, esrgan: EsrganService): 30 | self.esrgan = esrgan 31 | 32 | def handle(self, args): 33 | print('got %s %s' % (args.input_file, args.output_file)) 34 | self.esrgan.handle(EsrganOptions( 35 | input_file=args.input_file, 36 | output_file=args.output_file, 37 | )) 38 | -------------------------------------------------------------------------------- /vc/model/generation_result.py: -------------------------------------------------------------------------------- 1 | from flask_restful import fields 2 | 3 | from vc.db import db 4 | from vc.model.base import BaseModel 5 | 6 | 7 | class GenerationResult(db.Model, BaseModel): 8 | FIELDS = [ 9 | 'request_id', 10 | 'url', 11 | 'url_watermarked', 12 | ] 13 | 14 | request_id = db.Column( 15 | db.Integer, 16 | db.ForeignKey('generation_request.id'), 17 | nullable=False 18 | ) 19 | 20 | url = db.Column(db.String, nullable=True) 21 | url_watermarked = db.Column(db.String, nullable=True) 22 | 23 | public_schema = { 24 | 'id': fields.Integer, 25 | 'url': fields.String, 26 | 'url_watermarked': fields.String, 27 | 'created': fields.DateTime(), 28 | 'updated': fields.DateTime(), 29 | 'deleted': fields.DateTime(), 30 | } 31 | 32 | private_schema = { 33 | 'id': fields.Integer, 34 | 'url_watermarked': fields.String, 35 | 'created': fields.DateTime(), 36 | 'updated': fields.DateTime(), 37 | 'deleted': fields.DateTime(), 38 | } 39 | -------------------------------------------------------------------------------- /vc/event_listener/generation_request.py: -------------------------------------------------------------------------------- 1 | from injector import inject 2 | 3 | from vc.event import ( 4 | GenerationRequestCreatedEvent, 5 | GenerationRequestCancelledEvent, 6 | ) 7 | from vc.event_listener.base import VcEventListener 8 | from vc.service.job import JobService 9 | 10 | 11 | class GenerationRequestCreatedEventListener(VcEventListener): 12 | job_service: JobService 13 | 14 | @inject 15 | def __init__(self, job_service: JobService): 16 | self.job_service = job_service 17 | 18 | def on(self, event: GenerationRequestCreatedEvent): 19 | event.generation_request.hash = self.job_service.enqueue( 20 | 'vc.job.generation.GenerationJob', 21 | event.generation_request.id 22 | ) 23 | 24 | 25 | class GenerationRequestCancelledEventListener(VcEventListener): 26 | job_service: JobService 27 | 28 | @inject 29 | def __init__(self, job_service: JobService): 30 | self.job_service = job_service 31 | 32 | def on(self, event: GenerationRequestCancelledEvent): 33 | self.job_service.cancel(event.generation_request.hash) 34 | -------------------------------------------------------------------------------- /app/ts/helpers/auth.ts: -------------------------------------------------------------------------------- 1 | export class AuthHelper { 2 | private static token: string = null; 3 | private static callbacks: CallableFunction[] = []; 4 | private static authenticated = false; 5 | 6 | static getToken() { 7 | return AuthHelper.token; 8 | } 9 | 10 | static setToken(token: string) { 11 | AuthHelper.token = token 12 | || (new URLSearchParams(window.location.search)).get('token') 13 | || localStorage.getItem('vc-token'); 14 | } 15 | 16 | static clearToken() { 17 | AuthHelper.token = null; 18 | AuthHelper.authenticated = false; 19 | } 20 | 21 | static authenticate() { 22 | AuthHelper.authenticated = true; 23 | localStorage.setItem('vc-token', AuthHelper.token); 24 | for (const callback of AuthHelper.callbacks) { 25 | callback(AuthHelper.token); 26 | } 27 | } 28 | 29 | static isAuthenticated() { 30 | return AuthHelper.authenticated; 31 | } 32 | 33 | static listen(callback: CallableFunction) { 34 | this.callbacks.push(callback); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Alex Moon 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /app/ts/elements/generation-request-form/add-video-step.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CustomElement, 3 | Dispatch, 4 | DispatchEmitter, 5 | Listen 6 | } from 'custom-elements-ts'; 7 | import {VideoSpec} from "../../models/video-spec"; 8 | 9 | @CustomElement({ 10 | tag: 'vc-add-video-step', 11 | shadow: false, 12 | style: ``, 13 | template: ` 14 | <div class="add-video-step"> 15 | <div class="add-image"> 16 | Add step 17 | <img src="../../../assets/add-image.png" /> 18 | </div> 19 | </div> 20 | `, 21 | }) 22 | export class AddVideoStep extends HTMLElement { 23 | $root: HTMLElement 24 | private spec: VideoSpec; 25 | 26 | @Dispatch() addedImage: DispatchEmitter; 27 | 28 | constructor() { 29 | super(); 30 | } 31 | 32 | connectedCallback() { 33 | this.$root = this.querySelector('.add-spec'); 34 | } 35 | 36 | @Listen('click', '.add-image') 37 | addImage() { 38 | this.addedImage.emit(); 39 | } 40 | 41 | update(spec: VideoSpec) { 42 | this.spec = spec; 43 | this.draw(); 44 | } 45 | 46 | draw() { 47 | 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /vqgan/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Katherine Crowson 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /app/ts/managers/generation-request-manager.ts: -------------------------------------------------------------------------------- 1 | import {GenerationRequest} from "../models/generation-request"; 2 | import {BaseManager} from "./base-manager"; 3 | 4 | export class GenerationRequestManager extends BaseManager<GenerationRequest> { 5 | protected base_url = '/api/generation-request/' 6 | 7 | public index(): Promise<GenerationRequest[]> { 8 | let url = ''; 9 | if (this.isLocal) { 10 | url = '/assets/latest.json'; 11 | } 12 | return this.fetch(url) as Promise<GenerationRequest[]>; 13 | } 14 | 15 | public create(request: GenerationRequest) { 16 | return this.post(request); 17 | } 18 | 19 | public cancel(id: number) { 20 | return this.put(id + '/cancel'); 21 | } 22 | 23 | public retry(id: number) { 24 | return this.put(id + '/retry'); 25 | } 26 | 27 | public delete(id: number) { 28 | return this.put(id + '/delete'); 29 | } 30 | 31 | public publish(id: number) { 32 | return this.put(id + '/publish'); 33 | } 34 | 35 | public unpublish(id: number) { 36 | return this.put(id + '/unpublish'); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /app/scss/_header.scss: -------------------------------------------------------------------------------- 1 | .header { 2 | height: 64px; 3 | display: flex; 4 | background-color: #efedd4; 5 | box-sizing: border-box; 6 | padding: 16px; 7 | 8 | .header-first { 9 | flex: 0; 10 | display: flex; 11 | align-items: center; 12 | 13 | h1 { 14 | cursor: pointer; 15 | background: url('../assets/logo-sm.png'); 16 | background-size: contain; 17 | width: 64px; 18 | height: 64px; 19 | margin: 0; 20 | padding: 0; 21 | color: rgba(0, 0, 0, 0); 22 | } 23 | } 24 | 25 | .header-second { 26 | flex: 1; 27 | display: flex; 28 | align-items: center; 29 | justify-content: flex-end; 30 | 31 | .header-button { 32 | cursor: pointer; 33 | background-size: contain; 34 | width: 28px; 35 | height: 28px; 36 | margin: 18px 8px; 37 | padding: 0; 38 | color: rgba(0, 0, 0, 0); 39 | } 40 | 41 | .info { 42 | background: url('../assets/help.png'); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /vc/service/helper/rife/model/warplayer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 5 | backwarp_tenGrid = {} 6 | 7 | 8 | def warp(tenInput, tenFlow): 9 | k = (str(tenFlow.device), str(tenFlow.size())) 10 | if k not in backwarp_tenGrid: 11 | tenHorizontal = torch.linspace(-1.0, 1.0, tenFlow.shape[3], device=device).view( 12 | 1, 1, 1, tenFlow.shape[3]).expand(tenFlow.shape[0], -1, tenFlow.shape[2], -1) 13 | tenVertical = torch.linspace(-1.0, 1.0, tenFlow.shape[2], device=device).view( 14 | 1, 1, tenFlow.shape[2], 1).expand(tenFlow.shape[0], -1, -1, tenFlow.shape[3]) 15 | backwarp_tenGrid[k] = torch.cat( 16 | [tenHorizontal, tenVertical], 1).to(device) 17 | 18 | tenFlow = torch.cat([tenFlow[:, 0:1, :, :] / ((tenInput.shape[3] - 1.0) / 2.0), 19 | tenFlow[:, 1:2, :, :] / ((tenInput.shape[2] - 1.0) / 2.0)], 1) 20 | 21 | g = (backwarp_tenGrid[k] + tenFlow).permute(0, 2, 3, 1) 22 | return torch.nn.functional.grid_sample(input=tenInput, grid=g, mode='bilinear', padding_mode='border', align_corners=True) 23 | -------------------------------------------------------------------------------- /app/ts/elements/generation-requests.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement} from 'custom-elements-ts'; 2 | import {GenerationRequest as Model} from "../models/generation-request"; 3 | import {GenerationRequest} from "./generation-request"; 4 | import {BaseElement} from "./base-element"; 5 | 6 | @CustomElement({ 7 | tag: 'vc-generation-requests', 8 | shadow: false, 9 | style: ``, 10 | // @todo figure out how to make templateUrl work 11 | template: ` 12 | <div class="requests"></div> 13 | ` 14 | }) 15 | export class GenerationRequests extends BaseElement { 16 | $root: HTMLElement; 17 | 18 | requests: Model[] 19 | 20 | constructor() { 21 | super(); 22 | } 23 | 24 | connectedCallback() { 25 | this.$root = this.querySelector('.requests'); 26 | } 27 | 28 | update(requests: Model[]) { 29 | this.requests = requests; 30 | 31 | this.$root.innerHTML = ''; 32 | this.requests.forEach((request: Model) => { 33 | const $request = document.createElement('vc-generation-request') as GenerationRequest; 34 | this.$root.appendChild($request); 35 | $request.update(request); 36 | }); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /debug.html: -------------------------------------------------------------------------------- 1 | <!DOCTYPE html> 2 | <html> 3 | <head> 4 | <title>Debug</title> 5 | <style> 6 | html, body { margin: 0; padding: 0; font-family: sans-serif; } 7 | .container { padding: 16px; display: flex; flex-direction: column; } 8 | .side-by-side { display: flex; justify-content: center; } 9 | .panel { display: flex; flex-direction: column; } 10 | img { width: 533px; height: 300px; } 11 | .caption { margin-top: 8px; } 12 | 13 | .overlaid { width: 400px; height: 400px; position: relative; margin: 16px auto; } 14 | .overlaid img { position: absolute; top: 0; left: 0; opacity: 0.5; } 15 | </style> 16 | </head> 17 | <body> 18 | <div class="container"> 19 | <div class="side-by-side"> 20 | <div class="panel"> 21 | <img src="file:///opt/vc/output.png" /> 22 | <div class="caption">Before</div> 23 | </div> 24 | <div class="panel"> 25 | <img src="file:///opt/vc/debug.png" /> 26 | <div class="caption">After</div> 27 | </div> 28 | </div> 29 | <div class="overlaid"> 30 | <img src="file:///opt/vc/output.png" /> 31 | <img src="file:///opt/vc/debug.png" /> 32 | </div> 33 | </div> 34 | </body> 35 | </html> 36 | -------------------------------------------------------------------------------- /app/partials/footer.html: -------------------------------------------------------------------------------- 1 | <div class="footer"> 2 | <div> 3 | &copy; 2021 Alex Moon 4 | </div> 5 | <div class="buttons"> 6 | <a target="_blank" href="https://patreon.com/ajmoonne"> 7 | <div class="patreon footer-button">Patreon</div> 8 | </a> 9 | <a target="_blank" href="https://objkt.com/profile/ajmoon"> 10 | <div class="tezos footer-button">objkt.com</div> 11 | </a> 12 | <a target="_blank" href="https://github.com/alex-moon"> 13 | <div class="github footer-button">GitHub</div> 14 | </a> 15 | <a target="_blank" href="https://twitter.com/ajmoonne"> 16 | <div class="twitter footer-button">Twitter</div> 17 | </a> 18 | </div> 19 | </div> 20 | </div> 21 | <script async src="//static.getclicky.com/101349219.js"></script> 22 | <noscript><p><img alt="Clicky" width="1" height="1" src="//in.getclicky.com/101349219ns.gif" /></p></noscript> 23 | </body> 24 | </html> 25 | -------------------------------------------------------------------------------- /argument.yml: -------------------------------------------------------------------------------- 1 | depth_edge_model_ckpt: 'checkpoints/edge-model.pth' 2 | depth_feat_model_ckpt: 'checkpoints/depth-model.pth' 3 | rgb_feat_model_ckpt: 'checkpoints/color-model.pth' 4 | MiDaS_model_ckpt: 'MiDaS/model.pt' 5 | use_boostmonodepth: True 6 | fps: 40 7 | num_frames: 200 8 | x_shift: 0.00 9 | y_shift: 0.00 10 | z_shift: 0.05 11 | traj_type: 'double-straight-line' 12 | video_postfix: 'zoom-in' 13 | specific: '' 14 | longer_side_len: 400 15 | src_folder: 'input' 16 | depth_folder: 'depth' 17 | mesh_folder: 'mesh' 18 | video_folder: 'results' 19 | load_ply: False 20 | save_ply: True 21 | inference_video: True 22 | gpu_ids: 0 23 | offscreen_rendering: False 24 | img_format: '.png' 25 | depth_format: '.npy' 26 | require_midas: True 27 | depth_threshold: 0.04 28 | ext_edge_threshold: 0.002 29 | sparse_iter: 5 30 | filter_size: [7, 7, 5, 5, 5] 31 | sigma_s: 4.0 32 | sigma_r: 0.5 33 | redundant_number: 12 34 | background_thickness: 70 35 | context_thickness: 140 36 | background_thickness_2: 70 37 | context_thickness_2: 70 38 | discount_factor: 1.00 39 | log_depth: True 40 | largest_size: 512 41 | depth_edge_dilate: 10 42 | depth_edge_dilate_2: 5 43 | extrapolate_border: True 44 | extrapolation_thickness: 60 45 | repeat_inpaint_edge: True 46 | crop_border: [0.03, 0.03, 0.05, 0.03] 47 | anti_flickering: False 48 | -------------------------------------------------------------------------------- /app/scss/_info.scss: -------------------------------------------------------------------------------- 1 | @import "config"; 2 | 3 | $info-width: 400px; 4 | $info-height: calc(100vh - 118px); 5 | $border-radius: 8px; 6 | $transition: 0.5s; 7 | 8 | vc-info { 9 | .info-container { 10 | background: #f9f8ef; 11 | transition: $transition; 12 | overflow-y: auto; 13 | overflow-x: hidden; 14 | width: 0; 15 | 16 | color: black; 17 | display: flex; 18 | flex-direction: column; 19 | justify-content: flex-start; 20 | align-items: flex-start; 21 | box-sizing: border-box; 22 | 23 | height: 100%; 24 | 25 | &.expanded { 26 | width: $info-width; 27 | } 28 | 29 | @include mobile { 30 | width: 100% !important; 31 | height: 0; 32 | 33 | &.expanded { 34 | height: $info-height; 35 | } 36 | } 37 | 38 | .info { 39 | display: flex; 40 | flex-direction: column; 41 | 42 | width: $info-width - 64px; 43 | padding: 16px; 44 | 45 | @include mobile { 46 | width: calc(100% - 64px); 47 | padding-bottom: 64px; 48 | } 49 | 50 | .text { 51 | flex-grow: 1; 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /vc/cli.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import argparse 3 | 4 | from vc import create_app, injector 5 | from vc import command 6 | 7 | 8 | class CommandLoader: 9 | commands = { 10 | 'vqgan_clip': command.VqganClipCommand, 11 | 'inpainting': command.InpaintingCommand, 12 | 'esrgan': command.EsrganCommand, 13 | 'rife': command.RifeCommand, 14 | 'rife_steps': command.RifeStepsCommand, 15 | 'video': command.VideoCommand, 16 | 'trigger': command.TriggerCommand, 17 | } 18 | 19 | @classmethod 20 | def load(cls, key) -> command.BaseCommand: 21 | return injector.get(cls.commands[key]) 22 | 23 | 24 | if __name__ == '__main__': 25 | app = create_app() 26 | if len(sys.argv) < 2: 27 | print('Usage: %s <command> [args]' % sys.argv[0]) 28 | print('Available commands:') 29 | for key, command in CommandLoader.commands.items(): 30 | print('%s: %s' % (key, command.description)) 31 | exit() 32 | 33 | key = sys.argv[1] 34 | command = CommandLoader.load(key) 35 | parser = argparse.ArgumentParser(description=command.description) 36 | parser.add_argument(dest='key', type=str, default=key, nargs='?') 37 | for arg in command.args: 38 | parser.add_argument(**arg) 39 | args = parser.parse_args() 40 | command.run(args) 41 | -------------------------------------------------------------------------------- /vc/command/abme.py: -------------------------------------------------------------------------------- 1 | from injector import inject 2 | 3 | from vc.command.base import BaseCommand 4 | from vc.service.abme import AbmeService, AbmeOptions 5 | 6 | 7 | class AbmeCommand(BaseCommand): 8 | description = 'Runs abme on an input file' 9 | args = [ 10 | { 11 | 'dest': 'first_file', 12 | 'type': str, 13 | 'help': 'First file', 14 | 'default': 'first.png', 15 | 'nargs': '?', 16 | }, 17 | { 18 | 'dest': 'second_file', 19 | 'type': str, 20 | 'help': 'Second file', 21 | 'default': 'second.png', 22 | 'nargs': '?', 23 | }, 24 | { 25 | 'dest': 'output_file', 26 | 'type': str, 27 | 'help': 'Output file', 28 | 'default': 'debug.png', 29 | 'nargs': '?', 30 | }, 31 | ] 32 | 33 | abme: AbmeService 34 | 35 | @inject 36 | def __init__(self, abme: AbmeService): 37 | self.abme = abme 38 | 39 | def handle(self, args): 40 | print('got %s %s %s' % (args.first_file, args.second_file, args.output_file)) 41 | self.abme.handle(AbmeOptions( 42 | first_file=args.first_file, 43 | second_file=args.second_file, 44 | output_file=args.output_file, 45 | )) 46 | -------------------------------------------------------------------------------- /vc/command/rife.py: -------------------------------------------------------------------------------- 1 | from injector import inject 2 | 3 | from vc.command.base import BaseCommand 4 | from vc.service.rife import RifeService, RifeOptions 5 | 6 | 7 | class RifeCommand(BaseCommand): 8 | description = 'Runs rife on an input file' 9 | args = [ 10 | { 11 | 'dest': 'first_file', 12 | 'type': str, 13 | 'help': 'First file', 14 | 'default': 'first.png', 15 | 'nargs': '?', 16 | }, 17 | { 18 | 'dest': 'second_file', 19 | 'type': str, 20 | 'help': 'Second file', 21 | 'default': 'second.png', 22 | 'nargs': '?', 23 | }, 24 | { 25 | 'dest': 'output_file', 26 | 'type': str, 27 | 'help': 'Output file', 28 | 'default': 'debug.png', 29 | 'nargs': '?', 30 | }, 31 | ] 32 | 33 | rife: RifeService 34 | 35 | @inject 36 | def __init__(self, rife: RifeService): 37 | self.rife = rife 38 | 39 | def handle(self, args): 40 | print('got %s %s %s' % (args.first_file, args.second_file, args.output_file)) 41 | self.rife.handle(RifeOptions( 42 | first_file=args.first_file, 43 | second_file=args.second_file, 44 | output_file=args.output_file, 45 | )) 46 | -------------------------------------------------------------------------------- /app/ts/elements/chip.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement, Dispatch, DispatchEmitter, Listen, Prop, Toggle, Watch} from 'custom-elements-ts'; 2 | 3 | @CustomElement({ 4 | tag: 'vc-chip', 5 | shadow: false, 6 | style: ``, 7 | template: ` 8 | <div class="chip"></div> 9 | ` 10 | }) 11 | export class Chip extends HTMLElement { 12 | $root: HTMLElement; 13 | 14 | @Prop() text: string; 15 | @Toggle() removable: boolean; 16 | @Dispatch('chip.remove') onRemove: DispatchEmitter; 17 | 18 | constructor() { 19 | super(); 20 | } 21 | 22 | connectedCallback() { 23 | this.$root = this.querySelector('.chip'); 24 | } 25 | 26 | @Watch('text') 27 | protected update() { 28 | this.$root.innerHTML = this.text; 29 | if (this.removable) { 30 | const remove = document.createElement('button'); 31 | remove.classList.add('remove'); 32 | remove.classList.add('material-icons'); 33 | remove.innerHTML = 'highlight_off'; 34 | remove.addEventListener('click', this.onRemoveClick.bind(this)); 35 | this.$root.appendChild(remove); 36 | } 37 | } 38 | 39 | @Listen('click', '.remove') 40 | protected onRemoveClick(e: MouseEvent) { 41 | e.preventDefault(); 42 | this.onRemove.emit({detail: this.text}) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /app/ts/elements/base-element.ts: -------------------------------------------------------------------------------- 1 | import {Vc} from "../vc"; 2 | 3 | export interface ElOptions { 4 | class ?: string; 5 | innerText ?: string; 6 | attr ?: any; 7 | 8 | type ?: string; 9 | value ?: string; 10 | checked ?: boolean; 11 | } 12 | 13 | export abstract class BaseElement extends HTMLElement { 14 | protected vc: Vc; 15 | 16 | constructor() { 17 | super(); 18 | this.vc = Vc.instance; 19 | } 20 | 21 | protected el(tag: string, options: ElOptions = {}) { 22 | const result = document.createElement(tag); 23 | if (options.class) { 24 | result.classList.add(options.class); 25 | } 26 | if (options.type) { 27 | result.setAttribute('type', options.type); 28 | } 29 | if (options.value) { 30 | result.setAttribute('value', options.value); 31 | } 32 | if (options.checked && result instanceof HTMLInputElement) { 33 | result.checked = options.checked; 34 | } 35 | if (options.innerText) { 36 | result.innerText = options.innerText; 37 | } 38 | if (options.attr) { 39 | for (const [key, value] of Object.entries(options.attr)) { 40 | result.setAttribute(key, '' + value); 41 | } 42 | } 43 | return result; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /app/ts/elements/login-form.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement} from 'custom-elements-ts'; 2 | import {Login} from "./login"; 3 | import {GenerationRequestForm} from "./generation-request-form"; 4 | import {AuthHelper} from "../helpers/auth"; 5 | import {EnvHelper} from "../helpers/env"; 6 | import {BaseElement} from "./base-element"; 7 | 8 | @CustomElement({ 9 | tag: 'vc-login-form', 10 | shadow: false, 11 | style: ``, 12 | template: ` 13 | <div class="login-form"></div> 14 | ` 15 | }) 16 | export class LoginForm extends BaseElement { 17 | $root: HTMLElement 18 | $login: Login 19 | $form: GenerationRequestForm 20 | 21 | constructor() { 22 | super(); 23 | } 24 | 25 | connectedCallback() { 26 | this.$root = this.querySelector('.login-form'); 27 | this.draw(); 28 | AuthHelper.listen(this.draw.bind(this)); 29 | } 30 | 31 | draw() { 32 | this.$root.innerHTML = '' 33 | 34 | if (EnvHelper.useLocal) { 35 | return; 36 | } 37 | 38 | if (AuthHelper.isAuthenticated()) { 39 | this.$form = document.createElement('vc-generation-request-form') as GenerationRequestForm; 40 | this.$root.appendChild(this.$form); 41 | } else { 42 | this.$login = document.createElement('vc-login') as Login; 43 | this.$root.appendChild(this.$login); 44 | } 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "vc", 3 | "version": "1.0.0", 4 | "description": "VC (\"virtual content\") is a tiny JEMSCI app that generates explorable dream worlds.", 5 | "private": true, 6 | "directories": { 7 | "test": "tests" 8 | }, 9 | "scripts": { 10 | "serve": "webpack-dev-server --host 0.0.0.0" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "git+https://github.com/alex-moon/vc.git" 15 | }, 16 | "keywords": [ 17 | "vqgan", 18 | "clip", 19 | "machine", 20 | "learning", 21 | "artificial", 22 | "intelligence", 23 | "3d", 24 | "photo", 25 | "inpainting", 26 | "image", 27 | "super", 28 | "resolution" 29 | ], 30 | "author": "Alex Moon", 31 | "license": "ISC", 32 | "bugs": { 33 | "url": "https://github.com/alex-moon/vc/issues" 34 | }, 35 | "homepage": "https://github.com/alex-moon/vc#readme", 36 | "dependencies": { 37 | "copy-webpack-plugin": "^9.0.1", 38 | "css-loader": "^6.2.0", 39 | "custom-elements-ts": "^0.0.16", 40 | "dayjs": "^1.10.7", 41 | "html-webpack-plugin": "^5.3.2", 42 | "node-sass": "^6.0.1", 43 | "sass-loader": "^12.1.0", 44 | "style-loader": "^3.2.1", 45 | "ts-loader": "^9.2.5", 46 | "typescript": "^4.4.3" 47 | }, 48 | "devDependencies": { 49 | "webpack": "^5.52.1", 50 | "webpack-cli": "^4.8.0", 51 | "webpack-dev-server": "^4.2.1" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /app/ts/elements/generation-request-form/add-spec.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CustomElement, 3 | Dispatch, 4 | DispatchEmitter, 5 | Listen 6 | } from 'custom-elements-ts'; 7 | import {GenerationSpec} from "../../models/generation-spec"; 8 | 9 | @CustomElement({ 10 | tag: 'vc-add-spec', 11 | shadow: false, 12 | style: ``, 13 | template: ` 14 | <div class="add-spec"> 15 | <div class="add-image"> 16 | Add image 17 | <img src="../../../assets/add-image.png" /> 18 | </div> 19 | <div class="add-video"> 20 | Add video 21 | <img src="../../../assets/add-video.png" /> 22 | </div> 23 | </div> 24 | `, 25 | }) 26 | export class AddSpec extends HTMLElement { 27 | $root: HTMLElement 28 | private spec: GenerationSpec; 29 | 30 | @Dispatch() addedImage: DispatchEmitter; 31 | @Dispatch() addedVideo: DispatchEmitter; 32 | 33 | constructor() { 34 | super(); 35 | } 36 | 37 | connectedCallback() { 38 | this.$root = this.querySelector('.add-spec'); 39 | } 40 | 41 | @Listen('click', '.add-image') 42 | addImage() { 43 | this.addedImage.emit(); 44 | } 45 | 46 | @Listen('click', '.add-video') 47 | addVideo() { 48 | this.addedVideo.emit(); 49 | } 50 | 51 | update(spec: GenerationSpec) { 52 | this.spec = spec; 53 | this.draw(); 54 | } 55 | 56 | draw() { 57 | 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /vc/command/inpainting.py: -------------------------------------------------------------------------------- 1 | from math import pi 2 | from injector import inject 3 | 4 | from vc.command.base import BaseCommand 5 | from vc.service.inpainting import InpaintingService, InpaintingOptions 6 | 7 | 8 | class InpaintingCommand(BaseCommand): 9 | description = 'Runs inpainting on an input file' 10 | args = [ 11 | { 12 | 'dest': 'input_file', 13 | 'type': str, 14 | 'help': 'Input file', 15 | 'default': 'output.png', 16 | 'nargs': '?', 17 | }, 18 | { 19 | 'dest': 'output_file', 20 | 'type': str, 21 | 'help': 'Output file', 22 | 'default': 'debug.png', 23 | 'nargs': '?', 24 | }, 25 | ] 26 | 27 | inpainting: InpaintingService 28 | 29 | @inject 30 | def __init__(self, inpainting: InpaintingService): 31 | self.inpainting = inpainting 32 | 33 | def handle(self, args): 34 | print('got %s %s' % (args.input_file, args.output_file)) 35 | self.inpainting.handle(InpaintingOptions( 36 | input_file=args.input_file, 37 | output_filename=args.output_file, 38 | offscreen_rendering=True, 39 | x_shift=-0.01, 40 | y_shift=0.01, 41 | z_shift=-0.01, 42 | pan=-(2 * pi / 360), 43 | tilt=-(2 * pi / 360), 44 | roll=-(2 * pi / 360) 45 | )) 46 | -------------------------------------------------------------------------------- /vc/command/vqgan_clip.py: -------------------------------------------------------------------------------- 1 | from injector import inject 2 | 3 | from vc.command.base import BaseCommand 4 | from vc.service.vqgan_clip import VqganClipService, VqganClipOptions 5 | 6 | 7 | class VqganClipCommand(BaseCommand): 8 | description = 'Runs vqgan_clip on an input file' 9 | args = [ 10 | { 11 | 'dest': 'output_file', 12 | 'type': str, 13 | 'help': 'Output file', 14 | 'default': 'debug.png', 15 | 'nargs': '?', 16 | }, 17 | { 18 | 'dest': 'prompt', 19 | 'type': str, 20 | 'help': 'Text prompts with : and | as usual', 21 | 'default': 'two people embracing, friends who have not seen each other in a long time | animal_ | Bruce Davidson', 22 | 'nargs': '?', 23 | }, 24 | { 25 | 'dest': 'iterations', 26 | 'type': int, 27 | 'help': 'Iterations', 28 | 'default': 200, 29 | 'nargs': '?', 30 | } 31 | ] 32 | 33 | vqgan_clip: VqganClipService 34 | 35 | @inject 36 | def __init__(self, vqgan_clip: VqganClipService): 37 | self.vqgan_clip = vqgan_clip 38 | 39 | def handle(self, args): 40 | self.vqgan_clip.handle(VqganClipOptions( 41 | init_image=None, 42 | output_filename=args.output_file, 43 | prompts=args.prompt, 44 | max_iterations=args.iterations, 45 | )) 46 | -------------------------------------------------------------------------------- /app/scss/_theme.scss: -------------------------------------------------------------------------------- 1 | @import "normalize"; 2 | 3 | $border-width: 2px; 4 | $border-radius: 8px; 5 | $button-height: 48px; 6 | $button-width: 9em; 7 | $transition: 0.5s; 8 | $margin: 8px; 9 | 10 | button { 11 | background: #d7e6d6; 12 | border: solid 1px #5c6267; 13 | border-radius: $border-radius; 14 | height: $button-height; 15 | width: $button-width; 16 | cursor: pointer; 17 | display: flex; 18 | justify-content: center; 19 | align-items: center; 20 | 21 | &:hover, &:active { 22 | background: #f0f5f0; 23 | } 24 | 25 | &.warn { 26 | background: #e6cfb9; 27 | 28 | &:hover, &:active { 29 | background: #f5ede6; 30 | } 31 | } 32 | 33 | &[disabled] { 34 | background: none; 35 | border: none; 36 | opacity: 0.5; 37 | 38 | &:hover { 39 | background: none; 40 | } 41 | &:active { 42 | background: none; 43 | } 44 | } 45 | 46 | &.material-icons { 47 | display: inline-block; 48 | border: none; 49 | padding: 0; 50 | margin: 0; 51 | background: none; 52 | height: inherit; 53 | width: inherit; 54 | text-decoration: none; 55 | cursor: pointer; 56 | text-align: center; 57 | -webkit-appearance: none; 58 | -moz-appearance: none; 59 | 60 | opacity: 0.5; 61 | color: #63575b; 62 | 63 | &:hover { 64 | opacity: 1; 65 | } 66 | &:active { 67 | color: black; 68 | } 69 | } 70 | 71 | span.material-icons { 72 | margin-right: 8px; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /vc/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dotenv import load_dotenv 4 | from flask import Flask 5 | from flask_cors import CORS 6 | from flask_injector import FlaskInjector 7 | 8 | from .api import api 9 | from .controller import init_app 10 | from .db import db 11 | from .migrate import migrate 12 | from .injector import injector 13 | from .q import q 14 | from .services import modules 15 | 16 | 17 | def create_app(): 18 | # initialise os.environ 19 | load_dotenv(override=True) 20 | 21 | # initialise the app 22 | app = Flask(__name__) 23 | app.config.update(os.environ) 24 | app.config['SQLALCHEMY_ENGINE_OPTIONS'] = {'poolclass': NullPool} 25 | app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://%s:%s@%s:%s/%s' % ( 26 | app.config['DB_USER'], 27 | app.config['DB_PASS'], 28 | app.config['DB_HOST'], 29 | app.config['DB_PORT'], 30 | app.config['DB_NAME'], 31 | ) 32 | app.config['RQ_DEFAULT_URL'] = 'redis://:%s@%s:%s/1' % ( 33 | app.config['REDIS_PASS'], 34 | app.config['REDIS_HOST'], 35 | app.config['REDIS_PORT'], 36 | ) 37 | 38 | # CORS 39 | CORS(app, resources={r"/*": {"origins": "*"}}) 40 | 41 | # spin everything up 42 | api.init_app(app) 43 | db.init_app(app) 44 | migrate.init_app(app, db) 45 | controller.init_app(app) 46 | q.init_app(app) 47 | app.app_context().push() 48 | db.create_all() 49 | FlaskInjector(app=app, modules=modules, injector=injector) 50 | return app 51 | -------------------------------------------------------------------------------- /app/ts/elements/chipset.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement, Dispatch, DispatchEmitter, Toggle} from 'custom-elements-ts'; 2 | import {BaseElement} from "./base-element"; 3 | 4 | @CustomElement({ 5 | tag: 'vc-chipset', 6 | shadow: false, 7 | style: ``, 8 | template: ` 9 | <div class="chipset"></div> 10 | ` 11 | }) 12 | export class Chipset extends BaseElement { 13 | texts: string[]; 14 | 15 | $root: HTMLElement; 16 | 17 | @Toggle() removable: boolean; 18 | @Dispatch('chipset.remove') onRemove: DispatchEmitter; 19 | 20 | constructor() { 21 | super(); 22 | } 23 | 24 | connectedCallback() { 25 | this.$root = this.querySelector('.chipset'); 26 | 27 | if (this.texts) { 28 | this.populateChips(); 29 | } 30 | } 31 | 32 | update(texts: string[]) { 33 | this.texts = texts || []; 34 | 35 | if (this.$root) { 36 | this.populateChips(); 37 | } 38 | } 39 | 40 | populateChips() { 41 | this.$root.innerHTML = ''; 42 | 43 | for (const text of this.texts) { 44 | const chip = this.el('vc-chip', {attr: {text}}); 45 | if (this.removable) { 46 | chip.setAttribute('removable', ''); 47 | chip.addEventListener('chip.remove', this.onChipRemove.bind(this)); 48 | } 49 | this.$root.appendChild(chip); 50 | } 51 | } 52 | 53 | protected onChipRemove(e: Event) { 54 | this.onRemove.emit(e); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /vc/services.py: -------------------------------------------------------------------------------- 1 | from typing import Type 2 | from injector import singleton, Binder 3 | 4 | from vc import manager, event, event_listener 5 | from vc.event_listener import generation_request 6 | 7 | 8 | def bind_singleton(binder: Binder, obj): 9 | binder.bind(obj, to=obj, scope=singleton) 10 | 11 | 12 | def managers(binder: Binder): 13 | # @todo bind_manager with ModelFactory and ModelEventDispatcher 14 | bind_singleton(binder, manager.GenerationRequestManager) 15 | bind_singleton(binder, manager.GenerationResultManager) 16 | bind_singleton(binder, manager.UserManager) 17 | 18 | 19 | def bind_event_listener( 20 | binder: Binder, 21 | vc_event: Type[event.VcEvent], 22 | vc_event_listener: Type[event_listener.VcEventListener] 23 | ): 24 | bind_singleton(binder, vc_event_listener) 25 | binder.injector.get(event.VcEventDispatcher).register( 26 | vc_event.id, 27 | binder.injector.get(vc_event_listener).on 28 | ) 29 | 30 | 31 | def events(binder: Binder): 32 | bind_singleton(binder, event.VcEventDispatcher) 33 | bind_event_listener( 34 | binder, 35 | event.GenerationRequestCreatedEvent, 36 | event_listener.generation_request.GenerationRequestCreatedEventListener 37 | ) 38 | bind_event_listener( 39 | binder, 40 | event.GenerationRequestCancelledEvent, 41 | event_listener.generation_request.GenerationRequestCancelledEventListener 42 | ) 43 | 44 | 45 | modules = [ 46 | managers, 47 | events, 48 | ] 49 | -------------------------------------------------------------------------------- /vqgan/zoom.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Example "Zoom" movie generation 3 | # e.g. ./zoom.sh "A painting of zooming in to a surreal, alien world" Zoom.png 180 4 | 5 | TEXT="$1" 6 | FILENAME="$2" 7 | MAX_EPOCHS=$3 8 | 9 | LR=0.1 10 | OPTIMISER=Adam 11 | MAX_ITERATIONS=25 12 | SEED=`shuf -i 1-9999999999 -n 1` # Keep the same seed each epoch for more deterministic runs 13 | 14 | # Extract 15 | FILENAME_NO_EXT=${FILENAME%.*} 16 | FILE_EXTENSION=${FILENAME##*.} 17 | 18 | # Initial run 19 | python3 generate.py -p="$TEXT" -opt="$OPTIMISER" -lr=$LR -i=$MAX_ITERATIONS -se=$MAX_ITERATIONS --seed=$SEED -o="$FILENAME" 20 | cp "$FILENAME" "$FILENAME_NO_EXT"-0000."$FILE_EXTENSION" 21 | convert "$FILENAME" -distort SRT 1.01,0 -gravity center "$FILENAME" # Zoom 22 | convert "$FILENAME" -distort SRT 1 -gravity center "$FILENAME" # Rotate 23 | 24 | # Feedback image loop 25 | for (( i=1; i<=$MAX_EPOCHS; i++ )) 26 | do 27 | padded_count=$(printf "%04d" "$i") 28 | python3 generate.py -p="$TEXT" -opt="$OPTIMISER" -lr=$LR -i=$MAX_ITERATIONS -se=$MAX_ITERATIONS --seed=$SEED -ii="$FILENAME" -o="$FILENAME" 29 | cp "$FILENAME" "$FILENAME_NO_EXT"-"$padded_count"."$FILE_EXTENSION" 30 | convert "$FILENAME" -distort SRT 1.01,0 -gravity center "$FILENAME" # Zoom 31 | convert "$FILENAME" -distort SRT 1 -gravity center "$FILENAME" # Rotate 32 | done 33 | 34 | # Make video - Nvidia GPU expected 35 | ffmpeg -y -i "$FILENAME_NO_EXT"-%04d."$FILE_EXTENSION" -b:v 8M -c:v h264_nvenc -pix_fmt yuv420p -strict -2 -filter:v "minterpolate='mi_mode=mci:mc_mode=aobmc:vsbmc=1:fps=60'" video.mp4 36 | -------------------------------------------------------------------------------- /docker/nginx/ssl/vc.local.crt: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEBzCCAu+gAwIBAgIUV/kNB0Oy9d0dsLlScqKIr3CwphUwDQYJKoZIhvcNAQEL 3 | BQAwgZIxCzAJBgNVBAYTAkdCMRYwFAYDVQQIDA1UeW5lIGFuZCBXZWFyMRIwEAYD 4 | VQQHDAlHYXRlc2hlYWQxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0 5 | ZDERMA8GA1UEAwwIdmMubG9jYWwxITAfBgkqhkiG9w0BCQEWEmZvcm1zYXR6QGdt 6 | YWlsLmNvbTAeFw0yMTExMjcxNTQzMzlaFw0yMTEyMjcxNTQzMzlaMIGSMQswCQYD 7 | VQQGEwJHQjEWMBQGA1UECAwNVHluZSBhbmQgV2VhcjESMBAGA1UEBwwJR2F0ZXNo 8 | ZWFkMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxETAPBgNVBAMM 9 | CHZjLmxvY2FsMSEwHwYJKoZIhvcNAQkBFhJmb3Jtc2F0ekBnbWFpbC5jb20wggEi 10 | MA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDiuTK8GejdSdlPTqh4iq97kQAK 11 | dLfWoepEZX37eKGVl7ElAz2zfMKSMIBWKP47WpURsZAjazIzYeaJL3gNRTJENDWb 12 | mYh4aVqTBMJ8w9AD95uoyOGIpPvhdv2kUVYMEiqFNW28IDYb8SI5Y8cvGim/CWKW 13 | mpSGQThpo4swn2fv96XkwKHGRgMgkOTmbh15lrW/LM3nD2A4jNGpEkOVOIP/EY75 14 | LPZ6wO6PTRfXNd2VRf+NAS/bAQ1pDqkk1JRQElxTHdYx/GByppOGAfWXiUhmNU7j 15 | eP2YNkYMMDkF9xaHvAEwcvPjVglAdbZGHjpkcfyDI2ggm3ry9E9EVasky+pZAgMB 16 | AAGjUzBRMB0GA1UdDgQWBBRp5n8LEG3hFZ0/AxLLjAckjKdiyjAfBgNVHSMEGDAW 17 | gBRp5n8LEG3hFZ0/AxLLjAckjKdiyjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 18 | DQEBCwUAA4IBAQBI2Kg5ercnM5OtGfwRIG/P72QP/YLKlv6FegKt/gYA9oHhK8cA 19 | /HXtFTAG4EaO7uDH+wbWYKNx0ouJLp8WpBBAibvb7j0KlAgVLH8fEdmy2HPQpGxL 20 | TCLuGR2XQb9/En6hVVrP+WG8qL7B1guAKwT7U2nzwktJonM/WMVWMGr5bLhT4r92 21 | a46DtP/kxTyGbGSDXDgPO1jHmWV2JkmoDPSJFrv/V4kGXNs5bHKQph1qgei6+VDH 22 | p1bp+9rHr5XfwUwu4shfZ11wFcZswXd+b5z/JX1aIN1OSrrfrHHkFhcpCzzfQW6M 23 | 7H+8MrdwbqTZ1U3EU8QVnp2FvsB/NZqUTeZl 24 | -----END CERTIFICATE----- 25 | -------------------------------------------------------------------------------- /app/ts/elements/generation-request.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement, Listen} from 'custom-elements-ts'; 2 | import {GenerationRequest as Model} from "../models/generation-request"; 3 | import {GenerationRequestSummary} from "./generation-request/summary"; 4 | import {GenerationRequestDetails} from "./generation-request/details"; 5 | 6 | @CustomElement({ 7 | tag: 'vc-generation-request', 8 | shadow: false, 9 | style: ``, 10 | template: ` 11 | <div class="request"> 12 | <vc-generation-request-summary></vc-generation-request-summary> 13 | <vc-generation-request-details></vc-generation-request-details> 14 | </div> 15 | ` 16 | }) 17 | export class GenerationRequest extends HTMLElement { 18 | $root: HTMLElement 19 | $summary: GenerationRequestSummary 20 | $details: GenerationRequestDetails 21 | 22 | request: Model 23 | expanded = false 24 | 25 | constructor() { 26 | super(); 27 | } 28 | 29 | connectedCallback() { 30 | this.$root = this.querySelector('.request'); 31 | this.$summary = this.$root.querySelector('vc-generation-request-summary'); 32 | this.$details = this.$root.querySelector('vc-generation-request-details'); 33 | } 34 | 35 | update(request: Model) { 36 | this.request = request; 37 | this.$summary.update(this.request); 38 | this.$details.update(this.request); } 39 | 40 | @Listen('summary.expand', 'vc-generation-request-summary') 41 | protected onExpand(e: any) { 42 | this.$details.setAttribute('expanded', e.detail); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /Taskfile.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | tasks: 4 | start: scripts/aws.start.sh 5 | stop: scripts/aws.stop.sh 6 | sh: scripts/sh.sh 7 | build: scripts/build.sh 8 | run: scripts/run.sh 9 | db: scripts/db.sh 10 | dump: scripts/dump.sh 11 | migrate: scripts/migrate.sh 12 | restore: scripts/restore.sh 13 | 14 | webpack: npx webpack --node-env=local 15 | 16 | serve: 17 | cmds: 18 | - task: webpack 19 | - npx webpack serve --node-env=local 20 | 21 | # deploy_backend: scripts/backend.sh 22 | # 23 | # deploy_frontend: scripts/frontend.sh 24 | # 25 | # deploy_private: 26 | # deps: [deploy_backend, deploy_frontend] 27 | # 28 | # deploy.public: scripts/public.sh 29 | # 30 | # deploy: 31 | # deps: [deploy_private, deploy_public] 32 | 33 | deploy: 34 | env: 35 | app: vc 36 | cmds: 37 | - | 38 | ssh -t ajmoon " 39 | if [[ ! -d '/opt/$app' ]]; then 40 | mkdir /opt/$app 41 | cd /opt/$app 42 | git clone https://github.com/alex-moon/$app . 43 | fi 44 | cd /opt/$app 45 | git reset --hard HEAD 46 | git pull origin \$(git rev-parse --abbrev-ref HEAD) 47 | docker compose build 48 | docker stack deploy -c docker-compose.yml $app 49 | docker run --rm -i \ 50 | -v /var/run/docker.sock:/var/run/docker.sock \ 51 | -u root ubirak/docker-php:latest \ 52 | stack:converge $app 53 | docker system prune -f 54 | scripts/frontend.sh 55 | # scripts/backend.sh 56 | " 57 | -------------------------------------------------------------------------------- /app/scss/_footer.scss: -------------------------------------------------------------------------------- 1 | .footer { 2 | height: 64px; 3 | display: flex; 4 | align-items: center; 5 | justify-content: center; 6 | background-color: #535c69; 7 | color: white; 8 | box-sizing: border-box; 9 | padding: 16px; 10 | 11 | @include mobile { 12 | padding: 0; 13 | } 14 | 15 | a { 16 | text-decoration: none; 17 | color: #c5d9dd; 18 | &:hover { 19 | color: #d7e6d6; 20 | } 21 | &:active { 22 | color: #efedd4; 23 | } 24 | } 25 | 26 | .buttons { 27 | display: flex; 28 | margin-left: 16px; 29 | 30 | .footer-button { 31 | cursor: pointer; 32 | background-size: contain; 33 | width: 28px; 34 | height: 28px; 35 | margin: 18px 8px; 36 | padding: 0; 37 | color: rgba(0, 0, 0, 0); 38 | 39 | @include mobile { 40 | margin: 18px 4px; 41 | } 42 | } 43 | 44 | .footer-button:hover { 45 | filter: brightness(200); 46 | } 47 | 48 | .github.footer-button { 49 | background: url('../assets/github.png'); 50 | } 51 | 52 | .tezos.footer-button { 53 | background: url('../assets/tezos.png'); 54 | } 55 | 56 | .patreon.footer-button { 57 | background: url('../assets/patreon.png'); 58 | } 59 | 60 | .twitter.footer-button { 61 | background: url('../assets/twitter.png'); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /vqgan/random.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | text_one=("A painting of a" "A pencil art sketch of a" "An illustration of a" "A photograph of a") 4 | text_two=("spinning" "dreaming" "watering" "loving" "eating" "drinking" "sleeping" "repeating" "surreal" "psychedelic") 5 | text_three=("fish" "egg" "peacock" "watermelon" "pickle" "horse" "dog" "house" "kitchen" "bedroom" "door" "table" "lamp" "dresser" "watch" "logo" "icon" "tree" 6 | "grass" "flower" "plant" "shrub" "bloom" "screwdriver" "spanner" "figurine" "statue" "graveyard" "hotel" "bus" "train" "car" "lamp" "computer" "monitor") 7 | styles=("Art Nouveau" "Camille Pissarro" "Michelangelo Caravaggio" "Claude Monet" "Edgar Degas" "Edvard Munch" "Fauvism" "Futurism" "Impressionism" 8 | "Picasso" "Pop Art" "Modern art" "Surreal Art" "Sandro Botticelli" "oil paints" "watercolours" "weird bananas" "strange colours") 9 | 10 | pickword() { 11 | local array=("$@") 12 | ARRAY_RANGE=$((${#array[@]}-1)) 13 | RANDOM_ENTRY=`shuf -i 0-$ARRAY_RANGE -n 1` 14 | UPDATE=${array[$RANDOM_ENTRY]} 15 | } 16 | 17 | 18 | # Generate some images 19 | for number in {1..50} 20 | do 21 | # Make some random text 22 | pickword "${text_one[@]}" 23 | TEXT=$UPDATE 24 | pickword "${text_two[@]}" 25 | TEXT+=" "$UPDATE 26 | pickword "${text_three[@]}" 27 | TEXT+=" "$UPDATE 28 | pickword "${text_three[@]}" 29 | TEXT+=" and a "$UPDATE 30 | pickword "${styles[@]}" 31 | TEXT+=" in the style of "$UPDATE 32 | pickword "${styles[@]}" 33 | TEXT+=" and "$UPDATE 34 | 35 | python3 generate.py -p "$TEXT" -o "$number".png 36 | done 37 | 38 | 39 | -------------------------------------------------------------------------------- /vc/service/helper/vqgan.py: -------------------------------------------------------------------------------- 1 | import gc 2 | import sys 3 | 4 | sys.path.append('taming-transformers') 5 | 6 | from omegaconf import OmegaConf 7 | from taming.models import cond_transformer, vqgan 8 | import torch 9 | 10 | 11 | class VqganHelper: 12 | gumbel: bool 13 | 14 | def __init__(self): 15 | self.gumbel = False 16 | 17 | def load_vqgan_model(self, config_path, checkpoint_path): 18 | self.gumbel = False 19 | config = OmegaConf.load(config_path) 20 | if config.model.target == 'taming.models.vqgan.VQModel': 21 | model = vqgan.VQModel(**config.model.params) 22 | model.eval().requires_grad_(False) 23 | model.init_from_ckpt(checkpoint_path) 24 | elif config.model.target == 'taming.models.vqgan.GumbelVQ': 25 | model = vqgan.GumbelVQ(**config.model.params) 26 | model.eval().requires_grad_(False) 27 | model.init_from_ckpt(checkpoint_path) 28 | self.gumbel = True 29 | elif config.model.target == 'taming.models.cond_transformer.Net2NetTransformer': 30 | parent_model = cond_transformer.Net2NetTransformer( 31 | **config.model.params 32 | ) 33 | parent_model.eval().requires_grad_(False) 34 | parent_model.init_from_ckpt(checkpoint_path) 35 | model = parent_model.first_stage_model 36 | else: 37 | raise ValueError(f'unknown model type: {config.model.target}') 38 | 39 | del model.loss 40 | gc.collect() 41 | torch.cuda.empty_cache() 42 | 43 | return model 44 | -------------------------------------------------------------------------------- /requirements.api.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.13.0 2 | aiohttp==3.7.4.post0 3 | antlr4-python3-runtime==4.8 4 | async-timeout==3.0.1 5 | attrs==21.2.0 6 | backcall==0.2.0 7 | bcrypt 8 | boto3 9 | cachetools==4.2.2 10 | certifi==2021.5.30 11 | chardet==4.0.0 12 | charset-normalizer==2.0.3 13 | cryptography 14 | dacite==1.6.0 15 | decorator 16 | einops==0.3.0 17 | flask==2.1.3 18 | flask-injector==0.14.0 19 | flask-restful 20 | flask-rq 21 | flask-sqlalchemy==2.5.1 22 | flask-cors 23 | flask-httpauth 24 | flask-migrate 25 | freetype-py==2.2.0 26 | fsspec==2021.7.0 27 | ftfy==6.0.3 28 | future==0.18.2 29 | google-auth==1.34.0 30 | google-auth-oauthlib==0.4.4 31 | grpcio==1.39.0 32 | guppy3==3.1.1 33 | hurry.filesize==0.9 34 | idna==3.2 35 | idna-ssl==1.1.0 36 | importlib-metadata==4.6.1 37 | injector 38 | ipdb==0.13.13 39 | ipython 40 | ipython-genutils==0.2.0 41 | jedi==0.18.0 42 | jsonschema 43 | markdown==3.3.4 44 | markupsafe==2.0.1 45 | multidict==5.1.0 46 | oauthlib==3.1.1 47 | omegaconf==2.1.0 48 | packaging==21.0 49 | parso==0.8.2 50 | pexpect==4.8.0 51 | pickleshare==0.7.5 52 | prompt-toolkit==3.0.19 53 | protobuf==3.17.3 54 | psycopg2-binary 55 | ptyprocess==0.7.0 56 | pyasn1==0.4.8 57 | pyasn1-modules==0.2.8 58 | pydeprecate==0.3.1 59 | pyparsing==2.4.7 60 | python-dateutil 61 | python-dotenv 62 | pytz 63 | pyyaml==5.4.1 64 | redis==3.5.3 65 | regex==2021.7.6 66 | requests==2.26.0 67 | requests-oauthlib==1.3.0 68 | rq 69 | rsa==4.7.2 70 | six==1.16.0 71 | sqlalchemy<2.0 72 | tqdm==4.61.2 73 | traitlets==4.3.3 74 | typing-extensions>4 75 | urllib3==1.26.6 76 | uwsgi 77 | wcwidth==0.2.5 78 | werkzeug==2.0.3 79 | whistle 80 | yarl==1.6.3 81 | -------------------------------------------------------------------------------- /tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | IFS=$'\n' 4 | LR=0.1 5 | OPTIMISER=Adam 6 | MAX_ITERATIONS=20 7 | MAX_EPOCHS=100 8 | MAX_REPEATS=2 9 | SEED=`shuf -i 1-9999999999 -n 1` 10 | 11 | i=0 12 | function generate() { 13 | padded_count=$(printf "%04d" "$i") 14 | python3 generate.py -p="$1" \ 15 | -i=$MAX_ITERATIONS \ 16 | -se=$MAX_ITERATIONS \ 17 | --seed=$SEED \ 18 | -ii="input/input.png" \ 19 | -o="input/input.png" 20 | cp "input/input.png" "input/input-$padded_count.png" 21 | (( i ++ )) 22 | } 23 | 24 | function zoom() { 25 | for (( j=1; j<=$MAX_EPOCHS; j++ )); do 26 | generate "$1" 27 | 28 | # scale, rotate, translate: <coords from>, <scale (multiple)>, <rotate (degrees)>, <coords to> 29 | convert "input/input.png" -distort SRT "0,0 1.01 1 10,0" -gravity center "input/input.png" 30 | done 31 | } 32 | 33 | function zoom3d() { 34 | for (( j=1; j<=$MAX_EPOCHS; j++ )); do 35 | generate "$1" 36 | 37 | python3 zoom3d.py "input/input.png" 38 | mv results/input.png input/input.png 39 | done 40 | } 41 | 42 | for (( repeat=1; repeat<=$MAX_REPEATS; repeat++ )); do 43 | for test in $(cat tests.txt); do 44 | zoom3d "$test" 45 | for style in $(cat styles.txt); do 46 | if [[ -z "$style" ]]; then 47 | break 48 | fi 49 | zoom3d "$test | $style" 50 | done 51 | done 52 | done 53 | 54 | ffmpeg -y -i "input/input-%04d.png" -b:v 8M -c:v h264_nvenc -pix_fmt yuv420p -strict -2 -filter:v "minterpolate='mi_mode=mci:mc_mode=aobmc:vsbmc=1:fps=60'" results/video-$(date -Iseconds | sed 's/[^0-9]/-/g').mp4 55 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | services: 3 | flask: 4 | build: 5 | context: ./ 6 | dockerfile: docker/flask/Dockerfile 7 | image: vc.ajmoon.com/flask 8 | volumes: 9 | - .:/opt/vc 10 | - .bashrc:/root/.bashrc 11 | environment: 12 | SERVICE: flask 13 | links: 14 | - pgsql 15 | command: ./docker.flask.sh 16 | # command: sleep 999999999 17 | ports: 18 | - "15001:5000" 19 | 20 | nginx: 21 | build: 22 | context: . 23 | dockerfile: docker/nginx/Dockerfile 24 | image: vc.ajmoon.com/nginx 25 | links: 26 | - flask 27 | volumes: 28 | - .:/opt/vc 29 | - ./log/nginx/:/var/log/nginx 30 | - ./docker/nginx/conf/flask.conf:/etc/nginx/nginx.conf 31 | ports: 32 | - "10001:80" 33 | 34 | pgsql: 35 | image: postgis/postgis:9.5-2.5-alpine 36 | environment: 37 | POSTGRES_PASSWORD: ${DB_PASS} 38 | POSTGRES_USER: ${DB_USER} 39 | PGDATA: /var/lib/postgresql/data/pgdata 40 | volumes: 41 | - ./pgsql:/var/lib/postgresql/data 42 | ports: 43 | - "15433:5432" 44 | 45 | # worker: 46 | # image: vc.ajmoon.com/flask 47 | # volumes: 48 | # - .:/opt/vc 49 | # - .bashrc:/root/.bashrc 50 | # links: 51 | # - pgsql 52 | # - redis 53 | # depends_on: 54 | # - pgsql 55 | # environment: 56 | # SERVICE: worker 57 | # command: sleep 999999999 58 | # command: ./docker.worker.sh 59 | 60 | redis: 61 | build: 62 | context: . 63 | dockerfile: docker/redis/Dockerfile 64 | image: vc.ajmoon.com/redis 65 | ports: 66 | - "16380:6379" 67 | -------------------------------------------------------------------------------- /vc/api.py: -------------------------------------------------------------------------------- 1 | from flask import jsonify 2 | from flask_restful import Api 3 | from werkzeug.http import HTTP_STATUS_CODES 4 | from werkzeug.exceptions import HTTPException 5 | 6 | from vc.controller import ( 7 | GenerationRequestController, 8 | GenerationRequestsController, 9 | MeController, 10 | MeTokenController, 11 | UserController, 12 | UsersController, 13 | UserTokenController, 14 | GenerationRequestActionController, 15 | ) 16 | 17 | class VcApi(Api): 18 | def handle_error(self, err): 19 | print(err) 20 | 21 | if isinstance(err, HTTPException): 22 | return jsonify({ 23 | 'message': getattr( 24 | err, 25 | 'description', 26 | HTTP_STATUS_CODES.get(err.code, '') 27 | ) 28 | }), err.code 29 | 30 | # if not getattr(err, 'message', None): 31 | # return jsonify({ 32 | # 'message': 'Server has encountered some error' 33 | # }), 500 34 | 35 | return jsonify(**err.kwargs), err.http_status_code 36 | 37 | api = VcApi() 38 | api.add_resource(GenerationRequestController, '/generation-request/<int:id_>') 39 | api.add_resource(GenerationRequestActionController, '/generation-request/<int:id_>/<string:action>') 40 | api.add_resource(GenerationRequestsController, '/generation-request') 41 | api.add_resource(MeController, '/user/me') 42 | api.add_resource(MeTokenController, '/user/me/token') 43 | api.add_resource(UserController, '/user/<int:id_>') 44 | api.add_resource(UserTokenController, '/user/<int:id_>/token') 45 | api.add_resource(UsersController, '/user') 46 | -------------------------------------------------------------------------------- /vqgan/video_styler.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Video styler - Use all images in a directory and style them 3 | # video_styler.sh video.mp4 4 | 5 | # Style text 6 | TEXT="Oil painting of a woman in the foreground | pencil art landscape background" 7 | 8 | ## Input and output frame directories 9 | FRAMES_IN="/home/nerdy/github/VQGAN-CLIP/VideoFrames" 10 | FRAMES_OUT="/home/nerdy/github/VQGAN-CLIP/Saves/VideoStyleTesting" 11 | 12 | ## Output image size 13 | HEIGHT=640 14 | WIDTH=360 15 | 16 | ## Iterations 17 | ITERATIONS=25 18 | SAVE_EVERY=$ITERATIONS 19 | 20 | ## Optimiser & Learning rate 21 | OPTIMISER=Adagrad # Adam, AdamW, Adagrad, Adamax 22 | LR=0.2 23 | 24 | # Fixed seed 25 | SEED=`shuf -i 1-9999999999 -n 1` # Keep the same seed each frame for more deterministic runs 26 | 27 | # MAIN 28 | ############################ 29 | mkdir -p "$FRAMES_IN" 30 | mkdir -p "$FRAMES_OUT" 31 | 32 | # For cuDNN determinism 33 | export CUBLAS_WORKSPACE_CONFIG=:4096:8 34 | 35 | # Extract video into frames 36 | ffmpeg -y -i "$1" -q:v 2 "$FRAMES_IN"/frame-%04d.jpg 37 | 38 | # Style all the frames 39 | ls "$FRAMES_IN" | while read file; do 40 | # Set the output filename 41 | FILENAME="$FRAMES_OUT"/"$file"-"out".jpg 42 | 43 | # And imagine! 44 | echo "Input frame: $file" 45 | echo "Style text: $TEXT" 46 | echo "Output file: $FILENAME" 47 | 48 | python3 generate.py -p "$TEXT" -ii "$FRAMES_IN"/"$file" -o "$FILENAME" -opt "$OPTIMISER" -lr "$LR" -i "$ITERATIONS" -se "$SAVE_EVERY" -s "$HEIGHT" "$WIDTH" -sd "$SEED" -d True 49 | done 50 | 51 | ffmpeg -y -i "$FRAMES_OUT"/frame-%04d.jpg-out.jpg -b:v 8M -c:v h264_nvenc -pix_fmt yuv420p -strict -2 -filter:v "minterpolate='mi_mode=mci:mc_mode=aobmc:vsbmc=1:fps=60'" style_video.mp4 52 | -------------------------------------------------------------------------------- /docker/nginx/conf/flask.conf: -------------------------------------------------------------------------------- 1 | worker_processes 4; 2 | 3 | error_log /var/log/nginx/error.log; 4 | 5 | pid /var/run/nginx.pid; 6 | 7 | events { 8 | worker_connections 1024; 9 | } 10 | 11 | http { 12 | include /etc/nginx/mime.types; 13 | default_type text/plain; 14 | 15 | log_format main '$remote_addr - $remote_user [$time_local] "$request" ' 16 | '$status $body_bytes_sent "$http_referer" ' 17 | '"$http_user_agent" "$http_x_forwarded_for"'; 18 | 19 | access_log /var/log/nginx/access.log main; 20 | 21 | # client_body_temp_path /var/tmp/nginx/client_body_temp; 22 | 23 | sendfile on; 24 | keepalive_timeout 15; 25 | 26 | gzip on; 27 | gzip_min_length 1000; 28 | gzip_proxied expired no-cache no-store private auth; 29 | gzip_types text/plain text/xml text/css application/x-javascript application/javascript; 30 | 31 | client_max_body_size 20M; 32 | 33 | server { 34 | listen 80; 35 | server_name _; 36 | 37 | access_log /var/log/nginx/vc.access.log; 38 | error_log /var/log/nginx/vc.error.log; 39 | 40 | location /api/ { 41 | proxy_pass http://flask:5000/; 42 | proxy_redirect off; 43 | 44 | proxy_set_header Host $host; 45 | proxy_set_header X-Real-IP $remote_addr; 46 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 47 | proxy_set_header X-Forwarded-Proto $scheme; 48 | } 49 | 50 | location / { 51 | root /opt/vc/public; 52 | index index.html index.htm; 53 | try_files $uri $uri/ /index.html; 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /vqgan/opt_tester.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Using each optimiser, generate images using a range of learning rates 4 | # Produce a labelled montage to easily view the results 5 | 6 | TEXT="A painting in the style of Paul Gauguin" 7 | OUT_DIR="/home/nerdy/github/VQGAN-CLIP/Saves/OptimiserTesting-60it-Noise-NPW-1" 8 | ITERATIONS=60 9 | SAVE_EVERY=60 10 | HEIGHT=256 11 | WIDTH=256 12 | SEED=`shuf -i 1-9999999999 -n 1` # Keep the same seed each epoch for more deterministic runs 13 | 14 | # Main 15 | ################# 16 | 17 | export CUBLAS_WORKSPACE_CONFIG=:4096:8 18 | mkdir -p "$OUT_DIR" 19 | 20 | function do_optimiser_test () { 21 | OPTIMISER="$1" 22 | LR="$2" 23 | STEP="$3" 24 | NPW="$4" 25 | for i in {1..10} 26 | do 27 | PADDED_COUNT=$(printf "%03d" "$COUNT") 28 | echo "Loop for $OPTIMISER - $LR" 29 | python3 generate.py -p "$TEXT" -in pixels -o "$OUT_DIR"/"$PADDED_COUNT"-"$OPTIMISER"-"$LR"-"$NPW".png -opt "$OPTIMISER" -lr "$LR" -i "$ITERATIONS" -se "$SAVE_EVERY" -s "$HEIGHT" "$WIDTH" --seed "$SEED" -d True -iw 1 -nps 666 -npw "$NPW" -d True 30 | LR=$(echo $LR + $STEP | bc) 31 | ((COUNT++)) 32 | done 33 | } 34 | 35 | # Test optimisers 36 | COUNT=0 37 | do_optimiser_test "Adam" .1 .1 1 38 | COUNT=10 39 | do_optimiser_test "AdamW" .1 .1 1 40 | COUNT=20 41 | do_optimiser_test "Adamax" .1 .1 1 42 | COUNT=30 43 | do_optimiser_test "Adagrad" .1 .25 1 44 | COUNT=40 45 | do_optimiser_test "AdamP" .1 .25 1 46 | COUNT=50 47 | do_optimiser_test "RAdam" .1 .25 1 48 | COUNT=60 49 | do_optimiser_test "DiffGrad" .1 .25 1 50 | 51 | # Make montage 52 | mogrify -font Liberation-Sans -fill white -undercolor '#00000080' -pointsize 14 -gravity NorthEast -annotate +10+10 %t "$OUT_DIR"/*.png 53 | montage "$OUT_DIR"/*.png -geometry 256x256+1+1 -tile 10x7 collage.jpg 54 | -------------------------------------------------------------------------------- /vc/manager/user.py: -------------------------------------------------------------------------------- 1 | from uuid import uuid4 2 | from vc.db import db 3 | from vc.manager.base import Manager 4 | from vc.model.user import User 5 | from vc.service.helper.hash import HashHelper 6 | 7 | 8 | class UserManager(Manager): 9 | model_class = User 10 | 11 | def authenticate(self, token): 12 | try: 13 | hashed = HashHelper.get(token) 14 | return self.model_class.query.filter( 15 | self.model_class.deleted.__eq__(None), 16 | self.model_class.token.__eq__(hashed) 17 | ).first() 18 | except Exception as e: 19 | db.session.rollback() 20 | raise e 21 | 22 | def create(self, raw, user: User = None): 23 | raw = self.fields(raw, user) 24 | api_token = str(uuid4()) 25 | token = HashHelper.get(api_token) 26 | raw.update({"token": token}) 27 | try: 28 | # @todo ModelFactory.create here 29 | 30 | user = self.model_class(**raw) 31 | self.save(user) 32 | 33 | # @todo ModelEventDispatcher.dispatchCreated here 34 | 35 | user.api_token = api_token 36 | return user 37 | except Exception as e: 38 | db.session.rollback() 39 | raise e 40 | 41 | def regenerate_token(self, id_): 42 | api_token = str(uuid4()) 43 | token = HashHelper.get(api_token) 44 | try: 45 | user = self.find_or_throw(id_) 46 | user.token = token 47 | self.save(user) 48 | 49 | # @todo ModelEventDispatcher.dispatchUpdated here 50 | 51 | user.api_token = api_token 52 | return user 53 | except Exception as e: 54 | db.session.rollback() 55 | raise e 56 | -------------------------------------------------------------------------------- /docker/nginx/ssl/vc.local.key: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDiuTK8GejdSdlP 3 | Tqh4iq97kQAKdLfWoepEZX37eKGVl7ElAz2zfMKSMIBWKP47WpURsZAjazIzYeaJ 4 | L3gNRTJENDWbmYh4aVqTBMJ8w9AD95uoyOGIpPvhdv2kUVYMEiqFNW28IDYb8SI5 5 | Y8cvGim/CWKWmpSGQThpo4swn2fv96XkwKHGRgMgkOTmbh15lrW/LM3nD2A4jNGp 6 | EkOVOIP/EY75LPZ6wO6PTRfXNd2VRf+NAS/bAQ1pDqkk1JRQElxTHdYx/GByppOG 7 | AfWXiUhmNU7jeP2YNkYMMDkF9xaHvAEwcvPjVglAdbZGHjpkcfyDI2ggm3ry9E9E 8 | Vasky+pZAgMBAAECggEAOF5fNNEkSlBgDQ27aSzh8FDet7rJMj3TZuns5qbZVb3/ 9 | NvqVTpjIL1yLq+BJRSyCP4HsGkI+mvPOQ3Ff4bzAgKCSa3tCqN1YvZDDYg21HCdl 10 | thMrr7OFREs3SLFIOXwSwkfEXq+LhGrxXV9nWFii5HJK5rLMFo0En48fvhkD2G88 11 | CUM1V9JcgvPSFJ9MS/waIZfVu3ECF8pibgtHz8dLUpY1bZO8zojvEZC8/Y8iPrHM 12 | pjlPuv2kTGOaF0IZ6DqymWpHUdkmYtwLRYzweSLy4O/zUjj9g3/p1U3EFVEAtQ37 13 | 8e1VNhIBAczKqEtPZvBVK6Aul5N2FraYRHPdTQe2uQKBgQD87xJfx8/n8KjPpwJL 14 | XHWg13KnVolivm2L+EEEZ3JdhBiN8U0LnurylBMPpmKc8NEBSjXRAhTKRp6llsM/ 15 | Viv5FbaCxg5apnWkR4XblmlauTfaJka61rvD3QPZjgfbds7e09Ix1sGmgD3yh3nI 16 | YzbtEasewBho8W0lfz5WQIzARwKBgQDleMmmFxnwPkLw4oOn9LamxhL2J1cda9lK 17 | ZbjNYYJ66i9FSLZu/I0G+K86xsWeQMimh1PnBzfgT1ePPxF4gowg/80vuTjWEQgF 18 | LvVshNt9X4Uyl3v7e0oYxhItFL9nt3yYGo68Jqw39NKLk0+j9n0DwTL/EPsEBSmK 19 | MH3WBkLwXwKBgQDFqc+iKnS/vcMvBEU2v7rtR3HCdE5JbKgPiqLE3lLx4hmOU9Ty 20 | D+9sQ2vh/xVxsIClkWkdPZMhwK6amYdcJ+uVbPKEoGL3VL768yoYwSL0U2rigKcl 21 | vEpgdohbrDNXvzFL0h2h9ULA69d9gYIap+50tioUjeqFmKbDdv0wSqG0YQKBgERY 22 | ED7efIQ4FvnljO62DLxpG/aiSujhWSs2Vq8czp7LJt8MxAEUSPB9mrr3vp3ej6RB 23 | g/S4CqVNHDf2wb5Rq2icp0ECenIPSoM4tDHo8/JIQ8SIp42D79KnE3zHxCYg2QP4 24 | TPCu1/YmfZrCeiLMY/l9TjH3nbtZykB1JtVasg8jAoGBAKuivEszBT7jf8Rdxwnm 25 | KupHClKMQ1lSbrZrH/t4GPcxKcNcBX0KSRyzA22ScOzYyc6//RdcrVUPhdV5Pdau 26 | Gh6+2wa8g1dIAYpjjYpYpUTlOsyp5e1gJ8Ue94mavet3pnwCyuckzvneyErq2JEu 27 | KO1a0m2mdrn6J/XWKKkAKmj9 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /app/scss/generation_request/_step.scss: -------------------------------------------------------------------------------- 1 | vc-generation-request-details-step { 2 | .step { 3 | display: flex; 4 | background-color: #f0f5f0; 5 | padding: 8px; 6 | border-radius: $border-radius; 7 | border: solid $border-width #5c6267; 8 | margin: 8px; 9 | box-sizing: border-box; 10 | 11 | .number { 12 | font-size: 48px; 13 | color: #5c6267; 14 | display: flex; 15 | align-items: center; 16 | justify-content: center; 17 | 18 | span { 19 | width: 64px; 20 | display: flex; 21 | align-items: center; 22 | justify-content: center; 23 | } 24 | } 25 | 26 | .spec { 27 | padding: 8px 0; 28 | flex-grow: 1; 29 | 30 | .text-chipset { 31 | margin-bottom: 16px; 32 | h3 { 33 | margin: 0; 34 | } 35 | } 36 | 37 | .fields { 38 | display: flex; 39 | flex-flow: row wrap; 40 | 41 | .field { 42 | background: #f9f8ef; 43 | border: solid 1px #5c6267; 44 | border-radius: $border-radius; 45 | padding: 8px; 46 | position: relative; 47 | display: flex; 48 | margin: 8px 8px 0 0; 49 | 50 | label { 51 | font-weight: bold; 52 | display: block; 53 | margin-right: 8px; 54 | } 55 | 56 | span { 57 | 58 | } 59 | } 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /app/partials/header.html: -------------------------------------------------------------------------------- 1 | <!DOCTYPE html> 2 | <html> 3 | <head> 4 | <meta charset="utf-8"> 5 | <title>vc: Virtual Content by AJ Moon</title> 6 | <meta name="description" content="vc is a digital art project by North East England-based British/Australian software engineer Alex Moon, combining existing AI/machine learning technologies to create explorable dream worlds."> 7 | <meta name="viewport" content="width=device-width, initial-scale=1.0"> 8 | <link href="https://fonts.googleapis.com/css?family=Roboto+Slab" rel="stylesheet"> 9 | <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet"> 10 | 11 | <meta property='twitter:card' content='summary_large_image' /> 12 | <meta property='twitter:site' content='@ajmoonne' /> 13 | <meta property='twitter:creator' content='@ajmoonne' /> 14 | <meta property='twitter:title' content='vc: Virtual Content by AJ Moon' /> 15 | <meta property='twitter:description' content='vc is a digital art project by North East England-based British/Australian software engineer Alex Moon, combining existing AI/machine learning technologies to create explorable dream worlds.' /> 16 | <meta property='twitter:image' content='https://vc-ajmoon.com.s3.eu-west-1.amazonaws.com/assets/twitter-card.png' /> 17 | </head> 18 | <body> 19 | <div class="container"> 20 | <div class="header"> 21 | <div class="header-first"> 22 | <a href="/"> 23 | <h1>vc: Virtual Content</h1> 24 | </a> 25 | </div> 26 | <div class="header-second"> 27 | <a href="/news"> 28 | <div class="info header-button">Info</div> 29 | </a> 30 | </div> 31 | </div> 32 | -------------------------------------------------------------------------------- /vc/service/helper/abme/correlation_package/correlation_cuda_kernel.cuh: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include <ATen/ATen.h> 4 | #include <ATen/Context.h> 5 | #include <cuda_runtime.h> 6 | 7 | int correlation_forward_cuda_kernel(at::Tensor& output, 8 | int ob, 9 | int oc, 10 | int oh, 11 | int ow, 12 | int osb, 13 | int osc, 14 | int osh, 15 | int osw, 16 | 17 | at::Tensor& input1, 18 | int ic, 19 | int ih, 20 | int iw, 21 | int isb, 22 | int isc, 23 | int ish, 24 | int isw, 25 | 26 | at::Tensor& input2, 27 | int gc, 28 | int gsb, 29 | int gsc, 30 | int gsh, 31 | int gsw, 32 | 33 | at::Tensor& rInput1, 34 | at::Tensor& rInput2, 35 | int pad_size, 36 | int kernel_size, 37 | int max_displacement, 38 | int stride1, 39 | int stride2, 40 | int corr_type_multiply, 41 | cudaStream_t stream); 42 | 43 | 44 | int correlation_backward_cuda_kernel( 45 | at::Tensor& gradOutput, 46 | int gob, 47 | int goc, 48 | int goh, 49 | int gow, 50 | int gosb, 51 | int gosc, 52 | int gosh, 53 | int gosw, 54 | 55 | at::Tensor& input1, 56 | int ic, 57 | int ih, 58 | int iw, 59 | int isb, 60 | int isc, 61 | int ish, 62 | int isw, 63 | 64 | at::Tensor& input2, 65 | int gsb, 66 | int gsc, 67 | int gsh, 68 | int gsw, 69 | 70 | at::Tensor& gradInput1, 71 | int gisb, 72 | int gisc, 73 | int gish, 74 | int gisw, 75 | 76 | at::Tensor& gradInput2, 77 | int ggc, 78 | int ggsb, 79 | int ggsc, 80 | int ggsh, 81 | int ggsw, 82 | 83 | at::Tensor& rInput1, 84 | at::Tensor& rInput2, 85 | int pad_size, 86 | int kernel_size, 87 | int max_displacement, 88 | int stride1, 89 | int stride2, 90 | int corr_type_multiply, 91 | cudaStream_t stream); 92 | -------------------------------------------------------------------------------- /planning.py: -------------------------------------------------------------------------------- 1 | import json 2 | import csv 3 | from re import split 4 | 5 | 6 | def to_float(input: str): 7 | return float(input) if input else 0. 8 | 9 | 10 | steps = [] 11 | with open('planning/october.csv') as csv_file: 12 | reader = csv.DictReader(csv_file) 13 | fieldnames = reader.fieldnames 14 | ground = None 15 | for row in reader: 16 | row_ground = split('[^a-z]', row['ground'])[0] 17 | if ground is None: 18 | ground = row_ground 19 | 20 | if row_ground != ground: 21 | ground = row_ground 22 | steps.append({ 23 | 'texts': ['a burst of flames, a flash of burning fire'], 24 | 'x_velocity': to_float(row['x']), 25 | 'y_velocity': to_float(row['y']), 26 | 'z_velocity': to_float(row['z']), 27 | 'pan_velocity': to_float(row['pan']), 28 | 'tilt_velocity': to_float(row['tilt']), 29 | 'roll_velocity': to_float(row['roll']), 30 | 'upscale': True, 31 | 'interpolate': True, 32 | 'epochs': 5, 33 | 'iterations': 150, 34 | 'transition': 5, 35 | }) 36 | 37 | steps.append({ 38 | 'texts': [row['text']], 39 | 'styles': ['%s | %s' % (row['ground'], row['style'])], 40 | 'x_velocity': to_float(row['x']), 41 | 'y_velocity': to_float(row['y']), 42 | 'z_velocity': to_float(row['z']), 43 | 'pan_velocity': to_float(row['pan']), 44 | 'tilt_velocity': to_float(row['tilt']), 45 | 'roll_velocity': to_float(row['roll']), 46 | 'upscale': True, 47 | 'interpolate': True, 48 | 'epochs': 42, 49 | 'iterations': 75, 50 | }) 51 | 52 | print(json.dumps({ 53 | "spec": { 54 | "videos": [ 55 | { 56 | "steps": steps 57 | } 58 | ] 59 | } 60 | })) 61 | -------------------------------------------------------------------------------- /app/scss/generation_request_form/_image_spec_form.scss: -------------------------------------------------------------------------------- 1 | @import "../theme"; 2 | 3 | vc-image-spec-form { 4 | .image-spec-form { 5 | background-color: #c5d9dd; 6 | padding: 8px; 7 | border-radius: $border-radius; 8 | border: solid $border-width #5c6267; 9 | box-sizing: border-box; 10 | width: 100%; 11 | margin-bottom: 16px; 12 | 13 | button { 14 | width: 32px !important; 15 | height: 32px !important; 16 | font-size: 32px !important; 17 | line-height: 32px !important; 18 | } 19 | 20 | .spec-header { 21 | display: flex; 22 | justify-content: space-between; 23 | align-items: center; 24 | h3 { 25 | margin: 0; 26 | padding: 0; 27 | } 28 | } 29 | 30 | .texts, .styles { 31 | margin: 8px; 32 | } 33 | 34 | .text-input { 35 | cursor: text; 36 | background: #eaf1f2; 37 | border: solid 1px #5c6267; 38 | border-radius: $border-radius; 39 | padding: 8px; 40 | position: relative; 41 | 42 | &:hover { 43 | background-color: #f0f5f0; 44 | } 45 | 46 | label { 47 | cursor: text; 48 | font-size: medium; 49 | font-weight: bold; 50 | margin: 0; 51 | display: block; 52 | width: 100%; 53 | } 54 | 55 | textarea, input { 56 | border: none; 57 | background: none; 58 | resize: none; 59 | outline: none; 60 | width: calc(100% - 32px); 61 | margin-top: 4px; 62 | } 63 | 64 | textarea { 65 | height: 5em; 66 | } 67 | 68 | button { 69 | position: absolute; 70 | bottom: 8px; 71 | right: 8px; 72 | } 73 | } 74 | 75 | .options { 76 | padding: 8px; 77 | display: flex; 78 | flex-flow: row wrap; 79 | 80 | @include mobile { 81 | flex-flow: nowrap; 82 | flex-direction: column; 83 | } 84 | @import 'image_spec_option'; 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /app/ts/helpers/status.ts: -------------------------------------------------------------------------------- 1 | import {GenerationRequest} from "../models/generation-request"; 2 | 3 | const dayjs = require('dayjs'); 4 | 5 | export enum StatusField { 6 | QUEUED = 'created', 7 | STARTED = 'started', 8 | COMPLETED = 'completed', 9 | FAILED = 'failed', 10 | CANCELLED = 'cancelled', 11 | RETRIED = 'retried', 12 | } 13 | 14 | export class Status { 15 | static DATETIME_FORMAT = 'ddd D MMM [at] h:mma'; 16 | 17 | field: StatusField; 18 | value: string; 19 | constructor(field: StatusField, value: string) { 20 | this.field = field; 21 | this.value = value; 22 | } 23 | get readable() { 24 | return { 25 | [StatusField.QUEUED]: 'Queued', 26 | [StatusField.STARTED]: 'Started', 27 | [StatusField.COMPLETED]: 'Completed', 28 | [StatusField.FAILED]: 'Failed', 29 | [StatusField.CANCELLED]: 'Cancelled', 30 | [StatusField.RETRIED]: 'Restarted', 31 | }[this.field]; 32 | } 33 | get datetime() { 34 | return dayjs(this.value).format(Status.DATETIME_FORMAT); 35 | } 36 | } 37 | 38 | export class StatusHelper { 39 | static get(request: GenerationRequest) { 40 | const [field, value] = StatusHelper.getMostRecent(request); 41 | return new Status(field, value); 42 | } 43 | 44 | private static getMostRecent(request: GenerationRequest) { 45 | let latestValue = null; 46 | let latestField = null; 47 | for (const field of Object.values(StatusField)) { 48 | const value = (request as any)[field]; 49 | if (latestField === null) { 50 | latestField = field; 51 | } 52 | if (latestValue === null) { 53 | latestValue = value; 54 | } 55 | if (value > latestValue) { 56 | latestValue = value; 57 | latestField = field; 58 | } 59 | } 60 | return [latestField, latestValue]; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /vc/service/isr.py: -------------------------------------------------------------------------------- 1 | import os 2 | from dataclasses import dataclass 3 | 4 | import numpy as np 5 | # from ISR.models import RRDN 6 | from PIL import Image 7 | from injector import inject 8 | 9 | from vc.service import FileService 10 | from vc.service.helper.dimensions import DimensionsHelper 11 | 12 | 13 | @dataclass 14 | class IsrOptions: 15 | input_file: str = 'output.png' 16 | output_file: str = 'output-upscaled.png' 17 | 18 | 19 | class IsrService: 20 | BORDER = 2 21 | 22 | file_service: FileService 23 | 24 | @inject 25 | def __init__(self, file_service: FileService): 26 | self.file_service = file_service 27 | 28 | def handle(self, args: IsrOptions): 29 | image = Image.open(args.input_file) 30 | image = image.convert('RGB') 31 | image_array = np.array(image) 32 | 33 | # model = RRDN( 34 | # arch_params={'C': 4, 'D': 3, 'G': 32, 'G0': 32, 'T': 10, 'x': 4} 35 | # ) 36 | # model.model.load_weights( 37 | # 'weights/rrdn-C4-D3-G32-G032-T10-x4_epoch299.hdf5' 38 | # ) 39 | # 40 | # result = model.predict(image_array) 41 | result = image_array 42 | 43 | output = Image.fromarray(result) 44 | 45 | # Resize 46 | target_width = DimensionsHelper.width_large() 47 | target_height = DimensionsHelper.height_large() 48 | width = target_width + 2 * self.BORDER 49 | height = target_height + 2 * self.BORDER 50 | output.thumbnail((width, height), Image.ANTIALIAS) 51 | 52 | # Crop 53 | output.crop( 54 | self.BORDER, 55 | self.BORDER, 56 | target_width, 57 | target_height 58 | ) 59 | 60 | # Save 61 | output.save(args.output_file) 62 | 63 | if os.getenv('DEBUG_FILES'): 64 | self.file_service.put( 65 | args.output_file, 66 | 'isr-%s' % ( 67 | args.output_file 68 | ) 69 | ) 70 | -------------------------------------------------------------------------------- /vc/service/file.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from injector import inject 3 | from flask import Flask 4 | import boto3 5 | import os 6 | from vc.service.helper import DiagnosisHelper as dh 7 | 8 | 9 | class FileService: 10 | # URL_PATTERN = 'https://{bucket}.s3.{region}.amazonaws.com/{filename}' 11 | URL_PATTERN = 'https://{bucket}.{region}.digitaloceanspaces.com/{filename}' 12 | client = None # @todo how to typehint this for the IDE? 13 | bucket: str 14 | region: str 15 | 16 | @inject 17 | def __init__(self, app: Flask): 18 | self.bucket = app.config.get('AWS_BUCKET_NAME') 19 | self.region = app.config.get('AWS_BUCKET_REGION') 20 | self.client = boto3.client( 21 | 's3', 22 | region_name=self.region, 23 | endpoint_url='https://{region}.digitaloceanspaces.com'.format(region=self.region), 24 | aws_access_key_id=app.config.get('AWS_ACCESS_KEY_ID'), 25 | aws_secret_access_key=app.config.get('AWS_SECRET_ACCESS_KEY') 26 | ) 27 | 28 | def put(self, local_file, filename, now: datetime = None): 29 | filename = self.get_filename(filename, now) 30 | url = self.URL_PATTERN.format( 31 | bucket=self.bucket, 32 | region=self.region, 33 | filename=filename 34 | ) 35 | dh.debug("FileService", "put", os.path.abspath(local_file), url) 36 | try: 37 | self.client.upload_file( 38 | local_file, 39 | self.bucket, 40 | filename, 41 | ExtraArgs={'ACL': 'public-read'} 42 | ) 43 | return url 44 | except Exception as e: 45 | dh.debug("FileService", "put", "Exception", e) 46 | return None 47 | 48 | def get_filename(self, filename, now: datetime = None): 49 | if now is None: 50 | now = datetime.now() 51 | return '%s-%s' % ( 52 | now.strftime('%Y-%m-%d-%H-%M-%S'), 53 | filename 54 | ) 55 | -------------------------------------------------------------------------------- /vc/service/helper/abme/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import torch 3 | import torch.nn as nn 4 | import time 5 | import numpy as np 6 | 7 | TAG_CHAR = np.array([202021.25], np.float32) 8 | 9 | def load_flow(path): 10 | with open(path, 'rb') as f: 11 | magic = float(np.fromfile(f, np.float32, count = 1)[0]) 12 | if magic == 202021.25: 13 | w, h = np.fromfile(f, np.int32, count = 1)[0], np.fromfile(f, np.int32, count = 1)[0] 14 | data = np.fromfile(f, np.float32, count = h*w*2) 15 | data.resize((h, w, 2)) 16 | return data 17 | return None 18 | 19 | def tic(): 20 | global startTime_for_tictoc 21 | startTime_for_tictoc = time.time() 22 | 23 | def toc(): 24 | if 'startTime_for_tictoc' in globals(): 25 | print("Elapsed time is "+ str(time.time() - startTime_for_tictoc)+" seconds") 26 | #str(time.time() - startTime_for tictoc) 27 | else: 28 | print("Toc: start time not set") 29 | 30 | def warp(x,flo, return_mask=True): 31 | B, C, H, W = x.size() 32 | # mesh grid 33 | xx = torch.arange(0, W).view(1, 1, 1, W).expand(B, 1, H, W) 34 | yy = torch.arange(0, H).view(1, 1, H, 1).expand(B, 1, H, W) 35 | 36 | grid = torch.cat((xx, yy), 1).float() 37 | 38 | if x.is_cuda: 39 | grid = grid.to(x.device) 40 | 41 | vgrid = torch.autograd.Variable(grid) + flo 42 | 43 | # scale grid to [-1,1] 44 | vgrid[:, 0, :, :] = 2.0 * vgrid[:, 0, :, :] / max(W - 1, 1) - 1.0 45 | vgrid[:, 1, :, :] = 2.0 * vgrid[:, 1, :, :] / max(H - 1, 1) - 1.0 46 | 47 | vgrid = vgrid.permute(0, 2, 3, 1) 48 | output = nn.functional.grid_sample(x, vgrid, align_corners=True) 49 | mask = torch.autograd.Variable(torch.ones(x.size())).to(x.device) 50 | mask = nn.functional.grid_sample(mask, vgrid, align_corners=True) 51 | 52 | mask = mask.masked_fill_(mask < 0.999, 0) 53 | mask = mask.masked_fill_(mask > 0, 1) 54 | 55 | if return_mask: 56 | return output * mask, mask 57 | else: 58 | return output * mask 59 | -------------------------------------------------------------------------------- /app/scss/_generation_request_form.scss: -------------------------------------------------------------------------------- 1 | vc-generation-request-form { 2 | display: flex; 3 | 4 | .request-form { 5 | width: 100%; 6 | 7 | h2 { 8 | font-size: x-large; 9 | margin: 0; 10 | padding: 16px; 11 | display: flex; 12 | align-items: center; 13 | justify-content: space-between; 14 | cursor: pointer; 15 | height: 64px; 16 | box-sizing: border-box; 17 | 18 | @include mobile { 19 | font-size: large; 20 | } 21 | 22 | span { 23 | margin-left: 2px; 24 | } 25 | 26 | .greeting { 27 | font-weight: bold; 28 | } 29 | 30 | .expand { 31 | font-weight: normal; 32 | display: flex; 33 | align-items: center; 34 | } 35 | } 36 | form { 37 | padding: 0 16px; 38 | overflow: auto; 39 | height: 0; 40 | box-sizing: border-box; 41 | 42 | &.expanded { 43 | height: calc(100vh - 192px); 44 | padding: 0 16px 16px; 45 | } 46 | 47 | .steps { 48 | display: flex; 49 | flex-direction: column; 50 | 51 | @import "generation_request_form/add_spec"; 52 | @import "generation_request_form/image_spec_form"; 53 | @import "generation_request_form/video_spec_form"; 54 | } 55 | /* 56 | 57 | */ 58 | 59 | .actions { 60 | margin-top: 16px; 61 | display: flex; 62 | justify-content: flex-end; 63 | 64 | button { 65 | background-color: #eaf1f2; 66 | 67 | &:hover { 68 | background-color: #f0f5f0; 69 | } 70 | } 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /vc/service/provider/api.py: -------------------------------------------------------------------------------- 1 | from dataclasses import is_dataclass, asdict 2 | from flask import Flask 3 | from injector import inject 4 | from requests import Request, Session 5 | from requests.exceptions import RequestException 6 | from typing import List, Union, Dict 7 | from urllib.parse import urljoin 8 | 9 | from vc.exception import ThirdPartyException 10 | 11 | 12 | class ApiProvider: 13 | session: Session 14 | base_uri: str 15 | token: str 16 | 17 | @inject 18 | def __init__(self, app: Flask): 19 | self.session = Session() 20 | self.base_uri = app.config.get('API_URL') 21 | self.token = app.config.get('API_TOKEN') 22 | 23 | def get(self, url) -> Union[Dict, List]: 24 | return self.request('GET', url) 25 | 26 | def put(self, url, data=None) -> Union[Dict, List]: 27 | return self.request('PUT', url, data) 28 | 29 | def request(self, method, url, data=None) -> Union[Dict, List]: 30 | headers = { 31 | "Authorization": "Bearer %s" % self.token 32 | } 33 | 34 | data = self.marshall(data) if data else {} 35 | 36 | request = Request( 37 | method, 38 | urljoin(self.base_uri, url), 39 | json=data, 40 | headers=headers 41 | ) 42 | 43 | prepared_request = self.session.prepare_request(request) 44 | try: 45 | response = self.session.send(prepared_request) 46 | if response.status_code > 400: 47 | print("Error response from API: [%s] %s" % ( 48 | response.status_code, 49 | response.text[:1024] 50 | )) 51 | raise ThirdPartyException(response.text[:1024]) 52 | except RequestException as e: 53 | print("Error from API: %s" % e) 54 | raise ThirdPartyException.for_exception(e) 55 | return response.json() 56 | 57 | def marshall(self, data) -> Union[Dict, List, str]: 58 | if isinstance(data, list): 59 | return list(map(self.marshall, data)) 60 | if is_dataclass(data): 61 | return asdict(data) 62 | return data 63 | -------------------------------------------------------------------------------- /isr.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | 4 | import dotenv 5 | import boto3 6 | import numpy as np 7 | from ISR.models import RRDN 8 | from PIL import Image 9 | 10 | 11 | class FileUploader: 12 | def __init__(self): 13 | dotenv.load_dotenv() 14 | self.client = boto3.client( 15 | 's3', 16 | aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'), 17 | aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY') 18 | ) 19 | self.bucket = os.getenv('AWS_BUCKET_NAME') 20 | self.region = os.getenv('AWS_BUCKET_REGION') 21 | 22 | def put(self, local_file): 23 | now = datetime.now() 24 | filename = '%s-%s' % ( 25 | now.strftime('%Y-%m-%d-%H-%M-%S'), 26 | os.path.basename(local_file) 27 | ) 28 | self.client.upload_file( 29 | local_file, 30 | self.bucket, 31 | filename, 32 | ExtraArgs={'ACL': 'public-read'} 33 | ) 34 | 35 | 36 | input_dir = 'usteps' 37 | output_dir = 'ustepsu' 38 | 39 | for filename in os.listdir(input_dir): 40 | input_file = os.path.join(input_dir, filename) 41 | output_file = os.path.join(output_dir, filename) 42 | 43 | image = Image.open(input_file) 44 | image = image.convert('RGB') 45 | image_array = np.array(image) 46 | 47 | model = RRDN( 48 | arch_params={'C': 4, 'D': 3, 'G': 32, 'G0': 32, 'T': 10, 'x': 4} 49 | ) 50 | model.model.load_weights( 51 | 'weights/rrdn-C4-D3-G32-G032-T10-x4_epoch299.hdf5' 52 | ) 53 | result = model.predict(image_array) 54 | output = Image.fromarray(result) 55 | output.thumbnail((800, 800), Image.ANTIALIAS) 56 | 57 | output.save(output_file) 58 | 59 | local_file = os.path.abspath('output-upscaled.mp4') 60 | os.system( 61 | ' '.join( 62 | [ 63 | 'ffmpeg -y -i "%s/%%04d.png"' % output_dir, 64 | '-b:v 8M -c:v h264_nvenc -pix_fmt yuv420p -strict -2', 65 | '-filter:v "minterpolate=\'mi_mode=mci:mc_mode=aobmc:vsbmc=1:fps=60\'"', 66 | local_file 67 | ] 68 | ) 69 | ) 70 | 71 | uploader = FileUploader() 72 | uploader.put(local_file) 73 | 74 | -------------------------------------------------------------------------------- /nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name vc.ajmoon.com; 4 | return 301 https://$host$request_uri; 5 | } 6 | 7 | server { 8 | listen 443 ssl http2; 9 | port_in_redirect off; 10 | server_name vc.ajmoon.com; 11 | server_name_in_redirect off; 12 | 13 | # ssl_dhparam /etc/nginx/ssl/dhparam.pem; 14 | ssl_protocols TLSv1.2 TLSv1.1; 15 | ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA'; 16 | ssl_prefer_server_ciphers on; 17 | ssl_stapling on; 18 | ssl_stapling_verify on; 19 | 20 | access_log /var/log/nginx/vc.access.log; 21 | error_log /var/log/nginx/vc.error.log; 22 | 23 | ssl_certificate /etc/letsencrypt/live/vc.ajmoon.com/fullchain.pem; 24 | ssl_certificate_key /etc/letsencrypt/live/vc.ajmoon.com/privkey.pem; 25 | 26 | location /api/ { 27 | proxy_pass http://127.0.0.1:5000/; 28 | proxy_redirect off; 29 | 30 | proxy_set_header Host $host; 31 | proxy_set_header X-Real-IP $remote_addr; 32 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 33 | proxy_set_header X-Forwarded-Proto $scheme; 34 | 35 | add_header Last-Modified $date_gmt; 36 | add_header Cache-Control 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; 37 | if_modified_since off; 38 | expires off; 39 | etag off; 40 | } 41 | 42 | location / { 43 | root /opt/vc/public; 44 | index index.html index.htm; 45 | try_files $uri $uri.html $uri/ /index.html; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /vc/service/helper/rife/model/laplacian.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | import torch.nn as nn 4 | import torch.nn.functional as F 5 | 6 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 7 | 8 | import torch 9 | 10 | def gauss_kernel(size=5, channels=3): 11 | kernel = torch.tensor([[1., 4., 6., 4., 1], 12 | [4., 16., 24., 16., 4.], 13 | [6., 24., 36., 24., 6.], 14 | [4., 16., 24., 16., 4.], 15 | [1., 4., 6., 4., 1.]]) 16 | kernel /= 256. 17 | kernel = kernel.repeat(channels, 1, 1, 1) 18 | kernel = kernel.to(device) 19 | return kernel 20 | 21 | def downsample(x): 22 | return x[:, :, ::2, ::2] 23 | 24 | def upsample(x): 25 | cc = torch.cat([x, torch.zeros(x.shape[0], x.shape[1], x.shape[2], x.shape[3]).to(device)], dim=3) 26 | cc = cc.view(x.shape[0], x.shape[1], x.shape[2]*2, x.shape[3]) 27 | cc = cc.permute(0,1,3,2) 28 | cc = torch.cat([cc, torch.zeros(x.shape[0], x.shape[1], x.shape[3], x.shape[2]*2).to(device)], dim=3) 29 | cc = cc.view(x.shape[0], x.shape[1], x.shape[3]*2, x.shape[2]*2) 30 | x_up = cc.permute(0,1,3,2) 31 | return conv_gauss(x_up, 4*gauss_kernel(channels=x.shape[1])) 32 | 33 | def conv_gauss(img, kernel): 34 | img = torch.nn.functional.pad(img, (2, 2, 2, 2), mode='reflect') 35 | out = torch.nn.functional.conv2d(img, kernel, groups=img.shape[1]) 36 | return out 37 | 38 | def laplacian_pyramid(img, kernel, max_levels=3): 39 | current = img 40 | pyr = [] 41 | for level in range(max_levels): 42 | filtered = conv_gauss(current, kernel) 43 | down = downsample(filtered) 44 | up = upsample(down) 45 | diff = current-up 46 | pyr.append(diff) 47 | current = down 48 | return pyr 49 | 50 | class LapLoss(torch.nn.Module): 51 | def __init__(self, max_levels=5, channels=3): 52 | super(LapLoss, self).__init__() 53 | self.max_levels = max_levels 54 | self.gauss_kernel = gauss_kernel(channels=channels) 55 | 56 | def forward(self, input, target): 57 | pyr_input = laplacian_pyramid(img=input, kernel=self.gauss_kernel, max_levels=self.max_levels) 58 | pyr_target = laplacian_pyramid(img=target, kernel=self.gauss_kernel, max_levels=self.max_levels) 59 | return sum(torch.nn.functional.l1_loss(a, b) for a, b in zip(pyr_input, pyr_target)) 60 | -------------------------------------------------------------------------------- /vc/command/abme_steps.py: -------------------------------------------------------------------------------- 1 | from os import listdir 2 | from os.path import isfile, join, splitext 3 | from shutil import copyfile 4 | from injector import inject 5 | 6 | from vc.command.base import BaseCommand 7 | from vc.service.video import VideoService 8 | from vc.service.abme import AbmeService, AbmeOptions 9 | 10 | 11 | class AbmeStepsCommand(BaseCommand): 12 | description = 'Runs abme on steps files' 13 | args = [ 14 | { 15 | 'dest': 'steps_dir', 16 | 'type': str, 17 | 'help': 'Steps dir', 18 | 'default': 'steps', 19 | 'nargs': '?', 20 | }, 21 | { 22 | 'dest': 'usteps_dir', 23 | 'type': str, 24 | 'help': 'Usteps dir', 25 | 'default': 'usteps', 26 | 'nargs': '?', 27 | } 28 | ] 29 | 30 | abme: AbmeService 31 | video: VideoService 32 | 33 | @inject 34 | def __init__(self, abme: AbmeService, video: VideoService): 35 | self.abme = abme 36 | self.video = video 37 | 38 | def handle(self, args): 39 | input_files = [ 40 | f 41 | for f 42 | in sorted(listdir(args.steps_dir)) 43 | if isfile(join(args.steps_dir, f)) 44 | ] 45 | 46 | for file in input_files: 47 | num, ext = splitext(file) 48 | outnum = int(num) * 2 - 1 49 | outfile = join(args.usteps_dir, f'{outnum:04}.png') 50 | infile = join(args.steps_dir, file) 51 | print('copying', infile, outfile) 52 | copyfile(infile, outfile) 53 | 54 | if int(num) > 1: 55 | last_outnum = outnum - 2 56 | new_outnum = outnum - 1 57 | first_file = join(args.usteps_dir, f'{last_outnum:04}.png') 58 | output_file = join(args.usteps_dir, f'{new_outnum:04}.png') 59 | second_file = outfile 60 | print('running abme', first_file, second_file, output_file) 61 | self.abme.handle( 62 | AbmeOptions( 63 | first_file=first_file, 64 | second_file=second_file, 65 | output_file=output_file, 66 | ) 67 | ) 68 | 69 | self.video.make_unwatermarked_video( 70 | 'debug.mp4', 71 | args.usteps_dir, 72 | fps_multiple=2 73 | ) 74 | -------------------------------------------------------------------------------- /vc/command/rife_steps.py: -------------------------------------------------------------------------------- 1 | from os import listdir 2 | from os.path import isfile, join, splitext 3 | from shutil import copyfile 4 | from injector import inject 5 | 6 | from vc.command.base import BaseCommand 7 | from vc.service.video import VideoService 8 | from vc.service.rife import RifeService, RifeOptions 9 | 10 | 11 | class RifeStepsCommand(BaseCommand): 12 | description = 'Runs rife on steps files' 13 | args = [ 14 | { 15 | 'dest': 'steps_dir', 16 | 'type': str, 17 | 'help': 'Steps dir', 18 | 'default': 'steps', 19 | 'nargs': '?', 20 | }, 21 | { 22 | 'dest': 'usteps_dir', 23 | 'type': str, 24 | 'help': 'Usteps dir', 25 | 'default': 'usteps', 26 | 'nargs': '?', 27 | } 28 | ] 29 | 30 | rife: RifeService 31 | video: VideoService 32 | 33 | @inject 34 | def __init__(self, rife: RifeService, video: VideoService): 35 | self.rife = rife 36 | self.video = video 37 | 38 | def handle(self, args): 39 | input_files = [ 40 | f 41 | for f 42 | in sorted(listdir(args.steps_dir)) 43 | if isfile(join(args.steps_dir, f)) 44 | ] 45 | 46 | for file in input_files: 47 | num, ext = splitext(file) 48 | outnum = int(num) * 2 - 1 49 | outfile = join(args.usteps_dir, f'{outnum:04}.png') 50 | infile = join(args.steps_dir, file) 51 | print('copying', infile, outfile) 52 | copyfile(infile, outfile) 53 | 54 | if int(num) > 1: 55 | last_outnum = outnum - 2 56 | new_outnum = outnum - 1 57 | first_file = join(args.usteps_dir, f'{last_outnum:04}.png') 58 | second_file = outfile 59 | output_file = lambda i: join(args.usteps_dir, f'{i:04}.png'), 60 | print('running rife', first_file, second_file, output_file) 61 | self.rife.handle( 62 | RifeOptions( 63 | first_file=first_file, 64 | second_file=second_file, 65 | output_file=output_file 66 | ) 67 | ) 68 | 69 | self.video.make_unwatermarked_video( 70 | 'debug.mp4', 71 | args.usteps_dir, 72 | fps_multiple=2 73 | ) 74 | -------------------------------------------------------------------------------- /vc/service/helper/abme/correlation_package/correlation.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn.modules.module import Module 3 | from torch.autograd import Function 4 | import correlation_cuda 5 | 6 | class CorrelationFunction(Function): 7 | 8 | @staticmethod 9 | def forward(ctx, input1, input2, pad_size, kernel_size, max_displacement,stride1, stride2, corr_multiply): 10 | ctx.save_for_backward(input1, input2) 11 | ctx.pad_size = pad_size 12 | ctx.kernel_size = kernel_size 13 | ctx.max_displacement = max_displacement 14 | ctx.stride1 = stride1 15 | ctx.stride2 = stride2 16 | ctx.corr_multiply = corr_multiply 17 | 18 | with torch.cuda.device_of(input1): 19 | rbot1 = input1.new() 20 | rbot2 = input2.new() 21 | output = input1.new() 22 | 23 | correlation_cuda.forward(input1, input2, rbot1, rbot2, output, pad_size, kernel_size, max_displacement, stride1, stride2, corr_multiply) 24 | 25 | return output 26 | 27 | @staticmethod 28 | def backward(ctx, grad_output): 29 | input1, input2 = ctx.saved_tensors 30 | 31 | with torch.cuda.device_of(input1): 32 | rbot1 = input1.new() 33 | rbot2 = input2.new() 34 | 35 | grad_input1 = input1.new() 36 | grad_input2 = input2.new() 37 | 38 | correlation_cuda.backward(input1, input2, rbot1, rbot2, grad_output, grad_input1, grad_input2, ctx.pad_size, ctx.kernel_size, ctx.max_displacement, ctx.stride1, ctx.stride2, ctx.corr_multiply) 39 | 40 | return grad_input1, grad_input2, None, None, None, None, None, None 41 | 42 | 43 | class Correlation(Module): 44 | def __init__(self, pad_size=0, kernel_size=0, max_displacement=0, stride1=1, stride2=2, corr_multiply=1): 45 | super(Correlation, self).__init__() 46 | self.pad_size = pad_size 47 | self.kernel_size = kernel_size 48 | self.max_displacement = max_displacement 49 | self.stride1 = stride1 50 | self.stride2 = stride2 51 | self.corr_multiply = corr_multiply 52 | 53 | # @staticmethod 54 | def forward(self, input1, input2): 55 | 56 | input1 = input1.contiguous() 57 | input2 = input2.contiguous() 58 | result = CorrelationFunction.apply(input1, input2, self.pad_size, self.kernel_size, self.max_displacement,self.stride1, self.stride2, self.corr_multiply) 59 | 60 | return result 61 | 62 | -------------------------------------------------------------------------------- /vc/value_object/generation_spec.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional 3 | 4 | from flask_restful import fields 5 | 6 | 7 | @dataclass 8 | class ImageSpec: 9 | texts: List[str] = None 10 | styles: List[str] = None 11 | ground: Optional[str] = None 12 | iterations: int = 200 13 | upscale: bool = False 14 | 15 | schema = { 16 | 'texts': fields.List(fields.String, default=[]), 17 | 'styles': fields.List(fields.String, default=[]), 18 | 'ground': fields.String, 19 | 'iterations': fields.Integer(default=200), 20 | 'upscale': fields.Boolean(default=False), 21 | } 22 | 23 | 24 | @dataclass 25 | class VideoStepSpec(ImageSpec): 26 | iterations: int = 20 27 | interpolate: bool = False 28 | init_iterations: int = 200 29 | epochs: int = 42 30 | transition: int = 20 31 | x_velocity: float = 0. 32 | y_velocity: float = 0. 33 | z_velocity: float = 0. 34 | pan_velocity: float = 0. 35 | tilt_velocity: float = 0. 36 | roll_velocity: float = 0. 37 | random_walk: bool = False 38 | 39 | schema = { 40 | 'texts': fields.List(fields.String, default=[]), 41 | 'styles': fields.List(fields.String, default=[]), 42 | 'ground': fields.String, 43 | 'iterations': fields.Integer(default=20), 44 | 'upscale': fields.Boolean(default=False), 45 | 'interpolate': fields.Boolean(default=False), 46 | 'init_iterations': fields.Integer(default=200), 47 | 'epochs': fields.Integer(default=42), 48 | 'transition': fields.Integer(default=20), 49 | 'x_velocity': fields.Float(default=0.), 50 | 'y_velocity': fields.Float(default=0.), 51 | 'z_velocity': fields.Float(default=0.), 52 | 'pan_velocity': fields.Float(default=0.), 53 | 'tilt_velocity': fields.Float(default=0.), 54 | 'roll_velocity': fields.Float(default=0.), 55 | 'random_walk': fields.Boolean(default=False), 56 | } 57 | 58 | 59 | @dataclass 60 | class VideoSpec: 61 | steps: List[VideoStepSpec] 62 | 63 | schema = { 64 | 'steps': fields.List(fields.Nested(VideoStepSpec.schema)) 65 | } 66 | 67 | 68 | @dataclass 69 | class GenerationSpec: 70 | images: List[ImageSpec] = None 71 | videos: List[VideoSpec] = None 72 | 73 | schema = { 74 | 'images': fields.List(fields.Nested(ImageSpec.schema)), 75 | 'videos': fields.List(fields.Nested(VideoSpec.schema)), 76 | } 77 | -------------------------------------------------------------------------------- /app/ts/managers/base-manager.ts: -------------------------------------------------------------------------------- 1 | import {AuthHelper} from "../helpers/auth"; 2 | import {EnvHelper} from "../helpers/env"; 3 | import {BaseModel} from "../models/base-model"; 4 | 5 | export abstract class BaseManager<M extends BaseModel> { 6 | protected isLocal = EnvHelper.useLocal; 7 | protected host = EnvHelper.host; 8 | protected base_url: string; 9 | 10 | protected full_url(url = '') { 11 | if (url[0] !== '/') { 12 | url = this.host + this.base_url + url; 13 | } 14 | return url.replace(/\/$/g, ''); 15 | } 16 | 17 | protected async fetch(url = ''): Promise<M | M[]> { 18 | url = this.full_url(url) 19 | return new Promise((resolve, reject) => { 20 | fetch(url, { 21 | headers: { 22 | 'Authorization': 'Bearer ' + AuthHelper.getToken(), 23 | 'Content-Type': 'application/json', 24 | }, 25 | }).then((response) => { 26 | if (response.status > 200) { 27 | reject(response.json()); 28 | return; 29 | } 30 | resolve(response.json()); 31 | }) 32 | }); 33 | } 34 | 35 | protected async post(data: M, url = ''): Promise<M> { 36 | url = this.full_url(url) 37 | let status: number; 38 | return fetch(url, { 39 | method: 'POST', 40 | headers: { 41 | 'Authorization': 'Bearer ' + AuthHelper.getToken(), 42 | 'Content-Type': 'application/json', 43 | }, 44 | body: JSON.stringify(data), 45 | }).then((response) => { 46 | status = response.status; 47 | return response.json(); 48 | }).then((json) => { 49 | if (status > 200) { 50 | throw json; 51 | } 52 | return json; 53 | }); 54 | } 55 | 56 | protected async put(url: string, data ?: M): Promise<M> { 57 | url = this.full_url(url); 58 | let status: number; 59 | return fetch(url, { 60 | method: 'PUT', 61 | headers: { 62 | 'Authorization': 'Bearer ' + AuthHelper.getToken(), 63 | 'Content-Type': 'application/json', 64 | }, 65 | }).then((response) => { 66 | status = response.status; 67 | return response.json(); 68 | }).then((json) => { 69 | if (status > 200) { 70 | throw json; 71 | } 72 | return json; 73 | }); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /app/scss/news/_text.scss: -------------------------------------------------------------------------------- 1 | @import "../theme"; 2 | 3 | $avatar-height: 64px; 4 | 5 | .text { 6 | max-width: 800px; 7 | padding: 16px 16px 64px; 8 | 9 | .heading { 10 | height: $avatar-height; 11 | display: flex; 12 | .avatar { 13 | width: $avatar-height; 14 | height: $avatar-height; 15 | } 16 | .avatar.ajmoon { 17 | background: url('../assets/news/ajmoon.png'); 18 | } 19 | .content { 20 | flex-grow: 1; 21 | display: flex; 22 | flex-direction: column; 23 | padding-left: 16px; 24 | justify-content: center; 25 | 26 | h3 { 27 | font-weight: normal; 28 | margin: -8px 0 0; 29 | padding: 0; 30 | font-size: large; 31 | } 32 | .byline { 33 | display: flex; 34 | .author { 35 | 36 | } 37 | .published { 38 | margin-left: 0.5em; 39 | color: rgba(0, 0, 0, 0.5); 40 | } 41 | } 42 | } 43 | } 44 | 45 | h2, h3, p { 46 | font-weight: normal; 47 | margin: 16px 0; 48 | } 49 | 50 | h2 { 51 | margin: 0; 52 | } 53 | 54 | a.button { 55 | color: #535c69; 56 | background: #c5d9dd; 57 | padding: 8px; 58 | margin: 4px 0; 59 | border: solid 1px #535c69; 60 | border-radius: $border-radius; 61 | 62 | strong, span { 63 | display: block; 64 | } 65 | } 66 | 67 | a.button:hover { 68 | background: none; 69 | } 70 | 71 | ul { 72 | list-style: none; 73 | margin: 0; 74 | padding: 0; 75 | 76 | li { 77 | margin: 0; 78 | display: flex; 79 | flex-direction: column; 80 | } 81 | } 82 | 83 | .actions { 84 | margin: 4px 0; 85 | display: flex; 86 | justify-content: flex-end; 87 | 88 | a.button { 89 | width: auto; 90 | flex-grow: 1; 91 | background: #d7e6d6; 92 | border: solid 1px #5c6267; 93 | height: $button-height; 94 | cursor: pointer; 95 | display: flex; 96 | justify-content: center; 97 | align-items: center; 98 | } 99 | 100 | a.button:hover { 101 | background: none; 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /vqgan/vqgan.yml: -------------------------------------------------------------------------------- 1 | name: vqgan 2 | channels: 3 | - pytorch 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - _libgcc_mutex=0.1=conda_forge 8 | - _openmp_mutex=4.5=1_gnu 9 | - ca-certificates=2021.5.30=ha878542_0 10 | - certifi=2021.5.30=py39hf3d152e_0 11 | - ld_impl_linux-64=2.35.1=hea4e1c9_2 12 | - libffi=3.3=h58526e2_2 13 | - libgcc-ng=9.3.0=h2828fa1_19 14 | - libgomp=9.3.0=h2828fa1_19 15 | - libstdcxx-ng=9.3.0=h6de172a_19 16 | - ncurses=6.2=h58526e2_4 17 | - openssl=1.1.1k=h7f98852_0 18 | - pip=21.1.3=pyhd8ed1ab_0 19 | - python=3.9.5=h49503c6_0_cpython 20 | - python_abi=3.9=2_cp39 21 | - readline=8.1=h46c0cb4_0 22 | - setuptools=49.6.0=py39hf3d152e_3 23 | - sqlite=3.36.0=h9cd32fc_0 24 | - tk=8.6.10=h21135ba_1 25 | - tzdata=2021a=he74cb21_0 26 | - wheel=0.36.2=pyhd3deb0d_0 27 | - xz=5.2.5=h516909a_1 28 | - zlib=1.2.11=h516909a_1010 29 | - pip: 30 | - absl-py==0.13.0 31 | - aiohttp==3.7.4.post0 32 | - antlr4-python3-runtime==4.8 33 | - async-timeout==3.0.1 34 | - attrs==21.2.0 35 | - backcall==0.2.0 36 | - cachetools==4.2.2 37 | - chardet==4.0.0 38 | - decorator==5.0.9 39 | - einops==0.3.0 40 | - fsspec==2021.6.1 41 | - ftfy==6.0.3 42 | - future==0.18.2 43 | - google-auth==1.32.0 44 | - google-auth-oauthlib==0.4.4 45 | - grpcio==1.38.1 46 | - idna==2.10 47 | - imageio==2.9.0 48 | - imageio-ffmpeg==0.4.4 49 | - ipython==7.25.0 50 | - ipython-genutils==0.2.0 51 | - jedi==0.18.0 52 | - kornia==0.5.4 53 | - markdown==3.3.4 54 | - matplotlib-inline==0.1.2 55 | - multidict==5.1.0 56 | - numpy==1.21.0 57 | - oauthlib==3.1.1 58 | - omegaconf==2.1.0 59 | - packaging==20.9 60 | - parso==0.8.2 61 | - pexpect==4.8.0 62 | - pickleshare==0.7.5 63 | - pillow==8.2.0 64 | - prompt-toolkit==3.0.19 65 | - protobuf==3.17.3 66 | - ptyprocess==0.7.0 67 | - pyasn1==0.4.8 68 | - pyasn1-modules==0.2.8 69 | - pydeprecate==0.3.0 70 | - pygments==2.9.0 71 | - pyparsing==2.4.7 72 | - pytorch-lightning==1.3.7.post0 73 | - pyyaml==5.4.1 74 | - regex==2021.4.4 75 | - requests==2.25.1 76 | - requests-oauthlib==1.3.0 77 | - rsa==4.7.2 78 | - six==1.16.0 79 | - tensorboard==2.4.1 80 | - tensorboard-plugin-wit==1.8.0 81 | - torch==1.9.0+cu111 82 | - torchaudio==0.9.0 83 | - torchmetrics==0.3.2 84 | - torchvision==0.10.0+cu111 85 | - tqdm==4.61.1 86 | - traitlets==5.0.5 87 | - typing-extensions==3.10.0.0 88 | - urllib3==1.26.6 89 | - wcwidth==0.2.5 90 | - werkzeug==2.0.1 91 | - yarl==1.6.3 92 | prefix: /home/nerdy/anaconda3/envs/vqgan 93 | -------------------------------------------------------------------------------- /app/ts/elements/generation-request-form/image-spec-option.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CustomElement, 3 | Dispatch, 4 | DispatchEmitter, 5 | Prop, 6 | Watch 7 | } from 'custom-elements-ts'; 8 | import {BaseElement} from "../base-element"; 9 | 10 | @CustomElement({ 11 | tag: 'vc-image-spec-option', 12 | shadow: false, 13 | style: ``, 14 | template: ` 15 | <div class="image-spec-option"></div> 16 | `, 17 | }) 18 | export class ImageSpecOption extends BaseElement { 19 | $root: HTMLElement; 20 | 21 | @Prop() label: string; 22 | @Prop() type: string; 23 | @Prop() value: string; 24 | @Dispatch('change.value') change: DispatchEmitter; 25 | 26 | constructor() { 27 | super(); 28 | } 29 | 30 | connectedCallback() { 31 | this.$root = this.querySelector('.image-spec-option'); 32 | } 33 | 34 | @Watch('value') 35 | protected update() { 36 | this.$root.innerHTML = ''; 37 | const label = this.el('label', {innerText: this.label}); 38 | let input; 39 | switch (this.type) { 40 | case 'int': 41 | input = this.el('input', { 42 | type: 'number', 43 | value: this.value, 44 | attr: {step: '1'} 45 | }) 46 | break; 47 | case 'float': 48 | input = this.el('input', { 49 | type: 'number', 50 | value: this.value, 51 | attr: {step: '0.1'} 52 | }) 53 | break; 54 | case 'boolean': 55 | input = this.el('input', { 56 | type: 'checkbox', 57 | checked: this.value === 'true' 58 | }); 59 | break; 60 | default: 61 | throw new Error('Invalid type for image spec option: ' + this.type); 62 | } 63 | input.addEventListener('change', this.onChange.bind(this)); 64 | label.appendChild(input); 65 | this.$root.appendChild(label); 66 | } 67 | 68 | protected onChange(e: any) { 69 | e.preventDefault(); 70 | let value; 71 | switch (this.type) { 72 | case 'int': 73 | value = parseInt(e.currentTarget.value, 10); 74 | break; 75 | case 'float': 76 | value = parseFloat(e.currentTarget.value); 77 | break; 78 | case 'boolean': 79 | value = e.currentTarget.checked; 80 | break; 81 | default: 82 | throw new Error('Invalid type for image spec option: ' + this.type); 83 | } 84 | this.change.emit({detail: value}); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /vc/service/helper/rife/model/RIFE_HDv3.py: -------------------------------------------------------------------------------- 1 | from torch.nn.parallel import DistributedDataParallel as DDP 2 | from torch.optim import AdamW 3 | 4 | from .IFNet_HDv3 import * 5 | from .loss import * 6 | 7 | 8 | class Model: 9 | def __init__(self, local_rank=-1): 10 | self.flownet = IFNet() 11 | self.device() 12 | self.optimG = AdamW(self.flownet.parameters(), lr=1e-6, weight_decay=1e-4) 13 | self.epe = EPE() 14 | self.sobel = SOBEL() 15 | if local_rank != -1: 16 | self.flownet = DDP(self.flownet, device_ids=[local_rank], output_device=local_rank) 17 | 18 | def train(self): 19 | self.flownet.train() 20 | 21 | def eval(self): 22 | self.flownet.eval() 23 | 24 | def device(self): 25 | self.flownet.to(torch.device( 26 | 'cuda' if torch.cuda.is_available() else 'cpu' 27 | )) 28 | 29 | def load_model(self, path, rank=0): 30 | def convert(param): 31 | if rank == -1: 32 | return { 33 | k.replace("module.", ""): v 34 | for k, v in param.items() 35 | if "module." in k 36 | } 37 | else: 38 | return param 39 | if rank <= 0: 40 | if torch.cuda.is_available(): 41 | self.flownet.load_state_dict(convert(torch.load('{}/flownet.pkl'.format(path)))) 42 | else: 43 | self.flownet.load_state_dict(convert(torch.load('{}/flownet.pkl'.format(path), map_location ='cpu'))) 44 | 45 | def save_model(self, path, rank=0): 46 | if rank == 0: 47 | torch.save(self.flownet.state_dict(),'{}/flownet.pkl'.format(path)) 48 | 49 | def inference(self, img0, img1, scale=1.0): 50 | imgs = torch.cat((img0, img1), 1) 51 | scale_list = [4/scale, 2/scale, 1/scale] 52 | flow, mask, merged = self.flownet(imgs, scale_list) 53 | return merged[2] 54 | 55 | def update(self, imgs, gt, learning_rate=0, mul=1, training=True, flow_gt=None): 56 | for param_group in self.optimG.param_groups: 57 | param_group['lr'] = learning_rate 58 | 59 | if training: 60 | self.train() 61 | else: 62 | self.eval() 63 | scale = [4, 2, 1] 64 | flow, mask, merged = self.flownet(torch.cat((imgs, gt), 1), scale=scale, training=training) 65 | loss_l1 = (merged[2] - gt).abs().mean() 66 | loss_smooth = self.sobel(flow[2], flow[2]*0).mean() 67 | 68 | return merged[2], { 69 | 'mask': mask, 70 | 'flow': flow[2][:, :2], 71 | 'loss_l1': loss_l1, 72 | 'loss_cons': 0, 73 | 'loss_smooth': loss_smooth, 74 | } 75 | -------------------------------------------------------------------------------- /vc/model/generation_request.py: -------------------------------------------------------------------------------- 1 | from flask_restful import fields 2 | 3 | from vc.db import db 4 | from vc.model.base import BaseModel 5 | from vc.model.generation_result import GenerationResult 6 | from vc.value_object.generation_spec import GenerationSpec 7 | 8 | 9 | class GenerationRequest(db.Model, BaseModel): 10 | FIELDS = [ 11 | 'spec', 12 | ] 13 | GOD_FIELDS = [ 14 | 'steps_completed', 15 | ] 16 | 17 | spec = db.Column(db.JSON, nullable=False) 18 | 19 | user_id = db.Column(db.Integer, db.ForeignKey('user.id'), nullable=False) 20 | 21 | started = db.Column(db.DateTime) 22 | completed = db.Column(db.DateTime) 23 | failed = db.Column(db.DateTime) 24 | cancelled = db.Column(db.DateTime) 25 | retried = db.Column(db.DateTime) 26 | published = db.Column(db.DateTime) 27 | 28 | steps_completed = db.Column(db.Integer) 29 | steps_total = db.Column(db.Integer) 30 | 31 | name = db.Column(db.String) 32 | hash = db.Column(db.String) 33 | 34 | preview = db.Column(db.String) 35 | interim = db.Column(db.String) 36 | interim_watermarked = db.Column(db.String) 37 | 38 | results = db.relationship('GenerationResult', backref='request') 39 | 40 | private_schema = { 41 | 'id': fields.Integer, 42 | 'spec': fields.Nested(GenerationSpec.schema), 43 | 'created': fields.DateTime(), 44 | 'updated': fields.DateTime(), 45 | 'deleted': fields.DateTime(), 46 | 'started': fields.DateTime(), 47 | 'completed': fields.DateTime(), 48 | 'failed': fields.DateTime(), 49 | 'cancelled': fields.DateTime(), 50 | 'retried': fields.DateTime(), 51 | 'published': fields.DateTime(), 52 | 'steps_completed': fields.Integer, 53 | 'steps_total': fields.Integer, 54 | 'name': fields.String, 55 | 'preview': fields.String, 56 | 'interim': fields.String, 57 | 'interim_watermarked': fields.String, 58 | 'results': fields.List(fields.Nested(GenerationResult.public_schema)), 59 | } 60 | 61 | public_schema = { 62 | 'id': fields.Integer, 63 | 'created': fields.DateTime(), 64 | 'updated': fields.DateTime(), 65 | 'deleted': fields.DateTime(), 66 | 'started': fields.DateTime(), 67 | 'completed': fields.DateTime(), 68 | 'failed': fields.DateTime(), 69 | 'cancelled': fields.DateTime(), 70 | 'retried': fields.DateTime(), 71 | 'steps_completed': fields.Integer, 72 | 'steps_total': fields.Integer, 73 | 'name': fields.String, 74 | 'preview': fields.String, 75 | 'interim_watermarked': fields.String, 76 | 'results': fields.List(fields.Nested(GenerationResult.private_schema)), 77 | } 78 | -------------------------------------------------------------------------------- /app/ts/elements/generation-request-form/video-spec-form.ts: -------------------------------------------------------------------------------- 1 | import { 2 | CustomElement, 3 | Dispatch, 4 | DispatchEmitter, 5 | Listen 6 | } from 'custom-elements-ts'; 7 | import {VideoSpec} from "../../models/video-spec"; 8 | import {ImageSpec} from "../../models/image-spec"; 9 | import {ImageSpecForm} from "./image-spec-form"; 10 | import {BaseElement} from "../base-element"; 11 | import {AddVideoStep} from "./add-video-step"; 12 | 13 | @CustomElement({ 14 | tag: 'vc-video-spec-form', 15 | shadow: false, 16 | style: ``, 17 | template: ` 18 | <div class="video-spec-form"> 19 | <div class="spec-header"> 20 | <h3>Video</h3> 21 | <button class="material-icons"> 22 | highlight_off 23 | </button> 24 | </div> 25 | <div class="steps"> 26 | </div> 27 | `, 28 | }) 29 | export class VideoSpecForm extends BaseElement { 30 | $root: HTMLElement; 31 | $steps: HTMLElement; 32 | 33 | spec: VideoSpec; 34 | expanded = false; 35 | 36 | @Dispatch('spec.remove') onRemove: DispatchEmitter; 37 | 38 | constructor() { 39 | super(); 40 | } 41 | 42 | connectedCallback() { 43 | this.$root = this.querySelector('.video-spec-form'); 44 | this.$steps = this.$root.querySelector('.steps'); 45 | } 46 | 47 | public update(spec: VideoSpec) { 48 | this.spec = spec; 49 | this.draw(); 50 | } 51 | 52 | protected draw() { 53 | if (!this.spec) { 54 | return; 55 | } 56 | this.$steps.innerHTML = ''; 57 | 58 | for (const spec of this.spec.steps || []) { 59 | this.addImageSpecForm(spec); 60 | } 61 | 62 | this.addAddSpec(); 63 | } 64 | 65 | @Listen('click', '.spec-header button') 66 | protected onRemoveClicked(e: any) { 67 | this.onRemove.emit(); 68 | } 69 | 70 | protected addImageSpecForm(spec: ImageSpec) { 71 | const form = this.el('vc-image-spec-form', { 72 | attr: {video: true}, 73 | }) as ImageSpecForm; 74 | form.addEventListener('spec.remove', () => { 75 | this.spec.steps.splice(this.spec.steps.indexOf(spec), 1); 76 | this.draw(); 77 | }); 78 | this.$steps.appendChild(form); 79 | form.update(spec); 80 | } 81 | 82 | protected addAddSpec() { 83 | const addSpec = this.el('vc-add-video-step') as AddVideoStep; 84 | this.$steps.appendChild(addSpec); 85 | addSpec.update(this.spec); 86 | addSpec.addEventListener('added.image', (event) => { 87 | this.spec.steps = this.spec.steps || []; 88 | this.spec.steps.push(new ImageSpec()); 89 | addSpec.update(this.spec); 90 | this.draw(); 91 | }); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /vc/service/helper/esrgan/archs/discriminator_arch.py: -------------------------------------------------------------------------------- 1 | from basicsr.utils.registry import ARCH_REGISTRY 2 | from torch import nn as nn 3 | from torch.nn import functional as F 4 | from torch.nn.utils import spectral_norm 5 | 6 | 7 | @ARCH_REGISTRY.register() 8 | class UNetDiscriminatorSN(nn.Module): 9 | """Defines a U-Net discriminator with spectral normalization (SN)""" 10 | 11 | def __init__(self, num_in_ch, num_feat=64, skip_connection=True): 12 | super(UNetDiscriminatorSN, self).__init__() 13 | self.skip_connection = skip_connection 14 | norm = spectral_norm 15 | 16 | self.conv0 = nn.Conv2d(num_in_ch, num_feat, kernel_size=3, stride=1, padding=1) 17 | 18 | self.conv1 = norm(nn.Conv2d(num_feat, num_feat * 2, 4, 2, 1, bias=False)) 19 | self.conv2 = norm(nn.Conv2d(num_feat * 2, num_feat * 4, 4, 2, 1, bias=False)) 20 | self.conv3 = norm(nn.Conv2d(num_feat * 4, num_feat * 8, 4, 2, 1, bias=False)) 21 | # upsample 22 | self.conv4 = norm(nn.Conv2d(num_feat * 8, num_feat * 4, 3, 1, 1, bias=False)) 23 | self.conv5 = norm(nn.Conv2d(num_feat * 4, num_feat * 2, 3, 1, 1, bias=False)) 24 | self.conv6 = norm(nn.Conv2d(num_feat * 2, num_feat, 3, 1, 1, bias=False)) 25 | 26 | # extra 27 | self.conv7 = norm(nn.Conv2d(num_feat, num_feat, 3, 1, 1, bias=False)) 28 | self.conv8 = norm(nn.Conv2d(num_feat, num_feat, 3, 1, 1, bias=False)) 29 | 30 | self.conv9 = nn.Conv2d(num_feat, 1, 3, 1, 1) 31 | 32 | def forward(self, x): 33 | x0 = F.leaky_relu(self.conv0(x), negative_slope=0.2, inplace=True) 34 | x1 = F.leaky_relu(self.conv1(x0), negative_slope=0.2, inplace=True) 35 | x2 = F.leaky_relu(self.conv2(x1), negative_slope=0.2, inplace=True) 36 | x3 = F.leaky_relu(self.conv3(x2), negative_slope=0.2, inplace=True) 37 | 38 | # upsample 39 | x3 = F.interpolate(x3, scale_factor=2, mode='bilinear', align_corners=False) 40 | x4 = F.leaky_relu(self.conv4(x3), negative_slope=0.2, inplace=True) 41 | 42 | if self.skip_connection: 43 | x4 = x4 + x2 44 | x4 = F.interpolate(x4, scale_factor=2, mode='bilinear', align_corners=False) 45 | x5 = F.leaky_relu(self.conv5(x4), negative_slope=0.2, inplace=True) 46 | 47 | if self.skip_connection: 48 | x5 = x5 + x1 49 | x5 = F.interpolate(x5, scale_factor=2, mode='bilinear', align_corners=False) 50 | x6 = F.leaky_relu(self.conv6(x5), negative_slope=0.2, inplace=True) 51 | 52 | if self.skip_connection: 53 | x6 = x6 + x0 54 | 55 | # extra 56 | out = F.leaky_relu(self.conv7(x6), negative_slope=0.2, inplace=True) 57 | out = F.leaky_relu(self.conv8(out), negative_slope=0.2, inplace=True) 58 | out = self.conv9(out) 59 | 60 | return out 61 | -------------------------------------------------------------------------------- /app/ts/elements/news/nav.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement, Prop} from 'custom-elements-ts'; 2 | 3 | interface Article { 4 | url: string; 5 | title: string; 6 | hero ?: string; 7 | author ?: string; 8 | published ?: string; 9 | } 10 | 11 | @CustomElement({ 12 | tag: 'vc-nav', 13 | shadow: false, 14 | style: ``, 15 | template: ` 16 | <div class="nav-container"> 17 | <div class="nav"></div> 18 | </div> 19 | ` 20 | }) 21 | export class Nav extends HTMLElement { 22 | $root: HTMLElement; 23 | $nav: HTMLElement; 24 | 25 | @Prop() active: string; 26 | 27 | articles: Article[] = [ 28 | { 29 | url: 'foss', 30 | title: 'vc is open source now!', 31 | hero: 'foss', 32 | author: 'AJ Moon', 33 | published: '16 November, 2021', 34 | }, 35 | { 36 | url: '', 37 | title: 'Welcome to vc', 38 | hero: 'index', 39 | }, 40 | ]; 41 | 42 | constructor() { 43 | super(); 44 | } 45 | 46 | connectedCallback() { 47 | this.$root = this.querySelector('.nav-container'); 48 | this.$nav = this.$root.querySelector('.nav'); 49 | this.articles.forEach(this.addLink.bind(this)); 50 | } 51 | 52 | addLink(article: Article) { 53 | const a = document.createElement('a'); 54 | a.href = '/news/' + article.url; 55 | if (this.active === article.url) { 56 | a.classList.add('active'); 57 | } 58 | 59 | const hero = document.createElement('div'); 60 | hero.classList.add('hero'); 61 | const img = document.createElement('img'); 62 | img.src = '/assets/news/' + (article.hero || '../placeholder') + '.png'; 63 | hero.appendChild(img); 64 | a.appendChild(hero); 65 | 66 | const content = document.createElement('div'); 67 | content.classList.add('content'); 68 | 69 | const title = document.createElement('h3'); 70 | title.innerText = article.title; 71 | content.appendChild(title); 72 | 73 | const byline = document.createElement('div'); 74 | byline.classList.add('byline'); 75 | 76 | if (article.author) { 77 | const author = document.createElement('div'); 78 | author.classList.add('author'); 79 | author.innerText = article.author; 80 | byline.appendChild(author); 81 | } 82 | if (article.published) { 83 | const published = document.createElement('div'); 84 | published.classList.add('published'); 85 | published.innerText = article.published; 86 | byline.appendChild(published); 87 | } 88 | 89 | content.appendChild(byline); 90 | a.appendChild(content); 91 | this.$nav.appendChild(a); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /app/scss/generation_request/_details.scss: -------------------------------------------------------------------------------- 1 | $image-width: 400px; 2 | $video-width: 800px; 3 | 4 | vc-generation-request-details { 5 | .details { 6 | border-top: solid $border-width #63575b; 7 | background-color: #d7e6d6; 8 | border-radius: 0 0 $border-radius $border-radius; 9 | 10 | flex-direction: column; 11 | justify-content: space-between; 12 | overflow: hidden; 13 | display: none; 14 | 15 | &.expanded { 16 | display: flex; 17 | } 18 | 19 | .steps { 20 | flex-grow: 1; 21 | @import "step"; 22 | } 23 | 24 | .actions { 25 | background-color: #f0f5f0; 26 | border-bottom: solid $border-width #63575b; 27 | padding: 16px; 28 | display: flex; 29 | align-items: center; 30 | justify-content: flex-end; 31 | flex-flow: row wrap; 32 | 33 | button { 34 | width: auto; 35 | padding: 0 16px; 36 | font-size: x-large; 37 | margin: 0 8px 8px 0; 38 | 39 | @include mobile { 40 | flex-grow: 1; 41 | } 42 | } 43 | } 44 | 45 | .preview { 46 | display: flex; 47 | flex-flow: row wrap; 48 | justify-content: center; 49 | padding: 0 16px 16px; 50 | background-color: #f5ede6; 51 | border-bottom: solid $border-width #63575b; 52 | 53 | .panel { 54 | margin: 16px 16px 0 0; 55 | border-radius: $border-radius; 56 | border: solid $border-width #535c69; 57 | background-color: #535c69; 58 | 59 | @include mobile { 60 | width: 100%; 61 | height: 100%; 62 | } 63 | 64 | vc-chipset { 65 | display: block; 66 | width: $image-width; 67 | vc-chip { 68 | text-overflow: ellipsis; 69 | } 70 | } 71 | 72 | img, video { 73 | height: auto; 74 | border-radius: $border-radius; 75 | @include mobile { 76 | width: 100%; 77 | } 78 | } 79 | 80 | img { 81 | width: $image-width; 82 | } 83 | 84 | video { 85 | width: $video-width; 86 | } 87 | 88 | img, video { 89 | height: auto; 90 | border-radius: $border-radius; 91 | @include mobile { 92 | width: 100%; 93 | } 94 | } 95 | } 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /app/ts/elements/generation-request/step.ts: -------------------------------------------------------------------------------- 1 | import {CustomElement, Toggle} from 'custom-elements-ts'; 2 | import {Chipset} from "../chipset"; 3 | import {ImageSpec} from "../../models/image-spec"; 4 | import {DetailsHelper} from "../../helpers/details"; 5 | import {BaseElement} from "../base-element"; 6 | 7 | @CustomElement({ 8 | tag: 'vc-generation-request-details-step', 9 | shadow: false, 10 | style: ``, 11 | template: ` 12 | <div class="step"> 13 | <div class="number"><span></span></div> 14 | <div class="spec"> 15 | <div class="text-chipset"> 16 | <h3>Texts</h3> 17 | <vc-chipset class="texts"></vc-chipset> 18 | </div> 19 | <div class="text-chipset"> 20 | <h3>Styles</h3> 21 | <vc-chipset class="styles"></vc-chipset> 22 | </div> 23 | <div class="fields"></div> 24 | </div> 25 | </div> 26 | ` 27 | }) 28 | export class GenerationRequestDetailsStep extends BaseElement { 29 | $root: HTMLElement 30 | $number: HTMLElement 31 | $spec: HTMLElement 32 | $texts: Chipset; 33 | $styles: Chipset; 34 | $fields: HTMLElement; 35 | 36 | number: number; 37 | spec: ImageSpec 38 | @Toggle() expanded = false 39 | @Toggle() video = false; 40 | 41 | constructor() { 42 | super(); 43 | } 44 | 45 | connectedCallback() { 46 | this.$root = this.querySelector('.step'); 47 | this.$number = this.$root.querySelector('.number span'); 48 | this.$spec = this.$root.querySelector('.spec'); 49 | this.$texts = this.$spec.querySelector('.texts'); 50 | this.$styles = this.$spec.querySelector('.styles'); 51 | this.$fields = this.$spec.querySelector('.fields'); 52 | } 53 | 54 | update(number: number, spec: ImageSpec) { 55 | this.spec = spec; 56 | this.number = number; 57 | this.draw(); 58 | } 59 | 60 | draw() { 61 | this.$number.innerHTML = this.number + '.'; 62 | this.$texts.update(this.spec.texts); 63 | this.$styles.update(this.spec.styles); 64 | this.drawFields(); 65 | } 66 | 67 | drawFields() { 68 | for (const field of DetailsHelper.getFields(this.video)) { 69 | if (field.field in this.spec) { 70 | const fieldValue = (this.spec as any)[field.field]; 71 | if (fieldValue) { 72 | const div = this.el('div', {class: 'field'}); 73 | const label = this.el('label', { 74 | innerText: field.label, 75 | }); 76 | const span = this.el('span', { 77 | innerText: '' + fieldValue, 78 | }); 79 | div.appendChild(label); 80 | div.appendChild(span); 81 | this.$fields.appendChild(div); 82 | } 83 | } 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /vc/manager/base.py: -------------------------------------------------------------------------------- 1 | from typing import Type 2 | 3 | from injector import inject 4 | 5 | from vc.db import db 6 | from vc.event import VcEventDispatcher 7 | from vc.exception import NotFoundException 8 | from vc.model.user import User 9 | from vc.service.helper.tier import TierHelper 10 | 11 | 12 | class Manager: 13 | dispatcher: VcEventDispatcher 14 | model_class: Type[db.Model] 15 | 16 | @inject 17 | def __init__(self, dispatcher: VcEventDispatcher): 18 | self.dispatcher = dispatcher 19 | 20 | def all_query(self): 21 | return self.model_class.query.filter( 22 | self.model_class.deleted.__eq__(None) 23 | ).order_by( 24 | self.model_class.created.desc() 25 | ) 26 | 27 | def all(self): 28 | try: 29 | return self.all_query().all() 30 | except Exception as e: 31 | db.session.rollback() 32 | raise e 33 | 34 | def find_or_throw(self, id_, user: User = None): 35 | try: 36 | model = self.model_class.query.get(id_) 37 | if model is None: 38 | raise NotFoundException(self.model_class.__name__, id_) 39 | return model 40 | except Exception as e: 41 | db.session.rollback() 42 | raise e 43 | 44 | def create(self, raw, user: User = None): 45 | try: 46 | # @todo ModelFactory.create here 47 | 48 | model = self.model_class(**self.fields(raw, user)) 49 | self.save(model) 50 | 51 | # @todo ModelEventDispatcher.dispatchCreated here 52 | 53 | return model 54 | except Exception as e: 55 | db.session.rollback() 56 | raise e 57 | 58 | def update(self, id_, raw, user: User): 59 | try: 60 | model = self.find_or_throw(id_, user) 61 | model.__init__(**self.fields(raw, user)) 62 | self.save(model) 63 | 64 | # @todo ModelEventDispatcher.dispatchUpdated here 65 | 66 | return model 67 | except Exception as e: 68 | db.session.rollback() 69 | raise e 70 | 71 | def delete(self, id_, user: User): 72 | try: 73 | model = self.find_or_throw(id_, user) 74 | db.session.delete(model) 75 | self.commit() 76 | except Exception as e: 77 | db.session.rollback() 78 | raise e 79 | 80 | def save(self, model): 81 | try: 82 | db.session.add(model) 83 | self.commit() 84 | except Exception as e: 85 | db.session.rollback() 86 | raise e 87 | 88 | def commit(self): 89 | db.session.commit() 90 | 91 | def fields(self, raw, user: User = None): 92 | fields = self.model_class.FIELDS 93 | if user is None or TierHelper.is_god(user): 94 | fields += self.model_class.GOD_FIELDS 95 | return {k: raw[k] for k in raw.keys() & fields} 96 | -------------------------------------------------------------------------------- /vc/validator/generation_request.py: -------------------------------------------------------------------------------- 1 | from dacite import from_dict 2 | from injector import inject 3 | 4 | from vc.exception.tier_exception import TierException 5 | from vc.manager import GenerationRequestManager 6 | from vc.model.user import User 7 | from vc.service.helper.steps import ( 8 | Steps, 9 | ImageGenerationStep, 10 | VideoGenerationStep, 11 | ) 12 | from vc.service.helper.tier import TierHelper 13 | from vc.value_object import GenerationSpec 14 | 15 | 16 | class GenerationRequestValidator: 17 | manager: GenerationRequestManager 18 | 19 | @inject 20 | def __init__(self, manager: GenerationRequestManager): 21 | self.manager = manager 22 | 23 | def create(self, raw, user: User): 24 | spec = from_dict( 25 | data_class=GenerationSpec, 26 | data=raw['spec'] 27 | ) 28 | 29 | steps = Steps.total_steps(spec) 30 | 31 | if not TierHelper.is_coder(user): 32 | raise TierException('Coder') 33 | 34 | if not TierHelper.is_artist(user) and steps > 800: 35 | raise TierException( 36 | 'Artist', 37 | 'That spec would have %s steps - your tier is limited ' 38 | 'to 800 steps max' % steps 39 | ) 40 | 41 | if not TierHelper.is_god(user) and steps > 1600: 42 | raise TierException( 43 | 'God', 44 | 'That spec would have %s steps - your tier is limited ' 45 | 'to 1600 steps max' % steps 46 | ) 47 | 48 | self.queued(user) 49 | self.spec(spec, user) 50 | 51 | def queued(self, user: User): 52 | if not TierHelper.is_god(user): 53 | queued = self.manager.mine_queued(user) 54 | if len(queued) > 0: 55 | raise TierException( 56 | 'God', 57 | 'Your tier is limited to one request at a time - ' 58 | 'try cancelling an existing request first' 59 | ) 60 | 61 | def spec(self, spec: GenerationSpec, user: User): 62 | if not TierHelper.is_artist(user): 63 | for step in Steps.iterate_steps(spec): 64 | if isinstance(step, ImageGenerationStep): 65 | if step.spec.upscale: 66 | raise TierException( 67 | 'Artist', 68 | 'Your tier is restricted from upscaling' 69 | ) 70 | if isinstance(step, VideoGenerationStep): 71 | if step.upscaled: 72 | raise TierException( 73 | 'Artist', 74 | 'Your tier is restricted from upscaling' 75 | ) 76 | if step.interpolated: 77 | raise TierException( 78 | 'Artist', 79 | 'Your tier is restricted from interpolation' 80 | ) 81 | -------------------------------------------------------------------------------- /vc/service/helper/midas/midas/midas_net.py: -------------------------------------------------------------------------------- 1 | """MidashNet: Network for monocular depth estimation trained by mixing several datasets. 2 | This file contains code that is adapted from 3 | https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py 4 | """ 5 | import torch 6 | import torch.nn as nn 7 | 8 | from .base_model import BaseModel 9 | from .blocks import FeatureFusionBlock, Interpolate, _make_encoder 10 | 11 | 12 | class MidasNet(BaseModel): 13 | """Network for monocular depth estimation. 14 | """ 15 | 16 | def __init__(self, path=None, features=256, non_negative=True): 17 | """Init. 18 | 19 | Args: 20 | path (str, optional): Path to saved model. Defaults to None. 21 | features (int, optional): Number of features. Defaults to 256. 22 | backbone (str, optional): Backbone network for encoder. Defaults to resnet50 23 | """ 24 | print("Loading weights: ", path) 25 | 26 | super(MidasNet, self).__init__() 27 | 28 | use_pretrained = False if path is None else True 29 | 30 | self.pretrained, self.scratch = _make_encoder(backbone="resnext101_wsl", features=features, use_pretrained=use_pretrained) 31 | 32 | self.scratch.refinenet4 = FeatureFusionBlock(features) 33 | self.scratch.refinenet3 = FeatureFusionBlock(features) 34 | self.scratch.refinenet2 = FeatureFusionBlock(features) 35 | self.scratch.refinenet1 = FeatureFusionBlock(features) 36 | 37 | self.scratch.output_conv = nn.Sequential( 38 | nn.Conv2d(features, 128, kernel_size=3, stride=1, padding=1), 39 | Interpolate(scale_factor=2, mode="bilinear"), 40 | nn.Conv2d(128, 32, kernel_size=3, stride=1, padding=1), 41 | nn.ReLU(True), 42 | nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), 43 | nn.ReLU(True) if non_negative else nn.Identity(), 44 | ) 45 | 46 | if path: 47 | self.load(path) 48 | 49 | def forward(self, x): 50 | """Forward pass. 51 | 52 | Args: 53 | x (tensor): input data (image) 54 | 55 | Returns: 56 | tensor: depth 57 | """ 58 | 59 | layer_1 = self.pretrained.layer1(x) 60 | layer_2 = self.pretrained.layer2(layer_1) 61 | layer_3 = self.pretrained.layer3(layer_2) 62 | layer_4 = self.pretrained.layer4(layer_3) 63 | 64 | layer_1_rn = self.scratch.layer1_rn(layer_1) 65 | layer_2_rn = self.scratch.layer2_rn(layer_2) 66 | layer_3_rn = self.scratch.layer3_rn(layer_3) 67 | layer_4_rn = self.scratch.layer4_rn(layer_4) 68 | 69 | path_4 = self.scratch.refinenet4(layer_4_rn) 70 | path_3 = self.scratch.refinenet3(path_4, layer_3_rn) 71 | path_2 = self.scratch.refinenet2(path_3, layer_2_rn) 72 | path_1 = self.scratch.refinenet1(path_2, layer_1_rn) 73 | 74 | out = self.scratch.output_conv(path_1) 75 | 76 | return torch.squeeze(out, dim=1) 77 | --------------------------------------------------------------------------------