├── .dockerignore ├── .gitignore ├── .npmignore ├── .prettierrc ├── Dockerfile ├── LICENSE ├── README.md ├── docker-compose.yaml ├── novia.example.yaml ├── package-lock.json ├── package.json ├── src ├── config.ts ├── dvm │ ├── archive.ts │ ├── index.ts │ ├── publish.ts │ ├── recover.ts │ ├── types.ts │ └── upload.ts ├── entity │ ├── Queue.ts │ └── Video.ts ├── helpers │ ├── blossom.ts │ └── dvm.ts ├── index.ts ├── init-setup.ts ├── jobs │ ├── cleanDeletedVideos.ts │ ├── processExtendMetaData.ts │ ├── processMirrorJob.ts │ ├── processNostrUpload.ts │ ├── processShaHashes.ts │ ├── processVideoDownloadJob.ts │ ├── queue.ts │ └── results.ts ├── mikro-orm.config.ts ├── server.ts ├── types.ts ├── utils │ ├── array.ts │ ├── ffmpeg.ts │ ├── mapvideodata.ts │ ├── move.ts │ ├── utils.ts │ └── ytdlp.ts ├── validation │ ├── validateHashes.ts │ └── validateMetaData.ts └── video-indexer.ts └── tsconfig.json /.dockerignore: -------------------------------------------------------------------------------- 1 | .env 2 | build 3 | dist 4 | node_modules 5 | .vscode 6 | temp* 7 | novia.yaml 8 | novia.db 9 | media/ 10 | data/ 11 | TODO.md 12 | fetchtest/ 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | .vscode/ 3 | node_modules/ 4 | build/ 5 | tmp/ 6 | temp/ 7 | novia.db 8 | novia.yaml 9 | dist/ 10 | temp* 11 | media/ 12 | data/ -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | bun.lockb 2 | .prettierrc 3 | src/ 4 | tsconfig.json 5 | novia.* 6 | temp/ 7 | media/ 8 | Dockerfile 9 | .dockerignore 10 | docker-compose.yaml 11 | data/ 12 | TODO.md 13 | fetchtest/ -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 120, 3 | "useTabs": false, 4 | "tabWidth": 2 5 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:22 2 | 3 | RUN apt-get update -y && apt-get install -y wget ffmpeg coreutils 4 | 5 | RUN wget https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -O /usr/local/bin/yt-dlp 6 | RUN chmod a+rx /usr/local/bin/yt-dlp 7 | 8 | WORKDIR /app 9 | COPY . /app/ 10 | RUN npm install 11 | RUN npm run build 12 | 13 | # example media folder /app/media 14 | RUN mkdir media 15 | 16 | # ENV DEBUG=novia* 17 | 18 | ENTRYPOINT [ "node", "dist/index.js", "serve" ] 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Team Novia 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # novia - NOstr VIdeo Archive 2 | 3 | novia is the glue that connects video archive tools to NOSTR. It can be used 4 | as a **standalone archive tool** or extend other open source tools like: 5 | 6 | - tubearchivist 7 | - pinchflat 8 | 9 | novia is running as a service and is able to 10 | 11 | - download videos (with `yt-dlp`) by REST API or requests posted via NOSTR. 12 | - scan existing videos on disk downloaded by other tools 13 | - manage a library of the video metadata 14 | - publish video events to NOSTR relays and 15 | - answer requests for videos and upload them to blossom server. 16 | 17 | # Archive structure 18 | 19 | novia follows a few simple rules: 20 | 21 | - filesystem first - all video, image and metadata is stored in a folder structure. 22 | - Archive contents can be therefore be synched, copied and backuped in a conventional way. 23 | - The default folder structure is: 24 | ```sh 25 | ///videoId.mp4 (the video) 26 | ///videoId.webp (a thumbnail) 27 | ///videoId.info.json (ytdlp metadata) 28 | ``` 29 | A concrete example would be: 30 | ```sh 31 | youtube/UClw9f0QDkIw2jrQ2_5QSMGw/dnDC3uWjhlo/dnDC3uWjhlo.mp4 32 | youtube/UClw9f0QDkIw2jrQ2_5QSMGw/dnDC3uWjhlo/dnDC3uWjhlo.webp 33 | youtube/UClw9f0QDkIw2jrQ2_5QSMGw/dnDC3uWjhlo/dnDC3uWjhlo.info.json 34 | ``` 35 | - Other folder structures are supported as well, as long all the content files have a unique name 36 | - novia uses an SQLlite database as an index but the database could be restored from metadata at any time. 37 | 38 | # Components 39 | 40 | - **Filesystem scan** scans defined folders for video and metadata. 41 | - **Filesystem watcher** watches for changes in the folder and processes files immediately. 42 | - **Metadata extension** if a video doesn't have a thumbail or metadata it is fetched and stored in the filesystem alongside the video. 43 | - **Hashing** For every file a unique hash (SHA256) is created. 44 | - **Download** new videos based on a link and add them to the archive. 45 | - **Publish video events** of archived videos on NOSTR. 46 | - **Offer download services** to other users via NOSTR DVMs. 47 | - **Offer video upload services** to users requesting a specific video. 48 | 49 | # Services 50 | 51 | The novia service has several roles, that can be enabled/disabled for different setups: 52 | - **archive** Receive a URL and archive the video locally 53 | - **publish** If enabled, archived videos are published (publicly) to a relay. By default this is only meta data and a thumbnail. If you enable the setting `autopublish` videos up to a certain size will also be uploaded automatically. 54 | - **recover** A user might request a video to be recovered (i.e. uploaded). If `recover` is enabled novia will upload the requested video to a blossom server. 55 | - **fetch** Instructs the novia instance to download or mirror videos that other novia instances publish or upload. 56 | 57 | # Setting it up 58 | 59 | ## Running in Docker 60 | 61 | To use docker to run novia you have to mount the media folders as well as a folder with the config and database into the container. The easiest setup is as follows: 62 | 63 | - Create a `./data` folder 64 | - Create a folder `./data/media` where the video content will go. 65 | - Create a config file `./data/novia.yaml`: 66 | 67 | ```yaml 68 | mediaStores: 69 | - id: media 70 | type: local 71 | path: /data/media 72 | watch: true 73 | 74 | database: /data/novia.db 75 | 76 | download: 77 | enabled: true 78 | ytdlpPath: yt-dlp 79 | ytdlpCookies: ./cookies.txt 80 | tempPath: /tmp 81 | targetStoreId: media 82 | secret: false 83 | 84 | publish: 85 | enabled: true 86 | key: nsecxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 87 | thumbnailUpload: 88 | - https://nostr.download 89 | videoUpload: 90 | - url: https://nostr.download 91 | maxUploadSizeMB: 300 92 | cleanUpMaxAgeDays: 5 93 | cleanUpKeepSizeUnderMB: 2 94 | relays: 95 | - 96 | secret: false 97 | autoUpload: 98 | enabled: true 99 | maxVideoSizeMB: 100 100 | 101 | fetch: 102 | enabled: false 103 | fetchVideoLimitMB: 10 104 | relays: 105 | - 106 | match: 107 | - nostr 108 | - bitcoin 109 | 110 | server: 111 | port: 9090 112 | enabled: true 113 | 114 | ``` 115 | 116 | **Notice:** The paths point to `/data/` here. The service will automatically look for the config file in `/data/novia.db` or `./novia.db`. For docker setups it is easiest to just mount the `/data` folder. 117 | 118 | 119 | Now you can run docker to start the novia service: 120 | 121 | ```sh 122 | docker run -it --rm -p 9090:9090 -v ./data:/data teamnovia/novia 123 | ``` 124 | 125 | or for running in background 126 | 127 | ```sh 128 | docker run -it --rm -p 9090:9090 -v ./data:/data teamnovia/novia 129 | ``` 130 | 131 | Use the `-e DEBUG=novia*` for more debugging output in the logs. 132 | 133 | ## Running with nodejs locally 134 | 135 | ### Prerequsites 136 | 137 | Required NodeJS version is 21 (novia relies on it's websocket client support) 138 | 139 | There are a few tools that novia uses for video download, conversion and hash calculation. 140 | It expects the following tools to be installed: 141 | 142 | - yt-dlp (https://github.com/yt-dlp/yt-dlp) 143 | - ffmpeg (https://ffmpeg.org/) 144 | - shasum (https://www.gnu.org/software/coreutils/ usually preinstalled on most systems) 145 | 146 | ### Configuration 147 | 148 | When running with nodejs there is a helpful init tool that helps you to create the config file: 149 | 150 | ```bash 151 | npx novia init 152 | ``` 153 | 154 | After answering the questions you should get a `novia.yaml` that looks something like this: 155 | 156 | ```yaml 157 | mediaStores: 158 | - id: media 159 | type: local 160 | path: ./media 161 | watch: true 162 | database: ./novia.db 163 | download: 164 | enabled: true 165 | ytdlpPath: yt-dlp 166 | tempPath: ./temp 167 | targetStoreId: media 168 | publish: 169 | enabled: true 170 | key: nsec188j2c3e0tdk7w6vapd0fcdn0g9fq653vzpj2zufgwx659qt49euqjlwnu0 171 | thumbnailUpload: 172 | - https://nostr.download 173 | videoUpload: 174 | - url: https://nostr.download 175 | cleanUpKeepSizeUnderMB: 2 176 | cleanUpMaxAgeDays: 10 177 | maxUploadSizeMB: 500 178 | relays: 179 | - my-video-relay.org 180 | server: 181 | enabled: true 182 | port: 9090 183 | ``` 184 | 185 | If the media folder that you have specified doesn't already exist, go ahead and create it now. 186 | 187 | ```sh 188 | mkdir ./media 189 | ``` 190 | 191 | Then you can run `serve` to start the service and answer video requests. 192 | 193 | ```bash 194 | npx novia serve 195 | ``` 196 | 197 | # Known issues / limitations 198 | 199 | There are several issues that have not been solved yet, here the most important: 200 | 201 | - Currently all running novia instances download a video triggered by an archive request. There is no coordination or circuit 202 | breaker, when someone else is downloading the video. 203 | - There is currently no way to share an archived video with another 204 | archive (incl. metadata and thumbnail). 205 | - All novia instances that download a video also publish the video, i.e. there will be 206 | multiple video events from different novia instances. Additional checks if a video already 207 | exists, might be needed. 208 | - Blossom servers don't upload of support large files. A possible solution is chunking of 209 | files (cherry tree). 210 | - There are not many blossom servers that support large amounts of content. This will hopefully 211 | be improved with payed blossom servers (soon). 212 | 213 | # NOSTR events 214 | 215 | ## Video Events 216 | 217 | Novia creates video events according to nip71 (https://github.com/nostr-protocol/nips/blob/master/71.md) but with a few specifics: 218 | 219 | - Videos created with novia are usually **not online**, i.e. the events are created without the video `url` but only with the `x` tag which contains the videos's sha256 hash. Clients have to try requesting the video's hash from known blossom servers. 220 | - A c-tag `["c", "", "author"]` is used to mark the 221 | original author, e.g. Youtube channel. 222 | - Another c-tag `["c", "", "source"]` is used to store the source website where this video was archived from. This is usually the `extractor` fields from `yt-dlp`. 223 | - An `["l", "en", "ISO-639-1"]` is added to specify the language of the video if available. 224 | - An additional tag `["info", ""]` is used to store a blossom hash for video metadata that is created by `yt-dlp`. This can be used to restore an archive entry with the full meta data information that is not contained in the nostr event. 225 | 226 | ## DVM Archive (aka Download) Request (Kind 5205) 227 | 228 | | name | tag | description | 229 | | ----- | --- | -------------------------------------------------------------------- | 230 | | input | i | An input of type URL of a website a video should be downloaded from. | 231 | 232 | ```json 233 | { 234 | "kind": 5205, 235 | "tags": [["i", "https://www.youtube.com/watch?v=CQ4G2wLdGSE", "url"]] 236 | } 237 | ``` 238 | 239 | ## DVM Archive (aka Download) Repsonse (Kind 6205) 240 | 241 | | name | tag | description | 242 | | ------- | --- | ---------------------------------------- | 243 | | content | i | `JSON block with video data, see below.` | 244 | 245 | ```json 246 | { 247 | "kind": 6205, 248 | "content": "{\"eventId\":\"6641da6a8f8d20acdad49b2bdbef3f57cf51f4e145be6a472903560f51a7bd4b\",\"video\":\"33528c883ea0f8b74f5f7433c7797ca36b9747799231aa7a9489423cbabfb217\",\"thumb\":\"ce1681a9bd006a2a9456f92fecad47372a5eb921488826856451c5f0ed8fac29\",\"info\":\"c674b4a2d431c34372ac9364a1e9207ee6fee82b36d392b4b52cff0c007f0604\",\"naddr\":{\"identifier\":\"youtube-5hPtU8Jbpg0\",\"pubkey\":\"3d70ed1c5f9a9103487c16f575bcd21d7cf4642e2e86539915cee78b2d68948c\",\"relays\":[\"wss://vidono.apps.slidestr.net/\"],\"kind\":34235}}", 249 | "tags": [ 250 | ["request", "... "], 251 | ["e", "170d42b31da8bd582b6797b3a74a2df8238538a65433baee7e59f746df1de9f1"], 252 | ["p", "..."], 253 | ["i", "https://www.youtube.com/watch?v=5hPtU8Jbpg0", "url"], 254 | ["expiration", "1733264732"] 255 | ] 256 | } 257 | ``` 258 | 259 | ### JSON Content 260 | 261 | ```json 262 | { 263 | "eventId": "6641da6a8f8d20acdad49b2bdbef3f57cf51f4e145be6a472903560f51a7bd4b", 264 | "video": "33528c883ea0f8b74f5f7433c7797ca36b9747799231aa7a9489423cbabfb217", 265 | "thumb": "ce1681a9bd006a2a9456f92fecad47372a5eb921488826856451c5f0ed8fac29", 266 | "info": "c674b4a2d431c34372ac9364a1e9207ee6fee82b36d392b4b52cff0c007f0604", 267 | "naddr": { 268 | "identifier": "youtube-5hPtU8Jbpg0", 269 | "pubkey": "...", 270 | "relays": ["wss://some.relay.net/"], 271 | "kind": 34235 272 | } 273 | } 274 | ``` 275 | 276 | ## DVM Recover (aka Upload) Request (Kind 5206) 277 | 278 | | name | tag |  multipe | description | 279 | | ----- | ------ | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | 280 | | input | i | no | A video event the author is looking for (including event id and relay ) | 281 | | param | x | no | The sha256 hash of the video that the author is looking for (required). | 282 | | param | target | yes | A blossom server the author wants the video to be uploaded. There can be issues with authentication and the novia service can decide to upload on a different server. The target param can appear multiple times, to request upload to multiple servers. | 283 | 284 | ```json 285 | { 286 | "kind": 5206, 287 | "tags": [ 288 | ["i", "0d1664a9709d385e2dc50e24de0d82fc6394bf93dfc60707dcf0bba2013f14f9", "event", "wss://some-video-relay.net/"], 289 | ["param", "x", "9bc58f0248ecfe4e2f3aa850edcb17725b9ac91bbe1b8d337617f431c66b8366"], 290 | ["param", "target", "https://nostr.download/"] 291 | ] 292 | } 293 | ``` 294 | 295 | ## DVM Recover (aka Upload) Repsonse (Kind 6206) 296 | 297 | ```json 298 | { 299 | "kind": 6206, 300 | 301 | "content": "{\"eventId\":\"6a6fc428642d277bb487f2992c2e0c8d33895841a8ac5c6b4d214708340d78d1\",\"video\":\"bab588bb3cb018080a49921d8bbf1775cccbb16c8e934efe5b65ee56289d3892\",\"thumb\":\"05e12717f17a39cca9b44e2f4a745dd3314308fbc2e78e716b49f89dec879386\",\"info\":\"963d1681a8021bf315c7b633655cfdd53c3f530f80d8ce4b7c404b60a8cfe7a6\"}", 302 | "created_at": 1733177725, 303 | "id": "9fc75cbbc8a06f4069e37077e920e4a6b0f41af6a279b98493da6a6ed897d27c", 304 | "tags": [ 305 | ["request", "..."], 306 | ["e", "da766329f00d71b73c94317db31688d4e3f74c35a2523e1dc016806d5ee9d866"], 307 | ["p", "..."], 308 | ["i", "6a6fc428642d277bb487f2992c2e0c8d33895841a8ac5c6b4d214708340d78d1", "event", "wss://some-video-relay.net/"], 309 | ["expiration", "1733609725"] 310 | ] 311 | } 312 | ``` 313 | 314 | ### JSON Content 315 | 316 | ```json 317 | { 318 | "eventId": "6a6fc428642d277bb487f2992c2e0c8d33895841a8ac5c6b4d214708340d78d1", 319 | "video": "bab588bb3cb018080a49921d8bbf1775cccbb16c8e934efe5b65ee56289d3892", 320 | "thumb": "05e12717f17a39cca9b44e2f4a745dd3314308fbc2e78e716b49f89dec879386", 321 | "info": "963d1681a8021bf315c7b633655cfdd53c3f530f80d8ce4b7c404b60a8cfe7a6" 322 | } 323 | ``` 324 | 325 | ## DVM Mirror Request (Kind 5207) ??? 326 | 327 | A (private) request to another DVM that offers a video 328 | for mirroring. 329 | 330 | "Here is a video, that is only available for a short time. Please mirror it locally" 331 | 332 | (not implemented yet) 333 | 334 | ```json 335 | { 336 | "kind": 5207, 337 | "tags": [ 338 | ["i", "0d1664a9709d385e2dc50e24de0d82fc6394bf93dfc60707dcf0bba2013f14f9", "event", "wss://some-video-relay.net/"], 339 | ["param", "x", "9bc58f0248ecfe4e2f3aa850edcb17725b9ac91bbe1b8d337617f431c66b8366"], 340 | ["param", "target", "https://nostr.download/"] 341 | ] 342 | } 343 | ``` 344 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | 2 | services: 3 | novia: 4 | build: . 5 | ports: 6 | - 9090:9090 7 | volumes: 8 | - ./data:/data 9 | 10 | -------------------------------------------------------------------------------- /novia.example.yaml: -------------------------------------------------------------------------------- 1 | mediaStores: 2 | - id: media 3 | type: local 4 | path: ./data/media 5 | watch: true 6 | 7 | database: ./data/novia.db 8 | 9 | download: 10 | enabled: true 11 | ytdlpPath: yt-dlp 12 | tempPath: ./temp 13 | targetStoreId: media 14 | 15 | publish: 16 | enabled: true 17 | key: nsecXXXXXXXXXXXXXXXX 18 | blossomThumbnails: 19 | - https://nostr.download 20 | blossomVideos: 21 | - https://nostr.download 22 | relays: 23 | - 24 | videoBlobExpirationDays: 10 25 | videoBlobCutoffSizeLimitMB: 2 26 | 27 | server: 28 | port: 9090 29 | enabled: true 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "novia", 3 | "version": "0.0.9", 4 | "description": "", 5 | "main": "index.js", 6 | "type": "module", 7 | "scripts": { 8 | "build": "tsc && chmod +x dist/index.js", 9 | "start": "tsc && node dist/index.js serve", 10 | "mikro-orm": "mikro-orm", 11 | "migration:create": "mikro-orm migration:create", 12 | "migration:generate": "mikro-orm migration:generate", 13 | "migration:up": "mikro-orm migration:up", 14 | "migration:down": "mikro-orm migration:down", 15 | "mikro-orm:cache": "mikro-orm cache:generate --combined", 16 | "format": "prettier -w src/**.ts" 17 | }, 18 | "keywords": [], 19 | "author": "", 20 | "license": "ISC", 21 | "devDependencies": { 22 | "@types/commander": "^2.12.5", 23 | "@types/cors": "^2.8.17", 24 | "@types/debug": "^4.1.12", 25 | "@types/express": "^5.0.0", 26 | "@types/fs-extra": "^11.0.4", 27 | "@types/js-yaml": "^4.0.9", 28 | "@types/lodash": "^4.17.13", 29 | "@types/node": "^22.9.0", 30 | "@types/progress-stream": "^2.0.5", 31 | "prettier": "^3.3.3", 32 | "tsconfig-paths": "^4.2.0", 33 | "tsx": "^4.19.2", 34 | "typescript": "^5.6.3" 35 | }, 36 | "dependencies": { 37 | "@mikro-orm/cli": "^6.4.0", 38 | "@mikro-orm/core": "^6.4.0", 39 | "@mikro-orm/migrations": "^6.4.0", 40 | "@mikro-orm/reflection": "^6.4.0", 41 | "@mikro-orm/sqlite": "^6.4.0", 42 | "axios": "^1.7.7", 43 | "chokidar": "^4.0.1", 44 | "cors": "^2.8.5", 45 | "debug": "^4.3.7", 46 | "express": "^4.21.1", 47 | "fs-extra": "^11.2.0", 48 | "inquirer": "^12.1.0", 49 | "js-yaml": "^4.1.0", 50 | "lodash": "^4.17.21", 51 | "nostr-tools": "^2.10.3", 52 | "progress-stream": "^2.0.0", 53 | "reflect-metadata": "^0.2.2", 54 | "uuid": "^11.0.3" 55 | }, 56 | "bin": { 57 | "novia": "dist/index.js" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/config.ts: -------------------------------------------------------------------------------- 1 | import { accessSync, existsSync, mkdirSync, readFileSync } from "fs"; 2 | import { Config } from "./types.js"; 3 | import * as yaml from "js-yaml"; 4 | import { pathExists } from "fs-extra"; 5 | 6 | export function readConfigSync(): Config { 7 | const configPaths = ["./novia.yaml", "/data/novia.yaml"]; 8 | let config: Config | undefined; 9 | 10 | for (const configPath of configPaths) { 11 | if (existsSync(configPath)) { 12 | console.log(`Reading config from ${configPath}`); 13 | const fileContents = readFileSync(configPath, "utf8"); 14 | config = yaml.load(fileContents) as Config; 15 | break; 16 | } 17 | } 18 | 19 | if (!config) { 20 | console.error("Config not found! Using fallback config..."); 21 | 22 | config = { 23 | mediaStores: [{ id: "store", type: "local", path: "./media", watch: true }], 24 | download: { enabled: true, ytdlpPath: "ytdlp", tempPath: "./temp", targetStoreId: "store" }, 25 | database: "./novia.db", 26 | server: { enabled: true, port: 9090 }, 27 | }; 28 | } 29 | 30 | return config; 31 | } 32 | 33 | export async function validateConfig(config: Config) { 34 | if (config.mediaStores.find((ms) => ms.id == config.download?.targetStoreId) == undefined) { 35 | throw new Error(`Download store ${config.download?.targetStoreId} not found in media stores.`); 36 | } 37 | 38 | // Ensure media folder exists 39 | for (const store of config.mediaStores.filter((ms) => ms.type == "local")) { 40 | if (store.path) { 41 | accessSync(store.path); 42 | } else { 43 | throw new Error(`Media store ${store.id} has no path configured.`); 44 | } 45 | } 46 | 47 | if (config.download?.tempPath && (await pathExists(config.download?.tempPath))) { 48 | mkdirSync(config.download?.tempPath, { recursive: true }); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/dvm/archive.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { Config } from "../types.js"; 3 | import { ArchiveJobContext, DVM_VIDEO_ARCHIVE_RESULT_KIND, ONE_DAY_IN_SECONDS } from "./types.js"; 4 | import { decode, EventPointer } from "nostr-tools/nip19"; 5 | import debug from "debug"; 6 | import { publishStatusEvent } from "./publish.js"; 7 | import { processVideoDownloadJob } from "../jobs/processVideoDownloadJob.js"; 8 | import { processFile } from "../video-indexer.js"; 9 | import { doComputeSha256 } from "../jobs/processShaHashes.js"; 10 | import { doNostrUploadForVideo } from "../jobs/processNostrUpload.js"; 11 | import { ensureEncrypted, getInputTag, getRelays } from "../helpers/dvm.js"; 12 | import { finalizeEvent, nip19, SimplePool } from "nostr-tools"; 13 | import { unique } from "../utils/array.js"; 14 | import { now } from "../utils/utils.js"; 15 | 16 | const logger = debug("novia:dvm:archive"); 17 | 18 | export async function doWorkForArchive(context: ArchiveJobContext, config: Config, rootEm: EntityManager) { 19 | const secretKey = decode(config.publish?.key || "").data as Uint8Array; 20 | const relays = config.publish?.relays || []; 21 | 22 | const msg = `Starting archive download job for ${context.url}`; 23 | logger(msg); 24 | 25 | if (!config.download?.secret) { 26 | await publishStatusEvent(context, "processing", JSON.stringify({ msg }), [], secretKey, relays); 27 | } 28 | 29 | try { 30 | const targetVideoPath = await processVideoDownloadJob(config, context.url, false, async (dl) => { 31 | const msg = `Download in progress: ${dl.percentage}% done at ${dl.speedMiBps}MB/s`; 32 | logger(msg); 33 | if (!config.download?.secret) { 34 | await publishStatusEvent(context, "partial", JSON.stringify({ msg }), [], secretKey, relays); 35 | } 36 | }); 37 | logger(targetVideoPath); 38 | 39 | if (!config.download?.secret) { 40 | publishStatusEvent( 41 | context, 42 | "partial", 43 | JSON.stringify({ msg: "Download finished. Processing and uploading to NOSTR..." }), 44 | [], 45 | secretKey, 46 | relays, 47 | ); 48 | } 49 | 50 | if (!targetVideoPath) { 51 | throw new Error("Download of video has failed."); 52 | } 53 | const video = await processFile(rootEm, config.mediaStores, targetVideoPath, false); 54 | 55 | if (video) { 56 | await doComputeSha256(video, config.mediaStores); 57 | const nostrResult = await doNostrUploadForVideo(video, config); 58 | 59 | if (!nostrResult) { 60 | throw new Error("Could not create nostr video event"); 61 | } 62 | const { id } = nip19.decode(nostrResult.nevent).data as EventPointer; 63 | video.event = id; 64 | const em = rootEm.fork(); 65 | em.persistAndFlush(video); 66 | 67 | if (!config.download?.secret) { 68 | const resultEvent = { 69 | kind: DVM_VIDEO_ARCHIVE_RESULT_KIND, 70 | tags: [ 71 | ["request", JSON.stringify(context.request)], 72 | ["e", context.request.id], 73 | ["p", context.request.pubkey], 74 | getInputTag(context.request), 75 | ["expiration", `${now() + ONE_DAY_IN_SECONDS}`], 76 | ], 77 | content: JSON.stringify(nostrResult), 78 | created_at: now(), 79 | }; 80 | 81 | logger(resultEvent); 82 | 83 | const event = await ensureEncrypted(secretKey, resultEvent, context.request.pubkey, context.wasEncrypted); 84 | const result = finalizeEvent(event, secretKey); 85 | 86 | const pool = new SimplePool(); 87 | 88 | const pubRes = await Promise.all( 89 | pool 90 | .publish(unique([...getRelays(context.request), ...(config.publish?.relays || [])]), result) 91 | .map((p) => p.catch((e) => {})), 92 | ); 93 | logger(pubRes); 94 | } 95 | } 96 | } catch (e) { 97 | const msg = "Download from the video source failed."; 98 | publishStatusEvent(context, "error", JSON.stringify({ msg }), [], secretKey, relays); 99 | console.error(msg, e); 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/dvm/index.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import { NostrEvent, Filter, nip04, SimplePool, getPublicKey, nip19 } from "nostr-tools"; 3 | import { listBlobs, deleteBlob } from "../helpers/blossom.js"; 4 | import { Subscription } from "nostr-tools/abstract-relay"; 5 | import { getInput, getInputParam, getInputParams } from "../helpers/dvm.js"; 6 | import debug from "debug"; 7 | import { Config, PublishConfig } from "../types.js"; 8 | import { decode } from "nostr-tools/nip19"; 9 | import { EntityManager } from "@mikro-orm/sqlite"; 10 | import { 11 | DVM_VIDEO_ARCHIVE_REQUEST_KIND, 12 | DVM_VIDEO_ARCHIVE_RESULT_KIND, 13 | DVM_VIDEO_RECOVER_REQUEST_KIND as DVM_VIDEO_RECOVER_REQUEST_KIND, 14 | DVM_VIDEO_RECOVER_RESULT_KIND, 15 | HORIZONZAL_VIDEO_KIND, 16 | JobContext, 17 | ONE_HOUR_IN_MILLISECS, 18 | VERTICAL_VIDEO_KIND, 19 | } from "./types.js"; 20 | import { doWorkForRecover } from "./recover.js"; 21 | import { mergeServers, now } from "../utils/utils.js"; 22 | import { doWorkForArchive } from "./archive.js"; 23 | import { queueMirrorJob } from "../jobs/queue.js"; 24 | import { unique } from "../utils/array.js"; 25 | import { RecoverResult } from "../jobs/results.js"; 26 | 27 | const pool = new SimplePool(); 28 | 29 | const logger = debug("novia:dvm"); 30 | 31 | const subscriptions: { [key: string]: Subscription } = {}; 32 | 33 | const filters: Filter[] = [ 34 | { 35 | kinds: [ 36 | DVM_VIDEO_ARCHIVE_REQUEST_KIND, 37 | DVM_VIDEO_RECOVER_REQUEST_KIND, 38 | DVM_VIDEO_RECOVER_RESULT_KIND, 39 | HORIZONZAL_VIDEO_KIND, 40 | VERTICAL_VIDEO_KIND, 41 | ], 42 | since: now() - 60, 43 | }, 44 | ]; // look 60s back 45 | 46 | async function shouldAcceptJob(request: NostrEvent): Promise { 47 | const input = getInput(request); 48 | 49 | if (input.type === "event" && request.kind == DVM_VIDEO_RECOVER_REQUEST_KIND) { 50 | const x = getInputParam(request, "x"); 51 | const target = getInputParams(request, "target"); 52 | return { type: "recover", x, eventId: input.value, relay: input.relay, target, request, wasEncrypted: false }; 53 | } else if (input.type === "url" && request.kind == DVM_VIDEO_ARCHIVE_REQUEST_KIND) { 54 | // TODO check allowed URLs (regexs in config?) 55 | return { type: "archive", url: input.value, request, wasEncrypted: false }; 56 | } else throw new Error(`Unknown input type ${input.type} ${request.kind}`); 57 | } 58 | 59 | async function doWork(context: JobContext, config: Config, rootEm: EntityManager) { 60 | if (config.download?.enabled && context.type == "archive") { 61 | await doWorkForArchive(context, config, rootEm); 62 | } else if (context.type == "recover") { 63 | await doWorkForRecover(context, config, rootEm); 64 | } 65 | } 66 | 67 | async function ensureDecrypted(secretKey: Uint8Array, event: NostrEvent) { 68 | const encrypted = event.tags.some((t) => t[0] == "encrypted"); 69 | if (encrypted) { 70 | const encryptedTags = await nip04.decrypt(secretKey, event.pubkey, event.content); 71 | return { 72 | wasEncrypted: true, 73 | event: { 74 | ...event, 75 | tags: event.tags.filter((t) => t[0] !== "encrypted").concat(JSON.parse(encryptedTags)), 76 | }, 77 | }; 78 | } 79 | return { wasEncrypted: false, event }; 80 | } 81 | 82 | const seenEvents = new Set(); 83 | 84 | async function handleEvent(event: NostrEvent, config: Config, rootEm: EntityManager) { 85 | if (!seenEvents.has(event.id)) { 86 | try { 87 | seenEvents.add(event.id); 88 | if (event.kind === DVM_VIDEO_ARCHIVE_REQUEST_KIND || event.kind === DVM_VIDEO_RECOVER_REQUEST_KIND) { 89 | const secretKey = decode(config.publish?.key || "").data as Uint8Array; 90 | 91 | const { wasEncrypted, event: decryptedEvent } = await ensureDecrypted(secretKey, event); 92 | const context = await shouldAcceptJob(decryptedEvent); 93 | context.wasEncrypted = wasEncrypted; 94 | try { 95 | await doWork(context, config, rootEm); 96 | } catch (e) { 97 | if (e instanceof Error) { 98 | logger(`Failed to process request ${decryptedEvent.id} because`, e.message); 99 | console.log(e); 100 | } 101 | } 102 | } 103 | if (event.kind === DVM_VIDEO_ARCHIVE_RESULT_KIND) { 104 | // skip for new because we use the HORIZONZAL_VIDEO_KIND/VERTICAL_VIDEO_KIND 105 | } 106 | if (event.kind === DVM_VIDEO_RECOVER_RESULT_KIND) { 107 | try { 108 | const recoverResult = JSON.parse(event.content) as RecoverResult; 109 | // TODO we could use the hashes directly!? 110 | if (config.fetch?.enabled) { 111 | queueMirrorJob(rootEm, recoverResult.nevent); 112 | } 113 | } catch (e) { 114 | if (e instanceof Error) { 115 | logger(`Failed to process recover result ${event.id} because`, e.message); 116 | console.log(e); 117 | } 118 | } 119 | } 120 | 121 | if (event.kind === HORIZONZAL_VIDEO_KIND || event.kind === VERTICAL_VIDEO_KIND) { 122 | if (config.fetch?.enabled) { 123 | const relays = config.publish?.relays || []; // TODO use read/inbox relays 124 | const nevent = nip19.neventEncode({ id: event.id, author: event.pubkey, kind: event.kind, relays }); 125 | queueMirrorJob(rootEm, nevent); 126 | } 127 | } 128 | 129 | /* 130 | if (event.kind === kinds.GiftWrap) { 131 | const dmEvent = unwrapGiftWrapDM(event); 132 | await processPaymentAndRunJob(dmEvent.pubkey, dmEvent.content); 133 | } 134 | */ 135 | } catch (e) { 136 | if (e instanceof Error) { 137 | console.error(e); 138 | logger(`Skipped request ${event.id} because`, e.message); 139 | } 140 | } 141 | } 142 | } 143 | 144 | async function ensureSubscriptions(config: Config, rootEm: EntityManager) { 145 | const relays = unique([...(config.publish?.relays || []), ...(config.fetch?.relays || [])]); // TODO use read/inbox relays vs outbox/publish relays 146 | logger( 147 | `ensureSubscriptions`, 148 | JSON.stringify(Object.entries(subscriptions).map(([k, v]) => ({ k, closed: v.closed }))), 149 | ); 150 | for (const url of relays) { 151 | const existing = subscriptions[url]; 152 | 153 | if (!existing || existing.closed) { 154 | if (existing?.closed) { 155 | logger(`Reconnecting to ${url}`); 156 | } 157 | delete subscriptions[url]; 158 | try { 159 | const relay = await pool.ensureRelay(url); 160 | const sub = relay.subscribe(filters, { 161 | onevent: (e) => handleEvent(e, config, rootEm), 162 | onclose: () => { 163 | logger("Subscription to", url, "closed"); 164 | if (subscriptions[url] === sub) delete subscriptions[url]; 165 | }, 166 | }); 167 | 168 | logger("Subscribed to", url); 169 | subscriptions[url] = sub; 170 | 171 | logger( 172 | `subscriptions after set`, 173 | JSON.stringify(Object.entries(subscriptions).map(([k, v]) => ({ k, closed: v.closed }))), 174 | ); 175 | } catch (error: any) { 176 | logger("Failed to reconnect to", url, error.message); 177 | delete subscriptions[url]; 178 | } 179 | } 180 | } 181 | } 182 | 183 | async function cleanupBlobs(publishConfig: PublishConfig) { 184 | const secretKey = decode(publishConfig.key || "").data as Uint8Array; 185 | const pubkey = getPublicKey(secretKey); 186 | const uploadServers = mergeServers(...publishConfig.videoUpload.map((s) => s.url), ...publishConfig.thumbnailUpload); 187 | 188 | for (const server of uploadServers) { 189 | const blobs = await listBlobs(server, pubkey, secretKey); // TODO add from/until to filter by timestamp 190 | 191 | const serverConfig = publishConfig.videoUpload.find((s) => s.url.startsWith(server)); 192 | if (serverConfig?.cleanUpMaxAgeDays !== undefined) { 193 | const videoBlobCutoffSizeLimit = (serverConfig?.cleanUpKeepSizeUnderMB || 0) * 1024 * 1024; 194 | const videoBlobCutoffAgeLimit = now() - 60 * 60 * 24 * serverConfig.cleanUpMaxAgeDays; 195 | const videoBlobCutoffMimeType = "video/mp4"; 196 | 197 | for (const blob of blobs) { 198 | if ( 199 | blob.created < videoBlobCutoffAgeLimit && 200 | blob.size > videoBlobCutoffSizeLimit && 201 | blob.type == videoBlobCutoffMimeType 202 | ) { 203 | // delete >2MB videos 204 | logger(`Deleting expired blob ${blob.url}`); 205 | await deleteBlob(server, blob.sha256, secretKey); 206 | } 207 | } 208 | } 209 | 210 | // TODO stats for all blossom servers, maybe groups for images/videos 211 | const storedSize = blobs.reduce((prev, val) => prev + val.size, 0); 212 | console.log( 213 | `Currently stored ${blobs.length} blobs in ${Math.floor((100 * storedSize) / 1024 / 1024 / 1024) / 100} GB on ${server}.`, 214 | ); 215 | } 216 | } 217 | 218 | export async function startDVM(config: Config, rootEm: EntityManager) { 219 | if (config.publish) { 220 | await cleanupBlobs(config.publish); 221 | setInterval(() => config.publish && cleanupBlobs(config.publish), ONE_HOUR_IN_MILLISECS); 222 | } 223 | 224 | await ensureSubscriptions(config, rootEm); 225 | setInterval(() => ensureSubscriptions(config, rootEm), 30_000); // Ensure connections every 30s 226 | } 227 | -------------------------------------------------------------------------------- /src/dvm/publish.ts: -------------------------------------------------------------------------------- 1 | import debug from "debug"; 2 | import { now } from "../utils/utils.js"; 3 | import { DVM_STATUS_KIND, JobContext, ONE_DAY_IN_SECONDS } from "./types.js"; 4 | import { finalizeEvent, SimplePool } from "nostr-tools"; 5 | import { getRelays } from "../helpers/dvm.js"; 6 | import { unique } from "../utils/array.js"; 7 | 8 | const logger = debug("novia:dvm:publish"); 9 | 10 | export async function publishStatusEvent( 11 | context: JobContext, 12 | status: "payment-required" | "processing" | "error" | "success" | "partial", 13 | data = "", 14 | additionalTags: string[][] = [], 15 | secretKey: Uint8Array, 16 | relays: string[], 17 | ) { 18 | const tags = [ 19 | ["status", status], 20 | ["e", context.request.id], 21 | ["p", context.request.pubkey], 22 | ["expiration", `${now() + ONE_DAY_IN_SECONDS}`], 23 | ]; 24 | tags.push(...additionalTags); 25 | 26 | const statusEvent = { 27 | kind: DVM_STATUS_KIND, // DVM Status 28 | tags, 29 | content: data, 30 | created_at: now(), 31 | }; 32 | logger("statusEvent", statusEvent); 33 | 34 | // const event = await ensureEncrypted(resultEvent, context.request.pubkey, context.wasEncrypted); 35 | const result = finalizeEvent(statusEvent, secretKey); 36 | 37 | const pool = new SimplePool(); 38 | await Promise.all( 39 | pool.publish(unique([...getRelays(context.request), ...relays]), result).map((p) => p.catch((e) => {})), 40 | ); 41 | } 42 | -------------------------------------------------------------------------------- /src/dvm/recover.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { BlossomConfig, Config } from "../types.js"; 3 | import { DVM_VIDEO_RECOVER_RESULT_KIND, FIVE_DAYS_IN_SECONDS, RecoverJobContext } from "./types.js"; 4 | import debug from "debug"; 5 | import { decode, npubEncode } from "nostr-tools/nip19"; 6 | import { findFullPathsForVideo, formatDuration, mergeServers, now } from "../utils/utils.js"; 7 | import { Video } from "../entity/Video.js"; 8 | import { publishStatusEvent } from "./publish.js"; 9 | import { ensureEncrypted, getInputTag, getRelays } from "../helpers/dvm.js"; 10 | import { finalizeEvent, SimplePool } from "nostr-tools"; 11 | import { unique } from "../utils/array.js"; 12 | import { buildRecoverResult } from "../jobs/results.js"; 13 | import { uploadToBlossomServers } from "./upload.js"; 14 | import uniqBy from "lodash/uniqBy.js"; 15 | 16 | const logger = debug("novia:dvm:recover"); 17 | 18 | let uploadSpeed = 2 * 1024 * 1024; 19 | 20 | export async function doWorkForRecover(context: RecoverJobContext, config: Config, rootEm: EntityManager) { 21 | if (!config.publish) { 22 | throw new Error("publish config not found."); 23 | } 24 | logger(`Starting work for ${context.request.id}`); 25 | const secretKey = decode(config.publish?.key || "").data as Uint8Array; 26 | const relays = config.publish?.relays || []; 27 | 28 | const startTime = now(); 29 | 30 | const em = rootEm.fork(); 31 | const video = await em.findOne(Video, { videoSha256: context.x }); 32 | 33 | if (!video) { 34 | logger(`Requested video not found in database. Ignoring the request. eventID: ${context.eventId} x: ${context.x}`); 35 | return; // End processing 36 | } 37 | 38 | const fullPaths = findFullPathsForVideo(video, config.mediaStores); 39 | 40 | if (!fullPaths) { 41 | if (!config.publish.secret) { 42 | await publishStatusEvent( 43 | context, 44 | "error", 45 | JSON.stringify({ msg: "Requested video found in database but the file is currently not available." }), 46 | [], 47 | secretKey, 48 | relays, 49 | ); 50 | } 51 | 52 | return; 53 | } 54 | 55 | if (!config.publish.secret) { 56 | await publishStatusEvent( 57 | context, 58 | "processing", 59 | JSON.stringify({ 60 | msg: `Starting video upload. Estimated time ${formatDuration(Math.floor(video?.mediaSize / uploadSpeed))}...`, 61 | }), 62 | [], 63 | secretKey, 64 | relays, 65 | ); 66 | } 67 | 68 | const uploadServers = uniqBy( 69 | [...config.publish.videoUpload, ...context.target.map((s) => ({ url: s }) as BlossomConfig)].map((bc) => ({ 70 | ...bc, 71 | url: bc.url.replace(/\/$/, ""), 72 | })), 73 | (bc) => bc.url.toLocaleLowerCase(), 74 | ); 75 | 76 | console.log( 77 | `Request for video ${video.id} by ${npubEncode(context.request.pubkey)}. Uploading to ${uploadServers.join(", ")}`, 78 | ); 79 | 80 | const handleUploadProgress = async (server: BlossomConfig, percentCompleted: number, speedMBs: number) => { 81 | const msg = `Upload to ${server.url}: ${percentCompleted.toFixed(2)}% done at ${speedMBs.toFixed(2)}MB/s`; 82 | logger(msg); 83 | if (!config.publish?.secret) { 84 | await publishStatusEvent(context, "partial", JSON.stringify({ msg }), [], secretKey, relays); 85 | } 86 | }; 87 | 88 | const handleErrorMessage = async (msg: string) => { 89 | if (!config?.publish?.secret) { 90 | await publishStatusEvent( 91 | context, 92 | "error", 93 | JSON.stringify({ 94 | msg, 95 | }), 96 | [], 97 | secretKey, 98 | relays, 99 | ); 100 | } 101 | }; 102 | 103 | await uploadToBlossomServers(uploadServers, video, fullPaths, secretKey, handleUploadProgress, handleErrorMessage); 104 | 105 | if (!config?.publish?.secret) { 106 | const resultEvent = { 107 | kind: DVM_VIDEO_RECOVER_RESULT_KIND, 108 | tags: [ 109 | ["request", JSON.stringify(context.request)], 110 | ["e", context.request.id], 111 | ["p", context.request.pubkey], 112 | getInputTag(context.request), 113 | ["expiration", `${now() + FIVE_DAYS_IN_SECONDS}`], 114 | ], 115 | content: JSON.stringify(buildRecoverResult(context.eventId, context.relay ? [context.relay] : [], video)), 116 | created_at: now(), 117 | }; 118 | 119 | const event = await ensureEncrypted(secretKey, resultEvent, context.request.pubkey, context.wasEncrypted); 120 | const result = finalizeEvent(event, secretKey); 121 | 122 | // TODO add DVM error events for exeptions 123 | 124 | logger("Will publish event: ", result); 125 | 126 | const pool = new SimplePool(); 127 | await Promise.all( 128 | pool 129 | .publish(unique([...getRelays(context.request), ...(config.publish?.relays || [])]), result) 130 | .map((p) => p.catch((e) => {})), 131 | ); 132 | } 133 | 134 | const endTime = now(); 135 | 136 | if (endTime - startTime > 2) { 137 | // min. 2s download to get good data 138 | uploadSpeed = Math.floor(uploadSpeed * 0.3 + (0.7 * video.mediaSize) / (endTime - startTime)); 139 | logger(`Setting upload speed to ${uploadSpeed} bytes/s`); 140 | } 141 | logger(`${`Finished work for ${context.request.id} in ` + (endTime - startTime)} seconds`); 142 | } 143 | -------------------------------------------------------------------------------- /src/dvm/types.ts: -------------------------------------------------------------------------------- 1 | import { NostrEvent } from "nostr-tools"; 2 | 3 | export const ONE_HOUR_IN_MILLISECS = 60 * 60 * 1000; 4 | export const ONE_DAY_IN_SECONDS = 24 * 60 * 60; 5 | export const FIVE_DAYS_IN_SECONDS = 5 * ONE_DAY_IN_SECONDS; 6 | 7 | export const DVM_STATUS_KIND = 7000; 8 | export const DVM_VIDEO_ARCHIVE_REQUEST_KIND = 5205; 9 | export const DVM_VIDEO_ARCHIVE_RESULT_KIND = 6205; 10 | export const DVM_VIDEO_RECOVER_REQUEST_KIND = 5206; 11 | export const DVM_VIDEO_RECOVER_RESULT_KIND = 6206; 12 | 13 | export const HORIZONZAL_VIDEO_KIND = 34235; 14 | export const VERTICAL_VIDEO_KIND = 34236; 15 | 16 | export const BLOSSOM_AUTH_KIND = 24242; 17 | 18 | interface BaseJobContext { 19 | request: NostrEvent; 20 | wasEncrypted: boolean; 21 | } 22 | 23 | // Subtype for "archive" 24 | export interface ArchiveJobContext extends BaseJobContext { 25 | type: "archive"; 26 | url: string; 27 | } 28 | 29 | // Subtype for "recover" 30 | export interface RecoverJobContext extends BaseJobContext { 31 | type: "recover"; 32 | x: string; 33 | eventId: string; 34 | relay?: string; 35 | target: string[]; 36 | } 37 | 38 | // Union type 39 | export type JobContext = ArchiveJobContext | RecoverJobContext; 40 | -------------------------------------------------------------------------------- /src/dvm/upload.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import { Video } from "../entity/Video.js"; 3 | import { uploadFile } from "../helpers/blossom.js"; 4 | import { getMimeTypeByPath } from "../utils/utils.js"; 5 | import debug from "debug"; 6 | import { BlossomConfig } from "../types.js"; 7 | 8 | const logger = debug("novia:dvm:upload"); 9 | 10 | export async function uploadToBlossomServers( 11 | uploadServers: BlossomConfig[], 12 | video: Video, 13 | fullPaths: { 14 | videoPath: string; 15 | thumbPath: string; 16 | infoPath: string; 17 | }, 18 | secretKey: Uint8Array, 19 | onProgress?: (server: BlossomConfig, percentCompleted: number, speedMBs: number) => Promise, 20 | onError?: (msg: string) => Promise, 21 | ) { 22 | for (const server of uploadServers) { 23 | if (video.mediaSize > server.maxUploadSizeMB * 1024 * 1024) { 24 | logger( 25 | `Can not upload to ${server.url} because video exceeds maxUploadSizeMB: ${video.mediaSize} > ${server.maxUploadSizeMB}MB`, 26 | ); 27 | continue; 28 | } 29 | 30 | const resultTags: string[][] = []; 31 | 32 | try { 33 | const videoBlob = await uploadFile( 34 | fullPaths.videoPath, 35 | server.url, 36 | getMimeTypeByPath(fullPaths.videoPath), 37 | path.basename(fullPaths.videoPath), 38 | "Upload Video", 39 | secretKey, 40 | video.videoSha256, 41 | async (percentCompleted, speedMBs) => { 42 | if (onProgress) { 43 | await onProgress(server, percentCompleted, speedMBs); 44 | } 45 | }, 46 | ); 47 | logger(`Uploaded video file: ${videoBlob.url}`); 48 | } catch (err) { 49 | const msg = `Upload of video to ${server} failed.`; 50 | console.error(msg, err); 51 | onError && (await onError(msg)); 52 | } 53 | 54 | try { 55 | const thumbBlob = await uploadFile( 56 | fullPaths.thumbPath, 57 | server.url, 58 | getMimeTypeByPath(fullPaths.thumbPath), 59 | path.basename(fullPaths.thumbPath), 60 | "Upload Thumbnail", 61 | secretKey, 62 | video.thumbSha256, 63 | ); 64 | logger(`Uploaded thumbnail file: ${thumbBlob.url}`); 65 | } catch (err) { 66 | const msg = `Upload of tumbnails to ${server} failed.`; 67 | console.error(msg, err); 68 | } 69 | 70 | try { 71 | const infoBlob = await uploadFile( 72 | fullPaths.infoPath, 73 | server.url, 74 | getMimeTypeByPath(fullPaths.infoPath), 75 | path.basename(fullPaths.infoPath), 76 | "Upload info json", 77 | secretKey, 78 | video.infoSha256, 79 | ); 80 | logger(`Uploaded info json file: ${infoBlob.url}`); 81 | } catch (err) { 82 | const msg = `Upload of info json to ${server} failed.`; 83 | console.error(msg, err); 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/entity/Queue.ts: -------------------------------------------------------------------------------- 1 | // src/entity/Queue.ts 2 | 3 | import { Entity, PrimaryKey, Property } from "@mikro-orm/core"; 4 | 5 | @Entity() 6 | export class Queue { 7 | @PrimaryKey() 8 | id!: number; 9 | 10 | @Property({ default: "download" }) 11 | type!: "download" | "extendmetadata" | 'createHashes' | 'nostrUpload' | 'mirrorVideo' 12 | 13 | @Property() 14 | url!: string; 15 | 16 | @Property({ default: "local" }) 17 | owner: string = "local"; 18 | 19 | @Property({ default: "queued" }) 20 | status: "queued" | "completed" | "failed" | "processing" = "queued"; 21 | 22 | @Property() 23 | errorMessage: string = ""; 24 | 25 | @Property({ onCreate: () => new Date() }) 26 | addedAt!: Date; 27 | 28 | @Property({ nullable: true }) 29 | processedAt?: Date; 30 | } 31 | -------------------------------------------------------------------------------- /src/entity/Video.ts: -------------------------------------------------------------------------------- 1 | // src/entities/Video.ts 2 | import { Entity, PrimaryKey, Property } from "@mikro-orm/core"; 3 | import { v4 as uuidv4 } from "uuid"; 4 | 5 | @Entity() 6 | export class Video { 7 | @PrimaryKey({ type: "uuid" }) 8 | id: string = uuidv4(); 9 | 10 | @Property() 11 | store!: string; 12 | 13 | @Property() 14 | videoPath!: string; 15 | 16 | @Property() 17 | videoSha256: string = ""; 18 | 19 | @Property() 20 | infoPath: string = ""; 21 | 22 | @Property() 23 | infoSha256: string = ""; 24 | 25 | @Property() 26 | thumbPath: string = ""; 27 | 28 | @Property() 29 | thumbSha256: string = ""; 30 | 31 | @Property() 32 | addedAt: Date = new Date(); 33 | 34 | @Property() 35 | externalId!: string; 36 | 37 | @Property() 38 | source: string = ""; 39 | 40 | @Property({ type: "text[]", nullable: true }) 41 | category?: string[]; 42 | 43 | @Property() 44 | channelId!: string; 45 | 46 | @Property() 47 | channelName: string = ""; 48 | 49 | @Property({ type: "bigint" }) // Storing timestamp as bigint 50 | dateDownloaded!: number; 51 | 52 | @Property({ type: "integer" }) 53 | duration!: number; 54 | 55 | @Property({ type: "text" }) 56 | description!: string; 57 | 58 | @Property({ type: "integer" }) 59 | mediaSize!: number; 60 | 61 | @Property({ type: "date" }) 62 | published!: Date; 63 | 64 | @Property({ type: "text[]", nullable: true }) 65 | tags?: string[]; 66 | 67 | @Property() 68 | title!: string; 69 | 70 | @Property() 71 | vidType!: string; 72 | 73 | @Property({ type: "integer" }) 74 | likeCount: number = 0; 75 | 76 | @Property({ type: "integer" }) 77 | viewCount: number = 0; 78 | 79 | @Property({ type: "integer" }) 80 | ageLimit: number = 0; 81 | 82 | @Property({ type: "integer" }) 83 | width: number = 0; 84 | 85 | @Property({ type: "integer" }) 86 | height: number = 0; 87 | 88 | @Property() 89 | event: string = ""; 90 | 91 | @Property() 92 | language: string = ""; 93 | } 94 | -------------------------------------------------------------------------------- /src/helpers/blossom.ts: -------------------------------------------------------------------------------- 1 | import { finalizeEvent } from "nostr-tools"; 2 | import { createReadStream, createWriteStream, mkdirSync, statSync } from "fs"; 3 | import axios, { AxiosProgressEvent } from "axios"; 4 | import debug from "debug"; 5 | 6 | import { readFile } from "fs/promises"; 7 | import { createHash } from "crypto"; 8 | import progress_stream from "progress-stream"; 9 | import { basename, join, parse } from "path"; 10 | import { pipeline } from "stream"; 11 | import { promisify } from "util"; 12 | 13 | const logger = debug("novia:blossom"); 14 | export const BLOSSOM_AUTH_KIND = 24242; 15 | 16 | export type BlobDescriptor = { 17 | created: number; 18 | type?: string; 19 | sha256: string; 20 | size: number; 21 | url: string; 22 | }; 23 | 24 | const now = () => Math.floor(Date.now() / 1000); 25 | 26 | const tenMinutesFromNow = () => now() + 10 * 60; 27 | 28 | function createBlossemUploadAuthToken( 29 | size: number, 30 | blobHash: string, 31 | name: string, 32 | description: string, 33 | secretKey: Uint8Array, 34 | ): string { 35 | const authEvent = { 36 | created_at: now(), 37 | kind: BLOSSOM_AUTH_KIND, 38 | content: "Upload thumbail", 39 | tags: [ 40 | ["t", "upload"], 41 | ["size", `${size}`], 42 | ["x", blobHash], 43 | ["name", name], 44 | ["expiration", `${tenMinutesFromNow()}`], 45 | ], 46 | }; 47 | const signedEvent = finalizeEvent(authEvent, secretKey); 48 | logger(JSON.stringify(signedEvent)); 49 | return btoa(JSON.stringify(signedEvent)); 50 | } 51 | 52 | function createBlossemListAuthToken(secretKey: Uint8Array): string { 53 | const authEvent = { 54 | created_at: now(), 55 | kind: BLOSSOM_AUTH_KIND, 56 | content: "List Blobs", 57 | tags: [ 58 | ["t", "list"], 59 | ["expiration", `${tenMinutesFromNow()}`], 60 | ], 61 | }; 62 | const signedEvent = finalizeEvent(authEvent, secretKey); 63 | return btoa(JSON.stringify(signedEvent)); 64 | } 65 | 66 | function createBlossemDeleteAuthToken(blobHash: string, secretKey: Uint8Array): string { 67 | const authEvent = { 68 | created_at: now(), 69 | kind: BLOSSOM_AUTH_KIND, 70 | content: "Delete Blob", 71 | tags: [ 72 | ["t", "delete"], 73 | ["x", blobHash], 74 | ["expiration", `${tenMinutesFromNow()}`], 75 | ], 76 | }; 77 | const signedEvent = finalizeEvent(authEvent, secretKey); 78 | return btoa(JSON.stringify(signedEvent)); 79 | } 80 | 81 | /* 82 | export function decodeBlossemAuthToken(encodedAuthToken: string) { 83 | try { 84 | return JSON.parse(atob(encodedAuthToken).toString()) as SignedEvent; 85 | } catch (e: any) { 86 | logger("Failed to extract auth token ", encodedAuthToken); 87 | } 88 | } 89 | */ 90 | async function calculateSHA256(filePath: string): Promise { 91 | try { 92 | const fileBuffer = await readFile(filePath); 93 | const hash = createHash("sha256"); 94 | hash.update(fileBuffer); 95 | return hash.digest("hex"); 96 | } catch (error: any) { 97 | throw new Error(`Fehler beim Berechnen des SHA-256-Hash: ${error.message}`); 98 | } 99 | } 100 | 101 | export async function uploadFile( 102 | filePath: string, 103 | server: string, 104 | mimeType: string, 105 | name: string, 106 | actionDescription: string, 107 | secretKey: Uint8Array, 108 | hash?: string, 109 | onProgress?: (percentCompleted: number, speedMBs: number) => Promise, 110 | ): Promise { 111 | try { 112 | const stat = statSync(filePath); 113 | const fileSize = stat.size; 114 | 115 | hash = hash || (await calculateSHA256(filePath)); 116 | 117 | try { 118 | const test = await axios.head(`${server}/${hash}`); 119 | if (test.status == 200) { 120 | logger("File already exists. No upload needed."); 121 | // Return dummy blob descriptor 122 | return { 123 | url: `${server}/${hash}`, 124 | created: now(), 125 | sha256: hash, 126 | size: fileSize, 127 | }; 128 | } 129 | } catch (error) { 130 | // Ignore error, due to 404 or similar 131 | } 132 | 133 | const blossomAuthToken = createBlossemUploadAuthToken(fileSize, hash, name, actionDescription, secretKey); 134 | 135 | // Create a read stream for the thumbnail file 136 | const fileStream = createReadStream(filePath); 137 | const progress = progress_stream({ 138 | length: fileSize, 139 | time: 10000, // Emit progress events every 100ms 140 | }); 141 | 142 | // Variables to calculate speed 143 | let startTime = Date.now(); 144 | let previousBytes = 0; 145 | 146 | progress.on("progress", (prog) => { 147 | if (onProgress) { 148 | // Calculate upload speed in MB/s 149 | const currentTime = Date.now(); 150 | const timeElapsed = (currentTime - startTime) / 1000; // seconds 151 | const bytesSinceLast = prog.transferred - previousBytes; 152 | const speed = bytesSinceLast / timeElapsed / (1024 * 1024); // MB/s 153 | 154 | // Update for next calculation 155 | startTime = currentTime; 156 | previousBytes = prog.transferred; 157 | onProgress(prog.percentage, speed); 158 | } 159 | }); 160 | 161 | // Pipe the read stream through the progress stream 162 | const stream = fileStream.pipe(progress); 163 | 164 | // Upload thumbnail stream using axios 165 | const blob = await axios.put(`${server}/upload`, stream, { 166 | headers: { 167 | "Content-Type": mimeType, 168 | Authorization: "Nostr " + blossomAuthToken, 169 | "Content-Length": fileSize, 170 | }, 171 | maxContentLength: Infinity, 172 | maxBodyLength: Infinity, 173 | 174 | // This is required for the progress to work. It prevents buffering but could break 175 | // with some blossom servers that require redirects. 176 | // https://github.com/axios/axios/issues/1045 177 | maxRedirects: 0, 178 | }); 179 | 180 | logger(`File ${filePath} uploaded successfully.`); 181 | return blob.data; 182 | } catch (error: any) { 183 | throw new Error( 184 | `Failed to upload file ${filePath} to ${server}: ${error.message} (${JSON.stringify(error.response?.data)})`, 185 | ); 186 | } 187 | } 188 | 189 | export async function listBlobs(server: string, pubkey: string, secretKey: Uint8Array): Promise { 190 | const authToken = createBlossemListAuthToken(secretKey); 191 | const blobResult = await axios.get(`${server}/list/${pubkey}`, { 192 | headers: { 193 | "Content-Type": "application/json; charset=utf-8", 194 | Authorization: "Nostr " + authToken, 195 | }, 196 | }); 197 | if (blobResult.status !== 200) { 198 | logger(`Failed to list blobs: ${blobResult.status} ${blobResult.statusText}`); 199 | } 200 | return blobResult.data; 201 | } 202 | 203 | export async function deleteBlob(server: string, blobHash: string, secretKey: Uint8Array): Promise { 204 | const authToken = createBlossemDeleteAuthToken(blobHash, secretKey); 205 | const blobResult = await axios.delete(`${server}/${blobHash}`, { 206 | headers: { 207 | Authorization: "Nostr " + authToken, 208 | }, 209 | }); 210 | if (blobResult.status !== 200) { 211 | logger(`Failed to delete blobs: ${blobResult.status} ${blobResult.statusText}`); 212 | } 213 | } 214 | 215 | const streamPipeline = promisify(pipeline); 216 | 217 | export async function downloadFile( 218 | server: string, 219 | hash: string, 220 | targetPath: string, 221 | filename?: string, 222 | ): Promise { 223 | const fileUrl = `${server}/${hash}`; 224 | const destinationPath = join(targetPath, filename || hash); 225 | 226 | try { 227 | // Ensure the target directory exists 228 | mkdirSync(targetPath, { recursive: true }); 229 | 230 | // Initiate the download request 231 | const response = await axios({ 232 | method: "get", 233 | url: fileUrl, 234 | responseType: "stream", 235 | }); 236 | 237 | if (response.status !== 200) { 238 | throw new Error(`Failed to download file: ${response.statusText}`); 239 | } 240 | 241 | // Save the file to the target path 242 | const fileStream = createWriteStream(destinationPath); 243 | await streamPipeline(response.data, fileStream); 244 | 245 | logger(`File downloaded successfully to ${destinationPath}`); 246 | return destinationPath; 247 | } catch (error: any) { 248 | throw new Error( 249 | `Failed to download file from ${fileUrl} to ${destinationPath}: ${error.message}`, 250 | ); 251 | } 252 | } 253 | 254 | export async function downloadFromServers( 255 | servers: string[], 256 | hash: string, 257 | targetPath: string, 258 | filename?: string 259 | ): Promise { 260 | let lastError: Error | null = null; 261 | 262 | for (const server of servers) { 263 | try { 264 | logger(`Attempting to download from server: ${server}`); 265 | const filePath = await downloadFile(server, hash, targetPath, filename); 266 | logger(`Successfully downloaded from server: ${server}`); 267 | return filePath; 268 | } catch (error: any) { 269 | lastError = error; 270 | logger(`Failed to download from server: ${server} - ${error.message}`); 271 | } 272 | } 273 | 274 | // If no server succeeded, throw the last error 275 | if (lastError) { 276 | throw new Error( 277 | `Failed to download file from all servers: ${lastError.message}` 278 | ); 279 | } 280 | 281 | throw new Error("No servers were provided for the download."); 282 | } 283 | 284 | /** returns the last sha256 in a URL */ 285 | export function getHashFromURL(url: string | URL) { 286 | if (typeof url === "string") url = new URL(url); 287 | 288 | const hashes = Array.from(url.pathname.matchAll(/[0-9a-f]{64}/gi)); 289 | if (hashes.length > 0) return hashes[hashes.length - 1][0]; 290 | 291 | return null; 292 | } -------------------------------------------------------------------------------- /src/helpers/dvm.ts: -------------------------------------------------------------------------------- 1 | import { Event, EventTemplate, nip04 } from "nostr-tools"; 2 | 3 | export function getInputTag(e: Event) { 4 | const tag = e.tags.find((t) => t[0] === "i"); 5 | if (!tag) throw new Error("Missing tag"); 6 | return tag; 7 | } 8 | 9 | export function getInput(e: Event) { 10 | const tag = getInputTag(e); 11 | const [_, value, type, relay, marker] = tag; 12 | if (!value) throw new Error("Missing input value"); 13 | if (!type) throw new Error("Missing input type"); 14 | return { value, type, relay, marker }; 15 | } 16 | export function getRelays(event: Event) { 17 | return event.tags.find((t) => t[0] === "relays")?.slice(1) ?? []; 18 | } 19 | export function getOutputType(event: Event): string | undefined { 20 | return event.tags.find((t) => t[0] === "output")?.[1]; 21 | } 22 | 23 | export function getInputParams(e: Event, k: string) { 24 | return e.tags.filter((t) => t[0] === "param" && t[1] === k).map((t) => t[2]); 25 | } 26 | 27 | export function getInputParam(e: Event, k: string, defaultValue?: string) { 28 | const value = getInputParams(e, k)[0] || defaultValue; 29 | if (value === undefined) throw new Error(`Missing ${k} param`); 30 | return value; 31 | } 32 | 33 | export async function ensureEncrypted( 34 | secretKey: Uint8Array, 35 | event: EventTemplate, 36 | recipentPubKey: string, 37 | wasEncrypted: boolean, 38 | ) { 39 | if (!wasEncrypted) return event; 40 | 41 | const tagsToEncrypt = event.tags.filter((t) => t[0] !== "p" && t[0] !== "e"); 42 | const encText = await nip04.encrypt(secretKey, recipentPubKey, JSON.stringify(tagsToEncrypt)); 43 | 44 | return { 45 | ...event, 46 | content: encText, 47 | tags: (event.tags = [...event.tags.filter((t) => t[0] == "e"), ["p", recipentPubKey], ["encrypted"]]), 48 | }; 49 | } 50 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S node --no-warnings 2 | 3 | import { Command } from "commander"; 4 | import { EntityManager, MikroORM, raw } from "@mikro-orm/sqlite"; 5 | import ormConfig from "./mikro-orm.config.js"; 6 | import { Queue } from "./entity/Queue.js"; 7 | import { formatDuration, getMimeTypeByPath } from "./utils/utils.js"; 8 | import { Video } from "./entity/Video.js"; 9 | import { readConfigSync, validateConfig } from "./config.js"; 10 | import { scanDirectory, setupWatcher as setupNewVideoWatcher } from "./video-indexer.js"; 11 | import { Config, MediaStore } from "./types.js"; 12 | import { getPublicKey, nip19 } from "nostr-tools"; 13 | import debug from "debug"; 14 | import { startLocalServer } from "./server.js"; 15 | import { validateMissingSha256Hashes as validateHashes } from "./validation/validateHashes.js"; 16 | import { validateMetaData } from "./validation/validateMetaData.js"; 17 | import { queueAllVideosForNostrUpload, queueDownloadJob } from "./jobs/queue.js"; 18 | import { processCreateHashesJob } from "./jobs/processShaHashes.js"; 19 | import { processVideoDownloadJob } from "./jobs/processVideoDownloadJob.js"; 20 | import { processExtendMetaData } from "./jobs/processExtendMetaData.js"; 21 | import { processNostrUpload } from "./jobs/processNostrUpload.js"; 22 | import { startDVM } from "./dvm/index.js"; 23 | import { createNoviaConfig } from "./init-setup.js"; 24 | import { processMirrorJob } from "./jobs/processMirrorJob.js"; 25 | 26 | const logger = debug("novia"); 27 | 28 | const appConfig = readConfigSync(); 29 | 30 | // initialize the ORM, loading the config file dynamically 31 | const orm = MikroORM.initSync({ ...ormConfig, dbName: appConfig.database }); 32 | 33 | let processingInterval: NodeJS.Timeout | null = null; 34 | 35 | let inProgress = false; 36 | 37 | export async function processJob(rootEm: EntityManager, config: Config, job: Queue) { 38 | const em = rootEm.fork(); 39 | const { id, url, type } = job; 40 | console.log(`Processing ID: ${id}, URL: ${url}, type: ${type}`); 41 | 42 | job.status = "processing"; 43 | await em.persistAndFlush(job); 44 | 45 | try { 46 | if (type == "download") { 47 | await processVideoDownloadJob(config, job.url); 48 | } else if (type == "extendmetadata") { 49 | await processExtendMetaData(rootEm, config, job); 50 | } else if (type == "createHashes") { 51 | await processCreateHashesJob(rootEm, config, job); 52 | } else if (type == "nostrUpload") { 53 | await processNostrUpload(rootEm, config, job); 54 | } else if (type == "mirrorVideo") { 55 | await processMirrorJob(rootEm, config, job); 56 | } 57 | 58 | // Update status to 'completed' 59 | job.status = "completed"; 60 | job.processedAt = new Date(); 61 | await em.persistAndFlush(job); 62 | } catch (error) { 63 | // Update status to 'failed' 64 | job.status = "failed"; 65 | job.errorMessage = (error as Error).message || ""; 66 | await em.persistAndFlush(job); 67 | 68 | console.error(`Failed to process ID: ${id}, URL: ${url}`, error); 69 | } 70 | } 71 | 72 | async function startQueueProcessing() { 73 | console.log("Starting queue processing..."); 74 | 75 | processingInterval = setInterval(async () => { 76 | if (inProgress) { 77 | // console.log("There is a job in progress."); 78 | return; 79 | } 80 | inProgress = true; 81 | try { 82 | const em = orm.em.fork(); 83 | const jobs = await em.findAll(Queue, { 84 | where: { status: "queued" }, 85 | orderBy: { id: "ASC" }, 86 | limit: 100, 87 | }); 88 | 89 | for (const job of jobs) { 90 | await processJob(orm.em, appConfig, job); 91 | } 92 | } catch (err) { 93 | console.error(err); 94 | } 95 | inProgress = false; 96 | }, 500); // Check every second 97 | 98 | // Keep the process running 99 | process.stdin.resume(); 100 | } 101 | 102 | async function stopProcessing() { 103 | if (processingInterval) { 104 | clearInterval(processingInterval); 105 | processingInterval = null; 106 | console.log("Processing stopped."); 107 | } else { 108 | console.log("Processing is not running."); 109 | } 110 | } 111 | 112 | async function scanAllStoresForNewVideos(localStores: MediaStore[]) { 113 | for (const store of localStores) { 114 | if (store.path) { 115 | console.log(`Scanning path '${store.path}' for videos...'`); 116 | await scanDirectory(orm.em, localStores, store.path); 117 | } 118 | } 119 | } 120 | 121 | async function refreshMedia() { 122 | const localStores = appConfig.mediaStores.filter((s) => (s.type = "local")); 123 | 124 | await scanAllStoresForNewVideos(localStores); 125 | 126 | await validateMetaData(orm.em, localStores); 127 | 128 | // TODO this is dangerous when the files are missing 129 | // add some security checks. 130 | //await cleanDeletedVideos(orm.em, localStores); 131 | 132 | await validateHashes(orm.em, localStores); 133 | } 134 | 135 | const program = new Command(); 136 | 137 | program.name("queue-cli").description("CLI application for managing a job queue").version("1.0.0"); 138 | 139 | program 140 | .command("refresh") 141 | .description("Refresh the video index from disk") 142 | .action(async () => { 143 | refreshMedia(); 144 | process.exit(0); 145 | }); 146 | program 147 | .command("init") 148 | .description("Create an inital novia.yaml config file") 149 | .action(async () => { 150 | await createNoviaConfig(); 151 | process.exit(0); 152 | }); 153 | 154 | program 155 | .command("serve") 156 | .description("Run as a server and process downloads") 157 | .action(async () => { 158 | await validateConfig(appConfig); 159 | 160 | await setupNewVideoWatcher( 161 | orm.em, 162 | appConfig.mediaStores.filter((ms) => ms.type == "local"), 163 | ); 164 | 165 | if (appConfig.server?.enabled) { 166 | startLocalServer(orm.em, appConfig.mediaStores); 167 | } 168 | 169 | await startQueueProcessing(); 170 | 171 | // Scan for new videos and update missing content 172 | await refreshMedia(); 173 | 174 | const em = orm.em.fork(); 175 | await em.nativeDelete(Queue, { status: "completed" }); 176 | 177 | // Uploads to blossom and local nostr relay 178 | if (appConfig.publish?.enabled) { 179 | await queueAllVideosForNostrUpload(orm.em); 180 | } 181 | 182 | await printStats(); 183 | 184 | await startDVM(appConfig, orm.em); 185 | }); 186 | 187 | const queue = program.command("queue").description("Manage the job queue"); 188 | 189 | // Subcommand: queue add 190 | queue 191 | .command("add ") 192 | .description("Add URL to queue") 193 | .action(async (url) => { 194 | await queueDownloadJob(orm.em, url); 195 | process.exit(0); 196 | }); 197 | 198 | // Subcommand: queue ls 199 | queue 200 | .command("ls") 201 | .description("List the contents of the queue") 202 | .action(async () => { 203 | await listQueue(); 204 | process.exit(0); 205 | }); 206 | 207 | const videos = program.command("video").description("Manage videos"); 208 | 209 | // Subcommand: queue add 210 | videos 211 | .command("ls [text]") 212 | .description("List all videos") 213 | .action(async (text: string) => { 214 | await listVideos(text); 215 | process.exit(0); 216 | }); 217 | 218 | async function listQueue() { 219 | try { 220 | const em = orm.em.fork(); 221 | const jobs = await em.findAll(Queue, { orderBy: { id: "ASC" } }); 222 | 223 | if (jobs.length === 0) { 224 | console.log("The queue is empty."); 225 | } else { 226 | console.table( 227 | jobs.map((job) => ({ 228 | ID: job.id, 229 | URL: job.url, 230 | Status: job.status, 231 | AddedAt: job.addedAt, 232 | ProcessedAt: job.processedAt, 233 | })), 234 | ); 235 | } 236 | } catch (error) { 237 | console.error("Error listing the queue:", error); 238 | } 239 | } 240 | 241 | async function listVideos(seachText: string = "") { 242 | try { 243 | const em = orm.em.fork(); 244 | const videos = await em.findAll(Video, { 245 | where: { title: { $like: `%${seachText}%` } }, 246 | orderBy: { addedAt: "DESC" }, 247 | }); 248 | 249 | if (videos.length === 0) { 250 | console.log("No videos is found."); 251 | } else { 252 | console.table( 253 | videos.map((v) => ({ 254 | ID: v.id, 255 | Store: v.store, 256 | URL: v.videoPath, 257 | Title: v.title, 258 | Duration: formatDuration(v.duration), 259 | })), 260 | ); 261 | } 262 | } catch (error) { 263 | console.error("Error listing videos:", error); 264 | } 265 | } 266 | 267 | async function printStats() { 268 | const em = orm.em.fork(); 269 | 270 | const res = (await em 271 | .createQueryBuilder(Video, "v") 272 | .select(["store", raw("sum(media_size)/1024.0/1024/1024 as sizeGB"), raw("count(*) as count")]) 273 | .groupBy(["store"]) 274 | .execute()) as { index: number; store: string; sizeGB: number; count: number }[]; 275 | console.table(res.map((o) => ({ ...o, sizeGB: Math.floor(100 * o.sizeGB) / 100 }))); 276 | } 277 | 278 | async function startup() { 279 | await orm.schema.updateSchema(); 280 | //await orm.schema.refreshDatabase(); // ATTENTION DELETES EVERYTHING 281 | program.parse(process.argv); 282 | } 283 | startup(); 284 | /* 285 | console.log( 286 | nip19.nsecEncode( 287 | Uint8Array.from( 288 | Buffer.from( 289 | "xxx", 290 | "hex" 291 | ) 292 | ) 293 | ) 294 | ); 295 | */ 296 | 297 | if (appConfig.publish?.key) { 298 | const pubkeyHex = getPublicKey(nip19.decode(appConfig.publish?.key).data as Uint8Array); 299 | console.log(`Identity: ${nip19.npubEncode(pubkeyHex)} (Hex: ${pubkeyHex})`); 300 | } 301 | 302 | async function shutdown() { 303 | process.exit(); 304 | } 305 | 306 | process.on("SIGINT", shutdown); 307 | process.on("SIGTERM", shutdown); 308 | process.once("SIGUSR2", shutdown); 309 | 310 | if (global.WebSocket == undefined) { 311 | console.error("Websocket support is required. Use NodeJS >= v.21"); 312 | process.exit(1); 313 | } 314 | 315 | // TODO check for yt-dlp 316 | // TODO check for shasum 317 | // TODO check for ffmpeg 318 | -------------------------------------------------------------------------------- /src/init-setup.ts: -------------------------------------------------------------------------------- 1 | import { outputFile, pathExists } from "fs-extra"; 2 | import inquirer from "inquirer"; 3 | import { generateSecretKey, nip19 } from "nostr-tools"; 4 | import path from "path"; 5 | import { Config, DownloadConfig, PublishConfig, ServerConfig } from "./types.js"; 6 | import * as yaml from "js-yaml"; 7 | 8 | /** 9 | * Prompts the user for configuration settings and writes the novia.yaml file. 10 | */ 11 | export async function createNoviaConfig() { 12 | const configPath = path.resolve(process.cwd(), "novia.yaml"); 13 | 14 | // Check if novia.yaml exists 15 | if (await pathExists(configPath)) { 16 | const { overwrite } = await inquirer.prompt([ 17 | { 18 | type: "confirm", 19 | name: "overwrite", 20 | message: "Overwrite novia.yaml? Are you sure?", 21 | default: false, 22 | }, 23 | ]); 24 | 25 | if (!overwrite) { 26 | console.log("Operation cancelled. novia.yaml was not overwritten."); 27 | return; 28 | } 29 | } 30 | 31 | // Initial prompts 32 | const initialAnswers = await inquirer.prompt([ 33 | { 34 | type: "list", 35 | name: "secOption", 36 | message: "Enter nsec or generate a new private key?", 37 | choices: [ 38 | { name: "Enter private key manually", value: "manual" }, 39 | { name: "Generate a new private key", value: "generate" }, 40 | ], 41 | }, 42 | { 43 | type: "input", 44 | name: "storagePath", 45 | message: "Storage path for videos:", 46 | default: "./media", 47 | validate: (input: string) => input.trim() !== "" || "Storage path cannot be empty.", 48 | }, 49 | { 50 | type: "confirm", 51 | name: "publishEnabled", 52 | message: "Enable publishing of video events to NOSTR? (yes/no)", 53 | default: false, 54 | }, 55 | ]); 56 | 57 | // SEC handling 58 | let nsec: string; 59 | if (initialAnswers.secOption === "manual") { 60 | const { manualnsec } = await inquirer.prompt([ 61 | { 62 | type: "input", 63 | name: "manualnsec", 64 | message: "Enter your nsec (private key):", 65 | validate: (input: string) => 66 | /^nsec[a-z0-9]$/.test(input) || 67 | 'Invalid nsec format. It should start with "nsec" followed by numbers or characters.', 68 | }, 69 | ]); 70 | nsec = manualnsec; 71 | } else { 72 | const key = generateSecretKey(); 73 | nsec = nip19.nsecEncode(key); 74 | console.log(`Generated new nsec: ${nsec}`); 75 | } 76 | 77 | // Publish-related prompts 78 | let publishConfig: PublishConfig = { 79 | enabled: false, 80 | key: "", 81 | thumbnailUpload: [], 82 | videoUpload: [], 83 | relays: [], 84 | }; 85 | 86 | if (initialAnswers.publishEnabled) { 87 | const publishAnswers = await inquirer.prompt([ 88 | { 89 | type: "input", 90 | name: "relayUrls", 91 | message: "Relay URLs to publish to (comma-separated):", 92 | validate: (input: string) => 93 | input 94 | .split(",") 95 | .map((url) => url.trim()) 96 | .filter((url) => url !== "").length > 0 || "At least one relay URL is required.", 97 | }, 98 | { 99 | type: "input", 100 | name: "blossomThumbnails", 101 | message: "Blossom server to publish thumbnails:", 102 | default: "https://nostr.download", 103 | validate: (input: string) => input.trim() !== "" || "Blossom server URL cannot be empty.", 104 | }, 105 | { 106 | type: "input", 107 | name: "blossomVideos", 108 | message: "Blossom server to publish videos:", 109 | default: "https://nostr.download", 110 | validate: (input: string) => input.trim() !== "" || "Blossom server URL cannot be empty.", 111 | }, 112 | ]); 113 | 114 | // Process relay URLs 115 | const relayList = (publishAnswers.relayUrls as string) 116 | .split(",") 117 | .map((url) => url.trim()) 118 | .filter((url) => url !== ""); 119 | 120 | publishConfig = { 121 | enabled: true, 122 | key: nsec, 123 | thumbnailUpload: [publishAnswers.blossomThumbnails], 124 | videoUpload: [ 125 | { url: publishAnswers.blossomVideos, cleanUpKeepSizeUnderMB: 2, cleanUpMaxAgeDays: 10, maxUploadSizeMB: 500 }, 126 | ], 127 | autoUpload: { 128 | enabled: false, 129 | maxVideoSizeMB: 5, 130 | }, 131 | relays: relayList, 132 | }; 133 | } 134 | 135 | // Assemble the configuration object 136 | const config: Config = { 137 | mediaStores: [ 138 | { 139 | id: "media", 140 | type: "local", 141 | path: initialAnswers.storagePath, 142 | watch: true, 143 | }, 144 | ], 145 | database: "./novia.db", 146 | download: { 147 | enabled: true, 148 | ytdlpPath: "yt-dlp", 149 | tempPath: "./temp", 150 | targetStoreId: "media", 151 | } as DownloadConfig, 152 | publish: publishConfig.enabled ? publishConfig : undefined, 153 | server: { 154 | enabled: true, 155 | port: 9090, 156 | } as ServerConfig, 157 | }; 158 | 159 | // Convert the configuration object to YAML 160 | const yamlStr = yaml.dump(config); 161 | 162 | // Write the YAML to novia.yaml 163 | try { 164 | await outputFile(configPath, yamlStr); 165 | console.log(`Configuration written to ${configPath}`); 166 | } catch (error) { 167 | console.error("Failed to write novia.yaml:", error); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /src/jobs/cleanDeletedVideos.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { MediaStore } from "../types.js"; 3 | import { Video } from "../entity/Video.js"; 4 | import path from "path"; 5 | import { promises as fs } from "fs"; 6 | import { deleteBlob } from "../helpers/blossom.js"; 7 | 8 | /** 9 | * Cleans up the database by removing entries for videos that no longer exist on the filesystem. 10 | * @param em - The MikroORM EntityManager instance. 11 | */ 12 | export const cleanDeletedVideos = async (rootEm: EntityManager, stores: MediaStore[]) => { 13 | try { 14 | const em = rootEm.fork(); 15 | console.log("Starting cleanup of deleted videos..."); 16 | 17 | // Retrieve all Video entries from the database 18 | const allVideos = await em.find(Video, {}); 19 | 20 | if (allVideos.length === 0) { 21 | console.log("No videos found in the database to clean."); 22 | return; 23 | } 24 | 25 | // Array to hold videos that need to be removed 26 | const videosToRemove: Video[] = []; 27 | 28 | // Iterate over each Video entry 29 | for (const video of allVideos) { 30 | const store = stores.find((st) => st.id == video.store); 31 | if (!store || !store.path) { 32 | continue; // skip if store is not found 33 | } 34 | 35 | const fullPath = path.join(store.path, video.videoPath); 36 | try { 37 | // Check if the file exists 38 | await fs.access(fullPath); 39 | // If no error, the file exists; do nothing 40 | } catch (err) { 41 | // If an error occurs, the file does not exist 42 | console.log(`File not found. Removing from database: ${video.id} ${store.path} ${video.videoPath} ${fullPath}`); 43 | videosToRemove.push(video); 44 | } 45 | } 46 | 47 | if (videosToRemove.length > 0) { 48 | for (const video of videosToRemove) { 49 | // fetch event from relay (video.event) 50 | // TODO remove from blossom 51 | // deleteBlob(); 52 | 53 | // TODO remove from NOSTR relays (delete event) 54 | em.remove(video); 55 | } 56 | 57 | // Remove the videos from the database 58 | await em.flush(); 59 | console.log(`Removed ${videosToRemove.length} video(s) from the database.`); 60 | } else { 61 | console.log("No deleted videos found. Cleanup complete."); 62 | } 63 | } catch (error) { 64 | console.error("Error during cleanup of deleted videos:", error); 65 | } 66 | }; 67 | -------------------------------------------------------------------------------- /src/jobs/processExtendMetaData.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { Config } from "../types.js"; 3 | import { Queue } from "../entity/Queue.js"; 4 | import { Video } from "../entity/Video.js"; 5 | import { downloadYoutubeVideo } from "../utils/ytdlp.js"; 6 | import debug from "debug"; 7 | import path from "path"; 8 | import { rmSync } from "fs"; 9 | import { updateVideoMetaData } from "../video-indexer.js"; 10 | import { move } from "../utils/move.js"; 11 | import { queueSHAUpdateJob } from "./queue.js"; 12 | import { removeFieldsFromJson } from "../utils/utils.js"; 13 | 14 | const logger = debug("novia:processExtendMetaData"); 15 | 16 | export function extractDownloadUrlFromVideoPath(videoPath: string, source?: string): string | undefined { 17 | if (source == undefined || source == "youtube") { 18 | const youtubeId = path.parse(videoPath).name; 19 | return "https://www.youtube.com/watch?v=" + youtubeId; 20 | } 21 | return; 22 | } 23 | 24 | export async function processExtendMetaData(rootEm: EntityManager, config: Config, job: Queue) { 25 | const em = rootEm.fork(); 26 | 27 | const { url: videoId } = job; 28 | 29 | const video = await em.findOne(Video, videoId); 30 | 31 | if (video == null) { 32 | throw new Error(`Video with ID ${videoId} not found.`); 33 | } 34 | 35 | const url = extractDownloadUrlFromVideoPath(video.videoPath); 36 | 37 | if (url == undefined) { 38 | throw new Error(`Could not extract download url from path '${video.videoPath}'.`); 39 | } 40 | 41 | if (config.download == undefined) { 42 | throw new Error(`Download config is not defined.`); 43 | } 44 | 45 | logger("Downloading additional meta data (info/thumb) for " + url); 46 | 47 | const download = await downloadYoutubeVideo(url, true, config.download); 48 | 49 | 50 | if (!download.infoPath || !download.infoData) { 51 | throw new Error("Required files not found in the temporary directory."); 52 | } 53 | 54 | await removeFieldsFromJson(download.infoPath, [ 55 | "thumbnails", 56 | "formats", 57 | "automatic_captions", 58 | "heatmap", 59 | ]); 60 | 61 | const videoStore = config.mediaStores.find((ms) => ms.type == "local" && ms.id == video.store); 62 | if (!videoStore || !videoStore.path) { 63 | throw new Error(`Store for video ${videoId} not found.`); 64 | } 65 | 66 | const targetFolder = path.join(videoStore.path, path.parse(video.videoPath).dir); 67 | 68 | if (download.infoPath && !video.infoPath) { 69 | const targetInfoPath = path.join(targetFolder, `${path.parse(video.videoPath).name}.info.json`); 70 | logger(`move`, download.infoPath, targetInfoPath); 71 | await move(download.infoPath, targetInfoPath); 72 | } 73 | 74 | if (download.thumbnailPath && !video.thumbPath) { 75 | const targetThumbPath = path.join( 76 | targetFolder, 77 | `${path.parse(video.videoPath).name}${path.extname(download.thumbnailPath)}`, 78 | ); 79 | logger(`move`, download.thumbnailPath, targetThumbPath); 80 | await move(download.thumbnailPath, targetThumbPath); 81 | } 82 | 83 | // remove the temp dir 84 | rmSync(download.folder, { recursive: true }); 85 | 86 | await updateVideoMetaData(video, path.join(videoStore.path, video.videoPath), videoStore); 87 | 88 | await em.persistAndFlush(video); 89 | 90 | if (video.thumbPath && video.infoPath) { 91 | await queueSHAUpdateJob(rootEm, video.id); 92 | } else { 93 | console.warn("Thumbail or info.json still not found after extendMetaData for video " + video.id); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/jobs/processMirrorJob.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { Config } from "../types.js"; 3 | import { Video } from "../entity/Video.js"; 4 | import { Queue } from "../entity/Queue.js"; 5 | import debug from "debug"; 6 | import { EventPointer } from "nostr-tools/nip19"; 7 | import { nip19, NostrEvent, SimplePool } from "nostr-tools"; 8 | import { processFile } from "../video-indexer.js"; 9 | import { createTempDir } from "../utils/utils.js"; 10 | import { downloadFromServers, getHashFromURL } from "../helpers/blossom.js"; 11 | import { mapVideoData } from "../utils/mapvideodata.js"; 12 | import uniq from "lodash/uniq.js"; 13 | import { analyzeVideoFolder } from "../utils/ytdlp.js"; 14 | import { moveFilesToTargetFolder } from "./processVideoDownloadJob.js"; 15 | import { rmSync } from "fs"; 16 | 17 | const logger = debug("novia:mirrorjob"); 18 | 19 | function videoMatchesFetchCriteria(videoEvent: NostrEvent, matchCriteria: string[]): boolean { 20 | const title = videoEvent.tags.find((t) => t[0] == "title")?.[1]; 21 | const author = videoEvent.tags.find((t) => t[0] == "c" && t[2] == "author")?.[1]; 22 | 23 | if (matchCriteria.length == 0) return true; 24 | 25 | for (const regex of matchCriteria) { 26 | const r = new RegExp(regex); 27 | if (title && title.match(r)) { 28 | return true; 29 | } 30 | if (author && author.match(r)) { 31 | return true; 32 | } 33 | } 34 | return false; 35 | } 36 | 37 | export async function processMirrorJob(rootEm: EntityManager, config: Config, job: Queue) { 38 | logger(`starting processMirrorJob`); 39 | 40 | const em = rootEm.fork(); 41 | if (!config.fetch) { 42 | console.error("Fetch settings are not defined in config."); 43 | return; 44 | } 45 | if (!config.download) { 46 | console.error("Download settings are not defined in config."); 47 | return; 48 | } 49 | 50 | const { url: nevent } = job; 51 | 52 | const { id, relays } = nip19.decode(nevent).data as EventPointer; 53 | 54 | const pool = new SimplePool(); 55 | 56 | const effectiveRelays = uniq([...(relays || []), ...(config.fetch?.relays || []), ...(config.publish?.relays || [])]); 57 | logger(`Looking for video event ${nevent} on ${effectiveRelays?.join(", ")}`); 58 | 59 | const videoEvent = await pool.get(effectiveRelays, { ids: [id] }); 60 | logger(videoEvent); 61 | if (!videoEvent) { 62 | logger(`Video event ${id} not found on relays ${relays}.`); 63 | return; 64 | } 65 | 66 | const video = await em.findOne(Video, { event: videoEvent.id }); 67 | if (video) { 68 | logger(`Video for event ${id} already exists in the database. No mirroring needed.`); 69 | return; 70 | } 71 | 72 | logger(`Mirroring video event ${id}`); 73 | 74 | // check if video should be mirror (title filter, channel, pubkey of author) 75 | 76 | if (videoMatchesFetchCriteria(videoEvent, config.fetch.match || [])) { 77 | const videoData = mapVideoData(videoEvent); 78 | if (!videoData.x) { 79 | throw new Error("Video hash not found in video event."); 80 | } 81 | logger(`Downloading blob ${videoData.x}: ${videoData.title}`); 82 | 83 | const tempDir = createTempDir(config.download?.tempPath); 84 | 85 | // TODO get the blossom server from the uploader 10063, cache for 10min 86 | const blossomServers = uniq( 87 | [...(config.fetch.blossom || []), ...(config.publish?.videoUpload.map((s) => s.url) || [])].map((s) => 88 | s.replace(/\/$/, ""), 89 | ), 90 | ); 91 | await downloadFromServers(blossomServers, videoData.x, tempDir, `${videoData.x}.mp4`); 92 | 93 | const imageHash = videoData.image && getHashFromURL(videoData.image); 94 | if (imageHash) { 95 | await downloadFromServers(blossomServers, imageHash, tempDir, `${videoData.x}.jpg`); 96 | } else { 97 | logger(`Could not find sha265 hash in url ${videoData.image}`); 98 | } 99 | 100 | if (videoData.info) { 101 | await downloadFromServers(blossomServers, videoData.info, tempDir, `${videoData.x}.info.json`); 102 | } 103 | 104 | const download = await analyzeVideoFolder(tempDir, false, false); 105 | 106 | const { targetFolder, targetVideoPath } = await moveFilesToTargetFolder( 107 | config.mediaStores, 108 | config.download, 109 | download, 110 | false, 111 | ); 112 | 113 | if (!targetVideoPath) { 114 | throw Error("Error finding the stored video. " + targetVideoPath); 115 | } 116 | 117 | // remove the temp dir 118 | rmSync(download.folder, { recursive: true }); 119 | 120 | console.log(`Downloaded content saved to ${targetFolder}`); 121 | 122 | await processFile(rootEm, config.mediaStores, targetVideoPath, false, (video) => { 123 | video.event = id; 124 | if (imageHash) { 125 | video.thumbSha256 = imageHash; 126 | } 127 | if (videoData.x) { 128 | video.videoSha256 = videoData.x; 129 | } 130 | if (videoData.info) { 131 | video.infoSha256 = videoData.info; 132 | } 133 | }); 134 | 135 | if (config.publish && config.publish.enabled && config.publish.relays.length > 0) { 136 | // if publish is enabled republish the video event to the defined 137 | // publishing relays as well. 138 | pool.publish(config.publish.relays, videoEvent); 139 | } 140 | } 141 | } 142 | -------------------------------------------------------------------------------- /src/jobs/processNostrUpload.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { Config } from "../types.js"; 3 | import { Video } from "../entity/Video.js"; 4 | import { Queue } from "../entity/Queue.js"; 5 | import path from "path"; 6 | import { BlobDescriptor, uploadFile } from "../helpers/blossom.js"; 7 | import { EventTemplate, finalizeEvent, nip19, SimplePool } from "nostr-tools"; 8 | import debug from "debug"; 9 | import { findFullPathsForVideo, getMimeTypeByPath } from "../utils/utils.js"; 10 | import { buildArchiveResult } from "./results.js"; 11 | import { EventPointer } from "nostr-tools/nip19"; 12 | import { HORIZONZAL_VIDEO_KIND, VERTICAL_VIDEO_KIND } from "../dvm/types.js"; 13 | 14 | const logger = debug("novia:nostrupload"); 15 | 16 | function createOriginalWebUrl(video: Video) { 17 | if (video.source == "youtube") { 18 | return `https://www.youtube.com/watch?v=${video.externalId}`; 19 | } 20 | 21 | if (video.source == "tiktok") { 22 | return `https://www.tiktok.com/@${video.channelName}/video/${video.externalId}`; 23 | } 24 | 25 | if (video.source == "twitter") { 26 | return ``; // TODO fix? 27 | } 28 | // TODO add more 29 | return ``; 30 | } 31 | 32 | export function createTemplateVideoEvent(video: Video, thumbBlobs: BlobDescriptor[]): EventTemplate { 33 | const thumbUrls = thumbBlobs.map((thumbBlob) => 34 | thumbBlob.url.endsWith(".webp") || thumbBlob.url.endsWith(".jpg") ? thumbBlob.url : thumbBlob.url + ".webp", 35 | ); // TODO fix for other formats; 36 | 37 | const videoMimeType = getMimeTypeByPath(video.videoPath); 38 | 39 | const imeta = [ 40 | "imeta", 41 | `dim ${video.width}x${video.height}`, 42 | `x ${video.videoSha256}`, 43 | `m ${videoMimeType}`, // TODO extract from extension or add to DB 44 | ]; 45 | for (let i = 0; i < thumbUrls.length; i++) { 46 | imeta.push(`image ${thumbUrls[i]}`); 47 | } 48 | 49 | const event = { 50 | created_at: Math.floor(Date.now() / 1000), // TODO should this be today / now? 51 | kind: video.width >= video.height ? HORIZONZAL_VIDEO_KIND : VERTICAL_VIDEO_KIND, 52 | tags: [ 53 | ["d", `${video.source}-${video.externalId}`], 54 | ["x", video.videoSha256], // deprecated 55 | ["title", video.title], 56 | ["summary", video.description], // deprecated 57 | ["alt", video.description], // deprecated 58 | ["published_at", `${video.published.getTime()}`], 59 | ["client", "nostr-video-archive"], 60 | ["m", videoMimeType], 61 | ["size", `${video.mediaSize}`], 62 | ["duration", `${video.duration}`], 63 | ["c", video.channelName, "author"], // TODO check if c or l tag is better 64 | ["c", video.source, "source"], // TODO check if c or l tag is better 65 | imeta, 66 | ["r", createOriginalWebUrl(video)], 67 | ...(video.tags || []).map((tag) => ["t", tag]), 68 | ["client", "novia"], 69 | ["info", video.infoSha256], // non standard field - but there is not great way to store the info json data. 70 | ], 71 | content: video.title, 72 | }; 73 | 74 | if (video.language) { 75 | event.tags.push(["l", video.language, "ISO-639-1"]); 76 | } 77 | 78 | for (let i = 0; i < thumbUrls.length; i++) { 79 | event.tags.push(["thumb", thumbUrls[i]]); // deprecated? 80 | event.tags.push(["image", thumbUrls[i]]); // deprecated? 81 | } 82 | 83 | if (video.ageLimit >= 18) { 84 | event.tags.push(["content-warning", `NSFW adult content`]); 85 | } 86 | 87 | return event; 88 | } 89 | 90 | export async function doNostrUploadForVideo(video: Video, config: Config) { 91 | if (!config.publish) { 92 | console.error("Publish settings are not defined in config."); 93 | return; 94 | } 95 | 96 | const secretKey = nip19.decode(config.publish.key).data as Uint8Array; 97 | 98 | const fullPaths = findFullPathsForVideo(video, config.mediaStores); 99 | if (!fullPaths) { 100 | console.error("Could not resolve the full paths for the video. " + video.id); 101 | return; 102 | } 103 | 104 | const thumbnailServers = config.publish.thumbnailUpload; 105 | const thumbBlobs: BlobDescriptor[] = []; 106 | 107 | for (const blossomServer of thumbnailServers) { 108 | console.log(`Uploading ${fullPaths.thumbPath} to ${blossomServer}`); 109 | 110 | try { 111 | const thumbBlob = await uploadFile( 112 | fullPaths.thumbPath, 113 | blossomServer, 114 | getMimeTypeByPath(video.thumbPath), 115 | path.basename(video.thumbPath), 116 | "Upload Thumbnail", 117 | secretKey, 118 | video.thumbSha256, 119 | ); 120 | thumbBlobs.push(thumbBlob); 121 | } catch (err) { 122 | console.log(err); 123 | } 124 | 125 | console.log(`Uploading ${fullPaths.infoPath} to ${blossomServer}`); 126 | 127 | try { 128 | const infoBlob = await uploadFile( 129 | fullPaths.infoPath, 130 | blossomServer, 131 | getMimeTypeByPath(video.infoPath), 132 | path.basename(video.infoPath), 133 | "Upload info json", 134 | secretKey, 135 | video.infoSha256, 136 | ); 137 | console.log(infoBlob); 138 | } catch (err) { 139 | console.log(err); 140 | } 141 | } 142 | 143 | if (thumbBlobs.length == 0) { 144 | throw new Error(`Failed uploading thumbnails for video ${video.id}`); 145 | } 146 | 147 | if (config.publish.autoUpload && config.publish.autoUpload.enabled) { 148 | if (video.mediaSize < config.publish.autoUpload.maxVideoSizeMB * 1024 * 1024) { 149 | const videoServers = config.publish.videoUpload; 150 | 151 | for (const blossomServer of videoServers) { 152 | if (video.mediaSize < blossomServer.maxUploadSizeMB * 1024 * 1024) { 153 | console.log(`Uploading video ${fullPaths.videoPath} to ${blossomServer.url}`); 154 | 155 | try { 156 | const videoBlob = await uploadFile( 157 | fullPaths.videoPath, 158 | blossomServer.url, 159 | getMimeTypeByPath(video.videoPath), 160 | path.basename(video.videoPath), 161 | "Upload video", 162 | secretKey, 163 | video.videoSha256, 164 | ); 165 | console.log(videoBlob); 166 | } catch (err) { 167 | console.log(err); 168 | } 169 | } else { 170 | logger(`Skipping upload to ${blossomServer.url} due to size limit <${blossomServer.maxUploadSizeMB}MB`); 171 | } 172 | } 173 | } else { 174 | logger(`Skipping auto publishing: ${JSON.stringify(config.publish.autoUpload)}`); 175 | } 176 | } 177 | 178 | const event = createTemplateVideoEvent(video, thumbBlobs); 179 | const signedEvent = finalizeEvent({ ...event }, secretKey); 180 | logger(signedEvent); 181 | 182 | const pool = new SimplePool(); 183 | 184 | console.log(`Publishing video ${video.id} to NOSTR ${config.publish.relays.join(", ")}`); 185 | 186 | const data = await Promise.allSettled(pool.publish(config.publish.relays, signedEvent)); 187 | logger(data); 188 | return buildArchiveResult(signedEvent, config.publish.relays, video); 189 | } 190 | 191 | export async function processNostrUpload(rootEm: EntityManager, config: Config, job: Queue) { 192 | const em = rootEm.fork(); 193 | if (!config.publish) { 194 | console.error("Publish settings are not defined in config."); 195 | return; 196 | } 197 | 198 | const { url: videoId } = job; 199 | 200 | const video = await em.findOne(Video, videoId); 201 | if (video == null) { 202 | throw new Error(`Video with ID ${videoId} not found.`); 203 | } 204 | 205 | const result = await doNostrUploadForVideo(video, config); 206 | 207 | if (result && result.nevent) { 208 | const { id } = nip19.decode(result.nevent).data as EventPointer; 209 | video.event = id; 210 | em.persistAndFlush(video); 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/jobs/processShaHashes.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { Config, MediaStore } from "../types.js"; 3 | import { Queue } from "../entity/Queue.js"; 4 | import { Video } from "../entity/Video.js"; 5 | import path from "path"; 6 | import { computeSha256 } from "../utils/utils.js"; 7 | import { queueNostrUpload } from "./queue.js"; 8 | 9 | export async function doComputeSha256(video: Video, stores: MediaStore[]) { 10 | 11 | const store = stores.find((st) => st.id == video.store); 12 | if (!store || !store.path) { 13 | return; // skip if store is not found 14 | } 15 | 16 | const videoFullPath = path.join(store.path, video.videoPath); 17 | video.videoSha256 = await computeSha256(videoFullPath); 18 | 19 | const thumbFullPath = path.join(store.path, video.thumbPath); 20 | video.thumbSha256 = await computeSha256(thumbFullPath); 21 | 22 | const infoFullPath = path.join(store.path, video.infoPath); 23 | video.infoSha256 = await computeSha256(infoFullPath); 24 | } 25 | 26 | export async function processCreateHashesJob( 27 | rootEm: EntityManager, 28 | config: Config, 29 | job: Queue 30 | ) { 31 | const em = rootEm.fork(); 32 | 33 | const { url: videoId } = job; 34 | 35 | const video = await em.findOne(Video, videoId); 36 | 37 | if (video == null) { 38 | throw new Error(`Video with ID ${videoId} not found.`); 39 | } 40 | 41 | await doComputeSha256(video, config.mediaStores); 42 | 43 | await em.persistAndFlush(video); 44 | 45 | // TODO validate??? 46 | if (config.publish?.enabled) { 47 | await queueNostrUpload(rootEm, video.id); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/jobs/processVideoDownloadJob.ts: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import { extractThumbnails, ThumbnailContent } from "../utils/ffmpeg.js"; 3 | import { Config, DownloadConfig, MediaStore } from "../types.js"; 4 | import { removeFieldsFromJson } from "../utils/utils.js"; 5 | import { DownloadInfo, downloadYoutubeVideo, VideoContent } from "../utils/ytdlp.js"; 6 | import { mkdir } from "fs/promises"; 7 | import { move } from "../utils/move.js"; 8 | import { rmSync } from "fs"; 9 | 10 | export async function moveFilesToTargetFolder( 11 | mediaStores: MediaStore[], 12 | config: DownloadConfig, 13 | download: VideoContent, 14 | skipVideo: boolean = false, 15 | ) { 16 | if ((!skipVideo && !download.videoPath) || !download.infoPath || !download.infoData) { 17 | throw new Error("Required files not found in the temporary directory."); 18 | } 19 | 20 | await removeFieldsFromJson(download.infoPath, ["thumbnails", "formats", "automatic_captions", "heatmap"]); 21 | 22 | const videoId = download.infoData.id; 23 | 24 | const downloadStore = mediaStores.find((ms) => ms.type == "local" && ms.id == config?.targetStoreId); 25 | 26 | if (!downloadStore || !downloadStore.path) { 27 | throw new Error(`Download folder for store ${config?.targetStoreId} not found.`); 28 | } 29 | 30 | let generatedThumb: ThumbnailContent | undefined = undefined; 31 | if (download.videoPath && !download.thumbnailPath) { 32 | generatedThumb = await extractThumbnails(config, download.videoPath, 1, "webp"); 33 | download.thumbnailPath = generatedThumb.thumbnailPaths[0]; 34 | } 35 | 36 | const targetFolder = path.join( 37 | downloadStore.path, 38 | `${download.infoData.extractor.toLocaleLowerCase()}`, 39 | `${download.infoData.channel_id || download.infoData.uploader_id || download.infoData.uploader}`, 40 | `${videoId}`, 41 | ); 42 | await mkdir(targetFolder, { recursive: true }); 43 | 44 | const targetVideoPath = 45 | download.videoPath && path.join(targetFolder, `${videoId}${path.extname(download.videoPath)}`); 46 | 47 | if (download.videoPath && targetVideoPath) { 48 | await move(download.videoPath, targetVideoPath); 49 | } 50 | 51 | if (download.infoPath) { 52 | await move(download.infoPath, path.join(targetFolder, `${videoId}.info.json`)); 53 | } 54 | 55 | if (download.thumbnailPath) { 56 | await move(download.thumbnailPath, path.join(targetFolder, `${videoId}${path.extname(download.thumbnailPath)}`)); 57 | } 58 | 59 | return { 60 | generatedThumb, 61 | targetFolder, 62 | targetVideoPath, 63 | }; 64 | } 65 | 66 | export async function processVideoDownloadJob( 67 | config: Config, 68 | url: string, 69 | skipVideo: boolean = false, 70 | onProgress?: (info: DownloadInfo) => Promise, 71 | ) { 72 | if (config.download == undefined) { 73 | throw new Error(`Download config is not defined.`); 74 | } 75 | 76 | const download = await downloadYoutubeVideo(url, skipVideo, config.download, onProgress); 77 | 78 | const { generatedThumb, targetFolder, targetVideoPath } = await moveFilesToTargetFolder( 79 | config.mediaStores, 80 | config.download, 81 | download, 82 | skipVideo, 83 | ); 84 | 85 | // remove the temp dir 86 | rmSync(download.folder, { recursive: true }); 87 | 88 | if (generatedThumb && generatedThumb.tempDir) { 89 | rmSync(generatedThumb.tempDir, { recursive: true }); 90 | } 91 | console.log(`Downloaded content saved to ${targetFolder}`); 92 | 93 | return targetVideoPath; 94 | } 95 | -------------------------------------------------------------------------------- /src/jobs/queue.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import debug from "debug"; 3 | import { Queue } from "../entity/Queue.js"; 4 | import { Video } from "../entity/Video.js"; 5 | import { nip19, NostrEvent } from "nostr-tools"; 6 | 7 | const logger = debug("novia:queue"); 8 | 9 | export async function queueDownloadJob(rootEm: EntityManager, url: string) { 10 | logger("addToQueue function called"); 11 | try { 12 | const em = rootEm.fork(); 13 | 14 | const exitingJob = await em.findAll(Queue, { 15 | where: { 16 | $and: [{ type: "download" }, { status: "queued" }, { url }], 17 | }, 18 | }); 19 | 20 | if (exitingJob.length == 0) { 21 | const job = new Queue(); 22 | job.url = url; 23 | job.owner = "local"; 24 | job.type = "download"; 25 | em.persist(job); 26 | await em.flush(); 27 | console.log(`Added URL to queue: ${url}`); 28 | } 29 | } catch (error) { 30 | console.error("Error adding URL to queue:", error); 31 | } 32 | } 33 | 34 | export async function queueExtendMetaDataJob(rootEm: EntityManager, videoId: string) { 35 | const em = rootEm.fork(); 36 | try { 37 | const exitingJob = await em.findAll(Queue, { 38 | where: { 39 | $and: [{ type: "extendmetadata" }, { status: "queued" }, { url: videoId }], 40 | }, 41 | }); 42 | 43 | if (exitingJob.length == 0) { 44 | const q = new Queue(); 45 | q.owner = "local"; 46 | q.status = "queued"; 47 | q.type = "extendmetadata"; 48 | q.url = videoId; 49 | await em.persistAndFlush(q); 50 | } 51 | } catch (error) { 52 | console.error("Error adding extend meta job to queue:", error); 53 | } 54 | } 55 | 56 | export async function queueSHAUpdateJob(rootEm: EntityManager, videoId: string) { 57 | const em = rootEm.fork(); 58 | try { 59 | const exitingJob = await em.findAll(Queue, { 60 | where: { 61 | $and: [{ type: "createHashes" }, { status: "queued" }, { url: videoId }], 62 | }, 63 | }); 64 | 65 | if (exitingJob.length == 0) { 66 | const q = new Queue(); 67 | q.owner = "local"; 68 | q.status = "queued"; 69 | q.type = "createHashes"; 70 | q.url = videoId; 71 | await em.persistAndFlush(q); 72 | } 73 | } catch (error) { 74 | console.error("Error adding extend meta job to queue:", error); 75 | } 76 | } 77 | 78 | export async function queueNostrUpload(rootEm: EntityManager, videoId: string) { 79 | const em = rootEm.fork(); 80 | try { 81 | const exitingJob = await em.findAll(Queue, { 82 | where: { 83 | $and: [{ type: "nostrUpload" }, { status: "queued" }, { url: videoId }], 84 | }, 85 | }); 86 | 87 | if (exitingJob.length == 0) { 88 | const q = new Queue(); 89 | q.owner = "local"; 90 | q.status = "queued"; 91 | q.type = "nostrUpload"; 92 | q.url = videoId; 93 | await em.persistAndFlush(q); 94 | } 95 | } catch (error) { 96 | console.error("Error adding nostr upload job to queue:", error); 97 | } 98 | } 99 | 100 | export async function queueAllVideosForNostrUpload(rootEm: EntityManager) { 101 | const em = rootEm.fork(); 102 | 103 | const videos = await em.findAll(Video, { 104 | where: { 105 | $and: [ 106 | { thumbPath: { $ne: "" } }, 107 | { thumbSha256: { $ne: "" } }, 108 | { title: { $ne: "" } }, 109 | { event: { $eq: "" } }, // Skip when the event already has been published. 110 | ], 111 | }, 112 | }); 113 | 114 | // TODO check somehow which have already been uploaded!?!?! 115 | for (const video of videos) { 116 | await queueNostrUpload(rootEm, video.id); 117 | } 118 | } 119 | 120 | export async function queueMirrorJob(rootEm: EntityManager, nevent: string ) { 121 | logger("queueMirrorJob function called"); 122 | try { 123 | const em = rootEm.fork(); 124 | 125 | const exitingJob = await em.findAll(Queue, { 126 | where: { 127 | $and: [{ type: "mirrorVideo" }, { status: "queued" }, { url: nevent }], 128 | }, 129 | }); 130 | 131 | if (exitingJob.length == 0) { 132 | const job = new Queue(); 133 | job.url = nevent; 134 | job.owner = "local"; 135 | job.type = "mirrorVideo"; 136 | em.persist(job); 137 | await em.flush(); 138 | console.log(`Added mirror job to queue for: ${nevent}`); 139 | } 140 | } catch (error) { 141 | console.error("Error mirror job to queue:", error); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/jobs/results.ts: -------------------------------------------------------------------------------- 1 | import { nip19, NostrEvent } from "nostr-tools"; 2 | import { AddressPointer } from "nostr-tools/nip19"; 3 | import { Video } from "../entity/Video.js"; 4 | 5 | export type RecoverResult = { 6 | nevent: string; 7 | video: string; 8 | thumb: string; 9 | info: string; 10 | }; 11 | 12 | export type ArchiveResult = RecoverResult & { 13 | naddr: string; 14 | }; 15 | 16 | export const buildRecoverResult = (eventId: string, relays: string[], video: Video) => { 17 | return { 18 | nevent: nip19.neventEncode({ 19 | id: eventId, 20 | relays: relays, 21 | }), 22 | video: video.videoSha256, 23 | thumb: video.thumbSha256, 24 | info: video.infoSha256, 25 | } as RecoverResult; 26 | }; 27 | 28 | export const buildArchiveResult = (event: NostrEvent, relays: string[], video: Video) => { 29 | const identifier = event.tags.find((t) => t[0] == "d")![1]; 30 | 31 | return { 32 | ...buildRecoverResult(event.id, relays, video), 33 | naddr: nip19.naddrEncode({ 34 | identifier, 35 | pubkey: event.pubkey, 36 | relays: relays, 37 | kind: event.kind, 38 | } as AddressPointer), 39 | } as ArchiveResult; 40 | }; 41 | -------------------------------------------------------------------------------- /src/mikro-orm.config.ts: -------------------------------------------------------------------------------- 1 | import { Options, SqliteDriver } from "@mikro-orm/sqlite"; 2 | import { TsMorphMetadataProvider } from "@mikro-orm/reflection"; 3 | import { Queue } from "./entity/Queue.js"; 4 | import { Video } from "./entity/Video.js"; 5 | 6 | const ormConfig: Options = { 7 | // for simplicity, we use the SQLite database, as it's available pretty much everywhere 8 | driver: SqliteDriver, 9 | // dbName: "./novia.db", 10 | 11 | // folder-based discovery setup, using common filename suffix 12 | entities: [Queue, Video], // we will use the ts-morph reflection, an alternative to the default reflect-metadata provider 13 | // check the documentation for their differences: https://mikro-orm.io/docs/metadata-providers 14 | metadataProvider: TsMorphMetadataProvider, 15 | // enable debug mode to log SQL queries and discovery information 16 | debug: false, 17 | /* 18 | metadataCache: { 19 | enabled: true, 20 | adapter: GeneratedCacheAdapter, 21 | options: { 22 | data: { 23 | Queue: { 24 | _id: 0, 25 | properties: { 26 | id: { name: "id", kind: "scalar", primary: true, type: "number", array: false, runtimeType: "number" }, 27 | type: { 28 | name: "type", 29 | kind: "scalar", 30 | default: "download", 31 | getter: false, 32 | setter: false, 33 | type: '"download" | "extendmetadata" | "createHashes" | "nostrUpload"', 34 | array: false, 35 | runtimeType: '"download" | "extendmetadata" | "createHashes" | "nostrUpload"', 36 | }, 37 | url: { 38 | name: "url", 39 | kind: "scalar", 40 | getter: false, 41 | setter: false, 42 | type: "string", 43 | array: false, 44 | runtimeType: "string", 45 | }, 46 | owner: { 47 | name: "owner", 48 | kind: "scalar", 49 | default: "local", 50 | getter: false, 51 | setter: false, 52 | type: "string", 53 | runtimeType: "string", 54 | array: false, 55 | }, 56 | status: { 57 | name: "status", 58 | kind: "scalar", 59 | default: "queued", 60 | getter: false, 61 | setter: false, 62 | type: "string", 63 | runtimeType: '"queued" | "completed" | "failed" | "processing"', 64 | array: false, 65 | }, 66 | errorMessage: { 67 | name: "errorMessage", 68 | kind: "scalar", 69 | getter: false, 70 | setter: false, 71 | type: "string", 72 | default: "", 73 | runtimeType: "string", 74 | array: false, 75 | }, 76 | addedAt: { 77 | name: "addedAt", 78 | kind: "scalar", 79 | getter: false, 80 | setter: false, 81 | type: "Date", 82 | array: false, 83 | runtimeType: "Date", 84 | }, 85 | processedAt: { 86 | name: "processedAt", 87 | kind: "scalar", 88 | nullable: true, 89 | getter: false, 90 | setter: false, 91 | type: "Date", 92 | array: false, 93 | runtimeType: "Date", 94 | optional: true, 95 | }, 96 | }, 97 | primaryKeys: ["id"], 98 | filters: {}, 99 | hooks: {}, 100 | indexes: [], 101 | uniques: [], 102 | className: "Queue", 103 | path: "./dist/entity/Queue.js", 104 | name: "Queue", 105 | abstract: false, 106 | internal: true, 107 | constructorParams: [], 108 | toJsonParams: [], 109 | useCache: true, 110 | compositePK: false, 111 | simplePK: true, 112 | collection: "queue", 113 | }, 114 | Video: { 115 | _id: 1000, 116 | properties: { 117 | id: { name: "id", kind: "scalar", primary: true, type: "uuid", array: false, runtimeType: "string" }, 118 | store: { 119 | name: "store", 120 | kind: "scalar", 121 | getter: false, 122 | setter: false, 123 | type: "string", 124 | array: false, 125 | runtimeType: "string", 126 | }, 127 | videoPath: { 128 | name: "videoPath", 129 | kind: "scalar", 130 | getter: false, 131 | setter: false, 132 | type: "string", 133 | array: false, 134 | runtimeType: "string", 135 | }, 136 | videoSha256: { 137 | name: "videoSha256", 138 | kind: "scalar", 139 | getter: false, 140 | setter: false, 141 | type: "string", 142 | default: "", 143 | runtimeType: "string", 144 | array: false, 145 | }, 146 | infoPath: { 147 | name: "infoPath", 148 | kind: "scalar", 149 | getter: false, 150 | setter: false, 151 | type: "string", 152 | default: "", 153 | runtimeType: "string", 154 | array: false, 155 | }, 156 | infoSha256: { 157 | name: "infoSha256", 158 | kind: "scalar", 159 | getter: false, 160 | setter: false, 161 | type: "string", 162 | default: "", 163 | runtimeType: "string", 164 | array: false, 165 | }, 166 | thumbPath: { 167 | name: "thumbPath", 168 | kind: "scalar", 169 | getter: false, 170 | setter: false, 171 | type: "string", 172 | default: "", 173 | runtimeType: "string", 174 | array: false, 175 | }, 176 | thumbSha256: { 177 | name: "thumbSha256", 178 | kind: "scalar", 179 | getter: false, 180 | setter: false, 181 | type: "string", 182 | default: "", 183 | runtimeType: "string", 184 | array: false, 185 | }, 186 | addedAt: { 187 | name: "addedAt", 188 | kind: "scalar", 189 | getter: false, 190 | setter: false, 191 | type: "Date", 192 | runtimeType: "Date", 193 | array: false, 194 | }, 195 | externalId: { 196 | name: "externalId", 197 | kind: "scalar", 198 | getter: false, 199 | setter: false, 200 | type: "string", 201 | array: false, 202 | runtimeType: "string", 203 | }, 204 | source: { 205 | name: "source", 206 | kind: "scalar", 207 | getter: false, 208 | setter: false, 209 | type: "string", 210 | default: "", 211 | runtimeType: "string", 212 | array: false, 213 | }, 214 | category: { 215 | name: "category", 216 | kind: "scalar", 217 | type: "text[]", 218 | nullable: true, 219 | getter: false, 220 | setter: false, 221 | array: true, 222 | runtimeType: "string[]", 223 | optional: true, 224 | }, 225 | channelId: { 226 | name: "channelId", 227 | kind: "scalar", 228 | getter: false, 229 | setter: false, 230 | type: "string", 231 | array: false, 232 | runtimeType: "string", 233 | }, 234 | channelName: { 235 | name: "channelName", 236 | kind: "scalar", 237 | getter: false, 238 | setter: false, 239 | type: "string", 240 | default: "", 241 | runtimeType: "string", 242 | array: false, 243 | }, 244 | dateDownloaded: { 245 | name: "dateDownloaded", 246 | kind: "scalar", 247 | type: "bigint", 248 | getter: false, 249 | setter: false, 250 | array: false, 251 | runtimeType: "number", 252 | }, 253 | duration: { 254 | name: "duration", 255 | kind: "scalar", 256 | type: "integer", 257 | getter: false, 258 | setter: false, 259 | array: false, 260 | runtimeType: "number", 261 | }, 262 | description: { 263 | name: "description", 264 | kind: "scalar", 265 | type: "text", 266 | getter: false, 267 | setter: false, 268 | array: false, 269 | runtimeType: "string", 270 | }, 271 | mediaSize: { 272 | name: "mediaSize", 273 | kind: "scalar", 274 | type: "integer", 275 | getter: false, 276 | setter: false, 277 | array: false, 278 | runtimeType: "number", 279 | }, 280 | published: { 281 | name: "published", 282 | kind: "scalar", 283 | type: "date", 284 | getter: false, 285 | setter: false, 286 | array: false, 287 | runtimeType: "Date", 288 | }, 289 | tags: { 290 | name: "tags", 291 | kind: "scalar", 292 | type: "text[]", 293 | nullable: true, 294 | getter: false, 295 | setter: false, 296 | array: true, 297 | runtimeType: "string[]", 298 | optional: true, 299 | }, 300 | title: { 301 | name: "title", 302 | kind: "scalar", 303 | getter: false, 304 | setter: false, 305 | type: "string", 306 | array: false, 307 | runtimeType: "string", 308 | }, 309 | vidType: { 310 | name: "vidType", 311 | kind: "scalar", 312 | getter: false, 313 | setter: false, 314 | type: "string", 315 | array: false, 316 | runtimeType: "string", 317 | }, 318 | likeCount: { 319 | name: "likeCount", 320 | kind: "scalar", 321 | type: "integer", 322 | getter: false, 323 | setter: false, 324 | default: 0, 325 | array: false, 326 | runtimeType: "number", 327 | }, 328 | viewCount: { 329 | name: "viewCount", 330 | kind: "scalar", 331 | type: "integer", 332 | getter: false, 333 | setter: false, 334 | default: 0, 335 | array: false, 336 | runtimeType: "number", 337 | }, 338 | ageLimit: { 339 | name: "ageLimit", 340 | kind: "scalar", 341 | type: "integer", 342 | getter: false, 343 | setter: false, 344 | default: 0, 345 | array: false, 346 | runtimeType: "number", 347 | }, 348 | width: { 349 | name: "width", 350 | kind: "scalar", 351 | type: "integer", 352 | getter: false, 353 | setter: false, 354 | default: 0, 355 | array: false, 356 | runtimeType: "number", 357 | }, 358 | height: { 359 | name: "height", 360 | kind: "scalar", 361 | type: "integer", 362 | getter: false, 363 | setter: false, 364 | default: 0, 365 | array: false, 366 | runtimeType: "number", 367 | }, 368 | event: { 369 | name: "event", 370 | kind: "scalar", 371 | getter: false, 372 | setter: false, 373 | type: "string", 374 | default: "", 375 | runtimeType: "string", 376 | array: false, 377 | }, 378 | }, 379 | primaryKeys: ["id"], 380 | filters: {}, 381 | hooks: {}, 382 | indexes: [], 383 | uniques: [], 384 | className: "Video", 385 | path: "./dist/entity/Video.js", 386 | name: "Video", 387 | abstract: false, 388 | internal: true, 389 | constructorParams: [], 390 | toJsonParams: [], 391 | useCache: true, 392 | compositePK: false, 393 | simplePK: true, 394 | collection: "video", 395 | }, 396 | }, 397 | }, 398 | },*/ 399 | }; 400 | 401 | export default ormConfig; 402 | -------------------------------------------------------------------------------- /src/server.ts: -------------------------------------------------------------------------------- 1 | import { FilterQuery } from "@mikro-orm/core"; 2 | import express, { NextFunction, Request, Response } from "express"; 3 | import { access, constants, createReadStream, stat } from "fs"; 4 | import { Video } from "./entity/Video.js"; 5 | import { EntityManager } from "@mikro-orm/sqlite"; 6 | import debug from "debug"; 7 | import { MediaStore } from "./types.js"; 8 | import path from "path"; 9 | import { promisify } from "util"; 10 | import { queueDownloadJob } from "./jobs/queue.js"; 11 | import cors from "cors"; // Import the CORS package 12 | 13 | // Promisify fs functions for better async/await handling 14 | const accessAsync = promisify(access); 15 | const statAsync = promisify(stat); 16 | 17 | const logger = debug("novia:server"); 18 | 19 | export function startLocalServer(rootEm: EntityManager, mediaStores: MediaStore[], port: number = 9090) { 20 | const app = express(); 21 | 22 | // Middleware to parse JSON requests 23 | app.use(express.json()); 24 | 25 | // Apply CORS middleware with default settings (allow all origins) 26 | app.use(cors()); 27 | 28 | app.get("/add/:url", async (req: Request, res: Response, next: NextFunction) => { 29 | const { url } = req.params; 30 | await queueDownloadJob(rootEm, url); 31 | res.status(200); 32 | res.send(); 33 | }); 34 | 35 | app.get("/videos", async (req: Request, res: Response, next: NextFunction) => { 36 | const { search, store } = req.query; 37 | const queries: FilterQueryGET /videos
230 | GET /:hash (retreive a blob with a SHA256 hash)
231 | GET /add/:url (add a new video download, url need to be uri encoded)
232 | 233 |

Fetch URL

234 | 235 | 236 | 237 | 238 | `; 239 | res.send(htmlContent); 240 | }); 241 | 242 | app.use((err: Error, req: Request, res: Response, next: NextFunction) => { 243 | console.error("Unhandled error:", err.stack); 244 | res.status(500).json({ error: "Something went wrong!" }); 245 | }); 246 | 247 | app.listen(port, () => { 248 | console.log(`Server is running on port ${port}`); 249 | }); 250 | } 251 | -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- 1 | export interface MediaStore { 2 | id: string; 3 | type: "local" | "blossom"; 4 | path?: string; 5 | url?: string; 6 | watch?: boolean; 7 | } 8 | 9 | export interface DownloadConfig { 10 | enabled: boolean; 11 | ytdlpPath: string; 12 | ytdlpCookies?: string; 13 | tempPath: string; 14 | targetStoreId: string; 15 | secret?: boolean; 16 | } 17 | 18 | export interface BlossomConfig { 19 | url: string; 20 | maxUploadSizeMB: number; 21 | cleanUpMaxAgeDays: number; 22 | cleanUpKeepSizeUnderMB: number; 23 | } 24 | 25 | export interface PublishConfig { 26 | enabled: boolean; 27 | key: string; 28 | thumbnailUpload: string[]; 29 | videoUpload: BlossomConfig[]; 30 | relays: string[]; 31 | secret?: boolean; 32 | autoUpload?: { 33 | enabled: boolean; 34 | maxVideoSizeMB: number; 35 | }; 36 | } 37 | 38 | export interface ServerConfig { 39 | enabled: boolean; 40 | port: number; 41 | } 42 | 43 | export interface FetchConfig { 44 | enabled: boolean; 45 | match?: string[]; 46 | relays?: string[]; 47 | blossom?: string[]; 48 | } 49 | 50 | export interface Config { 51 | mediaStores: MediaStore[]; 52 | database: string; 53 | download?: DownloadConfig; 54 | publish?: PublishConfig; 55 | server?: ServerConfig; 56 | fetch?: FetchConfig; 57 | } 58 | -------------------------------------------------------------------------------- /src/utils/array.ts: -------------------------------------------------------------------------------- 1 | export function unique(arr: T[]): T[] { 2 | return Array.from(new Set(arr)); 3 | } 4 | -------------------------------------------------------------------------------- /src/utils/ffmpeg.ts: -------------------------------------------------------------------------------- 1 | import { exec } from "child_process"; 2 | import { mkdirSync } from "fs"; 3 | import path from "path"; 4 | import { promisify } from "util"; 5 | import { createTempDir } from "./utils.js"; 6 | import { DownloadConfig } from "../types.js"; 7 | 8 | type MetaData = { 9 | streams: { 10 | index: number; 11 | codec_name: string; 12 | codec_long_name: string; 13 | profile?: string; 14 | codec_type: string; 15 | codec_tag_string: string; 16 | codec_tag: string; 17 | width: number; 18 | height: number; 19 | coded_width?: number; 20 | coded_height?: number; 21 | closed_captions: number; 22 | film_grain: number; 23 | has_b_frames: number; 24 | sample_aspect_ratio?: string; 25 | display_aspect_ratio?: string; 26 | pix_fmt?: string; 27 | is_avc?: string; 28 | duration: string; 29 | bit_rate: string; 30 | bits_per_raw_sample: string; 31 | }[]; 32 | format: { 33 | filename: string; 34 | nb_streams: number; 35 | nb_programs: number; 36 | format_name: string; 37 | format_long_name: string; 38 | start_time: string; 39 | duration: string; 40 | size: string; 41 | bit_rate: string; 42 | probe_score: number; 43 | tags: { 44 | major_brand: string; 45 | minor_version: string; 46 | compatible_brands: string; 47 | creation_time: string; 48 | }; 49 | }; 50 | }; 51 | 52 | const execAsync = promisify(exec); 53 | 54 | export async function extractVideoMetadata(videoUrl: string): Promise { 55 | try { 56 | // Construct the command to extract metadata using ffprobe 57 | const command = `ffprobe -v quiet -print_format json -show_format -show_streams "${videoUrl}"`; 58 | 59 | // Execute the command 60 | const { stdout, stderr } = await execAsync(command); 61 | 62 | // Check for any errors 63 | if (stderr) { 64 | throw new Error(stderr); 65 | } 66 | 67 | // Parse the JSON output 68 | const metadata = JSON.parse(stdout); 69 | 70 | return metadata; 71 | } catch (error: any) { 72 | throw new Error(`Failed to extract video metadata: ${error.message}`); 73 | } 74 | } 75 | 76 | export type ThumbnailContent = { 77 | thumbnailPaths: string[]; 78 | tempDir: string; 79 | }; 80 | 81 | export async function extractThumbnails( 82 | config: DownloadConfig, 83 | videoUrl: string, 84 | numFrames: number = 1, 85 | outputFormat: 'jpg'|'png'|'webp' = "jpg", 86 | options: string = "", 87 | ): Promise { 88 | try { 89 | const tempDir = createTempDir(config.tempPath); 90 | 91 | // Construct the command to extract thumbnails using ffmpeg 92 | const filenameTemplate = "thumbnail%02d." + outputFormat; 93 | const command = `ffmpeg -v error -i "${videoUrl}" -vf "thumbnail" -frames:v ${numFrames} -ss 00:00:01 -vf fps=1/4 ${options} "${path.join(tempDir, filenameTemplate)}"`; 94 | 95 | // Execute the command 96 | const { stdout, stderr } = await execAsync(command); 97 | 98 | // Check for any errors 99 | if (stderr) { 100 | throw new Error(stderr); 101 | } 102 | 103 | // Generate array of thumbnail file paths 104 | const thumbnailPaths: string[] = []; 105 | for (let i = 1; i <= numFrames; i++) { 106 | thumbnailPaths.push(path.join(tempDir, `thumbnail${i.toString().padStart(2, "0")}.${outputFormat}`)); 107 | } 108 | 109 | return { thumbnailPaths, tempDir }; 110 | } catch (error: any) { 111 | throw new Error(`Failed to extract thumbnails: ${error.message}`); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/utils/mapvideodata.ts: -------------------------------------------------------------------------------- 1 | import { nip19, NostrEvent } from "nostr-tools"; 2 | import { HORIZONZAL_VIDEO_KIND } from "../dvm/types.js"; 3 | 4 | export type VideoFormat = "widescreen" | "vertical"; 5 | 6 | export type VideoData = { 7 | eventId: string; 8 | archivedByNpub: string; 9 | identifier: string; 10 | x?: string; 11 | url: string | undefined; 12 | published_at: number; 13 | published_year: string; 14 | image: string | undefined; 15 | author?: string; 16 | source?: string; 17 | title: string | undefined; 18 | duration: number; 19 | description?: string; 20 | size: number; 21 | originalUrl?: string; 22 | dim?: string; 23 | tags: string[]; 24 | format: VideoFormat; 25 | relayUrl?: string; 26 | contentWarning?: string; 27 | language?: string; 28 | info?: string; 29 | }; 30 | 31 | export const getTagValue = (ev: NostrEvent, tagKey: string, postfix?: string): string | undefined => { 32 | const tag = ev.tags.find((t) => t[0] == tagKey && (postfix == undefined || postfix == t[2])); 33 | if (!tag) return undefined; 34 | return tag[1]; 35 | }; 36 | 37 | export function mapVideoData(ev: NostrEvent): VideoData { 38 | const pub = parseInt(getTagValue(ev, "published_at") || "0", 10); 39 | 40 | //`dim ${video.width}x${video.height}`, 41 | 42 | const iMetaTags = ev.tags.filter((t) => t[0] == "imeta"); 43 | 44 | let dim = undefined; 45 | if (iMetaTags.length > 0) { 46 | const dimField = iMetaTags[0].find((s) => s.startsWith("dim ")); 47 | if (dimField) { 48 | dim = dimField.substring(4); 49 | } 50 | } 51 | 52 | return { 53 | eventId: ev.id, 54 | archivedByNpub: nip19.npubEncode(ev.pubkey), 55 | identifier: getTagValue(ev, "d") || ev.id, 56 | x: getTagValue(ev, "x"), 57 | url: getTagValue(ev, "url"), // todo add imeta parsing 58 | published_at: pub, 59 | published_year: `${new Date(pub * 1000).getUTCFullYear()}`, 60 | image: getTagValue(ev, "image"), // todo add imeta parsing 61 | title: getTagValue(ev, "title"), 62 | duration: parseInt(getTagValue(ev, "duration") || "0", 10), 63 | source: getTagValue(ev, "c", "source"), 64 | author: getTagValue(ev, "c", "author"), 65 | description: getTagValue(ev, "summary"), 66 | size: parseInt(getTagValue(ev, "size") || "0", 10), 67 | originalUrl: getTagValue(ev, "r"), 68 | tags: ev.tags.filter((t) => t[0] == "t").map((t) => t[1]), 69 | format: ev.kind == HORIZONZAL_VIDEO_KIND ? "widescreen" : "vertical", 70 | contentWarning: getTagValue(ev, "content-warning"), 71 | language: getTagValue(ev, "l", "ISO-639-1"), 72 | info: getTagValue(ev, "info"), 73 | dim, 74 | }; 75 | } 76 | -------------------------------------------------------------------------------- /src/utils/move.ts: -------------------------------------------------------------------------------- 1 | import { copyFile, mkdir, rename, unlink } from "fs/promises"; 2 | 3 | export async function move(sourceFilePath: string, targetFilePath: string) { 4 | try { 5 | await rename(sourceFilePath, targetFilePath); 6 | } catch (err) { 7 | const error = err as NodeJS.ErrnoException; 8 | if (error.code === "EXDEV") { 9 | // Cross-device error, so copy and delete 10 | await copyFile(sourceFilePath, targetFilePath); 11 | await unlink(sourceFilePath); 12 | } else { 13 | // Rethrow other errors 14 | throw error; 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/utils/utils.ts: -------------------------------------------------------------------------------- 1 | import { readFile, stat, writeFile } from "fs/promises"; 2 | import path, { parse, resolve } from "path"; 3 | import { MediaStore } from "../types.js"; 4 | import fsx from "fs-extra"; 5 | import { exec } from "child_process"; 6 | import util from "util"; 7 | import debug from "debug"; 8 | import { Video } from "../entity/Video.js"; 9 | import { unique } from "./array.js"; 10 | import { existsSync, mkdirSync } from "fs"; 11 | 12 | const logger = debug("novia:utils"); 13 | 14 | // Promisify exec for use with async/await 15 | const execPromise = util.promisify(exec); 16 | 17 | /** 18 | * Removes specified fields from each object in a JSON array and saves it back to the same file. 19 | * @param {string} filePath - The path to the JSON file. 20 | * @param {string[]} fieldsToRemove - An array of field names to remove from each object. 21 | */ 22 | export async function removeFieldsFromJson(filePath: string, fieldsToRemove: string[]) { 23 | try { 24 | // Resolve the absolute path 25 | const absolutePath = resolve(filePath); 26 | 27 | // Read the JSON file 28 | const data = await readFile(absolutePath, "utf-8"); 29 | 30 | // Parse the JSON data 31 | let jsonObject = JSON.parse(data); 32 | 33 | // Iterate over each object and remove the specified fields 34 | 35 | fieldsToRemove.forEach((field) => { 36 | if (field in jsonObject) { 37 | delete jsonObject[field]; 38 | } 39 | }); 40 | 41 | // Convert the modified array back to JSON string with indentation for readability 42 | const modifiedData = JSON.stringify(jsonObject, null, 2); 43 | 44 | // Write the modified JSON back to the same file 45 | await writeFile(absolutePath, modifiedData, "utf-8"); 46 | } catch (error) { 47 | console.error(`Error processing the file`, error); 48 | } 49 | } 50 | 51 | export function getFileStats(filePath: string) { 52 | try { 53 | const jsonFilePath = resolve(filePath); 54 | 55 | const stats = fsx.statSync(jsonFilePath, {}); 56 | 57 | return stats; 58 | } catch (error) { 59 | console.error("Error getting file stats:", error); 60 | } 61 | } 62 | 63 | export function findStoreForFilePath(stores: MediaStore[], filePath: string): MediaStore | undefined { 64 | return stores.find((st) => st.type == "local" && st.path && filePath.startsWith(st.path.replace(/^\.\//, ""))); 65 | } 66 | 67 | export function findFullPathsForVideo(video: Video, stores: MediaStore[]) { 68 | const store = stores.find((st) => st.id == video.store); 69 | if (!store || !store.path) { 70 | return undefined; // store not found 71 | } 72 | 73 | const videoPath = path.join(store.path, video.videoPath); 74 | const thumbPath = path.join(store.path, video.thumbPath); 75 | const infoPath = path.join(store.path, video.infoPath); 76 | 77 | return { videoPath, thumbPath, infoPath }; 78 | } 79 | 80 | export function removeStorePathPrefixFromFilePath(store: MediaStore, filePath: string): string { 81 | if (!store.path) return filePath; 82 | 83 | // remove the ./ prefix for relative paths 84 | let storePath = store.path.replace(/^\.\//, ""); 85 | 86 | // Add a trailing slash if needed 87 | storePath = storePath.endsWith("/") ? storePath : `${storePath}/`; 88 | 89 | if (filePath.startsWith(storePath)) { 90 | return filePath.substring(storePath.length); 91 | } 92 | return filePath; 93 | } 94 | 95 | /** 96 | * Formats a duration from seconds to "m:ss" format. 97 | * 98 | * @param {number} totalSeconds - The total duration in seconds. 99 | * @returns {string} The formatted duration as "m:ss". 100 | */ 101 | export function formatDuration(totalSeconds: number) { 102 | const minutes = Math.floor(totalSeconds / 60); 103 | const seconds = totalSeconds % 60; 104 | 105 | // Pad seconds with leading zero if less than 10 106 | const paddedSeconds = seconds.toString().padStart(2, "0"); 107 | 108 | return `${minutes}m${paddedSeconds}s`; 109 | } 110 | 111 | /** 112 | * Computes the SHA-256 hash of a file using the 'shasum -a 256' command. 113 | * 114 | * @param {string} filePath - The absolute or relative path to the file. 115 | * @returns {Promise} - A promise that resolves to the SHA-256 hash in hexadecimal format. 116 | */ 117 | export async function computeSha256(filePath: string) { 118 | try { 119 | logger(`shasum -a 256 "${filePath}"`); 120 | // Execute the shasum command 121 | const { stdout, stderr } = await execPromise(`shasum -a 256 "${filePath}"`); 122 | 123 | // Check for errors in stderr 124 | if (stderr) { 125 | throw new Error(stderr); 126 | } 127 | 128 | // The output format of shasum -a 256 is: 129 | const hash = stdout.split(" ")[0].trim(); 130 | 131 | // Validate the hash format (should be 64 hexadecimal characters) 132 | if (!/^[a-fA-F0-9]{64}$/.test(hash)) { 133 | throw new Error("Invalid SHA-256 hash format received."); 134 | } 135 | 136 | logger("Found hash: " + hash); 137 | 138 | return hash; 139 | } catch (error) { 140 | // Handle errors (e.g., file not found, shasum not installed) 141 | throw new Error(`Failed to compute SHA-256 hash: ${(error as Error).message}`); 142 | } 143 | } 144 | 145 | /** 146 | * Returns the MIME type based on the file extension. 147 | * 148 | * @param path - The file path or name. 149 | * @returns The corresponding MIME type as a string. 150 | */ 151 | export function getMimeTypeByPath(path: string): string { 152 | const { ext } = parse(path.toLowerCase()); 153 | 154 | switch (ext) { 155 | // Image MIME types 156 | case ".jpg": 157 | case ".jpeg": 158 | case ".image": 159 | return "image/jpeg"; 160 | case ".png": 161 | return "image/png"; 162 | case ".gif": 163 | return "image/gif"; 164 | case ".bmp": 165 | return "image/bmp"; 166 | case ".svg": 167 | return "image/svg+xml"; 168 | case ".webp": 169 | return "image/webp"; 170 | case ".tiff": 171 | case ".tif": 172 | return "image/tiff"; 173 | case ".ico": 174 | return "image/vnd.microsoft.icon"; 175 | case ".heic": 176 | return "image/heic"; 177 | case ".avif": 178 | return "image/avif"; 179 | 180 | // Video MIME types 181 | case ".mp4": 182 | return "video/mp4"; 183 | case ".avi": 184 | return "video/x-msvideo"; 185 | case ".mov": 186 | return "video/quicktime"; 187 | case ".wmv": 188 | return "video/x-ms-wmv"; 189 | case ".flv": 190 | return "video/x-flv"; 191 | case ".mkv": 192 | return "video/x-matroska"; 193 | case ".webm": 194 | return "video/webm"; 195 | case ".mpeg": 196 | case ".mpg": 197 | return "video/mpeg"; 198 | case ".3gp": 199 | return "video/3gpp"; 200 | case ".3g2": 201 | return "video/3gpp2"; 202 | case ".m4v": 203 | return "video/x-m4v"; 204 | 205 | case ".json": 206 | return "application/json"; 207 | case ".info.json": 208 | return "application/json"; 209 | 210 | default: 211 | return "application/octet-stream"; // Default binary type 212 | } 213 | } 214 | 215 | export const now = () => Math.floor(new Date().getTime() / 1000); 216 | 217 | export const mergeServers = (...aBunchOfServers: string[]) => { 218 | return unique(aBunchOfServers.filter((s) => !!s).map((s) => s.replace(/\/$/, ""))); 219 | }; 220 | 221 | export function createTempDir(tempBaseDir?: string): string { 222 | // Create a temporary directory with a random name 223 | const tempDir = path.join(tempBaseDir || process.cwd(), "temp_" + Math.random().toString(36).substring(2)); 224 | if (!existsSync(tempDir)) { 225 | mkdirSync(tempDir); 226 | } 227 | return tempDir; 228 | } 229 | -------------------------------------------------------------------------------- /src/utils/ytdlp.ts: -------------------------------------------------------------------------------- 1 | import { spawn } from "child_process"; 2 | import debug from "debug"; 3 | import { readFileSync, readdirSync } from "fs"; 4 | import path from "path"; 5 | import { DownloadConfig } from "../types.js"; 6 | import debounce from "lodash/debounce.js"; 7 | import { createTempDir } from "./utils.js"; 8 | 9 | const logger = debug("novia:ytdlp"); 10 | 11 | export interface YoutubeVideoInfo { 12 | id: string; 13 | title: string; 14 | thumbnail: string; 15 | description: string; 16 | channel_id: string; 17 | channel_url: string; 18 | duration: number; 19 | view_count: number; 20 | age_limit: number; 21 | webpage_url: string; 22 | categories: string[]; 23 | tags: string[]; 24 | playable_in_embed: boolean; 25 | live_status: string; 26 | release_timestamp: number; 27 | _format_sort_fields: string[]; 28 | comment_count: number; 29 | like_count: number; 30 | channel: string; 31 | channel_follower_count: number; 32 | channel_is_verified: boolean; 33 | uploader: string; 34 | uploader_id: string; 35 | uploader_url: string; 36 | upload_date: string; 37 | timestamp: number; 38 | availability: string; 39 | webpage_url_basename: string; 40 | webpage_url_domain: string; 41 | extractor: string; 42 | extractor_key: string; 43 | display_id: string; 44 | fulltitle: string; 45 | duration_string: string; 46 | release_date: string; 47 | release_year: number; 48 | is_live: boolean; 49 | was_live: boolean; 50 | epoch: number; 51 | format: string; 52 | format_id: string; 53 | ext: string; 54 | protocol: string; 55 | language: string; 56 | format_note: string; 57 | filesize_approx: number; 58 | tbr: number; 59 | width: number; 60 | height: number; 61 | resolution: string; 62 | fps: number; 63 | dynamic_range: string; 64 | vcodec: string; 65 | vbr: number; 66 | aspect_ratio: number; 67 | acodec: string; 68 | abr: number; 69 | asr: number; 70 | audio_channels: number; 71 | _type: string; 72 | _version: { 73 | version: string; 74 | release_git_head: string; 75 | repository: string; 76 | }; 77 | } 78 | 79 | export type VideoContent = { 80 | folder: string; 81 | videoPath?: string; 82 | infoData?: YoutubeVideoInfo; 83 | infoPath?: string; 84 | thumbnailPath?: string; 85 | }; 86 | 87 | /** 88 | * Finds a file based on the base name and desired extensions. 89 | * If not found, searches for any file with the desired extensions. 90 | * 91 | * @param {string} baseName - The base name of the file (without extension). 92 | * @param {string[]} preferredExtensions - Array of preferred extensions to look for first. 93 | * @param {string[]} fallbackExtensions - Array of fallback extensions to search if preferred not found. 94 | * @param {string[]} files - Array of all filenames in the directory. 95 | * @returns {string|null} - The found filename or null if not found. 96 | */ 97 | function findFile( 98 | baseName: string | undefined, 99 | preferredExtensions: string[], 100 | fallbackExtensions: string[], 101 | files: string[], 102 | ) { 103 | logger(baseName, preferredExtensions, fallbackExtensions, files); 104 | if (baseName) { 105 | // Attempt to find the file with the same base name and preferred extensions 106 | for (const ext of preferredExtensions) { 107 | const fileName = `${baseName}${ext}`; 108 | if (files.includes(fileName)) { 109 | return fileName; 110 | } 111 | } 112 | } 113 | // If not found, search for any file with the fallback extensions 114 | const possibleFallbackFiles = files.filter((file) => fallbackExtensions.some((ext) => file.endsWith(ext))); 115 | logger(possibleFallbackFiles); 116 | if (possibleFallbackFiles.length == 1) { 117 | return possibleFallbackFiles[0]; 118 | } 119 | return undefined; 120 | } 121 | 122 | export type AnalysisResult = { 123 | folder: string; 124 | videoPath: string | undefined; 125 | infoData: YoutubeVideoInfo | undefined; 126 | infoPath: string | undefined; 127 | thumbnailPath: string | undefined; 128 | }; 129 | 130 | export async function analyzeVideoFolder( 131 | folder: string, 132 | skipVideo = false, 133 | exactMetaMatch = true, 134 | videoFileName?: string, 135 | ): Promise { 136 | logger("analyzeVideoFolder", folder, skipVideo); 137 | // Read the temp directory to find the first .mp4, .json, and .webp files 138 | const files = readdirSync(folder); 139 | logger(files); 140 | const videoFile = files.find( 141 | (file) => 142 | (videoFileName == undefined || file.startsWith(videoFileName)) && 143 | (file.endsWith(".mp4") || file.endsWith(".webm")), 144 | ); 145 | 146 | if (!skipVideo && !videoFile) { 147 | throw new Error("Video not found in folder: " + folder); 148 | } 149 | const videoPath = videoFile ? path.join(folder, videoFile) : undefined; 150 | // 2. Extract the base name (filename without extension) 151 | const baseName = videoFile ? path.parse(videoFile).name : undefined; 152 | logger(`Base name extracted: ${baseName}`); 153 | 154 | const infoFile = findFile( 155 | baseName, 156 | [".info.json", ".json"], // Preferred extensions with base name 157 | exactMetaMatch ? [] : [".info.json", ".json"], // Fallback extensions (same in this case) 158 | files, 159 | ); 160 | 161 | const infoPath = infoFile && path.join(folder, infoFile); 162 | 163 | const thumbnailFile = findFile( 164 | baseName, 165 | [".webp", ".jpg", ".image"], // Preferred extensions with base name 166 | exactMetaMatch ? [] : [".webp", ".jpg", ".image"], // Fallback extensions (same in this case) 167 | files, 168 | ); 169 | 170 | const thumbnailPath = thumbnailFile && path.join(folder, thumbnailFile); 171 | 172 | // Read the JSON data from the info file 173 | const infoData = infoPath ? (JSON.parse(readFileSync(infoPath, "utf-8")) as YoutubeVideoInfo) : undefined; 174 | 175 | return { 176 | folder, 177 | videoPath, 178 | infoData, 179 | infoPath, 180 | thumbnailPath, 181 | }; 182 | } 183 | 184 | // Define an interface for the extracted download information 185 | export interface DownloadInfo { 186 | percentage: number; 187 | totalSizeMiB: number; 188 | speedMiBps: number; 189 | etaSeconds: number; // ETA in seconds 190 | fragCurrent: number; 191 | fragTotal: number; 192 | } 193 | 194 | /** 195 | * Parses a console output string to extract download information. 196 | * @param input - The console output string to parse, potentially containing multiple lines. 197 | * @returns An array of DownloadInfo objects with extracted numbers. 198 | */ 199 | function parseDownloadInfo(line: string): DownloadInfo | undefined { 200 | // Regular expression to match and capture the required parts of the string 201 | const regex = 202 | /^\[download\]\s+(\d+(?:\.\d+)?)%\s+of\s+~\s+(\d+(?:\.\d+)?)MiB\s+at\s+(\d+(?:\.\d+)?)MiB\/s\s+ETA\s+(\d{2}):(\d{2})\s+\(frag\s+(\d+)\/(\d+)\)(?:\s+\+\d+ms)?$/; 203 | 204 | const results: DownloadInfo[] = []; 205 | 206 | const trimmedLine = line.trim(); 207 | if (trimmedLine.length === 0) return; 208 | 209 | const match = trimmedLine.match(regex); 210 | if (!match) { 211 | // If the line doesn't match the expected format, ignore it 212 | return; 213 | } 214 | 215 | // Destructure the captured groups from the regex match 216 | const [ 217 | _fullMatch, 218 | percentageStr, 219 | totalSizeStr, 220 | speedStr, 221 | etaMinutesStr, 222 | etaSecondsStr, 223 | fragCurrentStr, 224 | fragTotalStr, 225 | ] = match; 226 | 227 | // Parse the captured strings into appropriate numerical types 228 | const percentage = parseFloat(percentageStr); 229 | const totalSizeMiB = parseFloat(totalSizeStr); 230 | const speedMiBps = parseFloat(speedStr); 231 | const etaMinutes = parseInt(etaMinutesStr, 10); 232 | const etaSeconds = parseInt(etaSecondsStr, 10); 233 | const fragCurrent = parseInt(fragCurrentStr, 10); 234 | const fragTotal = parseInt(fragTotalStr, 10); 235 | 236 | // Convert ETA to total seconds 237 | const totalEtaSeconds = etaMinutes * 60 + etaSeconds; 238 | 239 | return { 240 | percentage, 241 | totalSizeMiB, 242 | speedMiBps, 243 | etaSeconds: totalEtaSeconds, 244 | fragCurrent, 245 | fragTotal, 246 | }; 247 | } 248 | 249 | export async function downloadYoutubeVideo( 250 | videoUrl: string, 251 | skipVideo = false, 252 | config: DownloadConfig, 253 | onProgress?: (info: DownloadInfo) => Promise, 254 | ): Promise { 255 | 256 | const publishProgress = onProgress && debounce(onProgress, 5000, { 257 | maxWait: 0 258 | }); 259 | 260 | return new Promise((resolve, reject) => { 261 | try { 262 | const tempDir = createTempDir(config.tempPath); 263 | logger(`Temporary directory created at: ${tempDir}`); 264 | 265 | // Define yt-dlp arguments 266 | const args = [ 267 | "-f", 268 | "bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best", 269 | "--write-info-json", 270 | "--write-thumbnail", 271 | videoUrl, 272 | ]; 273 | 274 | if (skipVideo) { 275 | args.push("--skip-download"); 276 | } 277 | 278 | if (config.ytdlpCookies) { 279 | args.push("--cookies"); 280 | args.push(config.ytdlpCookies); 281 | } 282 | 283 | // Optionally, use the absolute path to yt-dlp 284 | const ytDlpPath = config.ytdlpPath || "yt-dlp"; 285 | logger(`Spawning yt-dlp '${config.ytdlpPath}' with args: ${args.join(" ")}`); 286 | 287 | // Spawn the yt-dlp process 288 | const ytDlp = spawn(ytDlpPath, args, { cwd: tempDir, shell: false }); 289 | 290 | let stdout = ""; 291 | let stderr = ""; 292 | 293 | ytDlp.stdout.on("data", async (data) => { 294 | // logger(data.toString()); 295 | const line = data.toString(); 296 | 297 | if (publishProgress) { 298 | const downloadInfo = parseDownloadInfo(line); 299 | if (downloadInfo) { 300 | await publishProgress(downloadInfo); 301 | } 302 | } 303 | 304 | 305 | stdout += line; 306 | }); 307 | 308 | ytDlp.stderr.on("data", (data) => { 309 | stderr += data.toString(); 310 | }); 311 | 312 | ytDlp.on("error", (err) => { 313 | logger(`Error spawning yt-dlp: ${err.message}`); 314 | reject(new Error(`Failed to spawn yt-dlp: ${err.message}`)); 315 | }); 316 | 317 | ytDlp.on("close", async (code) => { 318 | logger(`yt-dlp exited with code ${code}`); 319 | if (code !== 0) { 320 | logger(`yt-dlp stderr: ${stderr}`); 321 | return reject(new Error(stderr || `yt-dlp exited with code ${code}`)); 322 | } 323 | 324 | try { 325 | const result = await analyzeVideoFolder(tempDir, skipVideo, false); 326 | logger(`Video analysis result: ${JSON.stringify(result)}`); 327 | resolve(result); 328 | } catch (analyzeError) { 329 | logger(`Error analyzing video folder: ${(analyzeError as Error).message}`); 330 | reject(analyzeError); 331 | } 332 | }); 333 | } catch (error: any) { 334 | logger(`Unexpected error: ${error.message}`); 335 | reject(new Error(`Failed to download video: ${error.message}`)); 336 | } 337 | }); 338 | } 339 | -------------------------------------------------------------------------------- /src/validation/validateHashes.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { queueSHAUpdateJob } from "../jobs/queue.js"; 3 | import { Video } from "../entity/Video.js"; 4 | import { MediaStore } from "../types.js"; 5 | 6 | export const validateMissingSha256Hashes = async ( 7 | rootEm: EntityManager, 8 | stores: MediaStore[] 9 | ) => { 10 | const em = rootEm.fork(); 11 | 12 | // Retrieve all Video entries from the database 13 | const allVideos = await em.findAll(Video, { 14 | where: { 15 | $or: [ 16 | { 17 | $and: [{ videoSha256: { $eq: "" } }, { videoPath: { $ne: "" } }], 18 | }, 19 | { $and: [{ infoSha256: { $eq: "" } }, { infoPath: { $ne: "" } }] }, 20 | { 21 | $and: [{ thumbSha256: { $eq: "" } }, { thumbPath: { $ne: "" } }], 22 | }, 23 | ], 24 | }, 25 | }); 26 | 27 | if (allVideos.length === 0) { 28 | console.log(`No missing hashes to compute.`); 29 | return; 30 | } 31 | 32 | console.log( 33 | `Starting computing of hashes for (${allVideos.length} files) ...` 34 | ); 35 | 36 | for (const video of allVideos) { 37 | await queueSHAUpdateJob(rootEm, video.id); 38 | } 39 | }; 40 | -------------------------------------------------------------------------------- /src/validation/validateMetaData.ts: -------------------------------------------------------------------------------- 1 | import { EntityManager } from "@mikro-orm/sqlite"; 2 | import { MediaStore } from "../types.js"; 3 | import { Video } from "../entity/Video.js"; 4 | import { queueExtendMetaDataJob } from "../jobs/queue.js"; 5 | 6 | export const validateMetaData = async ( 7 | rootEm: EntityManager, 8 | stores: MediaStore[] 9 | ) => { 10 | const em = rootEm.fork(); 11 | console.log("Checking for missing metadata..."); 12 | 13 | // Retrieve all Video entries from the database 14 | const metaMissing = await em.findAll(Video, { 15 | where: { 16 | $or: [{ infoPath: { $eq: "" } }, { thumbPath: { $eq: "" } }], 17 | }, 18 | }); 19 | 20 | if (metaMissing.length === 0) { 21 | console.log("No videos found in the database to clean."); 22 | return; 23 | } 24 | 25 | for (const video of metaMissing) { 26 | await queueExtendMetaDataJob(rootEm, video.id); 27 | } 28 | await em.flush(); 29 | }; 30 | -------------------------------------------------------------------------------- /src/video-indexer.ts: -------------------------------------------------------------------------------- 1 | // src/index.ts 2 | import chokidar from "chokidar"; 3 | import path from "path"; 4 | import { Video } from "./entity/Video.js"; // Ensure the correct path and extension 5 | import { EntityManager } from "@mikro-orm/sqlite"; 6 | import { analyzeVideoFolder } from "./utils/ytdlp.js"; 7 | import { findStoreForFilePath, getFileStats, removeStorePathPrefixFromFilePath } from "./utils/utils.js"; 8 | import { MediaStore } from "./types.js"; 9 | import debug from "debug"; 10 | import { queueExtendMetaDataJob, queueSHAUpdateJob } from "./jobs/queue.js"; 11 | import debounce from "lodash/debounce.js"; 12 | import { readdir } from "fs/promises"; 13 | 14 | const logger = debug("novia:indexer"); 15 | 16 | export async function updateVideoMetaData(vid: Video, filePath: string, videoStore: MediaStore) { 17 | logger("updateVideoMetaData", vid, filePath, videoStore); 18 | const videoStats = await getFileStats(filePath); 19 | const relativePath = videoStore ? removeStorePathPrefixFromFilePath(videoStore, filePath) : filePath; 20 | 21 | const videoFolder = path.dirname(filePath); 22 | const videoFileName = path.parse(filePath).name; 23 | const meta = await analyzeVideoFolder(videoFolder, false, true, videoFileName); 24 | 25 | const inf = meta.infoData; 26 | 27 | // Create new Video entity and save to database 28 | vid.videoPath = relativePath; 29 | vid.store = videoStore?.id || ""; 30 | vid.externalId = inf?.id || ""; 31 | vid.source = inf?.extractor.toLocaleLowerCase() || ""; 32 | vid.category = inf?.categories; 33 | vid.channelId = inf?.channel_id || inf?.uploader_id || inf?.uploader || ""; 34 | vid.channelName = inf?.uploader || ""; 35 | vid.dateDownloaded = videoStats !== undefined ? videoStats.ctime.getTime() : Date.now(); 36 | vid.description = inf?.description || ""; 37 | vid.mediaSize = videoStats?.size || 0; 38 | vid.duration = inf?.duration || 0; 39 | vid.published = new Date(inf?.timestamp || 0); 40 | vid.tags = inf?.tags; 41 | vid.title = inf?.title || ""; 42 | vid.vidType = inf?._type || "video"; 43 | vid.likeCount = inf?.like_count || 0; 44 | vid.viewCount = inf?.view_count || 0; 45 | vid.ageLimit = inf?.age_limit || 0; 46 | vid.width = inf?.width || 0; // TODO for twitch clips the sizes are wrong 47 | vid.height = inf?.height || 0; 48 | vid.language = inf?.language || ""; 49 | 50 | vid.thumbPath = 51 | meta.thumbnailPath && videoStore ? removeStorePathPrefixFromFilePath(videoStore, meta.thumbnailPath) : ""; 52 | vid.infoPath = meta.infoPath && videoStore ? removeStorePathPrefixFromFilePath(videoStore, meta.infoPath) : ""; 53 | } 54 | 55 | /** 56 | * Processes a given file: 57 | * - Checks if it's an mp4 file 58 | * - Checks if it's already in the database 59 | * - If not, adds it to the database 60 | * @param filePath Absolute path to the file 61 | */ 62 | export const processFile = async ( 63 | rootEm: EntityManager, 64 | stores: MediaStore[], 65 | filePath: string, 66 | triggerJobs = true, 67 | /* Callback that allows callers to modify the new video object 68 | before it is persisted to the database */ 69 | onUpdate?: (video: Video) => void, 70 | ) => { 71 | try { 72 | if ( 73 | path.extname(filePath).toLowerCase().endsWith(".mp4") || 74 | path.extname(filePath).toLowerCase().endsWith(".webm") 75 | ) { 76 | // Check if the file already exists in the database 77 | const em = rootEm.fork(); 78 | 79 | const store = findStoreForFilePath(stores, filePath); 80 | if (!store) { 81 | throw new Error(`Store for video ${filePath} not found.`); 82 | } 83 | 84 | const relativePath = store ? removeStorePathPrefixFromFilePath(store, filePath) : filePath; 85 | 86 | const existingVideo = await em.findOne(Video, { 87 | videoPath: relativePath, 88 | }); 89 | 90 | if (!existingVideo) { 91 | const vid = new Video(); 92 | 93 | await updateVideoMetaData(vid, filePath, store); 94 | if (onUpdate) { 95 | onUpdate(vid); 96 | } 97 | await em.persistAndFlush(vid); 98 | 99 | if (triggerJobs) { 100 | if (!vid.thumbPath || !vid.infoPath) { 101 | await queueExtendMetaDataJob(rootEm, vid.id); 102 | } else { 103 | await queueSHAUpdateJob(rootEm, vid.id); 104 | } 105 | } 106 | 107 | console.log(`Added new mp4 file to database: ${filePath}`); 108 | return vid; 109 | } else { 110 | // TODO archive job fails here without any error or notice 111 | logger(`File already exists in database: ${filePath}`); 112 | } 113 | } 114 | } catch (error) { 115 | console.error(`Error processing file ${filePath}:`, error); 116 | } 117 | }; 118 | 119 | // Initialize and run the application 120 | export const setupWatcher = async (rootEm: EntityManager, storesToWatch: MediaStore[]) => { 121 | try { 122 | const foldersToWatch = storesToWatch.filter((st) => !!st.path && st.watch).map((st) => st.path as string); 123 | 124 | if (foldersToWatch.length === 0) { 125 | console.warn("No folders to watch."); 126 | return; 127 | } 128 | 129 | // Define glob patterns for relevant file types 130 | const fileGlobs = foldersToWatch.map((folder) => path.join(folder, "*.{mp4,m4a}")); 131 | 132 | // Initialize chokidar watcher with optimized settings 133 | const watcher = chokidar.watch(fileGlobs, { 134 | persistent: true, 135 | ignoreInitial: true, // Ignore existing files 136 | depth: 0, // Watch only the specified directories, not subdirectories 137 | awaitWriteFinish: { 138 | stabilityThreshold: 2000, // Time in ms for a file to be considered fully written 139 | pollInterval: 100, // Interval in ms for polling 140 | }, 141 | ignored: (filePath) => { 142 | const normalizedPath = path.normalize(filePath); 143 | const segments = normalizedPath.split(path.sep); 144 | 145 | // Ignore hidden directories and unwanted file types 146 | return segments.some((segment) => segment.startsWith(".")) || !/\.(mp4|m4a)$/.test(filePath); 147 | }, 148 | // Optionally, use polling to reduce file handle usage 149 | // usePolling: true, 150 | // interval: 500, 151 | }); 152 | 153 | // Debounce the processFile function to prevent rapid successive calls 154 | const debouncedProcessFile = debounce((filePath: string) => { 155 | processFile(rootEm, storesToWatch, filePath); 156 | }, 500); // Adjust the delay as needed 157 | 158 | // Listen for 'add' events (new files) 159 | watcher.on("add", (filePath) => { 160 | console.log(`New file detected: ${filePath}`); 161 | debouncedProcessFile(filePath); 162 | }); 163 | 164 | // Handle watcher errors 165 | watcher.on("error", (error) => { 166 | console.error("Error watching folders:", error); 167 | }); 168 | 169 | console.log(`Watching for new video files in: ${foldersToWatch.join(", ")} ...`); 170 | } catch (error) { 171 | console.error("Error initializing application:", error); 172 | process.exit(1); 173 | } 174 | }; 175 | 176 | /** 177 | * Recursively scans a directory and processes all mp4 files, 178 | * ignoring hidden directories (directories starting with '.') 179 | * @param dirPath Absolute path to the directory 180 | */ 181 | export const scanDirectory = async (rootEm: EntityManager, stores: MediaStore[], dirPath: string) => { 182 | try { 183 | const entries = await readdir(dirPath, { withFileTypes: true }); 184 | 185 | for (const entry of entries) { 186 | // Ignore hidden directories 187 | if (entry.isDirectory() && entry.name.startsWith(".")) { 188 | continue; 189 | } 190 | 191 | const fullPath = path.join(dirPath, entry.name); 192 | 193 | if (entry.isDirectory()) { 194 | // Recursively scan subdirectories 195 | await scanDirectory(rootEm, stores, fullPath); 196 | } else if (entry.isFile()) { 197 | // Process the file 198 | await processFile(rootEm, stores, fullPath); 199 | } 200 | } 201 | } catch (error) { 202 | console.error(`Error scanning directory ${dirPath}:`, error); 203 | } 204 | }; 205 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "NodeNext", 4 | "moduleResolution": "NodeNext", 5 | "target": "ESNext", 6 | "strict": true, 7 | "outDir": "dist", 8 | "declaration": true, 9 | "experimentalDecorators": true, 10 | "esModuleInterop": true 11 | }, 12 | "include": ["./src/**/*.ts"] 13 | } 14 | --------------------------------------------------------------------------------