├── start.sh ├── stop.sh ├── .prettierrc ├── images ├── breads-1.avif ├── breads-2.avif ├── breads-3.avif ├── breads-4.avif ├── breads-5.avif ├── breads-6.avif ├── breads-7.avif ├── breads-8.avif ├── breads-9.avif ├── cats-1.avif ├── cats-10.avif ├── cats-2.avif ├── cats-3.avif ├── cats-4.avif ├── cats-5.avif ├── cats-6.avif ├── cats-7.avif ├── cats-8.avif ├── cats-9.avif ├── coffee-1.avif ├── coffee-2.avif ├── coffee-3.avif ├── coffee-4.avif ├── coffee-5.avif ├── coffee-6.avif ├── coffee-7.avif ├── coffee-8.avif ├── coffee-9.avif ├── dogs-1.avif ├── dogs-10.avif ├── dogs-2.avif ├── dogs-3.avif ├── dogs-4.avif ├── dogs-5.avif ├── dogs-6.avif ├── dogs-7.avif ├── dogs-8.avif ├── dogs-9.avif ├── breads-10.avif ├── coffee-10.avif ├── flowers-1.avif ├── flowers-10.avif ├── flowers-2.avif ├── flowers-3.avif ├── flowers-4.avif ├── flowers-5.avif ├── flowers-6.avif ├── flowers-7.avif ├── flowers-8.avif ├── flowers-9.avif ├── aeroplane-1.avif ├── aeroplane-10.avif ├── aeroplane-2.avif ├── aeroplane-3.avif ├── aeroplane-4.avif ├── aeroplane-5.avif ├── aeroplane-6.avif ├── aeroplane-7.avif ├── aeroplane-8.avif ├── aeroplane-9.avif ├── buildings-1.avif ├── buildings-10.avif ├── buildings-2.avif ├── buildings-3.avif ├── buildings-4.avif ├── buildings-5.avif ├── buildings-6.avif ├── buildings-7.avif ├── buildings-8.avif ├── buildings-9.avif ├── cat-sref 29007055.webp ├── cat-sref 29338216.webp ├── cat-sref 29652225.webp ├── cat-sref 29885588.webp └── cat-sref 30172192.webp ├── tsconfig.build.json ├── nest-cli.json ├── Dockerfile ├── test ├── jest-e2e.json └── app.e2e-spec.ts ├── src ├── common.ts ├── file │ ├── chat.controller.ts │ ├── file.controller.ts │ ├── file.module.ts │ ├── file-scan.entity.ts │ ├── chat.service.ts │ └── file.service.ts ├── remote │ ├── embedding.service.ts │ ├── pg-vector-store.service.ts │ └── extract-image.service.ts ├── main.ts ├── page │ └── page.controller.ts ├── task │ └── task.service.ts ├── app.module.ts └── config │ └── config.service.ts ├── tsconfig.json ├── docker-compose.yml ├── .github └── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md ├── .eslintrc.js ├── sql └── project.sql ├── .gitignore ├── package.json ├── README_CN.md ├── README.md ├── .env.prod.example ├── .env.example ├── views └── index.hbs └── LICENSE /start.sh: -------------------------------------------------------------------------------- 1 | docker-compose up -d -------------------------------------------------------------------------------- /stop.sh: -------------------------------------------------------------------------------- 1 | docker-compose stop -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": true, 3 | "trailingComma": "all" 4 | } -------------------------------------------------------------------------------- /images/breads-1.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-1.avif -------------------------------------------------------------------------------- /images/breads-2.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-2.avif -------------------------------------------------------------------------------- /images/breads-3.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-3.avif -------------------------------------------------------------------------------- /images/breads-4.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-4.avif -------------------------------------------------------------------------------- /images/breads-5.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-5.avif -------------------------------------------------------------------------------- /images/breads-6.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-6.avif -------------------------------------------------------------------------------- /images/breads-7.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-7.avif -------------------------------------------------------------------------------- /images/breads-8.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-8.avif -------------------------------------------------------------------------------- /images/breads-9.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-9.avif -------------------------------------------------------------------------------- /images/cats-1.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-1.avif -------------------------------------------------------------------------------- /images/cats-10.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-10.avif -------------------------------------------------------------------------------- /images/cats-2.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-2.avif -------------------------------------------------------------------------------- /images/cats-3.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-3.avif -------------------------------------------------------------------------------- /images/cats-4.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-4.avif -------------------------------------------------------------------------------- /images/cats-5.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-5.avif -------------------------------------------------------------------------------- /images/cats-6.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-6.avif -------------------------------------------------------------------------------- /images/cats-7.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-7.avif -------------------------------------------------------------------------------- /images/cats-8.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-8.avif -------------------------------------------------------------------------------- /images/cats-9.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cats-9.avif -------------------------------------------------------------------------------- /images/coffee-1.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-1.avif -------------------------------------------------------------------------------- /images/coffee-2.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-2.avif -------------------------------------------------------------------------------- /images/coffee-3.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-3.avif -------------------------------------------------------------------------------- /images/coffee-4.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-4.avif -------------------------------------------------------------------------------- /images/coffee-5.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-5.avif -------------------------------------------------------------------------------- /images/coffee-6.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-6.avif -------------------------------------------------------------------------------- /images/coffee-7.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-7.avif -------------------------------------------------------------------------------- /images/coffee-8.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-8.avif -------------------------------------------------------------------------------- /images/coffee-9.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-9.avif -------------------------------------------------------------------------------- /images/dogs-1.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-1.avif -------------------------------------------------------------------------------- /images/dogs-10.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-10.avif -------------------------------------------------------------------------------- /images/dogs-2.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-2.avif -------------------------------------------------------------------------------- /images/dogs-3.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-3.avif -------------------------------------------------------------------------------- /images/dogs-4.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-4.avif -------------------------------------------------------------------------------- /images/dogs-5.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-5.avif -------------------------------------------------------------------------------- /images/dogs-6.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-6.avif -------------------------------------------------------------------------------- /images/dogs-7.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-7.avif -------------------------------------------------------------------------------- /images/dogs-8.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-8.avif -------------------------------------------------------------------------------- /images/dogs-9.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/dogs-9.avif -------------------------------------------------------------------------------- /images/breads-10.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/breads-10.avif -------------------------------------------------------------------------------- /images/coffee-10.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/coffee-10.avif -------------------------------------------------------------------------------- /images/flowers-1.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-1.avif -------------------------------------------------------------------------------- /images/flowers-10.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-10.avif -------------------------------------------------------------------------------- /images/flowers-2.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-2.avif -------------------------------------------------------------------------------- /images/flowers-3.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-3.avif -------------------------------------------------------------------------------- /images/flowers-4.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-4.avif -------------------------------------------------------------------------------- /images/flowers-5.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-5.avif -------------------------------------------------------------------------------- /images/flowers-6.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-6.avif -------------------------------------------------------------------------------- /images/flowers-7.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-7.avif -------------------------------------------------------------------------------- /images/flowers-8.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-8.avif -------------------------------------------------------------------------------- /images/flowers-9.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/flowers-9.avif -------------------------------------------------------------------------------- /images/aeroplane-1.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-1.avif -------------------------------------------------------------------------------- /images/aeroplane-10.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-10.avif -------------------------------------------------------------------------------- /images/aeroplane-2.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-2.avif -------------------------------------------------------------------------------- /images/aeroplane-3.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-3.avif -------------------------------------------------------------------------------- /images/aeroplane-4.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-4.avif -------------------------------------------------------------------------------- /images/aeroplane-5.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-5.avif -------------------------------------------------------------------------------- /images/aeroplane-6.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-6.avif -------------------------------------------------------------------------------- /images/aeroplane-7.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-7.avif -------------------------------------------------------------------------------- /images/aeroplane-8.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-8.avif -------------------------------------------------------------------------------- /images/aeroplane-9.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/aeroplane-9.avif -------------------------------------------------------------------------------- /images/buildings-1.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-1.avif -------------------------------------------------------------------------------- /images/buildings-10.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-10.avif -------------------------------------------------------------------------------- /images/buildings-2.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-2.avif -------------------------------------------------------------------------------- /images/buildings-3.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-3.avif -------------------------------------------------------------------------------- /images/buildings-4.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-4.avif -------------------------------------------------------------------------------- /images/buildings-5.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-5.avif -------------------------------------------------------------------------------- /images/buildings-6.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-6.avif -------------------------------------------------------------------------------- /images/buildings-7.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-7.avif -------------------------------------------------------------------------------- /images/buildings-8.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-8.avif -------------------------------------------------------------------------------- /images/buildings-9.avif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/buildings-9.avif -------------------------------------------------------------------------------- /images/cat-sref 29007055.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cat-sref 29007055.webp -------------------------------------------------------------------------------- /images/cat-sref 29338216.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cat-sref 29338216.webp -------------------------------------------------------------------------------- /images/cat-sref 29652225.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cat-sref 29652225.webp -------------------------------------------------------------------------------- /images/cat-sref 29885588.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cat-sref 29885588.webp -------------------------------------------------------------------------------- /images/cat-sref 30172192.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gcui-art/album-ai/HEAD/images/cat-sref 30172192.webp -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "exclude": ["node_modules", "test", "dist", "**/*spec.ts"] 4 | } 5 | -------------------------------------------------------------------------------- /nest-cli.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://json.schemastore.org/nest-cli", 3 | "collection": "@nestjs/schematics", 4 | "sourceRoot": "src", 5 | "compilerOptions": { 6 | "deleteOutDir": true 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:18.20.4-slim 2 | 3 | RUN mkdir -p /app 4 | 5 | WORKDIR /app 6 | 7 | COPY . ./ 8 | 9 | RUN rm -rf /app/node_modules && rm -rf /app/dist && npm install && npm run build 10 | 11 | CMD npm run start:prod 12 | 13 | EXPOSE 8080 14 | -------------------------------------------------------------------------------- /test/jest-e2e.json: -------------------------------------------------------------------------------- 1 | { 2 | "moduleFileExtensions": ["js", "json", "ts", "a", "node"], 3 | "rootDir": ".", 4 | "testEnvironment": "node", 5 | "testRegex": ".e2e-spec.ts$", 6 | "transform": { 7 | "^.+\\.(t|j)s$": "ts-jest" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/common.ts: -------------------------------------------------------------------------------- 1 | export enum FileStatus { 2 | Init = 'Init', 3 | Extract = 'Extract', 4 | Embedding = 'Embedding', 5 | } 6 | 7 | export const replacePlaceholders = (template: string, ...args: any[]) => { 8 | return template.replace(/{(\d+)}/g, (match, index) => { 9 | return typeof args[index] !== 'undefined' ? args[index] : match; 10 | }); 11 | }; 12 | -------------------------------------------------------------------------------- /src/file/chat.controller.ts: -------------------------------------------------------------------------------- 1 | import { Body, Controller, Post } from '@nestjs/common'; 2 | import { ChatService } from './chat.service'; 3 | 4 | @Controller('/api/v1') 5 | export class ChatController { 6 | constructor(private readonly chatService: ChatService) {} 7 | 8 | @Post('/chat') 9 | public async chat(@Body() input: { text: string }) { 10 | return this.chatService.chat(input); 11 | } 12 | } -------------------------------------------------------------------------------- /src/remote/embedding.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | import { openai } from '../app.module'; 3 | 4 | @Injectable() 5 | export class EmbeddingService { 6 | public async embeddingByOpenAI(text: string) { 7 | if (process.env.EMBEDDING_PROVIDER == 'openai') { 8 | const response = await openai.embeddings.create({ 9 | model: process.env.EMBEDDING_PROVIDER_MODEL, 10 | input: text, 11 | }); 12 | return response.data[0].embedding; 13 | } else { 14 | throw new Error(`no support. provider=${process.env.EMBEDDING_PROVIDER}`); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "declaration": true, 5 | "removeComments": true, 6 | "emitDecoratorMetadata": true, 7 | "experimentalDecorators": true, 8 | "allowSyntheticDefaultImports": true, 9 | "target": "ES2021", 10 | "sourceMap": true, 11 | "outDir": "./dist", 12 | "baseUrl": "./", 13 | "incremental": true, 14 | "skipLibCheck": true, 15 | "strictNullChecks": false, 16 | "noImplicitAny": false, 17 | "strictBindCallApply": false, 18 | "forceConsistentCasingInFileNames": false, 19 | "noFallthroughCasesInSwitch": false 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/main.ts: -------------------------------------------------------------------------------- 1 | import { NestFactory } from '@nestjs/core'; 2 | import * as process from 'node:process'; 3 | import { AppModule } from './app.module.js'; 4 | import { NestExpressApplication } from '@nestjs/platform-express'; 5 | import { join } from 'path'; 6 | 7 | async function bootstrap() { 8 | const app = await NestFactory.create(AppModule); 9 | app.useStaticAssets(join(__dirname, '..', 'public')); 10 | app.setBaseViewsDir(join(__dirname, '..', 'views')); 11 | app.setViewEngine('hbs'); 12 | await app.init(); 13 | await app.listen(process.env.SERVER_PORT); 14 | } 15 | 16 | bootstrap().then(() => { 17 | console.log('success launched!'); 18 | }); 19 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | postgres: 3 | image: pgvector/pgvector:pg16 4 | environment: 5 | POSTGRES_USER: root 6 | POSTGRES_PASSWORD: 123456 7 | POSTGRES_DB: album 8 | ports: 9 | - "5432:5432" 10 | volumes: 11 | - ./sql/project.sql:/docker-entrypoint-initdb.d/init.sql 12 | - /tmp/postgres/data:/var/lib/postgresql/data 13 | album-ai: 14 | depends_on: 15 | - postgres 16 | build: 17 | context: . 18 | dockerfile: Dockerfile 19 | restart: on-failure 20 | tty: true 21 | ports: 22 | - "8080:8080" 23 | env_file: 24 | - .env.prod 25 | volumes: 26 | - ./images:/home/images 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: '@typescript-eslint/parser', 3 | parserOptions: { 4 | project: 'tsconfig.json', 5 | tsconfigRootDir: __dirname, 6 | sourceType: 'module', 7 | }, 8 | plugins: ['@typescript-eslint/eslint-plugin'], 9 | extends: [ 10 | 'plugin:@typescript-eslint/recommended', 11 | 'plugin:prettier/recommended', 12 | ], 13 | root: true, 14 | env: { 15 | node: true, 16 | jest: true, 17 | }, 18 | ignorePatterns: ['.eslintrc.js'], 19 | rules: { 20 | '@typescript-eslint/interface-name-prefix': 'off', 21 | '@typescript-eslint/explicit-function-return-type': 'off', 22 | '@typescript-eslint/explicit-module-boundary-types': 'off', 23 | '@typescript-eslint/no-explicit-any': 'off', 24 | }, 25 | }; 26 | -------------------------------------------------------------------------------- /src/page/page.controller.ts: -------------------------------------------------------------------------------- 1 | import { Get, Controller, Render, Query, Logger } from '@nestjs/common'; 2 | import { ChatService } from '../file/chat.service'; 3 | 4 | @Controller('/') 5 | export class PageController { 6 | private readonly logger = new Logger(PageController.name); 7 | 8 | constructor(private readonly chatService: ChatService) {} 9 | 10 | @Get('/') 11 | @Render('index') 12 | async root(@Query('query') query?: string) { 13 | this.logger.log('query:', query); 14 | if (query) { 15 | const { content } = await this.chatService.chat({ text: query }); 16 | this.logger.log('Answer:', content); 17 | return { 18 | answer: content, 19 | query: query || null, 20 | }; 21 | } 22 | return { 23 | query: query || null, 24 | }; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /sql/project.sql: -------------------------------------------------------------------------------- 1 | \c album; 2 | 3 | CREATE TABLE file_album 4 | ( 5 | f_id BIGSERIAL PRIMARY KEY, 6 | file_name VARCHAR(255), 7 | path text, 8 | crc VARCHAR(64), 9 | status VARCHAR(32), 10 | content_type varchar(64), 11 | size integer, 12 | desc_ai text, 13 | desc_custom text, 14 | extra text, 15 | created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP, 16 | updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP 17 | ); 18 | 19 | create unique index uk_crc on file_album (crc); 20 | 21 | CREATE EXTENSION vector; 22 | 23 | create table file_vector_index 24 | ( 25 | id bigserial primary key, 26 | text varchar not null, 27 | metadata jsonb, 28 | embedding vector(3072) 29 | ); 30 | 31 | -------------------------------------------------------------------------------- /src/file/file.controller.ts: -------------------------------------------------------------------------------- 1 | import { Controller, Get, Param, Query, Res } from '@nestjs/common'; 2 | import { FileService } from './file.service.js'; 3 | import { Response } from 'express'; 4 | 5 | @Controller('/api/v1/file') 6 | export class FileController { 7 | constructor(private readonly fileService: FileService) {} 8 | 9 | @Get('/search') 10 | public async search(@Query('query') query: string) { 11 | return this.fileService.searchDetail(query); 12 | } 13 | 14 | @Get('/:fId/download') 15 | public async download(@Param('fId') fId: string, @Res() response: Response) { 16 | const file = await this.fileService.findFile(fId); 17 | response.download(file.path, (err) => { 18 | if (!err) { 19 | return; 20 | } 21 | response.send({ 22 | code: 500, 23 | msg: 'download error', 24 | }); 25 | }); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # compiled output 2 | /dist 3 | /node_modules 4 | /build 5 | 6 | # Logs 7 | logs 8 | *.log 9 | npm-debug.log* 10 | pnpm-debug.log* 11 | yarn-debug.log* 12 | yarn-error.log* 13 | lerna-debug.log* 14 | 15 | # OS 16 | .DS_Store 17 | 18 | # Tests 19 | /coverage 20 | /.nyc_output 21 | 22 | # IDEs and editors 23 | /.idea 24 | .project 25 | .classpath 26 | .c9/ 27 | *.launch 28 | .settings/ 29 | *.sublime-workspace 30 | 31 | # IDE - VSCode 32 | .vscode/* 33 | !.vscode/settings.json 34 | !.vscode/tasks.json 35 | !.vscode/launch.json 36 | !.vscode/extensions.json 37 | 38 | # dotenv environment variable files 39 | .env 40 | .env.development.local 41 | .env.test.local 42 | .env.production.local 43 | .env.local 44 | .env.prod 45 | 46 | # temp directory 47 | .temp 48 | .tmp 49 | 50 | # Runtime data 51 | pids 52 | *.pid 53 | *.seed 54 | *.pid.lock 55 | 56 | # Diagnostic reports (https://nodejs.org/api/report.html) 57 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 58 | -------------------------------------------------------------------------------- /src/file/file.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { TypeOrmModule } from '@nestjs/typeorm'; 3 | import { FileAlbum } from './file-scan.entity.js'; 4 | import { FileController } from './file.controller.js'; 5 | import { FileService } from './file.service.js'; 6 | import { EmbeddingService } from '../remote/embedding.service'; 7 | import { ExtractImageService } from '../remote/extract-image.service'; 8 | import { PgVectorStoreService } from '../remote/pg-vector-store.service'; 9 | import { ChatService } from './chat.service'; 10 | import { ChatController } from './chat.controller'; 11 | 12 | @Module({ 13 | imports: [TypeOrmModule.forFeature([FileAlbum])], 14 | controllers: [FileController, ChatController], 15 | providers: [ 16 | FileService, 17 | EmbeddingService, 18 | ExtractImageService, 19 | PgVectorStoreService, 20 | ChatService, 21 | ], 22 | exports: [FileService], 23 | }) 24 | export class FileModule {} 25 | -------------------------------------------------------------------------------- /src/task/task.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, Logger } from '@nestjs/common'; 2 | import { Cron } from '@nestjs/schedule'; 3 | import { FileService } from '../file/file.service.js'; 4 | import { configService } from '../config/config.service'; 5 | 6 | @Injectable() 7 | export class TaskService { 8 | private readonly logger = new Logger(TaskService.name); 9 | 10 | constructor(private readonly fileService: FileService) {} 11 | 12 | @Cron('0 */1 * * * *') 13 | async triggerScanFile() { 14 | this.logger.log( 15 | `Starting scan files, path: ${configService.getAlbumDir()}`, 16 | ); 17 | await this.fileService.scanFile(); 18 | } 19 | 20 | @Cron('0 */1 * * * *') 21 | async triggerExtractDesc() { 22 | this.logger.log('Starting extract desc'); 23 | await this.fileService.extractDesc(); 24 | } 25 | 26 | @Cron('0 */1 * * * *') 27 | async triggerEmbedding() { 28 | this.logger.log('Starting embedding'); 29 | await this.fileService.embedding(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /test/app.e2e-spec.ts: -------------------------------------------------------------------------------- 1 | import { Test, TestingModule } from '@nestjs/testing'; 2 | import { INestApplication } from '@nestjs/common'; 3 | import * as request from 'supertest'; 4 | import { AppModule } from './../src/app.module'; 5 | import { FileService } from '../src/file/file.service'; 6 | 7 | describe('AppController (e2e)', () => { 8 | let app: INestApplication; 9 | let fileService: FileService; 10 | 11 | beforeEach(async () => { 12 | const moduleFixture: TestingModule = await Test.createTestingModule({ 13 | imports: [AppModule], 14 | }).compile(); 15 | 16 | app = moduleFixture.createNestApplication(); 17 | await app.init(); 18 | 19 | fileService = moduleFixture.get(FileService); 20 | }); 21 | 22 | // it('/ (GET)', () => { 23 | // return request(app.getHttpServer()) 24 | // .get('/') 25 | // .expect(200) 26 | // .expect('Hello World!'); 27 | // }); 28 | 29 | it('test scan', async () => { 30 | await fileService.scanFile(); 31 | console.log('-------------'); 32 | }); 33 | }); 34 | -------------------------------------------------------------------------------- /src/file/file-scan.entity.ts: -------------------------------------------------------------------------------- 1 | import { 2 | Entity, 3 | PrimaryGeneratedColumn, 4 | Column, 5 | CreateDateColumn, 6 | UpdateDateColumn, 7 | } from 'typeorm'; 8 | 9 | @Entity('file_album') 10 | export class FileAlbum { 11 | @PrimaryGeneratedColumn({ name: 'f_id' }) 12 | fId: bigint; 13 | 14 | @Column({ name: 'file_name', type: 'varchar', length: 255 }) 15 | fileName: string; 16 | 17 | @Column({ name: 'path', type: 'text' }) 18 | path: string; 19 | 20 | @Column({ name: 'crc', type: 'varchar', length: 64 }) 21 | crc: string; 22 | 23 | @Column({ name: 'status', type: 'varchar', length: 32 }) 24 | status: string; 25 | 26 | @Column({ name: 'content_type', type: 'varchar', length: 64 }) 27 | contentType: string; 28 | 29 | @Column({ name: 'size', type: 'int' }) 30 | size: number; 31 | 32 | @Column({ name: 'desc_ai', type: 'text' }) 33 | descAi: string; 34 | 35 | @Column({ name: 'desc_custom', type: 'text' }) 36 | descCustom: string; 37 | 38 | @Column({ name: 'extra', type: 'text' }) 39 | extra: string; 40 | 41 | @CreateDateColumn({ name: 'created_at', type: 'timestamp with time zone' }) 42 | createdAt: Date; 43 | 44 | @UpdateDateColumn({ name: 'updated_at', type: 'timestamp with time zone' }) 45 | updatedAt: Date; 46 | } 47 | -------------------------------------------------------------------------------- /src/app.module.ts: -------------------------------------------------------------------------------- 1 | import { Module } from '@nestjs/common'; 2 | import { ConfigModule } from '@nestjs/config'; 3 | import { TypeOrmModule } from '@nestjs/typeorm'; 4 | import { ScheduleModule } from '@nestjs/schedule'; 5 | import { configService } from './config/config.service.js'; 6 | import { FileModule } from './file/file.module.js'; 7 | import { TaskService } from './task/task.service.js'; 8 | import OpenAI from 'openai'; 9 | import * as process from 'node:process'; 10 | import { EmbeddingService } from './remote/embedding.service'; 11 | import { HttpsProxyAgent } from 'https-proxy-agent'; 12 | import { ExtractImageService } from './remote/extract-image.service'; 13 | import { PgVectorStoreService } from './remote/pg-vector-store.service'; 14 | import { PageController } from './page/page.controller'; 15 | import { ChatService } from './file/chat.service'; 16 | import Anthropic from '@anthropic-ai/sdk'; 17 | 18 | @Module({ 19 | controllers: [PageController], 20 | imports: [ 21 | ConfigModule.forRoot({ isGlobal: true }), 22 | TypeOrmModule.forRoot(configService.getTypeOrmConfig()), 23 | ScheduleModule.forRoot(), 24 | FileModule, 25 | ], 26 | providers: [ 27 | TaskService, 28 | EmbeddingService, 29 | ExtractImageService, 30 | PgVectorStoreService, 31 | ChatService, 32 | ], 33 | }) 34 | export class AppModule {} 35 | 36 | export const openai = new OpenAI({ 37 | apiKey: process.env.OPENAI_API_KEY, 38 | httpAgent: process.env.PROXY_URL 39 | ? new HttpsProxyAgent(process.env.PROXY_URL) 40 | : undefined, 41 | }); 42 | 43 | export const anthropicClient = new Anthropic({ 44 | apiKey: process.env.ANTHROPIC_API_KEY, 45 | httpAgent: process.env.PROXY_URL 46 | ? new HttpsProxyAgent(process.env.PROXY_URL) 47 | : undefined, 48 | }); 49 | -------------------------------------------------------------------------------- /src/config/config.service.ts: -------------------------------------------------------------------------------- 1 | import { TypeOrmModuleOptions } from '@nestjs/typeorm'; 2 | import { FileAlbum } from '../file/file-scan.entity.js'; 3 | import { PoolConfig } from 'pg'; 4 | import { DistanceStrategy } from '@langchain/community/dist/vectorstores/pgvector'; 5 | 6 | export class ConfigService { 7 | public getTypeOrmConfig(): TypeOrmModuleOptions { 8 | return { 9 | type: 'postgres', 10 | host: process.env.POSTGRES_HOST, 11 | port: parseInt(process.env.POSTGRES_PORT), 12 | username: process.env.POSTGRES_USER, 13 | password: process.env.POSTGRES_PASSWORD, 14 | database: process.env.POSTGRES_DATABASE, 15 | entities: [FileAlbum], 16 | }; 17 | } 18 | 19 | public getPGvectorConfig() { 20 | return { 21 | postgresConnectionOptions: { 22 | type: 'postgres', 23 | host: process.env.POSTGRES_HOST, 24 | port: parseInt(process.env.POSTGRES_PORT), 25 | user: process.env.POSTGRES_USER, 26 | password: process.env.POSTGRES_PASSWORD, 27 | database: process.env.POSTGRES_DATABASE, 28 | } as PoolConfig, 29 | tableName: 'file_vector_index', 30 | columns: { 31 | idColumnName: 'id', 32 | vectorColumnName: 'embedding', 33 | contentColumnName: 'text', 34 | metadataColumnName: 'metadata', 35 | }, 36 | // supported distance strategies: cosine (default), innerProduct, or euclidean 37 | distanceStrategy: 'cosine' as DistanceStrategy, 38 | }; 39 | } 40 | 41 | public getAlbumDir() { 42 | return process.env.ALBUM_PATH || '/home/images'; 43 | } 44 | 45 | public getHostName() { 46 | return process.env.HOST_NAME || 'http://127.0.0.1:8080'; 47 | } 48 | } 49 | 50 | export const configService = new ConfigService(); 51 | -------------------------------------------------------------------------------- /src/remote/pg-vector-store.service.ts: -------------------------------------------------------------------------------- 1 | import { OpenAIEmbeddings } from '@langchain/openai'; 2 | import { configService } from '../config/config.service'; 3 | import { Injectable, OnModuleInit } from '@nestjs/common'; 4 | import { HttpsProxyAgent } from 'https-proxy-agent'; 5 | import { PGVectorStore } from '@langchain/community/vectorstores/pgvector'; 6 | import { similaritySearch } from '@langchain/community/dist/vectorstores/tests/convex/convex/lib'; 7 | 8 | @Injectable() 9 | export class PgVectorStoreService implements OnModuleInit { 10 | private pgvectorStore: PGVectorStore; 11 | 12 | async onModuleInit() { 13 | await this.init(); 14 | } 15 | 16 | public async init() { 17 | if (process.env.EMBEDDING_PROVIDER == 'openai') { 18 | this.pgvectorStore = await PGVectorStore.initialize( 19 | new OpenAIEmbeddings({ 20 | apiKey: process.env.OPENAI_API_KEY, 21 | model: process.env.EMBEDDING_PROVIDER_MODEL, 22 | configuration: { 23 | httpAgent: process.env.PROXY_URL 24 | ? new HttpsProxyAgent(process.env.PROXY_URL) 25 | : undefined, 26 | }, 27 | }), 28 | configService.getPGvectorConfig(), 29 | ); 30 | } else { 31 | throw new Error(`no support. provider=${process.env.EMBEDDING_PROVIDER}`); 32 | } 33 | } 34 | 35 | public async addDoc(pageContent: string, metadata: Record) { 36 | await this.pgvectorStore.addDocuments([ 37 | { 38 | pageContent: pageContent, 39 | metadata: metadata, 40 | }, 41 | ]); 42 | } 43 | 44 | public async addDocs( 45 | docs: { pageContent: string; metadata: Record }[], 46 | ) { 47 | await this.pgvectorStore.addDocuments(docs); 48 | } 49 | 50 | public async search(query: string, limit: number) { 51 | return this.pgvectorStore.similaritySearchWithScore(query, limit); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/file/chat.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | import { FileService } from './file.service'; 3 | import { anthropicClient, openai } from '../app.module'; 4 | import Anthropic from '@anthropic-ai/sdk'; 5 | import TextBlock = Anthropic.TextBlock; 6 | import { replacePlaceholders } from '../common'; 7 | 8 | @Injectable() 9 | export class ChatService { 10 | constructor(private readonly fileService: FileService) {} 11 | 12 | public async chat(input: { text: string }) { 13 | const res = await this.fileService.searchDetail(input.text); 14 | if (!res) { 15 | return { code: 404, msg: 'Not found' }; 16 | } 17 | const { results, urls } = res; 18 | if (!results || results.length == 0) { 19 | 20 | return { 21 | content: 'No relevant images were found in your album. Please add images or modify the search query.', 22 | }; 23 | } 24 | 25 | console.log( 26 | replacePlaceholders( 27 | process.env.CHAT_PROVIDER_PROMPT, 28 | input.text, 29 | JSON.stringify({ results, urls }), 30 | ), 31 | ); 32 | let content: string; 33 | if (process.env.CHAT_PROVIDER == 'openai') { 34 | const response = await openai.chat.completions.create({ 35 | model: process.env.CHAT_PROVIDER_MODEL, 36 | max_tokens: 1024, 37 | temperature: 0.5, 38 | messages: [ 39 | { 40 | role: 'user', 41 | content: replacePlaceholders( 42 | process.env.CHAT_PROVIDER_PROMPT, 43 | input.text, 44 | JSON.stringify({ results, urls }), 45 | ), 46 | }, 47 | ], 48 | }); 49 | content = response.choices[0].message.content; 50 | } else if (process.env.CHAT_PROVIDER == 'anthropic') { 51 | const response = await anthropicClient.messages.create({ 52 | model: process.env.CHAT_PROVIDER_MODEL, 53 | max_tokens: 1024, 54 | temperature: 0.5, 55 | messages: [ 56 | { 57 | role: 'user', 58 | content: replacePlaceholders( 59 | process.env.CHAT_PROVIDER_PROMPT, 60 | input.text, 61 | JSON.stringify({ results, urls }), 62 | ), 63 | }, 64 | ], 65 | }); 66 | content = (response.content[0] as TextBlock).text; 67 | } else { 68 | throw new Error(`no support. provider=${process.env.CHAT_PROVIDER}`); 69 | } 70 | return { 71 | content, 72 | }; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/remote/extract-image.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable } from '@nestjs/common'; 2 | import { anthropicClient, openai } from '../app.module'; 3 | import * as sharp from 'sharp'; 4 | import Anthropic from '@anthropic-ai/sdk'; 5 | import TextBlock = Anthropic.TextBlock; 6 | 7 | @Injectable() 8 | export class ExtractImageService { 9 | public async extractImageInfo( 10 | imageBase64: string, 11 | contentType: 'image/jpeg', 12 | ) { 13 | if (process.env.IMAGE_EXTRACT_PROVIDER == 'anthropic') { 14 | const res = await anthropicClient.messages.create({ 15 | model: process.env.IMAGE_EXTRACT_PROVIDER_MODEL, 16 | max_tokens: 1024, 17 | messages: [ 18 | { 19 | role: 'user', 20 | content: [ 21 | { 22 | type: 'image', 23 | source: { 24 | type: 'base64', 25 | media_type: contentType, 26 | data: imageBase64, 27 | }, 28 | }, 29 | { 30 | type: 'text', 31 | text: process.env.IMAGE_EXTRACT_PROVIDER_PROMPT, 32 | }, 33 | ], 34 | }, 35 | ], 36 | }); 37 | return (res.content[0] as TextBlock).text; 38 | } else if (process.env.IMAGE_EXTRACT_PROVIDER == 'openai') { 39 | const res = await openai.chat.completions.create({ 40 | model: process.env.IMAGE_EXTRACT_PROVIDER_MODEL, 41 | max_tokens: 300, 42 | messages: [ 43 | { 44 | role: 'user', 45 | content: [ 46 | { 47 | type: 'image_url', 48 | image_url: { 49 | url: `data:${contentType};base64,${imageBase64}`, 50 | }, 51 | }, 52 | { 53 | type: 'text', 54 | text: process.env.IMAGE_EXTRACT_PROVIDER_PROMPT, 55 | }, 56 | ], 57 | }, 58 | ], 59 | }); 60 | return res.choices[0].message.content; 61 | } else { 62 | throw new Error( 63 | `no support. provider=${process.env.IMAGE_EXTRACT_PROVIDER}`, 64 | ); 65 | } 66 | } 67 | 68 | public async compressImageToBuffer(inputImagePath: string, quality: number) { 69 | const image = sharp(inputImagePath); 70 | return await image 71 | .resize({ 72 | width: 512, 73 | height: 512, 74 | fit: sharp.fit.inside, 75 | withoutEnlargement: true, 76 | }) 77 | .jpeg({ quality }) 78 | .toBuffer(); 79 | } 80 | 81 | public async imageToBase64(imageBuffer: Buffer) { 82 | return imageBuffer.toString('base64'); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "album-ai", 3 | "version": "0.0.1", 4 | "description": "", 5 | "author": "", 6 | "private": true, 7 | "license": "UNLICENSED", 8 | "scripts": { 9 | "build": "nest build", 10 | "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", 11 | "start": "nest start", 12 | "start:dev": "nest start --watch", 13 | "start:debug": "nest start --debug --watch", 14 | "start:prod": "node dist/main", 15 | "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", 16 | "test": "jest", 17 | "test:watch": "jest --watch", 18 | "test:cov": "jest --coverage", 19 | "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand", 20 | "test:e2e": "jest --config ./test/jest-e2e.json" 21 | }, 22 | "dependencies": { 23 | "@anthropic-ai/sdk": "^0.24.3", 24 | "@langchain/community": "^0.2.17", 25 | "@langchain/openai": "^0.2.1", 26 | "@nestjs/common": "^10.0.0", 27 | "@nestjs/config": "^3.2.2", 28 | "@nestjs/core": "^10.0.0", 29 | "@nestjs/platform-express": "^10.0.0", 30 | "@nestjs/schedule": "^4.0.2", 31 | "@nestjs/typeorm": "^10.0.2", 32 | "@types/mime-types": "^2.1.4", 33 | "@types/pg": "^8.11.6", 34 | "crc": "^4.3.2", 35 | "glob": "^10.4.2", 36 | "hbs": "^4.2.0", 37 | "https-proxy-agent": "^7.0.5", 38 | "mime-types": "^2.1.35", 39 | "openai": "^4.52.3", 40 | "pg": "^8.12.0", 41 | "reflect-metadata": "^0.2.0", 42 | "rxjs": "^7.8.1", 43 | "sharp": "^0.33.4", 44 | "socks-proxy-agent": "^8.0.4", 45 | "typeorm": "^0.3.20" 46 | }, 47 | "devDependencies": { 48 | "@nestjs/cli": "^10.0.0", 49 | "@nestjs/schematics": "^10.0.0", 50 | "@nestjs/testing": "^10.0.0", 51 | "@types/express": "^4.17.17", 52 | "@types/jest": "^29.5.2", 53 | "@types/node": "^20.14.9", 54 | "@types/supertest": "^6.0.0", 55 | "@typescript-eslint/eslint-plugin": "^6.0.0", 56 | "@typescript-eslint/parser": "^6.0.0", 57 | "eslint": "^8.42.0", 58 | "eslint-config-prettier": "^9.0.0", 59 | "eslint-plugin-prettier": "^5.0.0", 60 | "jest": "^29.5.0", 61 | "prettier": "^3.0.0", 62 | "source-map-support": "^0.5.21", 63 | "supertest": "^6.3.3", 64 | "ts-jest": "^29.1.0", 65 | "ts-loader": "^9.4.3", 66 | "ts-node": "^10.9.1", 67 | "tsconfig-paths": "^4.2.0", 68 | "typescript": "^5.1.3" 69 | }, 70 | "jest": { 71 | "moduleFileExtensions": [ 72 | "js", 73 | "json", 74 | "ts" 75 | ], 76 | "rootDir": "src", 77 | "testRegex": ".*\\.spec\\.ts$", 78 | "transform": { 79 | "^.+\\.(t|j)s$": "ts-jest" 80 | }, 81 | "collectCoverageFrom": [ 82 | "**/*.(t|j)s" 83 | ], 84 | "coverageDirectory": "../coverage", 85 | "testEnvironment": "node" 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- 1 |
2 |

3 | Album AI 4 |

5 |

基于生成式AI和RAG技术，重新设计的图库，用自然语言直接和你的图库/相册对话。

6 |

👉 更新很快，欢迎 star。

7 |
8 |

9 | English 10 | | 简体中文 11 | | Demo 12 | | Discord 13 | 14 |

15 |

16 | Album AI - AI-First Album - Chat with your gallery using plain language! | Product Hunt 17 |

18 | 19 | https://github.com/user-attachments/assets/5cc72436-2749-479f-a1bf-06e0d06ce1e3 20 | 21 | ## 简介 22 | 23 | Album AI 是一个试验项目，使用刚刚发布的gpt-4o-mini和 Haiku 作为廉价的视觉模型，自动识别出相册中图像文件的元信息，然后借助RAG技术，实现了与相册对话的能力。 24 | 25 | 可以当做传统的相册使用，也可以作为一个图像知识库辅助LLM生成内容。 26 | 27 | ## 故事 28 | 29 | 作为一个摄影爱好者，面对几T的照片，我常常感到无从下手，现有的所有的相册管理软件都需要我付出额外的精力去维护它。Haiku和刚刚发布的gpt-4o-mini，让我看到了希望。所以我准备马上动手实现它，第一个版本我和我的伙伴，只用了不到24小时实现。 30 | 31 | 希望你们也喜欢它。我愿意听到你们任何赞美和反对。别忘了点个⭐️，或者分享让更多人知道。 32 | 33 | ## 在线 Demo 34 | 35 | [album.gcui.ai](http://album.gcui.ai) 36 | 37 | ## Features 38 | 39 | - 自动发现相册中的图片，使用一个 PgSQL 数据库存储 40 | - 使用GPT-4-o-mini，自动生成图像的元信息 41 | - 使用OpenAI的Embedding API，完成元信息向量化 42 | - 提供两个API： 43 | - Search: 传统的Search API，输入query，返回最相关的图像 44 | - Chat: RAG API，输入query，查询到图片并生成回复 45 | - 一键部署到 Render等支持Docker容器部署的平台 46 | - 宽松的开源协议，你可以随意的集成和修改（但如果要商业化请与我们取得联系） 47 | 48 | ## 如何开始使用？ 49 | 50 | 推荐在本地运行，如果要在服务器运行请自行部署，之后我们会完善这部分指南。 51 | 52 | ### 1. 克隆项目 53 | 54 | ```bash 55 | git clone git@github.com:gcui-art/album-ai.git 56 | cd album-ai 57 | ``` 58 | 59 | ### 2. 修改配置 60 | 61 | ```bash 62 | cp .env.prod.example .env.prod 63 | ``` 64 | 65 | 用编辑器打开`.env.prod`，修改里面的配置： 66 | 67 | ```bash 68 | HOST_NAME= # 你本地的IP地址，一般192.168.x.x:8080 69 | PROXY_URL= # (可选) 你本地的代理IP地址，一般 192.168.x.x:7890，无法直接访问OpenAI API时需要 70 | 71 | OPENAI_API_KEY= # 你的 openai api 秘钥 72 | ANTHROPIC_API_KEY= # 你的 Anthropic api 秘钥 73 | ``` 74 | 75 | ### 3. Build 并启动项目 76 | 77 | ```bash 78 | chmod a+x ./build.sh 79 | ./build.sh 80 | ``` 81 | 82 | ### 4. Enjoy! 83 | 84 | 浏览器中打开 `http://localhost:8080` ，访问demo。 85 | 86 | ### 5. 添加新照片 87 | 88 | 打开项目的`images`目录，将新照片添加到`images`目录下后台认为会自动进行元信息的识别和向量化。稍后就可以在demo中通过搜索和对话的方式来使用。 89 | 90 | ## API 说明 91 | 92 | Album AI 目前主要实现了以下 API: 93 | 94 | - `get` /api/v1/file/search: 搜索图片 95 | - `post`/api/v1/chat: 与图片对话 96 | 97 | ## 贡献指南 98 | 99 | 您有四种方式支持本项目： 100 | 101 | 1. Fork 项目并提交 PR：我们欢迎任何让Album AI变得更好的PR。 102 | 2. 提交Issue：我们欢迎任何合理的建议、bug反馈。 103 | 3. 推荐：向其他人推荐本项目；点击Star；使用本项目后放置外链。 104 | 105 | ## 许可证 106 | 107 | Apache 2.0 License 108 | 109 | ## 你有一个问题/建议/困难/Bug？ 110 | 111 | 我们使用 Github 的 Issue 来管理这些反馈，你可以提交一个。我们会经常来处理。 112 | 113 | ## 相关链接 114 | 115 | - 项目仓库: [github.com/gcui-art/album-ai](https://github.com/gcui-art/album-ai) 116 | - Album AI 主页: [album.gcui.ai](http://album.gcui.ai) 117 | - 演示站点: [album.gcui.ai](http://album.gcui.ai) 118 | - 作者：[@Kane](https://x.com/BlueeonY) 119 | 120 | ## 声明 121 | 122 | 如果要商业使用，请与我们取得联系。 123 | 124 | ## Star History 125 | 126 | 127 | 128 | 129 | 130 | Star History Chart 131 | 132 | 133 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 |

3 | Album AI 4 |

5 |

AI-First Album, Chat with your gallery/album using plain language!

6 |

👉 We update frequently, feel free to star us.

7 |
8 |

9 | English 10 | | 简体中文 11 | | Demo 12 | | Discord 13 | 14 |

15 |

16 | Album AI - AI-First Album - Chat with your gallery using plain language! | Product Hunt 17 |

18 | 19 | https://github.com/user-attachments/assets/5cc72436-2749-479f-a1bf-06e0d06ce1e3 20 | 21 | 22 | ## Introduction 23 | 24 | Album AI is an experimental project that uses the recently released gpt-4o-mini and Haiku as a visual model to automatically identify metadata from image files in the album. It then leverages RAG technology to enable conversations with the album. 25 | 26 | It can be used as a traditional photo album or as an image knowledge base to assist LLM in content generation. 27 | 28 | ## Story 29 | 30 | As a photography enthusiast facing terabytes of photos, I often felt overwhelmed. All existing photo management software required extra effort to maintain. Haiku and the newly released gpt-4o-mini gave me hope. So I decided to implement it immediately. My partner and I created the first version in less than 24 hours. 31 | 32 | We hope you'll like it too. We welcome any praise or criticism. Don't forget to give us a ⭐️ or share to let more people know about it. 33 | 34 | ## Live Demo 35 | [album.gcui.ai](http://album.gcui.ai) 36 | 37 | ## Features 38 | 39 | - Automatically discover images in the album, using a PgSQL database for storage. 40 | - Utilize GPT-4-o-mini to automatically generate metadata for images. 41 | - Use OpenAI's Embedding API for metadata vectorization. 42 | - Provide two APIs: 43 | - Search: A traditional Search API that takes a query and returns the most relevant images. 44 | - Chat: A RAG API that takes a query, retrieves images, and generates responses. 45 | - One-click deployment to platforms like Render that support Docker container deployment. 46 | - A permissive open-source license allows for integration and modification (please contact us for commercial use). 47 | 48 | ## How to start using? 49 | 50 | Recommended to run locally, if you want to run on a server, please deploy yourself, and we will improve this part of the guide. 51 | 52 | ### 1. Clone the project 53 | 54 | ```bash 55 | git clone git@github.com:gcui-art/album-ai.git 56 | cd album-ai 57 | ``` 58 | 59 | ### 2. Modify the .env 60 | 61 | ```bash 62 | cp .env.prod.example .env.prod 63 | ``` 64 | 65 | Open `.env.prod` with your favorite editor, modify the configuration: 66 | 67 | ```bash 68 | HOST_NAME= # Your local IP address, usually 192.168.x.x:8080 69 | PROXY_URL= # (Optional) Your local proxy IP address, usually 192.168.x.x:7890, required when accessing OpenAI API directly is not available 70 | 71 | OPENAI_API_KEY= # Your openai api key 72 | ANTHROPIC_API_KEY= # Your Anthropic api key 73 | ``` 74 | 75 | ### 3. Build and run the project 76 | 77 | ```bash 78 | chmod a+x ./build.sh 79 | ./build.sh 80 | ``` 81 | 82 | ### 4. Enjoy! 83 | 84 | Open the browser and visit `http://localhost:8080` to see the demo. 85 | 86 | ### 5. Add new photos 87 | 88 | Open the `images` directory in the project, add new photos to the `images` directory, and the background will automatically recognize and vectorize metadata. After that, you can use it in the demo through search and chat. 89 | 90 | ## API Reference 91 | 92 | Album AI currently implements the following APIs: 93 | 94 | - `get` /api/v1/file/search: Search for images 95 | - `post`/api/v1/chat: Chat with images 96 | 97 | ## Contribution 98 | 99 | There are four ways to support this project: 100 | 101 | 1. Fork the project and submit a PR: We welcome any PR to make Album AI better. 102 | 2. Submit an Issue: We welcome any reasonable suggestions or bug reports. 103 | 3. Recommend: Recommend the project to others; click Star; place a link to the project after using it. 104 | 105 | ## License 106 | 107 | Apache 2.0 License 108 | 109 | ## Do You have a question/suggestion/issue/Bug? 110 | 111 | We use Github's Issue to manage these feedbacks, you can submit one. We will often deal with them. 112 | 113 | ## Related links 114 | 115 | - Project repository: [github.com/gcui-art/album-ai](https://github.com/gcui-art/album-ai) 116 | - Album AI homepage: [album.gcui.ai](http://album.gcui.ai) 117 | - Live Demo: [album.gcui.ai](http://album.gcui.ai) 118 | - Author: [@Kane](https://x.com/BlueeonY) 119 | 120 | ## Disclaimer 121 | 122 | If you want to use it for commercial purposes, please contact us. 123 | 124 | ## Star History 125 | 126 | 127 | 128 | 129 | 130 | Star History Chart 131 | 132 | 133 | -------------------------------------------------------------------------------- /.env.prod.example: -------------------------------------------------------------------------------- 1 | # Configure the environment variables to be used inside the container here. 2 | 3 | 4 | # ------------------------------------------------------ 5 | # SET Server Port 6 | # ------------------------------------------------------ 7 | SERVER_PORT: 8080 8 | 9 | # ------------------------------------------------------ 10 | # SET Host and Proxy 11 | # HOST_NAME is the local network IP address of your machine, which you can obtain using Ifconfig/Ipconfig. 12 | # PROXY_URL is optional and serves as a proxy for accessing the OpenAI API when direct access is not possible. 13 | # ------------------------------------------------------ 14 | HOST_NAME: http://192.168.8.104:8080 # Needs to be modified to your own, required 15 | PROXY_URL: http://192.168.8.104:7890 # Needs to be modified to your own, optional 16 | 17 | # ------------------------------------------------------ 18 | # SET Postgres. 19 | # Postgres will be used for storing the image database and for vector queries. 20 | # ------------------------------------------------------ 21 | POSTGRES_HOST: postgres 22 | POSTGRES_PORT: 5432 23 | POSTGRES_USER: root 24 | POSTGRES_PASSWORD: 123456 25 | POSTGRES_DATABASE: album 26 | 27 | 28 | # ------------------------------------------------------ 29 | # SET OpenAI / Anthropic API Key， 30 | # Album-ai can use the models claude-3-haiku-20240307 and gpt-4o-mini as visual and dialogue models. 31 | # ------------------------------------------------------ 32 | OPENAI_API_KEY: # Your open ai key, required 33 | ANTHROPIC_API_KEY: # Your anthropic key, optional 34 | 35 | # ------------------------------------------------------ 36 | # similarity_score is a score that indicates the similarity between the query text and the document; 37 | # the smaller the score, the more similar they are. 38 | # ------------------------------------------------------ 39 | SIMILARITY_SCORE: 0.8 40 | 41 | # ------------------------------------------------------ 42 | # Maximum number of searches 43 | # ------------------------------------------------------ 44 | MAX_SEARCH_LIMIT: 10 45 | 46 | # ------------------------------------------------------ 47 | # SET Image Extract Model 48 | # ------------------------------------------------------ 49 | 50 | # The available model providers include: anthropic and openai. 51 | IMAGE_EXTRACT_PROVIDER: openai 52 | # The available vision model for anthropic is: claude-3-haiku-20240307 53 | # The available vision model for openai is: gpt-4o-mini 54 | IMAGE_EXTRACT_PROVIDER_MODEL: gpt-4o-mini 55 | 56 | # The prompt for image extract can be modified as needed. 57 | IMAGE_EXTRACT_PROVIDER_PROMPT: 'You are an advanced image recognition AI tasked with identifying and describing the contents of images. Your analysis will be used in an image recognition software. Follow these instructions carefully: 58 | 59 | 1. You will be presented with an image. 60 | 61 | 2. Analyze the image thoroughly, paying attention to all visible elements, including: 62 | - Objects 63 | - People 64 | - Text 65 | - Colors 66 | - Shapes 67 | - Patterns 68 | - Background 69 | - Foreground 70 | - Emotional and atmosphere 71 | - Any notable features or characteristics 72 | 73 | 3. If any aspect of the image is unclear or ambiguous, state this explicitly in the relevant section of your analysis. Use phrases like "unclear," "ambiguous," or "cannot be determined with certainty" when appropriate. 74 | 75 | 4. Remember to respect privacy and ethical considerations: 76 | - Do not attempt to identify specific individuals 77 | - Avoid making assumptions about race, ethnicity, or other sensitive characteristics unless they are clearly relevant to the image content 78 | - If the image contains potentially sensitive or inappropriate content, mention this in the Additional Note section without going into explicit detail 79 | 80 | 5. Provide your complete analysis(Markdown format) within tags. 81 | 82 | Remember, your goal is to provide an accurate, detailed, and objective description of the image contents that can be used by image recognition software. Focus on what you can see and reasonably infer from the image itself.' 83 | 84 | 85 | # ------------------------------------------------------ 86 | # SET Embedding Model provider and model name 87 | # ------------------------------------------------------ 88 | EMBEDDING_PROVIDER: openai 89 | EMBEDDING_PROVIDER_MODEL: text-embedding-3-large 90 | 91 | 92 | # ------------------------------------------------------ 93 | # SET Chat Model provider and model name 94 | # ------------------------------------------------------ 95 | 96 | # Available chat model providers include: Anthropic and OpenAI. 97 | CHAT_PROVIDER: openai 98 | 99 | # You can choose any model from the provider for conversations. 100 | CHAT_PROVIDER_MODEL: gpt-4o-mini 101 | 102 | # The prompt for chat can be modified as needed. 103 | CHAT_PROVIDER_PROMPT: "You are an artificial intelligence image album assistant, and your goal is to respond to user requests by combining relevant image data. 104 | The user request is: {0}, 105 | and the associated images are: {1}. 106 | 'fId' is the unique identification of the image. 107 | You need to return a friendly response in Markdown format and showcase the related images(use markdown image tag)." 108 | 109 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # Configure the environment variables to be used inside the container here. 2 | 3 | 4 | # ------------------------------------------------------ 5 | # SET Server Port 6 | # ------------------------------------------------------ 7 | SERVER_PORT: 8080 8 | 9 | # ------------------------------------------------------ 10 | # SET Host and Proxy 11 | # HOST_NAME is the local network IP address of your machine, which you can obtain using Ifconfig/Ipconfig. 12 | # PROXY_URL is optional and serves as a proxy for accessing the OpenAI API when direct access is not possible. 13 | # ------------------------------------------------------ 14 | HOST_NAME: http://192.168.8.104:8080 # Needs to be modified to your own, required 15 | PROXY_URL: http://192.168.8.104:7890 # Needs to be modified to your own, optional 16 | 17 | # ------------------------------------------------------ 18 | # SET Postgres. 19 | # Postgres will be used for storing the image database and for vector queries. 20 | # ------------------------------------------------------ 21 | POSTGRES_HOST: postgres 22 | POSTGRES_PORT: 5432 23 | POSTGRES_USER: root 24 | POSTGRES_PASSWORD: 123456 25 | POSTGRES_DATABASE: album 26 | 27 | 28 | # ------------------------------------------------------ 29 | # SET Album Path 30 | # You shouldn’t change the directory of images inside the container unless you’re sure of its purpose. 31 | # ------------------------------------------------------ 32 | ALBUM_PATH: /home/images 33 | 34 | 35 | # ------------------------------------------------------ 36 | # SET OpenAI / Anthropic API Key， 37 | # Album-ai can use the models claude-3-haiku-20240307 and gpt-4o-mini as visual and dialogue models. 38 | # ------------------------------------------------------ 39 | OPENAI_API_KEY: # Your open ai key, required 40 | ANTHROPIC_API_KEY: # Your anthropic key, optional 41 | 42 | # ------------------------------------------------------ 43 | # similarity_score is a score that indicates the similarity between the query text and the document; 44 | # the smaller the score, the more similar they are. 45 | # ------------------------------------------------------ 46 | SIMILARITY_SCORE: 0.8 47 | 48 | # ------------------------------------------------------ 49 | # Maximum number of searches 50 | # ------------------------------------------------------ 51 | MAX_SEARCH_LIMIT: 10 52 | 53 | # ------------------------------------------------------ 54 | # SET Image Extract Model 55 | # ------------------------------------------------------ 56 | 57 | # The available model providers include: anthropic and openai. 58 | IMAGE_EXTRACT_PROVIDER: openai 59 | # The available vision model for anthropic is: claude-3-haiku-20240307 60 | # The available vision model for openai is: gpt-4o-mini 61 | IMAGE_EXTRACT_PROVIDER_MODEL: gpt-4o-mini 62 | 63 | # The prompt for image extract can be modified as needed. 64 | IMAGE_EXTRACT_PROVIDER_PROMPT: 'You are an advanced image recognition AI tasked with identifying and describing the contents of images. Your analysis will be used in an image recognition software. Follow these instructions carefully: 65 | 66 | 1. You will be presented with an image. 67 | 68 | 2. Analyze the image thoroughly, paying attention to all visible elements, including: 69 | - Objects 70 | - People 71 | - Text 72 | - Colors 73 | - Shapes 74 | - Patterns 75 | - Background 76 | - Foreground 77 | - Emotional and atmosphere 78 | - Any notable features or characteristics 79 | 80 | 3. If any aspect of the image is unclear or ambiguous, state this explicitly in the relevant section of your analysis. Use phrases like "unclear," "ambiguous," or "cannot be determined with certainty" when appropriate. 81 | 82 | 4. Remember to respect privacy and ethical considerations: 83 | - Do not attempt to identify specific individuals 84 | - Avoid making assumptions about race, ethnicity, or other sensitive characteristics unless they are clearly relevant to the image content 85 | - If the image contains potentially sensitive or inappropriate content, mention this in the Additional Note section without going into explicit detail 86 | 87 | 5. Provide your complete analysis(Markdown format) within tags. 88 | 89 | Remember, your goal is to provide an accurate, detailed, and objective description of the image contents that can be used by image recognition software. Focus on what you can see and reasonably infer from the image itself.' 90 | 91 | 92 | # ------------------------------------------------------ 93 | # SET Embedding Model provider and model name 94 | # ------------------------------------------------------ 95 | EMBEDDING_PROVIDER: openai 96 | EMBEDDING_PROVIDER_MODEL: text-embedding-3-large 97 | 98 | 99 | # ------------------------------------------------------ 100 | # SET Chat Model provider and model name 101 | # ------------------------------------------------------ 102 | 103 | # Available chat model providers include: Anthropic and OpenAI. 104 | CHAT_PROVIDER: openai 105 | 106 | # You can choose any model from the provider for conversations. 107 | CHAT_PROVIDER_MODEL: gpt-4o-mini 108 | 109 | # The prompt for chat can be modified as needed. 110 | CHAT_PROVIDER_PROMPT: "You are an artificial intelligence image album assistant, and your goal is to respond to user requests by combining relevant image data. 111 | The user request is: {0}, 112 | and the associated images are: {1}. 113 | 'fId' is the unique identification of the image. 114 | You need to return a friendly response in Markdown format and showcase the related images(use markdown image tag)." 115 | 116 | -------------------------------------------------------------------------------- /src/file/file.service.ts: -------------------------------------------------------------------------------- 1 | import { Injectable, Logger } from '@nestjs/common'; 2 | import { InjectRepository } from '@nestjs/typeorm'; 3 | import { FileAlbum } from './file-scan.entity.js'; 4 | import { In, MoreThan, Repository } from 'typeorm'; 5 | import { glob } from 'glob'; 6 | import { configService } from '../config/config.service.js'; 7 | import * as fs from 'node:fs'; 8 | import * as path from 'node:path'; 9 | import crc32 from 'crc/crc32'; 10 | import * as mime from 'mime-types'; 11 | import { FileStatus } from '../common'; 12 | import { ExtractImageService } from '../remote/extract-image.service'; 13 | import { PgVectorStoreService } from '../remote/pg-vector-store.service'; 14 | 15 | @Injectable() 16 | export class FileService { 17 | private readonly logger = new Logger(FileService.name); 18 | 19 | private hasScanFileTask = false; 20 | private hasExtractDescTask = false; 21 | private hasEmbeddingTask = false; 22 | 23 | constructor( 24 | @InjectRepository(FileAlbum) 25 | private readonly fileRepository: Repository, 26 | private readonly extractImageService: ExtractImageService, 27 | private readonly pgVectorStoreService: PgVectorStoreService, 28 | ) {} 29 | 30 | public findAll() { 31 | return this.fileRepository.find(); 32 | } 33 | 34 | public async scanFile() { 35 | if (this.hasScanFileTask) { 36 | return; 37 | } 38 | 39 | this.hasScanFileTask = true; 40 | try { 41 | const dir = configService.getAlbumDir(); 42 | const filePaths = await glob(`${dir}/**/*`); 43 | 44 | const fileDetails = filePaths.map(async (filePath) => { 45 | const stats = fs.lstatSync(filePath); 46 | 47 | const fileBuffer = fs.readFileSync(filePath); 48 | const crc = crc32(fileBuffer).toString(16); 49 | 50 | return { 51 | path: filePath, 52 | name: path.basename(filePath), 53 | isDirectory: stats.isDirectory(), 54 | size: stats.size, 55 | crc: crc, 56 | fileType: mime.lookup(filePath) || 'unknown', 57 | }; 58 | }); 59 | 60 | const resList = await Promise.all(fileDetails); 61 | const dataList = resList.map((item) => { 62 | return { 63 | fileName: item.name, 64 | path: item.path, 65 | crc: item.crc, 66 | status: FileStatus.Init, 67 | crcFile: item.fileType, 68 | contentType: item.fileType, 69 | size: item.size, 70 | } as Partial; 71 | }); 72 | // save DB 73 | await this.fileRepository 74 | .createQueryBuilder() 75 | .insert() 76 | .into(FileAlbum) 77 | .values(dataList) 78 | .orIgnore() 79 | .execute(); 80 | } catch (err) { 81 | this.logger.warn('exec scanning file error', err); 82 | } finally { 83 | this.hasScanFileTask = false; 84 | } 85 | } 86 | 87 | public async extractDesc() { 88 | if (this.hasExtractDescTask) { 89 | return; 90 | } 91 | this.hasExtractDescTask = true; 92 | try { 93 | let cursorId = 0n; 94 | while (true) { 95 | const dataList = await this.fileRepository.find({ 96 | where: { 97 | fId: MoreThan(cursorId), 98 | status: FileStatus.Init, 99 | }, 100 | take: 10, 101 | order: { fId: 'asc' }, 102 | }); 103 | 104 | if (!dataList || dataList.length == 0) { 105 | return; 106 | } 107 | cursorId = dataList[dataList.length - 1].fId; 108 | 109 | for (const fileAlbum of dataList) { 110 | const imageBuf = await this.extractImageService.compressImageToBuffer( 111 | fileAlbum.path, 112 | 80, 113 | ); 114 | const base64 = await this.extractImageService.imageToBase64(imageBuf); 115 | const content = await this.extractImageService.extractImageInfo( 116 | base64, 117 | 'image/jpeg', 118 | ); 119 | 120 | await this.fileRepository.update( 121 | { 122 | fId: fileAlbum.fId, 123 | status: FileStatus.Init, 124 | }, 125 | { 126 | descAi: content, 127 | status: FileStatus.Extract, 128 | }, 129 | ); 130 | } 131 | } 132 | } catch (err) { 133 | this.logger.warn('exec extracting Desc error', err); 134 | } finally { 135 | this.hasExtractDescTask = false; 136 | } 137 | } 138 | 139 | public async embedding() { 140 | if (this.hasEmbeddingTask) { 141 | return; 142 | } 143 | this.hasEmbeddingTask = true; 144 | try { 145 | let cursorId = 0n; 146 | while (true) { 147 | const dataList = await this.fileRepository.find({ 148 | where: { 149 | fId: MoreThan(cursorId), 150 | status: FileStatus.Extract, 151 | }, 152 | take: 10, 153 | order: { fId: 'asc' }, 154 | }); 155 | 156 | if (!dataList || dataList.length == 0) { 157 | return; 158 | } 159 | cursorId = dataList[dataList.length - 1].fId; 160 | const docs = dataList.map((item) => { 161 | return { 162 | pageContent: item.descAi, 163 | metadata: { fId: item.fId.toString() }, 164 | } as { pageContent: string; metadata: Record }; 165 | }); 166 | await this.pgVectorStoreService.addDocs(docs); 167 | 168 | await this.fileRepository.update( 169 | { 170 | fId: In(dataList.map((item) => item.fId)), 171 | status: FileStatus.Extract, 172 | }, 173 | { 174 | status: FileStatus.Embedding, 175 | }, 176 | ); 177 | } 178 | } catch (err) { 179 | this.logger.warn('exec embedding error', err); 180 | } finally { 181 | this.hasEmbeddingTask = false; 182 | } 183 | } 184 | 185 | public async searchDetail(query: string, score?: number, limit?: number) { 186 | let results = await this.pgVectorStoreService.search( 187 | query, 188 | limit ?? parseInt(process.env.MAX_SEARCH_LIMIT), 189 | ); 190 | if (!results || results.length == 0) { 191 | return; 192 | } 193 | score = score ?? parseFloat(process.env.SIMILARITY_SCORE); 194 | results = results.filter((item) => item[1] < score); 195 | const fIds = results.map((item) => item[0].metadata.fId); 196 | const fileAlbums = await this.fileRepository.find({ 197 | where: { 198 | fId: In(fIds), 199 | }, 200 | }); 201 | const fileUrls = fIds.map((fId) => { 202 | return { 203 | fId, 204 | url: `${configService.getHostName()}/api/v1/file/${fId}/download`, 205 | }; 206 | }); 207 | return { 208 | results, 209 | fileAlbums, 210 | urls: fileUrls, 211 | }; 212 | } 213 | 214 | public async findFile(fId: string) { 215 | return await this.fileRepository.findOne({ 216 | where: { 217 | fId: BigInt(fId), 218 | }, 219 | }); 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /views/index.hbs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Album AI - AI-First Album/Gallery 5 | 9 | 10 | 13 | 14 | 19 | 23 | 27 | 28 | 29 |
30 | {{#if query}} 31 | 32 |
33 |
34 |

35 | ✨ Album AI 36 |

37 |
38 | 45 | 49 |
50 |
51 |
52 |
55 |

56 | Answer: 57 |

58 |

59 | 65 |
66 | {{else}} 67 | 68 |
69 |

✨ Album AI

70 |

AI-First Album, Chat with your 71 | Album.

72 |
73 |
74 | 81 | 85 |
86 |
87 |
90 | cat 94 | Let me bid farewell to hunger 98 | cute animals 102 | Some means of transportation 106 | I want to give a gift to my mom 110 | 让我感到温暖的照片 114 | 元気が出る飲み物をください。y 118 | Plantas que crecen en el agua 122 |
123 |
124 | 130 | 142 | Follow us on GitHub 143 | 144 |
145 |
146 | {{/if}} 147 |
148 | 170 | 171 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2024] [gcui-art/album-ai]\ 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. --------------------------------------------------------------------------------