tr]:last:border-b-0",
48 | className
49 | )}
50 | {...props}
51 | />
52 | )
53 | }
54 |
55 | function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
56 | return (
57 | |
65 | )
66 | }
67 |
68 | function TableHead({ className, ...props }: React.ComponentProps<"th">) {
69 | return (
70 | [role=checkbox]]:translate-y-[2px]",
74 | className
75 | )}
76 | {...props}
77 | />
78 | )
79 | }
80 |
81 | function TableCell({ className, ...props }: React.ComponentProps<"td">) {
82 | return (
83 | | [role=checkbox]]:translate-y-[2px]",
87 | className
88 | )}
89 | {...props}
90 | />
91 | )
92 | }
93 |
94 | function TableCaption({
95 | className,
96 | ...props
97 | }: React.ComponentProps<"caption">) {
98 | return (
99 |
104 | )
105 | }
106 |
107 | export {
108 | Table,
109 | TableHeader,
110 | TableBody,
111 | TableFooter,
112 | TableHead,
113 | TableRow,
114 | TableCell,
115 | TableCaption,
116 | }
117 |
--------------------------------------------------------------------------------
/admin/src/components/ui/tabs.tsx:
--------------------------------------------------------------------------------
1 | "use client"
2 |
3 | import * as React from "react"
4 | import * as TabsPrimitive from "@radix-ui/react-tabs"
5 |
6 | import { cn } from "@/lib/utils"
7 |
8 | function Tabs({
9 | className,
10 | ...props
11 | }: React.ComponentProps) {
12 | return (
13 |
18 | )
19 | }
20 |
21 | function TabsList({
22 | className,
23 | ...props
24 | }: React.ComponentProps) {
25 | return (
26 |
34 | )
35 | }
36 |
37 | function TabsTrigger({
38 | className,
39 | ...props
40 | }: React.ComponentProps) {
41 | return (
42 |
50 | )
51 | }
52 |
53 | function TabsContent({
54 | className,
55 | ...props
56 | }: React.ComponentProps) {
57 | return (
58 |
63 | )
64 | }
65 |
66 | export { Tabs, TabsList, TabsTrigger, TabsContent }
67 |
--------------------------------------------------------------------------------
/admin/src/components/ui/toaster.tsx:
--------------------------------------------------------------------------------
1 | "use client"
2 |
3 | import {
4 | Toast,
5 | ToastClose,
6 | ToastDescription,
7 | ToastProvider,
8 | ToastTitle,
9 | ToastViewport,
10 | } from "@/components/ui/toast"
11 | import { useToast } from "@/components/ui/use-toast"
12 |
13 | export function Toaster() {
14 | const { toasts } = useToast()
15 |
16 | return (
17 |
18 | {toasts.map(function ({ id, title, description, action, ...props }) {
19 | return (
20 |
21 |
22 | {title && {title}}
23 | {description && (
24 | {description}
25 | )}
26 |
27 | {action}
28 |
29 |
30 | )
31 | })}
32 |
33 |
34 | )
35 | }
36 |
--------------------------------------------------------------------------------
/admin/src/lib/utils.ts:
--------------------------------------------------------------------------------
1 | import { clsx, type ClassValue } from "clsx"
2 | import { twMerge } from "tailwind-merge"
3 |
4 | export function cn(...inputs: ClassValue[]) {
5 | return twMerge(clsx(inputs))
6 | }
7 |
--------------------------------------------------------------------------------
/admin/tsconfig.json:
--------------------------------------------------------------------------------
1 | {
2 | "compilerOptions": {
3 | "target": "ES2017",
4 | "lib": ["dom", "dom.iterable", "esnext"],
5 | "allowJs": true,
6 | "skipLibCheck": true,
7 | "strict": true,
8 | "noEmit": true,
9 | "esModuleInterop": true,
10 | "module": "esnext",
11 | "moduleResolution": "bundler",
12 | "resolveJsonModule": true,
13 | "isolatedModules": true,
14 | "jsx": "preserve",
15 | "incremental": true,
16 | "plugins": [
17 | {
18 | "name": "next"
19 | }
20 | ],
21 | "paths": {
22 | "@/*": ["./src/*"]
23 | }
24 | },
25 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
26 | "exclude": ["node_modules"]
27 | }
28 |
--------------------------------------------------------------------------------
/backend/.dockerignore:
--------------------------------------------------------------------------------
1 | __pycache__/
2 | *.py[cod]
3 | *$py.class
4 | *.so
5 | .Python
6 | venv/
7 | env/
8 | .env
9 | .env.local
10 | .pytest_cache/
11 | .coverage
12 | htmlcov/
13 | .tox/
14 | .nox/
15 | .hypothesis/
16 | .idea/
17 | .vscode/
18 | *.log
19 | uploads/
20 | test_uploads/
21 | *.db
22 |
--------------------------------------------------------------------------------
/backend/.env.example:
--------------------------------------------------------------------------------
1 | # Environment
2 | ENVIRONMENT=development
3 |
4 | # Database settings
5 | DATABASE_URL=postgresql://postgres:postgres@localhost:5432/importcsv
6 |
7 | # JWT settings
8 | SECRET_KEY=your-secret-key-for-jwt
9 | ACCESS_TOKEN_EXPIRE_MINUTES=30
10 |
11 | # Webhook settings
12 | WEBHOOK_SECRET=your-webhook-secret
13 |
14 | # Redis and RQ settings
15 | REDIS_URL=redis://localhost:6379
16 | RQ_DEFAULT_TIMEOUT=3600
17 | RQ_IMPORT_QUEUE=imports
18 |
19 | # Admin user (for initialization)
20 | ADMIN_EMAIL=admin@example.com
21 | ADMIN_PASSWORD=admin123
22 | ADMIN_NAME=Admin User
23 |
--------------------------------------------------------------------------------
/backend/.gitignore:
--------------------------------------------------------------------------------
1 | # Python
2 | __pycache__/
3 | **/__pycache__/
4 | *.py[cod]
5 | *$py.class
6 | *.so
7 | .Python
8 | **/*.pyc
9 | **/*.pyo
10 | **/*.pyd
11 | .coverage
12 | .coverage.*
13 | .cache
14 | nosetests.xml
15 | coverage.xml
16 | *.cover
17 | *.so
18 | .Python
19 | build/
20 | develop-eggs/
21 | dist/
22 | downloads/
23 | eggs/
24 | .eggs/
25 | lib/
26 | lib64/
27 | parts/
28 | sdist/
29 | var/
30 | wheels/
31 | *.egg-info/
32 | .installed.cfg
33 | *.egg
34 |
35 | # Virtual Environment
36 | venv/
37 | ENV/
38 | env/
39 | .env
40 | .venv
41 |
42 | # Database
43 | *.db
44 | *.sqlite3
45 | *.sqlite
46 |
47 | # Uploads and user content
48 | uploads/
49 |
50 | # Logs
51 | *.log
52 | logs/
53 |
54 | # IDE specific files
55 | .idea/
56 | .vscode/
57 | *.swp
58 | *.swo
59 | .DS_Store
60 |
61 | # Environment variables
62 | .env
63 | .env.local
64 | .env.development.local
65 | .env.test.local
66 | .env.production.local
67 |
68 | # Cache
69 | .pytest_cache/
70 | .coverage
71 | htmlcov/
72 | .tox/
73 | .nox/
74 |
75 | # Jupyter Notebook
76 | .ipynb_checkpoints
77 |
78 | # FastAPI specific
79 | __pycache__/
80 | .pytest_cache/
81 |
--------------------------------------------------------------------------------
/backend/.python-version:
--------------------------------------------------------------------------------
1 | 3.12
2 |
--------------------------------------------------------------------------------
/backend/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM python:3.11-slim
2 |
3 | WORKDIR /app
4 |
5 | # Install dependencies
6 | COPY requirements.txt .
7 | RUN pip install --no-cache-dir -r requirements.txt
8 |
9 | # Copy application code
10 | COPY . .
11 |
12 | # Create upload directory
13 | RUN mkdir -p uploads
14 |
15 | # Expose port
16 | EXPOSE 8000
17 |
18 | # Run the application
19 | CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
20 |
--------------------------------------------------------------------------------
/backend/app/api/routes.py:
--------------------------------------------------------------------------------
1 | from fastapi import APIRouter
2 |
3 | from app.api.v1 import auth, importers, imports
4 |
5 | api_router = APIRouter()
6 |
7 | # Include all API routes
8 | # Auth routes (FastAPI-Users authentication)
9 | api_router.include_router(auth.router, prefix="/v1/auth", tags=["Authentication"])
10 |
11 | # Other API routes
12 | api_router.include_router(importers.router, prefix="/v1/importers", tags=["Importers"])
13 | api_router.include_router(imports.router, prefix="/v1/imports", tags=["Imports"])
14 |
15 | # Key-authenticated routes (no user authentication required)
16 | api_router.include_router(imports.key_router, prefix="/v1/imports", tags=["Key Imports"])
17 |
--------------------------------------------------------------------------------
/backend/app/api/v1/importers.py:
--------------------------------------------------------------------------------
1 | import uuid
2 | from fastapi import APIRouter, Depends, HTTPException, status
3 | from sqlalchemy.orm import Session
4 | from typing import List
5 |
6 | from app.db.base import get_db
7 | from app.auth.users import get_current_active_user
8 | from app.models.user import User
9 | from app.schemas.importer import ImporterCreate, ImporterUpdate, Importer as ImporterSchema
10 | from app.services import importer as importer_service
11 |
12 | router = APIRouter()
13 |
14 | @router.get("/", response_model=List[ImporterSchema])
15 | async def read_importers(
16 | db: Session = Depends(get_db),
17 | skip: int = 0,
18 | limit: int = 100,
19 | current_user: User = Depends(get_current_active_user),
20 | ):
21 | """
22 | Retrieve importers
23 | """
24 | return importer_service.get_importers(db, str(current_user.id), skip, limit)
25 |
26 |
27 | @router.post("/", response_model=ImporterSchema)
28 | async def create_importer(
29 | importer_in: ImporterCreate,
30 | db: Session = Depends(get_db),
31 | current_user: User = Depends(get_current_active_user),
32 | ):
33 | """
34 | Create new importer
35 | """
36 | return importer_service.create_importer(db, str(current_user.id), importer_in)
37 |
38 |
39 | @router.get("/{importer_id}", response_model=ImporterSchema)
40 | async def read_importer(
41 | importer_id: uuid.UUID,
42 | db: Session = Depends(get_db),
43 | current_user: User = Depends(get_current_active_user),
44 | ):
45 | """
46 | Get importer by ID
47 | """
48 | importer = importer_service.get_importer(db, str(current_user.id), importer_id)
49 | if not importer:
50 | raise HTTPException(status_code=404, detail="Importer not found")
51 | return importer
52 |
53 |
54 | @router.put("/{importer_id}", response_model=ImporterSchema)
55 | async def update_importer(
56 | importer_id: uuid.UUID,
57 | importer_in: ImporterUpdate,
58 | db: Session = Depends(get_db),
59 | current_user: User = Depends(get_current_active_user),
60 | ):
61 | """
62 | Update an importer
63 | """
64 | importer = importer_service.update_importer(db, str(current_user.id), importer_id, importer_in)
65 | if not importer:
66 | raise HTTPException(status_code=404, detail="Importer not found")
67 | return importer
68 |
69 | @router.delete("/{importer_id}", status_code=status.HTTP_204_NO_CONTENT)
70 | async def delete_importer(
71 | importer_id: uuid.UUID,
72 | db: Session = Depends(get_db),
73 | current_user: User = Depends(get_current_active_user),
74 | ):
75 | """
76 | Delete an importer
77 | """
78 | importer = importer_service.delete_importer(db, str(current_user.id), importer_id)
79 | if not importer:
80 | raise HTTPException(status_code=404, detail="Importer not found")
81 | return None
82 |
--------------------------------------------------------------------------------
/backend/app/auth/__init__.py:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/backend/app/auth/users.py:
--------------------------------------------------------------------------------
1 | import uuid
2 | import logging
3 | from typing import Optional
4 |
5 | from fastapi import Depends, Request
6 | from fastapi_users import BaseUserManager, FastAPIUsers, UUIDIDMixin
7 | from fastapi_users.authentication import AuthenticationBackend, BearerTransport, CookieTransport, JWTStrategy
8 |
9 | from app.core.config import settings
10 | from app.db.users import get_user_db
11 | from app.models.user import User
12 |
13 | logger = logging.getLogger(__name__)
14 |
15 |
16 | class UserManager(UUIDIDMixin, BaseUserManager[User, uuid.UUID]):
17 | reset_password_token_secret = settings.SECRET_KEY
18 | verification_token_secret = settings.SECRET_KEY
19 |
20 | async def on_after_register(self, user: User, request: Optional[Request] = None):
21 | logger.info(f"User {user.id} has registered.")
22 |
23 | async def on_after_forgot_password(
24 | self, user: User, token: str, request: Optional[Request] = None
25 | ):
26 | # In production, you would send an email here
27 | logger.info(f"User {user.id} has forgot their password. Reset token: {token}")
28 |
29 | async def on_after_request_verify(
30 | self, user: User, token: str, request: Optional[Request] = None
31 | ):
32 | # In production, you would send an email here
33 | logger.info(f"Verification requested for user {user.id}. Verification token: {token}")
34 |
35 |
36 | async def get_user_manager(user_db=Depends(get_user_db)):
37 | yield UserManager(user_db)
38 |
39 |
40 | # Bearer transport for API access
41 | bearer_transport = BearerTransport(tokenUrl=f"{settings.API_V1_STR}/auth/login")
42 |
43 | # Cookie transport for web applications
44 | cookie_transport = CookieTransport(
45 | cookie_name="importcsv_auth",
46 | cookie_max_age=settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60,
47 | cookie_secure=settings.ENVIRONMENT == "production", # Only send over HTTPS in production
48 | cookie_httponly=True, # Prevent JavaScript access
49 | cookie_samesite="lax", # CSRF protection
50 | )
51 |
52 |
53 | def get_jwt_strategy() -> JWTStrategy:
54 | return JWTStrategy(
55 | secret=settings.SECRET_KEY,
56 | lifetime_seconds=settings.ACCESS_TOKEN_EXPIRE_MINUTES * 60,
57 | token_audience=["fastapi-users:auth"],
58 | )
59 |
60 |
61 | # JWT Authentication backend for API access
62 | jwt_backend = AuthenticationBackend(
63 | name="jwt",
64 | transport=bearer_transport,
65 | get_strategy=get_jwt_strategy,
66 | )
67 |
68 | # Cookie Authentication backend for web applications
69 | cookie_backend = AuthenticationBackend(
70 | name="cookie",
71 | transport=cookie_transport,
72 | get_strategy=get_jwt_strategy,
73 | )
74 |
75 | # Create FastAPIUsers instance
76 | fastapi_users = FastAPIUsers[User, uuid.UUID](get_user_manager, [jwt_backend, cookie_backend])
77 |
78 | # Export dependencies
79 | get_current_user = fastapi_users.current_user()
80 | get_current_active_user = fastapi_users.current_user(active=True)
81 | get_current_superuser = fastapi_users.current_user(active=True, superuser=True)
82 | get_optional_user = fastapi_users.current_user(optional=True)
83 |
--------------------------------------------------------------------------------
/backend/app/db/models.py:
--------------------------------------------------------------------------------
1 | # Import all models here to ensure they are registered with SQLAlchemy
2 | # before any relationships are resolved
3 |
4 | # This file is imported by base.py to ensure all models are loaded
5 | # Using import strings to avoid circular imports
6 |
7 | # Import the modules, not the classes directly
8 | import app.models.user
9 | import app.models.import_job
10 | import app.models.importer
11 | import app.models.webhook
12 |
13 | # Add any new model imports here
14 |
--------------------------------------------------------------------------------
/backend/app/db/users.py:
--------------------------------------------------------------------------------
1 | """User database utilities for FastAPI Users.
2 |
3 | This module provides the necessary database dependencies for FastAPI Users.
4 | """
5 | from fastapi import Depends
6 | from fastapi_users.db import SQLAlchemyUserDatabase
7 | from sqlalchemy.ext.asyncio import AsyncSession
8 |
9 | from app.db.base import get_async_session
10 | from app.models.user import User
11 |
12 | async def get_user_db(session: AsyncSession = Depends(get_async_session)):
13 | """Get a SQLAlchemy user database instance.
14 |
15 | This is a dependency that will be used by FastAPI Users.
16 | """
17 | yield SQLAlchemyUserDatabase(session, User)
18 |
--------------------------------------------------------------------------------
/backend/app/db/utils.py:
--------------------------------------------------------------------------------
1 | import logging
2 | from contextlib import contextmanager
3 | from sqlalchemy.orm import Session
4 |
5 | logger = logging.getLogger(__name__)
6 |
7 | @contextmanager
8 | def db_transaction(db: Session):
9 | """
10 | Context manager for database transactions.
11 |
12 | Usage:
13 | with db_transaction(db):
14 | # Your database operations here
15 | db.add(item)
16 | # No need to call db.commit() - it's handled by the context manager
17 |
18 | Args:
19 | db (Session): The SQLAlchemy database session
20 |
21 | Raises:
22 | Exception: Any exception that occurs during the transaction will be re-raised
23 | after the rollback is performed
24 | """
25 | try:
26 | yield
27 | db.commit()
28 | except Exception as e:
29 | db.rollback()
30 | logger.error(f"Transaction failed: {str(e)}")
31 | raise
32 |
--------------------------------------------------------------------------------
/backend/app/models/import_job.py:
--------------------------------------------------------------------------------
1 | import enum
2 | import uuid
3 |
4 | from sqlalchemy import Column, Integer, String, JSON, DateTime, ForeignKey, Enum, UUID
5 | from sqlalchemy.sql import func
6 | from sqlalchemy.orm import relationship
7 |
8 | from app.db.base import Base
9 |
10 | # Import models using strings in relationships to avoid circular imports
11 |
12 |
13 | class ImportStatus(str, enum.Enum):
14 | PENDING = "pending"
15 | PROCESSING = "processing"
16 | VALIDATING = "validating"
17 | VALIDATED = "validated"
18 | IMPORTING = "importing"
19 | COMPLETED = "completed"
20 | FAILED = "failed"
21 |
22 |
23 | class ImportJob(Base):
24 | __tablename__ = "import_jobs"
25 |
26 | id = Column(UUID, primary_key=True, default=uuid.uuid4)
27 | user_id = Column(UUID, ForeignKey("users.id"), nullable=False)
28 | importer_id = Column(UUID, ForeignKey("importers.id"), nullable=False)
29 | file_name = Column(String, nullable=False)
30 | file_path = Column(String, nullable=False)
31 | file_type = Column(String, nullable=False) # csv, xlsx, etc.
32 | status = Column(Enum(ImportStatus), default=ImportStatus.PENDING, nullable=False)
33 | row_count = Column(Integer, default=0, nullable=False)
34 | processed_rows = Column(Integer, default=0, nullable=False)
35 | error_count = Column(Integer, default=0, nullable=False)
36 | errors = Column(JSON, nullable=True) # Store validation errors
37 | column_mapping = Column(
38 | JSON, nullable=True
39 | ) # Mapping of file columns to schema fields
40 | file_metadata = Column(JSON, nullable=True) # Additional metadata
41 | processed_data = Column(
42 | JSON, nullable=True
43 | ) # Store processed data (valid and invalid records)
44 | error_message = Column(
45 | String, nullable=True
46 | ) # Store error message if processing fails
47 | created_at = Column(DateTime(timezone=True), server_default=func.now())
48 | updated_at = Column(DateTime(timezone=True), onupdate=func.now())
49 | completed_at = Column(DateTime(timezone=True), nullable=True)
50 |
51 | # Relationships - using simple string references
52 | user = relationship("User", back_populates="import_jobs")
53 | importer = relationship("Importer", back_populates="import_jobs")
54 | webhook_events = relationship("WebhookEvent", back_populates="import_job")
55 |
--------------------------------------------------------------------------------
/backend/app/models/importer.py:
--------------------------------------------------------------------------------
1 | import uuid
2 | from sqlalchemy import (
3 | Column,
4 | Integer,
5 | String,
6 | JSON,
7 | DateTime,
8 | ForeignKey,
9 | UUID,
10 | Boolean,
11 | )
12 | from sqlalchemy.sql import func
13 | from sqlalchemy.orm import relationship
14 |
15 | from app.db.base import Base
16 |
17 |
18 | class Importer(Base):
19 | __tablename__ = "importers"
20 |
21 | id = Column(UUID, primary_key=True, default=uuid.uuid4)
22 | key = Column(UUID, unique=True, index=True, default=uuid.uuid4)
23 | name = Column(String, index=True, nullable=False)
24 | description = Column(String, nullable=True)
25 | user_id = Column(UUID, ForeignKey("users.id"), nullable=False)
26 | fields = Column(JSON, nullable=False) # JSON structure defining the importer fields
27 |
28 | # Webhook settings
29 | webhook_url = Column(String, nullable=True) # URL where imported data is sent to
30 | webhook_enabled = Column(
31 | Boolean, default=True
32 | ) # Whether to use webhook or onData callback
33 | include_data_in_webhook = Column(
34 | Boolean, default=True
35 | ) # Whether to include processed data in webhook
36 | webhook_data_sample_size = Column(
37 | Integer, default=5
38 | ) # Number of rows to include in webhook sample
39 |
40 | # Import settings
41 | include_unmatched_columns = Column(
42 | Boolean, default=False
43 | ) # Include all unmatched columns in import
44 | filter_invalid_rows = Column(
45 | Boolean, default=False
46 | ) # Filter rows that fail validation
47 | disable_on_invalid_rows = Column(
48 | Boolean, default=False
49 | ) # Disable importing all data if there are invalid rows
50 |
51 | created_at = Column(DateTime(timezone=True), server_default=func.now())
52 | updated_at = Column(DateTime(timezone=True), onupdate=func.now())
53 |
54 | # Relationships - using simple string references
55 | user = relationship("User", back_populates="importers")
56 | import_jobs = relationship("ImportJob", back_populates="importer")
57 |
--------------------------------------------------------------------------------
/backend/app/models/token.py:
--------------------------------------------------------------------------------
1 | import uuid
2 | from sqlalchemy import (
3 | Column,
4 | String,
5 | DateTime,
6 | ForeignKey,
7 | UUID,
8 | )
9 | from sqlalchemy.sql import func
10 | from sqlalchemy.orm import relationship
11 |
12 | from app.db.base import Base
13 |
14 |
15 | class TokenBlacklist(Base):
16 | """
17 | Model for tracking revoked/blacklisted tokens
18 |
19 | This model serves two purposes:
20 | 1. Track individual revoked tokens by their JTI (token_id)
21 | 2. Track mass revocation events (like 'logout from all devices') using the invalidate_before field
22 | """
23 |
24 | __tablename__ = "token_blacklist"
25 |
26 | id = Column(UUID, primary_key=True, default=uuid.uuid4)
27 | token_id = Column(
28 | String, unique=True, index=True, nullable=False
29 | ) # JTI from the token
30 | created_at = Column(DateTime(timezone=True), server_default=func.now())
31 |
32 | # For mass revocation: invalidate all tokens issued before this time for a specific user
33 | invalidate_before = Column(DateTime(timezone=True), nullable=True)
34 |
35 | # Link to user for easier querying
36 | user_id = Column(UUID, ForeignKey("users.id"), nullable=True)
37 | user = relationship("User", back_populates="blacklisted_tokens")
38 |
--------------------------------------------------------------------------------
/backend/app/models/user.py:
--------------------------------------------------------------------------------
1 | import uuid
2 |
3 | from fastapi_users.db import SQLAlchemyBaseUserTable
4 | from sqlalchemy import Column, String, Boolean, DateTime, UUID
5 | from sqlalchemy.sql import func
6 | from sqlalchemy.orm import relationship
7 |
8 | from app.db.base import Base
9 |
10 |
11 | class User(SQLAlchemyBaseUserTable[uuid.UUID], Base):
12 | __tablename__ = "users"
13 |
14 | id = Column(UUID, primary_key=True, default=uuid.uuid4)
15 | email = Column(String, unique=True, index=True, nullable=False)
16 | hashed_password = Column(String, nullable=False)
17 | full_name = Column(String, nullable=True)
18 | is_active = Column(Boolean, default=True, nullable=False)
19 | is_superuser = Column(Boolean, default=False, nullable=False)
20 | is_verified = Column(Boolean, default=False, nullable=False)
21 | created_at = Column(DateTime(timezone=True), server_default=func.now())
22 | updated_at = Column(DateTime(timezone=True), onupdate=func.now())
23 |
24 | # Relationships - using simple string references
25 | importers = relationship("Importer", back_populates="user")
26 | import_jobs = relationship("ImportJob", back_populates="user")
27 | webhook_events = relationship("WebhookEvent", back_populates="user")
28 |
--------------------------------------------------------------------------------
/backend/app/models/webhook.py:
--------------------------------------------------------------------------------
1 | import enum
2 | import uuid
3 | from sqlalchemy import (
4 | Column,
5 | Integer,
6 | JSON,
7 | DateTime,
8 | ForeignKey,
9 | Enum,
10 | Boolean,
11 | UUID,
12 | )
13 | from sqlalchemy.sql import func
14 | from sqlalchemy.orm import relationship
15 |
16 | from app.db.base import Base
17 |
18 |
19 | class WebhookEventType(str, enum.Enum):
20 | IMPORT_STARTED = "import.started"
21 | IMPORT_VALIDATION_ERROR = "import.validation_error"
22 | IMPORT_PROGRESS = "import.progress"
23 | IMPORT_FINISHED = "import.finished"
24 | IMPORT_FAILED = "import.failed"
25 |
26 |
27 | class WebhookEvent(Base):
28 | __tablename__ = "webhook_events"
29 |
30 | id = Column(UUID, primary_key=True, default=uuid.uuid4)
31 | user_id = Column(UUID, ForeignKey("users.id"), nullable=False)
32 | import_job_id = Column(UUID, ForeignKey("import_jobs.id"), nullable=False)
33 | event_type = Column(Enum(WebhookEventType), nullable=False)
34 | payload = Column(JSON, nullable=False)
35 | delivered = Column(Boolean, default=False, nullable=False)
36 | delivery_attempts = Column(Integer, default=0, nullable=False)
37 | last_delivery_attempt = Column(DateTime(timezone=True), nullable=True)
38 | created_at = Column(DateTime(timezone=True), server_default=func.now())
39 |
40 | # Relationships - using simple string references
41 | user = relationship("User", back_populates="webhook_events")
42 | import_job = relationship("ImportJob", back_populates="webhook_events")
43 |
--------------------------------------------------------------------------------
/backend/app/schemas/auth.py:
--------------------------------------------------------------------------------
1 | from pydantic import BaseModel, EmailStr
2 |
3 | class TokenResponse(BaseModel):
4 | """Schema for token response containing access and refresh tokens"""
5 | access_token: str
6 | refresh_token: str
7 | token_type: str
8 |
9 |
10 | class RefreshTokenRequest(BaseModel):
11 | """Schema for refresh token request"""
12 | refresh_token: str
13 |
14 |
15 | class PasswordResetRequest(BaseModel):
16 | """Schema for password reset request"""
17 | email: EmailStr
18 |
--------------------------------------------------------------------------------
/backend/app/schemas/import_job.py:
--------------------------------------------------------------------------------
1 | import uuid
2 | from datetime import datetime
3 |
4 | from pydantic import BaseModel, computed_field
5 | from typing import Dict, Any, List, Optional
6 |
7 | from app.models.import_job import ImportStatus
8 |
9 |
10 | # Base ImportJob model
11 | class ImportJobBase(BaseModel):
12 | importer_id: uuid.UUID
13 | file_name: str
14 | file_type: str
15 |
16 |
17 | # ImportJob creation model
18 | class ImportJobCreate(ImportJobBase):
19 | pass
20 |
21 |
22 | # ImportJob update model
23 | class ImportJobUpdate(BaseModel):
24 | status: Optional[ImportStatus] = None
25 | processed_rows: Optional[int] = None
26 | error_count: Optional[int] = None
27 | errors: Optional[Dict[str, Any]] = None
28 | column_mapping: Optional[Dict[str, str]] = None
29 | file_metadata: Optional[Dict[str, Any]] = None
30 |
31 |
32 | # ImportJob in DB
33 | class ImportJobInDBBase(ImportJobBase):
34 | id: uuid.UUID
35 | user_id: uuid.UUID
36 | status: ImportStatus
37 | row_count: int
38 | processed_rows: int
39 | error_count: int
40 | errors: Optional[Dict[str, Any]] = None
41 | column_mapping: Optional[Dict[str, str]] = None
42 | file_metadata: Optional[Dict[str, Any]] = None
43 | created_at: datetime
44 | updated_at: Optional[datetime] = None
45 | completed_at: Optional[datetime] = None
46 |
47 | class Config:
48 | from_attributes = True
49 |
50 |
51 | # ImportJob to return via API (Uses computed_field with different names and aliases)
52 | class ImportJob(ImportJobInDBBase):
53 |
54 | @computed_field(alias="id") # Keep 'id' in JSON output using alias
55 | @property
56 | def id_str(self) -> str: # Change property name
57 | return str(self.id)
58 |
59 | @computed_field(alias="user_id") # Keep 'user_id' in JSON output
60 | @property
61 | def user_id_str(self) -> str: # Change property name
62 | return str(self.user_id)
63 |
64 | @computed_field(alias="importer_id") # Keep 'importer_id' in JSON output
65 | @property
66 | def importer_id_str(self) -> str: # Change property name
67 | return str(self.importer_id)
68 |
69 | class Config:
70 | from_attributes = True
71 | # Exclude the original UUID fields from the response if needed,
72 | # though aliasing might handle this implicitly. Let's try without exclude first.
73 | # exclude = {'id', 'user_id', 'importer_id'}
74 |
75 |
76 | # Column mapping model
77 | class ColumnMapping(BaseModel):
78 | file_column: str
79 | importer_field: str
80 | confidence: float = 0.0
81 |
82 |
83 | # Column mapping request
84 | class ColumnMappingRequest(BaseModel):
85 | mappings: List[ColumnMapping]
86 |
87 |
88 | # Import request for importer-key based authentication
89 | class ImportByKeyRequest(BaseModel):
90 | validData: List[Dict[str, Any]]
91 | invalidData: List[Dict[str, Any]] = []
92 | columnMapping: Dict[str, Any] = {}
93 | user: Dict[str, Any] = {}
94 | metadata: Dict[str, Any] = {}
95 | importer_key: uuid.UUID
96 |
97 |
98 | # Simplified response for import processing
99 | class ImportProcessResponse(BaseModel):
100 | success: bool
101 |
--------------------------------------------------------------------------------
/backend/app/schemas/schema.py:
--------------------------------------------------------------------------------
1 | import uuid
2 | from datetime import datetime
3 |
4 | from pydantic import BaseModel, Field
5 | from typing import Dict, Any, List, Optional, Literal
6 |
7 |
8 | # Schema field definition
9 | class SchemaField(BaseModel):
10 | name: str
11 | display_name: Optional[str] = None
12 | type: str # text, number, date, email, phone, boolean, select, custom_regex
13 | required: bool = False
14 | description: Optional[str] = None
15 | must_match: bool = False # Require that users must match this column
16 | not_blank: bool = False # Value cannot be blank
17 | example: Optional[str] = None # Example value for the field
18 | validation_error_message: Optional[str] = None # Custom validation error message
19 | validation_format: Optional[str] = (
20 | None # For date format, regex pattern, or select options
21 | )
22 | validation: Optional[Dict[str, Any]] = None # JSON Schema validation rules
23 | template: Optional[str] = (
24 | None # Template for boolean or select fields (e.g., 'true/false', 'yes/no', '1/0')
25 | )
26 |
27 | def dict(self, *args, **kwargs):
28 | # Ensure all fields are serializable
29 | result = super().dict(*args, **kwargs)
30 | # Remove None values to keep the JSON clean
31 | return {k: v for k, v in result.items() if v is not None}
32 |
33 | class Config:
34 | from_attributes = True
35 |
36 |
37 | # Base Schema model
38 | class SchemaBase(BaseModel):
39 | name: str
40 | description: Optional[str] = None
41 | fields: List[SchemaField]
42 |
43 | class Config:
44 | from_attributes = True
45 |
46 |
47 | # Schema creation model
48 | class SchemaCreate(SchemaBase):
49 | pass
50 |
51 |
52 | # Schema update model
53 | class SchemaUpdate(BaseModel):
54 | name: Optional[str] = None
55 | description: Optional[str] = None
56 | fields: Optional[List[SchemaField]] = None
57 |
58 |
59 | # Schema in DB
60 | class SchemaInDBBase(SchemaBase):
61 | id: uuid.UUID
62 | user_id: uuid.UUID
63 | created_at: datetime
64 | updated_at: Optional[datetime] = None
65 |
66 | class Config:
67 | from_attributes = True
68 |
69 |
70 | # Schema to return via API
71 | class Schema(SchemaInDBBase):
72 | # Convert UUID fields to strings for API responses
73 | id: str
74 | user_id: str
75 |
76 | class Config:
77 | from_attributes = True
78 |
--------------------------------------------------------------------------------
/backend/app/schemas/token.py:
--------------------------------------------------------------------------------
1 | from datetime import datetime
2 | from uuid import UUID
3 |
4 | from typing import Optional
5 | from pydantic import BaseModel
6 |
7 |
8 | class TokenPayload(BaseModel):
9 | sub: str
10 | exp: int
11 | iat: int
12 | jti: str
13 | aud: list[str]
14 |
15 |
16 | class TokenData(BaseModel):
17 | # JWT standard fields
18 | sub: str
19 | exp: int
20 | iat: int
21 | jti: str
22 | aud: Optional[list[str]] = None
23 |
24 | # Convenience fields for our application
25 | user_id: Optional[UUID] = None
26 | token_id: Optional[str] = None
27 | expires_at: Optional[datetime] = None
28 |
--------------------------------------------------------------------------------
/backend/app/schemas/user.py:
--------------------------------------------------------------------------------
1 | import uuid
2 | from datetime import datetime
3 |
4 | from pydantic import BaseModel, EmailStr, Field
5 | from typing import Optional
6 | from fastapi_users.schemas import BaseUser, BaseUserCreate, BaseUserUpdate
7 |
8 |
9 | class UserRead(BaseUser[uuid.UUID]):
10 | full_name: Optional[str] = None
11 | created_at: datetime
12 | updated_at: Optional[datetime] = None
13 |
14 | class Config:
15 | from_attributes = True
16 |
17 |
18 | class UserCreate(BaseUserCreate):
19 | full_name: Optional[str] = None
20 |
21 |
22 | class UserUpdate(BaseUserUpdate):
23 | full_name: Optional[str] = None
24 |
25 |
26 | # Public registration schema
27 | class UserRegister(BaseModel):
28 | email: EmailStr
29 | password: str = Field(..., min_length=8)
30 | full_name: Optional[str] = None
31 |
32 |
33 | # For backward compatibility with existing code
34 | class User(UserRead):
35 | pass
36 |
37 |
38 | # For internal use
39 | class UserInDB(UserRead):
40 | hashed_password: str
41 |
--------------------------------------------------------------------------------
/backend/app/schemas/webhook.py:
--------------------------------------------------------------------------------
1 | from pydantic import BaseModel, Field, HttpUrl
2 | from typing import Dict, Any, List, Optional
3 | from datetime import datetime
4 | import uuid
5 | from app.models.webhook import WebhookEventType
6 |
7 |
8 | # Base WebhookEvent model
9 | class WebhookEventBase(BaseModel):
10 | event_type: str
11 | payload: Dict[str, Any]
12 |
13 |
14 | # WebhookEvent creation model
15 | class WebhookEventCreate(WebhookEventBase):
16 | importer_id: Optional[uuid.UUID] = None
17 | import_job_id: Optional[uuid.UUID] = None
18 |
19 |
20 | # WebhookEvent in DB
21 | class WebhookEventInDBBase(WebhookEventBase):
22 | id: uuid.UUID
23 | user_id: uuid.UUID
24 | import_job_id: uuid.UUID
25 | delivered: bool
26 | delivery_attempts: int
27 | last_delivery_attempt: Optional[datetime] = None
28 | created_at: datetime
29 |
30 | class Config:
31 | from_attributes = True
32 |
33 |
34 | # WebhookEvent to return via API
35 | class WebhookEvent(WebhookEventInDBBase):
36 | pass
37 |
38 |
39 | # Webhook configuration
40 | class WebhookConfig(BaseModel):
41 | url: HttpUrl
42 | secret: str
43 | events: List[WebhookEventType]
44 | description: Optional[str] = None
45 | active: bool = True
46 |
47 |
48 | # Webhook configuration create
49 | class WebhookConfigCreate(WebhookConfig):
50 | pass
51 |
52 |
53 | # Webhook configuration update
54 | class WebhookConfigUpdate(BaseModel):
55 | url: Optional[HttpUrl] = None
56 | secret: Optional[str] = None
57 | events: Optional[List[WebhookEventType]] = None
58 | description: Optional[str] = None
59 | active: Optional[bool] = None
60 |
--------------------------------------------------------------------------------
/backend/app/services/auth.py:
--------------------------------------------------------------------------------
1 | """
2 | Modern authentication service using FastAPI-Users.
3 | """
4 | from typing import Optional, Dict, Any
5 | from uuid import UUID
6 |
7 | from fastapi import Depends
8 | from sqlalchemy.orm import Session
9 |
10 | from app.db.base import get_db
11 | from app.auth.users import get_current_active_user
12 | from app.models.user import User
13 |
14 |
15 | async def get_user_by_id(user_id: UUID, db: Session = Depends(get_db)) -> Optional[User]:
16 | """
17 | Get a user by ID
18 | """
19 | return db.query(User).filter(User.id == user_id).first()
20 |
21 |
22 | async def get_user_by_email(email: str, db: Session = Depends(get_db)) -> Optional[User]:
23 | """
24 | Get a user by email
25 | """
26 | return db.query(User).filter(User.email == email).first()
27 |
28 |
29 | async def get_current_user_data(
30 | current_user: User = Depends(get_current_active_user),
31 | ) -> Dict[str, Any]:
32 | """
33 | Get the current user's data
34 | """
35 | return {
36 | "id": current_user.id,
37 | "email": current_user.email,
38 | "full_name": current_user.full_name,
39 | "is_active": current_user.is_active,
40 | "is_superuser": current_user.is_superuser,
41 | "is_verified": current_user.is_verified,
42 | }
43 |
--------------------------------------------------------------------------------
/backend/app/worker.py:
--------------------------------------------------------------------------------
1 | """
2 | Worker module for processing background jobs with RQ (Redis Queue)
3 | """
4 | import os
5 | import sys
6 |
7 | # Set environment variable to prevent macOS fork safety issues
8 | # This is needed when running on macOS to prevent objc runtime errors
9 | os.environ['OBJC_DISABLE_INITIALIZE_FORK_SAFETY'] = 'YES'
10 |
11 | from redis import Redis
12 | from rq import Worker, Queue
13 |
14 | from app.core.config import settings
15 |
16 | # Try to import logging, but provide a fallback if not available
17 | try:
18 | from app.core.logging import setup_logging
19 | logger = setup_logging(__name__)
20 | except ImportError:
21 | import logging
22 | logger = logging.getLogger(__name__)
23 | logging.basicConfig(level=logging.INFO)
24 |
25 | # Connect to Redis
26 | redis_conn = Redis.from_url(settings.REDIS_URL)
27 |
28 | # Define queues to listen to
29 | QUEUES = [settings.RQ_IMPORT_QUEUE, 'default']
30 |
31 | def start_worker():
32 | """Start a worker process to listen for jobs"""
33 | logger.info(f"Starting RQ worker, listening to queues: {', '.join(QUEUES)}")
34 | logger.info(f"Using Redis at: {settings.REDIS_URL}")
35 |
36 | try:
37 | # Create queues from names
38 | queue_list = [Queue(name, connection=redis_conn) for name in QUEUES]
39 |
40 | # Create and start worker
41 | worker = Worker(queue_list, connection=redis_conn)
42 | logger.info(f"Worker started with ID: {worker.key}")
43 | worker.work(with_scheduler=True)
44 | except Exception as e:
45 | logger.error(f"Worker failed: {str(e)}")
46 | sys.exit(1)
47 |
48 | if __name__ == "__main__":
49 | start_worker()
50 |
--------------------------------------------------------------------------------
/backend/app/workers/__init__.py:
--------------------------------------------------------------------------------
1 | # Import worker package
2 |
--------------------------------------------------------------------------------
/backend/app/workers/import_worker.py:
--------------------------------------------------------------------------------
1 | """
2 | Import worker module for processing CSV import jobs from the Redis Queue.
3 |
4 | This module contains worker functions that are executed by Redis Queue workers.
5 | It delegates all business logic to the ImportService class.
6 | """
7 |
8 | import asyncio
9 | import logging
10 | from typing import Dict, Any, List
11 |
12 | from app.db.base import SessionLocal
13 | from app.services.import_service import import_service
14 |
15 | logger = logging.getLogger(__name__)
16 |
17 |
18 | def process_import_job(
19 | import_job_id: str,
20 | valid_data: List[Dict[str, Any]],
21 | invalid_data: List[Dict[str, Any]] = None,
22 | ) -> Dict[str, Any]:
23 | """
24 | Process a data import job in the background using Redis Queue.
25 |
26 | This worker function is called by RQ and delegates to the ImportService.
27 |
28 | Args:
29 | import_job_id (str): The ID of the import job
30 | valid_data (List[Dict[str, Any]]): List of valid data rows
31 | invalid_data (List[Dict[str, Any]], optional): List of invalid data rows
32 |
33 | Returns:
34 | Dict[str, Any]: Results of the import process
35 | """
36 | logger.info(f"RQ Worker: Starting import job {import_job_id}")
37 |
38 | # Create a new database session for this worker
39 | db = SessionLocal()
40 |
41 | try:
42 | # Since we're in a worker process, we need to create a new event loop
43 | loop = asyncio.new_event_loop()
44 | asyncio.set_event_loop(loop)
45 |
46 | try:
47 | # Process the import job using the import service
48 | import_job, processed_df = loop.run_until_complete(
49 | import_service.process_import_data(
50 | db=db,
51 | import_job_id=import_job_id,
52 | valid_data=valid_data,
53 | invalid_data=invalid_data if invalid_data else [],
54 | )
55 | )
56 |
57 | if not import_job:
58 | return {
59 | "status": "error",
60 | "message": f"Import job {import_job_id} not found or processing failed",
61 | }
62 |
63 | logger.info(f"RQ Worker: Import job {import_job_id} completed successfully")
64 | return {
65 | "status": "success",
66 | "message": "Import completed successfully",
67 | "total_rows": import_job.row_count,
68 | "processed_rows": import_job.processed_rows,
69 | "error_count": import_job.error_count,
70 | }
71 | finally:
72 | loop.close()
73 |
74 | except Exception as e:
75 | logger.error(
76 | f"RQ Worker: Error processing import job {import_job_id}: {str(e)}",
77 | exc_info=True,
78 | )
79 | return {"status": "error", "message": str(e)}
80 |
81 | finally:
82 | # Close the database session
83 | db.close()
84 |
--------------------------------------------------------------------------------
/backend/create_migration.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 | import subprocess
4 | from dotenv import load_dotenv
5 |
6 | # Load environment variables
7 | load_dotenv()
8 |
9 | def create_migration(message):
10 | """Create a new Alembic migration with the given message."""
11 | try:
12 | # Generate the migration
13 | result = subprocess.run(
14 | ["alembic", "revision", "--autogenerate", "-m", message],
15 | check=True,
16 | capture_output=True,
17 | text=True
18 | )
19 | print(result.stdout)
20 | print("Migration file created successfully!")
21 | except subprocess.CalledProcessError as e:
22 | print(f"Error creating migration: {e}")
23 | print(f"Error output: {e.stderr}")
24 | sys.exit(1)
25 |
26 | def apply_migrations():
27 | """Apply all pending migrations."""
28 | try:
29 | # Apply the migrations
30 | result = subprocess.run(
31 | ["alembic", "upgrade", "head"],
32 | check=True,
33 | capture_output=True,
34 | text=True
35 | )
36 | print(result.stdout)
37 | print("Migrations applied successfully!")
38 | except subprocess.CalledProcessError as e:
39 | print(f"Error applying migrations: {e}")
40 | print(f"Error output: {e.stderr}")
41 | sys.exit(1)
42 |
43 | if __name__ == "__main__":
44 | if len(sys.argv) < 2:
45 | print("Usage: python create_migration.py [--apply]")
46 | sys.exit(1)
47 |
48 | message = sys.argv[1]
49 | create_migration(message)
50 |
51 | # Check if --apply flag is provided
52 | if len(sys.argv) > 2 and sys.argv[2] == "--apply":
53 | apply_migrations()
54 |
--------------------------------------------------------------------------------
/backend/init_app.py:
--------------------------------------------------------------------------------
1 | import os
2 | import asyncio
3 | from app.db.init_db import init_db_and_create_superuser
4 | from dotenv import load_dotenv
5 |
6 | # Load environment variables
7 | load_dotenv()
8 |
9 | async def init():
10 | # Initialize database and create superuser
11 | await init_db_and_create_superuser()
12 | print("Database initialization completed")
13 |
14 | if __name__ == "__main__":
15 | asyncio.run(init())
16 |
--------------------------------------------------------------------------------
/backend/migrations/README:
--------------------------------------------------------------------------------
1 | Generic single-database configuration.
--------------------------------------------------------------------------------
/backend/migrations/env.py:
--------------------------------------------------------------------------------
1 | from logging.config import fileConfig
2 | import os
3 | import sys
4 | from dotenv import load_dotenv
5 |
6 | # Add the parent directory to sys.path
7 | sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
8 |
9 | from sqlalchemy import engine_from_config
10 | from sqlalchemy import pool
11 |
12 | from alembic import context
13 |
14 | # Load environment variables from .env file
15 | load_dotenv()
16 |
17 | # Import models to ensure they're registered with the Base metadata
18 | from app.models.user import User
19 | from app.models.importer import Importer
20 | from app.models.import_job import ImportJob
21 | from app.models.webhook import WebhookEvent
22 | from app.db.base import Base
23 | from app.core.config import settings
24 |
25 | # this is the Alembic Config object, which provides
26 | # access to the values within the .ini file in use.
27 | config = context.config
28 |
29 | # Set the SQLAlchemy URL from settings
30 | config.set_main_option("sqlalchemy.url", settings.DATABASE_URL)
31 |
32 | # Interpret the config file for Python logging.
33 | # This line sets up loggers basically.
34 | if config.config_file_name is not None:
35 | fileConfig(config.config_file_name)
36 |
37 | # add your model's MetaData object here
38 | # for 'autogenerate' support
39 | target_metadata = Base.metadata
40 |
41 |
42 | def run_migrations_offline() -> None:
43 | """Run migrations in 'offline' mode.
44 |
45 | This configures the context with just a URL
46 | and not an Engine, though an Engine is acceptable
47 | here as well. By skipping the Engine creation
48 | we don't even need a DBAPI to be available.
49 |
50 | Calls to context.execute() here emit the given string to the
51 | script output.
52 |
53 | """
54 | url = config.get_main_option("sqlalchemy.url")
55 | context.configure(
56 | url=url,
57 | target_metadata=target_metadata,
58 | literal_binds=True,
59 | dialect_opts={"paramstyle": "named"},
60 | )
61 |
62 | with context.begin_transaction():
63 | context.run_migrations()
64 |
65 |
66 | def run_migrations_online() -> None:
67 | """Run migrations in 'online' mode.
68 |
69 | In this scenario we need to create an Engine
70 | and associate a connection with the context.
71 |
72 | """
73 | connectable = engine_from_config(
74 | config.get_section(config.config_ini_section, {}),
75 | prefix="sqlalchemy.",
76 | poolclass=pool.NullPool,
77 | )
78 |
79 | with connectable.connect() as connection:
80 | context.configure(
81 | connection=connection, target_metadata=target_metadata
82 | )
83 |
84 | with context.begin_transaction():
85 | context.run_migrations()
86 |
87 |
88 | if context.is_offline_mode():
89 | run_migrations_offline()
90 | else:
91 | run_migrations_online()
92 |
--------------------------------------------------------------------------------
/backend/migrations/script.py.mako:
--------------------------------------------------------------------------------
1 | """${message}
2 |
3 | Revision ID: ${up_revision}
4 | Revises: ${down_revision | comma,n}
5 | Create Date: ${create_date}
6 |
7 | """
8 | from typing import Sequence, Union
9 |
10 | from alembic import op
11 | import sqlalchemy as sa
12 | ${imports if imports else ""}
13 |
14 | # revision identifiers, used by Alembic.
15 | revision: str = ${repr(up_revision)}
16 | down_revision: Union[str, None] = ${repr(down_revision)}
17 | branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
18 | depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
19 |
20 |
21 | def upgrade() -> None:
22 | """Upgrade schema."""
23 | ${upgrades if upgrades else "pass"}
24 |
25 |
26 | def downgrade() -> None:
27 | """Downgrade schema."""
28 | ${downgrades if downgrades else "pass"}
29 |
--------------------------------------------------------------------------------
/backend/migrations/versions/98f92bc3f715_add_webhook_data_fields.py:
--------------------------------------------------------------------------------
1 | """add_webhook_data_fields
2 |
3 | Revision ID: 98f92bc3f715
4 | Revises: 34203d60870b
5 | Create Date: 2025-04-24 15:13:23.547319
6 |
7 | """
8 | from typing import Sequence, Union
9 |
10 | from alembic import op
11 | import sqlalchemy as sa
12 |
13 |
14 | # revision identifiers, used by Alembic.
15 | revision: str = '98f92bc3f715'
16 | down_revision: Union[str, None] = '34203d60870b'
17 | branch_labels: Union[str, Sequence[str], None] = None
18 | depends_on: Union[str, Sequence[str], None] = None
19 |
20 |
21 | def upgrade() -> None:
22 | """Upgrade schema."""
23 | # ### commands auto generated by Alembic - please adjust! ###
24 | op.add_column('importers', sa.Column('include_data_in_webhook', sa.Boolean(), nullable=True))
25 | op.add_column('importers', sa.Column('webhook_data_sample_size', sa.Integer(), nullable=True))
26 | op.alter_column('importers', 'webhook_url',
27 | existing_type=sa.VARCHAR(),
28 | nullable=True)
29 | # ### end Alembic commands ###
30 |
31 |
32 | def downgrade() -> None:
33 | """Downgrade schema."""
34 | # ### commands auto generated by Alembic - please adjust! ###
35 | op.alter_column('importers', 'webhook_url',
36 | existing_type=sa.VARCHAR(),
37 | nullable=False)
38 | op.drop_column('importers', 'webhook_data_sample_size')
39 | op.drop_column('importers', 'include_data_in_webhook')
40 | # ### end Alembic commands ###
41 |
--------------------------------------------------------------------------------
/backend/migrations/versions/add_token_blacklist.py:
--------------------------------------------------------------------------------
1 | """Add token blacklist table
2 |
3 | Revision ID: add_token_blacklist
4 | Revises:
5 | Create Date: 2025-04-24
6 |
7 | """
8 | from alembic import op
9 | import sqlalchemy as sa
10 | from sqlalchemy.dialects import postgresql
11 | import uuid
12 |
13 |
14 | # revision identifiers, used by Alembic.
15 | revision = 'add_token_blacklist'
16 | down_revision = '98f92bc3f715' # Updated to depend on the previous migration
17 | branch_labels = None
18 | depends_on = None
19 |
20 |
21 | def upgrade():
22 | # Check if token_blacklist table exists
23 | conn = op.get_bind()
24 | inspector = sa.inspect(conn)
25 | tables = inspector.get_table_names()
26 |
27 | if 'token_blacklist' not in tables:
28 | # Create token_blacklist table
29 | op.create_table('token_blacklist',
30 | sa.Column('id', postgresql.UUID(as_uuid=True), primary_key=True, default=uuid.uuid4),
31 | sa.Column('token_id', sa.String(), nullable=False, index=True, unique=True),
32 | sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
33 | sa.Column('user_id', postgresql.UUID(as_uuid=True), sa.ForeignKey('users.id', ondelete='CASCADE'), nullable=True),
34 | )
35 |
36 | # Check if index exists before creating it
37 | indexes = inspector.get_indexes('token_blacklist')
38 | index_names = [index['name'] for index in indexes]
39 |
40 | if 'ix_token_blacklist_token_id' not in index_names:
41 | # Create index on token_id for faster lookups
42 | op.create_index(op.f('ix_token_blacklist_token_id'), 'token_blacklist', ['token_id'], unique=True)
43 | else:
44 | print("Table 'token_blacklist' already exists, skipping creation")
45 |
46 |
47 | def downgrade():
48 | # Check if token_blacklist table exists before dropping
49 | conn = op.get_bind()
50 | inspector = sa.inspect(conn)
51 | tables = inspector.get_table_names()
52 |
53 | if 'token_blacklist' in tables:
54 | # Check if index exists before dropping it
55 | indexes = inspector.get_indexes('token_blacklist')
56 | index_names = [index['name'] for index in indexes]
57 |
58 | if 'ix_token_blacklist_token_id' in index_names:
59 | # Drop index
60 | op.drop_index(op.f('ix_token_blacklist_token_id'), table_name='token_blacklist')
61 |
62 | # Drop table
63 | op.drop_table('token_blacklist')
64 |
--------------------------------------------------------------------------------
/backend/migrations/versions/c734150f184e_add_key_to_importer.py:
--------------------------------------------------------------------------------
1 | """Add key to importer
2 |
3 | Revision ID: c734150f184e
4 | Revises: add_token_blacklist
5 | Create Date: 2025-04-26 13:51:24.378207
6 |
7 | """
8 | from typing import Sequence, Union
9 |
10 | from alembic import op
11 | import sqlalchemy as sa
12 | from sqlalchemy.dialects import postgresql
13 |
14 | # revision identifiers, used by Alembic.
15 | revision: str = 'c734150f184e'
16 | down_revision: Union[str, None] = 'add_token_blacklist'
17 | branch_labels: Union[str, Sequence[str], None] = None
18 | depends_on: Union[str, Sequence[str], None] = None
19 |
20 |
21 | def upgrade() -> None:
22 | """Upgrade schema."""
23 | # ### commands auto generated by Alembic - please adjust! ###
24 | op.drop_index('ix_token_blacklist_token_id', table_name='token_blacklist')
25 | op.drop_table('token_blacklist')
26 | op.add_column('importers', sa.Column('key', sa.UUID(), nullable=True))
27 | op.create_index(op.f('ix_importers_key'), 'importers', ['key'], unique=True)
28 | # ### end Alembic commands ###
29 |
30 |
31 | def downgrade() -> None:
32 | """Downgrade schema."""
33 | # ### commands auto generated by Alembic - please adjust! ###
34 | op.drop_index(op.f('ix_importers_key'), table_name='importers')
35 | op.drop_column('importers', 'key')
36 | op.create_table('token_blacklist',
37 | sa.Column('id', sa.UUID(), autoincrement=False, nullable=False),
38 | sa.Column('token_id', sa.VARCHAR(), autoincrement=False, nullable=False),
39 | sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=False),
40 | sa.Column('user_id', sa.UUID(), autoincrement=False, nullable=True),
41 | sa.ForeignKeyConstraint(['user_id'], ['users.id'], name='token_blacklist_user_id_fkey', ondelete='CASCADE'),
42 | sa.PrimaryKeyConstraint('id', name='token_blacklist_pkey')
43 | )
44 | op.create_index('ix_token_blacklist_token_id', 'token_blacklist', ['token_id'], unique=True)
45 | # ### end Alembic commands ###
46 |
--------------------------------------------------------------------------------
/backend/pytest.ini:
--------------------------------------------------------------------------------
1 | [pytest]
2 | testpaths = tests
3 | python_files = test_*.py
4 | python_classes = Test*
5 | python_functions = test_*
6 | asyncio_mode = auto
7 | markers =
8 | asyncio: mark a test as an asyncio coroutine
9 | slow: mark test as slow
10 | integration: mark as integration test
11 | unit: mark as unit test
12 |
--------------------------------------------------------------------------------
/backend/requirements.txt:
--------------------------------------------------------------------------------
1 | aiohappyeyeballs==2.6.1
2 | aiohttp==3.11.16
3 | aiosignal==1.3.2
4 | alembic==1.15.2
5 | annotated-types==0.7.0
6 | anyio==4.9.0
7 | argon2-cffi==23.1.0
8 | argon2-cffi-bindings==21.2.0
9 | asgi-lifespan==2.1.0
10 | asttokens==3.0.0
11 | asyncpg==0.30.0
12 | attrs==25.3.0
13 | bcrypt==4.3.0
14 | certifi==2025.1.31
15 | cffi==1.17.1
16 | charset-normalizer==3.4.1
17 | click==8.1.8
18 | cryptography==44.0.2
19 | decorator==5.2.1
20 | distro==1.9.0
21 | dnspython==2.7.0
22 | ecdsa==0.19.1
23 | email_validator==2.2.0
24 | et_xmlfile==2.0.0
25 | executing==2.2.0
26 | fastapi==0.115.12
27 | fastapi-users==14.0.1
28 | fastapi-users-db-sqlalchemy==7.0.0
29 | filelock==3.18.0
30 | frozenlist==1.5.0
31 | fsspec==2025.3.2
32 | greenlet==3.2.0
33 | h11==0.14.0
34 | httpcore==1.0.8
35 | httpx==0.28.1
36 | huggingface-hub==0.30.2
37 | idna==3.10
38 | importlib_metadata==8.6.1
39 | iniconfig==2.1.0
40 | ipdb==0.13.13
41 | ipython==9.1.0
42 | ipython_pygments_lexers==1.1.1
43 | itsdangerous==2.2.0
44 | jedi==0.19.2
45 | Jinja2==3.1.6
46 | jiter==0.9.0
47 | jsonschema==4.23.0
48 | jsonschema-specifications==2024.10.1
49 |
50 | makefun==1.15.6
51 | Mako==1.3.10
52 | MarkupSafe==3.0.2
53 | matplotlib-inline==0.1.7
54 | multidict==6.4.3
55 | numpy==2.2.4
56 |
57 | openpyxl==3.1.5
58 | packaging==24.2
59 | pandas==2.2.3
60 | parso==0.8.4
61 | passlib==1.7.4
62 | pexpect==4.9.0
63 | pluggy==1.5.0
64 | prompt_toolkit==3.0.51
65 | propcache==0.3.1
66 | psycopg2-binary==2.9.10
67 | ptyprocess==0.7.0
68 | pure_eval==0.2.3
69 | pwdlib==0.2.1
70 | pyasn1==0.4.8
71 | pycparser==2.22
72 | pydantic==2.11.3
73 | pydantic-settings==2.8.1
74 | pydantic_core==2.33.1
75 | Pygments==2.19.1
76 | PyJWT==2.10.1
77 | pytest==8.3.5
78 | pytest-asyncio==0.26.0
79 | python-dateutil==2.9.0.post0
80 | python-dotenv==1.1.0
81 | python-jose==3.4.0
82 | python-multipart==0.0.20
83 | pytz==2025.2
84 | PyYAML==6.0.2
85 | redis==5.2.1
86 | referencing==0.36.2
87 | regex==2024.11.6
88 | requests==2.32.3
89 | rpds-py==0.24.0
90 | rq==2.3.2
91 | rsa==4.9
92 | six==1.17.0
93 | sniffio==1.3.1
94 | SQLAlchemy==2.0.40
95 | stack-data==0.6.3
96 | starlette==0.46.2
97 |
98 | tokenizers==0.21.1
99 | tqdm==4.67.1
100 | traitlets==5.14.3
101 | typing-inspection==0.4.0
102 | typing_extensions==4.13.2
103 | tzdata==2025.2
104 | urllib3==2.4.0
105 | uvicorn==0.34.1
106 | wcwidth==0.2.13
107 | yarl==1.19.0
108 | zipp==3.21.0
109 |
--------------------------------------------------------------------------------
/backend/scripts/reset_password.py:
--------------------------------------------------------------------------------
1 | # scripts/reset_password.py
2 | import sys
3 | import os
4 | import asyncio
5 | from sqlalchemy import create_engine
6 | from sqlalchemy.orm import sessionmaker
7 | from fastapi_users.password import PasswordHelper
8 |
9 | # Adjust the path to import from the app directory
10 | backend_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
11 | sys.path.append(backend_dir)
12 |
13 | from app.core.config import settings
14 | from app.models.user import User # Adjust import path if necessary
15 | from app.models.schema import Schema # Add import for Schema model
16 | from app.models.import_job import ImportJob # Add import for ImportJob model
17 | from app.models.webhook import WebhookEvent # Add import for WebhookEvent model
18 |
19 | # Create a password helper from FastAPI-Users
20 | password_helper = PasswordHelper()
21 |
22 | DATABASE_URL = settings.DATABASE_URL
23 | engine = create_engine(DATABASE_URL)
24 | SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
25 |
26 | def reset_user_password(email: str, new_password: str):
27 | db = SessionLocal()
28 | try:
29 | user = db.query(User).filter(User.email == email).first()
30 | if not user:
31 | print(f"Error: User with email {email} not found.")
32 | return
33 |
34 | # Use FastAPI-Users' password helper for hashing
35 | hashed_password = password_helper.hash(new_password)
36 | user.hashed_password = hashed_password
37 | user.is_active = True # Ensure the user is active
38 | db.add(user)
39 | db.commit()
40 | print(f"Password for user {email} has been reset and user activated successfully.")
41 |
42 | except Exception as e:
43 | db.rollback()
44 | print(f"An error occurred: {e}")
45 | finally:
46 | db.close()
47 |
48 | if __name__ == "__main__":
49 | if len(sys.argv) != 3:
50 | print("Usage: python scripts/reset_password.py ")
51 | sys.exit(1)
52 |
53 | user_email = sys.argv[1]
54 | new_password = sys.argv[2]
55 |
56 | reset_user_password(user_email, new_password)
57 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: "3.8"
2 |
3 | services:
4 | postgres:
5 | image: postgres:16
6 | container_name: importcsv-postgres
7 | environment:
8 | POSTGRES_USER: postgres
9 | POSTGRES_PASSWORD: postgres
10 | POSTGRES_DB: importcsv
11 | ports:
12 | - "5432:5432"
13 | volumes:
14 | - postgres_data:/var/lib/postgresql/data
15 | restart: unless-stopped
16 |
17 | redis:
18 | image: redis:7-alpine
19 | container_name: importcsv-redis
20 | ports:
21 | - "6379:6379"
22 | volumes:
23 | - redis_data:/data
24 | restart: unless-stopped
25 | command: redis-server --appendonly yes
26 |
27 | backend:
28 | build:
29 | context: ./backend
30 | dockerfile: Dockerfile
31 | container_name: importcsv-backend
32 | depends_on:
33 | - postgres
34 | - redis
35 | ports:
36 | - "8000:8000"
37 | environment:
38 | - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/importcsv
39 | - REDIS_URL=redis://redis:6379/0
40 | - ENVIRONMENT=development
41 | - SECRET_KEY=your-secret-key-at-least-32-characters-long
42 | - WEBHOOK_SECRET=your-webhook-secret-for-callbacks
43 | volumes:
44 | - ./backend:/app
45 | - ./backend/uploads:/app/uploads
46 | - backend_logs:/app/logs
47 | restart: unless-stopped
48 | command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload
49 |
50 | worker:
51 | build:
52 | context: ./backend
53 | dockerfile: Dockerfile
54 | container_name: importcsv-worker
55 | depends_on:
56 | - postgres
57 | - redis
58 | - backend
59 | environment:
60 | - DATABASE_URL=postgresql://postgres:postgres@postgres:5432/importcsv
61 | - REDIS_URL=redis://redis:6379/0
62 | - ENVIRONMENT=development
63 | - SECRET_KEY=your-secret-key-at-least-32-characters-long
64 | - WEBHOOK_SECRET=your-webhook-secret-for-callbacks
65 | volumes:
66 | - ./backend:/app
67 | - ./backend/uploads:/app/uploads
68 | restart: unless-stopped
69 | command: python -m app.worker
70 |
71 | admin:
72 | build:
73 | context: .
74 | dockerfile: admin/Dockerfile
75 | container_name: importcsv-admin
76 | depends_on:
77 | - backend
78 | ports:
79 | - "3000:3000"
80 | environment:
81 | - NEXT_PUBLIC_API_URL=http://backend:8000
82 | volumes:
83 | - ./admin/src:/app/src
84 | - ./admin/public:/app/public
85 | - /app/node_modules
86 | - /app/.next
87 | restart: unless-stopped
88 |
89 | volumes:
90 | postgres_data:
91 | redis_data:
92 | backend_logs:
93 |
--------------------------------------------------------------------------------
/docs/assets/demo.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/abhishekray07/importcsv/205669cc0d0bf442c5b2cacd22ef56bc3f2ec457/docs/assets/demo.mp4
--------------------------------------------------------------------------------
/docs/assets/importer.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/abhishekray07/importcsv/205669cc0d0bf442c5b2cacd22ef56bc3f2ec457/docs/assets/importer.png
--------------------------------------------------------------------------------
/docs/assets/mapping.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/abhishekray07/importcsv/205669cc0d0bf442c5b2cacd22ef56bc3f2ec457/docs/assets/mapping.png
--------------------------------------------------------------------------------
/docs/assets/schema.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/abhishekray07/importcsv/205669cc0d0bf442c5b2cacd22ef56bc3f2ec457/docs/assets/schema.png
--------------------------------------------------------------------------------
/docs/assets/validation.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/abhishekray07/importcsv/205669cc0d0bf442c5b2cacd22ef56bc3f2ec457/docs/assets/validation.png
--------------------------------------------------------------------------------
/docs/assets/webhooks.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/abhishekray07/importcsv/205669cc0d0bf442c5b2cacd22ef56bc3f2ec457/docs/assets/webhooks.png
--------------------------------------------------------------------------------
/frontend/.babelrc.json:
--------------------------------------------------------------------------------
1 | {
2 | "sourceType": "unambiguous",
3 | "presets": [
4 | [
5 | "@babel/preset-env",
6 | {
7 | "targets": {
8 | "chrome": 100
9 | }
10 | }
11 | ],
12 | "@babel/preset-typescript",
13 | "@babel/preset-react"
14 | ],
15 | "plugins": []
16 | }
17 |
--------------------------------------------------------------------------------
/frontend/.gitignore:
--------------------------------------------------------------------------------
1 | *.exe
2 | *.iml
3 | *.swp
4 | *.swo
5 | *.dll
6 | *.so
7 | *.dylib
8 | *.test
9 | *.out
10 | *.log
11 | *.env
12 | *.zip
13 |
14 | # debug
15 | npm-debug.log*
16 | yarn-debug.log*
17 | yarn-error.log*
18 |
19 | node_modules/
20 | vendor/
21 | build/
22 | .yarn/
23 | .pnp*
24 |
25 | .DS_Store
26 | .idea
27 | .vscode
28 |
29 | /coverage
30 | /.next/
31 | /out/
32 |
33 | # misc
34 | *.pem
35 |
36 | # local env files
37 | .env*.local
38 |
39 | # vercel
40 | .vercel
41 |
42 | # typescript
43 | *.tsbuildinfo
44 | next-env.d.ts
45 |
--------------------------------------------------------------------------------
/frontend/.prettierrc:
--------------------------------------------------------------------------------
1 | {
2 | "semi": true,
3 | "singleQuote": false,
4 | "tabWidth": 2,
5 | "jsxSingleQuote": false,
6 | "bracketSameLine": true,
7 | "printWidth": 150,
8 | "endOfLine": "auto",
9 | "importOrder": [
10 | "^[^/.]+$",
11 | "^@.+$",
12 | "^\\.\\./(.*)/[A-Z]((?!\\.)(?!types.).)+$",
13 | "^\\.\\./(.*)/((?!\\.)(?!types.).)+$",
14 | "^\\.$",
15 | "^\\./(.*)/[A-Z]((?!\\.)(?!types.).)+$",
16 | "^\\./(.*)/((?!\\.)(?!types.).)+$",
17 | "/types$",
18 | "\\.(css|scss)$",
19 | "\\.(svg|png|jpg|jpeg|gif)$",
20 | ".*"
21 | ],
22 | "importOrderSeparation": false,
23 | "importOrderSortSpecifiers": true,
24 | "importOrderCaseInsensitive": true
25 | }
26 |
--------------------------------------------------------------------------------
/frontend/.yarnrc.yml:
--------------------------------------------------------------------------------
1 | nodeLinker: node-modules
2 |
--------------------------------------------------------------------------------
/frontend/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2023 Portola Labs, Inc.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/frontend/postcss.config.cjs:
--------------------------------------------------------------------------------
1 | module.exports = {
2 | plugins: {
3 | 'tailwindcss/nesting': {},
4 | tailwindcss: {},
5 | autoprefixer: {},
6 | },
7 | }
8 |
--------------------------------------------------------------------------------
/frontend/rollup.config-js.js:
--------------------------------------------------------------------------------
1 | import peerDepsExternal from "rollup-plugin-peer-deps-external";
2 |
3 | import postcss from "rollup-plugin-postcss";
4 | import typescript from "rollup-plugin-typescript2";
5 | import commonjs from "@rollup/plugin-commonjs";
6 | import image from "@rollup/plugin-image";
7 | import json from "@rollup/plugin-json";
8 | import resolve, { nodeResolve } from "@rollup/plugin-node-resolve";
9 | import replace from "@rollup/plugin-replace";
10 |
11 | const packageJson = require("./package.json");
12 |
13 | export default {
14 | input: "src/js.tsx",
15 | output: [
16 | {
17 | file: "build/index.js",
18 | format: "umd",
19 | name: "CSVImporter",
20 | sourcemap: true,
21 | },
22 | ],
23 | plugins: [
24 | replace({
25 | "process.env.NODE_ENV": JSON.stringify("production"),
26 | preventAssignment: true,
27 | }),
28 | resolve({
29 | browser: true,
30 | }),
31 | commonjs(),
32 | typescript({ useTsconfigDeclarationDir: true }),
33 | image(),
34 | postcss({}),
35 | json(),
36 | ],
37 | };
--------------------------------------------------------------------------------
/frontend/rollup.config.js:
--------------------------------------------------------------------------------
1 | import peerDepsExternal from "rollup-plugin-peer-deps-external";
2 | import postcss from "rollup-plugin-postcss";
3 | import typescript from "rollup-plugin-typescript2";
4 | import commonjs from "@rollup/plugin-commonjs";
5 | import image from "@rollup/plugin-image";
6 | import json from "@rollup/plugin-json";
7 | import resolve from "@rollup/plugin-node-resolve";
8 |
9 | const packageJson = require("./package.json");
10 |
11 | export default {
12 | input: "src/index.ts",
13 | output: [
14 | {
15 | file: packageJson.main,
16 | format: "cjs",
17 | sourcemap: true,
18 | },
19 | {
20 | file: packageJson.module,
21 | format: "esm",
22 | sourcemap: true,
23 | },
24 | ],
25 | external: ["react", "react-dom", "react/jsx-runtime", "@emotion/react"],
26 | plugins: [
27 | peerDepsExternal(),
28 | resolve({
29 | browser: true,
30 | }),
31 | commonjs(),
32 | typescript({ useTsconfigDeclarationDir: true }),
33 | image(),
34 | postcss({}),
35 | json(),
36 | ],
37 | };
38 |
--------------------------------------------------------------------------------
/frontend/src/components/CSVImporter/style/csv-importer.css:
--------------------------------------------------------------------------------
1 | .CSVImporter {
2 | border: none;
3 | background-color: transparent;
4 | padding: 0 1rem;
5 | border-radius: 1.2rem;
6 | color: inherit;
7 | cursor: pointer;
8 | font-weight: 500;
9 | font-size: 14px;
10 | /* height: 2.4rem; */
11 | display: inline-flex;
12 | gap: 0.5rem;
13 | align-items: center;
14 | transition: filter 0.2s ease-out;
15 | }
16 |
17 | .CSVImporter svg {
18 | display: block;
19 | }
20 |
21 | .CSVImporter svg path {
22 | stroke: currentColor !important;
23 | }
24 |
25 | .CSVImporter:hover,
26 | .CSVImporter:active {
27 | filter: brightness(1.2);
28 | }
29 |
30 | .CSVImporter-dialog::backdrop {
31 | background-color: rgba(0, 0, 0, 0.85);
32 | }
33 |
34 | .CSVImporter-dialog {
35 | border-radius: 1rem;
36 | width: 80vw;
37 | max-height: 80vh;
38 | min-width: 907px;
39 | border: none;
40 | position: fixed;
41 | inset: 0;
42 | padding: 0;
43 | margin: auto;
44 | }
45 |
46 | .CSVImporter-div {
47 | border: none;
48 | display: block;
49 | width: 100%;
50 | max-height: 600px;
51 | overflow-y: auto;
52 | }
53 |
54 | @media (max-width: 768px) {
55 | .CSVImporter-dialog {
56 | width: 90vw;
57 | min-width: 950px;
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/frontend/src/i18n/de.ts:
--------------------------------------------------------------------------------
1 | const translations = {
2 | Upload: "Hochladen",
3 | "Select Header": "Kopfzeile auswählen",
4 | "Map Columns": "Spalten zuordnen",
5 | "Expected Column": "Erwartete Spalten",
6 | Required: "Erforderlich",
7 | "Drop your file here": "Datei hier ablegen",
8 | or: "oder",
9 | "Browse files": "Dateien durchsuchen",
10 | "Download Template": "Vorlage herunterladen",
11 | "Only CSV, XLS, and XLSX files can be uploaded": "Nur CSV-, XLS- und XLSX-Dateien können hochgeladen werden",
12 | "Select Header Row": "Kopfzeilenreihe auswählen",
13 | "Select the row which contains the column headers": "Wähle die Zeile, die die Spaltenüberschriften enthält",
14 | "Only the first sheet ("{{sheet}}") of the Excel file will be imported. To import multiple sheets, please upload each sheet individually.": "Nur das erste Blatt ("{{sheet}}") der Excel-Datei wird importiert. Um mehrere Blätter zu importieren, lade bitte jedes Blatt einzeln hoch.",
15 | "Cancel": "Abbrechen",
16 | "Continue": "Weiter",
17 | "Your File Column": "Deine Spalte der Datei",
18 | "Your Sample Data": "Deine Beispieldaten",
19 | "Destination Column": "Zielspalte",
20 | "Include": "Einfügen",
21 | "Submit": "Senden",
22 | "Loading...": "Laden...",
23 | "Please include all required columns": "Bitte alle erforderlichen Spalten einfügen",
24 | "Back": "Zurück",
25 | "- Select one -": "- Wähle eine aus -",
26 | "- empty -": "- leer -",
27 | "Import Successful": "Import erfolgreich",
28 | "Upload Successful": "Upload erfolgreich",
29 | "Done": "Fertig",
30 | "Upload another file": "Eine weitere Datei hochladen",
31 | };
32 |
33 | export default translations;
34 |
--------------------------------------------------------------------------------
/frontend/src/i18n/es.ts:
--------------------------------------------------------------------------------
1 | const translations = {
2 | Upload: "Subir",
3 | "Select Header": "Seleccionar encabezado",
4 | "Map Columns": "Mapear columnas",
5 | "Expected Column": "Columnas esperadas",
6 | Required: "Requerido",
7 | "Drop your file here": "Suelta tu archivo aquí",
8 | or: "o",
9 | "Browse files": "Examinar archivos",
10 | "Download Template": "Descargar plantilla",
11 | "Only CSV, XLS, and XLSX files can be uploaded": "Solo se pueden subir archivos CSV, XLS y XLSX",
12 | "Select Header Row": "Seleccionar fila de encabezado",
13 | "Select the row which contains the column headers": "Selecciona la fila que contiene los encabezados de las columnas",
14 | "Only the first sheet ("{{sheet}}") of the Excel file will be imported. To import multiple sheets, please upload each sheet individually.": " Solo se importará la primera hoja ("{{sheet}}") del archivo de Excel. Para importar varias hojas, sube cada hoja individualmente.",
15 | "Cancel": "Cancelar",
16 | "Continue": "Continuar",
17 | "Your File Column": "Tu columna del archivo",
18 | "Your Sample Data": "Tus datos de muestra",
19 | "Destination Column": "Columna de destino",
20 | "Include": "Incluir",
21 | "Submit": "Enviar",
22 | "Loading...": "Cargando...",
23 | "Please include all required columns": "Por favor incluye todas las columnas requeridas",
24 | "Back": "Atrás",
25 | "- Select one -": "- Selecciona uno -",
26 | "- empty -": "- vacío -",
27 | "Import Successful": "Importación exitosa",
28 | "Upload Successful": "Se ha subido con éxito",
29 | "Done": "Listo",
30 | "Upload another file": "Subir otro archivo",
31 | };
32 |
33 | export default translations;
34 |
--------------------------------------------------------------------------------
/frontend/src/i18n/fr.ts:
--------------------------------------------------------------------------------
1 | // Translations in french
2 | //TODO: Double the translations
3 | const translations = {
4 | Upload: "Télécharger",
5 | "Select Header": "Sélectionner l'en-tête",
6 | "Map Columns": "Mapper les colonnes",
7 | "Expected Column": "Colonne attendue",
8 | Required: "Requis",
9 | "Drop your file here": "Déposez votre fichier ici",
10 | or: "ou",
11 | "Browse files": "Parcourir les fichiers",
12 | "Download Template": "Télécharger le modèle",
13 | "Only CSV, XLS, and XLSX files can be uploaded": "Seuls les fichiers CSV, XLS et XLSX peuvent être téléchargés",
14 | "Select Header Row": "Sélectionner la ligne d'en-tête",
15 | "Select the row which contains the column headers": "Sélectionnez la ligne qui contient les en-têtes de colonne",
16 | "Only the first sheet ("{{sheet}}") of the Excel file will be imported. To import multiple sheets, please upload each sheet individually.": "Seule la première feuille ("{{sheet}}") du fichier Excel sera importée. Pour importer plusieurs feuilles, veuillez télécharger chaque feuille individuellement.",
17 | "Cancel": "Annuler",
18 | "Continue": "Continuer",
19 | "Your File Column": "Votre colonne de fichier",
20 | "Your Sample Data": "Vos données d'échantillon",
21 | "Destination Column": "Colonne de destination",
22 | "Include": "Inclure",
23 | "Submit": "Soumettre",
24 | "Loading...": "Chargement...",
25 | "Please include all required columns": "Veuillez inclure toutes les colonnes requises",
26 | "Back": "Retour",
27 | "- Select one -": "- Sélectionnez un -",
28 | "- empty -": "- vide -",
29 | "Import Successful": "Importation réussie",
30 | "Upload Successful": "Téléchargement réussi",
31 | "Done": "Terminé",
32 | "Upload another file": "Télécharger un autre fichier",
33 | };
34 |
35 | export default translations;
36 |
--------------------------------------------------------------------------------
/frontend/src/i18n/i18n.ts:
--------------------------------------------------------------------------------
1 | import i18, { Resource } from "i18next";
2 | import { initReactI18next } from "react-i18next";
3 | import esTranslation from "./es";
4 | import frTranslation from "./fr";
5 | import itTranslations from "./it";
6 | import deTranslations from "./de";
7 |
8 | const resources: Resource = {
9 | en: {
10 | translation: {},
11 | },
12 | fr: {
13 | translation: frTranslation,
14 | },
15 | es: {
16 | translation: esTranslation,
17 | },
18 | it: {
19 | translation: itTranslations,
20 | },
21 | de: {
22 | translation: deTranslations,
23 | },
24 | };
25 |
26 | i18.use(initReactI18next).init({
27 | resources,
28 | lng: "en",
29 | keySeparator: false,
30 | interpolation: {
31 | escapeValue: false,
32 | },
33 | });
34 |
35 | export default i18;
36 |
--------------------------------------------------------------------------------
/frontend/src/i18n/it.ts:
--------------------------------------------------------------------------------
1 | // Translations in Italian
2 | const translations = {
3 | Upload: "Caricare",
4 | "Select Header": "Seleziona intestazione",
5 | "Map Columns": "Mappa colonne",
6 | "Expected Column": "Colonna prevista",
7 | Required: "Richiesto",
8 | "Drop your file here": "Trascina il tuo file qui",
9 | or: "oppure",
10 | "Browse files": "Sfoglia file",
11 | "Download Template": "Scarica il modello",
12 | "Only CSV, XLS, and XLSX files can be uploaded": "Solo i file CSV, XLS e XLSX possono essere caricati",
13 | "Select Header Row": "Seleziona la riga di intestazione",
14 | "Select the row which contains the column headers": "Seleziona la riga che contiene le intestazioni delle colonne",
15 | "Only the first sheet ("{{sheet}}") of the Excel file will be imported. To import multiple sheets, please upload each sheet individually.": "Solo il primo foglio ("{{sheet}}") del file Excel verrà importato. Per importare più fogli, carica ogni foglio singolarmente.",
16 | "Cancel": "Annulla",
17 | "Continue": "Continua",
18 | "Your File Column": "La tua colonna di file",
19 | "Your Sample Data": "I tuoi dati di esempio",
20 | "Destination Column": "Colonna di destinazione",
21 | "Include": "Includere",
22 | "Submit": "Invia",
23 | "Loading...": "Caricamento...",
24 | "Please include all required columns": "Si prega di includere tutte le colonne richieste",
25 | "Back": "Indietro",
26 | "- Select one -": "- Selezionane uno -",
27 | "- empty -": "- vuoto -",
28 | "Import Successful": "Importazione riuscita",
29 | "Upload Successful": "Caricamento riuscito",
30 | "Done": "Fatto",
31 | "Upload another file": "Carica un altro file",
32 | };
33 |
34 | export default translations;
35 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Box/index.tsx:
--------------------------------------------------------------------------------
1 | import classes from "../../utils/classes";
2 | import { BoxProps } from "./types";
3 | import style from "./style/Box.module.scss";
4 |
5 | export default function Box({ className, variants = [], ...props }: BoxProps) {
6 | const variantStyles = classes(variants.map((c: keyof typeof style) => style[c]));
7 | const containerClasses = classes([style.box, variantStyles, className]);
8 |
9 | return ;
10 | }
11 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Box/style/Box.module.scss:
--------------------------------------------------------------------------------
1 | .box {
2 | display: block;
3 | margin: 0 auto;
4 | padding: var(--m);
5 | background-color: var(--color-background-modal);
6 | border-radius: var(--border-radius-5);
7 | box-shadow: 0 0 20px var(--color-background-modal-shadow);
8 | max-width: 100%;
9 |
10 | &.fluid {
11 | max-width: none;
12 | }
13 | &.mid {
14 | max-width: 440px;
15 | }
16 | &.wide {
17 | max-width: 660px;
18 | }
19 | &.space-l {
20 | padding: var(--m-l);
21 | }
22 | &.space-mid {
23 | padding: var(--m);
24 | }
25 | &.space-none {
26 | padding: 0;
27 | }
28 | &.bg-shade {
29 | background-color: var(--color-background-modal-shade);
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Box/types/index.ts:
--------------------------------------------------------------------------------
1 | export type BoxVariant = "fluid" | "mid" | "wide" | "space-l" | "space-mid" | "space-none" | "bg-shade";
2 |
3 | export type BoxProps = React.HTMLAttributes & {
4 | variants?: BoxVariant[];
5 | };
6 |
7 | export const boxVariants: BoxVariant[] = ["fluid", "mid", "wide", "space-l", "space-mid", "space-none", "bg-shade"];
8 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Checkbox/index.tsx:
--------------------------------------------------------------------------------
1 | import classes from "../../utils/classes";
2 | import { CheckboxProps } from "./types";
3 | import style from "./style/Checkbox.module.scss";
4 |
5 | export default function Checkbox({ label, className, ...props }: CheckboxProps) {
6 | const containerClasses = classes([style.container, className]);
7 |
8 | return (
9 |
13 | );
14 | }
15 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Checkbox/style/Checkbox.module.scss:
--------------------------------------------------------------------------------
1 | .container {
2 | display: inline-block;
3 | gap: var(--m-xs);
4 | align-items: center;
5 |
6 | &:has(input:not(:disabled)) {
7 | cursor: pointer;
8 | }
9 |
10 | input[type="checkbox"] {
11 | -webkit-appearance: none;
12 | appearance: none;
13 | background-color: transparent;
14 | margin: 0;
15 | color: var(--color-primary);
16 | width: var(--m);
17 | height: var(--m);
18 | border: 2px solid var(--color-border);
19 | display: grid;
20 | place-content: center;
21 | border-radius: var(--border-radius-1);
22 | cursor: pointer;
23 |
24 | &::before {
25 | content: "";
26 | width: var(--m-xs);
27 | height: var(--m-xs);
28 | }
29 |
30 | &:checked {
31 | background-color: var(--color-primary);
32 | border-color: var(--color-primary);
33 | }
34 |
35 | &:checked::before {
36 | clip-path: polygon(14% 44%, 0 65%, 50% 100%, 100% 16%, 80% 0%, 43% 62%);
37 | box-shadow: inset 1em 1em var(--color-text-on-primary);
38 | }
39 |
40 | &:not(:disabled) {
41 | &:focus-visible {
42 | outline: 1px solid var(--color-border);
43 | outline-offset: 3px;
44 | }
45 | }
46 |
47 | &:disabled {
48 | --container-color: var(--container-disabled);
49 | color: var(--container-disabled);
50 | cursor: default;
51 | background-color: var(--color-input-disabled);
52 | border-color: var(--color-border-soft);
53 |
54 | &:checked {
55 | &::before {
56 | clip-path: polygon(14% 44%, 0 65%, 50% 100%, 100% 16%, 80% 0%, 43% 62%);
57 | box-shadow: inset 1em 1em var(--color-border-soft);
58 | }
59 | }
60 | }
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Checkbox/types/index.ts:
--------------------------------------------------------------------------------
1 | import { InputHTMLAttributes, ReactElement } from "react";
2 |
3 | export type CheckboxProps = InputHTMLAttributes & {
4 | label?: string | ReactElement;
5 | };
6 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Errors/index.tsx:
--------------------------------------------------------------------------------
1 | import { sizes } from "../../settings/theme";
2 | import classes from "../../utils/classes";
3 | import style from "./style/Errors.module.scss";
4 | import { PiInfo } from "react-icons/pi";
5 |
6 | export default function Errors({ error, centered = false }: { error?: unknown; centered?: boolean }) {
7 | return error ? (
8 |
9 |
10 |
11 | {error.toString()}
12 |
13 |
10 | {steps.map((step, i) => {
11 | if (step.disabled) return null;
12 | const done = i < current;
13 |
14 | const Element = clickable ? "button" : "div";
15 |
16 | const buttonProps: any = clickable
17 | ? {
18 | onClick: () => setCurrent(i),
19 | type: "button",
20 | }
21 | : {};
22 |
23 | let displayNumber = i + 1;
24 | if (skipHeader && displayNumber > 1) {
25 | displayNumber--;
26 | }
27 |
28 | return (
29 |
33 |
34 | {step.label}
35 |
36 | );
37 | })}
38 |
31 | {options.map((option) => (
32 |
43 | ))}
44 |
45 | );
46 | }
47 |
48 | export default ToggleFilter;
49 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/ToggleFilter/style/ToggleFilter.module.scss:
--------------------------------------------------------------------------------
1 | .toggleFilter {
2 | display: flex;
3 | align-items: center;
4 | background-color: var(--color-input-background);
5 | border-radius: 20px;
6 | overflow: hidden;
7 | min-height: 36px;
8 | }
9 |
10 | .toggleOption {
11 | padding: 8px 16px;
12 | cursor: pointer;
13 |
14 | &.selected {
15 | background-color: var(--color-text-on-tertiary);
16 | border-radius: 20px;
17 | transition: background-color 0.2s, color 0.2s;
18 | }
19 |
20 | .defaultColor {
21 | color: var(--color-text);
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/ToggleFilter/types/index.ts:
--------------------------------------------------------------------------------
1 | export interface Option {
2 | label: string;
3 | filterValue: string,
4 | selected: boolean;
5 | color?: string;
6 | }
7 |
8 | export interface ToggleFilterProps {
9 | options: Option[];
10 | className?: string;
11 | onChange: (option: string) => void;
12 | }
--------------------------------------------------------------------------------
/frontend/src/importer/components/Tooltip/index.tsx:
--------------------------------------------------------------------------------
1 | import { useEffect, useRef, useState } from "react";
2 | import ReactDOM from "react-dom";
3 | import classes from "../../utils/classes";
4 | import getStringLengthOfChildren from "../../utils/getStringLengthOfChildren";
5 | import { AsMap, TooltipProps } from "./types";
6 | import style from "./style/Tooltip.module.scss";
7 | import { PiInfo } from "react-icons/pi";
8 |
9 | export default function Tooltip({ as, className, title, children, icon = , ...props }: TooltipProps) {
10 | const Tag: any = as || "span";
11 |
12 | const length = getStringLengthOfChildren(title);
13 | const wrapperClasses = classes([style.tooltip, className, length > 30 && style.multiline]);
14 |
15 | const [tooltipVisible, setTooltipVisible] = useState(false);
16 | const [position, setPosition] = useState({ top: 0, left: 0 });
17 | const targetRef = useRef(null);
18 |
19 | // Create a ref to attach the tooltip portal to
20 | const tooltipContainer = useRef(document.createElement("div"));
21 |
22 | useEffect(() => {
23 | // Appending the tooltip container to the body on mount
24 | document.body.appendChild(tooltipContainer.current);
25 |
26 | // Removing the tooltip container from the body on unmount
27 | return () => {
28 | document.body.removeChild(tooltipContainer.current);
29 | };
30 | }, []);
31 |
32 | const showTooltip = () => {
33 | if (targetRef.current) {
34 | const rect = targetRef.current.getBoundingClientRect();
35 | setPosition({
36 | top: rect.bottom + window.scrollY,
37 | left: rect.left + rect.width / 2 + window.scrollX,
38 | });
39 | setTooltipVisible(true);
40 | }
41 | };
42 |
43 | const hideTooltip = () => {
44 | setTooltipVisible(false);
45 | };
46 |
47 | const tooltipMessage = tooltipVisible && (
48 |
49 | {title}
50 |
51 | );
52 |
53 | return (
54 |
55 | {children}
56 |
57 | {icon}
58 | {tooltipMessage}
59 |
60 |
61 | );
62 | }
63 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Tooltip/style/Tooltip.module.scss:
--------------------------------------------------------------------------------
1 | $side: var(--m-xxxs);
2 | $height: calc($side * 1.732);
3 |
4 | .tooltip {
5 | display: inline-flex;
6 | align-items: center;
7 | gap: var(--m-xs);
8 |
9 | .icon {
10 | position: relative;
11 | display: block;
12 | cursor: pointer;
13 | }
14 |
15 | &.multiline .message {
16 | width: 260px;
17 | white-space: normal;
18 | }
19 | }
20 |
21 | .message {
22 | position: absolute;
23 | transform: translateX(-50%);
24 | background-color: var(--color-background-modal);
25 | z-index: 3;
26 | padding: var(--m-xxs) var(--m-xs);
27 | border-radius: var(--border-radius);
28 | margin-top: var(--m-xs);
29 | box-shadow: 0 0 0 1px var(--color-border), 0 5px 15px rgba(0, 0, 0, 0.2);
30 | max-width: 300px;
31 |
32 | &::after,
33 | &::before {
34 | position: absolute;
35 | top: calc($height * -1);
36 | left: 50%;
37 | border-left: $side solid transparent;
38 | border-right: $side solid transparent;
39 | border-bottom: $height solid var(--color-border);
40 | content: "";
41 | font-size: 0;
42 | line-height: 0;
43 | width: 0;
44 | transform: translateX(-50%);
45 | }
46 |
47 | &::after {
48 | top: calc($height * -1 + 2px);
49 | border-bottom: $height solid var(--color-background-modal);
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/Tooltip/types/index.ts:
--------------------------------------------------------------------------------
1 | import { ReactNode } from "react";
2 |
3 | export type AsMap = {
4 | div: React.HTMLProps;
5 | span: React.HTMLProps;
6 | p: React.HTMLProps;
7 | };
8 |
9 | export type TooltipProps = {
10 | as?: T;
11 | title?: string | ReactNode;
12 | icon?: ReactNode;
13 | } & AsMap[T];
14 |
--------------------------------------------------------------------------------
/frontend/src/importer/components/UploaderWrapper/UploaderWrapper.tsx:
--------------------------------------------------------------------------------
1 | import { useState } from "react";
2 | import { useDropzone } from "react-dropzone";
3 | import { useTranslation } from "react-i18next";
4 | import useThemeStore from "../../stores/theme";
5 | import { UploaderWrapperProps } from "./types";
6 | import { FileIcon, UploadIcon, Loader2 } from "lucide-react";
7 | import { Button } from "../../components/ui/button";
8 | import { cn } from "../../../utils/classes";
9 |
10 | export default function UploaderWrapper({ onSuccess, setDataError, ...props }: UploaderWrapperProps) {
11 | const [loading, setLoading] = useState(false);
12 | const theme = useThemeStore((state) => state.theme);
13 | const { t } = useTranslation();
14 |
15 | const { getRootProps, getInputProps, isDragActive, open } = useDropzone({
16 | noClick: true,
17 | noKeyboard: true,
18 | maxFiles: 1,
19 | accept: {
20 | "application/vnd.ms-excel": [".xls"],
21 | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": [".xlsx"],
22 | "text/csv": [".csv"],
23 | },
24 | onDropRejected: (fileRejections) => {
25 | setLoading(false);
26 | const errorMessage = fileRejections[0].errors[0].message;
27 | setDataError(errorMessage);
28 | },
29 | onDropAccepted: async ([file]) => {
30 | setLoading(true);
31 | onSuccess(file);
32 | setLoading(false);
33 | },
34 | });
35 |
36 | return (
37 |
38 |
45 |
46 | {isDragActive ? (
47 |
48 |
49 | {t("Drop your file here")}
50 |
51 | ) : loading ? (
52 |
53 |
54 | {t("Loading...")}
55 |
56 | ) : (
57 |
58 |
59 |
60 | {t("Drop your file here")}
61 | {t("Supports CSV, XLS, and XLSX files")}
62 |
63 |
64 | {t("or")}
65 |
73 |
74 |
75 | )}
76 |
77 | {t("Import Successful")}
17 |
18 | } onClick={reload}>
19 | {t("Upload another file")}
20 |
21 | {isModal && (
22 | } onClick={close}>
23 | {t("Done")}
24 |
25 | )}
26 |
27 |
28 |
29 | );
30 | }
31 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/complete/style/Complete.module.scss:
--------------------------------------------------------------------------------
1 | .content.content {
2 | max-width: 1000px;
3 | padding-top: var(--m);
4 | height: 100%;
5 | flex: 1 0 100px;
6 | box-shadow: none;
7 | background-color: transparent;
8 | align-self: center;
9 |
10 | display: flex;
11 | align-items: center;
12 | justify-content: center;
13 | font-size: var(--font-size-xl);
14 | flex-direction: column;
15 | gap: var(--m);
16 | text-align: center;
17 | position: relative;
18 |
19 | .icon {
20 | width: 64px;
21 | height: 64px;
22 | isolation: isolate;
23 | position: relative;
24 | display: flex;
25 | align-items: center;
26 | justify-content: center;
27 |
28 | &::before {
29 | content: "";
30 | position: absolute;
31 | inset: 0;
32 | border-radius: 50%;
33 | background-color: var(--color-green-ui);
34 | z-index: -1;
35 | }
36 |
37 | svg {
38 | width: 38%;
39 | height: 38%;
40 | object-fit: contain;
41 | color: var(--color-text-on-primary);
42 | }
43 | }
44 |
45 | .actions {
46 | display: flex;
47 | gap: var(--m-l);
48 | align-items: center;
49 | justify-content: center;
50 | margin-top: var(--m-xxl);
51 |
52 | & > * {
53 | flex: 1 0 190px;
54 | }
55 |
56 | button {
57 | width: 50%;
58 | }
59 | }
60 | }
61 |
62 | .spinner {
63 | border: 1px solid var(--color-border);
64 | margin-top: var(--m);
65 | padding: var(--m);
66 | border-radius: var(--border-radius-1);
67 | }
68 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/complete/types/index.ts:
--------------------------------------------------------------------------------
1 | export type CompleteProps = {
2 | reload: () => void;
3 | close: () => void;
4 | isModal: boolean;
5 | };
6 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/main/hooks/useMutableLocalStorage.ts:
--------------------------------------------------------------------------------
1 | import { useEffect, useState } from "react";
2 |
3 | export default function useMutableLocalStorage(key: string, initialValue: any) {
4 | // State to store our value
5 | // Pass initial state function to useState so logic is only executed once
6 | const getLocalStorage = () => {
7 | if (typeof window === "undefined") {
8 | return initialValue;
9 | }
10 | try {
11 | // Get from local storage by key
12 | const item = window.localStorage.getItem(key);
13 | // Parse stored json or if none return initialValue
14 | return item ? JSON.parse(item) : initialValue;
15 | } catch (error) {
16 | // If error also return initialValue
17 |
18 | return initialValue;
19 | }
20 | };
21 | const [storedValue, setStoredValue] = useState(getLocalStorage());
22 |
23 | useEffect(() => {
24 | setStoredValue(getLocalStorage());
25 | }, [key]);
26 |
27 | // Return a wrapped version of useState's setter function that ...
28 | // ... persists the new value to localStorage.
29 | const setValue = (value: any) => {
30 | try {
31 | // Allow value to be a function so we have same API as useState
32 | const valueToStore = value instanceof Function ? value(storedValue) : value;
33 | // Save state
34 | setStoredValue(valueToStore);
35 | // Save to local storage
36 | if (typeof window !== "undefined") {
37 | window.localStorage.setItem(key, JSON.stringify(valueToStore));
38 | }
39 | } catch (error) {
40 |
41 | }
42 | };
43 |
44 | return [storedValue, setValue];
45 | }
46 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/main/hooks/useStepNavigation.ts:
--------------------------------------------------------------------------------
1 | import { useEffect, useState } from "react";
2 | import { useTranslation } from "react-i18next";
3 | import useStepper from "../../../components/Stepper/hooks/useStepper";
4 | import { Steps } from "../types";
5 | import useMutableLocalStorage from "./useMutableLocalStorage";
6 |
7 | export const StepEnum = {
8 | Upload: 0,
9 | RowSelection: 1,
10 | MapColumns: 2,
11 | Validation: 3,
12 | Complete: 4,
13 | };
14 |
15 | const calculateNextStep = (nextStep: number, skipHeader: boolean) => {
16 | if (skipHeader) {
17 | switch (nextStep) {
18 | case StepEnum.Upload:
19 | case StepEnum.RowSelection:
20 | return StepEnum.MapColumns;
21 | case StepEnum.MapColumns:
22 | return StepEnum.Validation;
23 | case StepEnum.Validation:
24 | return StepEnum.Complete;
25 | default:
26 | return nextStep;
27 | }
28 | }
29 | return nextStep;
30 | };
31 |
32 | const getStepConfig = (skipHeader: boolean) => {
33 | return [
34 | { label: "Upload", id: Steps.Upload },
35 | { label: "Select Header", id: Steps.RowSelection, disabled: skipHeader },
36 | { label: "Map Columns", id: Steps.MapColumns },
37 | { label: "Validation", id: Steps.Validation },
38 | ];
39 | };
40 |
41 | function useStepNavigation(initialStep: number, skipHeader: boolean) {
42 | const [t] = useTranslation();
43 | const translatedSteps = getStepConfig(skipHeader).map((step) => ({
44 | ...step,
45 | label: t(step.label),
46 | }));
47 | const stepper = useStepper(translatedSteps, StepEnum.Upload, skipHeader);
48 | const [storageStep, setStorageStep] = useMutableLocalStorage(`tf_steps`, "");
49 | const [currentStep, setCurrentStep] = useState(initialStep);
50 |
51 | const goBack = (backStep = 0) => {
52 | backStep = backStep || currentStep - 1 || 0;
53 | setStep(backStep);
54 | };
55 |
56 | const goNext = (nextStep = 0) => {
57 | nextStep = nextStep || currentStep + 1 || 0;
58 | const calculatedStep = calculateNextStep(nextStep, skipHeader);
59 | setStep(calculatedStep);
60 | };
61 |
62 | const setStep = (newStep: number) => {
63 | setCurrentStep(newStep);
64 | setStorageStep(newStep);
65 | stepper.setCurrent(newStep);
66 | };
67 |
68 | useEffect(() => {
69 | stepper.setCurrent(storageStep || 0);
70 | setCurrentStep(storageStep || 0);
71 | }, [storageStep]);
72 |
73 | return {
74 | currentStep: storageStep || currentStep,
75 | setStep,
76 | goBack,
77 | goNext,
78 | stepper,
79 | stepId: stepper?.step?.id,
80 | setStorageStep,
81 | };
82 | }
83 |
84 | export default useStepNavigation;
85 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/main/style/Main.module.scss:
--------------------------------------------------------------------------------
1 | .wrapper {
2 | display: flex;
3 | flex-direction: column;
4 | max-height: 100%;
5 | padding: 16px 8px 8px 8px;
6 | }
7 |
8 | .content {
9 | padding: 20px;
10 | flex: 1;
11 | overflow: hidden;
12 | }
13 |
14 | .status {
15 | display: flex;
16 | align-items: center;
17 | justify-content: space-between;
18 | gap: var(--m);
19 | padding: 0 var(--m-s) var(--m-s) var(--m-s);
20 | }
21 |
22 | .spinner {
23 | border: 1px solid var(--color-border);
24 | margin-top: var(--m);
25 | padding: var(--m);
26 | border-radius: var(--border-radius-1);
27 | position: absolute;
28 | top: 50%;
29 | left: 50%;
30 | transform: translate(-50%, -50%);
31 | }
32 |
33 | $closeSide: calc(var(--m-xl) * 36 / 48);
34 |
35 | .close.close {
36 | position: absolute;
37 | right: var(--m-xs, 0.5rem);
38 | top: var(--m-xs, 0.5rem);
39 | border-radius: 50%;
40 | min-width: $closeSide;
41 | height: $closeSide;
42 | aspect-ratio: 1;
43 | font-size: var(--font-size-xl);
44 | padding: 0;
45 | }
46 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/main/types/index.ts:
--------------------------------------------------------------------------------
1 | export enum Steps {
2 | Upload = "upload",
3 | RowSelection = "row-selection",
4 | MapColumns = "map-columns",
5 | Validation = "validation",
6 | }
7 |
8 | export type FileRow = {
9 | index: number;
10 | values: string[];
11 | };
12 |
13 | export type FileData = {
14 | fileName: string;
15 | rows: FileRow[];
16 | sheetList: string[];
17 | errors: string[];
18 | };
19 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/map-columns/hooks/useNameChange.ts:
--------------------------------------------------------------------------------
1 | import { useEffect, useState } from "react";
2 |
3 | const useTransformValue = (initialValue: string) => {
4 | const [transformedValue, setTransformedValue] = useState("");
5 |
6 | useEffect(() => {
7 | const keyValue = initialValue.replace(/\s/g, "_").toLowerCase();
8 | setTransformedValue(keyValue);
9 | }, [initialValue]);
10 |
11 | const transformValue = (value: string) => {
12 | const keyValue = value.replace(/\s/g, "_").toLowerCase();
13 | setTransformedValue(keyValue);
14 | };
15 |
16 | return { transformedValue, transformValue };
17 | };
18 |
19 | export default useTransformValue;
20 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/map-columns/style/MapColumns.module.scss:
--------------------------------------------------------------------------------
1 | .content {
2 | height: 100%;
3 |
4 | form {
5 | display: flex;
6 | flex-direction: column;
7 | height: 100%;
8 | gap: 1rem;
9 |
10 | .tableWrapper {
11 | display: flex;
12 | max-height: 400px;
13 | overflow-y: auto;
14 | padding: 1px;
15 | border-radius: 0.5rem;
16 | border: 1px solid var(--color-border);
17 | background-color: var(--color-background);
18 | }
19 |
20 | .actions {
21 | display: flex;
22 | justify-content: space-between;
23 | align-items: center;
24 | padding-top: 0.5rem;
25 | }
26 | }
27 | }
28 |
29 | .samples {
30 | overflow: hidden;
31 | text-overflow: ellipsis;
32 | line-height: 1.2;
33 | white-space: nowrap;
34 |
35 | & > small {
36 | background-color: var(--color-input-background);
37 | font-family: monospace;
38 | padding: 0.25rem 0.5rem;
39 | border-radius: 0.25rem;
40 | font-size: 0.75rem;
41 | display: inline-block;
42 | color: var(--color-text);
43 |
44 | & + small {
45 | margin-left: 0.25rem;
46 | }
47 | }
48 | }
49 |
50 | .spinner {
51 | border: 1px solid var(--color-border);
52 | margin-top: var(--m);
53 | padding: var(--m);
54 | border-radius: var(--border-radius-1);
55 | }
56 |
57 | .errorContainer {
58 | display: flex;
59 | justify-content: center;
60 | max-width: 60vw;
61 | position: absolute;
62 | top: -30px;
63 | left: 50%;
64 | transform: translateX(-50%);
65 | }
66 |
67 | .schemalessTextInput {
68 | width: 210px;
69 | }
70 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/map-columns/types/index.ts:
--------------------------------------------------------------------------------
1 | import { Template } from "../../../types";
2 | import { FileData } from "../../main/types";
3 |
4 | export type TemplateColumnMapping = {
5 | key: string;
6 | include: boolean;
7 | selected?: boolean;
8 | };
9 |
10 | export type MapColumnsProps = {
11 | template: Template;
12 | data: FileData;
13 | columnMapping: { [index: number]: TemplateColumnMapping };
14 | selectedHeaderRow: number | null;
15 | skipHeaderRowSelection?: boolean;
16 | onSuccess: (columnMapping: { [index: number]: TemplateColumnMapping }) => void;
17 | onCancel: () => void;
18 | isSubmitting: boolean;
19 | importerKey?: string; // Key of the importer for API calls
20 | backendUrl?: string; // Backend URL for API calls
21 | };
22 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/row-selection/style/RowSelection.module.scss:
--------------------------------------------------------------------------------
1 | .content {
2 | flex-grow: 1;
3 | height: 100%;
4 |
5 | form {
6 | display: flex;
7 | flex-direction: column;
8 | height: 100%;
9 | gap: var(--m);
10 |
11 | .tableWrapper {
12 | display: flex;
13 | overflow-y: auto;
14 | padding: 1px;
15 | margin-right: -20px;
16 | padding-right: 21px;
17 | max-height: 400px;
18 | }
19 |
20 | .actions {
21 | display: flex;
22 | justify-content: space-between;
23 | }
24 | }
25 | }
26 |
27 | .samples {
28 | overflow: hidden;
29 | text-overflow: ellipsis;
30 | line-height: 1;
31 | white-space: nowrap;
32 |
33 | & > small {
34 | background-color: var(--color-input-background);
35 | font-family: monospace;
36 | padding: var(--m-xxxxs);
37 | border-radius: var(--border-radius-1);
38 | font-size: var(--font-size-xs);
39 | display: inline-block;
40 |
41 | & + small {
42 | margin-left: var(--m-xxxxs);
43 | }
44 | }
45 | }
46 | .spinner {
47 | border: 1px solid var(--color-border);
48 | margin-top: var(--m);
49 | padding: var(--m);
50 | border-radius: var(--border-radius-1);
51 | }
52 |
53 | .inputRadio {
54 | margin-right: 10px;
55 | }
56 |
57 | .headingCaption {
58 | padding: 12px 0 10px 0;
59 | color: var(--color-text-secondary);
60 | font-weight: 400;
61 | height: 48px;
62 | vertical-align: middle;
63 | text-align: center;
64 |
65 | // TODO: Hacky solution to update the tooltip title text, update this
66 | span > span:nth-child(1) > span {
67 | font-weight: 400;
68 | }
69 | }
70 |
71 | .warningIcon {
72 | margin-right: 7px;
73 | }
74 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/row-selection/types/index.ts:
--------------------------------------------------------------------------------
1 | import { FileData } from "../../main/types";
2 |
3 | export type RowSelectionProps = {
4 | data: FileData;
5 | onSuccess: () => void;
6 | onCancel: () => void;
7 | selectedHeaderRow: number | null;
8 | setSelectedHeaderRow: (id: number) => void;
9 | };
10 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/uploader/hooks/useTemplateTable.tsx:
--------------------------------------------------------------------------------
1 | import { useMemo } from "react";
2 | import { useTranslation } from "react-i18next";
3 | import Tooltip from "../../../components/Tooltip";
4 | import { TemplateColumn } from "../../../types";
5 | import { PiCheckBold } from "react-icons/pi";
6 |
7 | export default function useTemplateTable(fields: TemplateColumn[] = []) {
8 | if (!fields) {
9 | return [];
10 | }
11 | const { t } = useTranslation();
12 | const expectedColumnKey = t("Expected Column");
13 | const requiredKey = t("Required");
14 | const result = useMemo(() => {
15 | return fields.map((item) => ({
16 | [expectedColumnKey]: item?.description
17 | ? {
18 | raw: item.name,
19 | content: (
20 |
21 | {item.name}
22 |
23 | ),
24 | }
25 | : item.name,
26 | [requiredKey]: { raw: item?.required ? 1 : 0, content: item?.required ? : <> },
27 | }));
28 | }, [fields]);
29 |
30 | return result;
31 | }
32 |
--------------------------------------------------------------------------------
/frontend/src/importer/features/uploader/index.tsx:
--------------------------------------------------------------------------------
1 | import { useTranslation } from "react-i18next";
2 | import { Button } from "@chakra-ui/button";
3 | import Table from "../../components/Table";
4 | import UploaderWrapper from "../../components/UploaderWrapper/UploaderWrapper";
5 | import useThemeStore from "../../stores/theme";
6 | import useTemplateTable from "./hooks/useTemplateTable";
7 | import { UploaderProps } from "./types";
8 | import style from "./style/Uploader.module.scss";
9 | import { PiDownloadSimple } from "react-icons/pi";
10 |
11 |
12 | export default function Uploader({ template, skipHeaderRowSelection, onSuccess, showDownloadTemplateButton, setDataError }: UploaderProps) {
13 | const fields = useTemplateTable(template.columns);
14 | const theme = useThemeStore((state) => state.theme);
15 | const uploaderWrapper = ;
16 | showDownloadTemplateButton = showDownloadTemplateButton ?? true;
17 | const { t } = useTranslation();
18 |
19 | function downloadTemplate() {
20 | const { columns } = template;
21 | const csvData = `${columns.map((obj) => obj.name).join(",")}`;
22 |
23 | const link = document.createElement("a");
24 | link.href = URL.createObjectURL(new Blob([csvData], { type: "text/csv" }));
25 | link.download = "example.csv";
26 | link.click();
27 | }
28 |
29 | const downloadTemplateButton = showDownloadTemplateButton ? (
30 | }
33 | onClick={downloadTemplate}
34 | colorScheme={"secondary"}
35 | variant={theme === "light" ? "outline" : "solid"}
36 | _hover={
37 | theme === "light"
38 | ? {
39 | background: "var(--color-border)",
40 | color: "var(--color-text)",
41 | }
42 | : undefined
43 | }>
44 | {t("Download Template")}
45 |
46 | ) : null;
47 |
48 | return (
49 |
50 | {uploaderWrapper}
51 |
52 |
55 | {downloadTemplateButton}
56 |
57 | |