37 | );
38 | };
39 |
--------------------------------------------------------------------------------
/apps/demo/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "demo",
3 | "version": "0.1.2",
4 | "private": true,
5 | "scripts": {
6 | "dev": "next dev --turbopack",
7 | "build": "next build",
8 | "start": "next start",
9 | "lint": "next lint",
10 | "cron": "bash cron.sh",
11 | "migrate-dataqueue": "dataqueue-cli migrate"
12 | },
13 | "dependencies": {
14 | "@radix-ui/react-slot": "^1.2.3",
15 | "class-variance-authority": "^0.7.1",
16 | "clsx": "^2.1.1",
17 | "date-fns": "^4.1.0",
18 | "dotenv-cli": "^8.0.0",
19 | "lucide-react": "^0.525.0",
20 | "next": "15.3.8",
21 | "node-pg-migrate": "^8.0.3",
22 | "pg": "^8.16.3",
23 | "@nicnocquee/dataqueue": "workspace:*",
24 | "react": "^19.0.0",
25 | "react-dom": "^19.0.0",
26 | "tailwind-merge": "^3.3.1",
27 | "ts-node": "^10.9.2"
28 | },
29 | "devDependencies": {
30 | "@eslint/eslintrc": "^3",
31 | "@tailwindcss/postcss": "^4",
32 | "@types/node": "^20",
33 | "@types/react": "^19",
34 | "@types/react-dom": "^19",
35 | "eslint": "^9",
36 | "eslint-config-next": "15.3.4",
37 | "tailwindcss": "^4",
38 | "tw-animate-css": "^1.3.4",
39 | "typescript": "^5"
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/apps/website/components/ui/checkbox.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as CheckboxPrimitive from '@radix-ui/react-checkbox';
5 | import { Check } from 'lucide-react';
6 |
7 | import { cn } from '@/lib/utils';
8 |
9 | const Checkbox = React.forwardRef<
10 | React.ElementRef,
11 | React.ComponentPropsWithoutRef
12 | >(({ className, ...props }, ref) => (
13 |
21 |
24 |
25 |
26 |
27 | ));
28 | Checkbox.displayName = CheckboxPrimitive.Root.displayName;
29 |
30 | export { Checkbox };
31 |
--------------------------------------------------------------------------------
/apps/website/components/ui/slider.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as SliderPrimitive from '@radix-ui/react-slider';
5 |
6 | import { cn } from '@/lib/utils';
7 |
8 | const Slider = React.forwardRef<
9 | React.ElementRef,
10 | React.ComponentPropsWithoutRef
11 | >(({ className, ...props }, ref) => (
12 |
20 |
21 |
22 |
23 |
24 |
25 | ));
26 | Slider.displayName = SliderPrimitive.Root.displayName;
27 |
28 | export { Slider };
29 |
--------------------------------------------------------------------------------
/apps/website/components/ui/switch.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as SwitchPrimitives from '@radix-ui/react-switch';
5 |
6 | import { cn } from '@/lib/utils';
7 |
8 | const Switch = React.forwardRef<
9 | React.ElementRef,
10 | React.ComponentPropsWithoutRef
11 | >(({ className, ...props }, ref) => (
12 |
20 |
25 |
26 | ));
27 | Switch.displayName = SwitchPrimitives.Root.displayName;
28 |
29 | export { Switch };
30 |
--------------------------------------------------------------------------------
/apps/website/components/ui/badge.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import { cva, type VariantProps } from 'class-variance-authority';
3 |
4 | import { cn } from '@/lib/utils';
5 |
6 | const badgeVariants = cva(
7 | 'inline-flex items-center rounded-full border px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-ring focus:ring-offset-2',
8 | {
9 | variants: {
10 | variant: {
11 | default:
12 | 'border-transparent bg-primary text-primary-foreground hover:bg-primary/80',
13 | secondary:
14 | 'border-transparent bg-secondary text-secondary-foreground hover:bg-secondary/80',
15 | destructive:
16 | 'border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80',
17 | outline: 'text-foreground',
18 | },
19 | },
20 | defaultVariants: {
21 | variant: 'default',
22 | },
23 | },
24 | );
25 |
26 | export interface BadgeProps
27 | extends React.HTMLAttributes,
28 | VariantProps {}
29 |
30 | function Badge({ className, variant, ...props }: BadgeProps) {
31 | return (
32 |
33 | );
34 | }
35 |
36 | export { Badge, badgeVariants };
37 |
--------------------------------------------------------------------------------
/apps/website/components/ui/tooltip.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as TooltipPrimitive from '@radix-ui/react-tooltip';
5 |
6 | import { cn } from '@/lib/utils';
7 |
8 | const TooltipProvider = TooltipPrimitive.Provider;
9 |
10 | const Tooltip = TooltipPrimitive.Root;
11 |
12 | const TooltipTrigger = TooltipPrimitive.Trigger;
13 |
14 | const TooltipContent = React.forwardRef<
15 | React.ElementRef,
16 | React.ComponentPropsWithoutRef
17 | >(({ className, sideOffset = 4, ...props }, ref) => (
18 |
27 | ));
28 | TooltipContent.displayName = TooltipPrimitive.Content.displayName;
29 |
30 | export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider };
31 |
--------------------------------------------------------------------------------
/packages/dataqueue/migrations/1751131910823_initial.sql:
--------------------------------------------------------------------------------
1 | -- 001-initial.sql: Initial schema for dataqueue
2 |
3 | -- Up Migration
4 | CREATE TABLE IF NOT EXISTS job_queue (
5 | id SERIAL PRIMARY KEY,
6 | job_type VARCHAR(255) NOT NULL,
7 | payload JSONB NOT NULL,
8 | status VARCHAR(50) DEFAULT 'pending',
9 | created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
10 | updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
11 | locked_at TIMESTAMPTZ,
12 | locked_by VARCHAR(255),
13 | attempts INT DEFAULT 0,
14 | max_attempts INT DEFAULT 3,
15 | next_attempt_at TIMESTAMPTZ,
16 | priority INT DEFAULT 0,
17 | run_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
18 | pending_reason TEXT,
19 | error_history JSONB DEFAULT '[]'::jsonb
20 | );
21 |
22 | CREATE INDEX IF NOT EXISTS idx_job_queue_status ON job_queue(status);
23 | CREATE INDEX IF NOT EXISTS idx_job_queue_next_attempt ON job_queue(next_attempt_at);
24 | CREATE INDEX IF NOT EXISTS idx_job_queue_run_at ON job_queue(run_at);
25 | CREATE INDEX IF NOT EXISTS idx_job_queue_priority ON job_queue(priority);
26 |
27 | -- Down Migration
28 | DROP INDEX IF EXISTS idx_job_queue_priority;
29 | DROP INDEX IF EXISTS idx_job_queue_run_at;
30 | DROP INDEX IF EXISTS idx_job_queue_next_attempt;
31 | DROP INDEX IF EXISTS idx_job_queue_status;
32 | DROP TABLE IF EXISTS job_queue;
33 |
--------------------------------------------------------------------------------
/apps/website/components/ui/hover-card.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as HoverCardPrimitive from '@radix-ui/react-hover-card';
5 |
6 | import { cn } from '@/lib/utils';
7 |
8 | const HoverCard = HoverCardPrimitive.Root;
9 |
10 | const HoverCardTrigger = HoverCardPrimitive.Trigger;
11 |
12 | const HoverCardContent = React.forwardRef<
13 | React.ElementRef,
14 | React.ComponentPropsWithoutRef
15 | >(({ className, align = 'center', sideOffset = 4, ...props }, ref) => (
16 |
26 | ));
27 | HoverCardContent.displayName = HoverCardPrimitive.Content.displayName;
28 |
29 | export { HoverCard, HoverCardTrigger, HoverCardContent };
30 |
--------------------------------------------------------------------------------
/apps/demo/public/next.svg:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/packages/dataqueue/migrations/1751186053000_add_job_events_table.sql:
--------------------------------------------------------------------------------
1 | -- Up Migration
2 | CREATE TABLE IF NOT EXISTS job_events (
3 | id SERIAL PRIMARY KEY,
4 | job_id INTEGER NOT NULL,
5 | event_type TEXT NOT NULL,
6 | created_at TIMESTAMPTZ DEFAULT NOW(),
7 | metadata JSONB
8 | );
9 |
10 | ALTER TABLE job_events ADD CONSTRAINT fk_job_events_job_queue FOREIGN KEY (job_id) REFERENCES job_queue(id) ON DELETE CASCADE;
11 | CREATE INDEX idx_job_events_job_id ON job_events(job_id);
12 | CREATE INDEX idx_job_events_event_type ON job_events(event_type);
13 |
14 | ALTER TABLE job_queue ADD COLUMN completed_at TIMESTAMPTZ;
15 | ALTER TABLE job_queue ADD COLUMN started_at TIMESTAMPTZ;
16 | ALTER TABLE job_queue ADD COLUMN last_retried_at TIMESTAMPTZ;
17 | ALTER TABLE job_queue ADD COLUMN last_failed_at TIMESTAMPTZ;
18 | ALTER TABLE job_queue ADD COLUMN last_cancelled_at TIMESTAMPTZ;
19 |
20 | -- Down Migration
21 | DROP INDEX IF EXISTS idx_job_events_event_type;
22 | DROP INDEX IF EXISTS idx_job_events_job_id;
23 | DROP TABLE IF EXISTS job_events;
24 |
25 | ALTER TABLE job_queue DROP COLUMN IF EXISTS completed_at;
26 | ALTER TABLE job_queue DROP COLUMN IF EXISTS started_at;
27 | ALTER TABLE job_queue DROP COLUMN IF EXISTS last_retried_at;
28 | ALTER TABLE job_queue DROP COLUMN IF EXISTS last_failed_at;
29 | ALTER TABLE job_queue DROP COLUMN IF EXISTS last_cancelled_at;
--------------------------------------------------------------------------------
/apps/website/components/ui/popover.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as PopoverPrimitive from '@radix-ui/react-popover';
5 |
6 | import { cn } from '@/lib/utils';
7 |
8 | const Popover = PopoverPrimitive.Root;
9 |
10 | const PopoverTrigger = PopoverPrimitive.Trigger;
11 |
12 | const PopoverContent = React.forwardRef<
13 | React.ElementRef,
14 | React.ComponentPropsWithoutRef
15 | >(({ className, align = 'center', sideOffset = 4, ...props }, ref) => (
16 |
17 |
27 |
28 | ));
29 | PopoverContent.displayName = PopoverPrimitive.Content.displayName;
30 |
31 | export { Popover, PopoverTrigger, PopoverContent };
32 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/api/processor.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: Processor
3 | ---
4 |
5 | The `Processor` interface represents a job processor that can process jobs from the queue, either in the background or synchronously.
6 |
7 | ## Creating a processor
8 |
9 | Create a processor by calling `createProcessor` on the queue.
10 |
11 | ```ts
12 | const jobQueue = getJobQueue();
13 | const processor = queue.createProcessor(handlers, options);
14 | ```
15 |
16 | ### ProcessorOptions
17 |
18 | ```ts
19 | interface ProcessorOptions {
20 | workerId?: string;
21 | batchSize?: number;
22 | concurrency?: number;
23 | pollInterval?: number;
24 | onError?: (error: Error) => void;
25 | verbose?: boolean;
26 | jobType?: string | string[];
27 | }
28 | ```
29 |
30 | ## Methods
31 |
32 | ### startInBackground
33 |
34 | ```ts
35 | startInBackground(): void
36 | ```
37 |
38 | Start the job processor in the background. This will run continuously and process jobs as they become available. It polls for new jobs every `pollInterval` milliseconds (default: 5 seconds).
39 |
40 | ### stop
41 |
42 | ```ts
43 | stop(): void
44 | ```
45 |
46 | Stop the job processor that runs in the background.
47 |
48 | ### isRunning
49 |
50 | ```ts
51 | isRunning(): boolean
52 | ```
53 |
54 | Check if the job processor is running.
55 |
56 | ### start
57 |
58 | ```ts
59 | start(): Promise
60 | ```
61 |
62 | Start the job processor synchronously. This will process jobs immediately and then stop. Returns the number of jobs processed.
63 |
--------------------------------------------------------------------------------
/.github/workflows/ci.yml:
--------------------------------------------------------------------------------
1 | name: Checks and Tests
2 |
3 | on:
4 | push:
5 | branches:
6 | - main
7 | pull_request:
8 |
9 | concurrency:
10 | group: ${{ github.workflow }}-${{ github.ref }}
11 | cancel-in-progress: true
12 |
13 | jobs:
14 | ci:
15 | runs-on: ubuntu-latest
16 | services:
17 | postgres:
18 | image: postgres
19 | env:
20 | POSTGRES_USER: postgres
21 | POSTGRES_PASSWORD: postgres
22 | POSTGRES_DB: postgres
23 | options: >-
24 | --health-cmd pg_isready
25 | --health-interval 10s
26 | --health-timeout 5s
27 | --health-retries 5
28 | ports:
29 | - 5432:5432
30 | steps:
31 | - uses: actions/checkout@v4
32 |
33 | - name: Use Node.js
34 | uses: actions/setup-node@v4
35 | with:
36 | node-version: '20'
37 |
38 | - name: Set up pnpm
39 | uses: pnpm/action-setup@v4
40 |
41 | - name: Install dependencies
42 | run: pnpm install
43 |
44 | - name: Build
45 | working-directory: packages/dataqueue
46 | run: pnpm run build
47 |
48 | - name: Run check format
49 | run: pnpm run check-format
50 |
51 | - name: Run check exports
52 | working-directory: packages/dataqueue
53 | run: pnpm run check-exports
54 |
55 | - name: Run lint
56 | working-directory: packages/dataqueue
57 | run: pnpm run lint
58 |
59 | - name: Run test
60 | working-directory: packages/dataqueue
61 | run: pnpm run test
62 |
--------------------------------------------------------------------------------
/apps/docs/app/(docs)/[[...slug]]/page.tsx:
--------------------------------------------------------------------------------
1 | import { source } from '@/lib/source';
2 | import {
3 | DocsPage,
4 | DocsBody,
5 | DocsDescription,
6 | DocsTitle,
7 | } from 'fumadocs-ui/page';
8 | import { notFound } from 'next/navigation';
9 | import { createRelativeLink } from 'fumadocs-ui/mdx';
10 | import { getMDXComponents } from '@/mdx-components';
11 |
12 | export default async function Page(props: {
13 | params: Promise<{ slug?: string[] }>;
14 | }) {
15 | const params = await props.params;
16 | const page = source.getPage(params.slug);
17 | if (!page) notFound();
18 |
19 | const MDXContent = page.data.body;
20 |
21 | return (
22 |
23 | {page.data.title}
24 | {page.data.description}
25 |
26 |
32 |
33 |
34 | );
35 | }
36 |
37 | export async function generateStaticParams() {
38 | return source.generateParams();
39 | }
40 |
41 | export async function generateMetadata(props: {
42 | params: Promise<{ slug?: string[] }>;
43 | }) {
44 | const params = await props.params;
45 | const page = source.getPage(params.slug);
46 | if (!page) notFound();
47 |
48 | return {
49 | title: `${page.data.title} - DataQueue`,
50 | description: page.data.description,
51 | };
52 | }
53 |
--------------------------------------------------------------------------------
/packages/dataqueue/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # dataqueue
2 |
3 | ## 1.24.0
4 |
5 | ### Minor Changes
6 |
7 | - Added edit job feature.
8 | - Added force kill on timeout feature.
9 |
10 | ## 1.22.0
11 |
12 | ### Minor Changes
13 |
14 | - Added SSL support
15 |
16 | ## 1.21.0
17 |
18 | ### Minor Changes
19 |
20 | - Added getJobs by filters and updated the cancelAllUpcomingJobs runAt filter
21 |
22 | ## 1.20.0
23 |
24 | ### Minor Changes
25 |
26 | - Added new feature to add, search, and cancel jobs by tags
27 |
28 | ## 1.19.3
29 |
30 | ### Patch Changes
31 |
32 | - Fix CLI ESM
33 |
34 | ## 1.19.2
35 |
36 | ### Patch Changes
37 |
38 | - fix cli
39 |
40 | ## 1.19.1
41 |
42 | ### Patch Changes
43 |
44 | - Refactor the cli
45 |
46 | ## 1.19.0
47 |
48 | ### Minor Changes
49 |
50 | - Fixed cannot connect to database schema
51 |
52 | ## 1.18.2
53 |
54 | ### Patch Changes
55 |
56 | - Fixed the CLI
57 |
58 | ## 1.18.1
59 |
60 | ### Patch Changes
61 |
62 | - Removed the env check from cli
63 |
64 | ## 1.17.0
65 |
66 | ### Minor Changes
67 |
68 | - Update docs and cases
69 |
70 | ## 1.16.0
71 |
72 | ### Minor Changes
73 |
74 | - Rename the package
75 |
76 | ## 1.15.0
77 |
78 | ### Minor Changes
79 |
80 | - Added job events
81 |
82 | ## 1.14.0
83 |
84 | ### Minor Changes
85 |
86 | - Added per job timeout support
87 |
88 | ## 1.13.0
89 |
90 | ### Minor Changes
91 |
92 | - Added cli for migrations
93 |
94 | ## 1.12.0
95 |
96 | ### Minor Changes
97 |
98 | - Initial release
99 |
100 | ## 1.11.0
101 |
102 | ### Minor Changes
103 |
104 | - Initial release
105 |
--------------------------------------------------------------------------------
/apps/website/components/ui/avatar.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as AvatarPrimitive from '@radix-ui/react-avatar';
5 |
6 | import { cn } from '@/lib/utils';
7 |
8 | const Avatar = React.forwardRef<
9 | React.ElementRef,
10 | React.ComponentPropsWithoutRef
11 | >(({ className, ...props }, ref) => (
12 |
20 | ));
21 | Avatar.displayName = AvatarPrimitive.Root.displayName;
22 |
23 | const AvatarImage = React.forwardRef<
24 | React.ElementRef,
25 | React.ComponentPropsWithoutRef
26 | >(({ className, ...props }, ref) => (
27 |
32 | ));
33 | AvatarImage.displayName = AvatarPrimitive.Image.displayName;
34 |
35 | const AvatarFallback = React.forwardRef<
36 | React.ElementRef,
37 | React.ComponentPropsWithoutRef
38 | >(({ className, ...props }, ref) => (
39 |
47 | ));
48 | AvatarFallback.displayName = AvatarPrimitive.Fallback.displayName;
49 |
50 | export { Avatar, AvatarImage, AvatarFallback };
51 |
--------------------------------------------------------------------------------
/apps/website/components/ui/toggle.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as TogglePrimitive from '@radix-ui/react-toggle';
5 | import { cva, type VariantProps } from 'class-variance-authority';
6 |
7 | import { cn } from '@/lib/utils';
8 |
9 | const toggleVariants = cva(
10 | 'inline-flex items-center justify-center rounded-md text-sm font-medium ring-offset-background transition-colors hover:bg-muted hover:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 data-[state=on]:bg-accent data-[state=on]:text-accent-foreground',
11 | {
12 | variants: {
13 | variant: {
14 | default: 'bg-transparent',
15 | outline:
16 | 'border border-input bg-transparent hover:bg-accent hover:text-accent-foreground',
17 | },
18 | size: {
19 | default: 'h-10 px-3',
20 | sm: 'h-9 px-2.5',
21 | lg: 'h-11 px-5',
22 | },
23 | },
24 | defaultVariants: {
25 | variant: 'default',
26 | size: 'default',
27 | },
28 | },
29 | );
30 |
31 | const Toggle = React.forwardRef<
32 | React.ElementRef,
33 | React.ComponentPropsWithoutRef &
34 | VariantProps
35 | >(({ className, variant, size, ...props }, ref) => (
36 |
41 | ));
42 |
43 | Toggle.displayName = TogglePrimitive.Root.displayName;
44 |
45 | export { Toggle, toggleVariants };
46 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/api/job-options.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: JobOptions
3 | ---
4 |
5 | The `JobOptions` interface defines the options for creating a new job in the queue.
6 |
7 | ## Fields
8 |
9 | - `jobType`: _string_ — The type of the job.
10 | - `payload`: _any_ — The payload for the job, type-safe per job
11 | type.
12 | - `maxAttempts?`: _number_ — Maximum number of attempts for
13 | this job (default: 3).
14 | - `priority?`: _number_ — Priority of the job (higher runs
15 | first, default: 0).
16 | - `runAt?`: _Date | null_ — When to run the job (default: now).
17 | - `timeoutMs?`: _number_ — Timeout for this job in milliseconds.
18 | If not set, uses the processor default or unlimited.
19 | - `forceKillOnTimeout?`: _boolean_ — If true, the job will be forcefully terminated (using Worker Threads) when timeout is reached. If false (default), the job will only receive an AbortSignal and must handle the abort gracefully.
20 |
21 | **⚠️ Runtime Requirements**: This option requires **Node.js** and will **not work** in Bun or other runtimes without worker thread support. See [Force Kill on Timeout](/docs/usage/force-kill-timeout) for details.
22 |
23 | - `tags?`: _string[]_ — Tags for this job. Used for grouping, searching, or batch operations.
24 |
25 | ## Example
26 |
27 | ```ts
28 | const job = {
29 | jobType: 'email',
30 | payload: { to: 'user@example.com', subject: 'Hello' },
31 | maxAttempts: 5,
32 | priority: 10,
33 | runAt: new Date(Date.now() + 60000), // run in 1 minute
34 | timeoutMs: 30000, // 30 seconds
35 | forceKillOnTimeout: false, // Use graceful shutdown (default)
36 | tags: ['welcome', 'user'], // tags for grouping/searching
37 | };
38 | ```
39 |
--------------------------------------------------------------------------------
/apps/website/components/ui/radio-group.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as RadioGroupPrimitive from '@radix-ui/react-radio-group';
5 | import { Circle } from 'lucide-react';
6 |
7 | import { cn } from '@/lib/utils';
8 |
9 | const RadioGroup = React.forwardRef<
10 | React.ElementRef,
11 | React.ComponentPropsWithoutRef
12 | >(({ className, ...props }, ref) => {
13 | return (
14 |
19 | );
20 | });
21 | RadioGroup.displayName = RadioGroupPrimitive.Root.displayName;
22 |
23 | const RadioGroupItem = React.forwardRef<
24 | React.ElementRef,
25 | React.ComponentPropsWithoutRef
26 | >(({ className, ...props }, ref) => {
27 | return (
28 |
36 |
37 |
38 |
39 |
40 | );
41 | });
42 | RadioGroupItem.displayName = RadioGroupPrimitive.Item.displayName;
43 |
44 | export { RadioGroup, RadioGroupItem };
45 |
--------------------------------------------------------------------------------
/.github/workflows/changeset-version-pr.yml:
--------------------------------------------------------------------------------
1 | name: Changeset Version PR
2 |
3 | on:
4 | workflow_dispatch:
5 |
6 | jobs:
7 | version:
8 | runs-on: ubuntu-latest
9 | steps:
10 | - name: Checkout code
11 | uses: actions/checkout@v4
12 |
13 | - name: Set up Node.js
14 | uses: actions/setup-node@v4
15 | with:
16 | node-version: 20
17 |
18 | - name: Set up pnpm
19 | uses: pnpm/action-setup@v2
20 | with:
21 | version: 8
22 |
23 | - name: Install dependencies
24 | run: pnpm install
25 |
26 | - name: Run changeset version
27 | run: pnpm exec changeset version
28 |
29 | - name: Configure git
30 | run: |
31 | git config user.name "github-actions[bot]"
32 | git config user.email "github-actions[bot]@users.noreply.github.com"
33 |
34 | - name: Create branch, commit, and push
35 | id: create_branch
36 | run: |
37 | BRANCH="changeset-version-$(date +%s)"
38 | git checkout -b $BRANCH
39 | git add .
40 | git commit -m "chore: version packages with changeset"
41 | git push origin $BRANCH
42 | echo "branch=$BRANCH" >> $GITHUB_OUTPUT
43 | env:
44 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
45 |
46 | - name: Create Pull Request
47 | uses: peter-evans/create-pull-request@v6
48 | with:
49 | token: ${{ secrets.GITHUB_TOKEN }}
50 | branch: ${{ steps.create_branch.outputs.branch }}
51 | title: 'chore: version packages with changeset'
52 | body: 'Automated version bump and changelog update via changeset.'
53 | base: main
54 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/usage/cleanup-jobs.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: Cleanup Jobs
3 | ---
4 |
5 | If you have a lot of jobs, you may want to clean up old ones—for example, keeping only jobs from the last 30 days. You can do this by calling the `cleanupOldJobs` method. The example below shows an API route (`/api/cron/cleanup`) that can be triggered by a cron job:
6 |
7 | ```typescript title="@/app/api/cron/cleanup.ts"
8 | import { getJobQueue } from '@/lib/queue';
9 | import { NextResponse } from 'next/server';
10 |
11 | export async function GET(request: Request) {
12 | const authHeader = request.headers.get('authorization');
13 | if (authHeader !== `Bearer ${process.env.CRON_SECRET}`) {
14 | return NextResponse.json({ message: 'Unauthorized' }, { status: 401 });
15 | }
16 |
17 | try {
18 | // [!code highlight:5]
19 | const jobQueue = getJobQueue();
20 |
21 | // Clean up old jobs (keep only the last 30 days)
22 | const deleted = await jobQueue.cleanupOldJobs(30);
23 | console.log(`Deleted ${deleted} old jobs`);
24 |
25 | return NextResponse.json({
26 | message: 'Old jobs cleaned up',
27 | deleted,
28 | });
29 | } catch (error) {
30 | console.error('Error cleaning up jobs:', error);
31 | return NextResponse.json(
32 | { message: 'Failed to clean up jobs' },
33 | { status: 500 },
34 | );
35 | }
36 | }
37 | ```
38 |
39 | #### Scheduling the Cleanup Job with Cron
40 |
41 | Add the following to your `vercel.json` to call the cleanup route every day at midnight:
42 |
43 | ```json title="vercel.json"
44 | {
45 | "crons": [
46 | {
47 | "path": "/api/cron/cleanup",
48 | "schedule": "0 0 * * *"
49 | }
50 | ]
51 | }
52 | ```
53 |
--------------------------------------------------------------------------------
/apps/docs/README.md:
--------------------------------------------------------------------------------
1 | # docs
2 |
3 | This is a Next.js application generated with
4 | [Create Fumadocs](https://github.com/fuma-nama/fumadocs).
5 |
6 | Run development server:
7 |
8 | ```bash
9 | npm run dev
10 | # or
11 | pnpm dev
12 | # or
13 | yarn dev
14 | ```
15 |
16 | Open http://localhost:3000 with your browser to see the result.
17 |
18 | ## Explore
19 |
20 | In the project, you can see:
21 |
22 | - `lib/source.ts`: Code for content source adapter, [`loader()`](https://fumadocs.dev/docs/headless/source-api) provides the interface to access your content.
23 | - `app/layout.config.tsx`: Shared options for layouts, optional but preferred to keep.
24 |
25 | | Route | Description |
26 | | ------------------------- | ------------------------------------------------------ |
27 | | `app/(home)` | The route group for your landing page and other pages. |
28 | | `app/docs` | The documentation layout and pages. |
29 | | `app/api/search/route.ts` | The Route Handler for search. |
30 |
31 | ### Fumadocs MDX
32 |
33 | A `source.config.ts` config file has been included, you can customise different options like frontmatter schema.
34 |
35 | Read the [Introduction](https://fumadocs.dev/docs/mdx) for further details.
36 |
37 | ## Learn More
38 |
39 | To learn more about Next.js and Fumadocs, take a look at the following
40 | resources:
41 |
42 | - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js
43 | features and API.
44 | - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
45 | - [Fumadocs](https://fumadocs.vercel.app) - learn about Fumadocs
46 |
--------------------------------------------------------------------------------
/apps/demo/lib/queue.ts:
--------------------------------------------------------------------------------
1 | import { initJobQueue, JobHandlers } from '@nicnocquee/dataqueue';
2 | import { sendEmail } from './services/email';
3 | import { generateReport } from './services/generate-report';
4 | import { generateImageAi } from './services/generate-image-ai';
5 |
6 | // Define the job payload map for this app.
7 | // This will ensure that the job payload is typed correctly when adding jobs.
8 | // The keys are the job types, and the values are the payload types.
9 | export type JobPayloadMap = {
10 | send_email: {
11 | to: string;
12 | subject: string;
13 | body: string;
14 | };
15 | generate_report: {
16 | reportId: string;
17 | userId: string;
18 | };
19 | generate_image: {
20 | prompt: string;
21 | };
22 | };
23 |
24 | let jobQueue: ReturnType> | null = null;
25 |
26 | export const getJobQueue = () => {
27 | if (!jobQueue) {
28 | jobQueue = initJobQueue({
29 | databaseConfig: {
30 | connectionString: process.env.PG_DATAQUEUE_DATABASE, // Set this in your environment
31 | },
32 | verbose: process.env.NODE_ENV === 'development',
33 | });
34 | }
35 | return jobQueue;
36 | };
37 |
38 | // Object literal mapping for static enforcement
39 | export const jobHandlers: JobHandlers = {
40 | send_email: async (payload) => {
41 | const { to, subject, body } = payload;
42 | await sendEmail(to, subject, body);
43 | },
44 | generate_report: async (payload) => {
45 | const { reportId, userId } = payload;
46 | await generateReport(reportId, userId);
47 | },
48 | generate_image: async (payload, signal) => {
49 | const { prompt } = payload;
50 | await generateImageAi(prompt, signal);
51 | },
52 | };
53 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/usage/add-job.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: Add Job
3 | ---
4 |
5 | You can add jobs to the queue from your application logic, such as in a [server function](https://react.dev/reference/rsc/server-functions):
6 |
7 | ```typescript title="@/app/actions/send-email.ts"
8 | 'use server';
9 |
10 | import { getJobQueue } from '@/lib/queue';
11 | import { revalidatePath } from 'next/cache';
12 |
13 | export const sendEmail = async ({
14 | name,
15 | email,
16 | }: {
17 | name: string;
18 | email: string;
19 | }) => {
20 | // Add a welcome email job
21 | const jobQueue = getJobQueue(); // [!code highlight]
22 | try {
23 | const runAt = new Date(Date.now() + 5 * 1000); // Run 5 seconds from now
24 | // [!code highlight:10]
25 | const job = await jobQueue.addJob({
26 | jobType: 'send_email',
27 | payload: {
28 | to: email,
29 | subject: 'Welcome to our platform!',
30 | body: `Hi ${name}, welcome to our platform!`,
31 | },
32 | priority: 10, // Higher number = higher priority
33 | runAt: runAt,
34 | tags: ['welcome', 'user'], // Add tags for grouping/searching
35 | });
36 |
37 | revalidatePath('/');
38 | return { job };
39 | } catch (error) {
40 | console.error('Error adding job:', error);
41 | throw error;
42 | }
43 | };
44 | ```
45 |
46 | In the example above, a job is added to the queue to send an email. The job type is `send_email`, and the payload includes the recipient's email, subject, and body.
47 |
48 | When adding a job, you can set its `priority`, schedule when it should run using `runAt`, and specify a timeout in milliseconds with `timeoutMs`.
49 |
50 | You can also add `tags` (an array of strings) to group, search, or batch jobs by category. See [Tags](/api/tags) for more details.
51 |
--------------------------------------------------------------------------------
/apps/demo/app/page.tsx:
--------------------------------------------------------------------------------
1 | import Buttons from './buttons';
2 | import {
3 | CancelledJobs,
4 | CompletedJobs,
5 | FailedJobs,
6 | PendingJobs,
7 | ProcessingJobs,
8 | WillRetryFailedJobs,
9 | } from './queue/list';
10 | import { refresh } from './queue/refresh';
11 | import { RefreshPeriodically } from './refresh-periodically';
12 |
13 | export default function Home() {
14 | return (
15 |
16 |
17 |
Dataqueue Demo
18 |
19 | This is a demo of the job queue. When run via pnpm dev{' '}
20 | from the root, the api/cron/process endpoint will be
21 | called every 10 seconds.
22 |
23 |
24 |
25 |
26 |
27 |
28 |
Pending Jobs
29 |
30 |
31 |
32 |
Processing Jobs
33 |
34 |
35 |
36 |
Will Retry Failed Jobs
37 |
38 |
39 |
40 |
Failed Jobs
41 |
42 |
43 |
44 |
Cancelled Jobs
45 |
46 |
47 |
48 |
Completed Jobs
49 |
50 |
51 |
52 |
53 | );
54 | }
55 |
--------------------------------------------------------------------------------
/apps/website/components/ui/alert.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import { cva, type VariantProps } from 'class-variance-authority';
3 |
4 | import { cn } from '@/lib/utils';
5 |
6 | const alertVariants = cva(
7 | 'relative w-full rounded-lg border p-4 [&>svg~*]:pl-7 [&>svg+div]:translate-y-[-3px] [&>svg]:absolute [&>svg]:left-4 [&>svg]:top-4 [&>svg]:text-foreground',
8 | {
9 | variants: {
10 | variant: {
11 | default: 'bg-background text-foreground',
12 | destructive:
13 | 'border-destructive/50 text-destructive dark:border-destructive [&>svg]:text-destructive',
14 | },
15 | },
16 | defaultVariants: {
17 | variant: 'default',
18 | },
19 | },
20 | );
21 |
22 | const Alert = React.forwardRef<
23 | HTMLDivElement,
24 | React.HTMLAttributes & VariantProps
25 | >(({ className, variant, ...props }, ref) => (
26 |
32 | ));
33 | Alert.displayName = 'Alert';
34 |
35 | const AlertTitle = React.forwardRef<
36 | HTMLParagraphElement,
37 | React.HTMLAttributes
38 | >(({ className, ...props }, ref) => (
39 |
44 | ));
45 | AlertTitle.displayName = 'AlertTitle';
46 |
47 | const AlertDescription = React.forwardRef<
48 | HTMLParagraphElement,
49 | React.HTMLAttributes
50 | >(({ className, ...props }, ref) => (
51 |
56 | ));
57 | AlertDescription.displayName = 'AlertDescription';
58 |
59 | export { Alert, AlertTitle, AlertDescription };
60 |
--------------------------------------------------------------------------------
/apps/website/components/ui/scroll-area.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as ScrollAreaPrimitive from '@radix-ui/react-scroll-area';
5 |
6 | import { cn } from '@/lib/utils';
7 |
8 | const ScrollArea = React.forwardRef<
9 | React.ElementRef,
10 | React.ComponentPropsWithoutRef
11 | >(({ className, children, ...props }, ref) => (
12 |
17 |
18 | {children}
19 |
20 |
21 |
22 |
23 | ));
24 | ScrollArea.displayName = ScrollAreaPrimitive.Root.displayName;
25 |
26 | const ScrollBar = React.forwardRef<
27 | React.ElementRef,
28 | React.ComponentPropsWithoutRef
29 | >(({ className, orientation = 'vertical', ...props }, ref) => (
30 |
43 |
44 |
45 | ));
46 | ScrollBar.displayName = ScrollAreaPrimitive.ScrollAreaScrollbar.displayName;
47 |
48 | export { ScrollArea, ScrollBar };
49 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/usage/get-jobs.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: Get Jobs
3 | ---
4 |
5 | To get a job by its ID:
6 |
7 | ```typescript
8 | const job = await jobQueue.getJob(jobId);
9 | ```
10 |
11 | To get all jobs:
12 |
13 | ```typescript
14 | const jobs = await jobQueue.getAllJobs(limit, offset);
15 | ```
16 |
17 | To get jobs by status:
18 |
19 | ```typescript
20 | const jobs = await jobQueue.getJobsByStatus(status, limit, offset);
21 | ```
22 |
23 | ## Get Jobs by Tags
24 |
25 | You can get jobs by their tags using the `getJobsByTags` method:
26 |
27 | ```typescript
28 | const jobs = await jobQueue.getJobsByTags(['welcome', 'user'], 'all', 10, 0);
29 | ```
30 |
31 | - The first argument is an array of tags to match.
32 | - The second argument is the tag query mode. See [Tags](/api/tags) for more details.
33 | - The third and fourth arguments are optional for pagination.
34 |
35 | ## Get Jobs by Filter
36 |
37 | You can retrieve jobs using multiple filters with the `getJobs` method:
38 |
39 | ```typescript
40 | const jobs = await jobQueue.getJobs(
41 | {
42 | jobType: 'email',
43 | priority: 2,
44 | runAt: { gte: new Date('2024-01-01'), lt: new Date('2024-02-01') },
45 | tags: { values: ['welcome', 'user'], mode: 'all' },
46 | },
47 | 10,
48 | 0,
49 | );
50 | ```
51 |
52 | - The first argument is an optional filter object. You can filter by:
53 | - `jobType`: The job type (string).
54 | - `priority`: The job priority (number).
55 | - `runAt`: The scheduled time. You can use a `Date` for exact match, or an object with `gt`, `gte`, `lt`, `lte`, or `eq` for range queries.
56 | - `tags`: An object with `values` (array of tags) and `mode` (see [Tags](/api/tags)).
57 | - The second and third arguments are optional for pagination (`limit`, `offset`).
58 |
59 | You can combine any of these filters. If no filters are provided, all jobs are returned (with pagination if specified).
60 |
--------------------------------------------------------------------------------
/apps/demo/app/job/[id]/page.tsx:
--------------------------------------------------------------------------------
1 | import {
2 | Table,
3 | TableBody,
4 | TableCell,
5 | TableHead,
6 | TableHeader,
7 | TableRow,
8 | } from '@/components/ui/table';
9 | import { getJobQueue } from '@/lib/queue';
10 | import { notFound } from 'next/navigation';
11 |
12 | const JobPage = async ({ params }: { params: Promise<{ id: string }> }) => {
13 | const { id } = await params;
14 | if (!id) {
15 | notFound();
16 | }
17 | const jobQueue = getJobQueue();
18 | const job = await jobQueue.getJob(Number(id));
19 | if (!job) {
20 | notFound();
21 | }
22 | const events = await jobQueue.getJobEvents(Number(id));
23 | return (
24 |
65 | );
66 | }
67 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/api/job-record.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: JobRecord
3 | ---
4 |
5 | The `JobRecord` interface represents a job stored in the queue, including its status, attempts, and metadata.
6 |
7 | ## Fields
8 |
9 | - `id`: _number_ — Unique job ID.
10 | - `jobType`: _string_ — The type of the job.
11 | - `payload`: _any_ — The job payload.
12 | - `status`:
13 | _'pending' | 'processing' | 'completed' | 'failed' | 'cancelled'_ —
14 | Current job status.
15 | - `createdAt`: _Date_ — When the job was created.
16 | - `updated_at`: _Date_ — When the job was last updated.
17 | - `locked_at`: _Date | null_ — When the job was locked for
18 | processing.
19 | - `locked_by`: _string | null_ — Worker that locked the job.
20 | - `attempts`: _number_ — Number of attempts so far.
21 | - `maxAttempts`: _number_ — Maximum allowed attempts.
22 | - `nextAttemptAt`: _Date | null_ — When the next attempt is
23 | scheduled.
24 | - `priority`: _number_ — Job priority.
25 | - `runAt`: _Date_ — When the job is scheduled to run.
26 | - `pendingReason?`: _string | null_ — Reason for pending
27 | status.
28 | - `errorHistory?`: _\{ message: string; timestamp: string \}[]_ — Error history for the job.
29 | - `timeoutMs?`: _number | null_ — Timeout for this job in
30 | milliseconds.
31 | - `failureReason?`: _FailureReason | null_ — Reason for last
32 | failure, if any.
33 | - `completedAt`: _Date | null_ — When the job was completed.
34 | - `startedAt`: _Date | null_ — When the job was first picked up
35 | for processing.
36 | - `lastRetriedAt`: _Date | null_ — When the job was last
37 | retried.
38 | - `lastFailedAt`: _Date | null_ — When the job last failed.
39 | - `lastCancelledAt`: _Date | null_ — When the job was last
40 | cancelled.
41 | - `tags?`: _string[]_ — Tags for this job. Used for grouping, searching, or batch operations.
42 |
43 | ## Example
44 |
45 | ```json
46 | {
47 | "id": 1,
48 | "jobType": "email",
49 | "payload": { "to": "user@example.com", "subject": "Hello" },
50 | "status": "pending",
51 | "createdAt": "2024-06-01T12:00:00Z",
52 | "tags": ["welcome", "user"]
53 | }
54 | ```
55 |
--------------------------------------------------------------------------------
/apps/website/components/ui/button.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import { Slot } from '@radix-ui/react-slot';
3 | import { cva, type VariantProps } from 'class-variance-authority';
4 |
5 | import { cn } from '@/lib/utils';
6 |
7 | const buttonVariants = cva(
8 | 'inline-flex items-center justify-center whitespace-nowrap rounded-md text-sm font-medium ring-offset-background transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50',
9 | {
10 | variants: {
11 | variant: {
12 | default: 'bg-primary text-primary-foreground hover:bg-primary/90',
13 | destructive:
14 | 'bg-destructive text-destructive-foreground hover:bg-destructive/90',
15 | outline:
16 | 'border border-input bg-background hover:bg-accent hover:text-accent-foreground',
17 | secondary:
18 | 'bg-secondary text-secondary-foreground hover:bg-secondary/80',
19 | ghost: 'hover:bg-accent hover:text-accent-foreground',
20 | link: 'text-primary underline-offset-4 hover:underline',
21 | },
22 | size: {
23 | default: 'h-10 px-4 py-2',
24 | sm: 'h-9 rounded-md px-3',
25 | lg: 'h-11 rounded-md px-8',
26 | icon: 'h-10 w-10',
27 | },
28 | },
29 | defaultVariants: {
30 | variant: 'default',
31 | size: 'default',
32 | },
33 | },
34 | );
35 |
36 | export interface ButtonProps
37 | extends React.ButtonHTMLAttributes,
38 | VariantProps {
39 | asChild?: boolean;
40 | }
41 |
42 | const Button = React.forwardRef(
43 | ({ className, variant, size, asChild = false, ...props }, ref) => {
44 | const Comp = asChild ? Slot : 'button';
45 | return (
46 |
51 | );
52 | },
53 | );
54 | Button.displayName = 'Button';
55 |
56 | export { Button, buttonVariants };
57 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/api/tags.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: Tags
3 | ---
4 |
5 | The tags feature lets you group, search, and batch jobs using arbitrary string tags. Tags can be set when adding a job and used in various JobQueue methods.
6 |
7 | ## Tags in JobOptions
8 |
9 | You can assign tags to a job when adding it:
10 |
11 | ```typescript
12 | await jobQueue.addJob({
13 | jobType: 'email',
14 | payload: { to: 'user@example.com', subject: 'Hello' },
15 | tags: ['welcome', 'user'],
16 | });
17 | ```
18 |
19 | ## Tags in JobRecord
20 |
21 | The `tags` field is available on JobRecord objects:
22 |
23 | ```json
24 | {
25 | "id": 1,
26 | "jobType": "email",
27 | "tags": ["welcome", "user"]
28 | }
29 | ```
30 |
31 | ## Tag Query Methods
32 |
33 | ### getJobsByTags
34 |
35 | ```typescript
36 | const jobs = await jobQueue.getJobsByTags(['welcome', 'user'], 'all');
37 | ```
38 |
39 | ### Cancel jobs by tags
40 |
41 | You can cancel jobs by their tags using the `cancelAllUpcomingJobs` method with the `tags` filter (an object with `values` and `mode`):
42 |
43 | ```typescript
44 | // Cancel all jobs with both 'welcome' and 'user' tags
45 | await jobQueue.cancelAllUpcomingJobs({
46 | tags: { values: ['welcome', 'user'], mode: 'all' },
47 | });
48 |
49 | // Cancel all jobs with any of the tags
50 | await jobQueue.cancelAllUpcomingJobs({
51 | tags: { values: ['foo', 'bar'], mode: 'any' },
52 | });
53 |
54 | // Cancel all jobs with exactly the given tags
55 | await jobQueue.cancelAllUpcomingJobs({
56 | tags: { values: ['foo', 'bar'], mode: 'exact' },
57 | });
58 |
59 | // Cancel all jobs with none of the given tags
60 | await jobQueue.cancelAllUpcomingJobs({
61 | tags: { values: ['foo', 'bar'], mode: 'none' },
62 | });
63 | ```
64 |
65 | ## TagQueryMode
66 |
67 | The `mode` parameter controls how tags are matched:
68 |
69 | - `'exact'`: Jobs with exactly the same tags (no more, no less)
70 | - `'all'`: Jobs that have all the given tags (can have more)
71 | - `'any'`: Jobs that have at least one of the given tags
72 | - `'none'`: Jobs that have none of the given tags
73 |
74 | The default mode is `'all'`.
75 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/intro/overview.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: Overview
3 | ---
4 |
5 | DataQueue is a lightweight library that helps you manage your job queue in a Postgres database. It has three main components: the processor, the queue, and the job. It is not an external tool or service. You install DataQueue in your project and use it to add jobs to the queue, process them, and more, using your own existing Postgres database.
6 |
7 | 
8 |
9 | ## Processor
10 |
11 | The processor has these responsibilities:
12 |
13 | - retrieve a certain number of unclaimed, pending jobs from the Postgres database
14 | - run the defined job handlers for each job
15 | - update the job status accordingly
16 | - retry failed jobs
17 |
18 | The processor doesn't run in a separate process. It runs in the same process as your application. In a serverless environment, you can initiate and start the processor for example in an API route. In a long running process environment, you can start the processor when your application starts, and it will periodically check for jobs to process.
19 |
20 | For more information, see [Processor](/api/processor).
21 |
22 | ## Queue
23 |
24 | The queue is an abstraction over the Postgres database. It has these responsibilities:
25 |
26 | - add jobs to the database
27 | - retrieve jobs from the database
28 | - cancel pending jobs
29 | - edit pending jobs
30 |
31 | For more information, see [Queue](/api/queue).
32 |
33 | ## Job
34 |
35 | A job that you add to the queue needs to have a type and a payload. The type is a string that identifies the job, and the payload is the data that will be passed to the job handler of that job type.
36 |
37 | Once a job is added to the queue, it can be in one of these states:
38 |
39 | - `pending`: The job is waiting in the queue to be processed.
40 | - `processing`: The job is currently being worked on.
41 | - `completed`: The job finished successfully.
42 | - `failed`: The job did not finish successfully. It can be retried up to `maxAttempts` times.
43 | - `cancelled`: The job was cancelled before it finished.
44 |
45 | For more information, see [Job](/api/job).
46 |
--------------------------------------------------------------------------------
/apps/website/components/ui/tabs.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as TabsPrimitive from '@radix-ui/react-tabs';
5 |
6 | import { cn } from '@/lib/utils';
7 |
8 | const Tabs = TabsPrimitive.Root;
9 |
10 | const TabsList = React.forwardRef<
11 | React.ElementRef,
12 | React.ComponentPropsWithoutRef
13 | >(({ className, ...props }, ref) => (
14 |
22 | ));
23 | TabsList.displayName = TabsPrimitive.List.displayName;
24 |
25 | const TabsTrigger = React.forwardRef<
26 | React.ElementRef,
27 | React.ComponentPropsWithoutRef
28 | >(({ className, ...props }, ref) => (
29 |
37 | ));
38 | TabsTrigger.displayName = TabsPrimitive.Trigger.displayName;
39 |
40 | const TabsContent = React.forwardRef<
41 | React.ElementRef,
42 | React.ComponentPropsWithoutRef
43 | >(({ className, ...props }, ref) => (
44 |
52 | ));
53 | TabsContent.displayName = TabsPrimitive.Content.displayName;
54 |
55 | export { Tabs, TabsList, TabsTrigger, TabsContent };
56 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/usage/job-timeout.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: Job Timeout
3 | ---
4 |
5 | When you add a job to the queue, you can set a timeout for it. If the job doesn't finish before the timeout, it will be marked as `failed` and may be retried. See [Failed Jobs](/docs/usage/failed-jobs) for more information.
6 |
7 | When the timeout is reached, DataQueue does not actually stop the handler from running. You need to handle this in your handler by checking the `AbortSignal` at one or more points in your code. For example:
8 |
9 | ```typescript title="@lib/job-handlers.ts"
10 | const handler = async (payload, signal) => {
11 | // Simulate work
12 | // Do something that may take a long time
13 |
14 | // Check if the job is aborted
15 | if (signal.aborted) {
16 | return;
17 | }
18 |
19 | // Do something else
20 | // Check again if the job is aborted
21 | if (signal.aborted) {
22 | return;
23 | }
24 |
25 | // ...rest of your logic
26 | };
27 | ```
28 |
29 | If the job times out, the signal will be aborted and your handler should exit early. If your handler does not check for `signal.aborted`, it will keep running in the background even after the job is marked as failed due to timeout. For best results, always make your handlers abortable if they might run for a long time.
30 |
31 | ## Force Kill on Timeout
32 |
33 | If you need to forcefully terminate jobs that don't respond to the abort signal, you can use `forceKillOnTimeout: true`. This will run the handler in a Worker Thread and forcefully terminate it when the timeout is reached.
34 |
35 | **⚠️ Runtime Requirements**: `forceKillOnTimeout` requires **Node.js** and will **not work** in Bun or other runtimes without worker thread support. See [Force Kill on Timeout](/usage/force-kill-timeout) for details.
36 |
37 | **Important**: When using `forceKillOnTimeout`, your handler must be serializable. See [Force Kill on Timeout](/usage/force-kill-timeout) for details.
38 |
39 | ```typescript
40 | await queue.addJob({
41 | jobType: 'longRunningTask',
42 | payload: { data: '...' },
43 | timeoutMs: 5000,
44 | forceKillOnTimeout: true, // Forcefully terminate if timeout is reached
45 | });
46 | ```
47 |
--------------------------------------------------------------------------------
/packages/dataqueue/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@nicnocquee/dataqueue",
3 | "version": "1.24.0",
4 | "description": "PostgreSQL-based job queue for Node.js applications with support for serverless environments",
5 | "type": "module",
6 | "main": "dist/index.js",
7 | "exports": {
8 | ".": {
9 | "import": "./dist/index.js",
10 | "require": "./dist/index.cjs"
11 | }
12 | },
13 | "types": "dist/index.d.ts",
14 | "files": [
15 | "dist/",
16 | "src/",
17 | "migrations/"
18 | ],
19 | "scripts": {
20 | "build": "tsup",
21 | "ci": "npm run build && npm run check-format && npm run check-exports && npm run lint && npm run test",
22 | "lint": "tsc",
23 | "test": "vitest run --reporter=verbose",
24 | "format": "prettier --write .",
25 | "check-format": "prettier --check .",
26 | "check-exports": "attw --pack .",
27 | "local-release": "changeset version && changeset publish",
28 | "dev": "tsup --watch",
29 | "migrate": "node-pg-migrate -d $PG_DATAQUEUE_DATABASE -m ./migrations",
30 | "changeset:add": "changeset",
31 | "changeset:version": "changeset version && find .changeset -type f -name '*.md' ! -name 'README.md' -delete"
32 | },
33 | "keywords": [
34 | "nextjs",
35 | "postgresql",
36 | "job-queue",
37 | "background-jobs",
38 | "vercel"
39 | ],
40 | "author": "Nico Prananta",
41 | "license": "MIT",
42 | "dependencies": {
43 | "pg": "^8.0.0",
44 | "pg-connection-string": "^2.9.1",
45 | "ts-node": "^10.9.2"
46 | },
47 | "devDependencies": {
48 | "@arethetypeswrong/cli": "^0.18.2",
49 | "@changesets/cli": "^2.29.5",
50 | "@types/node": "^24.0.4",
51 | "@types/pg": "^8.15.4",
52 | "@vitejs/plugin-react": "^4.6.0",
53 | "jsdom": "^26.1.0",
54 | "node-pg-migrate": "^8.0.3",
55 | "pnpm": "^9.0.0",
56 | "prettier": "^3.6.2",
57 | "tsup": "^8.5.0",
58 | "turbo": "^1.13.0",
59 | "typescript": "^5.8.3",
60 | "vite": "^7.0.0",
61 | "vitest": "^3.2.4"
62 | },
63 | "peerDependencies": {
64 | "node-pg-migrate": "^8.0.3",
65 | "pg": "^8.0.0"
66 | },
67 | "bin": {
68 | "dataqueue-cli": "./cli.cjs"
69 | }
70 | }
71 |
--------------------------------------------------------------------------------
/apps/website/components/ui/accordion.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import * as AccordionPrimitive from '@radix-ui/react-accordion';
5 | import { ChevronDown } from 'lucide-react';
6 |
7 | import { cn } from '@/lib/utils';
8 |
9 | const Accordion = AccordionPrimitive.Root;
10 |
11 | const AccordionItem = React.forwardRef<
12 | React.ElementRef,
13 | React.ComponentPropsWithoutRef
14 | >(({ className, ...props }, ref) => (
15 |
20 | ));
21 | AccordionItem.displayName = 'AccordionItem';
22 |
23 | const AccordionTrigger = React.forwardRef<
24 | React.ElementRef,
25 | React.ComponentPropsWithoutRef
26 | >(({ className, children, ...props }, ref) => (
27 |
28 | svg]:rotate-180',
32 | className,
33 | )}
34 | {...props}
35 | >
36 | {children}
37 |
38 |
39 |
40 | ));
41 | AccordionTrigger.displayName = AccordionPrimitive.Trigger.displayName;
42 |
43 | const AccordionContent = React.forwardRef<
44 | React.ElementRef,
45 | React.ComponentPropsWithoutRef
46 | >(({ className, children, ...props }, ref) => (
47 |
52 |
{children}
53 |
54 | ));
55 |
56 | AccordionContent.displayName = AccordionPrimitive.Content.displayName;
57 |
58 | export { Accordion, AccordionItem, AccordionTrigger, AccordionContent };
59 |
--------------------------------------------------------------------------------
/apps/website/components/ui/card.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 |
3 | import { cn } from '@/lib/utils';
4 |
5 | const Card = React.forwardRef<
6 | HTMLDivElement,
7 | React.HTMLAttributes
8 | >(({ className, ...props }, ref) => (
9 |
17 | ));
18 | Card.displayName = 'Card';
19 |
20 | const CardHeader = React.forwardRef<
21 | HTMLDivElement,
22 | React.HTMLAttributes
23 | >(({ className, ...props }, ref) => (
24 |
29 | ));
30 | CardHeader.displayName = 'CardHeader';
31 |
32 | const CardTitle = React.forwardRef<
33 | HTMLParagraphElement,
34 | React.HTMLAttributes
35 | >(({ className, ...props }, ref) => (
36 |
44 | ));
45 | CardTitle.displayName = 'CardTitle';
46 |
47 | const CardDescription = React.forwardRef<
48 | HTMLParagraphElement,
49 | React.HTMLAttributes
50 | >(({ className, ...props }, ref) => (
51 |
56 | ));
57 | CardDescription.displayName = 'CardDescription';
58 |
59 | const CardContent = React.forwardRef<
60 | HTMLDivElement,
61 | React.HTMLAttributes
62 | >(({ className, ...props }, ref) => (
63 |
64 | ));
65 | CardContent.displayName = 'CardContent';
66 |
67 | const CardFooter = React.forwardRef<
68 | HTMLDivElement,
69 | React.HTMLAttributes
70 | >(({ className, ...props }, ref) => (
71 |
76 | ));
77 | CardFooter.displayName = 'CardFooter';
78 |
79 | export {
80 | Card,
81 | CardHeader,
82 | CardFooter,
83 | CardTitle,
84 | CardDescription,
85 | CardContent,
86 | };
87 |
--------------------------------------------------------------------------------
/packages/dataqueue/src/test-util.ts:
--------------------------------------------------------------------------------
1 | import { Pool } from 'pg';
2 | import { randomUUID } from 'crypto';
3 | import { fileURLToPath } from 'url';
4 | import { join, dirname } from 'path';
5 | import { runner } from 'node-pg-migrate';
6 |
7 | const __filename = fileURLToPath(import.meta.url);
8 | const __dirname = dirname(__filename);
9 |
10 | export async function createTestDbAndPool() {
11 | const baseDatabaseUrl =
12 | process.env.PG_TEST_URL ||
13 | 'postgres://postgres:postgres@localhost:5432/postgres';
14 | const dbName = `test_db_${randomUUID().replace(/-/g, '')}`;
15 |
16 | // 1. Connect to the default database to create a new test database
17 | const adminPool = new Pool({ connectionString: baseDatabaseUrl });
18 | await adminPool.query(`CREATE DATABASE ${dbName}`);
19 | await adminPool.end();
20 |
21 | // 2. Connect to the new test database
22 | const testDbUrl = baseDatabaseUrl.replace(/(\/)[^/]+$/, `/${dbName}`);
23 | const pool = new Pool({ connectionString: testDbUrl });
24 |
25 | // Wait a bit to ensure DB visibility
26 | await new Promise((r) => setTimeout(r, 50));
27 |
28 | // 3. Run migrations
29 | try {
30 | await runner({
31 | databaseUrl: testDbUrl,
32 | dir: join(__dirname, '../migrations'),
33 | direction: 'up',
34 | count: Infinity,
35 | migrationsTable: 'pgmigrations',
36 | verbose: false,
37 | logger: {
38 | debug: () => {},
39 | info: () => {},
40 | warn: () => {},
41 | error: () => {},
42 | },
43 | });
44 | } catch (error) {
45 | console.error(error);
46 | }
47 |
48 | return { pool, dbName, testDbUrl };
49 | }
50 |
51 | export async function destroyTestDb(dbName: string) {
52 | const baseDatabaseUrl =
53 | process.env.PG_TEST_URL ||
54 | 'postgres://postgres:postgres@localhost:5432/postgres';
55 | const adminPool = new Pool({ connectionString: baseDatabaseUrl });
56 | // Terminate all connections to the test database before dropping
57 | await adminPool.query(
58 | `
59 | SELECT pg_terminate_backend(pid)
60 | FROM pg_stat_activity
61 | WHERE datname = $1 AND pid <> pg_backend_pid()
62 | `,
63 | [dbName],
64 | );
65 | await adminPool.query(`DROP DATABASE IF EXISTS ${dbName}`);
66 | await adminPool.end();
67 | }
68 |
--------------------------------------------------------------------------------
/apps/demo/components/ui/button.tsx:
--------------------------------------------------------------------------------
1 | import * as React from 'react';
2 | import { Slot } from '@radix-ui/react-slot';
3 | import { cva, type VariantProps } from 'class-variance-authority';
4 |
5 | import { cn } from '@/lib/utils';
6 |
7 | const buttonVariants = cva(
8 | "inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
9 | {
10 | variants: {
11 | variant: {
12 | default:
13 | 'bg-primary text-primary-foreground shadow-xs hover:bg-primary/90',
14 | destructive:
15 | 'bg-destructive text-white shadow-xs hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60',
16 | outline:
17 | 'border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input dark:hover:bg-input/50',
18 | secondary:
19 | 'bg-secondary text-secondary-foreground shadow-xs hover:bg-secondary/80',
20 | ghost:
21 | 'hover:bg-accent hover:text-accent-foreground dark:hover:bg-accent/50',
22 | link: 'text-primary underline-offset-4 hover:underline',
23 | },
24 | size: {
25 | default: 'h-9 px-4 py-2 has-[>svg]:px-3',
26 | sm: 'h-8 rounded-md gap-1.5 px-3 has-[>svg]:px-2.5',
27 | lg: 'h-10 rounded-md px-6 has-[>svg]:px-4',
28 | icon: 'size-9',
29 | },
30 | },
31 | defaultVariants: {
32 | variant: 'default',
33 | size: 'default',
34 | },
35 | },
36 | );
37 |
38 | function Button({
39 | className,
40 | variant,
41 | size,
42 | asChild = false,
43 | ...props
44 | }: React.ComponentProps<'button'> &
45 | VariantProps & {
46 | asChild?: boolean;
47 | }) {
48 | const Comp = asChild ? Slot : 'button';
49 |
50 | return (
51 |
56 | );
57 | }
58 |
59 | export { Button, buttonVariants };
60 |
--------------------------------------------------------------------------------
/apps/docs/content/docs/intro/index.mdx:
--------------------------------------------------------------------------------
1 | ---
2 | title: About
3 | ---
4 |
5 | DataQueue is an open source lightweight job queue for Node.js/TypeScript projects, backed by PostgreSQL. It lets you easily schedule, process, and manage background jobs. It's ideal for serverless environments like Vercel, AWS Lambda, and more.
6 |
7 | ## Features
8 |
9 | - Simple API for adding and processing jobs
10 | - Strong typing for job types and payloads, preventing you from adding jobs with the wrong payload and ensuring handlers receive the correct type
11 | - Works in serverless environments
12 | - Supports job priorities, scheduling, canceling, and retries
13 | - Reclaims stuck jobs: No job will remain in the `processing` state indefinitely
14 | - Cleans up old jobs: Keeps only jobs from the last xxx days
15 |
16 | ## Who is this for?
17 |
18 | This package is for you if all of the following apply:
19 |
20 | | | |
21 | | --- | ---------------------------------------------------------------------------------------------- |
22 | | ☁️ | You deploy web apps to serverless platforms like Vercel, AWS Lambda, etc. |
23 | | 📝 | You use TypeScript |
24 | | ⚡ | You want your app to stay fast and responsive by offloading heavy tasks to the background |
25 | | 💾 | You use PostgreSQL as your database |
26 | | 🤷♂️ | You don't want to set up or maintain another queue system like Redis |
27 | | 💸 | You're on a budget and want to avoid paying for a job queue service or running your own server |
28 |
29 | ## Why PostgreSQL?
30 |
31 | Many people use Redis for job queues, but adding another tool to your stack can increase costs and maintenance. If you already use PostgreSQL, it makes sense to use it for job queues, thanks to [SKIP LOCKED](https://www.postgresql.org/docs/current/sql-select.html).
32 |
33 | The update process in DataQueue uses `FOR UPDATE SKIP LOCKED` to avoid race conditions and improve performance. If two jobs are scheduled at the same time, one will skip any jobs that are already being processed and work on other available jobs instead. This lets multiple workers handle different jobs at once without waiting or causing conflicts, making PostgreSQL a great choice for job queues and similar tasks.
34 |
--------------------------------------------------------------------------------
/apps/website/components/ui/input-otp.tsx:
--------------------------------------------------------------------------------
1 | 'use client';
2 |
3 | import * as React from 'react';
4 | import { OTPInput, OTPInputContext } from 'input-otp';
5 | import { Dot } from 'lucide-react';
6 |
7 | import { cn } from '@/lib/utils';
8 |
9 | const InputOTP = React.forwardRef<
10 | React.ElementRef,
11 | React.ComponentPropsWithoutRef
12 | >(({ className, containerClassName, ...props }, ref) => (
13 |
22 | ));
23 | InputOTP.displayName = 'InputOTP';
24 |
25 | const InputOTPGroup = React.forwardRef<
26 | React.ElementRef<'div'>,
27 | React.ComponentPropsWithoutRef<'div'>
28 | >(({ className, ...props }, ref) => (
29 |
30 | ));
31 | InputOTPGroup.displayName = 'InputOTPGroup';
32 |
33 | const InputOTPSlot = React.forwardRef<
34 | React.ElementRef<'div'>,
35 | React.ComponentPropsWithoutRef<'div'> & { index: number }
36 | >(({ index, className, ...props }, ref) => {
37 | const inputOTPContext = React.useContext(OTPInputContext);
38 | const { char, hasFakeCaret, isActive } = inputOTPContext.slots[index];
39 |
40 | return (
41 |