├── assets └── auto-extract-api.png ├── src ├── Schedulers │ ├── types.ts │ └── index.ts ├── Lists │ ├── types.ts │ └── index.ts ├── Projects │ ├── types.ts │ └── index.ts ├── Users │ ├── types.ts │ └── index.ts ├── Jobs │ ├── types.ts │ └── index.ts ├── Agents │ ├── types.ts │ └── index.ts ├── Workflows │ ├── types.ts │ └── index.ts ├── browser.ts ├── index.ts └── Api.ts ├── dist ├── Schedulers │ ├── types.d.ts │ └── index.d.ts ├── Lists │ ├── types.d.ts │ └── index.d.ts ├── Projects │ ├── types.d.ts │ └── index.d.ts ├── Jobs │ ├── index.d.ts │ └── types.d.ts ├── Users │ ├── index.d.ts │ └── types.d.ts ├── browser.d.ts ├── Workflows │ ├── index.d.ts │ └── types.d.ts ├── api.d.ts ├── Agents │ ├── types.d.ts │ └── index.d.ts ├── index.d.ts ├── index.modern.mjs ├── index.m.js ├── index.js └── index.umd.js ├── tsconfig.json ├── LICENSE ├── package.json ├── README.md ├── docs ├── schedulers.md ├── lists.md ├── users.md ├── jobs.md ├── listRows.md ├── projects.md ├── changeDetectionAgents.md ├── workflows.md ├── scrapingAgents.md └── crawlingAgents.md └── .gitignore /assets/auto-extract-api.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Agenty/scrapingai/HEAD/assets/auto-extract-api.png -------------------------------------------------------------------------------- /src/Schedulers/types.ts: -------------------------------------------------------------------------------- 1 | export type Scheduler = { 2 | type: string; 3 | expression: string; 4 | frequency: Number; 5 | description: string; 6 | is_enabled: boolean; 7 | }; 8 | -------------------------------------------------------------------------------- /dist/Schedulers/types.d.ts: -------------------------------------------------------------------------------- 1 | export type Scheduler = { 2 | type: string; 3 | expression: string; 4 | frequency: Number; 5 | description: string; 6 | is_enabled: boolean; 7 | }; 8 | -------------------------------------------------------------------------------- /src/Lists/types.ts: -------------------------------------------------------------------------------- 1 | export type List = { 2 | list_id?: number; 3 | account_id?: number; 4 | user_id?: number; 5 | name?: string; 6 | description?: string; 7 | created_at?: Date; 8 | updated_at?: Date; 9 | }; 10 | -------------------------------------------------------------------------------- /src/Projects/types.ts: -------------------------------------------------------------------------------- 1 | export type Project = { 2 | project_id?: number; 3 | user_id?: number; 4 | account_id?: number; 5 | name: string; 6 | description?: string; 7 | created_at?: Date; 8 | updated_at?: Date; 9 | }; 10 | -------------------------------------------------------------------------------- /dist/Lists/types.d.ts: -------------------------------------------------------------------------------- 1 | export type List = { 2 | list_id?: number; 3 | account_id?: number; 4 | user_id?: number; 5 | name?: string; 6 | description?: string; 7 | created_at?: Date; 8 | updated_at?: Date; 9 | }; 10 | -------------------------------------------------------------------------------- /dist/Projects/types.d.ts: -------------------------------------------------------------------------------- 1 | export type Project = { 2 | project_id?: number; 3 | user_id?: number; 4 | account_id?: number; 5 | name: string; 6 | description?: string; 7 | created_at?: Date; 8 | updated_at?: Date; 9 | }; 10 | -------------------------------------------------------------------------------- /src/Users/types.ts: -------------------------------------------------------------------------------- 1 | export type User = { 2 | user_id?: number; 3 | account_id?: number; 4 | name?: string; 5 | email?: string; 6 | role?: string; 7 | ip_address?: string; 8 | is_email_verified?: boolean; 9 | is_email_subscribed?: boolean; 10 | guid?: string; 11 | last_login_at?: Date; 12 | status?: String; 13 | avatar?: string; 14 | created_at?: Date; 15 | updated_at?: Date; 16 | }; 17 | -------------------------------------------------------------------------------- /dist/Jobs/index.d.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Job } from "./types"; 3 | declare class Jobs { 4 | private api; 5 | constructor(api: Api); 6 | get(job_id: number): Promise; 7 | list(params?: any): Promise; 8 | start(agent_id: string): Promise; 9 | stop(job_id: number): Promise; 10 | result(job_id: number): Promise; 11 | } 12 | export default Jobs; 13 | -------------------------------------------------------------------------------- /dist/Users/index.d.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { User } from "./types"; 3 | declare class Users { 4 | private api; 5 | constructor(api: Api); 6 | get(user_id: number): Promise; 7 | list(): Promise; 8 | create(data: User): Promise; 9 | update(user_id: number, data: User): Promise; 10 | delete(user_id: number): Promise; 11 | } 12 | export default Users; 13 | -------------------------------------------------------------------------------- /dist/Users/types.d.ts: -------------------------------------------------------------------------------- 1 | export type User = { 2 | user_id?: number; 3 | account_id?: number; 4 | name?: string; 5 | email?: string; 6 | role?: string; 7 | ip_address?: string; 8 | is_email_verified?: boolean; 9 | is_email_subscribed?: boolean; 10 | guid?: string; 11 | last_login_at?: Date; 12 | status?: String; 13 | avatar?: string; 14 | created_at?: Date; 15 | updated_at?: Date; 16 | }; 17 | -------------------------------------------------------------------------------- /dist/browser.d.ts: -------------------------------------------------------------------------------- 1 | import Api from "./api"; 2 | declare class Browser { 3 | private api; 4 | constructor(api: Api); 5 | screenshot(url: string, options?: any): Promise; 6 | pdf(url: string, options?: any): Promise; 7 | content(url: string, options?: any): Promise; 8 | extract(url: string, options?: any): Promise; 9 | scrape(url: string, options?: any): Promise; 10 | } 11 | export default Browser; 12 | -------------------------------------------------------------------------------- /dist/Schedulers/index.d.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Scheduler } from "./types"; 3 | declare class Schedulers { 4 | private api; 5 | constructor(api: Api); 6 | get(agent_id: string): Promise; 7 | create(agent_id: string, data: Scheduler): Promise; 8 | isEnabled(agent_id: string, data: Scheduler): Promise; 9 | delete(agent_id: string): Promise; 10 | } 11 | export default Schedulers; 12 | -------------------------------------------------------------------------------- /dist/Workflows/index.d.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Workflow } from "./types"; 3 | declare class Workflows { 4 | private api; 5 | constructor(api: Api); 6 | get(workflow_id: string): Promise; 7 | list(): Promise; 8 | create(data: Workflow): Promise; 9 | update(workflow_id: string, data: Workflow): Promise; 10 | delete(workflow_id: string): Promise; 11 | isEnabled(workflow_id: string, data: any): Promise; 12 | } 13 | export default Workflows; 14 | -------------------------------------------------------------------------------- /src/Jobs/types.ts: -------------------------------------------------------------------------------- 1 | export type Job = { 2 | job_id?: number; 3 | account_id: number; 4 | agent_id: string; 5 | type: string; 6 | status: string; 7 | priority: number; 8 | pages_total?: number; 9 | pages_processed?: number; 10 | pages_succeeded?: number; 11 | pages_failed?: number; 12 | pages_credit?: number; 13 | created_at?: Date; 14 | started_at?: Date; 15 | completed_at?: Date; 16 | stopped_at?: Date; 17 | is_scheduled?: boolean; 18 | queue_time?: string; 19 | run_duration?: string; 20 | error?: string; 21 | }; 22 | -------------------------------------------------------------------------------- /dist/api.d.ts: -------------------------------------------------------------------------------- 1 | declare class Api { 2 | private baseUrl; 3 | private browserUrl; 4 | private apiKey; 5 | constructor(apiKey: string); 6 | get(url: string, params?: any): Promise; 7 | post(url: string, data: any, params?: any): Promise; 8 | put(url: string, data: any, params?: any): Promise; 9 | patch(url: string, data: any, params?: any): Promise; 10 | delete(url: string, params?: any): Promise; 11 | browser(url: string, data: any, params?: any): Promise; 12 | } 13 | export default Api; 14 | -------------------------------------------------------------------------------- /src/Agents/types.ts: -------------------------------------------------------------------------------- 1 | import Scheduler from "../Schedulers"; 2 | 3 | export type Agent = { 4 | agent_id?: string; 5 | account_id?: number; 6 | user_id?: number; 7 | project_id?: number; 8 | name: string; 9 | description?: string; 10 | type: string; 11 | tags?: string[]; 12 | version?: number; 13 | config: Record; 14 | is_public?: boolean; 15 | is_managed?: boolean; 16 | icon?: string; 17 | scheduler?: Scheduler; 18 | scripts?: string; 19 | input?: Record; 20 | created_at?: Date; 21 | updated_at?: Date; 22 | }; 23 | -------------------------------------------------------------------------------- /dist/Jobs/types.d.ts: -------------------------------------------------------------------------------- 1 | export type Job = { 2 | job_id?: number; 3 | account_id: number; 4 | agent_id: string; 5 | type: string; 6 | status: string; 7 | priority: number; 8 | pages_total?: number; 9 | pages_processed?: number; 10 | pages_succeeded?: number; 11 | pages_failed?: number; 12 | pages_credit?: number; 13 | created_at?: Date; 14 | started_at?: Date; 15 | completed_at?: Date; 16 | stopped_at?: Date; 17 | is_scheduled?: boolean; 18 | queue_time?: string; 19 | run_duration?: string; 20 | error?: string; 21 | }; 22 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2016", 4 | "module": "commonjs", 5 | "declaration": true, 6 | "removeComments": true, 7 | "allowSyntheticDefaultImports": true, 8 | "esModuleInterop": true, 9 | "forceConsistentCasingInFileNames": true, 10 | "sourceMap": false, 11 | "outDir": "./dist", 12 | "rootDir": "src", 13 | "baseUrl": "./", 14 | "incremental": true, 15 | "strict": true, 16 | "skipLibCheck": true 17 | }, 18 | "include": ["src/**/*"], 19 | "exclude": ["node_modules", "tests", "docs", "**/*spec.ts"] 20 | } 21 | -------------------------------------------------------------------------------- /dist/Agents/types.d.ts: -------------------------------------------------------------------------------- 1 | import Scheduler from "../Schedulers"; 2 | export type Agent = { 3 | agent_id?: string; 4 | account_id?: number; 5 | user_id?: number; 6 | project_id?: number; 7 | name: string; 8 | description?: string; 9 | type: string; 10 | tags?: string[]; 11 | version?: number; 12 | config: Record; 13 | is_public?: boolean; 14 | is_managed?: boolean; 15 | icon?: string; 16 | scheduler?: Scheduler; 17 | scripts?: string; 18 | input?: Record; 19 | created_at?: Date; 20 | updated_at?: Date; 21 | }; 22 | -------------------------------------------------------------------------------- /src/Workflows/types.ts: -------------------------------------------------------------------------------- 1 | export type Workflow = { 2 | workflow_id?: string; 3 | account_id?: number; 4 | user_id?: number; 5 | agents: WorkflowAgents; 6 | name: string; 7 | trigger?: WorkflowTrigger; 8 | actions?: Array; 9 | is_enabled?: boolean; 10 | created_at?: Date; 11 | updated_at?: Date; 12 | }; 13 | 14 | type WorkflowAgents = { 15 | all?: boolean; 16 | selected?: string[]; 17 | }; 18 | 19 | type WorkflowTrigger = { 20 | when: string[]; 21 | }; 22 | 23 | type WorkflowAction = { 24 | type: string; 25 | connection_id?: string; 26 | config?: Record; 27 | }; 28 | -------------------------------------------------------------------------------- /dist/Workflows/types.d.ts: -------------------------------------------------------------------------------- 1 | export type Workflow = { 2 | workflow_id?: string; 3 | account_id?: number; 4 | user_id?: number; 5 | agents: WorkflowAgents; 6 | name: string; 7 | trigger?: WorkflowTrigger; 8 | actions?: Array; 9 | is_enabled?: boolean; 10 | created_at?: Date; 11 | updated_at?: Date; 12 | }; 13 | type WorkflowAgents = { 14 | all?: boolean; 15 | selected?: string[]; 16 | }; 17 | type WorkflowTrigger = { 18 | when: string[]; 19 | }; 20 | type WorkflowAction = { 21 | type: string; 22 | connection_id?: string; 23 | config?: Record; 24 | }; 25 | export {}; 26 | -------------------------------------------------------------------------------- /dist/Projects/index.d.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Project } from "./types"; 3 | declare class Projects { 4 | private api; 5 | constructor(api: Api); 6 | get(project_id: number): Promise; 7 | list(params?: any): Promise; 8 | create(project: Project): Promise; 9 | update(project_id: number, project: Project): Promise; 10 | delete(project_id: number): Promise; 11 | } 12 | declare class ProjectAgents { 13 | private api; 14 | constructor(api: Api); 15 | create(project_id: number, agents: any): Promise; 16 | delete(project_id: number, agent_id: string): Promise; 17 | } 18 | export { Projects, ProjectAgents }; 19 | -------------------------------------------------------------------------------- /src/Schedulers/index.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Scheduler } from "./types"; 3 | 4 | class Schedulers { 5 | private api: Api; 6 | constructor(api: Api) { 7 | this.api = api; 8 | } 9 | 10 | async get(agent_id: string): Promise { 11 | return this.api.get(`/scheduler/${agent_id}`); 12 | } 13 | 14 | async create(agent_id: string, data: Scheduler): Promise { 15 | return this.api.post(`/scheduler/${agent_id}`, data); 16 | } 17 | 18 | async isEnabled(agent_id: string, data: Scheduler): Promise { 19 | return this.api.patch(`/scheduler/${agent_id}`, data); 20 | } 21 | 22 | async delete(agent_id: string): Promise { 23 | return this.api.delete(`/scheduler/${agent_id}`); 24 | } 25 | } 26 | 27 | export default Schedulers; 28 | -------------------------------------------------------------------------------- /src/Users/index.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { User } from "./types"; 3 | 4 | class Users { 5 | private api: Api; 6 | constructor(api: Api) { 7 | this.api = api; 8 | } 9 | 10 | async get(user_id: number): Promise { 11 | return this.api.get(`/users/${user_id}`); 12 | } 13 | 14 | async list(): Promise { 15 | return this.api.get(`/users`); 16 | } 17 | 18 | async create(data: User): Promise { 19 | return this.api.post(`/users`, data); 20 | } 21 | 22 | async update(user_id: number, data: User): Promise { 23 | return this.api.put(`/users/${user_id}`, data); 24 | } 25 | 26 | async delete(user_id: number): Promise { 27 | return this.api.delete(`/users?user_id=${user_id}`); 28 | } 29 | } 30 | 31 | export default Users; 32 | -------------------------------------------------------------------------------- /src/Jobs/index.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Job } from "./types"; 3 | 4 | class Jobs { 5 | private api: Api; 6 | 7 | constructor(api: Api) { 8 | this.api = api; 9 | } 10 | 11 | async get(job_id: number): Promise { 12 | return this.api.get(`/jobs/${job_id}`); 13 | } 14 | 15 | async list(params: any = undefined): Promise { 16 | return this.api.get(`/jobs`, params); 17 | } 18 | 19 | async start(agent_id: string): Promise { 20 | return this.api.post(`/jobs/start`, { agent_id }); 21 | } 22 | 23 | async stop(job_id: number): Promise { 24 | return this.api.get(`/jobs/${job_id}/stop`); 25 | } 26 | 27 | async result(job_id: number): Promise { 28 | return this.api.get(`/jobs/${job_id}/result`); 29 | } 30 | } 31 | 32 | export default Jobs; 33 | -------------------------------------------------------------------------------- /dist/Lists/index.d.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { List } from "./types"; 3 | declare class Lists { 4 | private api; 5 | constructor(api: Api); 6 | list(params?: any): Promise; 7 | get(list_id: number): Promise; 8 | create(data: List): Promise; 9 | update(list_id: number, data: List): Promise; 10 | delete(list_id: number): Promise; 11 | } 12 | declare class ListRows { 13 | private api; 14 | constructor(api: Api); 15 | list(list_id: number): Promise; 16 | get(list_id: number, row_id: string): Promise; 17 | create(list_id: number, payload: any): Promise; 18 | update(list_id: number, row_id: string, data: any): Promise; 19 | delete(list_id: number, row_id: string): Promise; 20 | clear(list_id: number): Promise; 21 | } 22 | export { Lists, ListRows }; 23 | -------------------------------------------------------------------------------- /dist/index.d.ts: -------------------------------------------------------------------------------- 1 | import { ChangeDetectionAgents, CrawlingAgents, ScrapingAgents } from "./Agents"; 2 | import Jobs from "./Jobs"; 3 | import { Lists, ListRows } from "./Lists"; 4 | import { Projects, ProjectAgents } from "./Projects"; 5 | import Schedulers from "./Schedulers"; 6 | import Users from "./Users"; 7 | import Workflows from "./Workflows"; 8 | import Browser from "./browser"; 9 | declare class Agenty { 10 | private api; 11 | jobs: Jobs; 12 | lists: Lists; 13 | listRows: ListRows; 14 | users: Users; 15 | workflows: Workflows; 16 | schedulers: Schedulers; 17 | scrapingAgents: ScrapingAgents; 18 | crawlingAgents: CrawlingAgents; 19 | changeDetectionAgents: ChangeDetectionAgents; 20 | projects: Projects; 21 | projectAgents: ProjectAgents; 22 | browser: Browser; 23 | constructor(apiKey: string); 24 | } 25 | export default Agenty; 26 | -------------------------------------------------------------------------------- /src/browser.ts: -------------------------------------------------------------------------------- 1 | import Api from "./api"; 2 | 3 | class Browser { 4 | private api: Api; 5 | 6 | constructor(api: Api) { 7 | this.api = api; 8 | } 9 | 10 | async screenshot(url: string, options: any = undefined): Promise { 11 | return this.api.browser("/screenshot", { url: url, ...options }); 12 | } 13 | 14 | async pdf(url: string, options: any = undefined): Promise { 15 | return this.api.browser("/pdf", { url: url, ...options }); 16 | } 17 | 18 | async content(url: string, options: any = undefined): Promise { 19 | return this.api.browser("/content", { url: url, ...options }); 20 | } 21 | 22 | async extract(url: string, options: any = undefined): Promise { 23 | return this.api.browser("/extract", { url: url, ...options }); 24 | } 25 | 26 | async scrape(url: string, options: any = undefined): Promise { 27 | return this.api.browser("/scrape", { url: url, ...options }); 28 | } 29 | } 30 | 31 | export default Browser; 32 | -------------------------------------------------------------------------------- /src/Workflows/index.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Workflow } from "./types"; 3 | 4 | class Workflows { 5 | private api: Api; 6 | constructor(api: Api) { 7 | this.api = api; 8 | } 9 | 10 | async get(workflow_id: string): Promise { 11 | return this.api.get(`/workflows/${workflow_id}`); 12 | } 13 | 14 | async list(): Promise { 15 | return this.api.get(`/workflows`); 16 | } 17 | 18 | async create(data: Workflow): Promise { 19 | return this.api.post(`/workflows`, data); 20 | } 21 | 22 | async update(workflow_id: string, data: Workflow): Promise { 23 | return this.api.put(`/workflows/${workflow_id}`, data); 24 | } 25 | 26 | async delete(workflow_id: string): Promise { 27 | return this.api.delete(`/workflows/${workflow_id}`); 28 | } 29 | 30 | async isEnabled(workflow_id: string, data: any): Promise { 31 | return this.api.patch(`/workflows/${workflow_id}`, data); 32 | } 33 | } 34 | 35 | export default Workflows; 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Agenty 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /dist/Agents/index.d.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Agent } from "./types"; 3 | declare class ScrapingAgents { 4 | private api; 5 | constructor(api: Api); 6 | get(agent_id: string): Promise; 7 | list(): Promise; 8 | create(data: Agent): Promise; 9 | update(agent_id: string, data: Agent): Promise; 10 | delete(agent_id: string): Promise; 11 | } 12 | declare class CrawlingAgents { 13 | private api; 14 | constructor(api: Api); 15 | get(agent_id: string): Promise; 16 | list(): Promise; 17 | create(data: Agent): Promise; 18 | update(agent_id: string, data: Agent): Promise; 19 | delete(agent_id: string): Promise; 20 | } 21 | declare class ChangeDetectionAgents { 22 | private api; 23 | constructor(api: Api); 24 | get(agent_id: string): Promise; 25 | list(): Promise; 26 | create(data: Agent): Promise; 27 | update(agent_id: string, data: Agent): Promise; 28 | delete(agent_id: string): Promise; 29 | } 30 | export { ScrapingAgents, CrawlingAgents, ChangeDetectionAgents }; 31 | -------------------------------------------------------------------------------- /src/Projects/index.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Project } from "./types"; 3 | 4 | class Projects { 5 | private api: Api; 6 | 7 | constructor(api: Api) { 8 | this.api = api; 9 | } 10 | 11 | async get(project_id: number): Promise { 12 | return this.api.get(`/projects/${project_id}`); 13 | } 14 | 15 | async list(params: any = undefined): Promise { 16 | return this.api.get(`/projects`, params); 17 | } 18 | 19 | async create(project: Project): Promise { 20 | return this.api.post(`/projects`, project); 21 | } 22 | 23 | async update(project_id: number, project: Project): Promise { 24 | return this.api.put(`/projects/${project_id}`, project); 25 | } 26 | 27 | async delete(project_id: number): Promise { 28 | return this.api.delete(`/projects?id=${project_id}`); 29 | } 30 | } 31 | 32 | class ProjectAgents { 33 | private api: Api; 34 | 35 | constructor(api: Api) { 36 | this.api = api; 37 | } 38 | 39 | async create(project_id: number, agents: any): Promise { 40 | return this.api.post(`/projects/${project_id}/add`, agents); 41 | } 42 | 43 | async delete(project_id: number, agent_id: string): Promise { 44 | return this.api.delete( 45 | `/projects/${project_id}/remove?agent_id=${agent_id}` 46 | ); 47 | } 48 | } 49 | 50 | export { Projects, ProjectAgents }; 51 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "scrapingai", 3 | "version": "1.0.1", 4 | "description": "Build web scraping agents using AI to auto-extract the data from websites", 5 | "main": "dist/index.js", 6 | "module": "dist/index.m.js", 7 | "unpkg": "dist/index.umd.js", 8 | "types": "dist/index.d.ts", 9 | "scripts": { 10 | "build": "rimraf ./dist && microbundle --tsconfig tsconfig.json --no-sourcemap", 11 | "dev": "ts-node-dev lib/index.ts", 12 | "test": "npx playwright test", 13 | "dev-test": "ts-node-dev lib/test.ts" 14 | }, 15 | "exports": { 16 | "require": "./dist/index.js", 17 | "default": "./dist/index.modern.js" 18 | }, 19 | "files": [ 20 | "dist" 21 | ], 22 | "repository": { 23 | "type": "git", 24 | "url": "git+https://github.com/Agenty/scrapingai.git" 25 | }, 26 | "keywords": [ 27 | "scraping", 28 | "crawling", 29 | "scraping api", 30 | "web scraping", 31 | "data scraping", 32 | "url scraping", 33 | "web crawling", 34 | "agenty", 35 | "webscraper" 36 | ], 37 | "author": "https://github.com/vickyRathee", 38 | "license": "MIT", 39 | "bugs": { 40 | "url": "https://github.com/Agenty/scrapingai/issues" 41 | }, 42 | "homepage": "https://agenty.com/", 43 | "dependencies": { 44 | "axios": "^1.5.1" 45 | }, 46 | "devDependencies": { 47 | "@playwright/test": "^1.38.1", 48 | "@types/node": "^20.8.3", 49 | "dotenv": "^16.3.1", 50 | "microbundle": "^0.15.1", 51 | "rimraf": "^5.0.5", 52 | "ts-node": "^10.9.1", 53 | "ts-node-dev": "^2.0.0", 54 | "typescript": "^5.2.2" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { 2 | ChangeDetectionAgents, 3 | CrawlingAgents, 4 | ScrapingAgents, 5 | } from "./Agents"; 6 | import Jobs from "./Jobs"; 7 | import { Lists, ListRows } from "./Lists"; 8 | import { Projects, ProjectAgents } from "./Projects"; 9 | import Schedulers from "./Schedulers"; 10 | import Users from "./Users"; 11 | import Workflows from "./Workflows"; 12 | import Api from "./api"; 13 | import Browser from "./browser"; 14 | 15 | class Agenty { 16 | private api: Api; 17 | public jobs: Jobs; 18 | public lists: Lists; 19 | public listRows: ListRows; 20 | public users: Users; 21 | public workflows: Workflows; 22 | public schedulers: Schedulers; 23 | public scrapingAgents: ScrapingAgents; 24 | public crawlingAgents: CrawlingAgents; 25 | public changeDetectionAgents: ChangeDetectionAgents; 26 | public projects: Projects; 27 | public projectAgents: ProjectAgents; 28 | public browser: Browser; 29 | 30 | constructor(apiKey: string) { 31 | this.api = new Api(apiKey); 32 | this.jobs = new Jobs(this.api); 33 | this.lists = new Lists(this.api); 34 | this.listRows = new ListRows(this.api); 35 | this.users = new Users(this.api); 36 | this.workflows = new Workflows(this.api); 37 | this.schedulers = new Schedulers(this.api); 38 | this.scrapingAgents = new ScrapingAgents(this.api); 39 | this.crawlingAgents = new CrawlingAgents(this.api); 40 | this.changeDetectionAgents = new ChangeDetectionAgents(this.api); 41 | this.projects = new Projects(this.api); 42 | this.projectAgents = new ProjectAgents(this.api); 43 | this.browser = new Browser(this.api); 44 | } 45 | } 46 | 47 | export default Agenty; 48 | -------------------------------------------------------------------------------- /src/Lists/index.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { List } from "./types"; 3 | 4 | class Lists { 5 | private api: Api; 6 | 7 | constructor(api: Api) { 8 | this.api = api; 9 | } 10 | 11 | async list(params: any = undefined): Promise { 12 | return this.api.get(`/lists`, params); 13 | } 14 | 15 | async get(list_id: number): Promise { 16 | return this.api.get(`/lists/${list_id}`); 17 | } 18 | 19 | async create(data: List): Promise { 20 | return this.api.post(`/lists`, data); 21 | } 22 | 23 | async update(list_id: number, data: List) { 24 | return this.api.put(`/lists/${list_id}`, data); 25 | } 26 | 27 | async delete(list_id: number): Promise { 28 | return this.api.delete(`/lists/${list_id}`); 29 | } 30 | } 31 | 32 | class ListRows { 33 | private api: Api; 34 | 35 | constructor(api: Api) { 36 | this.api = api; 37 | } 38 | 39 | async list(list_id: number): Promise { 40 | return this.api.get(`/lists/${list_id}/rows`); 41 | } 42 | 43 | async get(list_id: number, row_id: string): Promise { 44 | return this.api.get(`/lists/${list_id}/rows/${row_id}`); 45 | } 46 | 47 | async create(list_id: number, payload: any): Promise { 48 | return this.api.post(`/lists/${list_id}/rows`, payload); 49 | } 50 | 51 | async update(list_id: number, row_id: string, data: any): Promise { 52 | return this.api.put(`/lists/${list_id}/rows/${row_id}`, data); 53 | } 54 | 55 | async delete(list_id: number, row_id: string): Promise { 56 | return this.api.delete(`/lists/${list_id}/rows/${row_id}`); 57 | } 58 | 59 | async clear(list_id: number): Promise { 60 | return this.api.delete(`/lists/${list_id}/rows/clear`); 61 | } 62 | } 63 | 64 | export { Lists, ListRows }; 65 | -------------------------------------------------------------------------------- /src/Api.ts: -------------------------------------------------------------------------------- 1 | import axios from "axios"; 2 | 3 | class Api { 4 | private baseUrl: string = "https://api.agenty.com/v2"; 5 | private browserUrl: string = "https://browser.agenty.com/api"; 6 | private apiKey: string; 7 | 8 | constructor(apiKey: string) { 9 | if (!apiKey) { 10 | throw new Error( 11 | "API key is required. Get your api key from here - https://cloud.agenty.com/settings/apikeys" 12 | ); 13 | } 14 | this.apiKey = apiKey; 15 | } 16 | 17 | async get(url: string, params?: any): Promise { 18 | const response = await axios.get(`${this.baseUrl}${url}`, { 19 | params: { 20 | apiKey: this.apiKey, 21 | ...params, 22 | }, 23 | }); 24 | return response.data; 25 | } 26 | 27 | async post(url: string, data: any, params?: any): Promise { 28 | const response = await axios.post(`${this.baseUrl}${url}`, data, { 29 | params: { 30 | apiKey: this.apiKey, 31 | ...params, 32 | }, 33 | }); 34 | return response.data; 35 | } 36 | 37 | async put(url: string, data: any, params?: any): Promise { 38 | const response = await axios.put(`${this.baseUrl}${url}`, data, { 39 | params: { 40 | apiKey: this.apiKey, 41 | ...params, 42 | }, 43 | }); 44 | return response.data; 45 | } 46 | 47 | async patch(url: string, data: any, params?: any): Promise { 48 | const response = await axios.patch(`${this.baseUrl}${url}`, data, { 49 | params: { 50 | apiKey: this.apiKey, 51 | ...params, 52 | }, 53 | }); 54 | return response.data; 55 | } 56 | 57 | async delete(url: string, params?: any): Promise { 58 | const response = await axios.delete(`${this.baseUrl}${url}`, { 59 | params: { 60 | apiKey: this.apiKey, 61 | ...params, 62 | }, 63 | }); 64 | return response.data; 65 | } 66 | 67 | async browser(url: string, data: any, params?: any): Promise { 68 | const response = await axios.post(`${this.browserUrl}${url}`, data, { 69 | params: { 70 | apiKey: this.apiKey, 71 | ...params, 72 | }, 73 | }); 74 | return response.data; 75 | } 76 | } 77 | 78 | export default Api; 79 | -------------------------------------------------------------------------------- /src/Agents/index.ts: -------------------------------------------------------------------------------- 1 | import Api from "../api"; 2 | import { Agent } from "./types"; 3 | 4 | class ScrapingAgents { 5 | private api: Api; 6 | constructor(api: Api) { 7 | this.api = api; 8 | } 9 | 10 | async get(agent_id: string): Promise { 11 | return this.api.get(`/agents/${agent_id}`); 12 | } 13 | 14 | async list(): Promise { 15 | return this.api.get(`/agents`); 16 | } 17 | 18 | async create(data: Agent): Promise { 19 | return this.api.post(`/agents`, data); 20 | } 21 | 22 | async update(agent_id: string, data: Agent): Promise { 23 | return this.api.put(`/agents/${agent_id}`, data); 24 | } 25 | 26 | async delete(agent_id: string): Promise { 27 | return this.api.delete(`/agents/${agent_id}`); 28 | } 29 | } 30 | 31 | class CrawlingAgents { 32 | private api: Api; 33 | constructor(api: Api) { 34 | this.api = api; 35 | } 36 | 37 | async get(agent_id: string): Promise { 38 | return this.api.get(`/agents/${agent_id}`); 39 | } 40 | 41 | async list(): Promise { 42 | return this.api.get(`/agents`); 43 | } 44 | 45 | async create(data: Agent): Promise { 46 | return this.api.post(`/agents`, data); 47 | } 48 | 49 | async update(agent_id: string, data: Agent): Promise { 50 | return this.api.put(`/agents/${agent_id}`, data); 51 | } 52 | 53 | async delete(agent_id: string): Promise { 54 | return this.api.delete(`/agents/${agent_id}`); 55 | } 56 | } 57 | 58 | class ChangeDetectionAgents { 59 | private api: Api; 60 | constructor(api: Api) { 61 | this.api = api; 62 | } 63 | 64 | async get(agent_id: string): Promise { 65 | return this.api.get(`/agents/${agent_id}`); 66 | } 67 | 68 | async list(): Promise { 69 | return this.api.get(`/agents`); 70 | } 71 | 72 | async create(data: Agent): Promise { 73 | return this.api.post(`/agents`, data); 74 | } 75 | 76 | async update(agent_id: string, data: Agent): Promise { 77 | return this.api.put(`/agents/${agent_id}`, data); 78 | } 79 | 80 | async delete(agent_id: string): Promise { 81 | return this.api.delete(`/agents/${agent_id}`); 82 | } 83 | } 84 | 85 | export { ScrapingAgents, CrawlingAgents, ChangeDetectionAgents }; 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # scarapingai 2 | 3 | [![version](https://img.shields.io/npm/v/scrapingai.svg)](https://www.npmjs.com/package/scrapingai) 4 | [![license](https://img.shields.io/npm/l/scrapingai.svg)](https://www.npmjs.com/package/scrapingai) 5 | 6 | > Extract data from websites automatically with AI or build [web scraping agents](https://agenty.com/products/scraping-agent) for bulk URL scraping. 7 | 8 | ![Auto extract website data with AI](/assets/auto-extract-api.png) 9 | 10 | ## Installation 11 | 12 | Install it via npm: 13 | 14 | ``` 15 | npm i scarapingai 16 | ``` 17 | 18 | ## Highlights 19 | - Built-in residential proxies and captcha handling 20 | - Smart ad blocker, popup blocker for better performance 21 | - Accept cookie consent automatically to close cookie banners 22 | - Compatible with Puppeteer, Playwright for browser automation and testing. 23 | - Background jobs for bulk URL scraping with automatic retry & error handling. 24 | 25 | ## Usage 26 | Get your [api key from here](https://cloud.agenty.com/settings/apikeys) 27 | 28 | ``` 29 | const agenty = new Agenty(API_KEY); 30 | const data = await agenty.browser.extract("https://example.com"); 31 | console.log(data); 32 | ``` 33 | 34 | ### Extract 35 | To auto-extract product, jobs listing, SEO meta data, schema JSON etc from given URL 36 | 37 | ``` 38 | const data = await agenty.browser.extract("https://example.com"); 39 | console.log(data); 40 | ``` 41 | 42 | ### Scrape 43 | To extract data from given CSS selector or custom jQuery function 44 | 45 | ``` 46 | const data = await agenty.browser.scrape("https://example.com"); 47 | console.log(data); 48 | ``` 49 | 50 | ### Screenshot 51 | To [capture a screenshot](https://agenty.com/tools/webpage-to-screenshot) for given URL 52 | 53 | 54 | ``` 55 | const data = await agenty.browser.screenshot("https://example.com"); 56 | console.log(data); 57 | ``` 58 | 59 | ### PDF 60 | To [convert webpage into PDF](https://agenty.com/tools/webpage-to-pdf). 61 | 62 | 63 | ``` 64 | const data = await agenty.browser.pdf("https://example.com"); 65 | console.log(data); 66 | ``` 67 | 68 | ### Content 69 | To get HTML content from a URL. 70 | 71 | ``` 72 | const data = await agenty.browser.content("https://example.com"); 73 | console.log(data); 74 | ``` 75 | 76 | 77 | ## License 78 | 79 | **scrapingai** is a project by [Agenty](https://agenty.com), released under the [MIT](https://github.com/Agenty/scrapingai/blob/main/LICENSE) License. 80 | -------------------------------------------------------------------------------- /docs/schedulers.md: -------------------------------------------------------------------------------- 1 | ## Schedulers 2 | 3 | ### Create a Scheduler 4 | ```js 5 | agenty.schedulers.create(agent_id, { 6 | "type": "cron", 7 | "expression": "0 20 * * MON,WED,FRI", 8 | "frequency": 0, 9 | "is_enabled": true, 10 | "description": "At 08:00 PM, only on Monday, Wednesday, and Friday" 11 | }) 12 | ``` 13 | **Parameters:** 14 | 15 | | Name | Type | Description | 16 | | ----------- | ------ | ----------------------------------- | 17 | | agent_id* | string | To create a scheduler on an agent | 18 | | type* | enum | Must be one of them(cron, interval) | 19 | | expression* | string | Must be a valid CRON expression | 20 | 21 | 22 | [For complete reference click here](https://agenty.com/docs/api#tag/Scheduler/operation/SchedulerController_createAgentSchedule) 23 | 24 | **Response:** 25 | ```json 26 | { 27 | "next_scheduled_run_at": "2023-10-13T20:00:00.000Z", 28 | "scheduler": { 29 | "expression": "0 20 * * MON,WED,FRI", 30 | "type": "cron", 31 | "is_enabled": true, 32 | "description": "At 08:00 PM, only on Monday, Wednesday, and Friday" 33 | }, 34 | "last_scheduled_run_at": null 35 | } 36 | ``` 37 | 38 | ### Get scheduled agent 39 | ```js 40 | agenty.schedulers.get(agent_id) 41 | ``` 42 | **Parameter:** 43 | 44 | | Name | Type | Description | 45 | | --------- | ------ | -------------------------------- | 46 | | agent_id* | string | To get an agent schedule details | 47 | 48 | **Response:** 49 | ```json 50 | { 51 | "scheduler": { 52 | "type": "cron", 53 | "expression": "0 20 * * MON,WED,FRI", 54 | "is_enabled": true, 55 | "description": "At 08:00 PM, only on Monday, Wednesday, and Friday" 56 | }, 57 | "next_scheduled_run_at": "2023-10-13T20:00:00.000Z", 58 | "last_scheduled_run_at": null 59 | } 60 | ``` 61 | 62 | ### Enable or disable an agent schedule 63 | ```js 64 | agenty.schedulers.isEnabled(agent_id, { 65 | "is_enabled": true, 66 | }) 67 | ``` 68 | 69 | **Parameter:** 70 | 71 | | Name | Type | Description | 72 | | --------- | ------ | ----------------------------------------------- | 73 | | agent_id* | string | Pass the agent_id to enable/disable a scheduler | 74 | 75 | **Payload:** 76 | 77 | | Name | Type | Description | 78 | | ----------- | ------- | ----------------------------------- | 79 | | is_enabled* | boolean | Enable or disable an agent schedule | 80 | 81 | 82 | **Response:** 83 | ```json 84 | { 85 | "is_enabled": false 86 | } 87 | ``` 88 | 89 | ### Delete scheduler for an agent 90 | ```js 91 | agenty.schedulers.delete(agent_id) 92 | ``` 93 | 94 | **Parameter:** 95 | 96 | | Name | Type | Description | 97 | | --------- | ------ | --------------------------- | 98 | | agent_id* | string | To delete an agent schedule | 99 | 100 | **Response:** 101 | ```json 102 | { 103 | "message": "Schedule removed successfully" 104 | } 105 | -------------------------------------------------------------------------------- /docs/lists.md: -------------------------------------------------------------------------------- 1 | ## Lists 2 | 3 | ### Create a list 4 | ```js 5 | agenty.lists.create({ 6 | name: "List-01", 7 | description: "the description" 8 | }) 9 | ``` 10 | **Parameters:** 11 | 12 | | Name | Type | Description | 13 | | ----------- | ------ | ----------------------- | 14 | | name* | string | Name of the list | 15 | | description | string | Description of the list | 16 | 17 | [For complete reference click here](https://agenty.com/docs/api#tag/Lists/operation/ListController_createList) 18 | **Response:** 19 | ```json 20 | { 21 | "name": "List-01", 22 | "account_id": 001, 23 | "user_id": 002, 24 | "list_id": 1 25 | } 26 | ``` 27 | 28 | ### Get Lists 29 | ```js 30 | agenty.lists.list() 31 | ``` 32 | 33 | **Response:** 34 | ```json 35 | { 36 | "total": 8, 37 | "limit": 1000, 38 | "offset": 0, 39 | "returned": 8, 40 | "result": [ 41 | { 42 | "list_id": 53, 43 | "account_id": 001, 44 | "name": "List-1", 45 | "description": "This is my list one", 46 | "created_at": "2023-06-15T09:29:18.000Z", 47 | "updated_at": "2023-10-18T05:39:55.000Z" 48 | }, 49 | { 50 | "list_id": 54, 51 | "account_id": 001, 52 | "name": "List-2", 53 | "description": null, 54 | "created_at": "2023-06-15T09:32:15.000Z", 55 | "updated_at": null 56 | }, 57 | ... 58 | ] 59 | } 60 | ``` 61 | 62 | ### Get list 63 | ```js 64 | agenty.lists.get(list_id) 65 | ``` 66 | 67 | **Parameter:** 68 | 69 | | Name | Type | Description | 70 | | -------- | ------ | ------------------- | 71 | | list_id* | number | To get a list by Id | 72 | 73 | 74 | **Response:** 75 | ```json 76 | { 77 | "list_id": 1, 78 | "account_id": 001, 79 | "name": "list-01", 80 | "description": "Your description", 81 | "created_at": "2023-06-15T09:29:18.000Z", 82 | "updated_at": null, 83 | "data": { "total": 0, "limit": 1000, "offset": 0, "returned": 0, "result": [] } 84 | } 85 | ``` 86 | 87 | ### Update list 88 | ```js 89 | agenty.lists.update(list_id, { 90 | name: "List-01 Update" 91 | }) 92 | ``` 93 | 94 | **Parameter:** 95 | 96 | | Name | Type | Description | 97 | | ----------- | ------ | ---------------------------------------- | 98 | | list_id* | number | To update a list name, description by Id | 99 | | name* | string | Name of the list | 100 | | description | string | Description of the list | 101 | 102 | [For complete reference click here](https://agenty.com/docs/api#tag/Lists/operation/ListController_updateList) 103 | **Response:** 104 | ```json 105 | { 106 | "message": "List with id: 1 updated successfully" 107 | } 108 | ``` 109 | 110 | ### Delete list 111 | ```js 112 | agenty.lists.delete(list_id) 113 | ``` 114 | 115 | **Parameter:** 116 | 117 | | Name | Type | Description | 118 | | -------- | ------ | ---------------------- | 119 | | list_id* | number | To delete a list by Id | 120 | 121 | 122 | **Response:** 123 | ```json 124 | { 125 | "message": "List with id: 1 deleted successfully" 126 | } 127 | ``` 128 | -------------------------------------------------------------------------------- /docs/users.md: -------------------------------------------------------------------------------- 1 | ## Users 2 | 3 | ### Create User 4 | ```js 5 | agenty.users.create({ 6 | name: "Manan", 7 | email: "manan.jain@agenty.com", 8 | role: "Admin" 9 | }); 10 | ``` 11 | 12 | **Parameters:** 13 | 14 | | Name | Type | Description | 15 | | ------ | ------ | ------------------------------------------------------------- | 16 | | name* | string | Name of your user | 17 | | email* | string | E-mail of your user | 18 | | role* | enum | Select one of them(Owner, Admin, Manager, Accountant, Viewer) | 19 | 20 | [For complete reference click here](https://agenty.com/docs/api#tag/Users/operation/UserController_createUser) 21 | 22 | **Response:** 23 | ```json 24 | { 25 | "name": "Manan", 26 | "email": "manan.jain@agenty.com", 27 | "role": "Admin", 28 | "status": "active", 29 | "ip_address": "103.237.159.37", 30 | "account_id": 001, 31 | "guid": "ln865x657sx16o911044dawyu953f3qea0mr", 32 | "is_email_subscribed": true, 33 | "user_id": 003 34 | } 35 | ``` 36 | 37 | ### Get User 38 | ```js 39 | agenty.users.get(user_id); 40 | ``` 41 | 42 | **Parameter:** 43 | 44 | | Name | Type | Description | 45 | | -------- | ------ | ------------------- | 46 | | user_id* | number | To get a user by Id | 47 | 48 | **Response:** 49 | ```json 50 | { 51 | "user_id": 003, 52 | "account_id": 001, 53 | "name": "Manan Jain", 54 | "email": "manan.jain@agenty.com", 55 | "role": "Admin", 56 | "status": "active", 57 | "created_at": "2023-10-11T07:17:25.000Z", 58 | "is_email_verified": null, 59 | "is_email_subscribed": 1, 60 | "last_login_at": null, 61 | "updated_at": "2023-10-11T07:17:58.000Z" 62 | } 63 | ``` 64 | 65 | ### Get Users 66 | ```js 67 | agenty.users.list(); 68 | ``` 69 | 70 | **Response:** 71 | ```json 72 | { 73 | "total": 4, 74 | "limit": 1000, 75 | "offset": 0, 76 | "returned": 4, 77 | "result": [ 78 | { 79 | "user_id": 003, 80 | "account_id": 001, 81 | "name": "Manan Jain", 82 | "email": "manan.jain@agenty.com", 83 | "role": "Owner", 84 | "status": "active", 85 | "created_at": "2023-06-14T05:02:52.000Z", 86 | "is_email_verified": 1, 87 | "is_email_subscribed": 1, 88 | "last_login_at": null, 89 | "updated_at": "2023-06-21T05:56:13.000Z" 90 | }, 91 | ... 92 | ] 93 | } 94 | ``` 95 | 96 | ### Update User 97 | ```js 98 | agenty.users.update(user_id, { 99 | name: "John Doe", 100 | email: "manan.jain@agenty.com", 101 | role: "Admin", 102 | status: "Active" 103 | }); 104 | ``` 105 | 106 | **Parameter:** 107 | 108 | | Name | Type | Description | 109 | | -------- | ------ | ---------------------- | 110 | | user_id* | number | To update a user by Id | 111 | 112 | [For complete reference click here](https://agenty.com/docs/api#tag/Users/operation/UserController_updateUserById) 113 | **Response:** 114 | ```json 115 | { 116 | "message": "User with id: 003 updated succesfully" 117 | } 118 | ``` 119 | 120 | ### Delete User 121 | ```js 122 | agenty.users.delete(user_id); 123 | ``` 124 | 125 | **Parameter:** 126 | 127 | | Name | Type | Description | 128 | | -------- | ------ | -------------------- | 129 | | user_id* | number | To delete user by Id | 130 | 131 | **Response:** 132 | ```json 133 | { 134 | "message": "1 user(s) with id(s): 003 deleted successfully" 135 | } 136 | ``` -------------------------------------------------------------------------------- /docs/jobs.md: -------------------------------------------------------------------------------- 1 | ## Jobs 2 | 3 | ### Start an agent job 4 | ```js 5 | agenty.jobs.start({ 6 | "agent_id": agent_id 7 | }) 8 | ``` 9 | 10 | **Parameter:** 11 | 12 | | Name | Type | Description | 13 | | --------- | ------ | -------------- | 14 | | agent_id* | string | ID of an agent | 15 | 16 | 17 | **Response:** 18 | ```json 19 | { 20 | "account_id": 001, 21 | "agent_id": "7hy654rfgt", 22 | "type": "scraping", 23 | "status": "started", 24 | "priority": 3, 25 | "created_at": "2023-10-10T05:06:35.008Z", 26 | "job_id": 19558 27 | } 28 | ``` 29 | 30 | ### Stop an agent job 31 | ```js 32 | agenty.jobs.stop(job_id) 33 | ``` 34 | 35 | **Parameter:** 36 | 37 | | Name | Type | Description | 38 | | ------- | ------ | ------------------ | 39 | | job_id* | number | ID of an agent job | 40 | 41 | 42 | **Response:** 43 | ```json 44 | { 45 | "job_id": 19541, 46 | "account_id": 001, 47 | "agent_id": "7hy654rfgt", 48 | "type": "scraping", 49 | "status": "completed", 50 | "priority": 3, 51 | "pages_total": 1, 52 | "pages_processed": 4, 53 | "pages_succeeded": 4, 54 | "pages_failed": 0, 55 | "pages_credit": 4, 56 | "created_at": "2023-10-09T08:55:59.000Z", 57 | "started_at": "2023-10-09T08:56:00.000Z", 58 | "completed_at": "2023-10-09T08:56:10.000Z", 59 | "stopped_at": null, 60 | "is_scheduled": 0, 61 | "worker_ip": null, 62 | "queue_time": "00:00:01", 63 | "run_duration": "00:00:10", 64 | "error": null, 65 | "ping_at": null 66 | } 67 | ``` 68 | 69 | ### Get jobs 70 | ```js 71 | agenty.jobs.list() 72 | ``` 73 | 74 | **Response:** 75 | ```json 76 | { "total": 229, 77 | "limit": 1000, 78 | "offset": 0, 79 | "returned": 229, 80 | "result":[ 81 | { 82 | "job_id": 19418, 83 | "account_id": 001, 84 | "agent_id": "7hy654rfgt", 85 | "type": "scraping", 86 | "status": "completed", 87 | "priority": 2, 88 | "pages_total": 1, 89 | "pages_processed": 1, 90 | "pages_succeeded": 1, 91 | "pages_failed": 0, 92 | "pages_credit": 0, 93 | "created_at": "2023-09-20T09:44:54.000Z", 94 | "started_at": "2023-09-20T09:44:54.000Z", 95 | "completed_at": "2023-09-20T09:45:04.000Z", 96 | "stopped_at": null, 97 | "is_scheduled": 0, 98 | "queue_time": "00:00:00", 99 | "run_duration": "00:00:10", 100 | "error": null, 101 | "ping_at": null 102 | }, 103 | ... 104 | ] 105 | } 106 | ``` 107 | 108 | ### Get job 109 | ```js 110 | agenty.jobs.get(job_id) 111 | ``` 112 | **Parameter:** 113 | 114 | | Name | Type | Description | 115 | | ------- | ------ | ------------------ | 116 | | job_id* | number | ID of an agent job | 117 | 118 | **Response:** 119 | ```json 120 | { 121 | "job_id": 19541, 122 | "account_id": 001, 123 | "agent_id": "7hy654rfgt", 124 | "type": "scraping", 125 | "status": "completed", 126 | "priority": 3, 127 | "pages_total": 1, 128 | "pages_processed": 4, 129 | "pages_succeeded": 4, 130 | "pages_failed": 0, 131 | "pages_credit": 4, 132 | "created_at": "2023-10-09T08:55:59.000Z", 133 | "started_at": "2023-10-09T08:56:00.000Z", 134 | "completed_at": "2023-10-09T08:56:10.000Z", 135 | "stopped_at": null, 136 | "is_scheduled": 0, 137 | "worker_ip": null, 138 | "queue_time": "00:00:01", 139 | "run_duration": "00:00:10", 140 | "error": null, 141 | "ping_at": null 142 | } 143 | ``` 144 | 145 | ### Get job result 146 | ```js 147 | agenty.jobs.result(job_id) 148 | ``` 149 | **Parameter:** 150 | 151 | | Name | Type | Description | 152 | | ------- | ------ | ------------------ | 153 | | job_id* | number | ID of an agent job | 154 | 155 | **Response:** 156 | ```json 157 | { 158 | "total": 10, 159 | "limit": 1000, 160 | "offset": 0, 161 | "returned": 10, 162 | "result": [ 163 | { 164 | "url": "https://books.toscrape.com/", 165 | "title": "A Light in the ...", 166 | "price": "£51.77" 167 | }, 168 | { 169 | "url": "https://books.toscrape.com/", 170 | "title": "Tipping the Velvet", 171 | "price": "£53.74" 172 | }, 173 | ... 174 | ] 175 | } 176 | ``` -------------------------------------------------------------------------------- /docs/listRows.md: -------------------------------------------------------------------------------- 1 | ## ListRows 2 | 3 | ### Add a new row in list 4 | ```js 5 | agenty.listRows.create(list_id, [ 6 | { 7 | field1: "Title-1", 8 | field2: "This is a discription" 9 | }, 10 | { 11 | field1: "Title-2", 12 | field2: "This is a discription" 13 | }, 14 | ... 15 | ] 16 | ) 17 | ``` 18 | 19 | **Parameter:** 20 | 21 | | Name | Type | Description | 22 | | -------- | ------ | ----------- | 23 | | list_id* | number | list id | 24 | 25 | **Payload:** 26 | 27 | | Name | Type | Description | 28 | | ------ | ---- | ----------------------------------------- | 29 | | field1 | any | Field name must be same as existing field | 30 | | field2 | any | Field name must be same as existing field | 31 | 32 | [For complete reference click here](https://agenty.com/docs/api#tag/Lists/operation/ListRowsController_addRow) 33 | **Response:** 34 | ```json 35 | [ 36 | { 37 | "_list_id": 2, 38 | "_id": "65262c3233ed3446a2dbf068", 39 | "field1": "Title-1", 40 | "field2": "This is a discription", 41 | "__v": 0 42 | }, 43 | { 44 | "_list_id": 2, 45 | "_id": "65262c3233ed3446a2dbf069", 46 | "field1": "Title-2", 47 | "field2": "This is a discription", 48 | "__v": 0 49 | }, 50 | ... 51 | ] 52 | ``` 53 | 54 | ### Get list rows 55 | ```js 56 | agenty.listRows.list(list_id) 57 | ``` 58 | 59 | **Parameter:** 60 | 61 | | Name | Type | Description | 62 | | -------- | ------ | ----------- | 63 | | list_id* | number | list id | 64 | 65 | 66 | **Response:** 67 | ```json 68 | { 69 | "total": 10, 70 | "limit": 1000, 71 | "offset": 0, 72 | "returned": 10, 73 | "result": [ 74 | { 75 | "_id": "65262c3233ed3446a2dbf068", 76 | "field1": "Title-1", 77 | "field2": "This is a discription", 78 | }, 79 | { 80 | "_id": "65262c3233ed3446a2dbf068", 81 | "field1": "Title-1", 82 | "field2": "This is a discription", 83 | }, 84 | ... 85 | ] 86 | } 87 | ``` 88 | 89 | ### Get list row 90 | ```js 91 | agenty.listRows.get(list_id, row_id) 92 | ``` 93 | 94 | **Parameters:** 95 | 96 | | Name | Type | Description | 97 | | -------- | ------ | ----------- | 98 | | list_id* | number | list id | 99 | | row_id* | number | row id | 100 | 101 | 102 | **Response:** 103 | ```json 104 | { 105 | "_id": "65262c3233ed3446a2dbf068", 106 | "field1": "Title-1", 107 | "field2": "This is a discription", 108 | } 109 | ``` 110 | 111 | 112 | ### Update list row 113 | ```js 114 | agenty.listRows.update(list_id, row_id,{ 115 | field1: "Title-2 Update" 116 | }) 117 | ``` 118 | 119 | **Parameter:** 120 | 121 | | Name | Type | Description | 122 | | -------- | ------ | --------------------------------- | 123 | | list_id* | number | list id to update list row | 124 | | row_id* | string | To update a single list row by Id | 125 | 126 | 127 | **Payload:** 128 | 129 | | Name | Type | Description | 130 | | ------- | ---- | -------------------- | 131 | | field1* | any | Update the row value | 132 | 133 | [For complete reference click here](https://agenty.com/docs/api#tag/Lists/operation/ListRowsController_updateListRowById) 134 | **Response:** 135 | ```json 136 | { 137 | "_id": "65262c3233ed3446a2dbf068", 138 | "field1": "Title-2 update", 139 | "field2": "This is a discription" 140 | } 141 | ``` 142 | 143 | ### Delete list row by row_id 144 | ```js 145 | agenty.listRows.delete(2, "65262c3233ed3446a2dbf068") 146 | ``` 147 | 148 | **Parameters:** 149 | 150 | | Name | Type | Description | 151 | | -------- | ------ | ----------------------------------- | 152 | | list_id* | number | list id to delete list row | 153 | | row_id* | string | To delete the single list row by Id | 154 | 155 | 156 | **Response:** 157 | ```json 158 | { 159 | "message": "Row with id: 65262c3233ed3446a2dbf068 deleted successfully" 160 | } 161 | ``` 162 | 163 | ### Clear rows 164 | ```js 165 | agenty.listRows.clear(list_id) 166 | ``` 167 | 168 | **Parameter:** 169 | 170 | | Name | Type | Description | 171 | | -------- | ------ | --------------------------------- | 172 | | list_id* | number | To clear the list data completely | 173 | 174 | **Response:** 175 | ```json 176 | { 177 | "message": "All rows deleted successfully" 178 | } 179 | ``` -------------------------------------------------------------------------------- /docs/projects.md: -------------------------------------------------------------------------------- 1 | ## Projects 2 | 3 | ### Create a project 4 | ```js 5 | agenty.projects.create({ 6 | name: "Project-1", 7 | description: "Your description" 8 | }) 9 | ``` 10 | **Parameters:** 11 | 12 | | Name | Type | Description | 13 | | ----------- | ------ | -------------------------- | 14 | | name* | string | Name of the project | 15 | | description | string | Description of the project | 16 | 17 | [For complete reference click here](https://agenty.com/docs/api#tag/Projects/operation/ProjectsController_createProject) 18 | 19 | **Response:** 20 | ```json 21 | { 22 | "name": "Project-1", 23 | "description": "Your description", 24 | "account_id": 001, 25 | "user_id": 002, 26 | "project_id": 1 27 | } 28 | ``` 29 | 30 | ### Get Projects 31 | ```js 32 | agenty.projects.list() 33 | ``` 34 | 35 | **Response:** 36 | ```json 37 | { 38 | "total": 3, 39 | "limit": 1000, 40 | "offset": 0, 41 | "returned": 3, 42 | "result": [ 43 | { 44 | "project_id": 1, 45 | "account_id": 001, 46 | "name": "E-commerce Project", 47 | "description": "All the e-commerce agents are here.", 48 | "created_at": "2023-10-20T09:34:56.000Z", 49 | "updated_at": "2023-10-30T11:04:11.000Z" 50 | }, 51 | { 52 | "project_id": 2, 53 | "account_id": 001, 54 | "name": "Germany Project", 55 | "description": "Agents in this project belongs to germany client", 56 | "created_at": "2023-10-20T10:59:30.000Z", 57 | "updated_at": "2023-10-30T05:37:38.000Z" 58 | }, 59 | ... 60 | ] 61 | } 62 | ``` 63 | 64 | ### Get project 65 | ```js 66 | agenty.projects.get(project_id) 67 | ``` 68 | 69 | **Parameter:** 70 | 71 | | Name | Type | Description | 72 | | ----------- | ------ | ---------------------- | 73 | | project_id* | number | To get a project by Id | 74 | 75 | 76 | **Response:** 77 | ```json 78 | { 79 | "project_id": 1, 80 | "account_id": 001, 81 | "user_id": 002, 82 | "name": "E-commerce Project", 83 | "description": "All the e-commerce agents are here.", 84 | "agents": [ 85 | { 86 | "project_id": 1, 87 | "agent_id": "0a7n0n5123", 88 | "name": "Crawling Agent Example", 89 | "description": "", 90 | "type": "crawling" 91 | }, 92 | ... 93 | ] 94 | } 95 | ``` 96 | 97 | ### Update project 98 | ```js 99 | agenty.projects.update(project_id, { 100 | name: "Project-1", 101 | description: "Let's change the description of this project" 102 | }) 103 | ``` 104 | 105 | **Parameter:** 106 | 107 | | Name | Type | Description | 108 | | ----------- | ------ | -------------------------- | 109 | | project_id* | number | To update a project by Id | 110 | | name* | string | Name of the project | 111 | | description | string | Description of the project | 112 | 113 | [For complete reference click here](https://agenty.com/docs/api#tag/Projects/operation/ProjectsController_updateProject) 114 | 115 | **Response:** 116 | ```json 117 | { 118 | "name": "Project-1", 119 | "description":"Let's change the description of this project", 120 | "project_id": 1, 121 | "account_id": 001 122 | } 123 | ``` 124 | 125 | ### Delete project 126 | ```js 127 | agenty.projects.delete(project_id) 128 | ``` 129 | 130 | **Parameter:** 131 | 132 | | Name | Type | Description | 133 | | ----------- | ------ | ------------------------- | 134 | | project_id* | number | To delete a project by Id | 135 | 136 | 137 | **Response:** 138 | ```json 139 | { 140 | "message": "Project with id: 1 deleted successfully" 141 | } 142 | ``` 143 | 144 | ### Add agent(s) in project 145 | ```js 146 | agenty.projectAgents.create(project_id, { 147 | "agents":[ agent_id, agent_id, ... ] 148 | }) 149 | ``` 150 | **Payload:** 151 | 152 | | Name | Type | Description | 153 | | ----------- | -------- | ----------------------------------- | 154 | | project_id* | number | project_id | 155 | | agents* | string[] | Pass the agent_id to add in project | 156 | 157 | 158 | **Response:** 159 | ```json 160 | { 161 | "message": "Agent(s) with id: 4rft6yhgt6,4rfder56hy added in Project: 'Project-1' successfully" 162 | } 163 | ``` 164 | 165 | ### Remove agent(s) from project 166 | ```js 167 | agenty.projectAgents.delete(project_id, agent_id) 168 | ``` 169 | **Parameters:** 170 | 171 | | Name | Type | Description | 172 | | ----------- | ------ | ------------------------------------------------- | 173 | | project_id* | number | project_id | 174 | | agent_id* | string | Pass the agent_id to remove agent(s) from project | 175 | 176 | **Response:** 177 | ```json 178 | { 179 | "message": "Agent(s) with id: 4rfder56hy removed successfully" 180 | } 181 | ``` -------------------------------------------------------------------------------- /dist/index.modern.mjs: -------------------------------------------------------------------------------- 1 | import t from"axios";class s{constructor(t){this.api=void 0,this.api=t}async get(t){return this.api.get(`/agents/${t}`)}async list(){return this.api.get("/agents")}async create(t){return this.api.post("/agents",t)}async update(t,s){return this.api.put(`/agents/${t}`,s)}async delete(t){return this.api.delete(`/agents/${t}`)}}class e{constructor(t){this.api=void 0,this.api=t}async get(t){return this.api.get(`/agents/${t}`)}async list(){return this.api.get("/agents")}async create(t){return this.api.post("/agents",t)}async update(t,s){return this.api.put(`/agents/${t}`,s)}async delete(t){return this.api.delete(`/agents/${t}`)}}class i{constructor(t){this.api=void 0,this.api=t}async get(t){return this.api.get(`/agents/${t}`)}async list(){return this.api.get("/agents")}async create(t){return this.api.post("/agents",t)}async update(t,s){return this.api.put(`/agents/${t}`,s)}async delete(t){return this.api.delete(`/agents/${t}`)}}class a{constructor(t){this.api=void 0,this.api=t}async get(t){return this.api.get(`/jobs/${t}`)}async list(t=void 0){return this.api.get("/jobs",t)}async start(t){return this.api.post("/jobs/start",{agent_id:t})}async stop(t){return this.api.get(`/jobs/${t}/stop`)}async result(t){return this.api.get(`/jobs/${t}/result`)}}class r{constructor(t){this.api=void 0,this.api=t}async list(t=void 0){return this.api.get("/lists",t)}async get(t){return this.api.get(`/lists/${t}`)}async create(t){return this.api.post("/lists",t)}async update(t,s){return this.api.put(`/lists/${t}`,s)}async delete(t){return this.api.delete(`/lists/${t}`)}}class n{constructor(t){this.api=void 0,this.api=t}async list(t){return this.api.get(`/lists/${t}/rows`)}async get(t,s){return this.api.get(`/lists/${t}/rows/${s}`)}async create(t,s){return this.api.post(`/lists/${t}/rows`,s)}async update(t,s,e){return this.api.put(`/lists/${t}/rows/${s}`,e)}async delete(t,s){return this.api.delete(`/lists/${t}/rows/${s}`)}async clear(t){return this.api.delete(`/lists/${t}/rows/clear`)}}class p{constructor(t){this.api=void 0,this.api=t}async get(t){return this.api.get(`/projects/${t}`)}async list(t=void 0){return this.api.get("/projects",t)}async create(t){return this.api.post("/projects",t)}async update(t,s){return this.api.put(`/projects/${t}`,s)}async delete(t){return this.api.delete(`/projects?id=${t}`)}}class c{constructor(t){this.api=void 0,this.api=t}async create(t,s){return this.api.post(`/projects/${t}/add`,s)}async delete(t,s){return this.api.delete(`/projects/${t}/remove?agent_id=${s}`)}}class h{constructor(t){this.api=void 0,this.api=t}async get(t){return this.api.get(`/scheduler/${t}`)}async create(t,s){return this.api.post(`/scheduler/${t}`,s)}async isEnabled(t,s){return this.api.patch(`/scheduler/${t}`,s)}async delete(t){return this.api.delete(`/scheduler/${t}`)}}class o{constructor(t){this.api=void 0,this.api=t}async get(t){return this.api.get(`/users/${t}`)}async list(){return this.api.get("/users")}async create(t){return this.api.post("/users",t)}async update(t,s){return this.api.put(`/users/${t}`,s)}async delete(t){return this.api.delete(`/users?user_id=${t}`)}}class u{constructor(t){this.api=void 0,this.api=t}async get(t){return this.api.get(`/workflows/${t}`)}async list(){return this.api.get("/workflows")}async create(t){return this.api.post("/workflows",t)}async update(t,s){return this.api.put(`/workflows/${t}`,s)}async delete(t){return this.api.delete(`/workflows/${t}`)}async isEnabled(t,s){return this.api.patch(`/workflows/${t}`,s)}}function l(){return l=Object.assign?Object.assign.bind():function(t){for(var s=1;s