├── .editorconfig ├── .eslintrc.js ├── .eslintrc.prepublish.js ├── .github ├── assets │ ├── banner.webp │ ├── example.svg │ └── operations.jpg ├── dependabot.yml └── workflows │ ├── auto-version.yml │ ├── ci.yml │ ├── release.yml │ └── security.yml ├── .gitignore ├── .npmignore ├── .prettierignore ├── .prettierrc.js ├── .vscode └── extensions.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.md ├── README.md ├── README_TEMPLATE.md ├── credentials ├── Scrappey.svg └── ScrappeyApi.credentials.ts ├── docs └── n8n-scanner.md ├── gulpfile.js ├── nodes └── Scrappey │ ├── GenericFunctions.ts │ ├── Scrappey.node.json │ ├── Scrappey.node.ts │ ├── Scrappey.svg │ ├── execute.ts │ ├── fields.ts │ ├── methods.ts │ ├── operators.ts │ ├── requestBodyBuilder.ts │ ├── types.ts │ └── utils.ts ├── package.json ├── pnpm-lock.yaml ├── scripts ├── check-versions.js ├── check-yaml.js ├── deploy.js ├── n8n-scanner.js └── update-node-json.js └── tsconfig.json /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = tab 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [package.json] 12 | indent_style = space 13 | indent_size = 2 14 | 15 | [*.md] 16 | trim_trailing_whitespace = false 17 | 18 | [*.yml] 19 | indent_style = space 20 | indent_size = 2 21 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @type {import('@types/eslint').ESLint.ConfigData} 3 | */ 4 | module.exports = { 5 | root: true, 6 | 7 | env: { 8 | browser: true, 9 | es6: true, 10 | node: true, 11 | }, 12 | 13 | parser: '@typescript-eslint/parser', 14 | 15 | parserOptions: { 16 | project: ['./tsconfig.json'], 17 | sourceType: 'module', 18 | extraFileExtensions: ['.json'], 19 | }, 20 | 21 | ignorePatterns: ['.eslintrc.js', '**/*.js', '**/node_modules/**', '**/dist/**'], 22 | 23 | overrides: [ 24 | { 25 | files: ['package.json'], 26 | plugins: ['eslint-plugin-n8n-nodes-base'], 27 | extends: ['plugin:n8n-nodes-base/community'], 28 | rules: { 29 | 'n8n-nodes-base/community-package-json-name-still-default': 'off', 30 | }, 31 | }, 32 | { 33 | files: ['./credentials/**/*.ts'], 34 | plugins: ['eslint-plugin-n8n-nodes-base'], 35 | extends: ['plugin:n8n-nodes-base/credentials'], 36 | rules: { 37 | 'n8n-nodes-base/cred-class-field-documentation-url-missing': 'off', 38 | 'n8n-nodes-base/cred-class-field-documentation-url-miscased': 'off', 39 | }, 40 | }, 41 | { 42 | files: ['./nodes/**/*.ts'], 43 | plugins: ['eslint-plugin-n8n-nodes-base'], 44 | extends: ['plugin:n8n-nodes-base/nodes'], 45 | rules: { 46 | 'n8n-nodes-base/node-execute-block-missing-continue-on-fail': 'off', 47 | 'n8n-nodes-base/node-resource-description-filename-against-convention': 'off', 48 | 'n8n-nodes-base/node-param-fixed-collection-type-unsorted-items': 'off', 49 | }, 50 | }, 51 | ], 52 | }; 53 | -------------------------------------------------------------------------------- /.eslintrc.prepublish.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @type {import('@types/eslint').ESLint.ConfigData} 3 | */ 4 | module.exports = { 5 | extends: './.eslintrc.js', 6 | 7 | overrides: [ 8 | { 9 | files: ['package.json'], 10 | plugins: ['eslint-plugin-n8n-nodes-base'], 11 | rules: { 12 | 'n8n-nodes-base/community-package-json-name-still-default': 'error', 13 | }, 14 | }, 15 | ], 16 | }; 17 | -------------------------------------------------------------------------------- /.github/assets/banner.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Automations-Project/n8n-nodes-scrappey/5dd50257f778b1152f6a3cf73880673d0f3a32c6/.github/assets/banner.webp -------------------------------------------------------------------------------- /.github/assets/operations.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Automations-Project/n8n-nodes-scrappey/5dd50257f778b1152f6a3cf73880673d0f3a32c6/.github/assets/operations.jpg -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | # Enable version updates for npm 4 | - package-ecosystem: 'npm' 5 | directory: '/' 6 | schedule: 7 | interval: 'weekly' 8 | day: 'monday' 9 | time: '09:00' 10 | open-pull-requests-limit: 10 11 | commit-message: 12 | prefix: 'chore' 13 | include: 'scope' 14 | reviewers: 15 | - 'Nskha' 16 | assignees: 17 | - 'Nskha' 18 | labels: 19 | - 'dependencies' 20 | - 'automated' 21 | ignore: 22 | # Ignore major version updates for n8n (requires manual testing) 23 | - dependency-name: 'n8n' 24 | update-types: ['version-update:semver-major'] 25 | # Ignore patch updates for TypeScript (can be noisy) 26 | - dependency-name: 'typescript' 27 | update-types: ['version-update:semver-patch'] 28 | 29 | # Enable version updates for GitHub Actions 30 | - package-ecosystem: 'github-actions' 31 | directory: '/' 32 | schedule: 33 | interval: 'weekly' 34 | day: 'monday' 35 | time: '09:00' 36 | commit-message: 37 | prefix: 'ci' 38 | include: 'scope' 39 | labels: 40 | - 'ci/cd' 41 | - 'automated' 42 | -------------------------------------------------------------------------------- /.github/workflows/auto-version.yml: -------------------------------------------------------------------------------- 1 | name: Auto Version & Release 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | paths-ignore: 8 | - 'README.md' 9 | - 'docs/**' 10 | - '*.md' 11 | 12 | jobs: 13 | auto-version: 14 | name: Auto Version Bump 15 | runs-on: ubuntu-latest 16 | if: ${{ !contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, '[skip version]') }} 17 | permissions: 18 | contents: write 19 | packages: write 20 | pull-requests: write 21 | actions: write 22 | outputs: 23 | version_bumped: ${{ steps.check_changes.outputs.has_changes }} 24 | new_version: ${{ steps.new_version.outputs.version }} 25 | steps: 26 | - name: Checkout code 27 | uses: actions/checkout@v4 28 | with: 29 | fetch-depth: 0 30 | token: ${{ secrets.GITHUB_TOKEN }} 31 | 32 | - name: Install pnpm 33 | uses: pnpm/action-setup@v2 34 | with: 35 | version: '9.1.4' 36 | 37 | - name: Setup Node.js 38 | uses: actions/setup-node@v4 39 | with: 40 | node-version: '18.x' 41 | cache: 'pnpm' 42 | 43 | - name: Install dependencies 44 | run: | 45 | # Try frozen lockfile first, regenerate if needed for auto-versioning 46 | echo "🔍 Attempting to install with frozen lockfile..." 47 | if ! pnpm install --frozen-lockfile; then 48 | echo "❌ Lockfile is outdated. Regenerating lockfile..." 49 | echo "📄 Current lockfile status:" 50 | ls -la pnpm-lock.yaml || echo "No lockfile found" 51 | 52 | # Clean install to fix lockfile issues 53 | rm -f pnpm-lock.yaml 54 | pnpm install 55 | 56 | echo "✅ Lockfile regenerated successfully" 57 | echo "📊 Lockfile will be committed with version bump" 58 | else 59 | echo "✅ Dependencies installed with frozen lockfile" 60 | fi 61 | 62 | - name: Determine version bump type 63 | id: version_type 64 | run: | 65 | COMMIT_MSG="${{ github.event.head_commit.message }}" 66 | echo "Commit message: $COMMIT_MSG" 67 | 68 | if [[ $COMMIT_MSG =~ \[major\] ]] || [[ $COMMIT_MSG =~ BREAKING[[:space:]]CHANGE ]]; then 69 | echo "type=major" >> $GITHUB_OUTPUT 70 | echo "📈 Major version bump detected" 71 | elif [[ $COMMIT_MSG =~ \[minor\] ]] || [[ $COMMIT_MSG =~ ^feat ]] || [[ $COMMIT_MSG =~ ^feat: ]] || [[ $COMMIT_MSG =~ ^feat\( ]]; then 72 | echo "type=minor" >> $GITHUB_OUTPUT 73 | echo "📊 Minor version bump detected" 74 | elif [[ $COMMIT_MSG =~ ^fix ]] || [[ $COMMIT_MSG =~ ^fix: ]] || [[ $COMMIT_MSG =~ ^fix\( ]] || [[ $COMMIT_MSG =~ \[patch\] ]]; then 75 | echo "type=patch" >> $GITHUB_OUTPUT 76 | echo "🔧 Patch version bump detected" 77 | else 78 | echo "type=patch" >> $GITHUB_OUTPUT 79 | echo "🔧 Default patch version bump (no specific pattern matched)" 80 | fi 81 | 82 | - name: Get current version 83 | id: current_version 84 | run: | 85 | CURRENT=$(node -p "require('./package.json').version") 86 | echo "current=${CURRENT}" >> $GITHUB_OUTPUT 87 | echo "Current version: ${CURRENT}" 88 | 89 | - name: Bump version 90 | id: new_version 91 | run: | 92 | TYPE="${{ steps.version_type.outputs.type }}" 93 | NEW_VERSION=$(pnpm version --no-git-tag-version $TYPE --preid='' | sed 's/^v//') 94 | echo "version=${NEW_VERSION}" >> $GITHUB_OUTPUT 95 | echo "New version: ${NEW_VERSION}" 96 | 97 | - name: Update node.json version 98 | run: | 99 | if [ -f "scripts/update-node-json.js" ]; then 100 | node scripts/update-node-json.js 101 | echo "📝 Updated node.json version" 102 | else 103 | echo "⚠️ scripts/update-node-json.js not found, skipping node.json update" 104 | fi 105 | 106 | - name: Check for changes 107 | id: check_changes 108 | run: | 109 | # Check if any files have been modified 110 | if git diff --quiet; then 111 | echo "has_changes=false" >> $GITHUB_OUTPUT 112 | echo "No changes to commit" 113 | else 114 | echo "has_changes=true" >> $GITHUB_OUTPUT 115 | echo "Changes detected:" 116 | git diff --name-only 117 | fi 118 | 119 | - name: Commit version bump 120 | if: steps.check_changes.outputs.has_changes == 'true' 121 | run: | 122 | git config --local user.email "action@github.com" 123 | git config --local user.name "GitHub Action" 124 | 125 | # Add all relevant files that might have changed 126 | git add package.json 127 | 128 | # Add lockfile if it was updated 129 | if [[ -n $(git diff --name-only pnpm-lock.yaml) ]]; then 130 | echo "📦 Including updated lockfile in commit" 131 | git add pnpm-lock.yaml 132 | fi 133 | 134 | # Add node.json if it exists and was updated 135 | if [ -f "nodes/Scrappey/Scrappey.node.json" ]; then 136 | git add nodes/Scrappey/Scrappey.node.json 137 | fi 138 | 139 | # Commit with comprehensive message 140 | COMMIT_FILES=$(git diff --cached --name-only | tr '\n' ' ') 141 | git commit -m "chore: bump version to v${{ steps.new_version.outputs.version }} [skip ci] 142 | 143 | Updated files: ${COMMIT_FILES} 144 | 145 | - Version bumped from ${{ steps.current_version.outputs.current }} to ${{ steps.new_version.outputs.version }} 146 | - Bump type: ${{ steps.version_type.outputs.type }}" 147 | 148 | - name: Create and push tag 149 | if: steps.check_changes.outputs.has_changes == 'true' 150 | run: | 151 | git tag "v${{ steps.new_version.outputs.version }}" 152 | git push origin HEAD 153 | git push origin "v${{ steps.new_version.outputs.version }}" 154 | 155 | - name: Create changelog 156 | if: steps.check_changes.outputs.has_changes == 'true' 157 | id: changelog 158 | run: | 159 | NEW_VERSION="v${{ steps.new_version.outputs.version }}" 160 | PREV_TAG=$(git describe --tags --abbrev=0 HEAD~1 2>/dev/null || echo "") 161 | 162 | if [ -z "$PREV_TAG" ]; then 163 | CHANGELOG="🎉 Initial release" 164 | else 165 | CHANGELOG=$(git log ${PREV_TAG}..HEAD~1 --pretty=format:"* %s" --no-merges | head -10) 166 | if [ -z "$CHANGELOG" ]; then 167 | CHANGELOG="* Minor improvements and bug fixes" 168 | fi 169 | fi 170 | 171 | # Use EOF delimiter to handle multi-line output safely 172 | echo "changelog<> $GITHUB_OUTPUT 173 | echo "$CHANGELOG" >> $GITHUB_OUTPUT 174 | echo "EOF" >> $GITHUB_OUTPUT 175 | 176 | - name: Create GitHub Release 177 | if: steps.check_changes.outputs.has_changes == 'true' 178 | uses: actions/github-script@v7 179 | with: 180 | script: | 181 | const changelog = `${{ steps.changelog.outputs.changelog }}`; 182 | const version = '${{ steps.new_version.outputs.version }}'; 183 | 184 | const releaseBody = `## 🚀 Release v${version} 185 | 186 | ### Changes 187 | ${changelog} 188 | 189 | ### Installation 190 | \`\`\`bash 191 | npm install @nskha/n8n-nodes-scrappey@${version} 192 | \`\`\` 193 | 194 | Or in n8n: Settings → Community Nodes → \`@nskha/n8n-nodes-scrappey\``; 195 | 196 | await github.rest.repos.createRelease({ 197 | owner: context.repo.owner, 198 | repo: context.repo.repo, 199 | tag_name: `v${version}`, 200 | name: `v${version}`, 201 | body: releaseBody, 202 | draft: false, 203 | prerelease: false 204 | }); 205 | 206 | - name: Output summary 207 | if: steps.check_changes.outputs.has_changes == 'true' 208 | run: | 209 | echo "## 🚀 Auto Version Bump Summary" >> $GITHUB_STEP_SUMMARY 210 | echo "" >> $GITHUB_STEP_SUMMARY 211 | echo "- **Previous Version**: ${{ steps.current_version.outputs.current }}" >> $GITHUB_STEP_SUMMARY 212 | echo "- **New Version**: ${{ steps.new_version.outputs.version }}" >> $GITHUB_STEP_SUMMARY 213 | echo "- **Bump Type**: ${{ steps.version_type.outputs.type }}" >> $GITHUB_STEP_SUMMARY 214 | echo "- **Tag Created**: v${{ steps.new_version.outputs.version }}" >> $GITHUB_STEP_SUMMARY 215 | echo "" >> $GITHUB_STEP_SUMMARY 216 | echo "### 📋 Changes:" >> $GITHUB_STEP_SUMMARY 217 | echo "${{ steps.changelog.outputs.changelog }}" >> $GITHUB_STEP_SUMMARY 218 | echo "" >> $GITHUB_STEP_SUMMARY 219 | echo "🎯 **GitHub release created automatically**" >> $GITHUB_STEP_SUMMARY 220 | 221 | - name: No changes summary 222 | if: steps.check_changes.outputs.has_changes == 'false' 223 | run: | 224 | echo "## ℹ️ No Version Bump Required" >> $GITHUB_STEP_SUMMARY 225 | echo "" >> $GITHUB_STEP_SUMMARY 226 | echo "No changes detected that require a version bump." >> $GITHUB_STEP_SUMMARY 227 | echo "Current version remains: ${{ steps.current_version.outputs.current }}" >> $GITHUB_STEP_SUMMARY 228 | 229 | trigger-release: 230 | name: Trigger Release Workflow 231 | runs-on: ubuntu-latest 232 | needs: auto-version 233 | if: needs.auto-version.outputs.version_bumped == 'true' 234 | permissions: 235 | actions: write 236 | steps: 237 | - name: Wait for tag to be available 238 | run: sleep 15 239 | 240 | - name: Trigger release workflow 241 | uses: actions/github-script@v7 242 | with: 243 | script: | 244 | const version = '${{ needs.auto-version.outputs.new_version }}'; 245 | 246 | console.log(`🚀 Triggering release workflow for version ${version}`); 247 | 248 | try { 249 | await github.rest.actions.createWorkflowDispatch({ 250 | owner: context.repo.owner, 251 | repo: context.repo.repo, 252 | workflow_id: 'release.yml', 253 | ref: 'main', 254 | inputs: { 255 | version_type: 'patch', 256 | force_version: version 257 | } 258 | }); 259 | 260 | console.log(`✅ Successfully triggered release workflow for version ${version}`); 261 | } catch (error) { 262 | console.log(`❌ Failed to trigger release workflow: ${error.message}`); 263 | 264 | // Try to find if the release workflow is already running 265 | try { 266 | const runs = await github.rest.actions.listWorkflowRuns({ 267 | owner: context.repo.owner, 268 | repo: context.repo.repo, 269 | workflow_id: 'release.yml', 270 | status: 'in_progress', 271 | per_page: 5 272 | }); 273 | 274 | if (runs.data.workflow_runs.length > 0) { 275 | console.log('🔄 Release workflow is already running, skipping trigger'); 276 | } else { 277 | console.log('⚠️ No running release workflow found, but trigger failed'); 278 | } 279 | } catch (listError) { 280 | console.log('Could not check workflow status'); 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [main, develop] 6 | pull_request: 7 | branches: [main, develop] 8 | 9 | jobs: 10 | lint-and-format: 11 | name: Lint & Format Check 12 | runs-on: ubuntu-latest 13 | permissions: 14 | contents: write 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v4 18 | with: 19 | token: ${{ secrets.GITHUB_TOKEN }} 20 | 21 | - name: Install pnpm 22 | uses: pnpm/action-setup@v2 23 | with: 24 | version: '9.1.4' 25 | 26 | - name: Setup Node.js 27 | uses: actions/setup-node@v4 28 | with: 29 | node-version: '18.x' 30 | cache: 'pnpm' 31 | 32 | - name: Install dependencies with lockfile validation 33 | run: | 34 | # Try frozen lockfile first 35 | if ! pnpm install --frozen-lockfile; then 36 | echo "❌ Lockfile is outdated. Regenerating lockfile..." 37 | pnpm install 38 | echo "✅ Lockfile regenerated successfully" 39 | 40 | # Check if lockfile was updated 41 | if [[ -n $(git diff --name-only pnpm-lock.yaml) ]]; then 42 | echo "🔄 Lockfile was updated. Committing changes..." 43 | git config --local user.email "action@github.com" 44 | git config --local user.name "GitHub Action" 45 | git add pnpm-lock.yaml 46 | git commit -m "chore: update pnpm-lock.yaml [skip ci]" || exit 0 47 | git push || echo "Failed to push lockfile changes" 48 | fi 49 | else 50 | echo "✅ Dependencies installed with frozen lockfile" 51 | fi 52 | 53 | - name: Run ESLint 54 | run: pnpm exec eslint "**/*.{ts,js}" --ignore-path .gitignore 55 | 56 | - name: Auto-format files 57 | run: pnpm exec prettier --write . 58 | 59 | - name: Check if formatting changed files 60 | id: format_check 61 | run: | 62 | if [[ -n $(git diff --name-only) ]]; then 63 | echo "formatted_files=true" >> $GITHUB_OUTPUT 64 | echo "Files were auto-formatted:" 65 | git diff --name-only 66 | else 67 | echo "formatted_files=false" >> $GITHUB_OUTPUT 68 | echo "✅ All files already properly formatted" 69 | fi 70 | 71 | - name: Commit auto-formatted files 72 | if: steps.format_check.outputs.formatted_files == 'true' 73 | run: | 74 | git config --local user.email "action@github.com" 75 | git config --local user.name "GitHub Action" 76 | git add . 77 | git commit -m "style: auto-format files with prettier [skip ci]" || exit 0 78 | git push || echo "Failed to push formatting changes" 79 | 80 | - name: TypeScript type check 81 | run: pnpm exec tsc --noEmit 82 | 83 | build: 84 | name: Build & Test 85 | runs-on: ubuntu-latest 86 | needs: lint-and-format 87 | strategy: 88 | matrix: 89 | node-version: [18.x, 20.x] 90 | steps: 91 | - name: Checkout code 92 | uses: actions/checkout@v4 93 | 94 | - name: Install pnpm 95 | uses: pnpm/action-setup@v2 96 | with: 97 | version: '9.1.4' 98 | 99 | - name: Setup Node.js ${{ matrix.node-version }} 100 | uses: actions/setup-node@v4 101 | with: 102 | node-version: ${{ matrix.node-version }} 103 | cache: 'pnpm' 104 | 105 | - name: Install dependencies 106 | run: | 107 | # For build jobs, use --no-frozen-lockfile to be more flexible 108 | pnpm install --no-frozen-lockfile 109 | 110 | - name: Build project 111 | run: pnpm run build 112 | 113 | - name: Check build artifacts 114 | run: | 115 | ls -la dist/ 116 | test -f dist/nodes/Scrappey/Scrappey.node.js 117 | test -f dist/credentials/ScrappeyApi.credentials.js 118 | 119 | - name: Upload build artifacts 120 | if: matrix.node-version == '18.x' 121 | uses: actions/upload-artifact@v4 122 | with: 123 | name: build-artifacts 124 | path: dist/ 125 | retention-days: 7 126 | 127 | security: 128 | name: Security Scan 129 | runs-on: ubuntu-latest 130 | permissions: 131 | actions: read 132 | contents: read 133 | security-events: write 134 | steps: 135 | - name: Checkout code 136 | uses: actions/checkout@v4 137 | 138 | - name: Install pnpm 139 | uses: pnpm/action-setup@v2 140 | with: 141 | version: '9.1.4' 142 | 143 | - name: Setup Node.js 144 | uses: actions/setup-node@v4 145 | with: 146 | node-version: '18.x' 147 | cache: 'pnpm' 148 | 149 | - name: Install dependencies 150 | run: | 151 | # For security scans, use --no-frozen-lockfile to be more flexible 152 | pnpm install --no-frozen-lockfile 153 | 154 | - name: Run security audit 155 | run: pnpm audit --audit-level high --prod || echo "⚠️ Security audit found issues in dev dependencies (non-critical for production)" 156 | 157 | - name: Run CodeQL Analysis 158 | uses: github/codeql-action/init@v3 159 | with: 160 | languages: javascript 161 | 162 | - name: Perform CodeQL Analysis 163 | uses: github/codeql-action/analyze@v3 164 | 165 | package-validation: 166 | name: Package Validation 167 | runs-on: ubuntu-latest 168 | needs: build 169 | steps: 170 | - name: Checkout code 171 | uses: actions/checkout@v4 172 | 173 | - name: Install pnpm 174 | uses: pnpm/action-setup@v2 175 | with: 176 | version: '9.1.4' 177 | 178 | - name: Setup Node.js 179 | uses: actions/setup-node@v4 180 | with: 181 | node-version: '18.x' 182 | cache: 'pnpm' 183 | 184 | - name: Install dependencies 185 | run: | 186 | # For package validation, use --no-frozen-lockfile to be more flexible 187 | pnpm install --no-frozen-lockfile 188 | 189 | - name: Build project 190 | run: pnpm run build 191 | 192 | - name: Pack package 193 | run: pnpm pack 194 | 195 | - name: Validate package contents 196 | run: | 197 | tar -tf *.tgz | grep -E "(dist/|package.json)" || exit 1 198 | echo "Package contents validated successfully" 199 | 200 | - name: Test package installation 201 | run: | 202 | mkdir test-install 203 | cd test-install 204 | npm init -y 205 | npm install ../nskha-n8n-nodes-scrappey-*.tgz 206 | node -e "console.log(require('@nskha/n8n-nodes-scrappey/package.json').version)" 207 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release & Deploy 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | workflow_dispatch: 8 | inputs: 9 | version_type: 10 | description: 'Version bump type' 11 | required: false 12 | default: 'auto' 13 | type: choice 14 | options: 15 | - 'auto' 16 | - 'patch' 17 | - 'minor' 18 | - 'major' 19 | force_version: 20 | description: 'Force specific version (e.g., 0.3.5) - overrides version_type' 21 | required: false 22 | type: string 23 | 24 | env: 25 | NODE_VERSION: '18.x' 26 | PNPM_VERSION: '9.1.4' 27 | 28 | jobs: 29 | detect-and-bump-version: 30 | name: Detect and Bump Version 31 | runs-on: ubuntu-latest 32 | outputs: 33 | new_version: ${{ steps.determine_version.outputs.new_version }} 34 | should_release: ${{ steps.determine_version.outputs.should_release }} 35 | version_changed: ${{ steps.determine_version.outputs.version_changed }} 36 | permissions: 37 | contents: write 38 | steps: 39 | - name: Checkout code 40 | uses: actions/checkout@v4 41 | with: 42 | fetch-depth: 0 43 | token: ${{ secrets.GITHUB_TOKEN }} 44 | 45 | - name: Install pnpm 46 | uses: pnpm/action-setup@v2 47 | with: 48 | version: ${{ env.PNPM_VERSION }} 49 | 50 | - name: Setup Node.js 51 | uses: actions/setup-node@v4 52 | with: 53 | node-version: ${{ env.NODE_VERSION }} 54 | cache: 'pnpm' 55 | 56 | - name: Install dependencies 57 | run: | 58 | # Use no-frozen-lockfile for releases to handle potential lockfile mismatches 59 | pnpm install --no-frozen-lockfile 60 | 61 | - name: Get current package.json version 62 | id: package_version 63 | run: | 64 | PACKAGE_VERSION=$(node -p "require('./package.json').version") 65 | echo "version=${PACKAGE_VERSION}" >> $GITHUB_OUTPUT 66 | echo "📦 Current package.json version: ${PACKAGE_VERSION}" 67 | 68 | - name: Get latest published versions 69 | id: published_versions 70 | run: | 71 | # Get latest version from npm 72 | NPM_VERSION="" 73 | if npm view @nskha/n8n-nodes-scrappey version 2>/dev/null; then 74 | NPM_VERSION=$(npm view @nskha/n8n-nodes-scrappey version 2>/dev/null || echo "0.0.0") 75 | else 76 | NPM_VERSION="0.0.0" 77 | fi 78 | echo "npm_version=${NPM_VERSION}" >> $GITHUB_OUTPUT 79 | echo "📦 Latest npm version: ${NPM_VERSION}" 80 | 81 | # Get latest version from GitHub packages 82 | GITHUB_VERSION="" 83 | if npm view @automations-project/n8n-nodes-scrappey version --registry=https://npm.pkg.github.com 2>/dev/null; then 84 | GITHUB_VERSION=$(npm view @automations-project/n8n-nodes-scrappey version --registry=https://npm.pkg.github.com 2>/dev/null || echo "0.0.0") 85 | else 86 | GITHUB_VERSION="0.0.0" 87 | fi 88 | echo "github_version=${GITHUB_VERSION}" >> $GITHUB_OUTPUT 89 | echo "📦 Latest GitHub package version: ${GITHUB_VERSION}" 90 | 91 | # Get latest git tag version 92 | GIT_VERSION=$(git describe --tags --abbrev=0 2>/dev/null | sed 's/^v//' || echo "0.0.0") 93 | echo "git_version=${GIT_VERSION}" >> $GITHUB_OUTPUT 94 | echo "🏷️ Latest git tag version: ${GIT_VERSION}" 95 | 96 | - name: Determine new version 97 | id: determine_version 98 | run: | 99 | PACKAGE_VERSION="${{ steps.package_version.outputs.version }}" 100 | NPM_VERSION="${{ steps.published_versions.outputs.npm_version }}" 101 | GITHUB_VERSION="${{ steps.published_versions.outputs.github_version }}" 102 | GIT_VERSION="${{ steps.published_versions.outputs.git_version }}" 103 | 104 | echo "Current versions:" 105 | echo " Package.json: ${PACKAGE_VERSION}" 106 | echo " NPM: ${NPM_VERSION}" 107 | echo " GitHub: ${GITHUB_VERSION}" 108 | echo " Git: ${GIT_VERSION}" 109 | 110 | # Function to compare versions (returns 0 if v1 > v2, 1 if v1 <= v2) 111 | version_gt() { 112 | [ "$(printf '%s\n%s\n' "$2" "$1" | sort -V | head -n1)" != "$1" ] 113 | } 114 | 115 | # Find the highest PUBLISHED version (exclude git tags - they're just markers) 116 | HIGHEST_PUBLISHED="0.0.0" 117 | for ver in "$NPM_VERSION" "$GITHUB_VERSION"; do 118 | if [ "$ver" != "0.0.0" ] && version_gt "$ver" "$HIGHEST_PUBLISHED"; then 119 | HIGHEST_PUBLISHED="$ver" 120 | fi 121 | done 122 | 123 | echo "🔍 Highest published version: ${HIGHEST_PUBLISHED}" 124 | 125 | # Determine version bump type 126 | if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.force_version }}" ]; then 127 | # Manual dispatch with forced version 128 | NEW_VERSION="${{ github.event.inputs.force_version }}" 129 | echo "🎯 Using forced version: ${NEW_VERSION}" 130 | elif [ "${{ github.event_name }}" = "push" ]; then 131 | # Triggered by tag push 132 | NEW_VERSION=${GITHUB_REF#refs/tags/v} 133 | echo "🏷️ Using tag version: ${NEW_VERSION}" 134 | else 135 | # Auto-determine version bump 136 | BUMP_TYPE="${{ github.event.inputs.version_type }}" 137 | if [ "$BUMP_TYPE" = "auto" ]; then 138 | # Analyze recent commits to determine bump type 139 | COMMIT_MSG=$(git log -1 --pretty=format:"%s") 140 | if [[ $COMMIT_MSG =~ \[major\] ]] || [[ $COMMIT_MSG =~ BREAKING[[:space:]]CHANGE ]]; then 141 | BUMP_TYPE="major" 142 | elif [[ $COMMIT_MSG =~ \[minor\] ]] || [[ $COMMIT_MSG =~ ^feat ]]; then 143 | BUMP_TYPE="minor" 144 | else 145 | BUMP_TYPE="patch" 146 | fi 147 | fi 148 | 149 | # Calculate new version based on highest published version 150 | IFS='.' read -r major minor patch <<< "$HIGHEST_PUBLISHED" 151 | case $BUMP_TYPE in 152 | major) 153 | NEW_VERSION="$((major + 1)).0.0" 154 | ;; 155 | minor) 156 | NEW_VERSION="${major}.$((minor + 1)).0" 157 | ;; 158 | patch) 159 | NEW_VERSION="${major}.${minor}.$((patch + 1))" 160 | ;; 161 | esac 162 | echo "📈 Auto-bumped ${BUMP_TYPE} version: ${NEW_VERSION}" 163 | fi 164 | 165 | # Validate new version is higher than all published versions 166 | SHOULD_RELEASE="true" 167 | VERSION_CHANGED="true" 168 | 169 | # Check against published versions only (not git tags) 170 | for ver in "$NPM_VERSION" "$GITHUB_VERSION"; do 171 | if [ "$ver" != "0.0.0" ] && ! version_gt "$NEW_VERSION" "$ver"; then 172 | echo "❌ New version ${NEW_VERSION} is not higher than published version ${ver}" 173 | SHOULD_RELEASE="false" 174 | fi 175 | done 176 | 177 | # Special case: if this is a manual release and the version already exists, 178 | # but we want to republish (e.g., failed previous attempt), allow it 179 | if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "$SHOULD_RELEASE" = "false" ]; then 180 | echo "🔄 Manual release mode - checking if we should allow republishing..." 181 | # Allow republishing if the new version equals the highest published version 182 | # but is higher than at least one published source 183 | if [ "$NEW_VERSION" = "$HIGHEST_PUBLISHED" ]; then 184 | for ver in "$NPM_VERSION" "$GITHUB_VERSION"; do 185 | if [ "$ver" != "0.0.0" ] && version_gt "$NEW_VERSION" "$ver"; then 186 | echo "✅ Allowing republish: ${NEW_VERSION} > ${ver}" 187 | SHOULD_RELEASE="true" 188 | break 189 | fi 190 | done 191 | fi 192 | fi 193 | 194 | # Check if package.json needs updating 195 | if [ "$PACKAGE_VERSION" = "$NEW_VERSION" ]; then 196 | VERSION_CHANGED="false" 197 | echo "ℹ️ Package.json already has the target version" 198 | fi 199 | 200 | echo "new_version=${NEW_VERSION}" >> $GITHUB_OUTPUT 201 | echo "should_release=${SHOULD_RELEASE}" >> $GITHUB_OUTPUT 202 | echo "version_changed=${VERSION_CHANGED}" >> $GITHUB_OUTPUT 203 | 204 | echo "✅ Final decision:" 205 | echo " New version: ${NEW_VERSION}" 206 | echo " Should release: ${SHOULD_RELEASE}" 207 | echo " Version changed: ${VERSION_CHANGED}" 208 | 209 | - name: Update package.json and node.json versions 210 | if: steps.determine_version.outputs.version_changed == 'true' 211 | run: | 212 | NEW_VERSION="${{ steps.determine_version.outputs.new_version }}" 213 | 214 | # Update package.json 215 | node -e " 216 | const pkg = require('./package.json'); 217 | pkg.version = '${NEW_VERSION}'; 218 | require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n'); 219 | " 220 | echo "📝 Updated package.json to version ${NEW_VERSION}" 221 | 222 | # Update node.json if script exists 223 | if [ -f "scripts/update-node-json.js" ]; then 224 | node scripts/update-node-json.js 225 | echo "📝 Updated node.json version" 226 | fi 227 | 228 | - name: Commit and tag new version 229 | if: steps.determine_version.outputs.version_changed == 'true' && github.event_name == 'workflow_dispatch' 230 | run: | 231 | NEW_VERSION="${{ steps.determine_version.outputs.new_version }}" 232 | 233 | git config --local user.email "action@github.com" 234 | git config --local user.name "GitHub Action" 235 | 236 | git add package.json 237 | if [ -f "nodes/Scrappey/Scrappey.node.json" ]; then 238 | git add nodes/Scrappey/Scrappey.node.json 239 | fi 240 | 241 | git commit -m "chore: bump version to v${NEW_VERSION} [skip ci]" || echo "No changes to commit" 242 | 243 | # Create and push tag 244 | git tag "v${NEW_VERSION}" || echo "Tag already exists" 245 | git push origin HEAD || echo "Failed to push commits" 246 | git push origin "v${NEW_VERSION}" || echo "Failed to push tag" 247 | 248 | build-and-test: 249 | name: Build and Test for Release 250 | runs-on: ubuntu-latest 251 | needs: detect-and-bump-version 252 | if: needs.detect-and-bump-version.outputs.should_release == 'true' 253 | steps: 254 | - name: Checkout code 255 | uses: actions/checkout@v4 256 | with: 257 | ref: ${{ github.event_name == 'workflow_dispatch' && 'main' || github.ref }} 258 | 259 | - name: Install pnpm 260 | uses: pnpm/action-setup@v2 261 | with: 262 | version: ${{ env.PNPM_VERSION }} 263 | 264 | - name: Setup Node.js 265 | uses: actions/setup-node@v4 266 | with: 267 | node-version: ${{ env.NODE_VERSION }} 268 | cache: 'pnpm' 269 | 270 | - name: Install dependencies 271 | run: | 272 | # Use no-frozen-lockfile for releases to handle potential lockfile mismatches 273 | pnpm install --no-frozen-lockfile 274 | 275 | - name: Run linting 276 | run: pnpm exec eslint "**/*.{ts,js}" --ignore-path .gitignore 277 | 278 | - name: Auto-format files 279 | run: pnpm exec prettier --write . 280 | 281 | - name: Type check 282 | run: pnpm exec tsc --noEmit 283 | 284 | - name: Build project 285 | run: pnpm run build 286 | 287 | - name: Pack package 288 | run: pnpm pack 289 | 290 | - name: Upload release artifacts 291 | uses: actions/upload-artifact@v4 292 | with: 293 | name: release-package 294 | path: | 295 | *.tgz 296 | dist/ 297 | retention-days: 30 298 | 299 | publish-github: 300 | name: Publish to GitHub Packages 301 | runs-on: ubuntu-latest 302 | needs: [detect-and-bump-version, build-and-test] 303 | if: needs.detect-and-bump-version.outputs.should_release == 'true' 304 | permissions: 305 | contents: read 306 | packages: write 307 | steps: 308 | - name: Checkout code 309 | uses: actions/checkout@v4 310 | with: 311 | ref: ${{ github.event_name == 'workflow_dispatch' && 'main' || github.ref }} 312 | 313 | - name: Install pnpm 314 | uses: pnpm/action-setup@v2 315 | with: 316 | version: ${{ env.PNPM_VERSION }} 317 | 318 | - name: Setup Node.js 319 | uses: actions/setup-node@v4 320 | with: 321 | node-version: ${{ env.NODE_VERSION }} 322 | cache: 'pnpm' 323 | registry-url: 'https://npm.pkg.github.com' 324 | scope: '@automations-project' 325 | 326 | - name: Install dependencies 327 | run: | 328 | # Use no-frozen-lockfile for releases to handle potential lockfile mismatches 329 | pnpm install --no-frozen-lockfile 330 | 331 | - name: Update package name and version for GitHub Packages 332 | run: | 333 | # Create a backup of original package.json 334 | cp package.json package.json.backup 335 | # Update package name and version for GitHub Packages 336 | node -e " 337 | const pkg = require('./package.json'); 338 | pkg.name = '@automations-project/n8n-nodes-scrappey'; 339 | pkg.version = '${{ needs.detect-and-bump-version.outputs.new_version }}'; 340 | require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2)); 341 | " 342 | echo "📝 Updated package.json for GitHub Packages: @automations-project/n8n-nodes-scrappey@${{ needs.detect-and-bump-version.outputs.new_version }}" 343 | 344 | - name: Build project 345 | run: pnpm run build 346 | 347 | - name: Configure npm for GitHub Packages 348 | run: | 349 | echo "@automations-project:registry=https://npm.pkg.github.com" >> .npmrc 350 | echo "//npm.pkg.github.com/:_authToken=${NODE_AUTH_TOKEN}" >> .npmrc 351 | env: 352 | NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 353 | 354 | - name: Verify package configuration 355 | run: | 356 | PACKAGE_NAME=$(node -p "require('./package.json').name") 357 | PACKAGE_VERSION=$(node -p "require('./package.json').version") 358 | EXPECTED_VERSION="${{ needs.detect-and-bump-version.outputs.new_version }}" 359 | echo "Package name: ${PACKAGE_NAME}" 360 | echo "Package version: ${PACKAGE_VERSION}" 361 | echo "Expected version: ${EXPECTED_VERSION}" 362 | if [ "$PACKAGE_VERSION" != "$EXPECTED_VERSION" ]; then 363 | echo "❌ Version mismatch!" 364 | exit 1 365 | fi 366 | echo "✅ GitHub package configuration verified" 367 | 368 | - name: Publish to GitHub Packages 369 | run: pnpm publish --no-git-checks 370 | env: 371 | NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} 372 | 373 | - name: Restore original package name 374 | run: mv package.json.backup package.json 375 | 376 | publish-npm: 377 | name: Publish to NPM Registry 378 | runs-on: ubuntu-latest 379 | needs: [detect-and-bump-version, build-and-test] 380 | if: needs.detect-and-bump-version.outputs.should_release == 'true' 381 | steps: 382 | - name: Checkout code 383 | uses: actions/checkout@v4 384 | with: 385 | ref: ${{ github.event_name == 'workflow_dispatch' && 'main' || github.ref }} 386 | 387 | - name: Install pnpm 388 | uses: pnpm/action-setup@v2 389 | with: 390 | version: ${{ env.PNPM_VERSION }} 391 | 392 | - name: Setup Node.js 393 | uses: actions/setup-node@v4 394 | with: 395 | node-version: ${{ env.NODE_VERSION }} 396 | cache: 'pnpm' 397 | registry-url: 'https://registry.npmjs.org' 398 | 399 | - name: Install dependencies 400 | run: | 401 | # Use no-frozen-lockfile for releases to handle potential lockfile mismatches 402 | pnpm install --no-frozen-lockfile 403 | 404 | - name: Update version in package.json 405 | run: | 406 | node -e " 407 | const pkg = require('./package.json'); 408 | pkg.version = '${{ needs.detect-and-bump-version.outputs.new_version }}'; 409 | require('fs').writeFileSync('package.json', JSON.stringify(pkg, null, 2)); 410 | " 411 | echo "📝 Updated package.json version to ${{ needs.detect-and-bump-version.outputs.new_version }}" 412 | 413 | - name: Build project 414 | run: pnpm run build 415 | 416 | - name: Verify package version 417 | run: | 418 | PACKAGE_VERSION=$(node -p "require('./package.json').version") 419 | EXPECTED_VERSION="${{ needs.detect-and-bump-version.outputs.new_version }}" 420 | echo "Package version: ${PACKAGE_VERSION}" 421 | echo "Expected version: ${EXPECTED_VERSION}" 422 | if [ "$PACKAGE_VERSION" != "$EXPECTED_VERSION" ]; then 423 | echo "❌ Version mismatch! Package has ${PACKAGE_VERSION}, expected ${EXPECTED_VERSION}" 424 | exit 1 425 | fi 426 | echo "✅ Version verified: ${PACKAGE_VERSION}" 427 | 428 | - name: Publish to NPM 429 | run: pnpm publish --access public --no-git-checks 430 | env: 431 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 432 | 433 | - name: Run n8n Security Scanner 434 | run: | 435 | echo "🔍 Running n8n community package security scan..." 436 | # Wait a bit for npm to propagate the package 437 | sleep 30 438 | node scripts/n8n-scanner.js || echo "⚠️ Scanner check completed (may show warnings for new packages)" 439 | 440 | create-github-release: 441 | name: Create GitHub Release 442 | runs-on: ubuntu-latest 443 | needs: [detect-and-bump-version, publish-github, publish-npm] 444 | if: needs.detect-and-bump-version.outputs.should_release == 'true' && (needs.publish-github.result == 'success' || needs.publish-npm.result == 'success') 445 | permissions: 446 | contents: write 447 | steps: 448 | - name: Checkout code 449 | uses: actions/checkout@v4 450 | with: 451 | fetch-depth: 0 452 | 453 | - name: Generate changelog 454 | id: changelog 455 | run: | 456 | NEW_VERSION="v${{ needs.detect-and-bump-version.outputs.new_version }}" 457 | PREV_TAG=$(git describe --tags --abbrev=0 HEAD~1 2>/dev/null || echo "") 458 | 459 | if [ -z "$PREV_TAG" ]; then 460 | CHANGELOG="🎉 Initial release" 461 | else 462 | CHANGELOG=$(git log ${PREV_TAG}..HEAD --pretty=format:"* %s" --no-merges | head -10) 463 | if [ -z "$CHANGELOG" ]; then 464 | CHANGELOG="* Minor improvements and bug fixes" 465 | fi 466 | fi 467 | 468 | echo "changelog<> $GITHUB_OUTPUT 469 | echo "$CHANGELOG" >> $GITHUB_OUTPUT 470 | echo "EOF" >> $GITHUB_OUTPUT 471 | 472 | - name: Create or Update GitHub Release 473 | uses: actions/github-script@v7 474 | with: 475 | script: | 476 | const changelog = `${{ steps.changelog.outputs.changelog }}`; 477 | const version = '${{ needs.detect-and-bump-version.outputs.new_version }}'; 478 | const tagName = `v${version}`; 479 | 480 | const releaseBody = `## 🚀 Release v${version} 481 | 482 | ### Changes 483 | ${changelog} 484 | 485 | ### Installation 486 | \`\`\`bash 487 | npm install @nskha/n8n-nodes-scrappey@${version} 488 | \`\`\` 489 | 490 | Or in n8n: Settings → Community Nodes → \`@nskha/n8n-nodes-scrappey\``; 491 | 492 | try { 493 | // Check if release already exists 494 | console.log(`🔍 Checking if release ${tagName} already exists...`); 495 | 496 | let existingRelease; 497 | try { 498 | const response = await github.rest.repos.getReleaseByTag({ 499 | owner: context.repo.owner, 500 | repo: context.repo.repo, 501 | tag: tagName, 502 | }); 503 | existingRelease = response.data; 504 | console.log(`✅ Found existing release: ${existingRelease.html_url}`); 505 | } catch (error) { 506 | if (error.status === 404) { 507 | console.log(`📝 Release ${tagName} does not exist, will create new one`); 508 | existingRelease = null; 509 | } else { 510 | throw error; 511 | } 512 | } 513 | 514 | if (existingRelease) { 515 | // Update existing release 516 | console.log(`🔄 Updating existing release ${tagName}...`); 517 | const updateResponse = await github.rest.repos.updateRelease({ 518 | owner: context.repo.owner, 519 | repo: context.repo.repo, 520 | release_id: existingRelease.id, 521 | name: tagName, 522 | body: releaseBody, 523 | draft: false, 524 | prerelease: false 525 | }); 526 | console.log(`✅ Successfully updated release: ${updateResponse.data.html_url}`); 527 | } else { 528 | // Create new release 529 | console.log(`🆕 Creating new release ${tagName}...`); 530 | const createResponse = await github.rest.repos.createRelease({ 531 | owner: context.repo.owner, 532 | repo: context.repo.repo, 533 | tag_name: tagName, 534 | name: tagName, 535 | body: releaseBody, 536 | draft: false, 537 | prerelease: false 538 | }); 539 | console.log(`✅ Successfully created release: ${createResponse.data.html_url}`); 540 | } 541 | } catch (error) { 542 | console.error(`❌ Error managing release: ${error.message}`); 543 | throw error; 544 | } 545 | 546 | notify: 547 | name: Notify Success 548 | runs-on: ubuntu-latest 549 | needs: [detect-and-bump-version, publish-github, publish-npm, create-github-release] 550 | if: always() && needs.detect-and-bump-version.outputs.should_release == 'true' 551 | steps: 552 | - name: Success notification 553 | if: needs.publish-github.result == 'success' && needs.publish-npm.result == 'success' 554 | run: | 555 | echo "🎉 Successfully published v${{ needs.detect-and-bump-version.outputs.new_version }}" 556 | echo "📦 Package published to GitHub Packages" 557 | echo "📦 Package published to npm.js" 558 | if [ "${{ needs.create-github-release.result }}" == "success" ]; then 559 | echo "🏷️ GitHub release created/updated" 560 | else 561 | echo "⚠️ GitHub release step was skipped or failed" 562 | fi 563 | 564 | - name: Partial success notification 565 | if: (needs.publish-github.result == 'success' && needs.publish-npm.result != 'success') || (needs.publish-github.result != 'success' && needs.publish-npm.result == 'success') 566 | run: | 567 | echo "⚠️ Partial success for v${{ needs.detect-and-bump-version.outputs.new_version }}" 568 | echo "GitHub Packages: ${{ needs.publish-github.result }}" 569 | echo "npm.js: ${{ needs.publish-npm.result }}" 570 | echo "GitHub Release: ${{ needs.create-github-release.result }}" 571 | 572 | - name: Failure notification 573 | if: needs.publish-github.result == 'failure' && needs.publish-npm.result == 'failure' 574 | run: | 575 | echo "❌ Publishing failed for v${{ needs.detect-and-bump-version.outputs.new_version }}" 576 | echo "Both GitHub Packages and npm.js publishing failed" 577 | echo "GitHub Release: ${{ needs.create-github-release.result }}" 578 | exit 1 579 | -------------------------------------------------------------------------------- /.github/workflows/security.yml: -------------------------------------------------------------------------------- 1 | name: Security Scan 2 | 3 | on: 4 | push: 5 | branches: [main, develop] 6 | pull_request: 7 | branches: [main, develop] 8 | workflow_dispatch: 9 | 10 | jobs: 11 | n8n-security-scan: 12 | name: n8n Community Package Security Scan 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout code 16 | uses: actions/checkout@v4 17 | 18 | - name: Setup Node.js 19 | uses: actions/setup-node@v4 20 | with: 21 | node-version: '18.x' 22 | 23 | - name: Run n8n Security Scanner 24 | run: | 25 | echo "🔍 Running n8n community package security scan..." 26 | echo "Package: @nskha/n8n-nodes-scrappey" 27 | echo "Checking published package security status..." 28 | 29 | # Run the scanner 30 | if node scripts/n8n-scanner.js; then 31 | echo "✅ Security scan passed successfully!" 32 | echo "SCAN_STATUS=passing" >> $GITHUB_ENV 33 | else 34 | echo "❌ Security scan failed or package not published" 35 | echo "SCAN_STATUS=failing" >> $GITHUB_ENV 36 | 37 | # Check if it's a "package not found" error (expected for unpublished packages) 38 | if npx @n8n/scan-community-package @nskha/n8n-nodes-scrappey 2>&1 | grep -q "404"; then 39 | echo "ℹ️ Package not published yet - this is expected for development" 40 | echo "SCAN_STATUS=not-published" >> $GITHUB_ENV 41 | exit 0 # Don't fail the workflow for unpublished packages 42 | else 43 | echo "⚠️ Real security issue detected!" 44 | exit 1 # Fail the workflow for actual security issues 45 | fi 46 | fi 47 | 48 | - name: Create security scan summary 49 | run: | 50 | echo "## 🔍 n8n Security Scan Results" >> $GITHUB_STEP_SUMMARY 51 | echo "" >> $GITHUB_STEP_SUMMARY 52 | echo "**Package:** \`@nskha/n8n-nodes-scrappey\`" >> $GITHUB_STEP_SUMMARY 53 | echo "**Status:** ${{ env.SCAN_STATUS }}" >> $GITHUB_STEP_SUMMARY 54 | echo "**Timestamp:** $(date -u)" >> $GITHUB_STEP_SUMMARY 55 | echo "" >> $GITHUB_STEP_SUMMARY 56 | 57 | case "${{ env.SCAN_STATUS }}" in 58 | "passing") 59 | echo "✅ **Result:** Package passed all security checks" >> $GITHUB_STEP_SUMMARY 60 | echo "" >> $GITHUB_STEP_SUMMARY 61 | echo "Your n8n community package is secure and ready for use!" >> $GITHUB_STEP_SUMMARY 62 | ;; 63 | "not-published") 64 | echo "ℹ️ **Result:** Package not yet published to npm" >> $GITHUB_STEP_SUMMARY 65 | echo "" >> $GITHUB_STEP_SUMMARY 66 | echo "This is expected for development versions. The security scan will run after publishing." >> $GITHUB_STEP_SUMMARY 67 | ;; 68 | "failing") 69 | echo "❌ **Result:** Security issues detected" >> $GITHUB_STEP_SUMMARY 70 | echo "" >> $GITHUB_STEP_SUMMARY 71 | echo "Please review and fix the security issues before proceeding." >> $GITHUB_STEP_SUMMARY 72 | ;; 73 | esac 74 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules/ 3 | package-lock.json 4 | yarn.lock 5 | .vscode/ 6 | package.json.backup 7 | 8 | # Build outputs 9 | dist/ 10 | .tmp/ 11 | tmp/ 12 | 13 | # Logs 14 | logs/ 15 | npm-debug.log* 16 | *.log 17 | 18 | # OS generated files 19 | .DS_Store 20 | .DS_Store? 21 | ._* 22 | .Spotlight-V100 23 | .Trashes 24 | ehthumbs.db 25 | Thumbs.db 26 | 27 | # IDE and editor files 28 | .vscode/ 29 | !.vscode/extensions.json 30 | .windsurf/ 31 | *.swp 32 | *.swo 33 | *~ 34 | 35 | # Environment and config files 36 | .env 37 | .env.local 38 | .env.*.local 39 | .npmrc 40 | .PublishKey 41 | 42 | # Runtime data 43 | pids 44 | *.pid 45 | *.seed 46 | *.pid.lock 47 | 48 | # Coverage directory used by tools like istanbul 49 | coverage/ 50 | *.lcov 51 | 52 | # Cache directories 53 | .cache/ 54 | .parcel-cache/ 55 | package-temp.json 56 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.tsbuildinfo 3 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | # Dependencies and lock files 2 | node_modules/ 3 | pnpm-lock.yaml 4 | package-lock.json 5 | yarn.lock 6 | 7 | # Build outputs 8 | dist/ 9 | build/ 10 | *.tgz 11 | 12 | # Git and CI 13 | .git/ 14 | .github/ 15 | 16 | # Logs 17 | *.log 18 | 19 | # IDE 20 | .vscode/ 21 | .idea/ 22 | 23 | # OS 24 | .DS_Store 25 | Thumbs.db 26 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | /** 3 | * https://prettier.io/docs/en/options.html#semicolons 4 | */ 5 | semi: true, 6 | 7 | /** 8 | * https://prettier.io/docs/en/options.html#trailing-commas 9 | */ 10 | trailingComma: 'all', 11 | 12 | /** 13 | * https://prettier.io/docs/en/options.html#bracket-spacing 14 | */ 15 | bracketSpacing: true, 16 | 17 | /** 18 | * https://prettier.io/docs/en/options.html#tabs 19 | */ 20 | useTabs: true, 21 | 22 | /** 23 | * https://prettier.io/docs/en/options.html#tab-width 24 | */ 25 | tabWidth: 2, 26 | 27 | /** 28 | * https://prettier.io/docs/en/options.html#arrow-function-parentheses 29 | */ 30 | arrowParens: 'always', 31 | 32 | /** 33 | * https://prettier.io/docs/en/options.html#quotes 34 | */ 35 | singleQuote: true, 36 | 37 | /** 38 | * https://prettier.io/docs/en/options.html#quote-props 39 | */ 40 | quoteProps: 'as-needed', 41 | 42 | /** 43 | * https://prettier.io/docs/en/options.html#end-of-line 44 | */ 45 | endOfLine: 'lf', 46 | 47 | /** 48 | * https://prettier.io/docs/en/options.html#print-width 49 | */ 50 | printWidth: 100, 51 | }; 52 | -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "dbaeumer.vscode-eslint", 4 | "EditorConfig.EditorConfig", 5 | "esbenp.prettier-vscode", 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | - Using welcoming and inclusive language 18 | - Being respectful of differing viewpoints and experiences 19 | - Gracefully accepting constructive criticism 20 | - Focusing on what is best for the community 21 | - Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | - The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | - Trolling, insulting/derogatory comments, and personal or political attacks 28 | - Public or private harassment 29 | - Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | - Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at jan@n8n.io. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Scrappey n8n Node 2 | 3 | Thank you for your interest in contributing to the Scrappey n8n Node! This document provides guidelines and information for contributors. 4 | 5 | ## 🚀 Getting Started 6 | 7 | ### Prerequisites 8 | 9 | - Node.js 18.10 or higher 10 | - pnpm 9.1+ (required package manager) 11 | - Git 12 | - n8n instance for testing (optional but recommended) 13 | 14 | ### Development Setup 15 | 16 | ```bash 17 | # 1. Fork and clone the repository 18 | git clone https://github.com/YOUR-USERNAME/n8n-nodes-scrappey.git 19 | cd n8n-nodes-scrappey 20 | 21 | # 2. Install dependencies 22 | pnpm install 23 | 24 | # 3. Build the project 25 | pnpm run build 26 | 27 | # 4. Set up development environment 28 | pnpm run start:dev 29 | ``` 30 | 31 | ## 📝 Development Guidelines 32 | 33 | ### Code Style 34 | 35 | - **TypeScript**: All new code must be written in TypeScript 36 | - **ESLint**: Follow the existing ESLint configuration 37 | - **Prettier**: Use Prettier for code formatting 38 | - **Naming**: Use descriptive variable and function names 39 | 40 | ### Code Quality Checklist 41 | 42 | Before submitting any code, ensure it passes: 43 | 44 | ```bash 45 | # Linting 46 | pnpm run lint 47 | 48 | # Type checking 49 | pnpm run type-check 50 | 51 | # Code formatting 52 | pnpm run format:check 53 | 54 | # Build verification 55 | pnpm run build 56 | 57 | # Full validation 58 | pnpm run validate 59 | ``` 60 | 61 | ### File Structure 62 | 63 | ``` 64 | nodes/Scrappey/ 65 | ├── Scrappey.node.ts # Main node definition 66 | ├── execute.ts # Operation dispatcher 67 | ├── RequestMethods.ts # Request handling logic 68 | ├── requestBodyBuilder.ts # Request construction 69 | ├── fields.ts # Node field definitions 70 | ├── GenericFunctions.ts # API utilities 71 | ├── utils.ts # Helper functions 72 | └── types.ts # Type definitions 73 | ``` 74 | 75 | ## 🔄 CI/CD Workflow 76 | 77 | ### Automated Processes 78 | 79 | Our CI/CD pipeline automatically handles: 80 | 81 | 1. **Code Quality**: Linting, formatting, and type checking 82 | 2. **Building**: Compiling TypeScript and copying assets 83 | 3. **Security**: Dependency auditing and CodeQL analysis 84 | 4. **Versioning**: Automatic version bumps based on commit messages 85 | 5. **Releases**: Publishing to GitHub Packages and npm 86 | 87 | ### Commit Message Conventions 88 | 89 | We use conventional commits for automatic versioning: 90 | 91 | - `feat: description` → **Minor version bump** (new features) 92 | - `fix: description` → **Patch version bump** (bug fixes) 93 | - `docs: description` → **No version bump** (documentation) 94 | - `chore: description` → **No version bump** (maintenance) 95 | - `BREAKING CHANGE` → **Major version bump** (breaking changes) 96 | - `[major]: description` → **Major version bump** (explicit major) 97 | 98 | ### Special Commit Flags 99 | 100 | - `[skip ci]` → Skip CI/CD pipeline 101 | - `[skip version]` → Skip automatic version bump 102 | 103 | ### Example Commits 104 | 105 | ```bash 106 | # New feature (minor version bump) 107 | git commit -m "feat: add support for custom user agents" 108 | 109 | # Bug fix (patch version bump) 110 | git commit -m "fix: resolve proxy connection timeout issue" 111 | 112 | # Breaking change (major version bump) 113 | git commit -m "feat: redesign API interface 114 | 115 | BREAKING CHANGE: The request configuration format has changed" 116 | 117 | # Documentation update (no version bump) 118 | git commit -m "docs: update installation instructions" 119 | 120 | # Skip automation 121 | git commit -m "chore: update dev dependencies [skip ci]" 122 | ``` 123 | 124 | ## 🧪 Testing 125 | 126 | ### Manual Testing 127 | 128 | 1. Build the node: `pnpm run build` 129 | 2. Install in local n8n: `pnpm run start:dev` 130 | 3. Test all three operation modes: 131 | - Request Builder 132 | - HTTP Auto-Retry 133 | - Browser Auto-Retry 134 | 135 | ### Test Cases to Verify 136 | 137 | - [ ] Basic HTTP requests work 138 | - [ ] Browser requests with anti-bot protection 139 | - [ ] Proxy configuration (credentials, Scrappey, custom) 140 | - [ ] Error handling for common scenarios 141 | - [ ] Auto-retry functionality with failed HTTP nodes 142 | - [ ] Custom headers and cookies 143 | - [ ] Session management 144 | - [ ] Country-specific proxy selection 145 | 146 | ## 📦 Release Process 147 | 148 | ### Automatic Releases (Recommended) 149 | 150 | 1. Merge approved PRs to `main` branch 151 | 2. Auto-versioning workflow triggers automatically 152 | 3. Version bump commits are created 153 | 4. Release workflow publishes to registries 154 | 155 | ### Manual Releases 156 | 157 | 1. Update version in `package.json` 158 | 2. Run `node scripts/update-node-json.js` 159 | 3. Create and push a version tag: `git tag v1.0.0 && git push origin v1.0.0` 160 | 4. GitHub Actions will handle the rest 161 | 162 | ## 🐛 Issue Reporting 163 | 164 | ### Bug Reports 165 | 166 | Please include: 167 | 168 | - n8n version 169 | - Node version 170 | - Scrappey node version 171 | - Steps to reproduce 172 | - Expected vs actual behavior 173 | - Error messages/logs 174 | - Workflow configuration (if applicable) 175 | 176 | ### Feature Requests 177 | 178 | Please include: 179 | 180 | - Use case description 181 | - Proposed solution 182 | - Alternatives considered 183 | - Implementation suggestions 184 | 185 | ## 📋 Pull Request Process 186 | 187 | 1. **Fork & Branch**: Create a feature branch from `main` 188 | 2. **Develop**: Make your changes following the guidelines 189 | 3. **Test**: Verify functionality manually 190 | 4. **Validate**: Run `pnpm run validate` to ensure quality 191 | 5. **Commit**: Use conventional commit messages 192 | 6. **Push**: Push to your fork 193 | 7. **PR**: Create a pull request with: 194 | - Clear description of changes 195 | - Link to related issues 196 | - Screenshots (if UI changes) 197 | - Test results 198 | 199 | ### PR Review Checklist 200 | 201 | - [ ] Code follows style guidelines 202 | - [ ] All CI checks pass 203 | - [ ] Functionality tested manually 204 | - [ ] Documentation updated (if needed) 205 | - [ ] No breaking changes (unless intentional) 206 | - [ ] Commit messages follow conventions 207 | 208 | ## 🏷️ Labeling System 209 | 210 | ### Issue Labels 211 | 212 | - `bug` - Something isn't working 213 | - `enhancement` - New feature or request 214 | - `documentation` - Documentation improvements 215 | - `good first issue` - Good for newcomers 216 | - `help wanted` - Extra attention is needed 217 | - `question` - Further information is requested 218 | 219 | ### PR Labels 220 | 221 | - `automated` - Created by automation 222 | - `dependencies` - Dependency updates 223 | - `ci/cd` - CI/CD related changes 224 | 225 | ## 🤝 Community 226 | 227 | ### Getting Help 228 | 229 | - **GitHub Issues**: [Report bugs or ask questions](https://github.com/Automations-Project/n8n-nodes-scrappey/issues) 230 | - **n8n Community**: [Join the discussion](https://community.n8n.io) 231 | - **Scrappey Support**: [API documentation](https://wiki.scrappey.com) 232 | 233 | ### Code of Conduct 234 | 235 | This project follows the [Code of Conduct](CODE_OF_CONDUCT.md). Please read and follow it in all interactions. 236 | 237 | ## 🙏 Recognition 238 | 239 | Contributors are recognized in: 240 | 241 | - Release notes 242 | - GitHub contributor graphs 243 | - Community highlights 244 | 245 | Thank you for making this project better! 🎉 246 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Nskha 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Banner](.github/assets/banner.webp) 2 | 3 | # Scrappey n8n Node 4 | 5 | [![CI](https://github.com/Automations-Project/n8n-nodes-scrappey/actions/workflows/ci.yml/badge.svg)](https://github.com/Automations-Project/n8n-nodes-scrappey/actions/workflows/ci.yml) 6 | [![Release](https://github.com/Automations-Project/n8n-nodes-scrappey/actions/workflows/release.yml/badge.svg)](https://github.com/Automations-Project/n8n-nodes-scrappey/actions/workflows/release.yml) 7 | [![Security Scan](https://github.com/Automations-Project/n8n-nodes-scrappey/actions/workflows/security.yml/badge.svg)](https://github.com/Automations-Project/n8n-nodes-scrappey/actions/workflows/security.yml) 8 | [![npm version](https://img.shields.io/npm/v/@nskha/n8n-nodes-scrappey?logo=npm)](https://www.npmjs.com/package/@nskha/n8n-nodes-scrappey) 9 | [![npm downloads](https://img.shields.io/npm/dm/@nskha/n8n-nodes-scrappey?logo=npm)](https://www.npmjs.com/package/@nskha/n8n-nodes-scrappey) 10 | [![n8n Node version](https://img.shields.io/github/package-json/v/Automations-Project/n8n-nodes-scrappey?logo=n8n&label=n8n%20node)](https://github.com/Automations-Project/n8n-nodes-scrappey) 11 | [![n8n compatibility](https://img.shields.io/github/v/release/n8n-io/n8n?logo=n8n&label=)](https://n8n.io) 12 | [![Node.js compatibility](https://img.shields.io/node/v/@nskha/n8n-nodes-scrappey?logo=node.js)](https://nodejs.org) 13 | [![TypeScript](https://img.shields.io/badge/TypeScript-5.5%2B-blue?logo=typescript)](https://www.typescriptlang.org) 14 | [![Dependencies](https://img.shields.io/librariesio/release/npm/@nskha/n8n-nodes-scrappey?logo=dependabot)](https://libraries.io/npm/@nskha%2Fn8n-nodes-scrappey) 15 | [![License](https://img.shields.io/github/license/Automations-Project/n8n-nodes-scrappey)](LICENSE.md) 16 | 17 | [![GitHub stars](https://img.shields.io/github/stars/Automations-Project/n8n-nodes-scrappey?style=social)](https://github.com/Automations-Project/n8n-nodes-scrappey/stargazers) 18 | [![GitHub forks](https://img.shields.io/github/forks/Automations-Project/n8n-nodes-scrappey?style=social)](https://github.com/Automations-Project/n8n-nodes-scrappey/network) 19 | [![GitHub issues](https://img.shields.io/github/issues/Automations-Project/n8n-nodes-scrappey)](https://github.com/Automations-Project/n8n-nodes-scrappey/issues) 20 | [![Last commit](https://img.shields.io/github/last-commit/Automations-Project/n8n-nodes-scrappey)](https://github.com/Automations-Project/n8n-nodes-scrappey/commits) 21 | 22 | > 🚀 **Advanced web scraping and anti-bot bypass node for n8n workflows** 23 | 24 | A powerful n8n community node that integrates with the [Scrappey.com API](https://scrappey.com) to provide advanced web scraping capabilities with built-in anti-bot protection bypass. Perfect for automating data extraction from protected websites, handling CAPTCHAs, and managing complex browser interactions. 25 | 26 | ## ✨ Key Features 27 | 28 | ### 🛠️ **Three Operation Modes** 29 | 30 | 1. **Request Builder** - Create fully customized HTTP/browser requests with granular control 31 | 2. **HTTP Auto-Retry** - Automatically retry failed HTTP requests through Scrappey's anti-bot network 32 | 3. **Browser Auto-Retry** - Advanced browser-based retry with full anti-bot protection 33 | 34 | ### 🔒 **Anti-Bot Protection Bypass** 35 | 36 | - **Cloudflare** challenge solving 37 | - **Datadome** bypass capabilities 38 | - **hCaptcha & reCAPTCHA** automatic solving 39 | - **JavaScript-heavy websites** full browser simulation 40 | - **Mouse movement simulation** for enhanced stealth 41 | 42 | ### 🌍 **Advanced Proxy Management** 43 | 44 | - **Residential proxies** with country targeting 45 | - **Datacenter proxies** for fast requests 46 | - **Mobile proxies** for mobile-specific content 47 | - **Custom proxy** support (SOCKS4/5, HTTP/HTTPS) 48 | - **150+ countries** available for geo-targeting 49 | 50 | ### ⚙️ **Flexible Configuration** 51 | 52 | - **Multiple request types**: Standard HTTP, Browser, Patched Chrome 53 | - **Custom headers & cookies** with field-based or JSON input 54 | - **Session management** for maintaining state across requests 55 | - **POST/PUT/PATCH support** with body or form parameters 56 | - **CSS selector waiting** for dynamic content 57 | - **XHR/Fetch interception** for API data extraction 58 | 59 | ## 🚀 Installation 60 | 61 | ### Method 1: n8n Community Nodes (Recommended) 62 | 63 | 1. Open your n8n instance 64 | 2. Go to **Settings** → **Community Nodes** 65 | 3. Enter: `@automations-project/n8n-nodes-scrappey` 66 | 4. Click **Install** 67 | 68 | ### Method 2: Manual Installation 69 | 70 | ```bash 71 | # Using npm 72 | npm install @automations-project/n8n-nodes-scrappey 73 | 74 | # Using pnpm 75 | pnpm add @automations-project/n8n-nodes-scrappey 76 | 77 | # Using yarn 78 | yarn add @automations-project/n8n-nodes-scrappey 79 | ``` 80 | 81 | ### Method 3: Development Installation 82 | 83 | ```bash 84 | # Clone the repository 85 | git clone https://github.com/Automations-Project/n8n-nodes-scrappey.git 86 | cd n8n-nodes-scrappey 87 | 88 | # Install dependencies 89 | pnpm install 90 | 91 | # Build the node 92 | pnpm run build 93 | 94 | # Link for development 95 | pnpm run start:dev 96 | ``` 97 | 98 | ## 🔧 Configuration 99 | 100 | ### 1. Set Up Scrappey API Credentials 101 | 102 | 1. Sign up at [Scrappey.com](/#) to get your API key. 103 | 2. In n8n, create new **Scrappey API** credentials 104 | 3. Enter your API key and optional proxy settings 105 | > 🎯 **Get Started Free!** Try Scrappey with **750 Direct requests** and **150 Browser requests** at no cost. 106 | > [Start your free trial →](https://nodes.n8n.community/scrappey/signup) 107 | > 108 | > **Affordable scaling**: For just €100, you can get 600,000 request credits including proxies, captcha etc... 109 | 110 | ### 2. Credential Options 111 | 112 | - **API Key** (required): Your Scrappey.com API key 113 | - **Custom Proxy** (optional): Your own proxy URL (SOCKS4/5, HTTP/HTTPS) 114 | - **Whitelisted Domains** (optional): JSON array of allowed domains for enhanced security 115 | 116 | ## 📋 Operation Modes 117 | ![Operations Types](.github/assets/operations.jpg) 118 | ### 🛠️ Request Builder (Manual) 119 | 120 | **Primary mode for creating custom requests with full control** 121 | 122 | ```typescript 123 | // Example configuration options: 124 | { 125 | "url": "https://example.com/api/data", 126 | "httpMethod": "GET", 127 | "request_type": "Browser", // or "Request", "PatchedChrome" 128 | "whichProxyToUse": "proxyFromScrappey", 129 | "proxyType": "residential", // residential, datacenter, mobile 130 | "customProxyCountry": "UnitedStates", 131 | "antibot": true, 132 | "mouseMovements": true, 133 | "datadome": true 134 | } 135 | ``` 136 | 137 | **Use Cases:** 138 | 139 | - Complex form submissions with CAPTCHA solving 140 | - JavaScript-heavy SPA scraping 141 | - API data extraction with anti-bot protection 142 | - Multi-step workflows with session management 143 | 144 | ### 🔁 HTTP Auto-Retry 145 | 146 | **Fallback solution for failed n8n HTTP Request nodes** 147 | ![Banner](.github/assets/example.svg) 148 | Connect the **error output** (red connector) of a standard HTTP Request node to this operation. It automatically retries the same request through Scrappey's network when blocked by: 149 | 150 | - Cloudflare challenges 151 | - Rate limiting 152 | - IP blocks 153 | - Basic anti-bot measures 154 | 155 | 156 | ### 🌐 Browser Auto-Retry 157 | 158 | **Advanced browser-based retry with full anti-bot protection** 159 | 160 | Similar to HTTP Auto-Retry but uses a full browser environment with: 161 | 162 | - Automatic CAPTCHA solving (hCaptcha, reCAPTCHA) 163 | - Mouse movement simulation 164 | - Datadome bypass enabled 165 | - JavaScript execution 166 | - 3 automatic retries 167 | 168 | ## 💡 Usage Examples 169 | 170 | ### Basic Web Scraping 171 | 172 | ```javascript 173 | // Request Builder - Simple GET request 174 | { 175 | "operation": "requestBuilder", 176 | "url": "https://httpbin.org/get", 177 | "httpMethod": "request.get", 178 | "request_type": "Request" 179 | } 180 | ``` 181 | 182 | ### Advanced Browser Automation 183 | 184 | ```javascript 185 | // Browser request with anti-bot protection 186 | { 187 | "operation": "requestBuilder", 188 | "url": "https://protected-site.com", 189 | "request_type": "Browser", 190 | "antibot": true, 191 | "mouseMovements": true, 192 | "datadome": true, 193 | "cssSelector": ".content-loaded", 194 | "proxyType": "residential", 195 | "customProxyCountry": "UnitedStates" 196 | } 197 | ``` 198 | 199 | ### Form Submission with CAPTCHA 200 | 201 | ```javascript 202 | // POST request with CAPTCHA solving 203 | { 204 | "operation": "requestBuilder", 205 | "url": "https://example.com/submit", 206 | "httpMethod": "request.post", 207 | "request_type": "Browser", 208 | "bodyOrParams": "body_used", 209 | "body_for_request": "{\"name\":\"John\",\"email\":\"john@example.com\"}", 210 | "antibot": true 211 | } 212 | ``` 213 | 214 | ### Auto-Retry Fallback 215 | 216 | ```javascript 217 | // Connect HTTP Request node error output to Scrappey node input 218 | // Set operation to "httpRequestAutoRetry" or "httpRequestAutoRetryBrowser" 219 | { 220 | "operation": "httpRequestAutoRetry", 221 | "whichProxyToUse": "proxyFromScrappey", 222 | "proxyType": "residential" 223 | } 224 | ``` 225 | 226 | ## 🔒 Error Handling 227 | 228 | The node provides detailed error messages for common Scrappey API error codes: 229 | 230 | | Code | Description | Solution | 231 | | --------- | ------------------ | ----------------------------------- | 232 | | CODE-0001 | Server overloaded | Retry after a few minutes | 233 | | CODE-0002 | Cloudflare blocked | Try different proxy or browser mode | 234 | | CODE-0003 | Too many attempts | Wait before retrying | 235 | | CODE-0004 | Invalid command | Check request configuration | 236 | | CODE-0005 | Tunnel failed | Retry with different proxy | 237 | 238 | ## 🏗️ Development 239 | 240 | ### Building from Source 241 | 242 | ```bash 243 | # Install dependencies 244 | pnpm install 245 | 246 | # Development build with watch 247 | pnpm run build:watch 248 | 249 | # Production build 250 | pnpm run build 251 | 252 | # Linting & formatting 253 | pnpm run lint 254 | pnpm run format 255 | 256 | # Type checking 257 | pnpm run type-check 258 | 259 | # Full validation 260 | pnpm run validate 261 | ``` 262 | 263 | ### Project Structure 264 | 265 | ``` 266 | n8n-nodes-scrappey/ 267 | ├── nodes/Scrappey/ # Main node implementation 268 | │ ├── Scrappey.node.ts # Node definition and execution 269 | │ ├── execute.ts # Operation dispatcher 270 | │ ├── RequestMethods.ts # HTTP/Browser request handlers 271 | │ ├── requestBodyBuilder.ts # Request body construction 272 | │ ├── fields.ts # Node field definitions 273 | │ ├── GenericFunctions.ts # API integration utilities 274 | │ └── utils.ts # Helper functions 275 | ├── credentials/ # Credential definitions 276 | │ └── ScrappeyApi.credentials.ts 277 | ├── scripts/ # Build and deployment scripts 278 | ├── .github/workflows/ # CI/CD pipelines 279 | └── dist/ # Built output 280 | ``` 281 | 282 | ### CI/CD Pipeline 283 | 284 | This project includes a comprehensive CI/CD setup: 285 | 286 | - **Continuous Integration**: Automated testing, linting, and building on every PR 287 | - **Auto-versioning**: Automatic version bumps based on commit messages 288 | - **Automated Releases**: Publishes to GitHub Packages and optionally npm 289 | - **Security Scanning**: CodeQL analysis and dependency auditing 290 | - **Dependabot**: Automated dependency updates 291 | 292 | #### Commit Message Conventions 293 | 294 | - `feat: description` → Minor version bump 295 | - `fix: description` → Patch version bump 296 | - `BREAKING CHANGE` or `[major]` → Major version bump 297 | - `[skip ci]` or `[skip version]` → Skip automation 298 | 299 | ## 🤝 Contributing 300 | 301 | 1. Fork the repository 302 | 2. Create a feature branch: `git checkout -b feature/amazing-feature` 303 | 3. Commit changes: `git commit -m 'feat: add amazing feature'` 304 | 4. Push to branch: `git push origin feature/amazing-feature` 305 | 5. Open a Pull Request 306 | 307 | ## 📄 License 308 | 309 | This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details. 310 | 311 | ## 🔗 Links 312 | 313 | - **Scrappey Website**: [https://scrappey.com](https://scrappey.com) 314 | - **Scrappey Documentation**: [https://wiki.scrappey.com](https://wiki.scrappey.com) 315 | - **n8n Community**: [https://community.n8n.io](https://community.n8n.io) 316 | - **GitHub Issues**: [Report bugs or request features](https://github.com/Automations-Project/n8n-nodes-scrappey/issues) 317 | - **Nskha Discord**: [⚠️Incative community](https://nskha.com/discord) 318 | 319 | --- 320 | 321 | **Made with ❤️ for the n8n community** 322 | -------------------------------------------------------------------------------- /README_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | # n8n-nodes-_node-name_ 2 | 3 | This is an n8n community node. It lets you use _app/service name_ in your n8n workflows. 4 | 5 | _App/service name_ is _one or two sentences describing the service this node integrates with_. 6 | 7 | [n8n](https://n8n.io/) is a [fair-code licensed](https://docs.n8n.io/reference/license/) workflow automation platform. 8 | 9 | [Installation](#installation) 10 | [Operations](#operations) 11 | [Credentials](#credentials) 12 | [Compatibility](#compatibility) 13 | [Usage](#usage) 14 | [Resources](#resources) 15 | [Version history](#version-history) 16 | 17 | ## Installation 18 | 19 | Follow the [installation guide](https://docs.n8n.io/integrations/community-nodes/installation/) in the n8n community nodes documentation. 20 | 21 | ## Operations 22 | 23 | _List the operations supported by your node._ 24 | 25 | ## Credentials 26 | 27 | _If users need to authenticate with the app/service, provide details here. You should include prerequisites (such as signing up with the service), available authentication methods, and how to set them up._ 28 | 29 | ## Compatibility 30 | 31 | _State the minimum n8n version, as well as which versions you test against. You can also include any known version incompatibility issues._ 32 | 33 | ## Usage 34 | 35 | _This is an optional section. Use it to help users with any difficult or confusing aspects of the node._ 36 | 37 | _By the time users are looking for community nodes, they probably already know n8n basics. But if you expect new users, you can link to the [Try it out](https://docs.n8n.io/try-it-out/) documentation to help them get started._ 38 | 39 | ## Resources 40 | 41 | - [n8n community nodes documentation](https://docs.n8n.io/integrations/community-nodes/) 42 | - _Link to app/service documentation._ 43 | 44 | ## Version history 45 | 46 | _This is another optional section. If your node has multiple versions, include a short description of available versions and what changed, as well as any compatibility impact._ 47 | -------------------------------------------------------------------------------- /credentials/ScrappeyApi.credentials.ts: -------------------------------------------------------------------------------- 1 | import { ICredentialType, INodeProperties, ICredentialTestRequest } from 'n8n-workflow'; 2 | 3 | export class ScrappeyApi implements ICredentialType { 4 | name = 'scrappeyApi'; 5 | displayName = 'Scrappey API'; 6 | icon = 'file:Scrappey.svg' as const; 7 | documentationUrl = 'https://wiki.scrappey.com'; 8 | properties: INodeProperties[] = [ 9 | { 10 | displayName: 'API Key', 11 | name: 'apiKey', 12 | type: 'string', 13 | default: '', 14 | required: true, 15 | typeOptions: { 16 | password: true, 17 | }, 18 | }, 19 | { 20 | displayName: 'Add your custom proxy', 21 | name: 'allowProxy', 22 | type: 'boolean', 23 | default: false, 24 | }, 25 | { 26 | displayName: 'Proxy URL Socks-5 Socks-4 Http Https', 27 | name: 'proxyUrl', 28 | type: 'string', 29 | default: '', 30 | required: false, 31 | hint: 'Optional. Proxy URL to be used for the scraping.', 32 | displayOptions: { 33 | show: { 34 | allowProxy: [true], 35 | }, 36 | }, 37 | }, 38 | { 39 | displayName: 'Whitelisted Domains', 40 | name: 'whitelistedDomains', 41 | type: 'string', 42 | placeholder: '["google.com","youtube.com","n8n.nskha.com"]', 43 | default: '', 44 | required: false, 45 | hint: "Optional. List of domains that are allowed to be scraped. If isn't set, all domains are allowed", 46 | }, 47 | ]; 48 | 49 | test: ICredentialTestRequest = { 50 | request: { 51 | baseURL: 'https://publisher.scrappey.com/api/v1', 52 | url: '/balance', 53 | method: 'GET', 54 | qs: { 55 | key: '={{$credentials.apiKey}}' 56 | }, 57 | }, 58 | }; 59 | } 60 | -------------------------------------------------------------------------------- /docs/n8n-scanner.md: -------------------------------------------------------------------------------- 1 | # n8n Community Package Scanner 2 | 3 | This document explains how to use the official n8n community package scanner to validate and security-check your n8n node package. 4 | 5 | ## What is the n8n Scanner? 6 | 7 | The `@n8n/scan-community-package` is an official tool from n8n that: 8 | 9 | - **Validates package structure** for n8n community nodes 10 | - **Performs security analysis** to detect malicious code 11 | - **Checks for vulnerabilities** in dependencies and code 12 | - **Ensures compliance** with n8n community standards 13 | 14 | ## Usage 15 | 16 | ### Local Development 17 | 18 | Run the scanner locally to check the published package: 19 | 20 | ```bash 21 | # Run the n8n scanner 22 | pnpm run scan:n8n 23 | ``` 24 | 25 | This will scan your published package `@nskha/n8n-nodes-scrappey` directly from the npm registry. 26 | 27 | ### Direct Usage 28 | 29 | You can also run the scanner directly: 30 | 31 | ```bash 32 | # Scan the published package 33 | npx @n8n/scan-community-package @nskha/n8n-nodes-scrappey 34 | ``` 35 | 36 | ### CI/CD Integration 37 | 38 | The scanner is automatically integrated into our workflows: 39 | 40 | - **CI Pipeline**: Runs on every commit (checks published version if available) 41 | - **Release Pipeline**: Runs after publishing to npm to validate the new version 42 | - **Behavior**: Continues on error (won't fail the build if package isn't published yet) 43 | 44 | ## How It Works 45 | 46 | The scanner works by: 47 | 48 | 1. **Downloading** the specified package from npm registry 49 | 2. **Analyzing** the package structure and code 50 | 3. **Checking** for security vulnerabilities and compliance 51 | 4. **Reporting** results with detailed feedback 52 | 53 | ## Example Results 54 | 55 | ### Successful Scan 56 | ```bash 57 | ✅ Downloaded @nskha/n8n-nodes-scrappey@0.3.7 58 | ✅ Analyzed @nskha/n8n-nodes-scrappey@0.3.7 59 | ✅ Package @nskha/n8n-nodes-scrappey@0.3.7 has passed all security checks 60 | ``` 61 | 62 | ### Failed Scan 63 | ```bash 64 | ❌ Failed to analyze @nskha/n8n-nodes-scrappey@0.3.7: AxiosError: Request failed with status code 404 65 | ``` 66 | 67 | ## What the Scanner Checks 68 | 69 | ### Security Analysis 70 | - **Malicious code patterns** in your source files 71 | - **Suspicious network requests** or file system access 72 | - **Obfuscated code** that might hide malicious intent 73 | - **Dependency vulnerabilities** in your package tree 74 | 75 | ### Package Structure 76 | - **Required files** are present (package.json, built files) 77 | - **n8n node structure** follows conventions 78 | - **Credential handling** is secure 79 | - **API usage** follows n8n guidelines 80 | 81 | ### Code Quality 82 | - **TypeScript compliance** for better maintainability 83 | - **Error handling** patterns are present 84 | - **Documentation** completeness 85 | - **Version compatibility** with n8n 86 | 87 | ## Common Issues and Solutions 88 | 89 | ### Package Not Found (404 Error) 90 | 91 | **Error**: `Request failed with status code 404` 92 | 93 | **Cause**: Package not published to npm registry 94 | 95 | **Solution**: 96 | - Ensure package is published to npm: `pnpm publish` 97 | - Check package name is correct: `@nskha/n8n-nodes-scrappey` 98 | - Wait a few minutes for npm to propagate new packages 99 | 100 | ### Network Connectivity Issues 101 | 102 | **Error**: Connection timeout or network errors 103 | 104 | **Cause**: Network connectivity issues 105 | 106 | **Solution**: 107 | - Check internet connection 108 | - Try again after a few minutes 109 | - Use VPN if corporate firewall blocks npm 110 | 111 | ### Scanner Tool Not Found 112 | 113 | **Error**: Command not found errors 114 | 115 | **Cause**: n8n scanner package not installed 116 | 117 | **Solution**: 118 | - The scanner is installed automatically via `npx` 119 | - Ensure you have Node.js and npm installed 120 | - Try: `npx @n8n/scan-community-package --help` 121 | 122 | ## Integration with Release Process 123 | 124 | ### Automated Scanning 125 | 126 | 1. **During CI**: Scanner attempts to check published version (expected to fail for unpublished packages) 127 | 2. **After Release**: Scanner runs automatically after npm publish 128 | 3. **Manual Check**: Run `pnpm run scan:n8n` anytime to check latest published version 129 | 130 | ### Release Workflow Integration 131 | 132 | The scanner is integrated into the release workflow: 133 | 134 | ```yaml 135 | - name: Run n8n Security Scanner 136 | run: | 137 | echo "🔍 Running n8n community package security scan..." 138 | # Wait a bit for npm to propagate the package 139 | sleep 30 140 | node scripts/n8n-scanner.js || echo "⚠️ Scanner check completed" 141 | ``` 142 | 143 | ## Best Practices 144 | 145 | ### For Developers 146 | 147 | 1. **Regular Scanning**: Run scanner after each release 148 | 2. **Fix Issues Promptly**: Address security findings immediately 149 | 3. **Monitor Dependencies**: Keep dependencies updated 150 | 4. **Secure Coding**: Follow n8n security guidelines 151 | 152 | ### For CI/CD 153 | 154 | 1. **Non-Blocking**: Scanner continues on error to avoid blocking releases 155 | 2. **Post-Publish**: Scanner runs after successful publishing 156 | 3. **Logging**: Results are captured in CI logs for review 157 | 158 | ## Manual Commands 159 | 160 | ```bash 161 | # Check current published version 162 | npx @n8n/scan-community-package @nskha/n8n-nodes-scrappey 163 | 164 | # Check specific version 165 | npx @n8n/scan-community-package @nskha/n8n-nodes-scrappey@0.3.7 166 | 167 | # Use local script 168 | pnpm run scan:n8n 169 | 170 | # Get scanner help 171 | npx @n8n/scan-community-package --help 172 | ``` 173 | 174 | ## Troubleshooting 175 | 176 | ### Scanner Always Fails 177 | 178 | 1. Verify package is published: `npm view @nskha/n8n-nodes-scrappey` 179 | 2. Check package name spelling 180 | 3. Ensure you have internet access 181 | 4. Try running scanner directly with npx 182 | 183 | ### Development vs Published Package 184 | 185 | - The scanner only works with **published** packages 186 | - For unpublished code, use other security tools like CodeQL (integrated in CI) 187 | - The scanner is most useful **after** publishing to validate releases 188 | 189 | ## Further Reading 190 | 191 | - [n8n Community Nodes Documentation](https://docs.n8n.io/integrations/community-nodes/) 192 | - [n8n Security Guidelines](https://docs.n8n.io/integrations/community-nodes/security/) 193 | - [npm Security Best Practices](https://docs.npmjs.com/security-best-practices) 194 | -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const { task, src, dest } = require('gulp'); 3 | 4 | task('build:icons', copyIcons); 5 | 6 | function copyIcons() { 7 | const nodeSource = path.resolve('nodes', '**', '*.{png,svg}'); 8 | const nodeDestination = path.resolve('dist', 'nodes'); 9 | 10 | src(nodeSource).pipe(dest(nodeDestination)); 11 | 12 | const credSource = path.resolve('credentials', '**', '*.{png,svg}'); 13 | const credDestination = path.resolve('dist', 'credentials'); 14 | 15 | return src(credSource).pipe(dest(credDestination)); 16 | } 17 | -------------------------------------------------------------------------------- /nodes/Scrappey/GenericFunctions.ts: -------------------------------------------------------------------------------- 1 | import { IExecuteFunctions, IHttpRequestMethods, ILoadOptionsFunctions } from 'n8n-workflow'; 2 | import { NodeApiError } from 'n8n-workflow'; 3 | 4 | interface RequestOptions { 5 | method?: IHttpRequestMethods; 6 | headers?: Record; 7 | body?: any; 8 | params?: Record; 9 | } 10 | 11 | export async function genericHttpRequest( 12 | this: IExecuteFunctions | ILoadOptionsFunctions, 13 | method: IHttpRequestMethods, 14 | endpoint: string, 15 | options: RequestOptions = {}, 16 | ): Promise { 17 | try { 18 | const credentials = await this.getCredentials('scrappeyApi'); 19 | const apiKey = credentials?.apiKey as string; 20 | 21 | // Set up base URL and add API key as query parameter 22 | let fullEndpoint = `https://publisher.scrappey.com/api/v1${endpoint}`; 23 | 24 | // Initialize params if they don't exist 25 | const params = options.params || {}; 26 | // Add API key to params 27 | params.key = apiKey; 28 | 29 | // Add query parameters to URL 30 | const searchParams = new URLSearchParams(params); 31 | fullEndpoint += fullEndpoint.includes('?') ? '&' : '?'; 32 | fullEndpoint += searchParams.toString(); 33 | 34 | const response = await this.helpers.httpRequest({ 35 | method, 36 | url: fullEndpoint, 37 | headers: { 38 | 'Content-Type': 'application/json', 39 | ...options.headers, 40 | }, 41 | body: options.body ? JSON.stringify(options.body) : undefined, 42 | json: true, 43 | }); 44 | return response as T; 45 | } catch (error: any) { 46 | // Error mapping for Scrappey API error codes 47 | const errorCodeMap: Record = { 48 | 'CODE-0001': { 49 | code: 'CODE-0001', 50 | message: 'Server is overloaded', 51 | details: 'All server capacity is used, please try again.', 52 | }, 53 | 'CODE-0002': { 54 | code: 'CODE-0002', 55 | message: 'Cloudflare blocked', 56 | details: 'Cloudflare blocked the request, preventing access to the resource.', 57 | }, 58 | 'CODE-0003': { 59 | code: 'CODE-0003', 60 | message: 'Cloudflare too many attempts, try again', 61 | details: 'Cloudflare has detected too many attempts from your IP; please try again later.', 62 | }, 63 | 'CODE-0004': { 64 | code: 'CODE-0004', 65 | message: 'Invalid cmd command', 66 | details: 'The provided cmd command is invalid and cannot be executed.', 67 | }, 68 | 'CODE-0005': { 69 | code: 'CODE-0005', 70 | message: 'Tunnel connection failed', 71 | details: 'Tunnel connection failed, and the request cannot be completed.', 72 | }, 73 | 'CODE-0006': { 74 | code: 'CODE-0006', 75 | message: 'ERR_HTTP_RESPONSE_CODE_FAILURE', 76 | details: 'Failure in the HTTP response code.', 77 | }, 78 | 'CODE-0007': { 79 | code: 'CODE-0007', 80 | message: 'Could not click turnstile button', 81 | details: 'System could not click the turnstile button, and the request cannot proceed.', 82 | }, 83 | 'CODE-0008': { 84 | code: 'CODE-0008', 85 | message: 'Ticketmaster blocked', 86 | details: 'Ticketmaster has blocked access to the resource.', 87 | }, 88 | 'CODE-0009': { 89 | code: 'CODE-0009', 90 | message: 'Error from ChatGPT, try again', 91 | details: 'An error occurred in the ChatGPT API, please try again.', 92 | }, 93 | 'CODE-0010': { 94 | code: 'CODE-0010', 95 | message: 'Blocked proxy on Datadome', 96 | details: 'Proxy blocked by Datadome, please try again later.', 97 | }, 98 | 'CODE-0011': { 99 | code: 'CODE-0011', 100 | message: 'Could not solve datadome', 101 | details: 'System could not solve Datadome protection, please try again.', 102 | }, 103 | 'CODE-0012': { 104 | code: 'CODE-0012', 105 | message: 'Could not parse datadome cookie', 106 | details: 'Issue parsing the Datadome cookie, please try again.', 107 | }, 108 | 'CODE-0013': { 109 | code: 'CODE-0013', 110 | message: 'Captcha solver datadome cookie error', 111 | details: "Captcha solver's Datadome cookie error, please retry the request.", 112 | }, 113 | 'CODE-0014': { 114 | code: 'CODE-0014', 115 | message: 'Could not load Datadome', 116 | details: 'Datadome protection could not be loaded, please try again.', 117 | }, 118 | 'CODE-0015': { 119 | code: 'CODE-0015', 120 | message: 'Socks4 With Authentication not Supported', 121 | details: 'Socks4 with authentication is not supported.', 122 | }, 123 | 'CODE-0016': { 124 | code: 'CODE-0016', 125 | message: 'Socks5 With Authentication not Supported', 126 | details: 'Socks5 with authentication is not supported.', 127 | }, 128 | 'CODE-0017': { 129 | code: 'CODE-0017', 130 | message: 'Cloudflare updated and is currently not solvable, try again later', 131 | details: 'Cloudflare updated and is currently unsolvable, try again later.', 132 | }, 133 | 'CODE-0018': { 134 | code: 'CODE-0018', 135 | message: 'Too high error rate for this URL', 136 | details: 137 | 'High error rate for this URL, affecting service. Contact support to lift the temporary ban.', 138 | }, 139 | 'CODE-0019': { 140 | code: 'CODE-0019', 141 | message: 'The proxy server is refusing connections', 142 | details: 'Proxy server refusing connections, check proxy settings.', 143 | }, 144 | 'CODE-0020': { 145 | code: 'CODE-0020', 146 | message: 'Could not find intercept request', 147 | details: 'System could not find the intercept request, please try again.', 148 | }, 149 | 'CODE-0021': { 150 | code: 'CODE-0021', 151 | message: 'Unknown error occurred with request', 152 | details: 'Unknown error occurred with the request, please try again.', 153 | }, 154 | 'CODE-0022': { 155 | code: 'CODE-0022', 156 | message: 'Captcha type solve_captcha is not found', 157 | details: 'Specified captcha type "solve_captcha" was not found, please try again.', 158 | }, 159 | 'CODE-0023': { 160 | code: 'CODE-0023', 161 | message: 'Turnstile solve_captcha was not found', 162 | details: 'Turnstile "solve_captcha" was not found, please try again.', 163 | }, 164 | 'CODE-0024': { 165 | code: 'CODE-0024', 166 | message: 'Proxy timeout - proxy too slow', 167 | details: 'Proxy connection timed out due to slowness, please try again.', 168 | }, 169 | 'CODE-0025': { 170 | code: 'CODE-0025', 171 | message: 'NS_ERROR_NET_TIMEOUT - proxy too slow', 172 | details: 'NS_ERROR_NET_TIMEOUT occurred due to a slow proxy, please try again.', 173 | }, 174 | 'CODE-0026': { 175 | code: 'CODE-0026', 176 | message: 'Internal browser error', 177 | details: 'Internal browser error occurred, please try again.', 178 | }, 179 | 'CODE-0027': { 180 | code: 'CODE-0027', 181 | message: 'No elements found for this CSS selector', 182 | details: 'No elements found for the given CSS selector, please retry.', 183 | }, 184 | 'CODE-0028': { 185 | code: 'CODE-0028', 186 | message: 'Could not solve perimeterx', 187 | details: 'System could not solve PerimeterX protection, please try again.', 188 | }, 189 | 'CODE-0029': { 190 | code: 'CODE-0029', 191 | message: 'Too many sessions open', 192 | details: 193 | 'Too many sessions open; sessions will automatically close after 240 seconds. Contact support for more sessions.', 194 | }, 195 | 'CODE-0030': { 196 | code: 'CODE-0030', 197 | message: 'Browser name must be: firefox, chrome or safari', 198 | details: 'Browser name must be one of: firefox, chrome, or safari.', 199 | }, 200 | 'CODE-0031': { 201 | code: 'CODE-0031', 202 | message: 'Request error, please try again', 203 | details: 'Request error, please try again.', 204 | }, 205 | 'CODE-0032': { 206 | code: 'CODE-0032', 207 | message: 'Turnstile captcha could not be solved', 208 | details: 'Turnstile captcha could not be solved, please try again with a different proxy.', 209 | }, 210 | 'CODE-0033': { 211 | code: 'CODE-0033', 212 | message: 'Mt captcha could not be solved', 213 | details: 'Mt captcha could not be solved, please try again.', 214 | }, 215 | 'CODE-0034': { 216 | code: 'CODE-0034', 217 | message: 'Datadome captcha could not be solved after 5 attempts', 218 | details: 219 | 'Datadome captcha could not be solved after 5 attempts, try different proxy settings.', 220 | }, 221 | 'CODE-0035': { 222 | code: 'CODE-0035', 223 | message: 'Could not load geetest', 224 | details: 'Could not load geetest, please try again.', 225 | }, 226 | 'CODE-0036': { 227 | code: 'CODE-0036', 228 | message: 'Keyboard action value not found', 229 | details: 'Keyboard action value not found.', 230 | }, 231 | 'CODE-0037': { 232 | code: 'CODE-0037', 233 | message: 'Datadome was blocked', 234 | details: 'Datadome was blocked, please try again with a different proxy.', 235 | }, 236 | 'CODE-10000': { 237 | code: 'CODE-10000', 238 | message: 'Unknown error - has to be specified', 239 | details: 'An unknown error occurred and needs to be specified.', 240 | }, 241 | }; 242 | 243 | 244 | if ( 245 | (error.message && error.message.includes('ERR_TUNNEL_CONNECTION_FAILED')) || 246 | error.message.includes('ERR_EMPTY_RESPONSE') 247 | ) { 248 | const errorDetails = errorCodeMap['CODE-0007']; 249 | throw new NodeApiError(this.getNode(), error, { 250 | message: errorDetails.message, 251 | description: errorDetails.details, 252 | httpCode: '500', 253 | }); 254 | } 255 | 256 | // Try to extract error code from response 257 | let errorCode = 'CODE-10000'; // Default unknown error 258 | if (error.response?.data?.code) { 259 | errorCode = error.response.data.code; 260 | } 261 | 262 | // Use mapped error if available, otherwise use a generic error 263 | const mappedError = errorCodeMap[errorCode] || errorCodeMap['CODE-10000']; 264 | 265 | throw new NodeApiError(this.getNode(), error, { 266 | message: mappedError.message, 267 | description: mappedError.details, 268 | httpCode: error.response?.status.toString() || '500', 269 | }); 270 | } 271 | } 272 | -------------------------------------------------------------------------------- /nodes/Scrappey/Scrappey.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "node": "n8n-nodes-scrappey", 3 | "nodeVersion": "0.3.13", 4 | "codexVersion": "0.3.13", 5 | "categories": [ 6 | "Development", 7 | "Web Scraping" 8 | ], 9 | "resources": { 10 | "credentialDocumentation": [ 11 | { 12 | "url": "https://nodes.n8n.community/scrappey/signup" 13 | } 14 | ], 15 | "primaryDocumentation": [ 16 | { 17 | "url": "https://wiki.scrappey.com" 18 | } 19 | ] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /nodes/Scrappey/Scrappey.node.ts: -------------------------------------------------------------------------------- 1 | import { INodeType, INodeTypeDescription } from 'n8n-workflow'; 2 | import { AdvancedSettingsForBrowser, publicFields } from './fields'; 3 | import { executeScrappey } from './execute'; 4 | import { scrappeyOperators } from './operators'; 5 | import { IExecuteFunctions, INodeExecutionData, IDataObject, NodeOperationError } from 'n8n-workflow'; 6 | 7 | export class Scrappey implements INodeType { 8 | public async execute(this: IExecuteFunctions): Promise { 9 | // Get all input items - this is crucial for item linking 10 | const items = this.getInputData(); 11 | const returnData: INodeExecutionData[] = []; 12 | 13 | const operation = this.getNodeParameter('scrappeyOperations', 0) as string; 14 | 15 | // Process each input item individually to maintain item relationships 16 | for (let itemIndex = 0; itemIndex < items.length; itemIndex++) { 17 | try { 18 | // This ensures that expressions like {{ $json.field }} work correctly 19 | const responseData = await executeScrappey.call(this, operation, itemIndex); 20 | 21 | if (Array.isArray(responseData)) { 22 | if (responseData.length > 0 && responseData[0].hasOwnProperty('json')) { 23 | responseData.forEach((item) => { 24 | returnData.push({ 25 | json: item.json, 26 | pairedItem: { item: itemIndex } 27 | }); 28 | }); 29 | } else { 30 | responseData.forEach((item) => { 31 | returnData.push({ 32 | json: item as IDataObject, 33 | pairedItem: { item: itemIndex } 34 | }); 35 | }); 36 | } 37 | } else { 38 | returnData.push({ 39 | json: responseData as IDataObject, 40 | pairedItem: { item: itemIndex } 41 | }); 42 | } 43 | } catch (error) { 44 | // This allows n8n to track which input item caused the error 45 | if (this.continueOnFail()) { 46 | returnData.push({ 47 | json: { 48 | error: error.message, 49 | // Include the original input data so it's not lost 50 | originalInput: items[itemIndex].json 51 | }, 52 | pairedItem: { item: itemIndex }, 53 | error: new NodeOperationError(this.getNode(), error as Error) 54 | }); 55 | } else { 56 | throw new NodeOperationError( 57 | this.getNode(), 58 | error as Error, 59 | { 60 | message: `Failed to process item ${itemIndex}`, 61 | description: `Error occurred while processing input item at index ${itemIndex}`, 62 | itemIndex 63 | } 64 | ); 65 | } 66 | } 67 | } 68 | 69 | return [returnData]; 70 | } 71 | 72 | description: INodeTypeDescription = { 73 | displayName: 'Scrappey', 74 | name: 'scrappey', 75 | icon: 'file:Scrappey.svg', 76 | group: ['web-scraping'], 77 | version: 1, 78 | subtitle: 79 | '={{ { requestBuilder: "🛠️ Request Builder", httpRequestAutoRetry: "🔁 Auto • HTTP Mode", httpRequestAutoRetryBrowser: "🌐 Auto • Browser Mode" }[$parameter["scrappeyOperations"]] }}', 80 | description: 'Make advanced web requests with anti-bot protection bypass using Scrappey API', 81 | defaults: { 82 | name: 'scrappey', 83 | }, 84 | inputs: ['main'] as unknown as INodeTypeDescription['inputs'], 85 | outputs: ['main'] as unknown as INodeTypeDescription['outputs'], 86 | credentials: [ 87 | { 88 | name: 'scrappeyApi', 89 | required: true, 90 | }, 91 | ], 92 | requestDefaults: { 93 | baseURL: 'https://api.scrappey.com', 94 | url: '', 95 | headers: { 96 | Accept: 'application/json', 97 | 'Content-Type': 'application/json', 98 | }, 99 | }, 100 | properties: [...scrappeyOperators, ...publicFields, ...AdvancedSettingsForBrowser], 101 | }; 102 | } -------------------------------------------------------------------------------- /nodes/Scrappey/execute.ts: -------------------------------------------------------------------------------- 1 | import { AutoRetryTypeBrowser, PostRequest, AutoRetryTypeRequest } from './methods'; 2 | import { IExecuteFunctions, NodeOperationError } from 'n8n-workflow'; 3 | 4 | export async function executeScrappey(this: IExecuteFunctions, operation: string, itemIndex: number = 0) { 5 | switch (operation) { 6 | case 'requestBuilder': 7 | return await PostRequest.call(this, itemIndex); 8 | case 'httpRequestAutoRetry': 9 | return await AutoRetryTypeRequest.call(this, itemIndex); 10 | case 'httpRequestAutoRetryBrowser': 11 | return await AutoRetryTypeBrowser.call(this, itemIndex); 12 | default: 13 | throw new NodeOperationError(this.getNode(), `Operation "${operation}" is not supported`, { 14 | description: 'Please select a valid operation from the available options.', 15 | itemIndex // item index in error for better debugging 16 | }); 17 | } 18 | } -------------------------------------------------------------------------------- /nodes/Scrappey/fields.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable n8n-nodes-base/node-param-display-name-miscased */ 2 | /* eslint-disable n8n-nodes-base/node-param-description-excess-final-period */ 3 | /* eslint-disable n8n-nodes-base/node-param-options-type-unsorted-items */ 4 | /* eslint-disable n8n-nodes-base/node-param-required-false */ 5 | import { INodeProperties } from 'n8n-workflow'; 6 | import { Static_Country_Proxies , generateUUID} from './utils'; 7 | 8 | export const publicFields: INodeProperties[] = [ 9 | { 10 | displayName: 11 | '⚠️This is a fallback solution and works only if the previous node is an HTTP node.

🚦 For best results, connect the error path of the HTTP node to this operation.

👉 See the example workflow.', 12 | name: 'affiliateMessage', 13 | type: 'notice', 14 | default: '', 15 | displayOptions: { 16 | show: { 17 | scrappeyOperations: ['httpRequestAutoRetry', 'httpRequestAutoRetryBrowser'], 18 | }, 19 | }, 20 | }, 21 | 22 | { 23 | displayName: 'URL', 24 | name: 'url', 25 | type: 'string', 26 | default: '', 27 | hint: 'URL of the page to scrape', 28 | placeholder: 'https://httpbin.rs/get', 29 | required: true, 30 | displayOptions: { 31 | show: { 32 | scrappeyOperations: ['requestBuilder'], 33 | }, 34 | }, 35 | }, 36 | { 37 | displayName: 'HTTP Method', 38 | name: 'httpMethod', 39 | type: 'options', 40 | default: 'request.get', 41 | hint: 'HTTP method to use with the URL', 42 | options: [ 43 | { 44 | name: 'GET', 45 | value: 'request.get', 46 | }, 47 | { 48 | name: 'POST', 49 | value: 'request.post', 50 | }, 51 | { 52 | name: 'PUT', 53 | value: 'request.put', 54 | }, 55 | { 56 | name: 'DELETE', 57 | value: 'request.delete', 58 | }, 59 | { 60 | name: 'PATCH', 61 | value: 'request.patch', 62 | }, 63 | { 64 | name: 'PUBLISH', 65 | value: 'request.publish', 66 | }, 67 | ], 68 | displayOptions: { 69 | show: { 70 | scrappeyOperations: ['requestBuilder'], 71 | }, 72 | }, 73 | }, 74 | { 75 | displayName: 'Request Type', 76 | name: 'request_type', 77 | type: 'options', 78 | default: 'Request', 79 | options: [ 80 | { 81 | name: 'Browser', 82 | value: 'Browser', 83 | }, 84 | { 85 | name: 'Request', 86 | value: 'Request', 87 | }, 88 | { 89 | name: 'Patched Chrome Browser', 90 | value: 'PatchedChrome', 91 | }, 92 | ], 93 | displayOptions: { 94 | show: { 95 | scrappeyOperations: ['requestBuilder'], 96 | }, 97 | }, 98 | }, 99 | { 100 | displayName: 'Which Proxy To Use', 101 | name: 'whichProxyToUse', 102 | type: 'options', 103 | options: [ 104 | { 105 | name: 'Proxy From Credentials', 106 | value: 'proxyFromCredentials', 107 | description: 'Use the proxy defined in credentials for this request', 108 | }, 109 | { 110 | name: 'Proxy From HTTP Request Node', 111 | value: 'proxyFromNode', 112 | description: 'Use the proxy defined in HTTP Request Node for this request', 113 | }, 114 | { 115 | name: 'Proxy From Scrappey', 116 | value: 'proxyFromScrappey', 117 | description: 'Use the proxy defined in Scrappey for this request', 118 | }, 119 | ], 120 | default: 'proxyFromCredentials', 121 | displayOptions: { 122 | show: { 123 | scrappeyOperations: [ 124 | 'requestBuilder', 125 | 'httpRequestAutoRetry', 126 | 'httpRequestAutoRetryBrowser', 127 | ], 128 | }, 129 | }, 130 | }, 131 | { 132 | displayName: 'Proxy Type', 133 | name: 'proxyType', 134 | type: 'options', 135 | default: '', 136 | hint: 'Proxy type to use for the request', 137 | options: [ 138 | { 139 | name: 'Residential proxy', 140 | value: '', 141 | }, 142 | { 143 | name: 'Premium residential proxy', 144 | value: 'premiumProxy', 145 | }, 146 | { 147 | name: 'Datacenter proxy', 148 | value: 'datacenter', 149 | }, 150 | { 151 | name: 'Mobile proxy', 152 | value: 'mobileProxy', 153 | }, 154 | ], 155 | displayOptions: { 156 | show: { 157 | scrappeyOperations: [ 158 | 'requestBuilder', 159 | 'httpRequestAutoRetry', 160 | 'httpRequestAutoRetryBrowser', 161 | ], 162 | whichProxyToUse: ['proxyFromScrappey'], 163 | }, 164 | }, 165 | }, 166 | { 167 | displayName: 'Custom proxy country', 168 | name: 'customProxyCountryBoolean', 169 | type: 'boolean', 170 | default: false, 171 | hint: 'Use a custom proxy country', 172 | required: false, 173 | displayOptions: { 174 | show: { 175 | scrappeyOperations: [ 176 | 'requestBuilder', 177 | 'httpRequestAutoRetry', 178 | 'httpRequestAutoRetryBrowser', 179 | ], 180 | whichProxyToUse: ['proxyFromScrappey'], 181 | }, 182 | }, 183 | }, 184 | { 185 | displayName: 'Custom Proxy Country', 186 | name: 'customProxyCountry', 187 | type: 'options', 188 | options: Static_Country_Proxies, 189 | default: '', 190 | hint: 'Specify a country for the proxy to use with this request', 191 | required: false, 192 | displayOptions: { 193 | show: { 194 | scrappeyOperations: [ 195 | 'requestBuilder', 196 | 'httpRequestAutoRetry', 197 | 'httpRequestAutoRetryBrowser', 198 | ], 199 | customProxyCountryBoolean: [true], 200 | }, 201 | }, 202 | }, 203 | { 204 | displayName: 'Custom Proxy', 205 | name: 'custom_proxy', 206 | type: 'boolean', 207 | default: false, 208 | hint: 'When enabled, the proxy defined in credentials will be used for this request.', 209 | required: false, 210 | displayOptions: { 211 | show: { 212 | scrappeyOperations: ['requestBuilder'], 213 | proxyType: [''], 214 | whichProxyToUse: ['proxyFromScrappey'], 215 | }, 216 | }, 217 | }, 218 | { 219 | displayName: 'Body OR Params?', 220 | name: 'bodyOrParams', 221 | type: 'options', 222 | default: 'params_used', 223 | hint: 'Select whether to use Body or Params for the request', 224 | options: [ 225 | { 226 | name: 'Body', 227 | value: 'body_used', 228 | }, 229 | { 230 | name: 'Params', 231 | value: 'params_used', 232 | }, 233 | ], 234 | displayOptions: { 235 | show: { 236 | scrappeyOperations: ['requestBuilder'], 237 | httpMethod: [ 238 | 'request.put', 239 | 'request.post', 240 | 'request.patch', 241 | 'request.delete', 242 | 'request.publish', 243 | ], 244 | }, 245 | }, 246 | }, 247 | { 248 | displayName: 'Params', 249 | name: 'params_for_request', 250 | type: 'string', 251 | default: '', 252 | hint: 'Parameters to use for the request', 253 | displayOptions: { 254 | show: { 255 | bodyOrParams: ['params_used'], 256 | scrappeyOperations: ['requestBuilder'], 257 | httpMethod: [ 258 | 'request.put', 259 | 'request.post', 260 | 'request.patch', 261 | 'request.delete', 262 | 'request.publish', 263 | ], 264 | }, 265 | }, 266 | placeholder: 'g-recaptcha-response=03AGdBq24JZ&submit=Submit', 267 | }, 268 | { 269 | displayName: 'Body', 270 | name: 'body_for_request', 271 | type: 'string', 272 | default: '', 273 | hint: 'Body to use for the request', 274 | displayOptions: { 275 | show: { 276 | bodyOrParams: ['body_used'], 277 | scrappeyOperations: ['requestBuilder'], 278 | httpMethod: [ 279 | 'request.put', 280 | 'request.post', 281 | 'request.patch', 282 | 'request.delete', 283 | 'request.publish', 284 | ], 285 | }, 286 | }, 287 | typeOptions: { 288 | rows: 4, 289 | editor: 'jsEditor', 290 | }, 291 | }, 292 | { 293 | displayName: 'User Session', 294 | name: 'userSession', 295 | type: 'string', 296 | default: generateUUID(), 297 | hint: 'User session identifier to use for the request', 298 | required: false, 299 | displayOptions: { 300 | show: { 301 | scrappeyOperations: ['requestBuilder'], 302 | }, 303 | }, 304 | typeOptions: { 305 | loadOptionsDependsOn: ['refreshSession'], 306 | }, 307 | }, 308 | 309 | { 310 | displayName: 'Headers Input Method', 311 | name: 'headersInputMethod', 312 | type: 'options', 313 | default: 'fields', 314 | hint: 'Choose how to input headers', 315 | options: [ 316 | { 317 | name: 'Using Fields Below', 318 | value: 'fields', 319 | }, 320 | { 321 | name: 'Using JSON', 322 | value: 'json', 323 | }, 324 | ], 325 | required: false, 326 | displayOptions: { 327 | show: { 328 | scrappeyOperations: ['requestBuilder'], 329 | }, 330 | }, 331 | }, 332 | { 333 | displayName: 'Custom Headers', 334 | name: 'customHeaders', 335 | type: 'fixedCollection', 336 | default: {}, 337 | hint: 'Custom headers to use for the request', 338 | options: [ 339 | { 340 | name: 'headers', 341 | displayName: 'Headers', 342 | values: [ 343 | { 344 | displayName: 'Header Key', 345 | name: 'header_key', 346 | type: 'string', 347 | default: '', 348 | }, 349 | { 350 | displayName: 'Header Value', 351 | name: 'header_value', 352 | type: 'string', 353 | default: '', 354 | description: 'Value to set for the header key.', 355 | }, 356 | ], 357 | }, 358 | ], 359 | required: false, 360 | displayOptions: { 361 | show: { 362 | scrappeyOperations: ['requestBuilder'], 363 | headersInputMethod: ['fields'], 364 | }, 365 | }, 366 | typeOptions: { 367 | multipleValues: true, 368 | }, 369 | }, 370 | { 371 | displayName: 'JSON Headers', 372 | name: 'jsonHeaders', 373 | type: 'string', 374 | default: '{"User-Agent": "Mozilla/5.0", "Accept": "application/json"}', 375 | hint: 'Enter headers as a JSON object', 376 | required: false, 377 | displayOptions: { 378 | show: { 379 | scrappeyOperations: ['requestBuilder'], 380 | headersInputMethod: ['json'], 381 | }, 382 | }, 383 | typeOptions: { 384 | rows: 4, 385 | }, 386 | }, 387 | { 388 | displayName: 'One String Cookie', 389 | name: 'oneStringCookie', 390 | type: 'boolean', 391 | default: false, 392 | hint: 'Use a single string format for cookies', 393 | required: false, 394 | displayOptions: { 395 | show: { 396 | scrappeyOperations: ['requestBuilder'], 397 | }, 398 | }, 399 | }, 400 | { 401 | displayName: 'Single String Cookie', 402 | name: 'cookie', 403 | type: 'string', 404 | default: '', 405 | placeholder: 'sessionid=abc123;csrftoken=xyz456;theme=light', 406 | hint: 'Cookie string to use for the request (format: name=value;name2=value2)', 407 | required: false, 408 | 409 | displayOptions: { 410 | show: { 411 | scrappeyOperations: ['requestBuilder'], 412 | oneStringCookie: [true], 413 | }, 414 | }, 415 | }, 416 | { 417 | displayName: 'Custom Cookies', 418 | name: 'customCookies', 419 | type: 'fixedCollection', 420 | default: {}, 421 | hint: 'Custom cookies to use for the request', 422 | options: [ 423 | { 424 | name: 'cookies', 425 | displayName: 'Cookies', 426 | values: [ 427 | { 428 | displayName: 'Cookie Key', 429 | name: 'cookie_key', 430 | type: 'string', 431 | placeholder: 'theme', 432 | default: '', 433 | }, 434 | { 435 | displayName: 'Cookie Value', 436 | name: 'cookie_value', 437 | type: 'string', 438 | default: '', 439 | placeholder: 'dark', 440 | description: 'Value to set for the cookie key.', 441 | }, 442 | ], 443 | }, 444 | ], 445 | required: false, 446 | displayOptions: { 447 | show: { 448 | scrappeyOperations: ['requestBuilder'], 449 | oneStringCookie: [false], 450 | }, 451 | }, 452 | typeOptions: { 453 | multipleValues: true, 454 | }, 455 | }, 456 | { 457 | displayName: 'Datadome', 458 | name: 'datadome', 459 | type: 'boolean', 460 | default: false, 461 | hint: 'Enable Datadome protection bypass. Get the best results by selecting a preconfigured option. Advanced includes all common antibot protections.', 462 | displayOptions: { 463 | show: { 464 | scrappeyOperations: ['requestBuilder'], 465 | request_type: ['Browser'], 466 | }, 467 | }, 468 | }, 469 | { 470 | displayName: 'Attempts', 471 | name: 'attempts', 472 | type: 'number', 473 | default: 1, 474 | hint: 'Number of attempts to make the request if it fails', 475 | typeOptions: { 476 | minValue: 1, 477 | maxValue: 3, 478 | }, 479 | required: false, 480 | displayOptions: { 481 | show: { 482 | scrappeyOperations: ['requestBuilder'], 483 | }, 484 | }, 485 | }, 486 | ]; 487 | 488 | export const AdvancedSettingsForBrowser: INodeProperties[] = [ 489 | { 490 | displayName: 'Antibot', 491 | name: 'antibot', 492 | type: 'boolean', 493 | default: false, 494 | hint: 'Enable automatic solving of hCaptcha and reCAPTCHA challenges', 495 | required: false, 496 | displayOptions: { 497 | show: { 498 | scrappeyOperations: ['requestBuilder'], 499 | request_type: ['Browser'], 500 | }, 501 | }, 502 | }, 503 | { 504 | displayName: 'Add Random mouse movement', 505 | name: 'addRandomMouseMovement', 506 | type: 'boolean', 507 | default: false, 508 | hint: 'Add random mouse movements to simulate human interaction during the session', 509 | required: false, 510 | displayOptions: { 511 | show: { 512 | scrappeyOperations: ['requestBuilder'], 513 | request_type: ['Browser'], 514 | }, 515 | }, 516 | }, 517 | { 518 | displayName: 'Record Video Session', 519 | name: 'recordVideoSession', 520 | type: 'boolean', 521 | default: false, 522 | hint: 'Record a video of the browser session for debugging purposes', 523 | required: false, 524 | displayOptions: { 525 | show: { 526 | scrappeyOperations: ['requestBuilder'], 527 | request_type: ['Browser'], 528 | }, 529 | }, 530 | }, 531 | 532 | { 533 | displayName: 'CSS Selector', 534 | name: 'cssSelector', 535 | type: 'string', 536 | default: '', 537 | placeholder: 538 | 'div[class="px-mobile-1 px-tablet-1 pt-mobile-0 pt-desktop-6 pt-tablet-6 pt-widescreen-6 pb-mobile-7 pb-desktop-6 pb-tablet-6 pb-widescreen-6"]', 539 | hint: 'CSS selector to target specific elements on the page', 540 | required: false, 541 | displayOptions: { 542 | show: { 543 | scrappeyOperations: ['requestBuilder'], 544 | request_type: ['Browser'], 545 | }, 546 | }, 547 | }, 548 | { 549 | displayName: 'Href (Optional)', 550 | name: 'href', 551 | type: 'string', 552 | default: '', 553 | placeholder: 'https://example.com', 554 | hint: 'URL to navigate to when the CSS selector is used', 555 | required: false, 556 | displayOptions: { 557 | show: { 558 | scrappeyOperations: ['requestBuilder'], 559 | request_type: ['Browser'], 560 | }, 561 | }, 562 | }, 563 | { 564 | displayName: 'Intercept XHR/Fetch Request', 565 | name: 'interceptXhrFetchRequest', 566 | type: 'string', 567 | default: '', 568 | placeholder: 'https://example.com/api/v2/Test', 569 | hint: 'Intercept and return data from a specific XHR/Fetch request rather than the main page. For example, instead of returning google.com content, it will return the data from google.com/result.json in text format.', 570 | required: false, 571 | displayOptions: { 572 | show: { 573 | scrappeyOperations: ['requestBuilder'], 574 | request_type: ['Browser'], 575 | }, 576 | }, 577 | }, 578 | ]; 579 | -------------------------------------------------------------------------------- /nodes/Scrappey/methods.ts: -------------------------------------------------------------------------------- 1 | import { IExecuteFunctions } from 'n8n-workflow'; 2 | import { handleBody, HTTPRequest_Extract_Parameters } from './requestBodyBuilder'; 3 | import type { ScrappeyRequestBody } from './types'; 4 | import { genericHttpRequest } from './GenericFunctions'; 5 | 6 | export const PostRequest = async function (this: IExecuteFunctions, itemIndex: number = 0) { 7 | const body = await handleBody(this, itemIndex); 8 | const response = await genericHttpRequest.call(this, 'POST', '', { body }); 9 | return response; 10 | }; 11 | 12 | export const AutoRetryTypeBrowser = async function (this: IExecuteFunctions, itemIndex: number = 0) { 13 | const prev_HTTPRequest = await HTTPRequest_Extract_Parameters(this, itemIndex); 14 | 15 | const proxyType = this.getNodeParameter('proxyType', itemIndex, '') as string; 16 | const whichProxyToUse = this.getNodeParameter( 17 | 'whichProxyToUse', 18 | itemIndex, 19 | 'proxyFromCredentials', 20 | ) as string; 21 | 22 | let body: ScrappeyRequestBody = { 23 | cmd: prev_HTTPRequest.cmd, 24 | url: prev_HTTPRequest.url as string, 25 | datadomeBypass: true, 26 | retries: 3, 27 | mouseMovements: true, 28 | automaticallySolveCaptchas: true, 29 | }; 30 | 31 | if (prev_HTTPRequest.processedHeaders) { 32 | body.customHeaders = prev_HTTPRequest.processedHeaders; 33 | } 34 | 35 | // Handle proxy settings based on the selected proxy source 36 | if (whichProxyToUse === 'proxyFromScrappey') { 37 | if (proxyType && proxyType.trim() !== '') { 38 | body[proxyType] = true; 39 | } 40 | 41 | const customProxyCountryBoolean = this.getNodeParameter( 42 | 'customProxyCountryBoolean', 43 | itemIndex, 44 | false, 45 | ) as boolean; 46 | 47 | if (customProxyCountryBoolean) { 48 | const customProxyCountry = this.getNodeParameter('customProxyCountry', itemIndex, '') as string; 49 | if (customProxyCountry && customProxyCountry.trim() !== '') { 50 | body.country = customProxyCountry; 51 | } 52 | } 53 | } else if (whichProxyToUse === 'proxyFromNode' && prev_HTTPRequest.processedProxy) { 54 | body.proxy = prev_HTTPRequest.processedProxy; 55 | } 56 | // For proxyFromCredentials, proxy is handled by the credentials 57 | 58 | if (prev_HTTPRequest.processedPostData) { 59 | body.postData = prev_HTTPRequest.processedPostData; 60 | 61 | if (prev_HTTPRequest.contentType && body.customHeaders) { 62 | body.customHeaders['content-type'] = prev_HTTPRequest.contentType; 63 | } 64 | } 65 | 66 | const response = await genericHttpRequest.call(this, 'POST', '', { body }); 67 | return response; 68 | }; 69 | 70 | export const AutoRetryTypeRequest = async function (this: IExecuteFunctions, itemIndex: number = 0) { 71 | const prev_HTTPRequest = await HTTPRequest_Extract_Parameters(this, itemIndex); 72 | 73 | const customProxyCountry = this.getNodeParameter('customProxyCountry', itemIndex, '') as string; 74 | const customProxyCountryBoolean = this.getNodeParameter( 75 | 'customProxyCountryBoolean', 76 | itemIndex, 77 | false, 78 | ) as boolean; 79 | const proxyType = this.getNodeParameter('proxyType', itemIndex, '') as string; 80 | 81 | let body: ScrappeyRequestBody = { 82 | cmd: prev_HTTPRequest.cmd, 83 | url: prev_HTTPRequest.url as string, 84 | requestType: 'request', // Add this to ensure it's a request type 85 | }; 86 | 87 | if (prev_HTTPRequest.processedHeaders) { 88 | body.customHeaders = prev_HTTPRequest.processedHeaders; 89 | } 90 | 91 | const whichProxyToUse = this.getNodeParameter( 92 | 'whichProxyToUse', 93 | itemIndex, 94 | 'proxyFromCredentials', 95 | ) as string; 96 | 97 | if (whichProxyToUse === 'proxyFromNode' && prev_HTTPRequest.processedProxy) { 98 | body.proxy = prev_HTTPRequest.processedProxy; 99 | } else if (whichProxyToUse === 'proxyFromScrappey') { 100 | if (customProxyCountryBoolean) { 101 | body.proxyCountry = customProxyCountry; 102 | } 103 | 104 | if (proxyType && proxyType.trim() !== '') { 105 | body[proxyType] = true; 106 | } 107 | } 108 | 109 | if (prev_HTTPRequest.processedPostData) { 110 | body.postData = prev_HTTPRequest.processedPostData; 111 | 112 | if (prev_HTTPRequest.contentType) { 113 | if (!body.customHeaders) { 114 | body.customHeaders = {}; 115 | } 116 | body.customHeaders['content-type'] = prev_HTTPRequest.contentType; 117 | } 118 | } 119 | 120 | const response = await genericHttpRequest.call(this, 'POST', '', { body }); 121 | return response; 122 | }; -------------------------------------------------------------------------------- /nodes/Scrappey/operators.ts: -------------------------------------------------------------------------------- 1 | import { INodeProperties } from 'n8n-workflow'; 2 | export const scrappeyOperators: INodeProperties[] = [ 3 | { 4 | displayName: 'Scrappey Operations', 5 | name: 'scrappeyOperations', 6 | type: 'options', 7 | default: 'requestBuilder', 8 | options: [ 9 | { 10 | name: 'Request Builder', 11 | value: 'requestBuilder', 12 | description: 13 | 'Create a customized HTTP or browser request with advanced configuration options', 14 | action: 'Build a request', 15 | }, 16 | { 17 | name: 'HTTP Request • Auto-Retry on Protection', 18 | value: 'httpRequestAutoRetry', 19 | description: 20 | 'Automatically retries an HTTP request when it is blocked by CAPTCHA, Cloudflare, or similar anti-bot measures, resending the identical payload, headers, cookies, and proxy settings', 21 | action: 'Handle Error HTTPs Node (Request)', 22 | }, 23 | { 24 | name: 'Browser Request • Auto-Retry & Anti-Bot', 25 | value: 'httpRequestAutoRetryBrowser', 26 | description: 27 | 'Executes a browser-based request with built-in anti-bot techniques (movement emulation, hCaptcha/Cloudflare bypass, etc.) and automatically retries if protection pages are encountered', 28 | action: 'Handle Error HTTPs Node (Browser)', 29 | }, 30 | ], 31 | }, 32 | ]; 33 | -------------------------------------------------------------------------------- /nodes/Scrappey/requestBodyBuilder.ts: -------------------------------------------------------------------------------- 1 | import { IExecuteFunctions, NodeOperationError } from 'n8n-workflow'; 2 | type BodyEntry = Record< 3 | string, 4 | string | number | boolean | Object | BodyEntry[] | Record 5 | >; 6 | let body: BodyEntry = {}; 7 | 8 | // Function to process URLs with multiple expressions like {{ $json.key }} or {{ $node["NodeName"].json["key"] }} 9 | const processUrlExpressions = ( 10 | url: string, 11 | eFn: IExecuteFunctions, 12 | itemIndex: number = 0, 13 | ): string => { 14 | if (typeof url !== 'string') return String(url); 15 | 16 | let processedUrl = url; 17 | eFn.logger.info(`Processing URL: ${url} for item ${itemIndex}`); 18 | 19 | // If the URL starts with '=', we need special handling 20 | if (processedUrl.trim().startsWith('=')) { 21 | eFn.logger.info('URL starts with =, special handling required'); 22 | processedUrl = processedUrl.trim().substring(1); 23 | 24 | // Find all n8n expressions in the URL - handles both simple and complex patterns 25 | const expressionRegex = /{{\s*(.*?)\s*}}/g; 26 | let match; 27 | const expressions: string[] = []; 28 | 29 | while ((match = expressionRegex.exec(processedUrl)) !== null) { 30 | if (match[1]) { 31 | expressions.push(match[1]); 32 | eFn.logger.info(`Found expression: ${match[1]}`); 33 | } 34 | } 35 | 36 | if (expressions.length > 0) { 37 | eFn.logger.info('URL contains expressions, processing each one'); 38 | for (const expr of expressions) { 39 | try { 40 | // Create a proper n8n expression to evaluate 41 | const fullExpr = `={{ ${expr} }}`; 42 | eFn.logger.info(`Evaluating expression: ${fullExpr} for item ${itemIndex}`); 43 | 44 | const value = eFn.evaluateExpression(fullExpr, itemIndex); 45 | eFn.logger.info(`Evaluated value: ${value}`); 46 | 47 | let stringValue = value !== undefined && value !== null ? String(value) : ''; 48 | 49 | // Remove any leading equals sign from the evaluated value to prevent double equals in URLs 50 | if (stringValue.startsWith('=')) { 51 | stringValue = stringValue.substring(1); 52 | eFn.logger.info(`Removed leading equals sign, new value: ${stringValue}`); 53 | } 54 | 55 | processedUrl = processedUrl.replace( 56 | new RegExp(`{{\\s*${expr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*}}`, 'g'), 57 | stringValue, 58 | ); 59 | } catch (error) { 60 | eFn.logger.error(`Error evaluating expression ${expr} for item ${itemIndex}`, error); 61 | // If evaluation fails, replace with empty string 62 | processedUrl = processedUrl.replace( 63 | new RegExp(`{{\\s*${expr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*}}`, 'g'), 64 | '', 65 | ); 66 | } 67 | } 68 | eFn.logger.info(`Final processed URL for item ${itemIndex}: ${processedUrl}`); 69 | return processedUrl; 70 | } else { 71 | // No expressions found, return the URL without the = prefix 72 | return processedUrl; 73 | } 74 | } 75 | 76 | // For URLs without '=' prefix, find and process expressions 77 | const expressionRegex = /{{\s*(.*?)\s*}}/g; 78 | let match; 79 | const expressions: string[] = []; 80 | 81 | while ((match = expressionRegex.exec(processedUrl)) !== null) { 82 | if (match[1]) { 83 | expressions.push(match[1]); 84 | eFn.logger.info(`Found expression: ${match[1]}`); 85 | } 86 | } 87 | 88 | if (expressions.length > 0) { 89 | eFn.logger.info('URL contains expressions, processing each one'); 90 | for (const expr of expressions) { 91 | try { 92 | // Create a proper n8n expression to evaluate 93 | const fullExpr = `={{ ${expr} }}`; 94 | eFn.logger.info(`Evaluating expression: ${fullExpr} for item ${itemIndex}`); 95 | 96 | const value = eFn.evaluateExpression(fullExpr, itemIndex); 97 | eFn.logger.info(`Evaluated value: ${value}`); 98 | 99 | let stringValue = value !== undefined && value !== null ? String(value) : ''; 100 | 101 | // Remove any leading equals sign from the evaluated value to prevent double equals in URLs 102 | if (stringValue.startsWith('=')) { 103 | stringValue = stringValue.substring(1); 104 | eFn.logger.info(`Removed leading equals sign, new value: ${stringValue}`); 105 | } 106 | 107 | processedUrl = processedUrl.replace( 108 | new RegExp(`{{\\s*${expr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*}}`, 'g'), 109 | stringValue, 110 | ); 111 | } catch (error) { 112 | eFn.logger.error(`Error evaluating expression ${expr} for item ${itemIndex}`, error); 113 | // If evaluation fails, replace with empty string 114 | processedUrl = processedUrl.replace( 115 | new RegExp(`{{\\s*${expr.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\s*}}`, 'g'), 116 | '', 117 | ); 118 | } 119 | } 120 | } 121 | 122 | eFn.logger.info(`Final processed URL for item ${itemIndex}: ${processedUrl}`); 123 | return processedUrl; 124 | }; 125 | 126 | const Request_Type_Choice = (choice: string, eFn: IExecuteFunctions, itemIndex: number) => { 127 | switch (choice) { 128 | case 'Browser': 129 | handleAdvancedBrowser(eFn, itemIndex); 130 | return body; 131 | case 'Request': 132 | body.requestType = 'request'; 133 | return body; 134 | 135 | case 'PatchedChrome': 136 | body.browser = [ 137 | { 138 | name: 'chrome', 139 | }, 140 | ]; 141 | body.noDriver = true; 142 | return body; 143 | 144 | default: 145 | return body; 146 | } 147 | }; 148 | 149 | const handleAdvancedBrowser = (eFn: IExecuteFunctions, itemIndex: number) => { 150 | const antibot = eFn.getNodeParameter('antibot', itemIndex, false) as boolean; 151 | const addRandomMouseMovement = eFn.getNodeParameter( 152 | 'addRandomMouseMovement', 153 | itemIndex, 154 | false, 155 | ) as boolean; 156 | const recordVideoSession = eFn.getNodeParameter('recordVideoSession', itemIndex, false) as boolean; 157 | const cssSelector = eFn.getNodeParameter('cssSelector', itemIndex, '') as string; 158 | const href = eFn.getNodeParameter('href', itemIndex, '') as string; 159 | const interceptXhrFetchRequest = eFn.getNodeParameter( 160 | 'interceptXhrFetchRequest', 161 | itemIndex, 162 | '', 163 | ) as string; 164 | 165 | if (antibot) body.automaticallySolveCaptchas = true; 166 | 167 | if (addRandomMouseMovement) body.mouseMovements = true; 168 | 169 | if (recordVideoSession) body.video = true; 170 | 171 | if (cssSelector && cssSelector.trim() !== '') body.cssSelector = cssSelector; 172 | 173 | if (href && href.trim() !== '') body.customAttribute = href; 174 | 175 | if (interceptXhrFetchRequest && interceptXhrFetchRequest.trim() !== '') 176 | body.interceptFetchRequest = interceptXhrFetchRequest; 177 | }; 178 | 179 | export const handleBody = async (eFn: IExecuteFunctions, itemIndex: number = 0) => { 180 | const credentials = await eFn.getCredentials('scrappeyApi'); 181 | // Reset body object for each call 182 | body = {}; 183 | 184 | const request_type = eFn.getNodeParameter('request_type', itemIndex, 'Request') as string; 185 | body = Request_Type_Choice(request_type, eFn, itemIndex); 186 | 187 | let url = eFn.getNodeParameter('url', itemIndex, '') as string; 188 | const httpMethod = eFn.getNodeParameter('httpMethod', itemIndex, '') as string; 189 | const proxyType = eFn.getNodeParameter('proxyType', itemIndex, '') as string; 190 | const bodyOrParams = eFn.getNodeParameter('bodyOrParams', itemIndex, '') as string; 191 | const params_for_request = eFn.getNodeParameter('params_for_request', itemIndex, '') as string; 192 | const body_for_request = eFn.getNodeParameter('body_for_request', itemIndex, '') as string; 193 | const userSession = eFn.getNodeParameter('userSession', itemIndex, '') as string; 194 | const headersInputMethod = eFn.getNodeParameter('headersInputMethod', itemIndex, 'fields') as string; 195 | const customHeaders = eFn.getNodeParameter('customHeaders', itemIndex, {}) as Record; 196 | const jsonHeaders = eFn.getNodeParameter('jsonHeaders', itemIndex, '') as string; 197 | const customCookies = eFn.getNodeParameter('customCookies', itemIndex, {}) as Record; 198 | const customProxyCountry = eFn.getNodeParameter('customProxyCountry', itemIndex, '') as string; 199 | const customProxyCountryBoolean = eFn.getNodeParameter( 200 | 'customProxyCountryBoolean', 201 | itemIndex, 202 | false, 203 | ) as boolean; 204 | const customProxy = eFn.getNodeParameter('custom_proxy', itemIndex, false) as boolean; 205 | const whichProxyToUse = eFn.getNodeParameter('whichProxyToUse', itemIndex, 'proxyFromCredentials') as string; 206 | const attempts = eFn.getNodeParameter('attempts', itemIndex, 3) as number; 207 | const datadome = eFn.getNodeParameter('datadome', itemIndex, false) as boolean; 208 | const oneStringCookie = eFn.getNodeParameter('oneStringCookie', itemIndex, false) as boolean; 209 | const cookie = eFn.getNodeParameter('cookie', itemIndex, '') as string; 210 | 211 | if (url && url.trim() !== '') { 212 | // Process URL expressions - starts with '=' or contains {{ $json.key }} 213 | url = processUrlExpressions(url, eFn, itemIndex); 214 | 215 | if (url.endsWith('/')) { 216 | url = url.slice(0, -1); 217 | } 218 | 219 | body.url = url; 220 | } 221 | 222 | if (httpMethod && httpMethod.trim() !== '') body.cmd = httpMethod; 223 | 224 | if (proxyType && proxyType.trim() !== '') body[proxyType] = true; 225 | 226 | if (userSession && userSession.trim() !== '') body.session = userSession; 227 | 228 | if (httpMethod !== 'request.get') { 229 | if (bodyOrParams === 'body_used') { 230 | body.postData = body_for_request; 231 | body.customHeaders = { 232 | 'content-type': 'application/json', 233 | }; 234 | } else { 235 | body.postData = params_for_request; 236 | } 237 | } 238 | 239 | // Handle headers based on input method 240 | if (headersInputMethod === 'fields') { 241 | if (customHeaders && Object.keys(customHeaders).length > 0) { 242 | const headersObj: Record = {}; 243 | 244 | if (customHeaders.headers && Array.isArray(customHeaders.headers)) { 245 | customHeaders.headers.forEach((header: any) => { 246 | if (header.header_key && header.header_value) { 247 | headersObj[header.header_key] = header.header_value; 248 | } 249 | }); 250 | body.customHeaders = headersObj; 251 | } 252 | } 253 | } else if (headersInputMethod === 'json') { 254 | if (jsonHeaders && jsonHeaders.trim() !== '') { 255 | try { 256 | const headersObj = JSON.parse(jsonHeaders); 257 | if (headersObj && typeof headersObj === 'object') { 258 | body.customHeaders = headersObj; 259 | } 260 | } catch (error) { 261 | throw new NodeOperationError(eFn.getNode(), 'Invalid JSON headers format', { 262 | description: `The provided JSON headers are not valid: ${error instanceof Error ? error.message : 'Unknown error'}`, 263 | itemIndex // Include item index in error 264 | }); 265 | } 266 | } 267 | } 268 | 269 | // Handle cookies 270 | if (oneStringCookie) { 271 | if (cookie && cookie.trim() !== '') { 272 | body.cookies = cookie; 273 | } 274 | } else if (customCookies && Object.keys(customCookies).length > 0) { 275 | let cookieString = ''; 276 | 277 | if (customCookies.cookies && Array.isArray(customCookies.cookies)) { 278 | customCookies.cookies.forEach((cookie: any) => { 279 | if (cookie.cookie_key && cookie.cookie_value) { 280 | if (cookieString) cookieString += '; '; 281 | cookieString += `${cookie.cookie_key}=${cookie.cookie_value}`; 282 | } 283 | }); 284 | } else { 285 | for (const [key, value] of Object.entries(customCookies)) { 286 | if (cookieString) cookieString += '; '; 287 | cookieString += `${key}=${value}`; 288 | } 289 | } 290 | 291 | if (cookieString) { 292 | body.cookies = cookieString; 293 | } 294 | } 295 | 296 | if (customProxyCountryBoolean) body.proxyCountry = customProxyCountry; 297 | 298 | // Handle proxy configuration 299 | if (whichProxyToUse === 'proxyFromCredentials' && credentials?.proxyUrl) { 300 | body.proxy = credentials.proxyUrl as string; 301 | } else if (whichProxyToUse === 'proxyFromScrappey') { 302 | if (customProxy && credentials?.proxyUrl) { 303 | body.proxy = credentials.proxyUrl as string; 304 | } 305 | } 306 | 307 | if (datadome && request_type === 'Browser') { 308 | body.datadomeBypass = true; 309 | } 310 | 311 | body.attempts = attempts; 312 | 313 | if (credentials?.whitelistedDomains) { 314 | // Ensure whitelistedDomains is passed as an array 315 | const domains = Array.isArray(credentials.whitelistedDomains) 316 | ? credentials.whitelistedDomains 317 | : typeof credentials.whitelistedDomains === 'string' 318 | ? credentials.whitelistedDomains.split(',').map((domain) => domain.trim()) 319 | : []; 320 | 321 | body.whitelistedDomains = domains; 322 | } 323 | 324 | return body; 325 | }; 326 | 327 | export const HTTPRequest_Extract_Parameters = async (eFn: IExecuteFunctions, itemIndex: number = 0) => { 328 | const method = eFn.getNodeParameter('method', itemIndex, '={{ $($prevNode.name).params.method }}'); 329 | 330 | const processedMethod = typeof method === 'string' ? method.toLowerCase() : method; 331 | const cmd = `request.${processedMethod}`; 332 | let urlRaw = eFn.getNodeParameter('url', itemIndex, '={{ $($prevNode.name).params.url }}') as string; 333 | 334 | urlRaw = processUrlExpressions(urlRaw, eFn, itemIndex); 335 | 336 | if (typeof urlRaw === 'string' && urlRaw.endsWith('/')) { 337 | urlRaw = urlRaw.slice(0, -1); 338 | } 339 | 340 | const url = urlRaw; 341 | 342 | const authentication = eFn.getNodeParameter( 343 | 'authentication', 344 | itemIndex, 345 | '={{ $($prevNode.name).params.authentication }}', 346 | ); 347 | const sendQuery = eFn.getNodeParameter( 348 | 'sendQuery', 349 | itemIndex, 350 | '={{ $($prevNode.name).params.sendQuery }}', 351 | ); 352 | 353 | const queryParameters = eFn.getNodeParameter( 354 | 'queryParameters', 355 | itemIndex, 356 | '={{ $($prevNode.name).params.queryParameters }}', 357 | ); 358 | const headerParameters = eFn.getNodeParameter( 359 | 'headerParameters', 360 | itemIndex, 361 | '={{ $($prevNode.name).params.headerParameters }}', 362 | ); 363 | const proxy = eFn.getNodeParameter('proxy', itemIndex, '={{ $($prevNode.name).params.options.proxy }}'); 364 | const bodyParameters = eFn.getNodeParameter( 365 | 'bodyParameters', 366 | itemIndex, 367 | '={{ $($prevNode.name).params.bodyParameters }}', 368 | ); 369 | 370 | let processedHeaders: Record | undefined = undefined; 371 | if (headerParameters) { 372 | const tempHeaders: Record = {}; 373 | 374 | if (typeof headerParameters === 'object') { 375 | const headerParams = headerParameters as any; 376 | 377 | if (Array.isArray(headerParams.parameters)) { 378 | for (const header of headerParams.parameters) { 379 | if (header.name && header.value) { 380 | tempHeaders[header.name] = header.value; 381 | } 382 | } 383 | } else if (typeof headerParams === 'object') { 384 | Object.assign(tempHeaders, headerParams); 385 | } 386 | } 387 | 388 | if (Object.keys(tempHeaders).length > 0) { 389 | processedHeaders = tempHeaders; 390 | } 391 | } 392 | 393 | let processedProxy: string | undefined; 394 | const whichProxyToUse = eFn.getNodeParameter( 395 | 'whichProxyToUse', 396 | itemIndex, 397 | 'proxyFromCredentials', 398 | ) as string; 399 | switch (whichProxyToUse) { 400 | case 'proxyFromCredentials': { 401 | const credentials = await eFn.getCredentials('scrappeyApi'); 402 | if (credentials?.proxyUrl) { 403 | processedProxy = String(credentials.proxyUrl); 404 | } 405 | break; 406 | } 407 | case 'proxyFromNode': { 408 | if (proxy) { 409 | processedProxy = proxy as string; 410 | } 411 | break; 412 | } 413 | case 'proxyFromScrappey': { 414 | // No processing needed here 415 | break; 416 | } 417 | } 418 | 419 | let processedPostData: string | undefined; 420 | let contentType: string | undefined; 421 | 422 | if (bodyParameters) { 423 | const bodyParams = bodyParameters; 424 | 425 | if ( 426 | typeof bodyParams === 'object' && 427 | bodyParams !== null && 428 | 'parameters' in bodyParams && 429 | Array.isArray(bodyParams.parameters) 430 | ) { 431 | const bodyParamsObj: Record = {}; 432 | for (const param of bodyParams.parameters) { 433 | if (param && typeof param === 'object' && 'name' in param && 'value' in param) { 434 | bodyParamsObj[param.name as string] = param.value as string; 435 | } 436 | } 437 | 438 | processedPostData = JSON.stringify(bodyParamsObj); 439 | contentType = 'application/json'; 440 | } else { 441 | processedPostData = typeof bodyParams === 'string' ? bodyParams : JSON.stringify(bodyParams); 442 | contentType = 'application/json'; 443 | } 444 | } 445 | 446 | return { 447 | method, 448 | cmd, 449 | url, 450 | authentication, 451 | proxy, 452 | processedProxy, 453 | processedHeaders, 454 | processedPostData, 455 | contentType, 456 | sendQuery, 457 | queryParameters, 458 | headerParameters, 459 | bodyParameters, 460 | }; 461 | }; -------------------------------------------------------------------------------- /nodes/Scrappey/types.ts: -------------------------------------------------------------------------------- 1 | export interface HTTPRequest_Body { 2 | cmd: string; 3 | method: string; 4 | url: string; 5 | datadomeBypass: string; 6 | retries: string; 7 | mouseMovements: string; 8 | automaticallySolveCaptchas: string; 9 | customHeaders: { [key: string]: string }; 10 | proxy: string; 11 | } 12 | 13 | export interface ScrappeyRequestBody { 14 | cmd: string; 15 | url: string; 16 | datadomeBypass?: boolean; 17 | retries?: number; 18 | mouseMovements?: boolean; 19 | automaticallySolveCaptchas?: boolean; 20 | customHeaders?: any; 21 | proxy?: string; 22 | bodyParameters?: any; 23 | postData?: string; 24 | proxyCountry?: string; 25 | // For proxy types and other dynamic properties 26 | [key: string]: string | boolean | number | any | undefined; 27 | } 28 | -------------------------------------------------------------------------------- /nodes/Scrappey/utils.ts: -------------------------------------------------------------------------------- 1 | import { IExecuteFunctions, INodePropertyOptions } from 'n8n-workflow'; 2 | 3 | export const Static_Country_Proxies: INodePropertyOptions[] = [ 4 | { name: 'Afghanistan', value: 'Afghanistan' }, 5 | { name: 'Albania', value: 'Albania' }, 6 | { name: 'Algeria', value: 'Algeria' }, 7 | { name: 'Argentina', value: 'Argentina' }, 8 | { name: 'Armenia', value: 'Armenia' }, 9 | { name: 'Aruba', value: 'Aruba' }, 10 | { name: 'Australia', value: 'Australia' }, 11 | { name: 'Austria', value: 'Austria' }, 12 | { name: 'Azerbaijan', value: 'Azerbaijan' }, 13 | { name: 'Bahamas', value: 'Bahamas' }, 14 | { name: 'Bahrain', value: 'Bahrain' }, 15 | { name: 'Bangladesh', value: 'Bangladesh' }, 16 | { name: 'Belarus', value: 'Belarus' }, 17 | { name: 'Belgium', value: 'Belgium' }, 18 | { name: 'Bosnia and Herzegovina', value: 'BosniaandHerzegovina' }, 19 | { name: 'Brazil', value: 'Brazil' }, 20 | { name: 'British Virgin Islands', value: 'BritishVirginIslands' }, 21 | { name: 'Brunei', value: 'Brunei' }, 22 | { name: 'Bulgaria', value: 'Bulgaria' }, 23 | { name: 'Cambodia', value: 'Cambodia' }, 24 | { name: 'Cameroon', value: 'Cameroon' }, 25 | { name: 'Canada', value: 'Canada' }, 26 | { name: 'Chile', value: 'Chile' }, 27 | { name: 'China', value: 'China' }, 28 | { name: 'Colombia', value: 'Colombia' }, 29 | { name: 'Costa Rica', value: 'CostaRica' }, 30 | { name: 'Croatia', value: 'Croatia' }, 31 | { name: 'Cuba', value: 'Cuba' }, 32 | { name: 'Cyprus', value: 'Cyprus' }, 33 | { name: 'Czechia', value: 'Czechia' }, 34 | { name: 'Denmark', value: 'Denmark' }, 35 | { name: 'Dominican Republic', value: 'DominicanRepublic' }, 36 | { name: 'Ecuador', value: 'Ecuador' }, 37 | { name: 'Egypt', value: 'Egypt' }, 38 | { name: 'El Salvador', value: 'ElSalvador' }, 39 | { name: 'Estonia', value: 'Estonia' }, 40 | { name: 'Ethiopia', value: 'Ethiopia' }, 41 | { name: 'Finland', value: 'Finland' }, 42 | { name: 'France', value: 'France' }, 43 | { name: 'Georgia', value: 'Georgia' }, 44 | { name: 'Germany', value: 'Germany' }, 45 | { name: 'Ghana', value: 'Ghana' }, 46 | { name: 'Greece', value: 'Greece' }, 47 | { name: 'Guatemala', value: 'Guatemala' }, 48 | { name: 'Guyana', value: 'Guyana' }, 49 | { name: 'Hashemite Kingdom of Jordan', value: 'HashemiteKingdomofJordan' }, 50 | { name: 'Hong Kong', value: 'HongKong' }, 51 | { name: 'Hungary', value: 'Hungary' }, 52 | { name: 'India', value: 'India' }, 53 | { name: 'Indonesia', value: 'Indonesia' }, 54 | { name: 'Iran', value: 'Iran' }, 55 | { name: 'Iraq', value: 'Iraq' }, 56 | { name: 'Ireland', value: 'Ireland' }, 57 | { name: 'Israel', value: 'Israel' }, 58 | { name: 'Italy', value: 'Italy' }, 59 | { name: 'Jamaica', value: 'Jamaica' }, 60 | { name: 'Japan', value: 'Japan' }, 61 | { name: 'Kazakhstan', value: 'Kazakhstan' }, 62 | { name: 'Kenya', value: 'Kenya' }, 63 | { name: 'Kosovo', value: 'Kosovo' }, 64 | { name: 'Kuwait', value: 'Kuwait' }, 65 | { name: 'Latvia', value: 'Latvia' }, 66 | { name: 'Liechtenstein', value: 'Liechtenstein' }, 67 | { name: 'Luxembourg', value: 'Luxembourg' }, 68 | { name: 'Macedonia', value: 'Macedonia' }, 69 | { name: 'Madagascar', value: 'Madagascar' }, 70 | { name: 'Malaysia', value: 'Malaysia' }, 71 | { name: 'Mauritius', value: 'Mauritius' }, 72 | { name: 'Mexico', value: 'Mexico' }, 73 | { name: 'Mongolia', value: 'Mongolia' }, 74 | { name: 'Montenegro', value: 'Montenegro' }, 75 | { name: 'Morocco', value: 'Morocco' }, 76 | { name: 'Mozambique', value: 'Mozambique' }, 77 | { name: 'Myanmar', value: 'Myanmar' }, 78 | { name: 'Nepal', value: 'Nepal' }, 79 | { name: 'Netherlands', value: 'Netherlands' }, 80 | { name: 'New Zealand', value: 'NewZealand' }, 81 | { name: 'Nigeria', value: 'Nigeria' }, 82 | { name: 'Norway', value: 'Norway' }, 83 | { name: 'Oman', value: 'Oman' }, 84 | { name: 'Pakistan', value: 'Pakistan' }, 85 | { name: 'Palestine', value: 'Palestine' }, 86 | { name: 'Panama', value: 'Panama' }, 87 | { name: 'Papua New Guinea', value: 'PapuaNewGuinea' }, 88 | { name: 'Paraguay', value: 'Paraguay' }, 89 | { name: 'Peru', value: 'Peru' }, 90 | { name: 'Philippines', value: 'Philippines' }, 91 | { name: 'Poland', value: 'Poland' }, 92 | { name: 'Portugal', value: 'Portugal' }, 93 | { name: 'Puerto Rico', value: 'PuertoRico' }, 94 | { name: 'Qatar', value: 'Qatar' }, 95 | { name: 'Republic of Lithuania', value: 'RepublicofLithuania' }, 96 | { name: 'Republic of Moldova', value: 'RepublicofMoldova' }, 97 | { name: 'Romania', value: 'Romania' }, 98 | { name: 'Russia', value: 'Russia' }, 99 | { name: 'Saudi Arabia', value: 'SaudiArabia' }, 100 | { name: 'Senegal', value: 'Senegal' }, 101 | { name: 'Serbia', value: 'Serbia' }, 102 | { name: 'Seychelles', value: 'Seychelles' }, 103 | { name: 'Singapore', value: 'Singapore' }, 104 | { name: 'Slovakia', value: 'Slovakia' }, 105 | { name: 'Slovenia', value: 'Slovenia' }, 106 | { name: 'Somalia', value: 'Somalia' }, 107 | { name: 'South Africa', value: 'SouthAfrica' }, 108 | { name: 'South Korea', value: 'SouthKorea' }, 109 | { name: 'Spain', value: 'Spain' }, 110 | { name: 'Sri Lanka', value: 'SriLanka' }, 111 | { name: 'Sudan', value: 'Sudan' }, 112 | { name: 'Suriname', value: 'Suriname' }, 113 | { name: 'Sweden', value: 'Sweden' }, 114 | { name: 'Switzerland', value: 'Switzerland' }, 115 | { name: 'Syria', value: 'Syria' }, 116 | { name: 'Taiwan', value: 'Taiwan' }, 117 | { name: 'Tajikistan', value: 'Tajikistan' }, 118 | { name: 'Thailand', value: 'Thailand' }, 119 | { name: 'Trinidad and Tobago', value: 'TrinidadandTobago' }, 120 | { name: 'Tunisia', value: 'Tunisia' }, 121 | { name: 'Turkey', value: 'Turkey' }, 122 | { name: 'Uganda', value: 'Uganda' }, 123 | { name: 'Ukraine', value: 'Ukraine' }, 124 | { name: 'United Arab Emirates', value: 'UnitedArabEmirates' }, 125 | { name: 'United Kingdom', value: 'UnitedKingdom' }, 126 | { name: 'United States', value: 'UnitedStates' }, 127 | { name: 'Uzbekistan', value: 'Uzbekistan' }, 128 | { name: 'Venezuela', value: 'Venezuela' }, 129 | { name: 'Vietnam', value: 'Vietnam' }, 130 | { name: 'Zambia', value: 'Zambia' }, 131 | ]; 132 | 133 | export const isExpression = (value: unknown): boolean => { 134 | if (typeof value !== 'string') return false; 135 | return value.trim().startsWith('={{') && value.trim().endsWith('}}'); 136 | }; 137 | export const evaluateExpression = ( 138 | eFn: IExecuteFunctions, 139 | value: unknown, 140 | itemIndex: number = 0, 141 | ): unknown => { 142 | if (!isExpression(value)) return value; 143 | 144 | const expressionString = (value as string).trim(); 145 | // Remove the '=' from '={{' as it's just a marker in the workflow JSON 146 | if (expressionString.startsWith('=')) { 147 | try { 148 | // Pass the expression without the leading '=' 149 | return eFn.evaluateExpression(expressionString.substring(1), itemIndex); 150 | } catch (error) { 151 | eFn.logger.warn(`Failed to evaluate expression: ${expressionString}`, error); 152 | return value; // Return original value if evaluation fails 153 | } 154 | } 155 | 156 | return value; 157 | }; 158 | export function generateUUID(): string { 159 | // Create an array of 16 random bytes. 160 | const randomBytes = new Uint8Array(16); 161 | crypto.getRandomValues(randomBytes); 162 | 163 | // Set the version number to 4. 164 | // The 7th byte (index 6) needs to have its most significant nibble set to 0100. 165 | randomBytes[6] = (randomBytes[6] & 0x0f) | 0x40; 166 | 167 | // Set the variant. 168 | // The 9th byte (index 8) needs to have its two most significant bits set to 10. 169 | randomBytes[8] = (randomBytes[8] & 0x3f) | 0x80; 170 | 171 | // Convert the byte array to a hexadecimal string. 172 | const byteToHex = (byte: number): string => { 173 | return ('0' + byte.toString(16)).slice(-2); 174 | }; 175 | 176 | const hexBytes = Array.from(randomBytes).map(byteToHex); 177 | 178 | // Format the hexadecimal string with hyphens. 179 | return [ 180 | hexBytes.slice(0, 4).join(''), 181 | hexBytes.slice(4, 6).join(''), 182 | hexBytes.slice(6, 8).join(''), 183 | hexBytes.slice(8, 10).join(''), 184 | hexBytes.slice(10).join(''), 185 | ].join('-'); 186 | } 187 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@nskha/n8n-nodes-scrappey", 3 | "version": "0.3.13", 4 | "description": "n8n node package for Scrappey API integration", 5 | "keywords": [ 6 | "n8n-community-node-package" 7 | ], 8 | "license": "MIT", 9 | "homepage": "", 10 | "author": { 11 | "name": "Nskha", 12 | "email": "github-public@admins.mozmail.com" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/Automations-Project/n8n-nodes-scrappey.git" 17 | }, 18 | "engines": { 19 | "node": ">=18.10", 20 | "pnpm": ">=9.1" 21 | }, 22 | "packageManager": "pnpm@9.1.4", 23 | "main": "index.js", 24 | "scripts": { 25 | "build": "tsc && gulp build:icons", 26 | "build:watch": "tsc --watch", 27 | "deploy": "node scripts/update-node-json.js && node scripts/deploy.js", 28 | "start:dev": "npm run build && mkdir -p ~/.n8n/custom/n8n-nodes-scrappey && cp -r dist package.json ~/.n8n/custom/n8n-nodes-scrappey/ && N8N_CUSTOM_EXTENSIONS=~/.n8n/custom n8n start", 29 | "clean": "rm -rf node_modules pnpm-lock.yaml package-lock.json dist", 30 | "lint": "eslint . --ext .ts,.js", 31 | "lint:fix": "eslint . --ext .ts,.js --fix", 32 | "format": "prettier --write .", 33 | "format:check": "prettier --check .", 34 | "type-check": "tsc --noEmit", 35 | "prepack": "npm run build", 36 | "test": "echo \"Warning: No tests specified. Please add tests for better CI/CD.\" && exit 0", 37 | "validate": "npm run lint && npm run format:check && npm run type-check && npm run build", 38 | "check-versions": "node scripts/check-versions.js", 39 | "check-workflows": "node scripts/check-yaml.js", 40 | "fix-lockfile": "rm -f pnpm-lock.yaml package-lock.json && pnpm install && echo \"✅ Lockfile regenerated successfully\"", 41 | "scan:n8n": "node scripts/n8n-scanner.js" 42 | }, 43 | "files": [ 44 | "dist" 45 | ], 46 | "n8n": { 47 | "n8nNodesApiVersion": 1, 48 | "credentials": [ 49 | "dist/credentials/ScrappeyApi.credentials.js" 50 | ], 51 | "nodes": [ 52 | "dist/nodes/Scrappey/Scrappey.node.js" 53 | ] 54 | }, 55 | "devDependencies": { 56 | "@typescript-eslint/parser": "^7.15.0", 57 | "eslint": "^8.56.0", 58 | "eslint-plugin-n8n-nodes-base": "^1.16.1", 59 | "gulp": "^4.0.2", 60 | "js-yaml": "^4.1.0", 61 | "n8n": "^1.88.0", 62 | "prettier": "^3.3.2", 63 | "typescript": "^5.5.3" 64 | }, 65 | "peerDependencies": { 66 | "n8n-workflow": "*" 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /scripts/check-versions.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { execSync } = require('child_process'); 4 | const fs = require('fs'); 5 | 6 | function execSafe(command) { 7 | try { 8 | return execSync(command, { encoding: 'utf-8' }).trim(); 9 | } catch (error) { 10 | return null; 11 | } 12 | } 13 | 14 | function getVersions() { 15 | const pkg = JSON.parse(fs.readFileSync('package.json', 'utf-8')); 16 | 17 | const versions = { 18 | packageJson: pkg.version, 19 | npmPublished: execSafe('npm view @nskha/n8n-nodes-scrappey version'), 20 | githubPublished: execSafe('npm view @automations-project/n8n-nodes-scrappey version --registry=https://npm.pkg.github.com'), 21 | latestGitTag: execSafe('git describe --tags --abbrev=0') || 'none' 22 | }; 23 | 24 | return versions; 25 | } 26 | 27 | function main() { 28 | console.log('🔍 Current Version Status:'); 29 | console.log('========================'); 30 | 31 | const versions = getVersions(); 32 | 33 | console.log(`📦 Package.json: ${versions.packageJson}`); 34 | console.log(`📦 NPM Published: ${versions.npmPublished || 'Not found'}`); 35 | console.log(`📦 GitHub Published: ${versions.githubPublished || 'Not found'}`); 36 | console.log(`🏷️ Latest Git Tag: ${versions.latestGitTag.replace('v', '') || 'none'}`); 37 | 38 | // Find highest version 39 | const allVersions = Object.values(versions) 40 | .filter(v => v && v !== 'none' && v !== 'Not found') 41 | .map(v => v.replace('v', '')); 42 | 43 | if (allVersions.length > 0) { 44 | const highest = allVersions.sort((a, b) => { 45 | const aParts = a.split('.').map(Number); 46 | const bParts = b.split('.').map(Number); 47 | 48 | for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) { 49 | const aPart = aParts[i] || 0; 50 | const bPart = bParts[i] || 0; 51 | if (aPart !== bPart) return bPart - aPart; 52 | } 53 | return 0; 54 | })[0]; 55 | 56 | console.log(`🔍 Highest Version: ${highest}`); 57 | 58 | // Suggest next version 59 | const [major, minor, patch] = highest.split('.').map(Number); 60 | console.log('\n💡 Next Versions:'); 61 | console.log(` Patch: ${major}.${minor}.${patch + 1}`); 62 | console.log(` Minor: ${major}.${minor + 1}.0`); 63 | console.log(` Major: ${major + 1}.0.0`); 64 | } 65 | } 66 | 67 | if (require.main === module) { 68 | main(); 69 | } 70 | 71 | module.exports = { getVersions }; 72 | -------------------------------------------------------------------------------- /scripts/check-yaml.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const yaml = require('js-yaml'); 4 | const fs = require('fs'); 5 | const path = require('path'); 6 | 7 | const workflowDir = '.github/workflows'; 8 | const files = [ 9 | 'auto-version.yml', 10 | 'release.yml', 11 | 'ci.yml' 12 | ]; 13 | 14 | console.log('🔍 Checking YAML syntax...\n'); 15 | 16 | let allValid = true; 17 | 18 | files.forEach(file => { 19 | const filePath = path.join(workflowDir, file); 20 | try { 21 | if (fs.existsSync(filePath)) { 22 | const content = fs.readFileSync(filePath, 'utf8'); 23 | yaml.load(content); 24 | console.log(`✅ ${file} - syntax OK`); 25 | } else { 26 | console.log(`⚠️ ${file} - file not found`); 27 | } 28 | } catch (error) { 29 | console.log(`❌ ${file} - syntax error:`); 30 | console.log(` ${error.message}`); 31 | allValid = false; 32 | } 33 | }); 34 | 35 | console.log('\n' + '='.repeat(40)); 36 | if (allValid) { 37 | console.log('🎉 All workflow files have valid YAML syntax!'); 38 | process.exit(0); 39 | } else { 40 | console.log('💥 Some workflow files have syntax errors!'); 41 | process.exit(1); 42 | } 43 | -------------------------------------------------------------------------------- /scripts/deploy.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { execSync } = require('child_process'); 4 | const fs = require('fs'); 5 | const path = require('path'); 6 | const readline = require('readline'); 7 | 8 | // --- Configuration --- 9 | const GITHUB_REGISTRY_URL = 'https://npm.pkg.github.com'; 10 | const NPMRC_FILENAME = '.npmrc'; 11 | const PACKAGE_JSON_FILENAME = 'package.json'; 12 | const CONFIG_FILENAME = '.PublishKey'; // Configuration file name 13 | 14 | // --- Helper Functions --- 15 | 16 | function printError(message) { 17 | console.error(`❌ Error: ${message}`); 18 | } 19 | 20 | function printSuccess(message) { 21 | console.log(`✅ ${message}`); 22 | } 23 | 24 | function printInfo(message) { 25 | console.log(`ℹ️ ${message}`); 26 | } 27 | 28 | const rl = readline.createInterface({ 29 | input: process.stdin, 30 | output: process.stdout, 31 | }); 32 | 33 | function question(query) { 34 | return new Promise((resolve) => rl.question(query, resolve)); 35 | } 36 | 37 | function hiddenQuestion(query) { 38 | return new Promise((resolve) => { 39 | const PWD = Symbol('password'); 40 | rl.stdoutMuted = true; 41 | 42 | rl.query = query; 43 | rl._writeToOutput = function _writeToOutput(stringToWrite) { 44 | if (rl.stdoutMuted) { 45 | if (rl.line.length === 0) { 46 | rl.output.write(rl.query); 47 | } 48 | } else { 49 | rl.output.write(stringToWrite); 50 | } 51 | }; 52 | 53 | rl.question(query, (value) => { 54 | rl[PWD] = value; 55 | rl.stdoutMuted = false; 56 | rl._writeToOutput = rl.constructor.prototype._writeToOutput; 57 | rl.history = rl.history.slice(1); 58 | rl.output.write('\n'); 59 | resolve(rl[PWD]); 60 | }); 61 | }); 62 | } 63 | 64 | function runCommand(command, args) { 65 | const fullCommand = `${command} ${args.join(' ')}`; 66 | printInfo(`Running command: ${fullCommand}`); 67 | try { 68 | execSync(fullCommand, { stdio: 'inherit' }); 69 | return true; 70 | } catch (error) { 71 | printError(`Command failed: ${fullCommand}`); 72 | return false; 73 | } 74 | } 75 | 76 | /** 77 | * Reads package.json, updates the name to include or replace the scope if necessary, 78 | * writes it back, verifies the write, and validates the final name. 79 | * @param {string} expectedScope - The GitHub organization or username. 80 | * @returns {boolean | null} True if successful, false on validation/write/verify error, null if file not found/read error. 81 | */ 82 | function updateAndValidatePackageJson(expectedScope) { 83 | const packageJsonPath = path.resolve(process.cwd(), PACKAGE_JSON_FILENAME); 84 | let packageData; 85 | let originalPackageName; 86 | let needsUpdate = false; 87 | let finalName = ''; // Store the name we intend to write 88 | 89 | if (!fs.existsSync(packageJsonPath)) { 90 | printError(`'${PACKAGE_JSON_FILENAME}' not found in the current directory (${process.cwd()}).`); 91 | return null; 92 | } 93 | 94 | // Read and Parse 95 | try { 96 | const packageContent = fs.readFileSync(packageJsonPath, 'utf-8'); 97 | packageData = JSON.parse(packageContent); 98 | originalPackageName = packageData.name; 99 | 100 | if (!originalPackageName) { 101 | printError(`'name' field missing in '${PACKAGE_JSON_FILENAME}'. Cannot proceed.`); 102 | return false; 103 | } 104 | } catch (error) { 105 | if (error instanceof SyntaxError) { 106 | printError(`Could not parse '${PACKAGE_JSON_FILENAME}'. Make sure it's valid JSON.`); 107 | } else { 108 | printError(`Could not read or parse '${PACKAGE_JSON_FILENAME}': ${error.message}`); 109 | } 110 | return null; // Indicate read/parse error 111 | } 112 | 113 | // Check and Prepare Update 114 | const requiredPrefix = `@${expectedScope}/`; 115 | const nameParts = originalPackageName.split('/'); // Split by '/' to check scope and package part 116 | const currentPackagePart = nameParts.length > 1 ? nameParts[1] : nameParts[0]; // Get the part after potential scope 117 | 118 | if (!originalPackageName.startsWith('@')) { 119 | // --- Case 1: No scope exists --- 120 | packageData.name = `${requiredPrefix}${originalPackageName}`; 121 | needsUpdate = true; 122 | printInfo( 123 | `Package name '${originalPackageName}' needs scope. Preparing update to '${packageData.name}'.`, 124 | ); 125 | } else if (!originalPackageName.startsWith(requiredPrefix)) { 126 | // --- Case 2: Different scope exists --- 127 | const oldScope = nameParts[0]; 128 | packageData.name = `${requiredPrefix}${currentPackagePart}`; 129 | needsUpdate = true; 130 | printInfo( 131 | `Package name '${originalPackageName}' has different scope. Preparing update to replace '${oldScope}' with '@${expectedScope}'. New name: '${packageData.name}'.`, 132 | ); 133 | } else { 134 | // --- Case 3: Correct scope already exists --- 135 | printInfo(`Package name '${originalPackageName}' is already correctly scoped.`); 136 | packageData.name = originalPackageName; // Ensure packageData.name is set even if no update needed 137 | } 138 | finalName = packageData.name; // Store the name that should be in the file 139 | 140 | // Write Update if Needed 141 | if (needsUpdate) { 142 | try { 143 | const updatedPackageContent = JSON.stringify(packageData, null, 2); // 2 spaces indentation 144 | fs.writeFileSync(packageJsonPath, updatedPackageContent + '\n', 'utf-8'); // Add trailing newline 145 | printSuccess( 146 | `Attempted to update package name in '${PACKAGE_JSON_FILENAME}' to '${finalName}'.`, 147 | ); 148 | 149 | // *** ADDED VERIFICATION STEP *** 150 | printInfo(`Verifying write operation for ${PACKAGE_JSON_FILENAME}...`); 151 | const writtenContent = fs.readFileSync(packageJsonPath, 'utf-8'); 152 | const writtenData = JSON.parse(writtenContent); 153 | if (writtenData.name === finalName) { 154 | printSuccess( 155 | `Verification successful: '${PACKAGE_JSON_FILENAME}' correctly updated on disk.`, 156 | ); 157 | } else { 158 | printError( 159 | `Verification failed: '${PACKAGE_JSON_FILENAME}' content on disk does not match expected name.`, 160 | ); 161 | printError(`Expected name: ${finalName}`); 162 | printError(`Found name: ${writtenData.name}`); 163 | return false; // Indicate verification error 164 | } 165 | // *** END VERIFICATION STEP *** 166 | } catch (writeError) { 167 | printError( 168 | `Failed to write updated name to '${PACKAGE_JSON_FILENAME}': ${writeError.message}`, 169 | ); 170 | return false; // Indicate write error 171 | } 172 | } 173 | 174 | // Final Validation (mostly for internal consistency) 175 | if (finalName !== packageData.name || !finalName.startsWith(requiredPrefix)) { 176 | printError(`Internal error: Package name validation failed.`); 177 | printError(`Current packageData.name: ${packageData.name}, Expected finalName: ${finalName}`); 178 | return false; 179 | } 180 | 181 | printSuccess(`Validated ${PACKAGE_JSON_FILENAME}: Package name is '${finalName}'.`); 182 | return true; // Indicate success 183 | } 184 | 185 | /** 186 | * Gets GitHub credentials, either from the config file or by prompting the user. 187 | * @returns {Promise<{githubScope: string, githubToken: string} | null>} Credentials or null on error. 188 | */ 189 | async function getCredentials() { 190 | const configPath = path.resolve(process.cwd(), CONFIG_FILENAME); 191 | let githubScope = ''; 192 | let githubToken = ''; 193 | 194 | if (fs.existsSync(configPath)) { 195 | // Config file exists, read from it 196 | printInfo(`Reading credentials from existing '${CONFIG_FILENAME}' file...`); 197 | try { 198 | const configContent = fs.readFileSync(configPath, 'utf-8'); 199 | const configData = JSON.parse(configContent); 200 | if (!configData.githubScope || !configData.githubToken) { 201 | throw new Error(`'${CONFIG_FILENAME}' is missing 'githubScope' or 'githubToken'.`); 202 | } 203 | githubScope = configData.githubScope; 204 | githubToken = configData.githubToken; 205 | printSuccess(`Credentials loaded successfully from '${CONFIG_FILENAME}'.`); 206 | return { githubScope, githubToken }; 207 | } catch (error) { 208 | printError(`Failed to read or parse '${CONFIG_FILENAME}': ${error.message}`); 209 | printInfo(`Please fix or delete '${CONFIG_FILENAME}' and run the script again.`); 210 | return null; // Indicate error 211 | } 212 | } else { 213 | // Config file does not exist, prompt user and create it 214 | printInfo(`'${CONFIG_FILENAME}' not found. Please provide your GitHub credentials.`); 215 | try { 216 | githubScope = ( 217 | await question( 218 | 'Enter your GitHub organization name or username (the scope for your package): ', 219 | ) 220 | ).trim(); 221 | if (!githubScope) { 222 | throw new Error('GitHub scope cannot be empty.'); 223 | } 224 | 225 | printInfo('\nEnter your GitHub Personal Access Token (PAT).'); 226 | printInfo("Ensure it has the 'write:packages' scope."); 227 | printInfo('Input will be hidden for security.'); 228 | githubToken = (await hiddenQuestion('GitHub PAT: ')).trim(); 229 | if (!githubToken) { 230 | throw new Error('GitHub PAT cannot be empty.'); 231 | } 232 | 233 | // Save credentials to the config file 234 | const configData = { githubScope, githubToken }; 235 | const configContent = JSON.stringify(configData, null, 2); // Pretty print JSON 236 | fs.writeFileSync(configPath, configContent + '\n', 'utf-8'); 237 | printSuccess(`Credentials saved to '${CONFIG_FILENAME}' for future use.`); 238 | printInfo( 239 | `IMPORTANT: Add '${CONFIG_FILENAME}' to your .gitignore file to avoid committing your token!`, 240 | ); 241 | 242 | return { githubScope, githubToken }; 243 | } catch (error) { 244 | printError(`Failed to get or save credentials: ${error.message}`); 245 | // Clean up potentially partially created file on error? Maybe not necessary. 246 | return null; // Indicate error 247 | } 248 | } 249 | } 250 | 251 | // --- Main Script Logic --- 252 | async function main() { 253 | printInfo('--- GitHub Package Publishing Script (Node.js) ---'); 254 | 255 | const npmrcPath = path.resolve(process.cwd(), NPMRC_FILENAME); 256 | let credentials = null; 257 | 258 | try { 259 | // 1. Get Credentials (from file or prompt) 260 | credentials = await getCredentials(); 261 | if (!credentials) { 262 | throw new Error('Could not obtain GitHub credentials.'); 263 | } 264 | const { githubScope, githubToken } = credentials; 265 | 266 | // 2. Update and Validate package.json name 267 | printInfo(`\nUpdating and validating ${PACKAGE_JSON_FILENAME}...`); 268 | const validationResult = updateAndValidatePackageJson(githubScope); 269 | if (validationResult === null) { 270 | // File not found or read error 271 | throw new Error(`Failed to process ${PACKAGE_JSON_FILENAME}.`); 272 | } else if (!validationResult) { 273 | // Validation, write, or verification error 274 | throw new Error( 275 | `Invalid ${PACKAGE_JSON_FILENAME} configuration, update failed, or verification failed.`, 276 | ); 277 | } 278 | // If validationResult is true, continue 279 | 280 | // 3. Create .npmrc 281 | const npmrcContent = `@${githubScope}:registry=${GITHUB_REGISTRY_URL}\n//npm.pkg.github.com/:_authToken=${githubToken}\n`; 282 | printInfo(`\nCreating '${NPMRC_FILENAME}' for authentication...`); 283 | try { 284 | // Overwrite if exists, create if not 285 | fs.writeFileSync(npmrcPath, npmrcContent, { encoding: 'utf-8' }); 286 | printSuccess(`'${NPMRC_FILENAME}' created/updated successfully.`); 287 | } catch (writeError) { 288 | throw new Error(`Failed to create/update '${NPMRC_FILENAME}': ${writeError.message}`); 289 | } 290 | 291 | // 4. Run npm publish 292 | printInfo('\nAttempting to publish package via npm...'); 293 | const publishSuccess = runCommand('npm', ['publish']); 294 | 295 | if (publishSuccess) { 296 | printSuccess('\nPackage published successfully to GitHub Packages!'); 297 | printInfo(`The '${NPMRC_FILENAME}' file has been left in the directory.`); // Inform user it's kept 298 | } else { 299 | printError(`The '${NPMRC_FILENAME}' file has been left in the directory for debugging.`); // Inform user it's kept on failure 300 | throw new Error('Package publishing failed.'); 301 | } 302 | } catch (error) { 303 | printError(`Script failed: ${error.message}`); 304 | // .npmrc cleanup is removed from here 305 | process.exitCode = 1; // Indicate failure 306 | } finally { 307 | // .npmrc cleanup is removed from here 308 | rl.close(); // Close the readline interface 309 | } 310 | } 311 | 312 | main(); 313 | -------------------------------------------------------------------------------- /scripts/n8n-scanner.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const { execSync } = require('child_process'); 4 | const fs = require('fs'); 5 | 6 | /** 7 | * Run n8n Community Package Scanner 8 | * This script runs the official n8n scanner against the published package 9 | */ 10 | 11 | function printInfo(msg) { 12 | console.log(`\x1b[36mℹ ${msg}\x1b[0m`); 13 | } 14 | 15 | function printSuccess(msg) { 16 | console.log(`\x1b[32m✅ ${msg}\x1b[0m`); 17 | } 18 | 19 | function printError(msg) { 20 | console.log(`\x1b[31m❌ ${msg}\x1b[0m`); 21 | } 22 | 23 | function printWarning(msg) { 24 | console.log(`\x1b[33m⚠️ ${msg}\x1b[0m`); 25 | } 26 | 27 | function execSafe(command, options = {}) { 28 | try { 29 | const result = execSync(command, { 30 | encoding: 'utf-8', 31 | stdio: options.silent ? 'pipe' : 'inherit', 32 | ...options 33 | }); 34 | return result; 35 | } catch (error) { 36 | if (!options.silent) { 37 | printError(`Command failed: ${command}`); 38 | printError(error.message); 39 | } 40 | throw error; 41 | } 42 | } 43 | 44 | function main() { 45 | printInfo('🔍 n8n Community Package Scanner'); 46 | printInfo('================================'); 47 | 48 | try { 49 | // Read package.json to get the package name 50 | const pkg = JSON.parse(fs.readFileSync('package.json', 'utf-8')); 51 | const packageName = pkg.name; 52 | 53 | printInfo(`Scanning package: ${packageName}`); 54 | printInfo(`Current version: ${pkg.version}`); 55 | printInfo(''); 56 | 57 | // Run the scanner directly against the published package 58 | printInfo('Running n8n security scanner...'); 59 | execSafe(`npx @n8n/scan-community-package ${packageName}`); 60 | 61 | printSuccess('n8n security scan completed successfully!'); 62 | printInfo('Your package passed all security checks ✨'); 63 | 64 | } catch (error) { 65 | printError(`Scanner failed: ${error.message}`); 66 | printWarning('Common issues:'); 67 | printWarning('- Package not yet published to npm'); 68 | printWarning('- Network connectivity issues'); 69 | printWarning('- Package name typo'); 70 | printWarning(''); 71 | printWarning('If package is not published yet, this is expected.'); 72 | process.exit(1); 73 | } 74 | } 75 | 76 | main(); 77 | -------------------------------------------------------------------------------- /scripts/update-node-json.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | const fs = require('fs'); 4 | const path = require('path'); 5 | 6 | // Read package.json version 7 | const pkg = require(path.resolve(__dirname, '../package.json')); 8 | const nodeJsonPath = path.resolve(__dirname, '../nodes/Scrappey/Scrappey.node.json'); 9 | 10 | try { 11 | const nodeJson = JSON.parse(fs.readFileSync(nodeJsonPath, 'utf-8')); 12 | nodeJson.nodeVersion = pkg.version; 13 | nodeJson.codexVersion = pkg.version; 14 | fs.writeFileSync(nodeJsonPath, JSON.stringify(nodeJson, null, 2) + '\n', 'utf-8'); 15 | console.log(`↪ Updated Scrappey.node.json to version ${pkg.version}`); 16 | } catch (error) { 17 | console.error(`❌ Could not update Scrappey.node.json: ${error.message}`); 18 | } 19 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "strict": true, 4 | "module": "commonjs", 5 | "moduleResolution": "node", 6 | "target": "es2019", 7 | "lib": ["es2019", "es2020", "es2022.error", "dom"], 8 | "removeComments": true, 9 | "useUnknownInCatchVariables": false, 10 | "forceConsistentCasingInFileNames": true, 11 | "noImplicitAny": true, 12 | "noImplicitReturns": true, 13 | "noUnusedLocals": true, 14 | "strictNullChecks": true, 15 | "preserveConstEnums": true, 16 | "esModuleInterop": true, 17 | "resolveJsonModule": true, 18 | "incremental": true, 19 | "declaration": true, 20 | "sourceMap": true, 21 | "skipLibCheck": true, 22 | "outDir": "./dist/" 23 | }, 24 | "include": [ 25 | "credentials/**/*", 26 | "nodes/**/*", 27 | "nodes/**/*.json", 28 | "package.json", 29 | "GenericFunctions.ts" 30 | ] 31 | } 32 | --------------------------------------------------------------------------------