├── .editorconfig ├── .eslintignore ├── .eslintrc.yml ├── .gitattributes ├── .gitignore ├── .npmignore ├── AUTHORS ├── README.md ├── index.js ├── lib ├── PayOwl.js ├── PayOwl.spec.js ├── Provider.js ├── cache.js ├── detect.js ├── index.js └── providers │ ├── Laterpay.js │ ├── Laterpay.spec.js │ ├── LeakyPaywall.js │ ├── MPPGlobalSolutions.js │ ├── MediaPass.js │ ├── MemberGate.js │ ├── Pelcro.js │ ├── PianoMedia.js │ ├── Pigeon.js │ ├── Recurly.js │ ├── RevenueWire.js │ ├── SubscriptionGenius.js │ ├── Vindicia.js │ └── index.js ├── package-lock.json └── package.json /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | charset = utf-8 6 | #trim_trailing_whitespace = true 7 | insert_final_newline = true 8 | indent_style = space 9 | indent_size = 2 10 | 11 | [*.md] 12 | trim_trailing_whitespace = false 13 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rodrigogs/payowl/23481596a2cce5f97a15b06b0fff5305ad0aed02/.eslintignore -------------------------------------------------------------------------------- /.eslintrc.yml: -------------------------------------------------------------------------------- 1 | extends: airbnb-base 2 | plugins: 3 | - import 4 | rules: 5 | no-underscore-dangle: off 6 | globals: 7 | expect: on 8 | process: on 9 | describe: on 10 | before: on 11 | after: on 12 | beforeEach: on 13 | suite: on 14 | test: on 15 | it: on 16 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # These settings are for any web project 2 | 3 | # Handle line endings automatically for files detected as text 4 | # and leave all files detected as binary untouched. 5 | # * text=auto 6 | # NOTE - originally I had the above line un-commented. it caused me a lot of grief related to line endings because I was dealing with WordPress plugins and the website changing line endings out if a user modified a plugin through the web interface. commenting this line out seems to have alleviated the git chaos where simply switching to a branch caused it to believe 500 files were modified. 7 | 8 | # 9 | # The above will handle all files NOT found below 10 | # 11 | 12 | # 13 | ## These files are text and should be normalized (Convert crlf => lf) 14 | # 15 | 16 | # source code 17 | *.php text 18 | *.css text 19 | *.sass text 20 | *.scss text 21 | *.less text 22 | *.styl text 23 | *.js text 24 | *.coffee text 25 | *.json text 26 | *.htm text 27 | *.html text 28 | *.xml text 29 | *.svg text 30 | *.txt text 31 | *.ini text 32 | *.inc text 33 | *.pl text 34 | *.rb text 35 | *.py text 36 | *.scm text 37 | *.sql text 38 | *.sh text 39 | *.bat text 40 | 41 | # templates 42 | *.ejs text 43 | *.hbt text 44 | *.jade text 45 | *.haml text 46 | *.hbs text 47 | *.dot text 48 | *.tmpl text 49 | *.phtml text 50 | 51 | # server config 52 | .htaccess text 53 | 54 | # git config 55 | .gitattributes text 56 | .gitignore text 57 | .gitconfig text 58 | 59 | # code analysis config 60 | .jshintrc text 61 | .jscsrc text 62 | .jshintignore text 63 | .csslintrc text 64 | 65 | # misc config 66 | *.yaml text 67 | *.yml text 68 | .editorconfig text 69 | 70 | # build config 71 | *.npmignore text 72 | *.bowerrc text 73 | 74 | # Heroku 75 | Procfile text 76 | .slugignore text 77 | 78 | # Documentation 79 | *.md text 80 | LICENSE text 81 | AUTHORS text 82 | 83 | 84 | # 85 | ## These files are binary and should be left untouched 86 | # 87 | 88 | # (binary is a macro for -text -diff) 89 | *.png binary 90 | *.jpg binary 91 | *.jpeg binary 92 | *.gif binary 93 | *.ico binary 94 | *.mov binary 95 | *.mp4 binary 96 | *.mp3 binary 97 | *.flv binary 98 | *.fla binary 99 | *.swf binary 100 | *.gz binary 101 | *.zip binary 102 | *.7z binary 103 | *.ttf binary 104 | *.eot binary 105 | *.woff binary 106 | *.pyc binary 107 | *.pdf binary 108 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea/ 2 | /node_modules/ 3 | /coverage 4 | *.iml 5 | .env 6 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /.env 3 | coverage 4 | *.iml 5 | .codeclimate 6 | .editorconfig 7 | .eslintrc.yml 8 | .gitattributes 9 | .gitignore 10 | README.md 11 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Rodrigo Gomes da Silva (https://github.com/rodrigogs) 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # payowl 2 | Detect websites protected by paywall applications. 3 | 4 | ## Install 5 | ```bash 6 | npm install payowl 7 | ``` 8 | 9 | ## Usage 10 | 11 | ### Basic 12 | ```javascript 13 | const PayOwl = require('payowl'); 14 | 15 | const payowl = new PayOwl(); 16 | const provider = await payowl.detect('http://example.com'); 17 | console.log(provider); // Paywall provider name or null if no paywall was found 18 | ``` 19 | 20 | ### Advanced 21 | ```javascript 22 | const PayOwl = require('payowl'); 23 | 24 | const cacheProvider = { 25 | async init() { 26 | // IMPL 27 | }, 28 | async get() { 29 | // IMPL 30 | }, 31 | async set() { 32 | // IMPL 33 | }, 34 | }; 35 | 36 | const payowl = await PayOwl.createInstance(cacheProvider); 37 | const provider = await payowl.detect('http://example.com'); 38 | console.log(provider); // Paywall provider name or null if no paywall was found 39 | ``` 40 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib'); 2 | -------------------------------------------------------------------------------- /lib/PayOwl.js: -------------------------------------------------------------------------------- 1 | const cache = require('./cache'); 2 | const detect = require('./detect'); 3 | 4 | /** 5 | * @class PayOwl 6 | * 7 | * @param {Object} [options] 8 | * @param {Object} [options.cacheProvider] Cache implementation. 9 | * @param {Function} [options.cacheProvider.init] Optional function to initialize cache. 10 | * @param {Function} options.cacheProvider.get Function to retrieve data from cache. 11 | * @param {Function} options.cacheProvider.set Function to store data on cache. 12 | * @param {String[]} [options.excludes = []] Providers to exclude. 13 | * Valid provider names: 14 | * > Laterpay 15 | * > LeakyPaywall 16 | * > MediaPass 17 | * > MemberGate 18 | * > MPPGlobalSolutions 19 | * > Pelcro 20 | * > PianoMedia 21 | * > Pigeon 22 | * > Recurly 23 | * > RevenueWire 24 | * > SubscriptionGenius 25 | * > Vindicia 26 | * @param {Number} [options.requestTimeout = 10000] Request timeout. 27 | */ 28 | class PayOwl { 29 | constructor({ 30 | cacheProvider = cache, 31 | excludes = [], 32 | requestTimeout = 10000, 33 | } = {}) { 34 | this._cache = cacheProvider; 35 | this._excludes = excludes; 36 | this._requestTimeout = requestTimeout; 37 | } 38 | 39 | static async createInstance(options) { 40 | const instance = new PayOwl(options); 41 | await instance.init(); 42 | 43 | return instance; 44 | } 45 | 46 | async init() { 47 | if (this._cache.init) await this._cache.init(); 48 | } 49 | 50 | detect(url) { 51 | return detect(url, { 52 | cache: this._cache, 53 | excludes: this._excludes, 54 | requestTimeout: this._requestTimeout, 55 | }); 56 | } 57 | } 58 | 59 | module.exports = PayOwl; 60 | -------------------------------------------------------------------------------- /lib/PayOwl.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @jest-environment node 3 | */ 4 | 5 | const PayOwl = require('./PayOwl'); 6 | 7 | describe('PayOwl', () => { 8 | it('Should detect Laterpay provider on bauhaus-movement.com', async () => { 9 | const payowl = await PayOwl.createInstance(); 10 | const detectedProvider = await payowl.detect('http://bauhaus-movement.com/'); 11 | expect(detectedProvider).toBe('Laterpay'); 12 | }, 10000); 13 | }); 14 | -------------------------------------------------------------------------------- /lib/Provider.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios'); 2 | const conditionalRace = require('conditional-race'); 3 | 4 | class Provider { 5 | constructor(name, expressions = [], options = {}) { 6 | this._name = name; 7 | this._expressions = expressions; 8 | this._options = options; 9 | } 10 | 11 | get name() { 12 | return this._name; 13 | } 14 | 15 | static async _onError(err, type) { 16 | if (type) { 17 | return console.error(err); 18 | } 19 | throw err; 20 | } 21 | 22 | static async _getPageContent(url, { requestTimeout: timeout }) { 23 | try { 24 | return await axios.get(url, { timeout }).then(response => response.data); 25 | } catch (err) { 26 | await Provider._onError(err, 'loading-page'); 27 | return null; 28 | } 29 | } 30 | 31 | static async _lookForRegex(content, regexp) { 32 | try { 33 | const regex = new RegExp(regexp); 34 | return regex.test(content); 35 | } catch (err) { 36 | await Provider._onError(err, 'building-regex'); 37 | return false; 38 | } 39 | } 40 | 41 | static async _lookForResult(content, func) { 42 | try { 43 | return !!(await func(content)); 44 | } catch (err) { 45 | await Provider._onError(err, 'executing-function'); 46 | return false; 47 | } 48 | } 49 | 50 | async _lookForExpressions(content) { 51 | const verifications = []; 52 | 53 | this._expressions.forEach((exp) => { 54 | if (typeof exp === 'string') { 55 | verifications.push(Provider._lookForRegex(content, exp)); 56 | } 57 | if (typeof exp === 'function') { 58 | verifications.push(Provider._lookForResult(content, exp)); 59 | } 60 | }); 61 | 62 | const passed = await conditionalRace(verifications, result => !result); 63 | return passed !== false; 64 | } 65 | 66 | async detect(url, { requestTimeout }) { 67 | const content = await Provider._getPageContent(url, { requestTimeout }); 68 | if (!content) return false; 69 | return this._lookForExpressions(content); 70 | } 71 | } 72 | 73 | module.exports = Provider; 74 | -------------------------------------------------------------------------------- /lib/cache.js: -------------------------------------------------------------------------------- 1 | const keys = []; 2 | const data = []; 3 | 4 | const cache = { 5 | get(host) { 6 | return data[keys.indexOf(host)]; 7 | }, 8 | 9 | set(detected) { 10 | const { host } = detected; 11 | 12 | let index = keys.indexOf(host); 13 | if (index === -1) { 14 | keys.push(host); 15 | index = keys.length; 16 | } 17 | 18 | data[index] = detected; 19 | }, 20 | }; 21 | 22 | module.exports = cache; 23 | -------------------------------------------------------------------------------- /lib/detect.js: -------------------------------------------------------------------------------- 1 | const { URL } = require('url'); 2 | const conditionalRace = require('conditional-race'); 3 | 4 | const allProviders = require('./providers'); 5 | 6 | /** 7 | * Detects if a website is protected by a paywall. 8 | * 9 | * @async 10 | * 11 | * @param {String} url Website url. 12 | * @param {Object} options Options object. 13 | * @param {Object} options.cache Cache implementation. 14 | * @param {Function} options.cache.get Function to retrieve data from cache. 15 | * @param {Function} options.cache.set Function to store data on cache. 16 | * @param {String[]} options.excludes Providers to exclude. 17 | * @param {Number} options.requestTimeout Request timeout. 18 | * Valid provider names: 19 | * > Laterpay 20 | * > LeakyPaywall 21 | * > MediaPass 22 | * > MemberGate 23 | * > MPPGlobalSolutions 24 | * > Pelcro 25 | * > PianoMedia 26 | * > Pigeon 27 | * > Recurly 28 | * > RevenueWire 29 | * > SubscriptionGenius 30 | * > Vindicia 31 | * @return {Promise} detected paywall 32 | */ 33 | const detect = async (url, { cache, excludes, requestTimeout }) => { 34 | const { host } = new URL(url); 35 | 36 | const providers = Object.keys(allProviders) 37 | .filter(provider => !excludes.includes(provider)) 38 | .map(provider => new allProviders[provider]()); 39 | 40 | const verifying = providers.map(async (provider) => { 41 | const cached = await cache.get(host); 42 | if (cached) return cached; 43 | 44 | const detected = await provider.detect(url, { requestTimeout }); 45 | if (!detected) return null; 46 | 47 | await cache.set(host, provider.name); 48 | 49 | return provider.name; 50 | }); 51 | 52 | const detected = await conditionalRace(verifying, result => !!result); 53 | if (!detected) return null; 54 | 55 | return detected; 56 | }; 57 | 58 | module.exports = detect; 59 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./PayOwl'); 2 | -------------------------------------------------------------------------------- /lib/providers/Laterpay.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class Laterpay extends Provider { 4 | constructor() { 5 | super('Laterpay', ['src=.+laterpay\\.net']); 6 | } 7 | } 8 | 9 | module.exports = Laterpay; 10 | -------------------------------------------------------------------------------- /lib/providers/Laterpay.spec.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @jest-environment node 3 | */ 4 | 5 | const Laterpay = require('./Laterpay'); 6 | 7 | describe('Laterpay', () => { 8 | it('Should verify http://bauhaus-movement.com/ as a valid site', async () => { 9 | const laterpay = new Laterpay(); 10 | const result = await laterpay.detect('http://bauhaus-movement.com/'); 11 | 12 | expect(result).toBe(true); 13 | }); 14 | 15 | it('Should verify https://msn.com/ as an invalid site', async () => { 16 | const laterpay = new Laterpay(); 17 | const result = await laterpay.detect('http://msn.com/'); 18 | 19 | expect(result).toBe(false); 20 | }); 21 | }); 22 | -------------------------------------------------------------------------------- /lib/providers/LeakyPaywall.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class LeakyPaywall extends Provider { 4 | } 5 | 6 | module.exports = LeakyPaywall; 7 | -------------------------------------------------------------------------------- /lib/providers/MPPGlobalSolutions.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class MPPGlobalSolutions extends Provider { 4 | } 5 | 6 | module.exports = MPPGlobalSolutions; 7 | -------------------------------------------------------------------------------- /lib/providers/MediaPass.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class MediaPass extends Provider { 4 | } 5 | 6 | module.exports = MediaPass; 7 | -------------------------------------------------------------------------------- /lib/providers/MemberGate.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class MemberGate extends Provider { 4 | } 5 | 6 | module.exports = MemberGate; 7 | -------------------------------------------------------------------------------- /lib/providers/Pelcro.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class Pelcro extends Provider { 4 | } 5 | 6 | module.exports = Pelcro; 7 | -------------------------------------------------------------------------------- /lib/providers/PianoMedia.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class PianoMedia extends Provider { 4 | } 5 | 6 | module.exports = PianoMedia; 7 | -------------------------------------------------------------------------------- /lib/providers/Pigeon.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class Pigeon extends Provider { 4 | } 5 | 6 | module.exports = Pigeon; 7 | -------------------------------------------------------------------------------- /lib/providers/Recurly.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class Recurly extends Provider { 4 | } 5 | 6 | module.exports = Recurly; 7 | -------------------------------------------------------------------------------- /lib/providers/RevenueWire.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class RevenueWire extends Provider { 4 | } 5 | 6 | module.exports = RevenueWire; 7 | -------------------------------------------------------------------------------- /lib/providers/SubscriptionGenius.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class SubscriptionGenius extends Provider { 4 | } 5 | 6 | module.exports = SubscriptionGenius; 7 | -------------------------------------------------------------------------------- /lib/providers/Vindicia.js: -------------------------------------------------------------------------------- 1 | const Provider = require('../Provider'); 2 | 3 | class Vindicia extends Provider { 4 | } 5 | 6 | module.exports = Vindicia; 7 | -------------------------------------------------------------------------------- /lib/providers/index.js: -------------------------------------------------------------------------------- 1 | const Laterpay = require('./Laterpay'); 2 | const LeakyPaywall = require('./LeakyPaywall'); 3 | const MediaPass = require('./MediaPass'); 4 | const MemberGate = require('./MemberGate'); 5 | const MPPGlobalSolutions = require('./MPPGlobalSolutions'); 6 | const Pelcro = require('./Pelcro'); 7 | const PianoMedia = require('./PianoMedia'); 8 | const Pigeon = require('./Pigeon'); 9 | const Recurly = require('./Recurly'); 10 | const RevenueWire = require('./RevenueWire'); 11 | const SubscriptionGenius = require('./SubscriptionGenius'); 12 | const Vindicia = require('./Vindicia'); 13 | 14 | module.exports = { 15 | Laterpay, 16 | LeakyPaywall, 17 | MediaPass, 18 | MemberGate, 19 | MPPGlobalSolutions, 20 | Pelcro, 21 | PianoMedia, 22 | Pigeon, 23 | Recurly, 24 | RevenueWire, 25 | SubscriptionGenius, 26 | Vindicia, 27 | }; 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "payowl", 3 | "description": "Detect websites protected by paywall applications", 4 | "version": "0.0.1-ALPHA", 5 | "license": "BSD-3-Clause", 6 | "scripts": { 7 | "lint": "eslint . --ext .js", 8 | "lint:fix": "eslint . --ext .js --fix", 9 | "test": "jest", 10 | "coverage": "jest --coverage" 11 | }, 12 | "dependencies": { 13 | "axios": "^0.18.0", 14 | "conditional-race": "^1.0.0" 15 | }, 16 | "devDependencies": { 17 | "dotenv": "^6.2.0", 18 | "eslint": "^5.12.0", 19 | "eslint-config-airbnb-base": "^13.1.0", 20 | "eslint-plugin-import": "^2.14.0", 21 | "jest": "^23.6.0" 22 | }, 23 | "engines": { 24 | "node": ">=7.6.0" 25 | } 26 | } 27 | --------------------------------------------------------------------------------