├── .deepsource.toml ├── .github └── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── custom.md │ └── feature_request.md ├── .gitignore ├── .travis.yml ├── AUTHORS.rst ├── CONTRIBUTING.rst ├── HISTORY.rst ├── LICENSE ├── MANIFEST.in ├── README.md ├── README.rst ├── TODO.rst ├── docs ├── Makefile ├── concepts │ ├── formats.rst │ ├── selectors.rst │ └── structure.rst ├── conf.py ├── contributing │ ├── authors.rst │ ├── guide.rst │ └── history.rst ├── framework │ ├── basic.rst │ ├── commands.rst │ ├── config.rst │ └── images │ │ └── architecture.jpg ├── implementation │ ├── classes.rst │ ├── cli.rst │ ├── commands.rst │ ├── images │ │ ├── class.png │ │ ├── genconfig.jpg │ │ ├── generate.jpg │ │ ├── run.jpg │ │ ├── tree.jpg │ │ └── web.jpg │ ├── interaction.rst │ ├── selectors.rst │ └── utils.rst ├── index.rst ├── intro │ ├── existing.rst │ ├── images │ │ ├── commits.png │ │ ├── gantt.png │ │ ├── plagiarism.jpg │ │ └── weekly.png │ ├── install.rst │ ├── overview.rst │ ├── references.rst │ ├── requirements.rst │ ├── timeline.rst │ └── tutorials │ │ ├── link_crawler.rst │ │ ├── results.rst │ │ └── single_linear.rst └── make.bat ├── examples ├── nba.json └── output_nba_players.json ├── requirements.txt ├── scrapple.sh ├── scrapple ├── __init__.py ├── cmd.py ├── commands │ ├── __init__.py │ ├── command.py │ ├── genconfig.py │ ├── generate.py │ ├── run.py │ ├── templates │ │ ├── complete.html │ │ ├── error.html │ │ └── home.html │ └── web.py ├── selectors │ ├── __init__.py │ ├── css.py │ ├── selector.py │ └── xpath.py ├── templates │ ├── configs │ │ ├── crawler.txt │ │ └── scraper.txt │ └── scripts │ │ └── generate.txt └── utils │ ├── __init__.py │ ├── config.py │ ├── dynamicdispatch.py │ ├── exceptions.py │ ├── form.py │ └── text.py ├── setup.py ├── test_requirements.txt ├── tests ├── __init__.py ├── expected_result1.json ├── expected_result2_20180428.json ├── expected_result3.json ├── expected_result4.json ├── project1.json ├── project2.json ├── project3.json ├── project4.json ├── test_cmd.py ├── test_genconfig.py ├── test_generate.py └── test_run.py └── tox.ini /.deepsource.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/.deepsource.toml -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/.github/ISSUE_TEMPLATE/custom.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/.gitignore -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/.travis.yml -------------------------------------------------------------------------------- /AUTHORS.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/AUTHORS.rst -------------------------------------------------------------------------------- /CONTRIBUTING.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/CONTRIBUTING.rst -------------------------------------------------------------------------------- /HISTORY.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/HISTORY.rst -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/MANIFEST.in -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/README.md -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/README.rst -------------------------------------------------------------------------------- /TODO.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/TODO.rst -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/concepts/formats.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/concepts/formats.rst -------------------------------------------------------------------------------- /docs/concepts/selectors.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/concepts/selectors.rst -------------------------------------------------------------------------------- /docs/concepts/structure.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/concepts/structure.rst -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/conf.py -------------------------------------------------------------------------------- /docs/contributing/authors.rst: -------------------------------------------------------------------------------- 1 | .. _contributing-authors: 2 | 3 | .. include:: ../../AUTHORS.rst -------------------------------------------------------------------------------- /docs/contributing/guide.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/contributing/guide.rst -------------------------------------------------------------------------------- /docs/contributing/history.rst: -------------------------------------------------------------------------------- 1 | .. _contributing-history: 2 | 3 | .. include:: ../../HISTORY.rst -------------------------------------------------------------------------------- /docs/framework/basic.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/framework/basic.rst -------------------------------------------------------------------------------- /docs/framework/commands.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/framework/commands.rst -------------------------------------------------------------------------------- /docs/framework/config.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/framework/config.rst -------------------------------------------------------------------------------- /docs/framework/images/architecture.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/framework/images/architecture.jpg -------------------------------------------------------------------------------- /docs/implementation/classes.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/classes.rst -------------------------------------------------------------------------------- /docs/implementation/cli.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/cli.rst -------------------------------------------------------------------------------- /docs/implementation/commands.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/commands.rst -------------------------------------------------------------------------------- /docs/implementation/images/class.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/images/class.png -------------------------------------------------------------------------------- /docs/implementation/images/genconfig.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/images/genconfig.jpg -------------------------------------------------------------------------------- /docs/implementation/images/generate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/images/generate.jpg -------------------------------------------------------------------------------- /docs/implementation/images/run.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/images/run.jpg -------------------------------------------------------------------------------- /docs/implementation/images/tree.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/images/tree.jpg -------------------------------------------------------------------------------- /docs/implementation/images/web.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/images/web.jpg -------------------------------------------------------------------------------- /docs/implementation/interaction.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/interaction.rst -------------------------------------------------------------------------------- /docs/implementation/selectors.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/selectors.rst -------------------------------------------------------------------------------- /docs/implementation/utils.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/implementation/utils.rst -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/index.rst -------------------------------------------------------------------------------- /docs/intro/existing.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/existing.rst -------------------------------------------------------------------------------- /docs/intro/images/commits.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/images/commits.png -------------------------------------------------------------------------------- /docs/intro/images/gantt.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/images/gantt.png -------------------------------------------------------------------------------- /docs/intro/images/plagiarism.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/images/plagiarism.jpg -------------------------------------------------------------------------------- /docs/intro/images/weekly.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/images/weekly.png -------------------------------------------------------------------------------- /docs/intro/install.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/install.rst -------------------------------------------------------------------------------- /docs/intro/overview.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/overview.rst -------------------------------------------------------------------------------- /docs/intro/references.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/references.rst -------------------------------------------------------------------------------- /docs/intro/requirements.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/requirements.rst -------------------------------------------------------------------------------- /docs/intro/timeline.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/timeline.rst -------------------------------------------------------------------------------- /docs/intro/tutorials/link_crawler.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/tutorials/link_crawler.rst -------------------------------------------------------------------------------- /docs/intro/tutorials/results.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/tutorials/results.rst -------------------------------------------------------------------------------- /docs/intro/tutorials/single_linear.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/intro/tutorials/single_linear.rst -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/docs/make.bat -------------------------------------------------------------------------------- /examples/nba.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/examples/nba.json -------------------------------------------------------------------------------- /examples/output_nba_players.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/examples/output_nba_players.json -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/requirements.txt -------------------------------------------------------------------------------- /scrapple.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple.sh -------------------------------------------------------------------------------- /scrapple/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/__init__.py -------------------------------------------------------------------------------- /scrapple/cmd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/cmd.py -------------------------------------------------------------------------------- /scrapple/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/__init__.py -------------------------------------------------------------------------------- /scrapple/commands/command.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/command.py -------------------------------------------------------------------------------- /scrapple/commands/genconfig.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/genconfig.py -------------------------------------------------------------------------------- /scrapple/commands/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/generate.py -------------------------------------------------------------------------------- /scrapple/commands/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/run.py -------------------------------------------------------------------------------- /scrapple/commands/templates/complete.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/templates/complete.html -------------------------------------------------------------------------------- /scrapple/commands/templates/error.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/templates/error.html -------------------------------------------------------------------------------- /scrapple/commands/templates/home.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/templates/home.html -------------------------------------------------------------------------------- /scrapple/commands/web.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/commands/web.py -------------------------------------------------------------------------------- /scrapple/selectors/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapple/selectors/css.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/selectors/css.py -------------------------------------------------------------------------------- /scrapple/selectors/selector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/selectors/selector.py -------------------------------------------------------------------------------- /scrapple/selectors/xpath.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/selectors/xpath.py -------------------------------------------------------------------------------- /scrapple/templates/configs/crawler.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/templates/configs/crawler.txt -------------------------------------------------------------------------------- /scrapple/templates/configs/scraper.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/templates/configs/scraper.txt -------------------------------------------------------------------------------- /scrapple/templates/scripts/generate.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/templates/scripts/generate.txt -------------------------------------------------------------------------------- /scrapple/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scrapple/utils/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/utils/config.py -------------------------------------------------------------------------------- /scrapple/utils/dynamicdispatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/utils/dynamicdispatch.py -------------------------------------------------------------------------------- /scrapple/utils/exceptions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/utils/exceptions.py -------------------------------------------------------------------------------- /scrapple/utils/form.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/utils/form.py -------------------------------------------------------------------------------- /scrapple/utils/text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/scrapple/utils/text.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/setup.py -------------------------------------------------------------------------------- /test_requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/test_requirements.txt -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/expected_result1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/expected_result1.json -------------------------------------------------------------------------------- /tests/expected_result2_20180428.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/expected_result2_20180428.json -------------------------------------------------------------------------------- /tests/expected_result3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/expected_result3.json -------------------------------------------------------------------------------- /tests/expected_result4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/expected_result4.json -------------------------------------------------------------------------------- /tests/project1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/project1.json -------------------------------------------------------------------------------- /tests/project2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/project2.json -------------------------------------------------------------------------------- /tests/project3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/project3.json -------------------------------------------------------------------------------- /tests/project4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/project4.json -------------------------------------------------------------------------------- /tests/test_cmd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/test_cmd.py -------------------------------------------------------------------------------- /tests/test_genconfig.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/test_genconfig.py -------------------------------------------------------------------------------- /tests/test_generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/test_generate.py -------------------------------------------------------------------------------- /tests/test_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tests/test_run.py -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AlexMathew/scrapple/HEAD/tox.ini --------------------------------------------------------------------------------