2 |
3 |
4 |
5 |
9 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/images.py:
--------------------------------------------------------------------------------
1 | """
2 | Images
3 | """
4 | from scrapely.extractors import extract_image_url
5 | from slybot.fieldtypes.url import UrlFieldTypeProcessor
6 |
7 | class ImagesFieldTypeProcessor(UrlFieldTypeProcessor):
8 | name = 'image'
9 | description = 'extracts image URLs'
10 |
11 | def extract(self, text):
12 | return extract_image_url(text)
13 |
14 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/example2.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [],
4 | "init_requests": [
5 | {
6 | "type": "start",
7 | "url": "http://www.example.com/index.html"
8 | }
9 | ],
10 | "exclude_patterns": [],
11 | "follow_patterns": [],
12 | "links_to_follow": "patterns",
13 | "respect_nofollow": true
14 | }
15 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/allowed_domains.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "allowed_domains": [
7 | "www.ebay.com",
8 | "www.yahoo.com"
9 | ],
10 | "exclude_patterns": [],
11 | "respect_nofollow": true,
12 | "follow_patterns": [],
13 | "links_to_follow": "none"
14 | }
15 |
--------------------------------------------------------------------------------
/slybot/README.rst:
--------------------------------------------------------------------------------
1 | ==============
2 | Slybot crawler
3 | ==============
4 |
5 | Slybot is a Python web crawler for doing web scraping. It's implemented on top of the
6 | `Scrapy`_ web crawling framework and the `Scrapely`_ extraction library.
7 |
8 | The documentation (including installation and usage) can be found at:
9 | http://slybot.readthedocs.org/
10 |
11 | .. _Scrapely: https://github.com/scrapy/scrapely
12 | .. _Scrapy: http://scrapy.org
13 |
--------------------------------------------------------------------------------
/slyd/tests/settings.py:
--------------------------------------------------------------------------------
1 | """
2 | Slyd test settings
3 |
4 | Imports slyd settings and adds necessary overrides for test setup
5 | """
6 | from slyd.settings import *
7 |
8 | LOG_LEVEL = 'DEBUG'
9 |
10 | # testing never makes remote requests. A cache may serve stale content.
11 | HTTPCACHE_ENABLED = False
12 |
13 | RESOURCE_DIR = join(dirname(__file__), 'resources')
14 | DATA_DIR = join(RESOURCE_DIR, 'data')
15 | SPEC_DATA_DIR = join(DATA_DIR, 'projects')
16 |
--------------------------------------------------------------------------------
/slybot/slybot/settings.py:
--------------------------------------------------------------------------------
1 | SPIDER_MANAGER_CLASS = 'slybot.spidermanager.SlybotSpiderManager'
2 | EXTENSIONS = {'slybot.closespider.SlybotCloseSpider': 1}
3 | ITEM_PIPELINES = ['slybot.dupefilter.DupeFilterPipeline']
4 | SPIDER_MIDDLEWARES = {'slybot.spiderlets.SpiderletsMiddleware': 999} # as close as possible to spider output
5 | SLYDUPEFILTER_ENABLED = True
6 | PROJECT_DIR = 'slybot-project'
7 |
8 | try:
9 | from local_slybot_settings import *
10 | except ImportError:
11 | pass
12 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/topbar-project.handlebars:
--------------------------------------------------------------------------------
1 |
6 |
7 |
8 |
2 |
5 |
10 |
--------------------------------------------------------------------------------
/slybot/debian/rules:
--------------------------------------------------------------------------------
1 | #!/usr/bin/make -f
2 | # -*- makefile -*-
3 | # Sample debian/rules that uses debhelper.
4 | # This file was originally written by Joey Hess and Craig Small.
5 | # As a special exception, when this file is copied by dh-make into a
6 | # dh-make output file, you may use that output file without restriction.
7 | # This special exception was added by Craig Small in version 0.37 of dh-make.
8 |
9 | # Uncomment this to turn on verbose mode.
10 | #export DH_VERBOSE=1
11 |
12 | %:
13 | dh $@
14 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/price.py:
--------------------------------------------------------------------------------
1 | """
2 | Price field types
3 | """
4 | from scrapely import extractors
5 |
6 | class PriceTypeProcessor(object):
7 | """Extracts price from text"""
8 | name = "price"
9 | description = "extracts a price decimal number in the text passed"
10 |
11 | def extract(self, htmlregion):
12 | return extractors.contains_any_numbers(htmlregion.text_content)
13 |
14 | def adapt(self, text, htmlpage):
15 | return extractors.extract_price(text)
16 |
17 |
--------------------------------------------------------------------------------
/slyd/media/js/controllers/controllers.js:
--------------------------------------------------------------------------------
1 | ASTool.BaseControllerMixin = Ember.Mixin.create({
2 |
3 | openAccordion: function(accordionNumber) {
4 | $( ".accordion" ).accordion("option", "active", accordionNumber);
5 | },
6 |
7 | getUnusedName: function(baseName, usedNames) {
8 | var i = 1;
9 | var newName = baseName;
10 | while(usedNames.any(function(usedName) {
11 | return usedName == newName
12 | })) {
13 | newName = baseName + '_' + i++;
14 | }
15 | return newName;
16 | },
17 | });
18 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/topbar-projects.handlebars:
--------------------------------------------------------------------------------
1 |
6 | {{view ASTool.TextField value=spiderPage width='320px' name='spiderPageTextField' placeholder='Enter page url' action="addSpider"}}
7 | {{view ASTool.ButtonView action="addSpider" icon="ui-icon-circle-plus" label="New spider" class="blue-button" disabled=createSpiderDisabled}}
8 |
9 |
2 |
5 |
10 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/cargurus.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [
3 | ],
4 | "start_urls": [
5 | "http://www.cargurus.com/Cars/sitemap.html"
6 | ],
7 | "exclude_patterns": [
8 | "-Pictures-",
9 | "-Specs-",
10 | "-Price-",
11 | "_v",
12 | "-Videos-"
13 | ],
14 | "follow_patterns": [
15 | "-Overview-",
16 | "-Reviews-",
17 | "/rss/"
18 | ],
19 | "links_to_follow": "patterns",
20 | "respect_nofollow": false
21 | }
22 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/navigation.handlebars:
--------------------------------------------------------------------------------
1 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/pinterest.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://pinterest.com/popular/"
5 | ],
6 | "init_requests": [
7 | {
8 | "username": "test",
9 | "loginurl": "https://pinterest.com/login/",
10 | "password": "testpass",
11 | "type": "login"
12 | }
13 | ],
14 | "exclude_patterns": [],
15 | "respect_nofollow": true,
16 | "follow_patterns": [],
17 | "links_to_follow": "patterns"
18 | }
19 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/topbar-extraction.handlebars:
--------------------------------------------------------------------------------
1 |
6 | {{view ASTool.TextField value=projectSite width='320px' name='projectSiteTextField' placeholder='Enter site url' action="createProject" id="projectSiteTextField"}}
7 | {{view ASTool.ButtonView action="createProject" icon="ui-icon-circle-plus" label="Start" class="blue-button" disabled=createProjectDisabled}}
8 |
9 |
2 |
5 |
18 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/seedsofchange.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.seedsofchange.com/garden_center/browse_category.aspx?id=123"
5 | ],
6 | "exclude_patterns": [
7 | "/tellafriend.aspx.+"
8 | ],
9 | "follow_patterns": [
10 | "/garden_center/browse_category.aspx.+",
11 | "/garden_center/detailedCategoryDisplay.aspx.+",
12 | "/garden_center/product_details.aspx.+"
13 | ],
14 | "links_to_follow": "patterns",
15 | "respect_nofollow": true
16 | }
17 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/toolbox.handlebars:
--------------------------------------------------------------------------------
1 |
6 |
7 | {{#view ASTool.LabelWithTooltip title=controller.url}}
8 | {{controller.url}}
9 | {{/view}}
10 |
11 |
12 | {{#if showContinueBrowsing}}
13 | {{view ASTool.ButtonView action="continueBrowsing" label="Continue browsing" class="blue-button"}}
14 | {{/if}}
15 |
16 |
17 |
2 |
16 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/example.com.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": ["http://www.example.com/index.html"],
4 | "init_requests": [
5 | {
6 | "type": "start",
7 | "url": "http://www.example.com/products.csv",
8 | "link_extractor": {
9 | "type": "column",
10 | "value": 1,
11 | "delimiter": ","
12 | }
13 | }
14 | ],
15 | "exclude_patterns": [],
16 | "follow_patterns": [],
17 | "links_to_follow": "patterns",
18 | "respect_nofollow": true
19 | }
20 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/toolbox-projects.handlebars:
--------------------------------------------------------------------------------
1 |
3 |
15 |
4 |
8 |
9 | {{view ASTool.PinToolBoxButton action="editAnnotation" class="textless-button pin-button"}}
10 |
11 |
5 | 
6 |
7 |
6 |
12 | {{yield}}
13 |
14 |
2 |
3 |
--------------------------------------------------------------------------------
/slyd/slyd/settings.py:
--------------------------------------------------------------------------------
1 | """Scrapy settings"""
2 | from os.path import join, dirname
3 |
4 | EXTENSIONS = {
5 | 'scrapy.contrib.logstats.LogStats': None,
6 | 'scrapy.webservice.WebService': None,
7 | 'scrapy.telnet.TelnetConsole': None,
8 | 'scrapy.contrib.throttle.AutoThrottle': None
9 | }
10 |
11 | LOG_LEVEL = 'DEBUG'
12 |
13 | # location of slybot projects - assumes a subdir per project
14 | DATA_DIR = join(dirname(dirname(__file__)), 'data')
15 | SPEC_DATA_DIR = join(DATA_DIR, 'projects')
16 |
17 |
18 | # recommended for development - use scrapy to cache http responses
19 | HTTPCACHE_ENABLED = True
20 | HTTPCACHE_DIR = join(DATA_DIR, 'cache')
21 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/url.py:
--------------------------------------------------------------------------------
1 | from urlparse import urljoin
2 | from scrapy.utils.url import safe_download_url
3 | from scrapy.utils.markup import unquote_markup
4 | from slybot.baseurl import get_base_url
5 |
6 | class UrlFieldTypeProcessor(object):
7 | """Renders URLs as links"""
8 |
9 | name = 'url'
10 | description = 'URL'
11 | limit = 80
12 |
13 | def extract(self, text):
14 | return text
15 |
16 | def adapt(self, text, htmlpage):
17 | text = text.encode(htmlpage.encoding)
18 | joined = urljoin(get_base_url(htmlpage).encode(htmlpage.encoding), text)
19 | return safe_download_url(unquote_markup(joined, encoding=htmlpage.encoding))
20 |
21 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/test_fieldtypes.py:
--------------------------------------------------------------------------------
1 | from unittest import TestCase
2 | from scrapely.htmlpage import HtmlPage
3 |
4 | from slybot.fieldtypes import UrlFieldTypeProcessor
5 |
6 | class FieldTypesUrlEncoding(TestCase):
7 | def test_not_standard_chars_in_url(self):
8 | body = u''
9 | url = u'fotos/produtos/Mam\xe3e noel.jpg'
10 | htmlpage = HtmlPage(url=u"http://www.example.com/", body=body, encoding='cp1252')
11 | processor = UrlFieldTypeProcessor()
12 | self.assertEqual(processor.adapt(url, htmlpage), u'http://www.example.com/fotos/produtos/Mam%C3%A3e%20noel.jpg')
13 |
14 |
--------------------------------------------------------------------------------
/slybot/debian/control:
--------------------------------------------------------------------------------
1 | Source: python-slybot
2 | Section: python
3 | Priority: extra
4 | Maintainer: Scrapinghub Team Open project
4 |
5 | {{#each}}
6 |
14 |
7 | {{view ASTool.ButtonView action="openProject" argument=this label=this minWidth='85%' class="light-button"}}
8 | {{view ASTool.ButtonView action="deleteProject" argument=this icon="ui-icon-trash" class="red-button"}}
9 |
10 | {{else}}
11 | No projects have been created yet.
12 | {{/each}} 13 |
2 |
18 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/toolbox-project.handlebars:
--------------------------------------------------------------------------------
1 | Items
3 |
4 |
17 |
5 | {{#each model}}
6 | {{view ASTool.EditItemView item=this}}
7 | {{else}}
8 |
11 | No items have been defined yet.
9 | {{/each}} 10 |
12 | {{view ASTool.ButtonView action="addItem" icon="ui-icon-circle-plus" label="Item" class="blue-button"}}
13 | {{view ASTool.ButtonView action="saveChanges" icon="ui-icon-circle-triangle-w" label="Save changes" class="blue-button"}}
14 | {{view ASTool.ButtonView action="undoChanges" icon="ui-icon-arrowreturnthick-1-w" label="Discard changes" class="red-button"}}
15 |
16 |
2 |
22 |
23 |
--------------------------------------------------------------------------------
/slyd/tests/mockserver.py:
--------------------------------------------------------------------------------
1 | from twisted.web.server import Site
2 | from twisted.web.resource import Resource
3 |
4 | class Root(Resource):
5 |
6 | def __init__(self):
7 | Resource.__init__(self)
8 | self.putChild("status", Status())
9 |
10 | def getChild(self, name, request):
11 | return self
12 |
13 | def render(self, request):
14 | return 'Slyd mock HTTP server\n'
15 |
16 | # TODO: make PR for scrapy to share code
17 | if __name__ == "__main__":
18 | root = Root()
19 | factory = Site(root)
20 | httpPort = reactor.listenTCP(8998, factory)
21 |
22 | def print_listening():
23 | httpHost = httpPort.getHost()
24 | print("Mock server running at http://%s:%d/" % (
25 | httpHost.host, httpHost.port))
26 | reactor.callWhenRunning(print_listening)
27 | reactor.run()
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/ebay.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "init_requests": [
7 | {
8 | "type": "form",
9 | "form_url": "http://http://www.ebay.com/sch/ebayadvsearch/?rt=nc",
10 | "xpath": "//form[@name='adv_search_from']",
11 | "fields": [
12 | {
13 | "xpath": ".//*[@name='_nkw']",
14 | "type": "constants",
15 | "value": ["Cars"]
16 | },
17 | {
18 | "xpath": ".//*[@name='_in_kw']",
19 | "type": "iterate"
20 | }
21 | ]
22 | }
23 | ],
24 | "exclude_patterns": [],
25 | "respect_nofollow": true,
26 | "follow_patterns": [],
27 | "links_to_follow": "none"
28 | }
29 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/item.handlebars:
--------------------------------------------------------------------------------
1 |
3 | {{#view ASTool.RenameTextField value=name}}
4 | Project {{name}}
5 | {{/view}}
6 |
7 |
8 | Spiders
9 |
10 |
21 |
11 | {{#each}}
12 |
20 |
13 | {{view ASTool.ButtonView action="editSpider" argument=this label=this minWidth="82%" class="light-button"}}
14 | {{view ASTool.ButtonView action="deleteSpider" argument=this icon="ui-icon-trash" class="red-button"}}
15 |
16 | {{else}}
17 | No spiders for this project.
18 | {{/each}} 19 |
2 | [spider] [options]")
10 | parser.add_option("--settings", help="Give specific settings module (must be on python path)", default='slybot.settings')
11 | parser.add_option("--logfile", help="Specify log file")
12 | parser.add_option("-a", help="Add spider arguments", dest="spargs", action="append", default=[], metavar="NAME=VALUE")
13 |
14 | opts, args = parser.parse_args()
15 |
16 | try:
17 | project_specs = args[0]
18 | if not os.path.exists(project_specs) or len(args) > 2:
19 | parser.print_help()
20 | return
21 | except IndexError:
22 | parser.print_help()
23 | return
24 |
25 |
26 | if opts.settings:
27 | os.environ["SCRAPY_SETTINGS_MODULE"] = opts.settings
28 |
29 | command_spec = ["scrapy", "crawl", args[1]] if len(args) == 2 else ["scrapy", "list"]
30 | if project_specs.endswith(".zip"):
31 | command_spec.extend([
32 | "-s", "PROJECT_ZIPFILE=%s" % project_specs,
33 | "-s", "SPIDER_MANAGER_CLASS=slybot.spidermanager.ZipfileSlybotSpiderManager",
34 | ])
35 | else:
36 | command_spec.extend([
37 | "-s", "PROJECT_DIR=%s" % project_specs,
38 | "-s", "SPIDER_MANAGER_CLASS=slybot.spidermanager.SlybotSpiderManager",
39 | ])
40 |
41 | if opts.logfile:
42 | command_spec.append("--logfile=%s" % opts.logfile)
43 |
44 | for sparg in opts.spargs:
45 | command_spec.append("-a")
46 | command_spec.append(sparg)
47 |
48 | subprocess.call(command_spec)
49 |
50 | main()
51 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/floating-annotation-widget.handlebars:
--------------------------------------------------------------------------------
1 | {{#unless view.creatingField}}
2 | ",
18 | "extractors": [],
19 | "original_body": "",
20 | "page_id": "",
21 | "page_type": "item",
22 | "scrapes": "default",
23 | "url": "http://site1.com",
24 | },
25 | {
26 | "name": "t2",
27 | "annotated_body": "",
28 | "extractors": [],
29 | "original_body": "",
30 | "page_id": "",
31 | "page_type": "item",
32 | "scrapes": "default",
33 | "url": "http://site2.com"
34 | }
35 | ]
36 | };
37 |
38 | var fetchedPageJson = {
39 | page: "
Type extractors override the type specified in the item definition.', 9 | select_item: 'You can choose what item type is extracted by this template using the combobox. You can also create and modify items by clicking on the Edit Items button.', 10 | variant: 'By selecting a different variant than Base(0) in your annotation, the resulting extracted data will be assigned to the base item special field variants, which is a list of objects similar to an item.', 11 | ignored_subregions: 'Allows you to define subregions that should be excluded from the extraction process.', 12 | selected_region_ancestors: 'Refine your selection by navigating its ancestors.', 13 | selected_region_children: 'Refine your selection by navigating its children.', 14 | sticky_fields: 'Required attributes are not extracted, but they must be present for a page to match the template.', 15 | annotation_widget: 'Select the attribute you want to extract and an item field to map it.
Choose -just required- to indicate that the template must match a particular feature without generating any extracted data.
You can create new fields by clicking the + field button or by seleting the -create new- option from the field combobox.', 16 | 17 | // Other messages. 18 | confirm_change_selection: 'If you select a different region you will lose all current attribute mappings and ignored subregions, proceed anyway?', 19 | }); 20 | -------------------------------------------------------------------------------- /slyd/media/js/controllers/projects-controller.js: -------------------------------------------------------------------------------- 1 | ASTool.ProjectsIndexController = Em.ArrayController.extend(ASTool.BaseControllerMixin, { 2 | needs: ['application'], 3 | 4 | projectSite: null, 5 | 6 | createProjectDisabled: function() { 7 | return Em.isEmpty(this.get('projectSite')); 8 | }.property('projectSite'), 9 | 10 | actions: { 11 | 12 | openProject: function(projectName) { 13 | this.set('slyd.project', projectName); 14 | this.transitionToRoute('project', { id: projectName }); 15 | }, 16 | 17 | deleteProject: function(projectName) { 18 | if (confirm('Are you sure you want to delete this project? This operation cannot be undone.')) { 19 | this.get('slyd').deleteProject(projectName); 20 | this.removeObject(projectName); 21 | } 22 | }, 23 | 24 | createProject: function() { 25 | var newProjectName = this.getUnusedName('new_project', this.get('content')); 26 | this.get('slyd').createProject(newProjectName).then(function() { 27 | this.set('slyd.project', newProjectName); 28 | // Initialize items spec. 29 | this.get('slyd').saveItems([ 30 | ASTool.Item.create({ name: 'default', fields: [ ] 31 | }) 32 | ]); 33 | // Initialize extractors spec. 34 | this.get('slyd').saveExtractors([]); 35 | // Setup automatic creation of an initial spider. 36 | this.set('controllers.application.siteWizard', this.get('projectSite')); 37 | this.set('projectSite', null); 38 | this.transitionToRoute('project', { id: newProjectName }); 39 | }.bind(this)); 40 | } 41 | }, 42 | 43 | animateProjectSiteInput: function() { 44 | var animateBorderColor = function () { 45 | $('#projectSiteTextField') 46 | .animate({ 'border-color': 'rgba(88,150,220,0.4)', 'background-color': 'rgba(130,210,230,0.1)' }, 1000) 47 | .animate({ 'border-color': '#BBBBB', 'background-color': '#FFFFFF' }, 1000, animateBorderColor) 48 | }; 49 | Em.run.schedule('afterRender', this, function() { 50 | $('#projectSiteTextField') 51 | .hide() 52 | .show('fast', animateBorderColor) 53 | .click(function(e) { 54 | $('#projectSiteTextField').stop(true) 55 | .css({ 'border-color': '#BBBBB', 'background-color': '#FFFFFF' }); 56 | }); 57 | }); 58 | }, 59 | 60 | willEnter: function() { 61 | this.get('documentView').showSpider(); 62 | if (Em.isEmpty(this.get('content'))) { 63 | this.animateProjectSiteInput(); 64 | } 65 | } 66 | }); 67 | -------------------------------------------------------------------------------- /slybot/slybot/item.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from collections import defaultdict 3 | 4 | from scrapy.item import DictItem, Field 5 | from scrapely.descriptor import ItemDescriptor, FieldDescriptor 6 | 7 | from slybot.fieldtypes import FieldTypeManager 8 | 9 | class SlybotItem(DictItem): 10 | # like DictItem.__setitem__ but doesn't check the field is declared 11 | def __setitem__(self, name, value): 12 | self._values[name] = value 13 | @classmethod 14 | def create_iblitem_class(cls, schema): 15 | class IblItem(cls): 16 | fields = defaultdict(dict) 17 | version_fields = [] 18 | for _name, _meta in schema['fields'].items(): 19 | fields[_name] = Field(_meta) 20 | if not _meta.get("vary", False): 21 | version_fields.append(_name) 22 | version_fields = sorted(version_fields) 23 | return IblItem 24 | 25 | def create_slybot_item_descriptor(schema): 26 | field_type_manager = FieldTypeManager() 27 | descriptors = [] 28 | for pname, pdict in schema['fields'].items(): 29 | required = pdict['required'] 30 | pclass = field_type_manager.type_processor_class(pdict['type']) 31 | processor = pclass() 32 | descriptor = SlybotFieldDescriptor(pname, pname, processor, required) 33 | descriptors.append(descriptor) 34 | return ItemDescriptor("", "", descriptors) 35 | 36 | class SlybotFieldDescriptor(FieldDescriptor): 37 | """Extends the scrapely field descriptor to use slybot fieldtypes and 38 | to be created from a slybot item schema 39 | """ 40 | 41 | def __init__(self, name, description, field_type_processor, required=False): 42 | """Create a new SlybotFieldDescriptor with the given name and description. 43 | The field_type_processor is used for extraction and is publicly available 44 | """ 45 | FieldDescriptor.__init__(self, name, description, 46 | field_type_processor.extract, required) 47 | # add an adapt method 48 | self.adapt = field_type_processor.adapt 49 | 50 | def create_item_version(item): 51 | """Item version based on hashlib.sha1 algorithm""" 52 | if not item.version_fields: 53 | return 54 | _hash = hashlib.sha1() 55 | for attrname in item.version_fields: 56 | _hash.update(repr(item.get(attrname))) 57 | return _hash.digest() 58 | 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | portia 2 | ====== 3 | Visual scraping for Scrapy. 4 | 5 | 6 | Overview 7 | ======== 8 | 9 | Portia is a tool for visually scraping web sites without any programming knowledge. Just annotate web pages with a point and click editor to indicate what data you want to extract, and portia will learn how to scrape similar pages 10 | from the site. 11 | 12 | Portia has a web based UI served by a [Twisted] server, so you can install it on almost any modern platform. 13 | 14 | Requirements 15 | ============ 16 | 17 | * Python 2.7 18 | * Works on Linux, Windows, Mac OSX, BSD 19 | * Supported browsers: Latest versions of Chrome (recommended) or Firefox 20 | 21 | 22 | Repository structure 23 | ==================== 24 | 25 | There are two main components in this repository, __slyd__ and __slybot__: 26 | 27 | ###slyd 28 | 29 | The visual editor used to create your scraping projects. 30 | 31 | ###slybot 32 | 33 | The Python web crawler that performs the actual site scraping. It's implemented on top of the [Scrapy] web crawling 34 | framework and the [Scrapely] extraction library. It uses projects created with __slyd__ as input. 35 | 36 | 37 | How to install portia 38 | ============================= 39 | 40 | The recommended way to install dependencies is to use __virtualenv__ and then do: 41 | 42 | cd slyd 43 | pip install -r requirements.txt 44 | 45 | As __slybot__ is a __slyd__ dependency, it will also get installed. 46 | 47 | Running portia 48 | ============== 49 | 50 | First, you need to start the ui and create a project. Run __slyd__ using: 51 | 52 | cd slyd 53 | twistd -n slyd 54 | 55 | and point your browser to: `http://localhost:9001/static/main.html` 56 | 57 | Choose the site you want to scrape and create a project. Every project is created with a default spider named after the domain of the site you are scraping. When you are ready, you can run your project with __slybot__ to do the actual crawling/extraction. 58 | 59 | Projects created with __slyd__ can be found at: 60 | 61 | slyd/data/projects 62 | 63 | To run one of those projects use: 64 | 65 | portiacrawl project_path spidername 66 | 67 | Where `spidername` should be one of the project spiders. If you don't remember the name of the spider, just use: 68 | 69 | portiacrawl project_path 70 | 71 | and you will get the list of spiders for that project. 72 | 73 | 74 | [Twisted]: https://twistedmatrix.com 75 | [Scrapely]: https://github.com/scrapy/scrapely 76 | [Scrapy]: http://scrapy.org 77 | -------------------------------------------------------------------------------- /slybot/slybot/tests/test_schema_validation.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from unittest import TestCase 4 | from os.path import dirname, join 5 | 6 | from slybot.validation.schema import get_schema_validator, \ 7 | ValidationError, validate_project_schema 8 | from slybot.utils import open_project_from_dir 9 | 10 | _TEST_PROJECT_DIR = join(dirname(__file__), "data/SampleProject") 11 | 12 | class JsonSchemaTest(TestCase): 13 | 14 | def assertRaisesRegexp(self, eclass, pattern, func, *args): 15 | """assertRaisesRegexp is not provided in python versions below 2.7""" 16 | try: 17 | func(*args) 18 | except eclass, e: 19 | m = re.search(pattern, e.message) 20 | if not m: 21 | raise AssertionError('"%s" does not match "%s"' % (pattern, e.message)) 22 | else: 23 | raise AssertionError("%s not raised" % eclass.__name__) 24 | 25 | def test_regex_formatting_wrong(self): 26 | obj = { 27 | "0": { 28 | "regular_expression": "Item: (\d+" 29 | } 30 | } 31 | validator = get_schema_validator("extractors") 32 | self.assertRaisesRegexp(ValidationError, "Invalid regular expression", 33 | validator.validate, obj) 34 | 35 | def test_regex_formatting_ok(self): 36 | obj = { 37 | "0": { 38 | "regular_expression": "Item: (\d+)" 39 | } 40 | } 41 | validator = get_schema_validator("extractors") 42 | self.assertEqual(validator.validate(obj), None) 43 | 44 | def test_valid_url(self): 45 | obj = { 46 | "start_urls": ['http://www.example.com/'], 47 | "links_to_follow": "none", 48 | "respect_nofollow": True, 49 | "templates": [], 50 | } 51 | validator = get_schema_validator("spider") 52 | self.assertEqual(validator.validate(obj), None) 53 | 54 | def test_invalid_url(self): 55 | obj = { 56 | "start_urls": ['www.example.com'], 57 | "links_to_follow": "none", 58 | "respect_nofollow": True, 59 | "templates": [], 60 | } 61 | validator = get_schema_validator("spider") 62 | self.assertRaisesRegexp(ValidationError, "Invalid url:", validator.validate, obj) 63 | 64 | def test_test_project(self): 65 | specs = open_project_from_dir(_TEST_PROJECT_DIR) 66 | self.assertTrue(validate_project_schema(specs)) 67 | 68 | -------------------------------------------------------------------------------- /slyd/tests/test_bot.py: -------------------------------------------------------------------------------- 1 | import json 2 | from os.path import join 3 | from twisted.trial import unittest 4 | from twisted.internet.defer import inlineCallbacks 5 | from twisted.web.server import Site 6 | from twisted.web.static import File 7 | from twisted.internet import reactor 8 | from slyd.bot import create_bot_resource 9 | from .utils import TestSite, test_spec_manager 10 | from .settings import RESOURCE_DIR 11 | 12 | 13 | class BotTest(unittest.TestCase): 14 | def setUp(self): 15 | # configure bot resource 16 | sm = test_spec_manager() 17 | self.bot_resource = create_bot_resource(sm) 18 | self.botsite = TestSite(self.bot_resource) 19 | 20 | # configure fake website to crawl 21 | docroot = join(RESOURCE_DIR, 'docroot') 22 | factory = Site(File(docroot)) 23 | self.listen_port = reactor.listenTCP(8997, factory) 24 | 25 | 26 | def _fetch(self, url, **params): 27 | req = dict(params) 28 | req.setdefault('request', {})['url'] = url 29 | request_json = json.dumps(req) 30 | return self.botsite.post('fetch', data=request_json) 31 | 32 | @inlineCallbacks 33 | def test_fetch(self): 34 | # test status code 35 | result = yield self._fetch("http://localhost:8997/notexists") 36 | self.assertEqual(result.responseCode, 200) 37 | status = json.loads(result.value())['response']['status'] 38 | self.assertEqual(status, 404) 39 | 40 | # get an existing file 41 | test_url = "http://localhost:8997/test.html" 42 | result = yield self._fetch(test_url) 43 | self.assertEqual(result.responseCode, 200) 44 | value = json.loads(result.value()) 45 | # expect 200 response and base href added 46 | self.assertEqual(value['response']['status'], 200) 47 | self.assertIn('
.50 pounds")
14 | u'45.50'
15 | """
16 | ereg = re.compile(pattern, re.S)
17 | def _extractor(txt):
18 | m = ereg.search(txt)
19 | if m:
20 | return htmlregion(u"".join(filter(None, m.groups() or m.group())))
21 |
22 | _extractor.__name__ = "Regex: %s" % pattern.encode("utf-8")
23 | return _extractor
24 |
25 | class PipelineExtractor:
26 | def __init__(self, *extractors):
27 | self.extractors = extractors
28 |
29 | def __call__(self, value):
30 | for extractor in self.extractors:
31 | value = extractor(value) if value else value
32 | return value
33 |
34 | @property
35 | def __name__(self):
36 | return repr(self.extractors)
37 |
38 |
39 | def apply_extractors(descriptor, template_extractors, extractors):
40 | field_type_manager = FieldTypeManager()
41 |
42 | for field_name, field_extractors in template_extractors.items():
43 | equeue = []
44 | for eid in field_extractors:
45 | extractor_doc = extractors[eid]
46 | if "regular_expression" in extractor_doc:
47 | equeue.append(create_regex_extractor(extractor_doc["regular_expression"]))
48 | elif "type_extractor" in extractor_doc: # overrides default one
49 | descriptor.attribute_map[field_name] = SlybotFieldDescriptor(field_name,
50 | field_name, field_type_manager.type_processor_class(extractor_doc["type_extractor"])())
51 | if not field_name in descriptor.attribute_map:
52 | # if not defined type extractor, use text type by default, as it is by far the most commonly used
53 | descriptor.attribute_map[field_name] = SlybotFieldDescriptor(field_name,
54 | field_name, field_type_manager.type_processor_class("text")())
55 |
56 | if equeue:
57 | equeue.insert(0, descriptor.attribute_map[field_name].extractor)
58 | descriptor.attribute_map[field_name].extractor = PipelineExtractor(*equeue)
59 |
60 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/text.py:
--------------------------------------------------------------------------------
1 | """
2 | Text types
3 | """
4 | from scrapely.extractors import text as extract_text, safehtml
5 |
6 | class _BaseTextProcessor(object):
7 | """basic text processor, defines identity functions, some of which
8 | are overridden in subclasses
9 | """
10 | def extract(self, text):
11 | """Matches and extracts any string, as it is"""
12 | return text
13 |
14 | def adapt(self, text, htmlpage):
15 | return text
16 |
17 | class RawFieldTypeProcessor(_BaseTextProcessor):
18 | """Extracts the raw data, without processing. Data is escaped for presentation
19 |
20 | >>> from scrapely.extractors import htmlregion
21 | >>> r = RawFieldTypeProcessor()
22 | >>> html = htmlregion(u'Choose an item field
3 |
4 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/ebay3.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "init_requests": [
7 | {
8 | "type": "form",
9 | "form_url": "http://www.ebay.com/sch/ebayadvsearch/?rt=nc",
10 | "xpath": "//form[@name='adv_search_from']",
11 | "fields": [
12 | {
13 | "xpath": ".//*[@name='_nkw']",
14 | "type": "constants",
15 | "value": ["{search_string}"]
16 | },
17 | {
18 | "xpath": ".//*[@name='_in_kw']",
19 | "type": "iterate"
20 | }
21 | ]
22 | }
23 | ],
24 | "exclude_patterns": [],
25 | "respect_nofollow": true,
26 | "follow_patterns": [],
27 | "scrapes": "default",
28 | "links_to_follow": "none"
29 | }
30 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/ebay4.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "init_requests": [
7 | {
8 | "type": "form",
9 | "form_url": "http://www.ebay.com/sch/ebayadvsearch/?rt=nc",
10 | "xpath": "//form[@name='adv_search_from']",
11 | "fields": [
12 | {
13 | "xpath": ".//*[@name='_nkw']",
14 | "type": "constants",
15 | "value": ["{search_string}"]
16 | },
17 | {
18 | "xpath": ".//*[@name='_in_kw']",
19 | "type": "iterate"
20 | }
21 | ]
22 | }
23 | ],
24 | "exclude_patterns": [],
25 | "respect_nofollow": true,
26 | "follow_patterns": [],
27 | "scrapes": "default",
28 | "links_to_follow": "none"
29 | }
30 |
--------------------------------------------------------------------------------
/slybot/setup.py:
--------------------------------------------------------------------------------
1 | from slybot import __version__
2 | try:
3 | from setuptools import setup
4 | except ImportError:
5 | from distutils.core import setup
6 |
7 | install_requires = ['Scrapy', 'scrapely', 'loginform', 'lxml']
8 | tests_requires = ['jsonschema'] + install_requires
9 |
10 | setup(name='slybot',
11 | version=__version__,
12 | license='BSD',
13 | description='Slybot crawler',
14 | author='Scrapy project',
15 | author_email='info@scrapy.org',
16 | url='http://github.com/scrapy/slybot',
17 | packages=['slybot', 'slybot.fieldtypes', 'slybot.tests', 'slybot.linkextractor'],
18 | platforms = ['Any'],
19 | scripts = ['bin/slybot', 'bin/portiacrawl'],
20 | install_requires = install_requires,
21 | tests_requires = tests_requires,
22 | classifiers = ['Development Status :: 4 - Beta',
23 | 'License :: OSI Approved :: BSD License',
24 | 'Operating System :: OS Independent',
25 | 'Programming Language :: Python']
26 | )
27 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/number.py:
--------------------------------------------------------------------------------
1 | """
2 | Numeric data extraction
3 | """
4 |
5 | from scrapely.extractors import contains_any_numbers, extract_number
6 |
7 | class NumberTypeProcessor(object):
8 | """NumberTypeProcessor
9 |
10 | Extracts a number from text
11 |
12 | >>> from scrapely.extractors import htmlregion
13 | >>> n = NumberTypeProcessor()
14 | >>> n.extract(htmlregion(u"there are no numbers here"))
15 | >>> n.extract(htmlregion(u"foo 34"))
16 | u'foo 34'
17 | >>> n.adapt(u"foo 34", None)
18 | u'34'
19 |
20 | If more than one number is present, nothing is extracted
21 | >>> n.adapt(u"34 48", None) is None
22 | True
23 | """
24 | name = 'number'
25 | description = 'extracts a single number in the text passed'
26 |
27 | def extract(self, htmlregion):
28 | """Only matches and extracts strings with at least one number"""
29 | return contains_any_numbers(htmlregion.text_content)
30 |
31 | def adapt(self, text, htmlpage):
32 | return extract_number(text)
33 |
34 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/data/SampleProject/spiders/ebay2.json:
--------------------------------------------------------------------------------
1 | {
2 | "templates": [],
3 | "start_urls": [
4 | "http://http://www.ebay.com/sch/ebayadvsearch/?rt=nc"
5 | ],
6 | "init_requests": [
7 | {
8 | "type": "form",
9 | "form_url": "http://http://www.ebay.com/sch/ebayadvsearch/?rt=nc",
10 | "xpath": "//form[@name='adv_search_from']",
11 | "fields": [
12 | {
13 | "xpath": ".//*[@name='_nkw']",
14 | "type": "inurl",
15 | "value": "file://tmp/test_params.txt"
16 | },
17 | {
18 | "name": "_nkw2",
19 | "type": "inurl",
20 | "value": "file://tmp/test_params.txt"
21 | },
22 | {
23 | "xpath": ".//*[@name='_in_kw']",
24 | "type": "iterate"
25 | }
26 | ]
27 | }
28 | ],
29 | "exclude_patterns": [],
30 | "respect_nofollow": true,
31 | "follow_patterns": [],
32 | "scrapes": "default",
33 | "links_to_follow": "none"
34 | }
35 |
--------------------------------------------------------------------------------
/slybot/slybot/dupefilter.py:
--------------------------------------------------------------------------------
1 | """
2 | Duplicates filter middleware for autoscraping
3 | """
4 | from scrapy.exceptions import NotConfigured
5 | from scrapy.exceptions import DropItem
6 |
7 | from slybot.item import create_item_version
8 |
9 | class DupeFilterPipeline(object):
10 | def __init__(self, settings):
11 | if not settings.getbool('SLYDUPEFILTER_ENABLED'):
12 | raise NotConfigured
13 | self._itemversion_cache = {}
14 |
15 | @classmethod
16 | def from_crawler(cls, crawler):
17 | return cls(crawler.settings)
18 |
19 | def process_item(self, item, spider):
20 | """Checks whether a scrapy item is a dupe, based on version (not vary)
21 | fields of the item class"""
22 | if not hasattr(item, 'version_fields'):
23 | return item
24 | version = create_item_version(item)
25 | if version in self._itemversion_cache:
26 | old_url = self._itemversion_cache[version]
27 | raise DropItem("Duplicate product scraped at <%s>, first one was scraped at <%s>" % (item["url"], old_url))
28 | self._itemversion_cache[version] = item["url"]
29 | return item
30 |
31 |
--------------------------------------------------------------------------------
/slybot/slybot/linkextractor/regex.py:
--------------------------------------------------------------------------------
1 | import re
2 | from scrapy.link import Link
3 |
4 | from .base import BaseLinkExtractor
5 |
6 | # Based on http://blog.mattheworiordan.com/post/13174566389/url-regular-expression-for-links-with-or-without-the
7 | # leaves aside the fragment part, not needed for link extraction
8 | URL_DEFAULT_REGEX = r'(?:[A-Za-z0-9.\-]+|(?:www.|[-;:&=\+\$,\w]+@)[A-Za-z0-9.\-]+)(?:(?:\/[\+~%\/.\w\-_]*)?\??(?:[\-\+=&;%@.\w_]*)(?:#[.\!\/\w]*)?)?'
9 |
10 | class RegexLinkExtractor(BaseLinkExtractor):
11 | def __init__(self, regex=None, **kwargs):
12 | super(RegexLinkExtractor, self).__init__(**kwargs)
13 | self.allowed_schemes = filter(lambda x: x and isinstance(x, basestring), self.allowed_schemes)
14 | regex = regex or '(?:%s)://%s' % ('|'.join(self.allowed_schemes), URL_DEFAULT_REGEX)
15 | self.regex = re.compile(regex)
16 |
17 | def _extract_links(self, response):
18 | """First extract regex groups(). If empty, extracts from regex group()"""
19 | for s in self.regex.finditer(response.body):
20 | if s.groups():
21 | for url in s.groups():
22 | yield Link(url)
23 | else:
24 | yield Link(s.group())
25 |
26 |
27 |
--------------------------------------------------------------------------------
/slybot/slybot/linkextractor/ecsv.py:
--------------------------------------------------------------------------------
1 | import csv
2 | from cStringIO import StringIO
3 |
4 | from scrapy.link import Link
5 |
6 | from .base import BaseLinkExtractor
7 |
8 | # see http://docs.python.org/2/library/csv.html#csv-fmt-params
9 | _FORMAT_PARAMETERS = (
10 | ('delimiter', ','),
11 | ('quotechar', '"'),
12 | ('doublequote', True),
13 | ('escapechar', None),
14 | ('lineterminator', '\r\n'),
15 | ('skipinitialspace', False),
16 | ('strict', False),
17 | )
18 |
19 | class CsvLinkExtractor(BaseLinkExtractor):
20 | def __init__(self, column=0, **kwargs):
21 | self.fmtparams = dict((key, kwargs.pop(key, default)) for key, default in _FORMAT_PARAMETERS)
22 | for key, val in self.fmtparams.items():
23 | if isinstance(val, unicode):
24 | self.fmtparams[key] = val.encode()
25 | super(CsvLinkExtractor, self).__init__(**kwargs)
26 | self.allowed_schemes = filter(lambda x: x and isinstance(x, basestring), self.allowed_schemes)
27 | self.column = column
28 |
29 | def _extract_links(self, response):
30 | buff = StringIO(response.body)
31 | reader = csv.reader(buff, **self.fmtparams)
32 | for row in reader:
33 | if len(row) > self.column:
34 | yield Link(row[self.column])
35 |
36 |
--------------------------------------------------------------------------------
/slybot/slybot/linkextractor/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Link extraction for auto scraping
3 | """
4 | from scrapy.utils.misc import load_object
5 |
6 | from .base import BaseLinkExtractor, ALLOWED_SCHEMES
7 | from .html import HtmlLinkExtractor
8 | from .xml import XmlLinkExtractor, RssLinkExtractor, SitemapLinkExtractor, AtomLinkExtractor
9 | from .regex import RegexLinkExtractor
10 | from .ecsv import CsvLinkExtractor
11 |
12 | _TYPE_MAP = (
13 | # type, class, ignore value
14 | ('regex', RegexLinkExtractor, False),
15 | ('xpath', XmlLinkExtractor, False),
16 | ('column', CsvLinkExtractor, False),
17 | ('html', HtmlLinkExtractor, True),
18 | ('rss', RssLinkExtractor, True),
19 | ('sitemap', SitemapLinkExtractor, True),
20 | ('atom', AtomLinkExtractor, True),
21 | )
22 | def create_linkextractor_from_specs(specs):
23 | """Return a link extractor instance from specs. By default, return a HtmlLinkExtractor.
24 | """
25 | specs = specs.copy()
26 | ltype, value = specs.pop('type'), specs.pop('value')
27 | if ltype == 'module':
28 | cls = load_object(value)
29 | return cls(**specs)
30 | for key, cls, ignore in _TYPE_MAP:
31 | if key == ltype:
32 | if ignore:
33 | return cls(**specs)
34 | return cls(value, **specs)
35 | raise ValueError("Invalid link extractor type specification")
36 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/annotation-widget.handlebars:
--------------------------------------------------------------------------------
1 | {{#unless view.creatingField}}
2 | Item {{name}}
5 | {{#if fields}} 6 |
7 |
8 |
19 | {{else}}
20 |
23 | | Field | Type | Required | Vary | |
{{name}} |
12 | {{type}} |
13 | {{#if required}}yes{{else}}no{{/if}} |
14 | {{#if vary}}yes{{else}}no{{/if}} |
15 | {{view ASTool.ButtonView icon="ui-icon-circle-check" action="fieldSelected" argument=name class="blue-button"}} | 16 |
The item has no fields.
21 | {{/if}} 22 |
3 | {{view ASTool.ButtonView action="editAnnotation" argument=this icon="ui-icon-gear" class="blue-button"}}
4 | {{view ASTool.ButtonView action="deleteAnnotation" argument=this icon="ui-icon-trash" class="red-button"}}
5 |
6 | {{#unless view.hasMultipleMappings}}
7 | {{view view.attributeSelect}}
8 | {{view view.fieldSelect}}
9 |
10 |
13 | {{else}}
14 | Attribute value:
11 | {{view.attributeValue}} 12 |
15 | {{#each view.mappings}}
16 |
22 | {{/unless}}
23 | {{else}}
24 |
17 |
20 | {{/each}}
21 | {{this.name}}
18 | {{this.mappedField}}
19 | {{view ASTool.ButtonView icon="ui-icon-circle-plus" class="blue-button fix_plus_icon" disabled=view.createFieldDisabled}}
25 |
26 |
27 |
28 |
29 | {{view view.fieldTextField}}
30 | {{view view.typeSelect}}
31 |
32 | {{/unless}}
--------------------------------------------------------------------------------
/slyd/slyd/resource.py:
--------------------------------------------------------------------------------
1 | import json, errno
2 | from twisted.web.resource import Resource, NoResource, ErrorPage
3 |
4 |
5 | class SlydJsonResource(Resource):
6 | """Base Resource for Slyd Resources
7 |
8 | This sets the content type to JSON and handles errors
9 | """
10 |
11 | def render(self, request):
12 | request.setResponseCode(200)
13 | request.setHeader('Content-Type', 'application/json')
14 | try:
15 | return Resource.render(self, request)
16 | except IOError as ex:
17 | if ex.errno == errno.ENOENT:
18 | return NoResource().render(request)
19 | else:
20 | raise
21 | except ErrorPage as ex:
22 | return ex.render(request)
23 |
24 | def error(self, request, status, why):
25 | raise ErrorPage(request, status, why)
26 |
27 | def bad_request(self, why):
28 | self.error(400, "Bad Request", why)
29 |
30 | def read_json(self, request):
31 | try:
32 | return json.load(request.content)
33 | except ValueError as ex:
34 | self.bad_request("Error parsing json. %s" % ex.message)
35 |
36 |
37 | class SlydJsonObjectResource(SlydJsonResource):
38 | """Extends SlydJsonResource, converting
39 | the returned data to JSON
40 | """
41 |
42 | def render(self, request):
43 | resp = SlydJsonResource.render(self, request)
44 | if resp is not None:
45 | return json.dumps(resp)
46 |
--------------------------------------------------------------------------------
/slyd/media/js/controllers/navigation-controller.js:
--------------------------------------------------------------------------------
1 | ASTool.NavigationController = Em.Controller.extend({
2 |
3 | needs: ['application', 'project_index', 'spider_index', 'template_index', 'annotation'],
4 |
5 | currentPathBinding: 'controllers.application.currentPath',
6 |
7 | splittedPath: function() {
8 | return this.get('currentPath').split('.').filter(function(pathElem) {
9 | return pathElem != 'index';
10 | });
11 | }.property('currentPath'),
12 |
13 | previousRoutes: function() {
14 | var splitted = this.get('splittedPath');
15 | return splitted.slice(0, splitted.length - 1).map(function(route) {
16 | return { route: route, label: this.labelForRoute(route) };
17 | }.bind(this));
18 | }.property('splittedPath'),
19 |
20 | currentRoute: function() {
21 | var splitted = this.get('splittedPath');
22 | var route = splitted[splitted.length - 1];
23 | return { route: route, label: this.labelForRoute(route) };
24 | }.property('splittedPath'),
25 |
26 | labelForRoute: function(route) {
27 | var controllerName = this.get('needs').find(function(name) {
28 | return name.indexOf(route) == 0;
29 | });
30 | if (controllerName) {
31 | var controller = this.get('controllers.' + controllerName);
32 | return controller.get('navigationLabel');
33 | } else {
34 | return '';
35 | }
36 | },
37 |
38 | actions: {
39 |
40 | gotoRoute: function(route) {
41 | ASTool.ToolboxViewMixin.expandToolbox = true;
42 | this.transitionToRoute(route);
43 | },
44 | },
45 | });
46 |
--------------------------------------------------------------------------------
/slybot/slybot/utils.py:
--------------------------------------------------------------------------------
1 | from urlparse import urlparse
2 | import os
3 | import json
4 |
5 | from scrapely.htmlpage import HtmlPage
6 |
7 | def iter_unique_scheme_hostname(urls):
8 | """Return an iterator of tuples (scheme, hostname) over the given urls,
9 | filtering dupes
10 | """
11 | scheme_hostname = set()
12 | for x in urls:
13 | p = urlparse(x)
14 | scheme_hostname.add((p.scheme, p.hostname))
15 | return list(scheme_hostname)
16 |
17 | def open_project_from_dir(project_dir):
18 | specs = {"spiders": {}}
19 | with open(os.path.join(project_dir, "project.json")) as f:
20 | specs["project"] = json.load(f)
21 | with open(os.path.join(project_dir, "items.json")) as f:
22 | specs["items"] = json.load(f)
23 | with open(os.path.join(project_dir, "extractors.json")) as f:
24 | specs["extractors"] = json.load(f)
25 | for fname in os.listdir(os.path.join(project_dir, "spiders")):
26 | if fname.endswith(".json"):
27 | spider_name = os.path.splitext(fname)[0]
28 | with open(os.path.join(project_dir, "spiders", fname)) as f:
29 | try:
30 | specs["spiders"][spider_name] = json.load(f)
31 | except ValueError, e:
32 | raise ValueError("Error parsing spider (invalid JSON): %s: %s" % (fname, e))
33 | return specs
34 |
35 | def htmlpage_from_response(response):
36 | return HtmlPage(response.url, response.headers, \
37 | response.body_as_unicode(), encoding=response.encoding)
38 |
--------------------------------------------------------------------------------
/slybot/slybot/linkextractor/xml.py:
--------------------------------------------------------------------------------
1 | """
2 | Link extraction for auto scraping
3 | """
4 | from scrapy.link import Link
5 | from scrapy.selector import XmlXPathSelector
6 |
7 | from slybot.linkextractor.base import BaseLinkExtractor
8 |
9 | class XmlLinkExtractor(BaseLinkExtractor):
10 | """Link extractor for XML sources"""
11 | def __init__(self, xpath, **kwargs):
12 | self.remove_namespaces = kwargs.pop('remove_namespaces', False)
13 | super(XmlLinkExtractor, self).__init__(**kwargs)
14 | self.xpath = xpath
15 |
16 | def _extract_links(self, response):
17 | xxs = XmlXPathSelector(response)
18 | if self.remove_namespaces:
19 | xxs.remove_namespaces()
20 | for url in xxs.select(self.xpath).extract():
21 | yield Link(url.encode(response.encoding))
22 |
23 | class RssLinkExtractor(XmlLinkExtractor):
24 | """Link extraction from RSS feeds"""
25 | def __init__(self, **kwargs):
26 | super(RssLinkExtractor, self).__init__("//item/link/text()", **kwargs)
27 |
28 | class SitemapLinkExtractor(XmlLinkExtractor):
29 | """Link extraction for sitemap.xml feeds"""
30 | def __init__(self, **kwargs):
31 | kwargs['remove_namespaces'] = True
32 | super(SitemapLinkExtractor, self).__init__("//urlset/url/loc/text() | //sitemapindex/sitemap/loc/text()", **kwargs)
33 |
34 | class AtomLinkExtractor(XmlLinkExtractor):
35 | def __init__(self, **kwargs):
36 | kwargs['remove_namespaces'] = True
37 | super(AtomLinkExtractor, self).__init__("//link/@href", **kwargs)
38 |
39 |
--------------------------------------------------------------------------------
/slyd/media/js/templates/edit-item.handlebars:
--------------------------------------------------------------------------------
1 |
2 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | Copyright (c) Scrapinghub.
2 | All rights reserved.
3 |
4 | Redistribution and use in source and binary forms, with or without modification,
5 | are permitted provided that the following conditions are met:
6 |
7 | 1. Redistributions of source code must retain the above copyright notice,
8 | this list of conditions and the following disclaimer.
9 |
10 | 2. Redistributions in binary form must reproduce the above copyright
11 | notice, this list of conditions and the following disclaimer in the
12 | documentation and/or other materials provided with the distribution.
13 |
14 | 3. Neither the name of Portia nor the names of its contributors may be used
15 | to endorse or promote products derived from this software without
16 | specific prior written permission.
17 |
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
22 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
25 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
27 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 |
--------------------------------------------------------------------------------
/slybot/slybot/tests/test_dupefilter.py:
--------------------------------------------------------------------------------
1 | from unittest import TestCase
2 | from os.path import dirname
3 |
4 | from scrapy.http import HtmlResponse
5 | from scrapy.settings import Settings
6 | from scrapy.item import DictItem
7 | from scrapy.exceptions import DropItem
8 |
9 | from slybot.spidermanager import SlybotSpiderManager
10 | from slybot.dupefilter import DupeFilterPipeline
11 |
12 | _PATH = dirname(__file__)
13 |
14 | class DupeFilterTest(TestCase):
15 | smanager = SlybotSpiderManager("%s/data/SampleProject" % _PATH)
16 |
17 | def test_dupefilter(self):
18 | name = "seedsofchange2"
19 | spider = self.smanager.create(name)
20 | spec = self.smanager._specs["spiders"][name]
21 | t1, t2 = spec["templates"]
22 |
23 | dupefilter = DupeFilterPipeline(Settings({"SLYDUPEFILTER_ENABLED": True}))
24 |
25 | response1 = HtmlResponse(url=t1["url"], body=t1["original_body"].encode('utf-8'))
26 | response2 = HtmlResponse(url=t2["url"], body=t2["original_body"].encode('utf-8'))
27 |
28 | result1 = spider.handle_html(response1)
29 | for item1 in result1:
30 | if isinstance(item1, DictItem):
31 | break
32 |
33 | result2 = spider.handle_html(response2)
34 | for item2 in result2:
35 | if isinstance(item2, DictItem):
36 | break
37 |
38 | self.assertEqual(item1, dupefilter.process_item(item1, spider))
39 | self.assertEqual(item2, dupefilter.process_item(item2, spider))
40 |
41 | self.assertRaises(DropItem, dupefilter.process_item, item1, spider)
42 |
43 |
--------------------------------------------------------------------------------
/slyd/slyd/utils.py:
--------------------------------------------------------------------------------
1 | """
2 | html page utils
3 | """
4 | from scrapely.htmlpage import HtmlTagType
5 |
6 |
7 | def _quotify(mystr):
8 | """
9 | quotifies an html tag attribute value.
10 | Assumes then, that any ocurrence of ' or " in the
11 | string is escaped if original string was quoted
12 | with it.
13 | So this function does not altere the original string
14 | except for quotation at both ends, and is limited just
15 | to guess if string must be quoted with '"' or "'"
16 | """
17 | quote = '"'
18 | l = len(mystr)
19 | for i in range(l):
20 | if mystr[i] == "\\" and i + 1 < l and mystr[i+1] == "'":
21 | quote = "'"; break
22 | elif mystr[i] == "\\" and i + 1 < l and mystr[i+1] == '"':
23 | quote = '"'; break
24 | elif mystr[i] == "'":
25 | quote = '"'; break
26 | elif mystr[i] == '"':
27 | quote = "'"; break
28 | return quote + mystr + quote
29 |
30 |
31 | def serialize_tag(tag):
32 | """
33 | Converts a tag into a string when a slice [tag.start:tag.end]
34 | over the source can't be used because tag has been modified
35 | """
36 | out = "<"
37 | if tag.tag_type == HtmlTagType.CLOSE_TAG:
38 | out += "/"
39 | out += tag.tag
40 |
41 | attributes = []
42 | for key, val in tag.attributes.iteritems():
43 | aout = key
44 | if val is not None:
45 | aout += "=" + _quotify(val)
46 | attributes.append(aout)
47 | if attributes:
48 | out += " " + " ".join(attributes)
49 |
50 | if tag.tag_type == HtmlTagType.UNPAIRED_TAG:
51 | out += "/"
52 | return out + ">"
53 |
--------------------------------------------------------------------------------
/slyd/media/css/breadcrumb.css:
--------------------------------------------------------------------------------
1 | /* Breadcrumb Styles
2 | ---------------------------------------------------------------------*/
3 |
4 | .breadCrumb
5 | {
6 | margin: 0;
7 | float: left;
8 | display: block;
9 | height: 21px;
10 | overflow: hidden;
11 | width: 400px;
12 | padding:3px;
13 | padding-top:4px;
14 | background:#fff;
15 | }
16 | .breadCrumb ul
17 | {
18 | margin: 0;
19 | padding: 0;
20 | height: 21px;
21 | display: block;
22 | }
23 | .breadCrumb ul li
24 | {
25 | display: block;
26 | float: left;
27 | position: relative;
28 | height: 21px;
29 | overflow: hidden;
30 | line-height: 21px;
31 | margin: 0px 6px 0px 0;
32 | padding: 0px 10px 0px 0;
33 | font-size: 1em;
34 | background: url(../images/Chevron.gif) no-repeat 100% 0;
35 | }
36 | .breadCrumb ul li div.chevronOverlay
37 | {
38 | position: absolute;
39 | right: 0;
40 | top: 0;
41 | z-index: 2;
42 | }
43 | .breadCrumb ul li span
44 | {
45 | display: block;
46 | overflow: hidden;
47 | }
48 | .breadCrumb ul li a
49 | {
50 | display: block;
51 | position: relative;
52 | height: 21px;
53 | line-height: 21px;
54 | overflow: hidden;
55 | float: left;
56 | }
57 | .breadCrumb ul li.first a
58 | {
59 | height: 16px !important;
60 | text-indent:-1000em;
61 | width:16px;
62 | padding: 0;
63 | margin-top: 2px;
64 | overflow: hidden;
65 | background:url(../images/IconHome.gif) no-repeat 0 0;
66 | }
67 | .breadCrumb ul li.first a:hover
68 | {
69 | background-position: 0 -16px;
70 | }
71 | .breadCrumb ul li.last
72 | {
73 | background: none;
74 | margin-right: 0;
75 | padding-right: 0;
76 | }
77 | .chevronOverlay
78 | {
79 | display: none;
80 | background: url(../images/ChevronOverlay.png) no-repeat 100% 0;
81 | width: 13px;
82 | height: 20px;
83 | }
84 |
--------------------------------------------------------------------------------
/slybot/slybot/spidermanager.py:
--------------------------------------------------------------------------------
1 | import tempfile, shutil, atexit
2 | from zipfile import ZipFile
3 |
4 | from zope.interface import implements
5 | from scrapy.interfaces import ISpiderManager
6 | from scrapy.utils.misc import load_object
7 |
8 | from slybot.spider import IblSpider
9 | from slybot.utils import open_project_from_dir
10 |
11 | class SlybotSpiderManager(object):
12 |
13 | implements(ISpiderManager)
14 |
15 | def __init__(self, datadir, spider_cls=None):
16 | self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
17 | self._specs = open_project_from_dir(datadir)
18 |
19 | @classmethod
20 | def from_crawler(cls, crawler):
21 | datadir = crawler.settings['PROJECT_DIR']
22 | spider_cls = crawler.settings['SLYBOT_SPIDER_CLASS']
23 | return cls(datadir, spider_cls)
24 |
25 | def create(self, name, **args):
26 | spec = self._specs["spiders"][name]
27 | items = self._specs["items"]
28 | extractors = self._specs["extractors"]
29 | return self.spider_cls(name, spec, items, extractors, **args)
30 |
31 | def list(self):
32 | return self._specs["spiders"].keys()
33 |
34 | class ZipfileSlybotSpiderManager(SlybotSpiderManager):
35 |
36 | def __init__(self, datadir, zipfile=None, spider_cls=None):
37 | if zipfile:
38 | datadir = tempfile.mkdtemp(prefix='slybot-')
39 | ZipFile(zipfile).extractall(datadir)
40 | atexit.register(shutil.rmtree, datadir)
41 | super(ZipfileSlybotSpiderManager, self).__init__(datadir, spider_cls)
42 |
43 | @classmethod
44 | def from_crawler(cls, crawler):
45 | s = crawler.settings
46 | sm = cls(s['PROJECT_DIR'], s['PROJECT_ZIPFILE'], s['SLYBOT_SPIDER_CLASS'])
47 | return sm
48 |
--------------------------------------------------------------------------------
/slyd/media/js/app.js:
--------------------------------------------------------------------------------
1 | /*************************** Application **************************/
2 | ASTool = Em.Application.create({
3 | LOG_TRANSITIONS: true,
4 | ready: function(){
5 |
6 | }
7 | });
8 |
9 | // Leave 'null' for using window.location. Define it to override.
10 | var SLYD_URL = null;
11 |
12 | Ember.Application.initializer({
13 | name: 'slydApiInitializer',
14 |
15 | initialize: function(container, application) {
16 | container.register('api:slyd', ASTool.SlydApi);
17 | application.inject('route', 'slyd', 'api:slyd');
18 | application.inject('adapter', 'slyd', 'api:slyd');
19 | application.inject('controller', 'slyd', 'api:slyd');
20 | }
21 | });
22 |
23 |
24 | Ember.Application.initializer({
25 | name: 'documentViewInitializer',
26 |
27 | initialize: function(container, application) {
28 | container.register('document:view', ASTool.DocumentView);
29 | application.inject('controller', 'documentView', 'document:view');
30 | }
31 | });
32 |
33 |
34 | Ember.Application.initializer({
35 | name: 'annotationsStoreInitializer',
36 |
37 | initialize: function(container, application) {
38 | container.register('annotations:store', ASTool.AnnotationsStore);
39 | application.inject('route', 'annotationsStore', 'annotations:store');
40 | application.inject('controller', 'annotationsStore', 'annotations:store');
41 | }
42 | });
43 |
44 |
45 | function s4() {
46 | return Math.floor((1 + Math.random()) * 0x10000)
47 | .toString(16)
48 | .substring(1);
49 | };
50 |
51 | function guid() {
52 | return s4() + s4() + '-' + s4() + '-' + s4() + '-' +
53 | s4() + '-' + s4() + s4() + s4();
54 | }
55 |
56 | function shortGuid() {
57 | return s4() + '.' + s4() + '.' + s4();
58 | }
59 |
60 | ASTool.guid = guid;
61 | ASTool.shortGuid = shortGuid;
--------------------------------------------------------------------------------
/slyd/bin/sh2sly:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """
3 | Downloads AS projects from scrapinghub in slybot format and saves them locally
4 |
5 | Used for testing.
6 |
7 | This uses the python-scrapinghub library
8 | """
9 |
10 |
11 | def sh2sly(apikey, projects, destination):
12 | from scrapinghub import Connection, Project
13 | import tempfile, zipfile, os
14 | conn = Connection(apikey)
15 | if not projects:
16 | projects = conn.project_ids()
17 | #projects = xrange(1300)
18 | out = tempfile.TemporaryFile()
19 | for projectid in projects:
20 | project = Project(conn, projectid)
21 | project.autoscraping_project_slybot(outputfile=out)
22 | try:
23 | zf = zipfile.ZipFile(out)
24 | scount = len([f for f in zf.namelist() if f.startswith('spiders/')])
25 | if scount == 0:
26 | print "skipping %s - no spiders" % projectid
27 | continue
28 | pdest = os.path.join(destination, str(projectid))
29 | zf.extractall(pdest)
30 | print "extracted project with %d spiders to %s" % (scount, pdest)
31 | except zipfile.BadZipfile:
32 | print "error with zipfile in project %s, skipping" % projectid
33 | out.truncate()
34 |
35 |
36 | def main():
37 | import argparse
38 | parser = argparse.ArgumentParser(
39 | description='fetch slybot projects from scrapinghub')
40 | parser.add_argument('-p', '--project', type=int, action='append',
41 | help='scrapinghub project, may be repeated, default is all')
42 | parser.add_argument('-k', '--apikey',
43 | help='api key')
44 | parser.add_argument('destination')
45 | args = parser.parse_args()
46 | sh2sly(args.apikey, args.project, args.destination)
47 |
48 |
49 | if __name__ == '__main__':
50 | main()
51 |
--------------------------------------------------------------------------------
/slyd/media/js/controllers/items-controller.js:
--------------------------------------------------------------------------------
1 | ASTool.ItemsController = Em.ArrayController.extend(ASTool.BaseControllerMixin, {
2 |
3 | needs: ['application', 'annotation'],
4 |
5 | documentView: null,
6 |
7 | addItem: function() {
8 | var newItem = ASTool.Item.create({ name: ASTool.shortGuid() });
9 | this.addField(newItem);
10 | this.pushObject(newItem);
11 | },
12 |
13 | addField: function(owner, name, type) {
14 | var newField = ASTool.ItemField.create({ name: name || 'new field',
15 | type: type || 'text',
16 | required: false,
17 | vary: false });
18 | owner.set('fields', owner.fields || []);
19 | owner.fields.pushObject(newField);
20 | },
21 |
22 | saveChanges: function() {
23 | this.get('slyd').saveItems(this.content.toArray()).then(function() {
24 | this.transitionToRoute('template');
25 | }.bind(this));
26 | },
27 |
28 | actions: {
29 |
30 | addItem: function() {
31 | this.addItem();
32 | },
33 |
34 | addField: function(item) {
35 | this.addField(item);
36 | },
37 |
38 | deleteItem: function(item) {
39 | this.removeObject(item);
40 | },
41 |
42 | deleteField: function(item, field) {
43 | item.get('fields').removeObject(field);
44 | },
45 |
46 | saveChanges: function() {
47 | this.saveChanges();
48 | },
49 |
50 | undoChanges: function() {
51 | this.get('slyd').loadItems().then(function(items) {
52 | this.set('content', items);
53 | this.transitionToRoute('template');
54 | }.bind(this));
55 | },
56 | },
57 |
58 | willEnter: function() {
59 | if (this.get('documentView.canvas')) {
60 | this.set('documentView.canvas.interactionsBlocked', true);
61 | }
62 | },
63 |
64 | willLeave: function() {
65 | if (this.get('documentView.canvas')) {
66 | this.set('documentView.canvas.interactionsBlocked', false);
67 | }
68 | },
69 | });
--------------------------------------------------------------------------------
/slybot/bin/portiacrawl:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 | """Allow to easily run slybot spiders on console. If spider is not given, print a list of available spiders inside the project"""
3 | import os
4 | import subprocess
5 | from optparse import OptionParser
6 |
7 | def main():
8 | parser = OptionParser(description=__doc__,
9 | usage="%prog
3 |
4 | {{#view ASTool.InlineTextField value=name}}
5 | Item {{name}}
6 | {{/view}}
7 |
8 |
9 | {{view ASTool.ButtonView action="deleteItem" argument=this icon="ui-icon-trash" class="red-button"}}
10 |
11 |
12 |
13 | {{#if view.item.fields}}
14 |
15 |
30 | {{else}}
31 |
37 | | Field | Type | Required | Vary | |
| 19 | {{#view ASTool.InlineTextField value=name}} 20 | {{name}} 21 | {{/view}} 22 | | 23 |{{view ASTool.TypeSelect name="fieldType" value=type itemField=this}} | 24 |{{view ASTool.CheckBox checked=required name="fieldRequired"}} | 25 |{{view ASTool.CheckBox checked=vary name="fieldVary"}} | 26 |{{view ASTool.ButtonView icon="ui-icon-trash" class="red-button" action="deleteField" argument=view.item argument2=this}} | 27 |
No fields defined yet.
32 | {{/if}} 33 |
34 | {{view ASTool.ButtonView action="addField" argument=this icon="ui-icon-circle-plus" label="Field" class="blue-button"}}
35 |
36 |
3 | Map attribute
4 | To field
5 |
15 | {{#unless view.hasMultipleMappings}}
16 |
6 | {{view ASTool.InlineHelp message="annotation_widget"}}
7 |
8 |
9 | {{view ASTool.ButtonView action="editAnnotation" argument=controller.floatingAnnotation icon="ui-icon-gear" class="clear-button"}}
10 |
11 |
12 | {{view ASTool.ButtonView action="deleteAnnotation" argument=controller.floatingAnnotation icon="ui-icon-trash" class="clear-button"}}
13 |
14 |
17 | {{view view.attributeSelect}}
18 | {{view view.fieldSelect}}
19 |
23 | {{else}}
24 |
20 | {{view ASTool.ButtonView icon="ui-icon-circle-plus" class="clear-button" label="field"}}
21 |
22 |
25 | {{#each view.mappings}}
26 |
32 | {{/unless}}
33 | {{else}}
34 |
35 |
36 |
27 |
30 | {{/each}}
31 | {{this.name}}
28 | {{this.mappedField}}
29 |
37 | {{view view.fieldTextField width=115 class="darktextfield"}}
38 | {{view view.typeSelect}}
39 | {{view ASTool.ButtonView icon="ui-icon-circle-check" class="clear-button" disabled=view.createFieldDisabled}}
40 |
41 | {{/unless}}
42 |
--------------------------------------------------------------------------------
/slybot/slybot/fieldtypes/__init__.py:
--------------------------------------------------------------------------------
1 | """
2 | Field Types
3 |
4 | Spiders extracts items of a given type. These item types are defined by a
5 | schema, which specifies the type of each field in the item. This module
6 | contains FieldProcessor implementations, which are the classes responsible for
7 | custom processing of these types.
8 |
9 | We keep the types of scrapers supported flexible and allow different methods
10 | for each. In the future, we expect many different types, for example one might
11 | be a mechanical turk scraper and the fields would have to do user validation
12 | and provide error messages.
13 | """
14 | from .text import (
15 | RawFieldTypeProcessor,
16 | TextFieldTypeProcessor,
17 | SafeHtmlFieldTypeProcessor
18 | )
19 | from .images import ImagesFieldTypeProcessor
20 | from .url import UrlFieldTypeProcessor
21 | from .number import NumberTypeProcessor
22 | from .point import GeoPointFieldTypeProcessor
23 | from .price import PriceTypeProcessor
24 |
25 | class FieldTypeManager(object):
26 | _TYPEMAP = dict((c.name, c) for c in (
27 | RawFieldTypeProcessor, TextFieldTypeProcessor,
28 | ImagesFieldTypeProcessor, NumberTypeProcessor,
29 | UrlFieldTypeProcessor, SafeHtmlFieldTypeProcessor,
30 | GeoPointFieldTypeProcessor, PriceTypeProcessor,
31 | ))
32 | _names = sorted(_TYPEMAP.keys())
33 |
34 | def available_type_names(self):
35 | """Find the names of all field types available. """
36 | return self._names
37 |
38 | def type_processor_class(self, name):
39 | """Retrieve the class for the given extractor
40 |
41 | This can be useful to introspect on the constructor arguments. If no
42 | suitable type is found, it will default to the RawFieldTypeProcessor
43 | (no processing of extracted data is done).
44 | """
45 | return self._TYPEMAP.get(name, RawFieldTypeProcessor)
46 |
47 | def all_processor_classes(self):
48 | """Retrieve all processor classes registered"""
49 | return self._TYPEMAP.values()
50 |
51 |
--------------------------------------------------------------------------------
/slybot/slybot/validation/schema.py:
--------------------------------------------------------------------------------
1 | """Simple validation of specifications passed to slybot"""
2 | from os.path import dirname, join
3 | import json, re
4 | from urlparse import urlparse
5 |
6 | from jsonschema import Draft3Validator, ValidationError, RefResolver
7 |
8 | _PATH = dirname(__file__)
9 |
10 | def load_schemas():
11 | filename = join(_PATH, "schemas.json")
12 | return dict((s["id"], s) for s in json.load(open(filename)))
13 |
14 | _SCHEMAS = load_schemas()
15 |
16 | class SlybotJsonSchemaValidator(Draft3Validator):
17 | DEFAULT_TYPES = Draft3Validator.DEFAULT_TYPES.copy()
18 | DEFAULT_TYPES.update({
19 | "mapping": dict,
20 | })
21 | def validate_format(self, fmt, instance, schema):
22 | if schema["type"] != "string":
23 | raise ValidationError("Invalid keyword 'format' for type '%s'" % schema["type"])
24 |
25 | if fmt == "regex":
26 | try:
27 | re.compile(instance)
28 | except:
29 | raise ValidationError("Invalid regular expression: %s" % repr(instance))
30 | elif fmt == "url":
31 | parsed = urlparse(instance)
32 | if not parsed.scheme or not parsed.netloc:
33 | raise ValidationError("Invalid url: '%s'" % repr(instance))
34 |
35 | return None
36 |
37 | def get_schema_validator(schema):
38 | resolver = RefResolver("", schema, _SCHEMAS)
39 | return SlybotJsonSchemaValidator(_SCHEMAS[schema], resolver=resolver)
40 |
41 | def validate_project_schema(specs):
42 |
43 | project = specs["project"]
44 | get_schema_validator("project").validate(project)
45 |
46 | items = specs["items"]
47 | get_schema_validator("items").validate(items)
48 |
49 | extractors = specs["extractors"]
50 | get_schema_validator("extractors").validate(extractors)
51 |
52 | spider_schema_validator = get_schema_validator("spider")
53 | for spider in specs["spiders"].values():
54 | spider_schema_validator.validate(spider)
55 |
56 | return True
57 |
58 |
--------------------------------------------------------------------------------
/slyd/slyd/tap.py:
--------------------------------------------------------------------------------
1 | """
2 | The module is used by the Twisted plugin system
3 | (twisted.plugins.slyd_plugin) to register twistd command to manage
4 | slyd server. The command can be used with 'twistd slyd'.
5 | """
6 | from os.path import join, dirname
7 | from twisted.python import usage
8 | from twisted.web.resource import Resource
9 | from twisted.application.internet import TCPServer
10 | from twisted.web.server import Site
11 | from twisted.web.static import File
12 |
13 | DEFAULT_PORT = 9001
14 | DEFAULT_DOCROOT = join(dirname(dirname(__file__)), 'media')
15 |
16 |
17 | class Options(usage.Options):
18 | optParameters = [
19 | ['port', 'p', DEFAULT_PORT, 'Port number to listen on.', int],
20 | ['docroot', 'd', DEFAULT_DOCROOT, 'Default doc root for static media.'],
21 | ]
22 |
23 |
24 | def create_root(config):
25 | from scrapy import log
26 | from scrapy.settings import CrawlerSettings
27 | from slyd.crawlerspec import (CrawlerSpecManager,
28 | create_crawler_spec_resource)
29 | from slyd.bot import create_bot_resource
30 | import slyd.settings
31 | from slyd.projects import ProjectsResource
32 |
33 | root = Resource()
34 | root.putChild("static", File(config['docroot']))
35 |
36 | crawler_settings = CrawlerSettings(settings_module=slyd.settings)
37 | spec_manager = CrawlerSpecManager(crawler_settings)
38 |
39 | # add project management at /projects
40 | projects = ProjectsResource(crawler_settings)
41 | root.putChild('projects', projects)
42 |
43 | # add crawler at /projects/PROJECT_ID/bot
44 | log.msg("Slybot specs loading from %s/[PROJECT]" % spec_manager.basedir,
45 | level=log.DEBUG)
46 | projects.putChild("bot", create_bot_resource(spec_manager))
47 |
48 | # add spec at /projects/PROJECT_ID/spec
49 | spec = create_crawler_spec_resource(spec_manager)
50 | projects.putChild("spec", spec)
51 | return root
52 |
53 |
54 | def makeService(config):
55 | root = create_root(config)
56 | site = Site(root)
57 | return TCPServer(config['port'], site)
58 |
--------------------------------------------------------------------------------
/slybot/slybot/closespider.py:
--------------------------------------------------------------------------------
1 | """
2 | This extension closes spiders after they have been crawling inefficiently for a
3 | while
4 | Each SLYCLOSE_SPIDER_CHECK_PERIOD seconds, it checks that at least SLYCLOSE_SPIDER_PERIOD_ITEMS
5 | have been extracted along the last time interval of same length.
6 | """
7 |
8 | from twisted.internet import task
9 |
10 | from scrapy.xlib.pydispatch import dispatcher
11 | from scrapy import signals
12 | from scrapy.exceptions import NotConfigured
13 |
14 | DEFAULT_CHECK_PERIOD = 3600
15 | DEFAULT_PERIOD_MIN_ITEMS = 200
16 |
17 | class SlybotCloseSpider(object):
18 |
19 | def __init__(self, crawler):
20 | if not crawler.settings.getbool("SLYCLOSE_SPIDER_ENABLED"):
21 | raise NotConfigured
22 |
23 | self.crawler = crawler
24 | self.check_period = crawler.settings.getint("SLYCLOSE_SPIDER_CHECK_PERIOD", DEFAULT_CHECK_PERIOD)
25 | self.period_items = crawler.settings.getint("SLYCLOSE_SPIDER_PERIOD_ITEMS", DEFAULT_PERIOD_MIN_ITEMS)
26 |
27 | self.items_in_period = 0
28 |
29 | dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
30 | dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
31 | dispatcher.connect(self.item_scraped, signal=signals.item_scraped)
32 |
33 | def spider_opened(self, spider):
34 | self.task = task.LoopingCall(self._check_crawled_items, spider)
35 | self.task.start(self.check_period, now=False)
36 |
37 | def spider_closed(self, spider):
38 | if self.task.running:
39 | self.task.stop()
40 |
41 | def item_scraped(self, item, spider):
42 | self.items_in_period += 1
43 |
44 | def _check_crawled_items(self, spider):
45 | if self.items_in_period >= self.period_items:
46 | self.items_in_period = 0
47 | else:
48 | spider.log("Closing spider because of low item throughput. Items in last period: %d" % self.items_in_period)
49 | self.crawler.engine.close_spider(spider, 'slybot_fewitems_scraped')
50 |
51 | @classmethod
52 | def from_crawler(cls, crawler):
53 | return cls(crawler)
54 |
--------------------------------------------------------------------------------
/slyd/media/tests/fixtures.js:
--------------------------------------------------------------------------------
1 | var spiderNamesJson = ['spider1'];
2 |
3 | var spider1Json = {
4 | "allowed_domains": [],
5 | "exclude_patterns": [],
6 | "follow_patterns": [],
7 | "init_requests": [],
8 | "links_to_follow": "none",
9 | "respect_nofollow": true,
10 | "start_urls": [
11 | "http://site1.com",
12 | "http://site2.com",
13 | ],
14 | "templates": [
15 | {
16 | "name": "t1",
17 | "annotated_body": "Some content
More content
Some content
More content
Some content
More content
Some content
More content
HELLO
",
40 | items: [],
41 | };
42 |
43 | var itemsJson = {
44 | "default": {
45 | "fields": {
46 | "description": {
47 | "required": false,
48 | "type": "safe html",
49 | "vary": false
50 | },
51 | "images": {
52 | "required": true,
53 | "type": "image",
54 | "vary": true
55 | },
56 | }
57 | }
58 | };
--------------------------------------------------------------------------------
/slybot/slybot/baseurl.py:
--------------------------------------------------------------------------------
1 | """
2 | html page utils
3 | """
4 | import urlparse, re
5 | from scrapely.htmlpage import parse_html, HtmlTagType
6 |
7 | ABSURLRE = re.compile("^https?\:\/\/")
8 | DOCTYPERE = re.compile("", re.S | re.I)
9 |
10 | def _is_abs_url(url):
11 | return bool(ABSURLRE.match(url))
12 |
13 | def insert_base_url(html, base):
14 | """
15 | Inserts the given base url if does not exist in html source,
16 | or replace the existing if needed
17 | """
18 | baseurl = baseelement = headelement = htmlelement = None
19 | for element in parse_html(html):
20 | if getattr(element, "tag", None) == "base":
21 | baseurl = element.attributes.get("href", None)
22 | baseelement = element
23 | elif getattr(element, "tag", None) == "head" and \
24 | element.tag_type == HtmlTagType.OPEN_TAG:
25 | headelement = element
26 | elif getattr(element, "tag", None) == "html" and \
27 | element.tag_type == HtmlTagType.OPEN_TAG:
28 | htmlelement = element
29 |
30 | if baseurl:
31 | if not _is_abs_url(baseurl):
32 | absurl = urlparse.urljoin(base, baseurl)
33 | # replace original base tag
34 | basetag = 'Type extractors override the type specified in the item definition.', 9 | select_item: 'You can choose what item type is extracted by this template using the combobox. You can also create and modify items by clicking on the Edit Items button.', 10 | variant: 'By selecting a different variant than Base(0) in your annotation, the resulting extracted data will be assigned to the base item special field variants, which is a list of objects similar to an item.', 11 | ignored_subregions: 'Allows you to define subregions that should be excluded from the extraction process.', 12 | selected_region_ancestors: 'Refine your selection by navigating its ancestors.', 13 | selected_region_children: 'Refine your selection by navigating its children.', 14 | sticky_fields: 'Required attributes are not extracted, but they must be present for a page to match the template.', 15 | annotation_widget: 'Select the attribute you want to extract and an item field to map it.
Choose -just required- to indicate that the template must match a particular feature without generating any extracted data.
You can create new fields by clicking the + field button or by seleting the -create new- option from the field combobox.', 16 | 17 | // Other messages. 18 | confirm_change_selection: 'If you select a different region you will lose all current attribute mappings and ignored subregions, proceed anyway?', 19 | }); 20 | -------------------------------------------------------------------------------- /slyd/media/js/controllers/projects-controller.js: -------------------------------------------------------------------------------- 1 | ASTool.ProjectsIndexController = Em.ArrayController.extend(ASTool.BaseControllerMixin, { 2 | needs: ['application'], 3 | 4 | projectSite: null, 5 | 6 | createProjectDisabled: function() { 7 | return Em.isEmpty(this.get('projectSite')); 8 | }.property('projectSite'), 9 | 10 | actions: { 11 | 12 | openProject: function(projectName) { 13 | this.set('slyd.project', projectName); 14 | this.transitionToRoute('project', { id: projectName }); 15 | }, 16 | 17 | deleteProject: function(projectName) { 18 | if (confirm('Are you sure you want to delete this project? This operation cannot be undone.')) { 19 | this.get('slyd').deleteProject(projectName); 20 | this.removeObject(projectName); 21 | } 22 | }, 23 | 24 | createProject: function() { 25 | var newProjectName = this.getUnusedName('new_project', this.get('content')); 26 | this.get('slyd').createProject(newProjectName).then(function() { 27 | this.set('slyd.project', newProjectName); 28 | // Initialize items spec. 29 | this.get('slyd').saveItems([ 30 | ASTool.Item.create({ name: 'default', fields: [ ] 31 | }) 32 | ]); 33 | // Initialize extractors spec. 34 | this.get('slyd').saveExtractors([]); 35 | // Setup automatic creation of an initial spider. 36 | this.set('controllers.application.siteWizard', this.get('projectSite')); 37 | this.set('projectSite', null); 38 | this.transitionToRoute('project', { id: newProjectName }); 39 | }.bind(this)); 40 | } 41 | }, 42 | 43 | animateProjectSiteInput: function() { 44 | var animateBorderColor = function () { 45 | $('#projectSiteTextField') 46 | .animate({ 'border-color': 'rgba(88,150,220,0.4)', 'background-color': 'rgba(130,210,230,0.1)' }, 1000) 47 | .animate({ 'border-color': '#BBBBB', 'background-color': '#FFFFFF' }, 1000, animateBorderColor) 48 | }; 49 | Em.run.schedule('afterRender', this, function() { 50 | $('#projectSiteTextField') 51 | .hide() 52 | .show('fast', animateBorderColor) 53 | .click(function(e) { 54 | $('#projectSiteTextField').stop(true) 55 | .css({ 'border-color': '#BBBBB', 'background-color': '#FFFFFF' }); 56 | }); 57 | }); 58 | }, 59 | 60 | willEnter: function() { 61 | this.get('documentView').showSpider(); 62 | if (Em.isEmpty(this.get('content'))) { 63 | this.animateProjectSiteInput(); 64 | } 65 | } 66 | }); 67 | -------------------------------------------------------------------------------- /slybot/slybot/item.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from collections import defaultdict 3 | 4 | from scrapy.item import DictItem, Field 5 | from scrapely.descriptor import ItemDescriptor, FieldDescriptor 6 | 7 | from slybot.fieldtypes import FieldTypeManager 8 | 9 | class SlybotItem(DictItem): 10 | # like DictItem.__setitem__ but doesn't check the field is declared 11 | def __setitem__(self, name, value): 12 | self._values[name] = value 13 | @classmethod 14 | def create_iblitem_class(cls, schema): 15 | class IblItem(cls): 16 | fields = defaultdict(dict) 17 | version_fields = [] 18 | for _name, _meta in schema['fields'].items(): 19 | fields[_name] = Field(_meta) 20 | if not _meta.get("vary", False): 21 | version_fields.append(_name) 22 | version_fields = sorted(version_fields) 23 | return IblItem 24 | 25 | def create_slybot_item_descriptor(schema): 26 | field_type_manager = FieldTypeManager() 27 | descriptors = [] 28 | for pname, pdict in schema['fields'].items(): 29 | required = pdict['required'] 30 | pclass = field_type_manager.type_processor_class(pdict['type']) 31 | processor = pclass() 32 | descriptor = SlybotFieldDescriptor(pname, pname, processor, required) 33 | descriptors.append(descriptor) 34 | return ItemDescriptor("", "", descriptors) 35 | 36 | class SlybotFieldDescriptor(FieldDescriptor): 37 | """Extends the scrapely field descriptor to use slybot fieldtypes and 38 | to be created from a slybot item schema 39 | """ 40 | 41 | def __init__(self, name, description, field_type_processor, required=False): 42 | """Create a new SlybotFieldDescriptor with the given name and description. 43 | The field_type_processor is used for extraction and is publicly available 44 | """ 45 | FieldDescriptor.__init__(self, name, description, 46 | field_type_processor.extract, required) 47 | # add an adapt method 48 | self.adapt = field_type_processor.adapt 49 | 50 | def create_item_version(item): 51 | """Item version based on hashlib.sha1 algorithm""" 52 | if not item.version_fields: 53 | return 54 | _hash = hashlib.sha1() 55 | for attrname in item.version_fields: 56 | _hash.update(repr(item.get(attrname))) 57 | return _hash.digest() 58 | 59 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | portia 2 | ====== 3 | Visual scraping for Scrapy. 4 | 5 | 6 | Overview 7 | ======== 8 | 9 | Portia is a tool for visually scraping web sites without any programming knowledge. Just annotate web pages with a point and click editor to indicate what data you want to extract, and portia will learn how to scrape similar pages 10 | from the site. 11 | 12 | Portia has a web based UI served by a [Twisted] server, so you can install it on almost any modern platform. 13 | 14 | Requirements 15 | ============ 16 | 17 | * Python 2.7 18 | * Works on Linux, Windows, Mac OSX, BSD 19 | * Supported browsers: Latest versions of Chrome (recommended) or Firefox 20 | 21 | 22 | Repository structure 23 | ==================== 24 | 25 | There are two main components in this repository, __slyd__ and __slybot__: 26 | 27 | ###slyd 28 | 29 | The visual editor used to create your scraping projects. 30 | 31 | ###slybot 32 | 33 | The Python web crawler that performs the actual site scraping. It's implemented on top of the [Scrapy] web crawling 34 | framework and the [Scrapely] extraction library. It uses projects created with __slyd__ as input. 35 | 36 | 37 | How to install portia 38 | ============================= 39 | 40 | The recommended way to install dependencies is to use __virtualenv__ and then do: 41 | 42 | cd slyd 43 | pip install -r requirements.txt 44 | 45 | As __slybot__ is a __slyd__ dependency, it will also get installed. 46 | 47 | Running portia 48 | ============== 49 | 50 | First, you need to start the ui and create a project. Run __slyd__ using: 51 | 52 | cd slyd 53 | twistd -n slyd 54 | 55 | and point your browser to: `http://localhost:9001/static/main.html` 56 | 57 | Choose the site you want to scrape and create a project. Every project is created with a default spider named after the domain of the site you are scraping. When you are ready, you can run your project with __slybot__ to do the actual crawling/extraction. 58 | 59 | Projects created with __slyd__ can be found at: 60 | 61 | slyd/data/projects 62 | 63 | To run one of those projects use: 64 | 65 | portiacrawl project_path spidername 66 | 67 | Where `spidername` should be one of the project spiders. If you don't remember the name of the spider, just use: 68 | 69 | portiacrawl project_path 70 | 71 | and you will get the list of spiders for that project. 72 | 73 | 74 | [Twisted]: https://twistedmatrix.com 75 | [Scrapely]: https://github.com/scrapy/scrapely 76 | [Scrapy]: http://scrapy.org 77 | -------------------------------------------------------------------------------- /slybot/slybot/tests/test_schema_validation.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from unittest import TestCase 4 | from os.path import dirname, join 5 | 6 | from slybot.validation.schema import get_schema_validator, \ 7 | ValidationError, validate_project_schema 8 | from slybot.utils import open_project_from_dir 9 | 10 | _TEST_PROJECT_DIR = join(dirname(__file__), "data/SampleProject") 11 | 12 | class JsonSchemaTest(TestCase): 13 | 14 | def assertRaisesRegexp(self, eclass, pattern, func, *args): 15 | """assertRaisesRegexp is not provided in python versions below 2.7""" 16 | try: 17 | func(*args) 18 | except eclass, e: 19 | m = re.search(pattern, e.message) 20 | if not m: 21 | raise AssertionError('"%s" does not match "%s"' % (pattern, e.message)) 22 | else: 23 | raise AssertionError("%s not raised" % eclass.__name__) 24 | 25 | def test_regex_formatting_wrong(self): 26 | obj = { 27 | "0": { 28 | "regular_expression": "Item: (\d+" 29 | } 30 | } 31 | validator = get_schema_validator("extractors") 32 | self.assertRaisesRegexp(ValidationError, "Invalid regular expression", 33 | validator.validate, obj) 34 | 35 | def test_regex_formatting_ok(self): 36 | obj = { 37 | "0": { 38 | "regular_expression": "Item: (\d+)" 39 | } 40 | } 41 | validator = get_schema_validator("extractors") 42 | self.assertEqual(validator.validate(obj), None) 43 | 44 | def test_valid_url(self): 45 | obj = { 46 | "start_urls": ['http://www.example.com/'], 47 | "links_to_follow": "none", 48 | "respect_nofollow": True, 49 | "templates": [], 50 | } 51 | validator = get_schema_validator("spider") 52 | self.assertEqual(validator.validate(obj), None) 53 | 54 | def test_invalid_url(self): 55 | obj = { 56 | "start_urls": ['www.example.com'], 57 | "links_to_follow": "none", 58 | "respect_nofollow": True, 59 | "templates": [], 60 | } 61 | validator = get_schema_validator("spider") 62 | self.assertRaisesRegexp(ValidationError, "Invalid url:", validator.validate, obj) 63 | 64 | def test_test_project(self): 65 | specs = open_project_from_dir(_TEST_PROJECT_DIR) 66 | self.assertTrue(validate_project_schema(specs)) 67 | 68 | -------------------------------------------------------------------------------- /slyd/tests/test_bot.py: -------------------------------------------------------------------------------- 1 | import json 2 | from os.path import join 3 | from twisted.trial import unittest 4 | from twisted.internet.defer import inlineCallbacks 5 | from twisted.web.server import Site 6 | from twisted.web.static import File 7 | from twisted.internet import reactor 8 | from slyd.bot import create_bot_resource 9 | from .utils import TestSite, test_spec_manager 10 | from .settings import RESOURCE_DIR 11 | 12 | 13 | class BotTest(unittest.TestCase): 14 | def setUp(self): 15 | # configure bot resource 16 | sm = test_spec_manager() 17 | self.bot_resource = create_bot_resource(sm) 18 | self.botsite = TestSite(self.bot_resource) 19 | 20 | # configure fake website to crawl 21 | docroot = join(RESOURCE_DIR, 'docroot') 22 | factory = Site(File(docroot)) 23 | self.listen_port = reactor.listenTCP(8997, factory) 24 | 25 | 26 | def _fetch(self, url, **params): 27 | req = dict(params) 28 | req.setdefault('request', {})['url'] = url 29 | request_json = json.dumps(req) 30 | return self.botsite.post('fetch', data=request_json) 31 | 32 | @inlineCallbacks 33 | def test_fetch(self): 34 | # test status code 35 | result = yield self._fetch("http://localhost:8997/notexists") 36 | self.assertEqual(result.responseCode, 200) 37 | status = json.loads(result.value())['response']['status'] 38 | self.assertEqual(status, 404) 39 | 40 | # get an existing file 41 | test_url = "http://localhost:8997/test.html" 42 | result = yield self._fetch(test_url) 43 | self.assertEqual(result.responseCode, 200) 44 | value = json.loads(result.value()) 45 | # expect 200 response and base href added 46 | self.assertEqual(value['response']['status'], 200) 47 | self.assertIn('
45
test
') 23 | >>> r.extract(html) 24 | u'test
' 25 | >>> r.adapt(html, None) 26 | u'test
' 27 | """ 28 | name = 'raw html' 29 | description = 'raw html as it appears in the page' 30 | 31 | class TextFieldTypeProcessor(_BaseTextProcessor): 32 | """Extracts strings, removing all HTML markup 33 | 34 | >>> from scrapely.extractors import htmlregion 35 | >>> p = TextFieldTypeProcessor() 36 | >>> html = htmlregion(u'test
!') 37 | >>> extracted = p.extract(html) 38 | >>> extracted 39 | u'test !' 40 | >>> p.adapt(extracted, None) 41 | u'test !' 42 | >>> html = htmlregion(u'\\n
') 43 | >>> p.extract(html) 44 | u'' 45 | """ 46 | name = 'text' 47 | description = 'extracts text from web pages, cleaning all markup' 48 | 49 | def extract(self, htmlregion): 50 | return extract_text(htmlregion.text_content) 51 | 52 | 53 | class SafeHtmlFieldTypeProcessor(_BaseTextProcessor): 54 | """Extracts strings, with only a safe subset of HTML remaining 55 | 56 | Extraction checks for presence of text content, and adapt transforms the HTML 57 | >>> from scrapely.extractors import htmlregion 58 | >>> p = SafeHtmlFieldTypeProcessor() 59 | >>> html = htmlregion(u'
test