").append($(element).eq(0).clone()).html();
72 | return elementHTML;
73 | }
74 | else if(this.clickElementUniquenessType === 'uniqueHTML') {
75 |
76 | // get element without text
77 | var $element = $(element).eq(0).clone();
78 |
79 | var removeText = function($element) {
80 | $element.contents()
81 | .filter(function() {
82 | if(this.nodeType !== 3) {
83 | removeText($(this));
84 | }
85 | return this.nodeType == 3; //Node.TEXT_NODE
86 | }).remove();
87 | };
88 | removeText($element);
89 |
90 | var elementHTML = $("
").append($element).html();
91 | return elementHTML;
92 | }
93 | else if(this.clickElementUniquenessType === 'uniqueCSSSelector') {
94 | var cs = new CssSelector({
95 | enableSmartTableSelector: false,
96 | parent: $("body")[0],
97 | enableResultStripping:false
98 | });
99 | var CSSSelector = cs.getCssSelector([element]);
100 | return CSSSelector;
101 | }
102 | else {
103 | throw "Invalid clickElementUniquenessType "+this.clickElementUniquenessType;
104 | }
105 | };
106 |
107 | UniqueElementList.prototype.isAdded = function(element) {
108 |
109 | var elementUniqueId = this.getElementUniqueId(element);
110 | var isAdded = elementUniqueId in this.addedElements;
111 | return isAdded;
112 | };
113 |
--------------------------------------------------------------------------------
/extension/background_page/background_script.js:
--------------------------------------------------------------------------------
1 | var config = new Config();
2 | var store;
3 | config.loadConfiguration(function () {
4 | console.log("initial configuration", config);
5 | store = new Store(config);
6 | });
7 |
8 | chrome.storage.onChanged.addListener(function () {
9 | config.loadConfiguration(function () {
10 | console.log("configuration changed", config);
11 | store = new Store(config);
12 | });
13 | });
14 |
15 | var sendToActiveTab = function(request, callback) {
16 | chrome.tabs.query({
17 | active: true,
18 | currentWindow: true
19 | }, function (tabs) {
20 | if (tabs.length < 1) {
21 | this.console.log("couldn't find active tab");
22 | }
23 | else {
24 | var tab = tabs[0];
25 | chrome.tabs.sendMessage(tab.id, request, callback);
26 | }
27 | });
28 | };
29 |
30 | chrome.runtime.onMessage.addListener(
31 | function (request, sender, sendResponse) {
32 |
33 | console.log("chrome.runtime.onMessage", request);
34 |
35 | if (request.createSitemap) {
36 | store.createSitemap(request.sitemap, sendResponse);
37 | return true;
38 | }
39 | else if (request.saveSitemap) {
40 | store.saveSitemap(request.sitemap, sendResponse);
41 | return true;
42 | }
43 | else if (request.deleteSitemap) {
44 | store.deleteSitemap(request.sitemap, sendResponse);
45 | return true;
46 | }
47 | else if (request.getAllSitemaps) {
48 | store.getAllSitemaps(sendResponse);
49 | return true;
50 | }
51 | else if (request.sitemapExists) {
52 | store.sitemapExists(request.sitemapId, sendResponse);
53 | return true;
54 | }
55 | else if (request.getSitemapData) {
56 | store.getSitemapData(new Sitemap(request.sitemap), sendResponse);
57 | return true;
58 | }
59 | else if (request.scrapeSitemap) {
60 | var sitemap = new Sitemap(request.sitemap);
61 | var queue = new Queue();
62 | var browser = new ChromePopupBrowser({
63 | pageLoadDelay: request.pageLoadDelay
64 | });
65 |
66 | var scraper = new Scraper({
67 | queue: queue,
68 | sitemap: sitemap,
69 | browser: browser,
70 | store: store,
71 | requestInterval: request.requestInterval,
72 | requestIntervalRandomness: request.requestIntervalRandomness
73 | });
74 |
75 | try {
76 | scraper.run(function () {
77 | browser.close();
78 | var notification = chrome.notifications.create("scraping-finished", {
79 | type: 'basic',
80 | iconUrl: 'assets/images/icon128.png',
81 | title: 'Scraping finished!',
82 | message: 'Finished scraping ' + sitemap._id
83 | }, function(id) {
84 | // notification showed
85 | });
86 | // table selector can dynamically add columns (addMissingColumns Feature)
87 | var selectors = sitemap.selectors;
88 | sendResponse(selectors);
89 | });
90 | }
91 | catch (e) {
92 | console.log("Scraper execution cancelled", e);
93 | }
94 |
95 | return true;
96 | }
97 | else if(request.previewSelectorData) {
98 | chrome.tabs.query({
99 | active: true,
100 | currentWindow: true
101 | }, function (tabs) {
102 | if (tabs.length < 1) {
103 | this.console.log("couldn't find active tab");
104 | }
105 | else {
106 | var tab = tabs[0];
107 | chrome.tabs.sendMessage(tab.id, request, sendResponse);
108 | }
109 | });
110 | return true;
111 | }
112 | else if(request.backgroundScriptCall) {
113 |
114 | var backgroundScript = getBackgroundScript("BackgroundScript");
115 | var deferredResponse = backgroundScript[request.fn](request.request)
116 | deferredResponse.done(function(response){
117 | sendResponse(response);
118 | });
119 |
120 | return true;
121 | }
122 | }
123 | );
124 |
--------------------------------------------------------------------------------
/tests/spec/Selector/SelectorHTMLSpec.js:
--------------------------------------------------------------------------------
1 | describe("HTML Selector", function () {
2 |
3 | beforeEach(function () {
4 |
5 | });
6 |
7 | it("should extract single html element", function () {
8 |
9 | var selector = new Selector({
10 | id: 'a',
11 | type: 'SelectorHTML',
12 | multiple: false,
13 | selector: "div"
14 | });
15 |
16 | var dataDeferred = selector.getData($("#selector-html-single-html"));
17 |
18 | waitsFor(function() {
19 | return dataDeferred.state() === 'resolved';
20 | }, "wait for data extraction", 5000);
21 |
22 | runs(function () {
23 | dataDeferred.done(function(data) {
24 | expect(data).toEqual([
25 | {
26 | a: "aaa
bbbccc"
27 | }
28 | ]);
29 | });
30 | });
31 | });
32 |
33 | it("should extract multiple html elements", function () {
34 |
35 | var selector = new Selector({
36 | id: 'a',
37 | type: 'SelectorHTML',
38 | multiple: true,
39 | selector: "div"
40 | });
41 |
42 | var dataDeferred = selector.getData($("#selector-html-multiple-html"));
43 |
44 | waitsFor(function() {
45 | return dataDeferred.state() === 'resolved';
46 | }, "wait for data extraction", 5000);
47 |
48 | runs(function () {
49 | dataDeferred.done(function(data) {
50 | expect(data).toEqual([
51 | {
52 | a: "aaa
bbbccc"
53 | },
54 | {
55 | a: "ddd
eeefff"
56 | }
57 | ]);
58 | });
59 | });
60 | });
61 |
62 | it("should extract null when there are no elements", function () {
63 |
64 | var selector = new Selector({
65 | id: 'a',
66 | type: 'SelectorHTML',
67 | multiple: false,
68 | selector: "div"
69 | });
70 |
71 | var dataDeferred = selector.getData($("#selector-html-single-not-exist"));
72 |
73 | waitsFor(function() {
74 | return dataDeferred.state() === 'resolved';
75 | }, "wait for data extraction", 5000);
76 |
77 | runs(function () {
78 | dataDeferred.done(function(data) {
79 | expect(data).toEqual([
80 | {
81 | a: null
82 | }
83 | ]);
84 | });
85 | });
86 | });
87 |
88 | it("should extract empty string when there is no regex match", function () {
89 |
90 | var selector = new Selector({
91 | id: 'a',
92 | type: 'SelectorHTML',
93 | multiple: false,
94 | selector: "div",
95 | textmanipulation: { regex: "wontmatch" }
96 | });
97 |
98 | var dataDeferred = selector.getData($("#selector-html-single-html"));
99 |
100 | waitsFor(function() {
101 | return dataDeferred.state() === 'resolved';
102 | }, "wait for data extraction", 5000);
103 |
104 | runs(function () {
105 | dataDeferred.done(function(data) {
106 | expect(data).toEqual([
107 | {
108 | a: ''
109 | }
110 | ]);
111 | });
112 | });
113 | });
114 |
115 | it("should extract html+text using regex", function () {
116 |
117 | var selector = new Selector({
118 | id: 'a',
119 | type: 'SelectorHTML',
120 | multiple: false,
121 | selector: "div",
122 | textmanipulation: { regex: "
\\w+" }
123 | });
124 |
125 | var dataDeferred = selector.getData($("#selector-html-single-html"));
126 |
127 | waitsFor(function() {
128 | return dataDeferred.state() === 'resolved';
129 | }, "wait for data extraction", 5000);
130 |
131 | runs(function () {
132 | dataDeferred.done(function(data) {
133 | expect(data).toEqual([
134 | {
135 | a: 'bbb'
136 | }
137 | ]);
138 | });
139 | });
140 | });
141 |
142 | it("should return only one data column", function () {
143 | var selector = new Selector({
144 | id: 'id',
145 | type: 'SelectorHTML',
146 | multiple: true,
147 | selector: "div"
148 | });
149 |
150 | var columns = selector.getDataColumns();
151 | expect(columns).toEqual(['id']);
152 | });
153 | });
--------------------------------------------------------------------------------
/tests/spec/Selector/SelectorImageSpec.js:
--------------------------------------------------------------------------------
1 | describe("Image Selector", function () {
2 |
3 | var $el;
4 |
5 | beforeEach(function () {
6 |
7 | this.addMatchers(selectorMatchers);
8 |
9 | $el = jQuery("#tests").html("");
10 | if($el.length === 0) {
11 | $el = $("").appendTo("body");
12 | }
13 | });
14 |
15 | it("should extract single image", function () {
16 |
17 | var selector = new Selector({
18 | id: 'img',
19 | type: 'SelectorImage',
20 | multiple: false,
21 | selector: "img"
22 | });
23 |
24 | var dataDeferred = selector.getData($("#selector-image-one-image"));
25 |
26 | waitsFor(function() {
27 | return dataDeferred.state() === 'resolved';
28 | }, "wait for data extraction", 5000);
29 |
30 | runs(function () {
31 | dataDeferred.done(function(data) {
32 | expect(data).toEqual([
33 | {
34 | 'img-src': "http://aa/"
35 | }
36 | ]);
37 | });
38 | });
39 | });
40 |
41 | it("should extract multiple images", function () {
42 |
43 | var selector = new Selector({
44 | id: 'img',
45 | type: 'SelectorImage',
46 | multiple: true,
47 | selector: "img"
48 | });
49 |
50 | var dataDeferred = selector.getData($("#selector-image-multiple-images"));
51 |
52 | waitsFor(function() {
53 | return dataDeferred.state() === 'resolved';
54 | }, "wait for data extraction", 5000);
55 |
56 | runs(function () {
57 | dataDeferred.done(function(data) {
58 | expect(data).toEqual([
59 | {
60 | 'img-src': "http://aa/"
61 | },
62 | {
63 | 'img-src': "http://bb/"
64 | }
65 | ]);
66 | });
67 | });
68 | });
69 |
70 | it("should return only src column", function () {
71 | var selector = new Selector({
72 | id: 'id',
73 | type: 'SelectorImage',
74 | multiple: true,
75 | selector: "img"
76 | });
77 |
78 | var columns = selector.getDataColumns();
79 | expect(columns).toEqual(['id-src']);
80 | });
81 |
82 | it("should return empty array when no images are found", function () {
83 | var selector = new Selector({
84 | id: 'img',
85 | type: 'SelectorImage',
86 | multiple: true,
87 | selector: "img.not-exist"
88 | });
89 |
90 | var dataDeferred = selector.getData($("#not-exist"));
91 |
92 | waitsFor(function() {
93 | return dataDeferred.state() === 'resolved';
94 | }, "wait for data extraction", 5000);
95 |
96 | runs(function () {
97 | dataDeferred.done(function(data) {
98 | expect(data).toEqual([]);
99 | });
100 | });
101 | });
102 |
103 | it("should be able to download image as base64", function() {
104 |
105 | var deferredImage = SelectorImage.downloadImageBase64("../docs/images/chrome-store-logo.png");
106 |
107 | waitsFor(function() {
108 | return deferredImage.state() === 'resolved';
109 | }, "wait for data extraction", 5000);
110 |
111 | runs(function () {
112 | deferredImage.done(function(imageResponse) {
113 | expect(imageResponse.imageBase64.length > 100).toEqual(true);
114 | });
115 | });
116 | });
117 |
118 | it("should be able to get data with image data attached", function() {
119 |
120 | $el.append('
');
121 |
122 | var selector = new Selector({
123 | id: 'img',
124 | type: 'SelectorImage',
125 | multiple: true,
126 | selector: "img",
127 | downloadImage: true
128 | });
129 | var deferredData = selector.getData($el[0]);
130 |
131 | waitsFor(function() {
132 | return deferredData.state() === 'resolved';
133 | }, "wait for data extraction", 5000);
134 |
135 | runs(function () {
136 | deferredData.done(function(data) {
137 | expect(!!data[0]['_imageBase64-img']).toEqual(true);
138 | expect(!!data[0]['_imageMimeType-img']).toEqual(true);
139 | });
140 | });
141 | });
142 | });
143 |
--------------------------------------------------------------------------------
/docs/Selectors/Link selector.md:
--------------------------------------------------------------------------------
1 | # Link selector
2 |
3 | Link selector is used for link selection and website navigation. If you use
4 | *Link selector* without any child selectors then it will extract the link and
5 | the href attribute of the link. If you add child selectors to *Link selector*
6 | then these child selectors will be used in the page that this link was leading
7 | to. If you are selecting multiple links then check *multiple* property.
8 |
9 | Note! Link selector works only with `` tags with `href` attribute. If the
10 | link selector is not working for you then you can try these workarounds:
11 |
12 | 1. Check that the link in the url bar changes after clicking an item (changes
13 | only after hash tag doesn't count). If the link doesn't change then the site
14 | is probably using ajax for data loading. Instead of using link selector you
15 | should use [Element click selector] [element-click].
16 | 2. If the site opens a popup then you should use
17 | [Link popup selector] [link-popup]
18 | 3. The site might be using JavaScript `window.location` to change the URL. Web
19 | Scraper cannot handle this kind of navigation right now.
20 |
21 | ## Configuration options
22 |
23 | * selector - [CSS selector] [css-selector] for the link element from which the
24 | link for navigation will be extracted.
25 | * multiple - multiple records are being extracted. Usually should be checked.
26 | * delay - delay the extraction
27 |
28 | ## Use cases
29 |
30 | **Navigate through multiple levels of navigation**
31 |
32 | For example an e-commerce site has multi level navigation -
33 | `categories -> subcategories`. To scrape data from all categories and
34 | subcategories you can create two *Link selectors*. One selector would select
35 | category links and the other selector would select subcategory links that are
36 | available in the category pages. The subcategory *Link selector* should be made
37 | as a child of the category *Link selector*. The selectors for data extraction
38 | from subcategory pages should be made as a child selectors to the subcategory
39 | selector.
40 |
41 | ![Fig. 1: Multiple link selectors for category navigation][multiple-level-link-selectors]
42 |
43 | **Handle pagination**
44 |
45 | For example an e-commerce site has multiple categories. Each category has a
46 | list of items and pagination links. Also some pages are not directly available
47 | from the category but are available from pagination pages (you can see
48 | pagination links 1-5, but not 6-8). You can start by building a sitemap that
49 | visits each category and extract items from category page. This sitemap will
50 | extract items only from the first pagination page. To extract items from all of
51 | the pagination links including the ones that are not visible at the beginning
52 | you need to create another *Link selector* that selects the pagination links.
53 | Figure 2 shows how the link selector should be created in the sitemap. When
54 | the scraper opens a category link it will extract items that are available in
55 | the page. After that it will find the pagination links and also visit those. If
56 | the pagination link selector is made a child to itself it will recursively
57 | discover all pagination pages. Figure 3 shows a selector graph where you can
58 | see how pagination links discover more pagination links and more data.
59 |
60 | ![Fig. 2: Sitemap with Link selector for pagination][pagination-link-selectors]
61 | ![Fig. 3: Selector graph with pagination][pagination-selector-graph]
62 |
63 | [multiple-level-link-selectors]: ../images/selectors/link/multiple-level-link-selectors.png?raw=true
64 | [pagination-link-selectors]: ../images/selectors/link/pagination-link-selectors.png?raw=true
65 | [pagination-selector-graph]: ../images/selectors/link/pagination-selector-graph.png?raw=true
66 | [element-click]: Element%20click%20selector.md
67 | [link-popup]: Link%20popup%20selector.md
68 | [css-selector]: ../CSS%20selector.md
--------------------------------------------------------------------------------
/docs/Selectors/Text selector.md:
--------------------------------------------------------------------------------
1 | # Text selector
2 |
3 | Text selector is used for text selection. The text selector will extract text
4 | from the selected element and from all its child elements. HTML will be
5 | stripped and only text will be returned. Selector will ignore text within
6 | `