├── index.js ├── license.md ├── package.json └── readme.md /index.js: -------------------------------------------------------------------------------- 1 | var fs = require("fs"); 2 | var http = require("http"); 3 | var https = require("https"); 4 | var path = require("path"); 5 | var url = require("url"); 6 | var events = require("events").EventEmitter; 7 | var util = require("util"); 8 | 9 | var cheerio = require("cheerio"); 10 | 11 | // The "Image" class. 12 | function Image(image, address){ 13 | 14 | var at = this.attributes = image.attribs; 15 | 16 | this.name = path.basename(at.src, path.extname(at.src)); 17 | this.saveTo = path.dirname(require.main.filename) + "/"; 18 | this.extension = path.extname(at.src); 19 | this.address = url.resolve(address, at.src); 20 | this.fromAddress = address; 21 | } 22 | 23 | Image.prototype.save = function(callback){ 24 | 25 | var parsedUrl = url.parse(this.address); 26 | 27 | // Make a reference to the current instance. 28 | var ref = this; 29 | 30 | // Support HTTPS. 31 | var protocol = http; 32 | if(parsedUrl.protocol == "https:") { 33 | protocol = https; 34 | } 35 | 36 | var request = protocol.request(this.address, function(response){ 37 | 38 | if(response.statusCode != 200){ 39 | 40 | console.error("Image scraper(3): image couldn't be found. (statusCode:" + response.statusCode + ")"); 41 | return request.end(); 42 | } 43 | else{ 44 | 45 | var imageFile = fs.createWriteStream(path.normalize(ref.saveTo + ref.name + ref.extension)); 46 | 47 | imageFile.on("error", function(e){ 48 | 49 | console.error("Image scraper(4): error while loading image: " + e + "."); 50 | }); 51 | 52 | response.on("data", function(data){ 53 | 54 | imageFile.write(data); 55 | }); 56 | 57 | response.on("end", function(){ 58 | 59 | imageFile.end(); 60 | 61 | if(typeof(callback) == "function") callback.call(ref); 62 | }); 63 | } 64 | }); 65 | 66 | request.end(); 67 | request.on("error", function(e){ 68 | 69 | console.error(e); 70 | }); 71 | }; 72 | 73 | function Scraper(address){ 74 | 75 | events.call(this); 76 | this.address = address; 77 | } 78 | 79 | // Inherit the methods of "events". 80 | util.inherits(Scraper, events); 81 | 82 | Scraper.prototype.scrape = function(callback){ 83 | 84 | if(typeof(callback) == "function"){ 85 | 86 | this.on("image", callback); 87 | } 88 | 89 | var parsedUrl = url.parse(this.address); 90 | 91 | // Make a reference to the current instance. 92 | var ref = this; 93 | 94 | // Support HTTPS. 95 | var protocol = http; 96 | if(parsedUrl.protocol == "https:") { 97 | protocol = https; 98 | } 99 | 100 | var request = protocol.request(this.address, function(response){ 101 | 102 | if(response.statusCode != 200){ 103 | console.error("Image scraper(1): web page couldn't be found. (statusCode:" + response.statusCode + ")"); 104 | ref.emit("end"); 105 | request.end(); 106 | return process.exit(1); 107 | } 108 | else{ 109 | 110 | response.setEncoding("utf8"); 111 | 112 | var previous = "", 113 | current; 114 | 115 | response.on("data", function(data){ 116 | var current = previous + data; 117 | 118 | current.replace(//ig, function(m){ 119 | 120 | var image = new Image(cheerio.load(m)("img")[0], ref.address); 121 | 122 | ref.emit("image", image); 123 | }); 124 | 125 | previous = data; 126 | }); 127 | 128 | response.on("end", function(){ 129 | ref.emit("end"); 130 | }); 131 | } 132 | }); 133 | request.end(); 134 | 135 | request.on("error", function(e){ 136 | 137 | console.error("Image scraper(2): error while loading web page: " + e + "."); 138 | }); 139 | }; 140 | 141 | module.exports = Scraper; -------------------------------------------------------------------------------- /license.md: -------------------------------------------------------------------------------- 1 | MIT: 2 | 3 | copyright (c) 2013 Léon van Velzen 4 | 5 | Permission is hereby granted, free of charge, to any person 6 | obtaining a copy of this software and associated documentation 7 | files (the "Software"), to deal in the Software without 8 | restriction, including without limitation the rights to use, 9 | copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the 11 | Software is furnished to do so, subject to the following 12 | conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | OTHER DEALINGS IN THE SOFTWARE. 25 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "image-scraper", 3 | "version": "0.2.0", 4 | "description": "Scrape images from a website easily.", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/leon-vv/node-image-scraper.git" 12 | }, 13 | "dependencies":{ 14 | "cheerio":">=0.12.0" 15 | }, 16 | "keywords": [ 17 | "scrape", 18 | "image", 19 | "web-scraper", 20 | "scraper" 21 | ], 22 | "author": "Léon van Velzen", 23 | "license": "MIT" 24 | } 25 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | Node Image Scraper 2 | ======================== 3 | Scrape images from the web easily. 4 | Usage 5 | ------------------------ 6 | The image-scrape module provides a class which needs to be constructed with a url: 7 | ```JavaScript 8 | var Scraper = require("image-scraper"); 9 | 10 | var scraper = new Scraper("https://apod.nasa.gov/apod/astropix.html"); 11 | ``` 12 | The url can be changed easily: 13 | ```JavaScript 14 | scraper.address = "http://www.npmjs.org"; 15 | ``` 16 | The scraper object provides the .scrape() method. 17 | The scrape method accepts one optional argument: a callback function. 18 | 19 | This callback function can also be set using the .on() method. 20 | 21 | ```JavaScript 22 | scraper.on("image", function(image){ 23 | 24 | // Do something. 25 | }); 26 | ``` 27 | As soon as 'scraper.scrape()' is called, the callback will be fired with every image found on the webpage, note however that dynamically generated images will not be found. 28 | 29 | The image object that is passed to the callback has the following properties and methods: 30 | ```JavaScript 31 | // The attributes found in the image tag, which is parsed by Cheerio (https://npmjs.org/package/cheerio). 32 | image.attributes; 33 | // The basename of the image. 34 | image.name; 35 | // Absolute path to the folder the image will be saved to. 36 | image.saveTo; 37 | // Extension of the image file. 38 | image.extension; 39 | // The absolute URL of the image. 40 | image.address; 41 | // The URL of the page the image is scraped from. 42 | image.fromAddress; 43 | // Save the image. 44 | image.save(); 45 | ``` 46 | The behaviour of image.save() can be changed by setting the name, saveTo, and extension properties. 47 | The saveTo property is by default set to the current directory, and the other two properties are by default set to the data found in the src attribute of the img tag. 48 | 49 | Thus, the smallest program that scans a webpage for images and saves them in the current directory looks as follows: 50 | ```JavaScript 51 | var Scraper = require('image-scraper'); 52 | var scraper = new Scraper('https://apod.nasa.gov/apod/astropix.html'); 53 | 54 | scraper.scrape(function(image) { 55 | image.save(); 56 | }); 57 | ``` 58 | --------------------------------------------------------------------------------