idMap) {
45 | this.idMap = idMap;
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/handler/PatternRequestMatcher.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.handler;
2 |
3 | import us.codecraft.webmagic.Request;
4 |
5 | import java.util.regex.Pattern;
6 |
7 | /**
8 | * Created with IntelliJ IDEA.
9 | * User: Sebastian MA
10 | * Date: April 03, 2014
11 | * Time: 10:00
12 | *
13 | * A PatternHandler is in charge of both page extraction and data processing by implementing
14 | * its two abstract methods.
15 | */
16 | public abstract class PatternRequestMatcher implements RequestMatcher {
17 |
18 | /**
19 | * match pattern. only matched page should be handled.
20 | */
21 | protected String pattern;
22 |
23 | private Pattern patternCompiled;
24 |
25 | /**
26 | * @param pattern url pattern to handle
27 | */
28 | public PatternRequestMatcher(String pattern) {
29 | this.pattern = pattern;
30 | this.patternCompiled = Pattern.compile(pattern);
31 | }
32 |
33 | @Override
34 | public boolean match(Request request) {
35 | return patternCompiled.matcher(request.getUrl()).matches();
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/webmagic-extension/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | us.codecraft
5 | webmagic-parent
6 | 0.5.4-SNAPSHOT
7 |
8 | 4.0.0
9 |
10 | webmagic-extension
11 |
12 |
13 |
14 | redis.clients
15 | jedis
16 | 2.0.0
17 |
18 |
19 | us.codecraft
20 | webmagic-core
21 | ${project.version}
22 |
23 |
24 | junit
25 | junit
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/HuxiuProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.samples;
2 |
3 | import us.codecraft.webmagic.Site;
4 | import us.codecraft.webmagic.Page;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.processor.PageProcessor;
7 |
8 | import java.util.List;
9 |
10 | /**
11 | * @author code4crafter@gmail.com
12 | */
13 | public class HuxiuProcessor implements PageProcessor {
14 | @Override
15 | public void process(Page page) {
16 | List requests = page.getHtml().links().regex(".*article.*").all();
17 | page.addTargetRequests(requests);
18 | page.putField("title",page.getHtml().xpath("//div[@class='clearfix neirong']//h1/text()"));
19 | page.putField("content",page.getHtml().xpath("//div[@id='neirong_box']/tidyText()"));
20 | }
21 |
22 | @Override
23 | public Site getSite() {
24 | return Site.me().setDomain("www.huxiu.com").addStartUrl("http://www.huxiu.com/");
25 | }
26 |
27 | public static void main(String[] args) {
28 | Spider.create(new HuxiuProcessor()).run();
29 | }
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/webmagic-extension/pom.xml.versionsBackup:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | us.codecraft
5 | webmagic-parent
6 | 0.5.2
7 |
8 | 4.0.0
9 |
10 | webmagic-extension
11 |
12 |
13 |
14 | redis.clients
15 | jedis
16 | 2.0.0
17 |
18 |
19 | us.codecraft
20 | webmagic-core
21 | ${project.version}
22 |
23 |
24 | junit
25 | junit
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/webmagic-extension/pom.xml.releaseBackup:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | us.codecraft
5 | webmagic-parent
6 | 0.5.3-SNAPSHOT
7 |
8 | 4.0.0
9 |
10 | webmagic-extension
11 |
12 |
13 |
14 | redis.clients
15 | jedis
16 | 2.0.0
17 |
18 |
19 | us.codecraft
20 | webmagic-core
21 | ${project.version}
22 |
23 |
24 | junit
25 | junit
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/webmagic-extension/src/test/java/us/codecraft/webmagic/model/GithubRepo.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model;
2 |
3 | import us.codecraft.webmagic.Site;
4 | import us.codecraft.webmagic.model.annotation.ExtractBy;
5 | import us.codecraft.webmagic.model.annotation.HelpUrl;
6 | import us.codecraft.webmagic.model.annotation.TargetUrl;
7 |
8 | /**
9 | * @author code4crafter@gmail.com
10 | * @since 0.3.2
11 | */
12 | @TargetUrl("https://github.com/\\w+/\\w+")
13 | @HelpUrl({"https://github.com/\\w+\\?tab=repositories", "https://github.com/\\w+", "https://github.com/explore/*"})
14 | public class GithubRepo extends BaseRepo{
15 |
16 | @ExtractBy("//ul[@class='pagehead-actions']/li[2]//a[@class='social-count']/text()")
17 | private int fork;
18 |
19 | public static void main(String[] args) {
20 | OOSpider.create(Site.me().setSleepTime(100)
21 | , new ConsolePageModelPipeline(), GithubRepo.class)
22 | .addUrl("https://github.com/code4craft").thread(10).run();
23 | }
24 |
25 | public int getStar() {
26 | return star;
27 | }
28 |
29 | public int getFork() {
30 | return fork;
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/webmagic-samples/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | webmagic-parent
5 | us.codecraft
6 | 0.5.4-SNAPSHOT
7 |
8 | 4.0.0
9 |
10 | webmagic-samples
11 |
12 |
13 |
14 | us.codecraft
15 | webmagic-core
16 | ${project.version}
17 |
18 |
19 | us.codecraft
20 | webmagic-extension
21 | ${project.version}
22 |
23 |
24 | junit
25 | junit
26 |
27 |
28 |
29 |
30 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/OschinaAnswer.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.samples;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.model.*;
6 | import us.codecraft.webmagic.model.annotation.ExtractBy;
7 | import us.codecraft.webmagic.model.annotation.HelpUrl;
8 | import us.codecraft.webmagic.model.annotation.TargetUrl;
9 |
10 | /**
11 | * @author code4crafter@gmail.com
12 | */
13 | @TargetUrl("http://www.oschina.net/question/\\d+_\\d+*")
14 | @HelpUrl("http://www.oschina.net/question/*")
15 | @ExtractBy(value = "//ul[@class='list']/li[@class='Answer']", multi = true)
16 | public class OschinaAnswer implements AfterExtractor{
17 |
18 | @ExtractBy("//img/@title")
19 | private String user;
20 |
21 | @ExtractBy("//div[@class='detail']")
22 | private String content;
23 |
24 | public static void main(String[] args) {
25 | OOSpider.create(Site.me().addStartUrl("http://www.oschina.net/question/567527_120597"), OschinaAnswer.class).run();
26 | }
27 |
28 | @Override
29 | public void afterProcess(Page page) {
30 |
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/main/java/us/codecraft/webmagic/model/DynamicClass.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model;
2 |
3 | import java.util.Date;
4 |
5 | /**
6 | * @author code4crafter@gmail.com
7 | */
8 | public class DynamicClass {
9 |
10 | private String className;
11 |
12 | private String sourceCode;
13 |
14 | private Date addTime;
15 |
16 | private Date updateTime;
17 |
18 | public String getClassName() {
19 | return className;
20 | }
21 |
22 | public void setClassName(String className) {
23 | this.className = className;
24 | }
25 |
26 | public String getSourceCode() {
27 | return sourceCode;
28 | }
29 |
30 | public void setSourceCode(String sourceCode) {
31 | this.sourceCode = sourceCode;
32 | }
33 |
34 | public Date getAddTime() {
35 | return addTime;
36 | }
37 |
38 | public void setAddTime(Date addTime) {
39 | this.addTime = addTime;
40 | }
41 |
42 | public Date getUpdateTime() {
43 | return updateTime;
44 | }
45 |
46 | public void setUpdateTime(Date updateTime) {
47 | this.updateTime = updateTime;
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/IteyeBlogProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.samples;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.processor.PageProcessor;
7 |
8 | /**
9 | * @author code4crafter@gmail.com
10 | */
11 | public class IteyeBlogProcessor implements PageProcessor {
12 |
13 | private Site site;
14 |
15 | @Override
16 | public void process(Page page) {
17 | page.addTargetRequests(page.getHtml().links().regex(".*yanghaoli\\.iteye\\.com/blog/\\d+").all());
18 | page.putField("title",page.getHtml().xpath("//title").toString());
19 | page.putField("content",page.getHtml().smartContent().toString());
20 | }
21 |
22 | @Override
23 | public Site getSite() {
24 | if (site == null) {
25 | site = Site.me().setDomain("yanghaoli.iteye.com").addStartUrl("http://yanghaoli.iteye.com/");
26 | }
27 | return site;
28 | }
29 |
30 | public static void main(String[] args) {
31 | Spider.create(new IteyeBlogProcessor()).thread(5).run();
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/model/Extractor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model;
2 |
3 | import us.codecraft.webmagic.selector.Selector;
4 |
5 | /**
6 | * The object contains 'ExtractBy' information.
7 | * @author code4crafter@gmail.com
8 | * @since 0.2.0
9 | */
10 | class Extractor {
11 |
12 | protected Selector selector;
13 |
14 | protected final Source source;
15 |
16 | protected final boolean notNull;
17 |
18 | protected final boolean multi;
19 |
20 | static enum Source {Html, Url, RawHtml}
21 |
22 | public Extractor(Selector selector, Source source, boolean notNull, boolean multi) {
23 | this.selector = selector;
24 | this.source = source;
25 | this.notNull = notNull;
26 | this.multi = multi;
27 | }
28 |
29 | Selector getSelector() {
30 | return selector;
31 | }
32 |
33 | Source getSource() {
34 | return source;
35 | }
36 |
37 | boolean isNotNull() {
38 | return notNull;
39 | }
40 |
41 | boolean isMulti() {
42 | return multi;
43 | }
44 |
45 | void setSelector(Selector selector) {
46 | this.selector = selector;
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/webmagic-core/src/test/java/us/codecraft/webmagic/downloader/MockGithubDownloader.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.downloader;
2 |
3 | import org.apache.commons.io.IOUtils;
4 | import us.codecraft.webmagic.Page;
5 | import us.codecraft.webmagic.Request;
6 | import us.codecraft.webmagic.Task;
7 | import us.codecraft.webmagic.selector.PlainText;
8 |
9 | import java.io.IOException;
10 | import java.io.InputStream;
11 |
12 | /**
13 | * @author code4crafter@gmail.com
14 | */
15 | public class MockGithubDownloader implements Downloader {
16 |
17 | @Override
18 | public Page download(Request request, Task task) {
19 | Page page = new Page();
20 | InputStream resourceAsStream = this.getClass().getResourceAsStream("/html/mock-github.html");
21 | try {
22 | page.setRawText(IOUtils.toString(resourceAsStream));
23 | } catch (IOException e) {
24 | e.printStackTrace();
25 | }
26 | page.setRequest(new Request("https://github.com/code4craft/webmagic"));
27 | page.setUrl(new PlainText("https://github.com/code4craft/webmagic"));
28 | return page;
29 | }
30 |
31 | @Override
32 | public void setThread(int threadNum) {
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/OschinaPageProcesser.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.samples;
2 |
3 | import us.codecraft.webmagic.Site;
4 | import us.codecraft.webmagic.Page;
5 | import us.codecraft.webmagic.processor.PageProcessor;
6 |
7 | import java.util.List;
8 |
9 | /**
10 | * @author code4crafter@gmail.com
11 | */
12 | public class OschinaPageProcesser implements PageProcessor {
13 |
14 | @Override
15 | public void process(Page page) {
16 | List strings = page.getHtml().regex("]*href=[\"']{1}(http://www\\.oschina\\.net/question/[\\w]+)[\"']{1}").all();
17 | page.addTargetRequests(strings);
18 | page.putField("title", page.getHtml().xpath("//div[@class='QTitle']/h1/a"));
19 | page.putField("content", page.getHtml().xpath("//div[@class='Question']//div[@class='Content']/div[@class='detail']"));
20 | }
21 |
22 | @Override
23 | public Site getSite() {
24 | return Site.me().setDomain("www.oschina.net").addStartUrl("http://www.oschina.net/").
25 | setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-admin/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | webmagic-avalon
5 | us.codecraft
6 | 0.5.4-SNAPSHOT
7 |
8 | 4.0.0
9 |
10 | us.codecraft
11 | webmagic-admin
12 | war
13 |
14 |
15 |
16 | us.codecraft
17 | webmagic-avalon-common
18 | ${project.version}
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 | maven-deploy-plugin
27 |
28 | true
29 |
30 |
31 |
32 |
33 |
34 |
35 |
--------------------------------------------------------------------------------
/webmagic-avalon/forger/src/main/java/us/codecraft/forger/property/AnnotationPropertyLoader.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.forger.property;
2 |
3 | import java.lang.reflect.Field;
4 | import java.util.ArrayList;
5 | import java.util.List;
6 |
7 | /**
8 | * @author code4crafter@gmail.com
9 | */
10 | public class AnnotationPropertyLoader extends AbstractPropertyLoader {
11 |
12 | @Override
13 | public List getProperties(Class clazz) {
14 | Field[] fields = clazz.getDeclaredFields();
15 | List properties = new ArrayList(fields.length);
16 | for (Field field : fields) {
17 | Inject inject = field.getAnnotation(Inject.class);
18 | if (inject != null) {
19 | if (!field.isAccessible()) {
20 | field.setAccessible(true);
21 | }
22 | Property property = Property.fromField(field);
23 | if (inject.value().length() > 0) {
24 | property.setName(inject.value());
25 | }
26 | property.setObjectFormatter(getObjectFormatter(field));
27 | properties.add(property);
28 | }
29 | }
30 | return properties;
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/main/resources/sql/mysql/schema.sql:
--------------------------------------------------------------------------------
1 | CREATE TABLE `DynamicClass` (
2 | `Id` int(11) unsigned NOT NULL AUTO_INCREMENT,
3 | `ClassName` varchar(200) NOT NULL,
4 | `SourceCode` text NOT NULL,
5 | `AddTime` datetime NOT NULL,
6 | `UpdateTime` datetime NOT NULL,
7 | PRIMARY KEY (`Id`),
8 | UNIQUE KEY `un_class_name` (`ClassName`)
9 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
10 |
11 | CREATE TABLE `Spider` (
12 | `Id` int(11) unsigned NOT NULL AUTO_INCREMENT,
13 | `PageProcessorId` int(11) unsigned NOT NULL AUTO_INCREMENT,
14 | `PipelineId` int(11) unsigned NOT NULL AUTO_INCREMENT,
15 | `SchedulerId` int(11) unsigned NOT NULL AUTO_INCREMENT,
16 | `Config` text NOT NULL,
17 | `AddTime` datetime NOT NULL,
18 | `UpdateTime` datetime NOT NULL,
19 | PRIMARY KEY (`Id`),
20 | UNIQUE KEY `un_class_name` (`ClassName`)
21 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
22 |
23 | CREATE TABLE `PageProcessor` (
24 | `Id` int(11) unsigned NOT NULL AUTO_INCREMENT,
25 | `ClassName` varchar(200) NOT NULL,
26 | `Params` text NOT NULL,
27 | `AddTime` datetime NOT NULL,
28 | `UpdateTime` datetime NOT NULL,
29 | PRIMARY KEY (`Id`),
30 | UNIQUE KEY `un_class_name` (`ClassName`)
31 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/model/formatter/ObjectFormatters.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.formatter;
2 |
3 | import java.util.Map;
4 | import java.util.concurrent.ConcurrentHashMap;
5 |
6 | /**
7 | * @author code4crafter@gmail.com
8 | * @since 0.3.2
9 | */
10 | public class ObjectFormatters {
11 |
12 | private static Map> formatterMap = new ConcurrentHashMap>();
13 |
14 | static {
15 | for (Class basicTypeFormatter : BasicTypeFormatter.basicTypeFormatters) {
16 | put(basicTypeFormatter);
17 | }
18 | put(DateFormatter.class);
19 | }
20 |
21 | public static void put(Class objectFormatter) {
22 | try {
23 | formatterMap.put(objectFormatter.newInstance().clazz(), objectFormatter);
24 | } catch (InstantiationException e) {
25 | e.printStackTrace();
26 | } catch (IllegalAccessException e) {
27 | e.printStackTrace();
28 | }
29 | }
30 |
31 | public static Class get(Class clazz){
32 | return formatterMap.get(clazz);
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/webmagic-extension/src/test/java/us/codecraft/webmagic/monitor/SpiderMonitorTest.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.monitor;
2 |
3 | import org.junit.Test;
4 | import us.codecraft.webmagic.Spider;
5 | import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
6 | import us.codecraft.webmagic.processor.example.OschinaBlogPageProcessor;
7 |
8 | /**
9 | * @author code4crafer@gmail.com
10 | * @since 0.5.0
11 | */
12 | public class SpiderMonitorTest {
13 |
14 | @Test
15 | public void testInherit() throws Exception {
16 | SpiderMonitor spiderMonitor = new SpiderMonitor(){
17 | @Override
18 | protected SpiderStatusMXBean getSpiderStatusMBean(Spider spider, MonitorSpiderListener monitorSpiderListener) {
19 | return new CustomSpiderStatus(spider, monitorSpiderListener);
20 | }
21 | };
22 |
23 | Spider oschinaSpider = Spider.create(new OschinaBlogPageProcessor())
24 | .addUrl("http://my.oschina.net/flashsword/blog").thread(2);
25 | Spider githubSpider = Spider.create(new GithubRepoPageProcessor())
26 | .addUrl("https://github.com/code4craft");
27 |
28 | spiderMonitor.register(oschinaSpider, githubSpider);
29 |
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/main/resources/config/spring/applicationContext-service.xml:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/main/resources/config/spring/applicationContext-component.xml:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
14 |
15 |
16 |
17 | web_messages
18 |
19 |
20 |
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-admin/src/main/webapp/static/js/jquery.flot.resize.min.js:
--------------------------------------------------------------------------------
1 | (function(n,p,u){var w=n([]),s=n.resize=n.extend(n.resize,{}),o,l="setTimeout",m="resize",t=m+"-special-event",v="delay",r="throttleWindow";s[v]=250;s[r]=true;n.event.special[m]={setup:function(){if(!s[r]&&this[l]){return false}var a=n(this);w=w.add(a);n.data(this,t,{w:a.width(),h:a.height()});if(w.length===1){q()}},teardown:function(){if(!s[r]&&this[l]){return false}var a=n(this);w=w.not(a);a.removeData(t);if(!w.length){clearTimeout(o)}},add:function(b){if(!s[r]&&this[l]){return false}var c;function a(d,h,g){var f=n(this),e=n.data(this,t);e.w=h!==u?h:f.width();e.h=g!==u?g:f.height();c.apply(this,arguments)}if(n.isFunction(b)){c=b;return a}else{c=b.handler;b.handler=a}}};function q(){o=p[l](function(){w.each(function(){var d=n(this),a=d.width(),b=d.height(),c=n.data(this,t);if(a!==c.w||b!==c.h){d.trigger(m,[c.w=a,c.h=b])}});q()},s[v])}})(jQuery,this);(function(b){var a={};function c(f){function e(){var h=f.getPlaceholder();if(h.width()==0||h.height()==0){return}f.resize();f.setupGrid();f.draw()}function g(i,h){i.getPlaceholder().resize(e)}function d(i,h){i.getPlaceholder().unbind("resize",e)}f.hooks.bindEvents.push(g);f.hooks.shutdown.push(d)}b.plot.plugins.push({init:c,options:a,name:"resize",version:"1.0"})})(jQuery);
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/IteyeBlog.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.samples;
2 |
3 | import us.codecraft.webmagic.Site;
4 | import us.codecraft.webmagic.model.annotation.ExtractBy;
5 | import us.codecraft.webmagic.model.OOSpider;
6 | import us.codecraft.webmagic.model.annotation.TargetUrl;
7 |
8 | /**
9 | * @author code4crafter@gmail.com
10 | * Date: 13-8-2
11 | * Time: 上午7:52
12 | */
13 | @TargetUrl("http://*.iteye.com/blog/*")
14 | public class IteyeBlog implements Blog{
15 |
16 | @ExtractBy("//title")
17 | private String title;
18 |
19 | @ExtractBy(value = "div#blog_content",type = ExtractBy.Type.Css)
20 | private String content;
21 |
22 | @Override
23 | public String toString() {
24 | return "IteyeBlog{" +
25 | "title='" + title + '\'' +
26 | ", content='" + content + '\'' +
27 | '}';
28 | }
29 |
30 | public static void main(String[] args) {
31 | OOSpider.create(Site.me().addStartUrl("http://flashsword20.iteye.com/blog"), IteyeBlog.class).run();
32 | }
33 |
34 | public String getTitle() {
35 | return title;
36 | }
37 |
38 | public String getContent() {
39 | return content;
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/webmagic-avalon/forger/src/main/java/us/codecraft/forger/property/format/Formatter.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.forger.property.format;
2 |
3 | import java.lang.annotation.ElementType;
4 | import java.lang.annotation.Retention;
5 | import java.lang.annotation.Target;
6 |
7 | /**
8 | * Define how the result string is convert to an object for field.
9 | *
10 | * @author code4crafter@gmail.com
11 | * @since 0.3.2
12 | */
13 | @Retention(java.lang.annotation.RetentionPolicy.RUNTIME)
14 | @Target({ElementType.FIELD})
15 | public @interface Formatter {
16 |
17 | /**
18 | * Set formatter params.
19 | *
20 | * @return formatter params
21 | */
22 | String[] value();
23 |
24 | /**
25 | * Specific the class of field of class of elements in collection for field.
26 | * It is not necessary to be set because we can detect the class by class of field,
27 | * unless you use a collection as a field.
28 | *
29 | * @return the class of field
30 | */
31 | Class subClazz() default String.class;
32 |
33 | /**
34 | * If there are more than one formatter for a class, just specify the implement.
35 | * @return implement
36 | */
37 | Class formatter() default TypeFormatter.class;
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/webmagic-core/src/test/java/us/codecraft/webmagic/example/GithubRepoPageProcessorTest.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.example;
2 |
3 | import org.junit.Test;
4 | import us.codecraft.webmagic.ResultItems;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.Task;
7 | import us.codecraft.webmagic.downloader.MockGithubDownloader;
8 | import us.codecraft.webmagic.pipeline.Pipeline;
9 | import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;
10 |
11 | import static org.assertj.core.api.Assertions.assertThat;
12 |
13 | /**
14 | * @author code4crafter@gmail.com
15 | * Date: 16/1/19
16 | * Time: 上午7:27
17 | */
18 | public class GithubRepoPageProcessorTest {
19 |
20 | @Test
21 | public void test_github() throws Exception {
22 | Spider.create(new GithubRepoPageProcessor()).addPipeline(new Pipeline() {
23 | @Override
24 | public void process(ResultItems resultItems, Task task) {
25 | assertThat(((String) resultItems.get("name")).trim()).isEqualTo("webmagic");
26 | assertThat(((String) resultItems.get("author")).trim()).isEqualTo("code4craft");
27 | }
28 | }).setDownloader(new MockGithubDownloader()).test("https://github.com/code4craft/webmagic");
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/webmagic-extension/src/test/java/us/codecraft/webmagic/scheduler/RedisSchedulerTest.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.scheduler;
2 |
3 | import org.junit.Before;
4 | import org.junit.Ignore;
5 | import org.junit.Test;
6 | import us.codecraft.webmagic.Request;
7 | import us.codecraft.webmagic.Site;
8 | import us.codecraft.webmagic.Task;
9 |
10 | /**
11 | * @author code4crafter@gmail.com
12 | */
13 | public class RedisSchedulerTest {
14 |
15 | private RedisScheduler redisScheduler;
16 |
17 | @Before
18 | public void setUp() {
19 | redisScheduler = new RedisScheduler("localhost");
20 | }
21 |
22 | @Ignore("environment depended")
23 | @Test
24 | public void test() {
25 | Task task = new Task() {
26 | @Override
27 | public String getUUID() {
28 | return "1";
29 | }
30 |
31 | @Override
32 | public Site getSite() {
33 | return null;
34 | }
35 | };
36 | Request request = new Request("http://www.ibm.com/developerworks/cn/java/j-javadev2-22/");
37 | request.putExtra("1","2");
38 | redisScheduler.push(request, task);
39 | Request poll = redisScheduler.poll(task);
40 | System.out.println(poll);
41 |
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/MultiPageModel.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic;
2 |
3 | import us.codecraft.webmagic.utils.Experimental;
4 |
5 | import java.util.Collection;
6 |
7 | /**
8 | * Extract an object of more than one pages, such as news and articles.
9 | *
10 | * @author code4crafter@gmail.com
11 | * @since 0.2.0
12 | */
13 | @Experimental
14 | public interface MultiPageModel {
15 |
16 | /**
17 | * Page key is the identifier for the object.
18 | *
19 | * @return page key
20 | */
21 | public String getPageKey();
22 |
23 | /**
24 | * page is the identifier of a page in pages for one object.
25 | *
26 | * @return page
27 | */
28 | public String getPage();
29 |
30 | /**
31 | * other pages to be extracted.
32 | * It is used to judge whether an object contains more than one page, and whether the pages of the object are all extracted.
33 | *
34 | * @return other pages
35 | */
36 | public Collection getOtherPages();
37 |
38 | /**
39 | * Combine multiPageModels to a whole object.
40 | *
41 | * @param multiPageModel multiPageModel
42 | * @return multiPageModel combined
43 | */
44 | public MultiPageModel combine(MultiPageModel multiPageModel);
45 |
46 | }
47 |
--------------------------------------------------------------------------------
/release.properties:
--------------------------------------------------------------------------------
1 | #release configuration
2 | #Thu Jan 21 19:19:07 CST 2016
3 | scm.commentPrefix=[maven-release-plugin]
4 | pushChanges=true
5 | project.dev.us.codecraft\:webmagic-extension=0.5.4-SNAPSHOT
6 | project.scm.us.codecraft\:webmagic-parent.developerConnection=scm\:git\:git@github.com\:code4craft/webmagic.git
7 | project.rel.us.codecraft\:webmagic-core=0.5.3
8 | project.rel.us.codecraft\:webmagic-extension=0.5.3
9 | project.rel.us.codecraft\:webmagic-parent=0.5.3
10 | scm.tag=webmagic-parent-0.5.3
11 | remoteTagging=true
12 | project.scm.us.codecraft\:webmagic-parent.tag=webmagic-parent-0.5.3
13 | exec.additionalArguments=-Psonatype-oss-release -P development
14 | project.dev.us.codecraft\:webmagic-core=0.5.4-SNAPSHOT
15 | scm.url=scm\:git\:git@github.com\:code4craft/webmagic.git
16 | scm.tagNameFormat=@{project.artifactId}-@{project.version}
17 | project.scm.us.codecraft\:webmagic-extension.empty=true
18 | project.scm.us.codecraft\:webmagic-parent.url=git@github.com\:code4craft/webmagic.git
19 | preparationGoals=clean verify
20 | project.scm.us.codecraft\:webmagic-core.empty=true
21 | project.scm.us.codecraft\:webmagic-parent.connection=scm\:git\:git@github.com\:code4craft/webmagic.git
22 | exec.snapshotReleasePluginAllowed=false
23 | project.dev.us.codecraft\:webmagic-parent=0.5.4-SNAPSHOT
24 | completedPhase=end-release
25 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/utils/MultiKeyMapBase.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.utils;
2 |
3 | /**
4 | * @author code4crafter@gmail.com
5 | */
6 |
7 | import java.util.HashMap;
8 | import java.util.Map;
9 |
10 | /**
11 | * multi-key map, some basic objects *
12 | *
13 | * @author yihua.huang
14 | */
15 | public abstract class MultiKeyMapBase {
16 |
17 | protected static final Class DEFAULT_CLAZZ = HashMap.class;
18 | @SuppressWarnings("rawtypes")
19 | private Class protoMapClass = DEFAULT_CLAZZ;
20 |
21 | public MultiKeyMapBase() {
22 | }
23 |
24 | @SuppressWarnings("rawtypes")
25 | public MultiKeyMapBase(Class protoMapClass) {
26 | this.protoMapClass = protoMapClass;
27 | }
28 |
29 | @SuppressWarnings("unchecked")
30 | protected Map newMap() {
31 | try {
32 | return (Map) protoMapClass.newInstance();
33 | } catch (InstantiationException e) {
34 | throw new IllegalArgumentException("wrong proto type map "
35 | + protoMapClass);
36 | } catch (IllegalAccessException e) {
37 | throw new IllegalArgumentException("wrong proto type map "
38 | + protoMapClass);
39 | }
40 | }
41 | }
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/DianpingFtlDataScanner.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.samples;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.model.AfterExtractor;
6 | import us.codecraft.webmagic.model.OOSpider;
7 | import us.codecraft.webmagic.model.annotation.ExtractBy;
8 | import us.codecraft.webmagic.model.annotation.TargetUrl;
9 |
10 | import java.util.List;
11 |
12 | /**
13 | * @author yihua.huang@dianping.com
14 | * Date: 13-8-13
15 | * Time: 上午10:13
16 | */
17 | @TargetUrl("http://*.alpha.dp/*")
18 | public class DianpingFtlDataScanner implements AfterExtractor {
19 |
20 | @ExtractBy(value = "(DP\\.data\\(\\{.*\\}\\));", type = ExtractBy.Type.Regex, notNull = true, multi = true)
21 | private List data;
22 |
23 | public static void main(String[] args) {
24 | OOSpider.create(Site.me().addStartUrl("http://w.alpha.dp/").setSleepTime(0), DianpingFtlDataScanner.class)
25 | .thread(5).run();
26 | }
27 |
28 | @Override
29 | public void afterProcess(Page page) {
30 | if (data.size() > 1) {
31 | System.err.println(page.getUrl());
32 | }
33 | if (data.size() > 0 && data.get(0).length() > 100) {
34 | System.err.println(page.getUrl());
35 | }
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/scheduler/QueueScheduler.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.scheduler;
2 |
3 | import org.apache.http.annotation.ThreadSafe;
4 | import us.codecraft.webmagic.Request;
5 | import us.codecraft.webmagic.Task;
6 |
7 | import java.util.concurrent.BlockingQueue;
8 | import java.util.concurrent.LinkedBlockingQueue;
9 |
10 |
11 | /**
12 | * Basic Scheduler implementation.
13 | * Store urls to fetch in LinkedBlockingQueue and remove duplicate urls by HashMap.
14 | *
15 | * @author code4crafter@gmail.com
16 | * @since 0.1.0
17 | */
18 | @ThreadSafe
19 | public class QueueScheduler extends DuplicateRemovedScheduler implements MonitorableScheduler {
20 |
21 | private BlockingQueue queue = new LinkedBlockingQueue();
22 |
23 | @Override
24 | public void pushWhenNoDuplicate(Request request, Task task) {
25 | queue.add(request);
26 | }
27 |
28 | @Override
29 | public synchronized Request poll(Task task) {
30 | return queue.poll();
31 | }
32 |
33 | @Override
34 | public int getLeftRequestsCount(Task task) {
35 | return queue.size();
36 | }
37 |
38 | @Override
39 | public int getTotalRequestsCount(Task task) {
40 | return getDuplicateRemover().getTotalRequestsCount(task);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/webmagic-core/src/test/java/us/codecraft/webmagic/selector/SelectorTest.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.selector;
2 |
3 | import org.junit.Test;
4 |
5 | import java.util.List;
6 |
7 | import static org.assertj.core.api.Assertions.assertThat;
8 |
9 | /**
10 | * @author code4crafter@gmail.com
11 | */
12 | public class SelectorTest {
13 |
14 | private String html = "";
15 |
16 | @Test
17 | public void testChain() throws Exception {
18 | Html selectable = new Html(html);
19 | List linksWithoutChain = selectable.links().all();
20 | Selectable xpath = selectable.xpath("//div");
21 | List linksWithChainFirstCall = xpath.links().all();
22 | List linksWithChainSecondCall = xpath.links().all();
23 | assertThat(linksWithoutChain).hasSameSizeAs(linksWithChainFirstCall);
24 | assertThat(linksWithChainFirstCall).hasSameSizeAs(linksWithChainSecondCall);
25 | }
26 |
27 | @Test
28 | public void testNodes() throws Exception {
29 | Html selectable = new Html(html);
30 | List links = selectable.xpath("//a").nodes();
31 | assertThat(links.get(0).links().get()).isEqualTo("http://whatever.com/aaa");
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/F58PageProcesser.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.samples;
2 |
3 | import us.codecraft.webmagic.Site;
4 | import us.codecraft.webmagic.Page;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.processor.PageProcessor;
7 | import us.codecraft.webmagic.scheduler.RedisScheduler;
8 |
9 | import java.util.List;
10 |
11 | /**
12 | * @author code4crafter@gmail.com
13 | * Date: 13-4-21
14 | * Time: 下午1:48
15 | */
16 | public class F58PageProcesser implements PageProcessor {
17 |
18 | @Override
19 | public void process(Page page) {
20 | List strings = page.getHtml().links().regex(".*/yewu/.*").all();
21 | page.addTargetRequests(strings);
22 | page.putField("title",page.getHtml().regex("(.*)"));
23 | page.putField("body",page.getHtml().xpath("//dd"));
24 | }
25 |
26 | @Override
27 | public Site getSite() {
28 | return Site.me().setDomain("sh.58.com").addStartUrl("http://sh1.51a8.com/").setCycleRetryTimes(2); //To change body of implemented methods use File | Settings | File Templates.
29 | }
30 |
31 | public static void main(String[] args) {
32 | Spider.create(new F58PageProcesser()).setScheduler(new RedisScheduler("localhost")).run();
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/webmagic-core/src/test/java/us/codecraft/webmagic/pipeline/FilePipelineTest.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.pipeline;
2 |
3 | import org.junit.BeforeClass;
4 | import org.junit.Test;
5 | import us.codecraft.webmagic.Request;
6 | import us.codecraft.webmagic.ResultItems;
7 | import us.codecraft.webmagic.Site;
8 | import us.codecraft.webmagic.Task;
9 |
10 | import java.util.UUID;
11 |
12 | /**
13 | * Created by ywooer on 2014/5/6 0006.
14 | */
15 | public class FilePipelineTest {
16 |
17 | private static ResultItems resultItems;
18 | private static Task task;
19 |
20 | @BeforeClass
21 | public static void before() {
22 | resultItems = new ResultItems();
23 | resultItems.put("content", "webmagic 爬虫工具");
24 | Request request = new Request("http://www.baidu.com");
25 | resultItems.setRequest(request);
26 |
27 | task = new Task() {
28 | @Override
29 | public String getUUID() {
30 | return UUID.randomUUID().toString();
31 | }
32 |
33 | @Override
34 | public Site getSite() {
35 | return null;
36 | }
37 | };
38 | }
39 | @Test
40 | public void testProcess() {
41 | FilePipeline filePipeline = new FilePipeline();
42 | filePipeline.process(resultItems, task);
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/Formatter.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.annotation;
2 |
3 | import us.codecraft.webmagic.model.formatter.ObjectFormatter;
4 |
5 | import java.lang.annotation.ElementType;
6 | import java.lang.annotation.Retention;
7 | import java.lang.annotation.Target;
8 |
9 | /**
10 | * Define how the result string is convert to an object for field.
11 | *
12 | * @author code4crafter@gmail.com
13 | * @since 0.3.2
14 | */
15 | @Retention(java.lang.annotation.RetentionPolicy.RUNTIME)
16 | @Target({ElementType.FIELD})
17 | public @interface Formatter {
18 |
19 | /**
20 | * Set formatter params.
21 | *
22 | * @return formatter params
23 | */
24 | String[] value() default "";
25 |
26 | /**
27 | * Specific the class of field of class of elements in collection for field.
28 | * It is not necessary to be set because we can detect the class by class of field,
29 | * unless you use a collection as a field.
30 | *
31 | * @return the class of field
32 | */
33 | Class subClazz() default Void.class;
34 |
35 | /**
36 | * If there are more than one formatter for a class, just specify the implement.
37 | * @return implement
38 | */
39 | Class formatter() default ObjectFormatter.class;
40 |
41 | }
42 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/TargetUrl.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.annotation;
2 |
3 | import java.lang.annotation.ElementType;
4 | import java.lang.annotation.Retention;
5 | import java.lang.annotation.Target;
6 |
7 | /**
8 | * Define the url patterns for class.
9 | * All urls matching the pattern will be crawled and extracted for new objects.
10 | *
11 | * @author code4crafter@gmail.com
12 | * @since 0.2.0
13 | */
14 | @Retention(java.lang.annotation.RetentionPolicy.RUNTIME)
15 | @Target({ElementType.TYPE})
16 | public @interface TargetUrl {
17 |
18 | /**
19 | * The url patterns for class.
20 | * Use regex expression with some changes:
21 | * "." stand for literal character "." instead of "any character".
22 | * "*" stand for any legal character for url in 0-n length ([^"'#]*) instead of "any length".
23 | *
24 | * @return the url patterns for class
25 | */
26 | String[] value();
27 |
28 | /**
29 | * Define the region for url extracting.
30 | * Only support XPath.
31 | * When sourceRegion is set, the urls will be extracted only from the region instead of entire content.
32 | *
33 | * @return the region for url extracting
34 | */
35 | String sourceRegion() default "";
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/BaiduNews.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.samples;
2 |
3 | import us.codecraft.webmagic.Site;
4 | import us.codecraft.webmagic.model.OOSpider;
5 | import us.codecraft.webmagic.model.annotation.ExtractBy;
6 |
7 | /**
8 | * @author code4crafter@gmail.com
9 | * @date 14-4-9
10 | */
11 | public class BaiduNews {
12 |
13 | @ExtractBy("//h3[@class='c-title']/a/text()")
14 | private String name;
15 |
16 | @ExtractBy("//div[@class='c-summary']/text()")
17 | private String description;
18 |
19 | @Override
20 | public String toString() {
21 | return "BaiduNews{" +
22 | "name='" + name + '\'' +
23 | ", description='" + description + '\'' +
24 | '}';
25 | }
26 |
27 | public static void main(String[] args) {
28 | OOSpider ooSpider = OOSpider.create(Site.me().setSleepTime(0), BaiduNews.class);
29 | //single download
30 | BaiduNews baike = ooSpider.get("http://news.baidu.com/ns?tn=news&cl=2&rn=20&ct=1&fr=bks0000&ie=utf-8&word=httpclient");
31 | System.out.println(baike);
32 |
33 | ooSpider.close();
34 | }
35 |
36 | public String getName() {
37 | return name;
38 | }
39 |
40 | public String getDescription() {
41 | return description;
42 | }
43 | }
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/HelpUrl.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.annotation;
2 |
3 | import java.lang.annotation.ElementType;
4 | import java.lang.annotation.Retention;
5 | import java.lang.annotation.Target;
6 |
7 | /**
8 | * Define the 'help' url patterns for class.
9 | * All urls matching the pattern will be crawled and but not extracted for new objects.
10 | *
11 | * @author code4crafter@gmail.com
12 | * @since 0.2.0
13 | */
14 | @Retention(java.lang.annotation.RetentionPolicy.RUNTIME)
15 | @Target({ElementType.TYPE})
16 | public @interface HelpUrl {
17 |
18 | /**
19 | * The url patterns to crawl.
20 | * Use regex expression with some changes:
21 | * "." stand for literal character "." instead of "any character".
22 | * "*" stand for any legal character for url in 0-n length ([^"'#]*) instead of "any length".
23 | *
24 | * @return the url patterns for class
25 | */
26 | String[] value();
27 |
28 | /**
29 | * Define the region for url extracting.
30 | * Only support XPath.
31 | * When sourceRegion is set, the urls will be extracted only from the region instead of entire content.
32 | *
33 | * @return the region for url extracting
34 | */
35 | String sourceRegion() default "";
36 | }
37 |
--------------------------------------------------------------------------------
/webmagic-scripts/src/main/java/us/codecraft/webmagic/scripts/ScriptEnginePool.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.scripts;
2 |
3 | import javax.script.ScriptEngine;
4 | import javax.script.ScriptEngineManager;
5 | import java.util.concurrent.LinkedBlockingQueue;
6 | import java.util.concurrent.atomic.AtomicInteger;
7 |
8 | /**
9 | * @author code4crafter@gmail.com
10 | * @since 0.4.1
11 | */
12 | public class ScriptEnginePool {
13 |
14 | private final int size;
15 |
16 | private final AtomicInteger availableCount;
17 |
18 | private final LinkedBlockingQueue scriptEngines = new LinkedBlockingQueue();
19 |
20 | public ScriptEnginePool(Language language,int size) {
21 | this.size = size;
22 | this.availableCount = new AtomicInteger(size);
23 | for (int i=0;i spiderMap;
24 |
25 | public Worker(int poolSize) {
26 | this.poolSize = poolSize;
27 | this.executorService = initExecutorService();
28 | this.spiderMap = new ConcurrentHashMap();
29 | }
30 |
31 | public Worker() {
32 | this(DEFAULT_POOL_SIZE);
33 | }
34 |
35 | protected ExecutorService initExecutorService() {
36 | return Executors.newFixedThreadPool(poolSize);
37 | }
38 |
39 | public void addSpider(Spider spider) {
40 | spider.setExecutorService(executorService);
41 | spiderMap.put(spider.getUUID(), spider);
42 | }
43 |
44 | public Spider getSpider(String uuid){
45 | return spiderMap.get(uuid);
46 | }
47 |
48 | }
49 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/selector/ReplaceSelector.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.selector;
2 |
3 | import java.util.List;
4 | import java.util.regex.Matcher;
5 | import java.util.regex.Pattern;
6 | import java.util.regex.PatternSyntaxException;
7 |
8 | /**
9 | * Replace selector.
10 | *
11 | * @author code4crafter@gmail.com
12 | * @since 0.1.0
13 | */
14 | public class ReplaceSelector implements Selector {
15 |
16 | private String regexStr;
17 |
18 | private String replacement;
19 |
20 | private Pattern regex;
21 |
22 | public ReplaceSelector(String regexStr, String replacement) {
23 | this.regexStr = regexStr;
24 | this.replacement = replacement;
25 | try {
26 | regex = Pattern.compile(regexStr);
27 | } catch (PatternSyntaxException e) {
28 | throw new IllegalArgumentException("invalid regex", e);
29 | }
30 | }
31 |
32 | @Override
33 | public String select(String text) {
34 | Matcher matcher = regex.matcher(text);
35 | return matcher.replaceAll(replacement);
36 | }
37 |
38 | @Override
39 | public List selectList(String text) {
40 | throw new UnsupportedOperationException();
41 | }
42 |
43 | @Override
44 | public String toString() {
45 | return regexStr + "_" + replacement;
46 | }
47 |
48 | }
49 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/utils/IPUtils.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.utils;
2 |
3 | import java.net.Inet6Address;
4 | import java.net.InetAddress;
5 | import java.net.NetworkInterface;
6 | import java.net.SocketException;
7 | import java.util.Enumeration;
8 |
9 | /**
10 | * @author code4crafer@gmail.com
11 | * @since 0.5.0
12 | */
13 | public abstract class IPUtils {
14 |
15 | public static String getFirstNoLoopbackIPAddresses() throws SocketException {
16 |
17 | Enumeration networkInterfaces = NetworkInterface.getNetworkInterfaces();
18 |
19 | InetAddress localAddress = null;
20 | while (networkInterfaces.hasMoreElements()) {
21 | NetworkInterface networkInterface = networkInterfaces.nextElement();
22 | Enumeration inetAddresses = networkInterface.getInetAddresses();
23 | while (inetAddresses.hasMoreElements()) {
24 | InetAddress address = inetAddresses.nextElement();
25 | if (!address.isLoopbackAddress() && !Inet6Address.class.isInstance(address)) {
26 | return address.getHostAddress();
27 | } else if (!address.isLoopbackAddress()) {
28 | localAddress = address;
29 | }
30 | }
31 | }
32 |
33 | return localAddress.getHostAddress();
34 | }
35 |
36 | }
37 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/model/samples/JokejiModel.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.samples;
2 |
3 | import us.codecraft.webmagic.Site;
4 | import us.codecraft.webmagic.model.ConsolePageModelPipeline;
5 | import us.codecraft.webmagic.model.OOSpider;
6 | import us.codecraft.webmagic.model.annotation.ExtractBy;
7 | import us.codecraft.webmagic.model.annotation.HelpUrl;
8 | import us.codecraft.webmagic.model.annotation.TargetUrl;
9 | import us.codecraft.webmagic.scheduler.RedisScheduler;
10 |
11 | /**
12 | * @author code4crafter@gmail.com
13 | */
14 | @TargetUrl("http://www.jokeji.cn/jokehtml/jy/\\d+.htm")
15 | @HelpUrl("http://www.jokeji.cn/list\\w+.htm")
16 | public class JokejiModel {
17 |
18 | @ExtractBy("//title/regex('([^_]+)',1)")
19 | private String title;
20 |
21 | @ExtractBy("//div[@class=mob_txt]/tidyText()")
22 | private String content;
23 |
24 | public static void main(String[] args) {
25 | OOSpider.create(Site.me().setDomain("www.jokeji.cn").setCharset("gbk").setSleepTime(100).setTimeOut(3000)
26 | .setUserAgent("Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)")
27 | , new ConsolePageModelPipeline(), JokejiModel.class).addUrl("http://www.jokeji.cn/").thread(2)
28 | .scheduler(new RedisScheduler("127.0.0.1"))
29 | .run();
30 | }
31 |
32 | }
33 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/test/java/us/codecraft/webmagic/Foo.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic;
2 |
3 | import us.codecraft.forger.property.Inject;
4 | import us.codecraft.forger.property.format.Formatter;
5 |
6 | /**
7 | * @author code4crafter@gmail.com
8 | */
9 | public class Foo {
10 |
11 | @Formatter("")
12 | @Inject("fooa")
13 | private String foo;
14 |
15 | public static final String SOURCE_CODE="package us.codecraft.webmagic;\n" +
16 | "\n" +
17 | "import us.codecraft.forger.property.Inject;\n" +
18 | "import us.codecraft.forger.property.format.Formatter;\n" +
19 | "\n" +
20 | "/**\n" +
21 | " * @author code4crafter@gmail.com\n" +
22 | " */\n" +
23 | "public class Foo {\n" +
24 | "\n" +
25 | " @Formatter(\"\")\n" +
26 | " @Inject(\"fooa\")\n" +
27 | " private String foo;\n" +
28 | "\n" +
29 | " public String getFoo() {\n" +
30 | " return foo;\n" +
31 | " }\n" +
32 | "\n" +
33 | " public String foo() {\n" +
34 | " return foo;\n" +
35 | " }\n" +
36 | "}";
37 |
38 | public String getFoo() {
39 | return foo;
40 | }
41 |
42 | public String foo() {
43 | return foo;
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/utils/FilePersistentBase.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.utils;
2 |
3 | import java.io.File;
4 |
5 | /**
6 | * Base object of file persistence.
7 | *
8 | * @author code4crafter@gmail.com <br>
9 | * @since 0.2.0
10 | */
11 | public class FilePersistentBase {
12 |
13 | protected String path;
14 |
15 | public static String PATH_SEPERATOR = "/";
16 |
17 | static {
18 | String property = System.getProperties().getProperty("file.separator");
19 | if (property != null) {
20 | PATH_SEPERATOR = property;
21 | }
22 | }
23 |
24 | public void setPath(String path) {
25 | if (!path.endsWith(PATH_SEPERATOR)) {
26 | path += PATH_SEPERATOR;
27 | }
28 | this.path = path;
29 | }
30 |
31 | public File getFile(String fullName) {
32 | checkAndMakeParentDirecotry(fullName);
33 | return new File(fullName);
34 | }
35 |
36 | public void checkAndMakeParentDirecotry(String fullName) {
37 | int index = fullName.lastIndexOf(PATH_SEPERATOR);
38 | if (index > 0) {
39 | String path = fullName.substring(0, index);
40 | File file = new File(path);
41 | if (!file.exists()) {
42 | file.mkdirs();
43 | }
44 | }
45 | }
46 |
47 | public String getPath() {
48 | return path;
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/webmagic-core/src/test/java/us/codecraft/webmagic/selector/ExtractorsTest.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.selector;
2 |
3 | import org.junit.Test;
4 |
5 | import static org.assertj.core.api.Assertions.assertThat;
6 | import static us.codecraft.webmagic.selector.Selectors.*;
7 |
8 | /**
9 | * @author code4crafter@gmail.com <br>
10 | */
11 | public class ExtractorsTest {
12 |
13 | String html = "<div><h1>test<a href=\"xxx\">aabbcc</a></h1></div>";
14 |
15 | String html2 = "<title>aabbcc";
16 |
17 | @Test
18 | public void testEach() {
19 | assertThat($("div h1 a").select(html)).isEqualTo("aabbcc");
20 | assertThat($("div h1 a", "href").select(html)).isEqualTo("xxx");
21 | assertThat($("div h1 a", "innerHtml").select(html)).isEqualTo("aabbcc");
22 | assertThat(xpath("//a/@href").select(html)).isEqualTo("xxx");
23 | assertThat(regex("a href=\"(.*)\"").select(html)).isEqualTo("xxx");
24 | assertThat(regex("(a href)=\"(.*)\"", 2).select(html)).isEqualTo("xxx");
25 | }
26 |
27 | @Test
28 | public void testCombo() {
29 | assertThat(and($("title"), regex("aa(bb)cc")).select(html2)).isEqualTo("bb");
30 | OrSelector or = or($("div h1 a", "innerHtml"), xpath("//title"));
31 | assertThat(or.select(html)).isEqualTo("aabbcc");
32 | assertThat(or.select(html2)).isEqualTo("aabbcc");
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/webmagic-samples/src/test/java/us/codecraft/webmagic/processor/SinablogProcessorTest.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.processor;
2 |
3 | import org.junit.Ignore;
4 | import org.junit.Test;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.pipeline.FilePipeline;
7 | import us.codecraft.webmagic.pipeline.JsonFilePipeline;
8 | import us.codecraft.webmagic.samples.SinaBlogProcessor;
9 | import us.codecraft.webmagic.scheduler.FileCacheQueueScheduler;
10 |
11 | import java.io.IOException;
12 |
13 | /**
14 | * @author code4crafter@gmail.com
15 | * Date: 13-6-9
16 | * Time: 上午8:02
17 | */
18 | public class SinablogProcessorTest {
19 |
20 | @Ignore
21 | @Test
22 | public void test() throws IOException {
23 | SinaBlogProcessor sinaBlogProcessor = new SinaBlogProcessor();
24 | //pipeline是抓取结束后的处理
25 | //默认放到/data/webmagic/ftl/[domain]目录下
26 | JsonFilePipeline pipeline = new JsonFilePipeline("/data/webmagic/");
27 | //Spider.me()是简化写法，其实就是new一个啦
28 | //Spider.pipeline()设定一个pipeline，支持链式调用
29 | //ConsolePipeline输出结果到控制台
30 | //FileCacheQueueSchedular保存url，支持断点续传，临时文件输出到/data/temp/webmagic/cache目录
31 | //Spider.run()执行
32 | Spider.create(sinaBlogProcessor).pipeline(new FilePipeline()).pipeline(pipeline).scheduler(new FileCacheQueueScheduler("/data/temp/webmagic/cache/")).
33 | run();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/webmagic-selenium/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | webmagic-parent
5 | us.codecraft
6 | 0.5.4-SNAPSHOT
7 |
8 | 4.0.0
9 |
10 | webmagic-selenium
11 |
12 |
13 |
14 | org.seleniumhq.selenium
15 | selenium-java
16 | 2.46.0
17 |
18 |
19 | us.codecraft
20 | webmagic-core
21 | ${project.version}
22 |
23 |
24 | com.github.detro
25 | phantomjsdriver
26 | 1.2.0
27 |
28 |
29 |
30 |
31 |
32 | junit
33 | junit
34 |
35 |
36 |
37 |
38 |
39 |
40 | maven-deploy-plugin
41 |
42 | true
43 |
44 |
45 |
46 |
47 |
48 |
49 |
--------------------------------------------------------------------------------
/webmagic-selenium/src/test/java/us/codecraft/webmagic/samples/GooglePlayProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.samples;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.downloader.selenium.SeleniumDownloader;
7 | import us.codecraft.webmagic.pipeline.FilePipeline;
8 | import us.codecraft.webmagic.processor.PageProcessor;
9 |
10 | /**
11 | *
12 | * Using Selenium with PhantomJS to fetch web-page with JS
13 | *
14 | * @author bob.li.0718@gmail.com
15 | * Date: 15-7-11
16 | */
17 | public class GooglePlayProcessor implements PageProcessor {
18 |
19 | private Site site;
20 |
21 | @Override
22 | public void process(Page page) {
23 |
24 | page.putField("whole-html", page.getHtml().toString());
25 |
26 | }
27 |
28 | @Override
29 | public Site getSite() {
30 | if (null == site) {
31 | site = Site.me().setDomain("play.google.com").setSleepTime(300);
32 | }
33 | return site;
34 | }
35 |
36 | public static void main(String[] args) {
37 | Spider.create(new GooglePlayProcessor())
38 | .thread(5)
39 | .addPipeline(
40 | new FilePipeline(
41 | "/Users/Bingo/Documents/workspace/webmagic/webmagic-selenium/data/"))
42 | .setDownloader(new SeleniumDownloader())
43 | .addUrl("https://play.google.com/store/apps/details?id=com.tencent.mm")
44 | .runAsync();
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/selector/OrSelector.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.selector;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 |
6 | /**
7 | * All extractors will do extracting separately,
8 | * and the results of extractors will combined as the final result.
9 | * @author code4crafter@gmail.com
10 | * @since 0.2.0
11 | */
12 | public class OrSelector implements Selector {
13 |
14 | private List selectors = new ArrayList();
15 |
16 | public OrSelector(Selector... selectors) {
17 | for (Selector selector : selectors) {
18 | this.selectors.add(selector);
19 | }
20 | }
21 |
22 | public OrSelector(List selectors) {
23 | this.selectors = selectors;
24 | }
25 |
26 | @Override
27 | public String select(String text) {
28 | for (Selector selector : selectors) {
29 | String result = selector.select(text);
30 | if (result != null) {
31 | return result;
32 | }
33 | }
34 | return null;
35 | }
36 |
37 | @Override
38 | public List selectList(String text) {
39 | List results = new ArrayList();
40 | for (Selector selector : selectors) {
41 | List strings = selector.selectList(text);
42 | results.addAll(strings);
43 | }
44 | return results;
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/KaichibaProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.samples;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.processor.PageProcessor;
7 |
8 | /**
9 | * @author code4crafter@gmail.com
10 | * Date: 13-5-20
11 | * Time: 下午5:31
12 | */
13 | public class KaichibaProcessor implements PageProcessor {
14 | @Override
15 | public void process(Page page) {
16 | //http://progressdaily.diandian.com/post/2013-01-24/40046867275
17 | int i = Integer.valueOf(page.getUrl().regex("shop/(\\d+)").toString()) + 1;
18 | page.addTargetRequest("http://kaichiba.com/shop/" + i);
19 | page.putField("title",page.getHtml().xpath("//Title"));
20 | page.putField("items", page.getHtml().xpath("//li[@class=\"foodTitle\"]").replace("^\\s+", "").replace("\\s+$", "").replace(".*?", ""));
21 | }
22 |
23 | @Override
24 | public Site getSite() {
25 | return Site.me().setDomain("kaichiba.com").addStartUrl("http://kaichiba.com/shop/41725781").setCharset("utf-8").
26 | setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
27 | }
28 |
29 | public static void main(String[] args) {
30 | Spider.create(new KaichibaProcessor()).run();
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/example/GithubRepoPageMapper.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.example;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.model.PageMapper;
7 | import us.codecraft.webmagic.processor.PageProcessor;
8 |
9 | /**
10 | * @author code4crafter@gmail.com
11 | * @since 0.3.2
12 | */
13 | public class GithubRepoPageMapper implements PageProcessor {
14 |
15 | private Site site = Site.me().setRetryTimes(3).setSleepTime(0);
16 |
17 | private PageMapper githubRepoPageMapper = new PageMapper(GithubRepo.class);
18 |
19 | @Override
20 | public void process(Page page) {
21 | page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
22 | page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+)").all());
23 | GithubRepo githubRepo = githubRepoPageMapper.get(page);
24 | if (githubRepo == null) {
25 | page.setSkip(true);
26 | } else {
27 | page.putField("repo", githubRepo);
28 | }
29 |
30 | }
31 |
32 | @Override
33 | public Site getSite() {
34 | return site;
35 | }
36 |
37 | public static void main(String[] args) {
38 | Spider.create(new GithubRepoPageMapper()).addUrl("https://github.com/code4craft").thread(5).run();
39 | }
40 | }
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/model/annotation/ExtractByUrl.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model.annotation;
2 |
3 | import java.lang.annotation.ElementType;
4 | import java.lang.annotation.Retention;
5 | import java.lang.annotation.Target;
6 |
7 | /**
8 | * Define a extractor to extract data in url of current page. Only regex can be used.
9 | *
10 | * @author code4crafter@gmail.com
11 | * @since 0.2.0
12 | */
13 | @Retention(java.lang.annotation.RetentionPolicy.RUNTIME)
14 | @Target({ElementType.FIELD})
15 | public @interface ExtractByUrl {
16 |
17 | /**
18 | * Extractor expression, only regex can be used
19 | *
20 | * @return extractor expression
21 | */
22 | String value() default "";
23 |
24 | /**
25 | * Define whether the field can be null.
26 | * If set to 'true' and the extractor get no result, the entire class will be discarded.
27 | *
28 | * @return whether the field can be null
29 | */
30 | boolean notNull() default false;
31 |
32 | /**
33 | * Define whether the extractor return more than one result.
34 | * When set to 'true', the extractor return a list of string (so you should define the field as List).
35 | *
36 | * Deprecated since 0.4.2. This option is determined automatically by the class of field.
37 | * @deprecated since 0.4.2
38 | * @return whether the extractor return more than one result
39 | */
40 | boolean multi() default false;
41 |
42 | }
43 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/processor/SimplePageProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.processor;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.utils.UrlUtils;
6 |
7 | import java.util.List;
8 |
9 | /**
10 | * A simple PageProcessor.
11 | *
12 | * @author code4crafter@gmail.com
13 | * @since 0.1.0
14 | */
15 | public class SimplePageProcessor implements PageProcessor {
16 |
17 | private String urlPattern;
18 |
19 | private Site site;
20 |
21 | public SimplePageProcessor(String startUrl, String urlPattern) {
22 | this.site = Site.me().addStartUrl(startUrl).
23 | setDomain(UrlUtils.getDomain(startUrl));
24 | //compile "*" expression to regex
25 | this.urlPattern = "(" + urlPattern.replace(".", "\\.").replace("*", "[^\"'#]*") + ")";
26 |
27 | }
28 |
29 | @Override
30 | public void process(Page page) {
31 | List requests = page.getHtml().links().regex(urlPattern).all();
32 | //add urls to fetch
33 | page.addTargetRequests(requests);
34 | //extract by XPath
35 | page.putField("title", page.getHtml().xpath("//title"));
36 | page.putField("html", page.getHtml().toString());
37 | //extract by Readability
38 | page.putField("content", page.getHtml().smartContent());
39 | }
40 |
41 | @Override
42 | public Site getSite() {
43 | //settings
44 | return site;
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/OschinaBlogPageProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.processor.example;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.processor.PageProcessor;
7 |
8 | import java.util.List;
9 |
10 | /**
11 | * @author code4crafter@gmail.com
12 | */
13 | public class OschinaBlogPageProcessor implements PageProcessor {
14 |
15 | private Site site = Site.me().setDomain("my.oschina.net");
16 |
17 | @Override
18 | public void process(Page page) {
19 | List links = page.getHtml().links().regex("http://my\\.oschina\\.net/flashsword/blog/\\d+").all();
20 | page.addTargetRequests(links);
21 | page.putField("title", page.getHtml().xpath("//div[@class='BlogEntity']/div[@class='BlogTitle']/h1/text()").toString());
22 | if (page.getResultItems().get("title") == null) {
23 | //skip this page
24 | page.setSkip(true);
25 | }
26 | page.putField("content", page.getHtml().smartContent().toString());
27 | page.putField("tags", page.getHtml().xpath("//div[@class='BlogTags']/a/text()").all());
28 | }
29 |
30 | @Override
31 | public Site getSite() {
32 | return site;
33 |
34 | }
35 |
36 | public static void main(String[] args) {
37 | Spider.create(new OschinaBlogPageProcessor()).addUrl("http://my.oschina.net/flashsword/blog").run();
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/selector/Selectors.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.selector;
2 |
3 | /**
4 | * Convenient methods for selectors.
5 | *
6 | * @author code4crafter@gmail.com
7 | * @since 0.2.1
8 | */
9 | public abstract class Selectors {
10 |
11 | public static RegexSelector regex(String expr) {
12 | return new RegexSelector(expr);
13 | }
14 |
15 | public static RegexSelector regex(String expr, int group) {
16 | return new RegexSelector(expr,group);
17 | }
18 |
19 | public static SmartContentSelector smartContent() {
20 | return new SmartContentSelector();
21 | }
22 |
23 | public static CssSelector $(String expr) {
24 | return new CssSelector(expr);
25 | }
26 |
27 | public static CssSelector $(String expr, String attrName) {
28 | return new CssSelector(expr, attrName);
29 | }
30 |
31 | public static XpathSelector xpath(String expr) {
32 | return new XpathSelector(expr);
33 | }
34 |
35 | /**
36 | * @Deprecated
37 | * @see #xpath(String)
38 | * @param expr expr
39 | * @return new selector
40 | */
41 | public static XpathSelector xsoup(String expr) {
42 | return new XpathSelector(expr);
43 | }
44 |
45 | public static AndSelector and(Selector... selectors) {
46 | return new AndSelector(selectors);
47 | }
48 |
49 | public static OrSelector or(Selector... selectors) {
50 | return new OrSelector(selectors);
51 | }
52 |
53 | }
--------------------------------------------------------------------------------
/webmagic-avalon/forger/src/test/java/us/codecraft/forger/Foo.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.forger;
2 |
3 | import us.codecraft.forger.property.Inject;
4 | import us.codecraft.forger.property.format.Formatter;
5 |
6 | /**
7 | * @author code4crafter@gmail.com
8 | */
9 | public class Foo implements Fooable{
10 |
11 | @Formatter("")
12 | @Inject("fooa")
13 | private String foo;
14 |
15 | public static final String SOURCE_CODE="import us.codecraft.forger.*;\n" +
16 | "import us.codecraft.forger.property.Inject;\n" +
17 | "import us.codecraft.forger.property.Inject;\n" +
18 | "import us.codecraft.forger.property.format.Formatter;\n" +
19 | "\n" +
20 | "/**\n" +
21 | " * @author code4crafter@gmail.com\n" +
22 | " */\n" +
23 | "public class Foo implements Fooable{\n" +
24 | "\n" +
25 | " @Formatter(\"\")\n" +
26 | " @Inject(\"fooa\")\n" +
27 | " private String foo;\n" +
28 | "\n" +
29 | " public String getFoo() {\n" +
30 | " return foo;\n" +
31 | " }\n" +
32 | "\n" +
33 | " @Override\n" +
34 | " public String foo() {\n" +
35 | " return foo;\n" +
36 | " }\n" +
37 | "}";
38 |
39 | public String getFoo() {
40 | return foo;
41 | }
42 |
43 | @Override
44 | public String foo() {
45 | return foo;
46 | }
47 | }
48 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/selector/BaseElementSelector.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.selector;
2 |
3 | import org.jsoup.Jsoup;
4 | import org.jsoup.nodes.Element;
5 |
6 | import java.util.ArrayList;
7 | import java.util.List;
8 |
9 | /**
10 | * @author code4crafter@gmail.com
11 | * @since 0.3.0
12 | */
13 | public abstract class BaseElementSelector implements Selector, ElementSelector {
14 |
15 | @Override
16 | public String select(String text) {
17 | if (text != null) {
18 | return select(Jsoup.parse(text));
19 | }
20 | return null;
21 | }
22 |
23 | @Override
24 | public List selectList(String text) {
25 | if (text != null) {
26 | return selectList(Jsoup.parse(text));
27 | } else {
28 | return new ArrayList();
29 | }
30 | }
31 |
32 | public Element selectElement(String text) {
33 | if (text != null) {
34 | return selectElement(Jsoup.parse(text));
35 | }
36 | return null;
37 | }
38 |
39 | public List selectElements(String text) {
40 | if (text != null) {
41 | return selectElements(Jsoup.parse(text));
42 | } else {
43 | return new ArrayList();
44 | }
45 | }
46 |
47 | public abstract Element selectElement(Element element);
48 |
49 | public abstract List selectElements(Element element);
50 |
51 | public abstract boolean hasAttribute();
52 |
53 | }
54 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-admin/src/main/webapp/static/css/jquery.cleditor.css:
--------------------------------------------------------------------------------
1 | .cleditorMain {border:1px solid #999; padding:0 1px 1px; background-color:white}
2 | .cleditorMain iframe {border:none; margin:0; padding:0}
3 | .cleditorMain textarea {border:none; margin:0; padding:0; overflow-y:scroll; font:10pt Arial,Verdana; resize:none; outline:none /* webkit grip focus */}
4 | .cleditorToolbar {background: url('../img/toolbar.gif') repeat}
5 | .cleditorGroup {float:left; height:26px}
6 | .cleditorButton {float:left; width:24px; height:24px; margin:1px 0 1px 0; background: url('../img/buttons.gif')}
7 | .cleditorDisabled {opacity:0.3; filter:alpha(opacity=30)}
8 | .cleditorDivider {float:left; width:1px; height:23px; margin:1px 0 1px 0; background:#CCC}
9 | .cleditorPopup {border:solid 1px #999; background-color:white; position:absolute; font:10pt Arial,Verdana; cursor:default; z-index:10000}
10 | .cleditorList div {padding:2px 4px 2px 4px}
11 | .cleditorList p,
12 | .cleditorList h1,
13 | .cleditorList h2,
14 | .cleditorList h3,
15 | .cleditorList h4,
16 | .cleditorList h5,
17 | .cleditorList h6,
18 | .cleditorList font {padding:0; margin:0; background-color:Transparent}
19 | .cleditorColor {width:150px; padding:1px 0 0 1px}
20 | .cleditorColor div {float:left; width:14px; height:14px; margin:0 1px 1px 0}
21 | .cleditorPrompt {background-color:#F6F7F9; padding:4px; font-size:8.5pt}
22 | .cleditorPrompt input,
23 | .cleditorPrompt textarea {font:8.5pt Arial,Verdana;}
24 | .cleditorMsg {background-color:#FDFCEE; width:150px; padding:4px; font-size:8.5pt}
25 |
--------------------------------------------------------------------------------
/webmagic-scripts/src/test/java/us/codecraft/webmagic/scripts/ScriptProcessorTest.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.scripts;
2 |
3 | import org.junit.Ignore;
4 | import org.junit.Test;
5 | import us.codecraft.webmagic.Spider;
6 |
7 | /**
8 | * @author code4crafter@gmail.com
9 | * @since 0.4.1
10 | */
11 | @Ignore
12 | public class ScriptProcessorTest {
13 |
14 | @Test
15 | public void testJavaScriptProcessor() {
16 | ScriptProcessor pageProcessor = ScriptProcessorBuilder.custom().language(Language.JavaScript).scriptFromClassPathFile("js/oschina.js").build();
17 | pageProcessor.getSite().setSleepTime(0);
18 | Spider.create(pageProcessor).addUrl("http://my.oschina.net/flashsword/blog").setSpawnUrl(false).run();
19 | }
20 |
21 | @Test
22 | public void testRubyProcessor() {
23 | ScriptProcessor pageProcessor = ScriptProcessorBuilder.custom().language(Language.JRuby).scriptFromClassPathFile("ruby/oschina.rb").build();
24 | pageProcessor.getSite().setSleepTime(0);
25 | Spider.create(pageProcessor).addUrl("http://my.oschina.net/flashsword/blog").setSpawnUrl(false).run();
26 | }
27 |
28 |
29 | @Test
30 | public void testPythonProcessor() {
31 | ScriptProcessor pageProcessor = ScriptProcessorBuilder.custom().language(Language.Jython).scriptFromClassPathFile("python/oschina.py").build();
32 | pageProcessor.getSite().setSleepTime(0);
33 | Spider.create(pageProcessor).addUrl("http://my.oschina.net/flashsword/blog").setSpawnUrl(false).run();
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/webmagic-avalon/forger/src/main/java/us/codecraft/forger/property/Property.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.forger.property;
2 |
3 | import us.codecraft.forger.property.format.ObjectFormatter;
4 |
5 | import java.lang.reflect.Field;
6 |
7 | /**
8 | * @author code4crafter@gmail.com
9 | */
10 | public class Property {
11 |
12 | private String name;
13 |
14 | private PropertyType type;
15 |
16 | private Field field;
17 |
18 | private ObjectFormatter objectFormatter;
19 |
20 | public ObjectFormatter getObjectFormatter() {
21 | return objectFormatter;
22 | }
23 |
24 | public Property setObjectFormatter(ObjectFormatter objectFormatter) {
25 | this.objectFormatter = objectFormatter;
26 | return this;
27 | }
28 |
29 | public String getName() {
30 | return name;
31 | }
32 |
33 | public Property setName(String name) {
34 | this.name = name;
35 | return this;
36 | }
37 |
38 | public PropertyType getType() {
39 | return type;
40 | }
41 |
42 | public Property setType(PropertyType type) {
43 | this.type = type;
44 | return this;
45 | }
46 |
47 | public Field getField() {
48 | return field;
49 | }
50 |
51 | public Property setField(Field field) {
52 | this.field = field;
53 | return this;
54 | }
55 |
56 | public static Property fromField(Field field) {
57 | return new Property().setName(field.getName()).setType(PropertyType.from(field.getType())).setField(field);
58 | }
59 |
60 | }
61 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/main/resources/config/mapper/DynamicClass.xml:
--------------------------------------------------------------------------------
1 |
2 |
17 |
19 |
20 |
21 |
22 |
23 | insert into DynamicClass (`ClassName`,`SourceCode`,`AddTime`,`UpdateTime`)
24 | values (#{className},#{sourceCode},now(),now())
25 |
26 |
27 |
28 | insert into DynamicClass (`ClassName`,`SourceCode`,`AddTime`,`UpdateTime`)
29 | values (#{className},#{sourceCode},now(),now())
30 |
31 |
32 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/handler/CompositePipeline.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.handler;
2 |
3 | import us.codecraft.webmagic.ResultItems;
4 | import us.codecraft.webmagic.Task;
5 | import us.codecraft.webmagic.pipeline.Pipeline;
6 |
7 | import java.util.ArrayList;
8 | import java.util.List;
9 |
10 | /**
11 | * @author code4crafer@gmail.com
12 | */
13 | public class CompositePipeline implements Pipeline {
14 |
15 | private List subPipelines = new ArrayList();
16 |
17 | @Override
18 | public void process(ResultItems resultItems, Task task) {
19 | for (SubPipeline subPipeline : subPipelines) {
20 | if (subPipeline.match(resultItems.getRequest())) {
21 | RequestMatcher.MatchOther matchOtherProcessorProcessor = subPipeline.processResult(resultItems, task);
22 | if (matchOtherProcessorProcessor == null || matchOtherProcessorProcessor != RequestMatcher.MatchOther.YES) {
23 | return;
24 | }
25 | }
26 | }
27 | }
28 |
29 | public CompositePipeline addSubPipeline(SubPipeline subPipeline) {
30 | this.subPipelines.add(subPipeline);
31 | return this;
32 | }
33 |
34 | public CompositePipeline setSubPipeline(SubPipeline... subPipelines) {
35 | this.subPipelines = new ArrayList();
36 | for (SubPipeline subPipeline : subPipelines) {
37 | this.subPipelines.add(subPipeline);
38 | }
39 | return this;
40 | }
41 |
42 | }
43 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/model/FieldExtractor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.model;
2 |
3 | import us.codecraft.webmagic.model.formatter.ObjectFormatter;
4 | import us.codecraft.webmagic.selector.Selector;
5 |
6 | import java.lang.reflect.Field;
7 | import java.lang.reflect.Method;
8 |
9 | /**
10 | * Wrapper of field and extractor.
11 | * @author code4crafter@gmail.com
12 | * @since 0.2.0
13 | */
14 | class FieldExtractor extends Extractor {
15 |
16 | private final Field field;
17 |
18 | private Method setterMethod;
19 |
20 | private ObjectFormatter objectFormatter;
21 |
22 | public FieldExtractor(Field field, Selector selector, Source source, boolean notNull, boolean multi) {
23 | super(selector, source, notNull, multi);
24 | this.field = field;
25 | }
26 |
27 | Field getField() {
28 | return field;
29 | }
30 |
31 | Selector getSelector() {
32 | return selector;
33 | }
34 |
35 | Source getSource() {
36 | return source;
37 | }
38 |
39 | void setSetterMethod(Method setterMethod) {
40 | this.setterMethod = setterMethod;
41 | }
42 |
43 | Method getSetterMethod() {
44 | return setterMethod;
45 | }
46 |
47 | boolean isNotNull() {
48 | return notNull;
49 | }
50 |
51 | ObjectFormatter getObjectFormatter() {
52 | return objectFormatter;
53 | }
54 |
55 | void setObjectFormatter(ObjectFormatter objectFormatter) {
56 | this.objectFormatter = objectFormatter;
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/processor/example/GithubRepoPageProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.processor.example;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.Spider;
6 | import us.codecraft.webmagic.processor.PageProcessor;
7 |
8 | /**
9 | * @author code4crafter@gmail.com
10 | * @since 0.3.2
11 | */
12 | public class GithubRepoPageProcessor implements PageProcessor {
13 |
14 | private Site site = Site.me().setRetryTimes(3).setSleepTime(0);
15 |
16 | @Override
17 | public void process(Page page) {
18 | page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/[\\w\\-]+/[\\w\\-])").all());
19 | page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/[\\w\\-])").all());
20 | page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());
21 | page.putField("name", page.getHtml().xpath("//h1[@class='entry-title public']/strong/a/text()").toString());
22 | if (page.getResultItems().get("name")==null){
23 | //skip this page
24 | page.setSkip(true);
25 | }
26 | page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"));
27 | }
28 |
29 | @Override
30 | public Site getSite() {
31 | return site;
32 | }
33 |
34 | public static void main(String[] args) {
35 | Spider.create(new GithubRepoPageProcessor()).addUrl("https://github.com/code4craft").thread(5).run();
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/webmagic-core/src/main/java/us/codecraft/webmagic/selector/XpathSelector.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.selector;
2 |
3 | import org.apache.commons.collections.CollectionUtils;
4 | import org.jsoup.nodes.Element;
5 | import us.codecraft.xsoup.XPathEvaluator;
6 | import us.codecraft.xsoup.Xsoup;
7 |
8 | import java.util.List;
9 |
10 | /**
11 | * XPath selector based on Xsoup.
12 | *
13 | * @author code4crafter@gmail.com
14 | * @since 0.3.0
15 | */
16 | public class XpathSelector extends BaseElementSelector {
17 |
18 | private XPathEvaluator xPathEvaluator;
19 |
20 | public XpathSelector(String xpathStr) {
21 | this.xPathEvaluator = Xsoup.compile(xpathStr);
22 | }
23 |
24 | @Override
25 | public String select(Element element) {
26 | return xPathEvaluator.evaluate(element).get();
27 | }
28 |
29 | @Override
30 | public List selectList(Element element) {
31 | return xPathEvaluator.evaluate(element).list();
32 | }
33 |
34 | @Override
35 | public Element selectElement(Element element) {
36 | List elements = selectElements(element);
37 | if (CollectionUtils.isNotEmpty(elements)){
38 | return elements.get(0);
39 | }
40 | return null;
41 | }
42 |
43 | @Override
44 | public List selectElements(Element element) {
45 | return xPathEvaluator.evaluate(element).getElements();
46 | }
47 |
48 | @Override
49 | public boolean hasAttribute() {
50 | return xPathEvaluator.hasAttribute();
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/webmagic-extension/src/test/java/us/codecraft/webmagic/processor/GithubRepoProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.processor;
2 |
3 | import junit.framework.Assert;
4 | import org.junit.Test;
5 | import us.codecraft.webmagic.*;
6 | import us.codecraft.webmagic.downloader.MockGithubDownloader;
7 | import us.codecraft.webmagic.model.OOSpider;
8 | import us.codecraft.webmagic.pipeline.Pipeline;
9 |
10 | /**
11 | * @author code4crafter@gmail.com
12 | */
13 | public class GithubRepoProcessor implements PageProcessor {
14 | @Override
15 | public void process(Page page) {
16 | page.putField("star",page.getHtml().xpath("//ul[@class='pagehead-actions']/li[2]//a[@class='social-count js-social-count']/text()").toString());
17 | page.putField("fork",page.getHtml().xpath("//ul[@class='pagehead-actions']/li[3]//a[@class='social-count']/text()").toString());
18 | }
19 |
20 | @Override
21 | public Site getSite() {
22 | return Site.me().addStartUrl("https://github.com/code4craft/webmagic");
23 | }
24 |
25 | @Test
26 | public void test() {
27 | OOSpider.create(new GithubRepoProcessor()).addPipeline(new Pipeline() {
28 | @Override
29 | public void process(ResultItems resultItems, Task task) {
30 | Assert.assertEquals("78",((String)resultItems.get("star")).trim());
31 | Assert.assertEquals("65",((String)resultItems.get("fork")).trim());
32 | }
33 | }).setDownloader(new MockGithubDownloader()).test("https://github.com/code4craft/webmagic");
34 | }
35 |
36 | }
37 |
--------------------------------------------------------------------------------
/webmagic-extension/src/main/java/us/codecraft/webmagic/example/AppStore.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.example;
2 |
3 | import us.codecraft.webmagic.Site;
4 | import us.codecraft.webmagic.model.OOSpider;
5 | import us.codecraft.webmagic.model.annotation.ExtractBy;
6 | import us.codecraft.webmagic.utils.Experimental;
7 |
8 | import java.util.List;
9 |
10 | /**
11 | * @author code4crafter@gmail.com
12 | * @since 0.4.1
13 | */
14 | @Experimental
15 | public class AppStore {
16 |
17 | @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..trackName")
18 | private String trackName;
19 |
20 | @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..description")
21 | private String description;
22 |
23 | @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..userRatingCount")
24 | private int userRatingCount;
25 |
26 | @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..screenshotUrls")
27 | private List screenshotUrls;
28 |
29 | @ExtractBy(type = ExtractBy.Type.JsonPath, value = "$..supportedDevices")
30 | private List supportedDevices;
31 |
32 | public static void main(String[] args) {
33 | AppStore appStore = OOSpider.create(Site.me(), AppStore.class).get("http://itunes.apple.com/lookup?id=653350791&country=cn&entity=software");
34 | System.out.println(appStore.trackName);
35 | System.out.println(appStore.description);
36 | System.out.println(appStore.userRatingCount);
37 | System.out.println(appStore.screenshotUrls);
38 | System.out.println(appStore.supportedDevices);
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/webmagic-samples/src/main/java/us/codecraft/webmagic/samples/QzoneBlogProcessor.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.samples;
2 |
3 | import us.codecraft.webmagic.Page;
4 | import us.codecraft.webmagic.Site;
5 | import us.codecraft.webmagic.processor.PageProcessor;
6 |
7 | import java.util.List;
8 |
9 | /**
10 | * @author code4crafter@gmail.com
11 | */
12 | public class QzoneBlogProcessor implements PageProcessor {
13 | @Override
14 | public void process(Page page) {
15 | //http://progressdaily.diandian.com/post/2013-01-24/40046867275
16 |
17 | //http://b1.cnc.qzone.qq.com/cgi-bin/blognew/get_abs?hostUin=233017404&uin=233017404&blogType=0&statYear=2013&source=0&statYear=2013&g_tk=291639571&g_tk=291639571&reqInfo=7&pos=0&num=15&source=0&rand=0.46480297949165106
18 | // &cateName=&cateHex=&statYear=2013&reqInfo=7&pos=0&num=15&sortType=0&source=0&rand=0.46480297949165106&g_tk=291639571&verbose=1&ref=qzone
19 | List requests = page.getHtml().regex("]*href=[\"']{1}(http://17dujingdian\\.com/post/[^#]*?)[\"']{1}").all();
20 | page.addTargetRequests(requests);
21 | page.putField("title",page.getHtml().xpath("//div[@id='content']//h2/a"));
22 | page.putField("content",page.getHtml().smartContent());
23 | }
24 |
25 | @Override
26 | public Site getSite() {
27 | return Site.me().setDomain("www.diandian.com").addStartUrl("http://17dujingdian.com/").
28 | setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/main/resources/config/spring/applicationContext-myBatis.xml:
--------------------------------------------------------------------------------
1 |
2 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 | sqlserver
15 | db2
16 | oracle
17 | mysql
18 | h2
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/main/resources/config/spring/applicationContext-webmvc.xml:
--------------------------------------------------------------------------------
1 |
2 |
12 |
13 |
14 |
15 |
16 |
18 |
19 |
20 | text/html;charset=UTF-8
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/webmagic-avalon/webmagic-avalon-common/src/main/java/us/codecraft/webmagic/service/impl/DynamicClassServiceImpl.java:
--------------------------------------------------------------------------------
1 | package us.codecraft.webmagic.service.impl;
2 |
3 | import org.codehaus.groovy.control.CompilationFailedException;
4 | import org.springframework.beans.factory.annotation.Autowired;
5 | import org.springframework.stereotype.Service;
6 | import us.codecraft.forger.Forger;
7 | import us.codecraft.forger.ForgerFactory;
8 | import us.codecraft.webmagic.dao.DynamicClassDao;
9 | import us.codecraft.webmagic.exception.DynamicClassCompileException;
10 | import us.codecraft.webmagic.model.DynamicClass;
11 | import us.codecraft.webmagic.service.DynamicClassService;
12 |
13 | /**
14 | * @author code4crafter@gmail.com
15 | */
16 | @Service
17 | public class DynamicClassServiceImpl implements DynamicClassService {
18 |
19 | @Autowired
20 | private DynamicClassDao dynamicClassDao;
21 |
22 | @Autowired
23 | private ForgerFactory forgerFactory;
24 |
25 | @Override
26 | public Class compileAndSave(String sourceCode) throws DynamicClassCompileException {
27 | Forger