index writer plugins send documents to index back-ends (Solr, etc.).
9 |
10 |
11 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/metadata/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | A Multi-valued Metadata container, and set
4 | of constant fields for Nutch Metadata.
5 |
6 |
7 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/net/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Web-related interfaces: URL {@link org.apache.nutch.net.URLFilter filters}
20 | * and {@link org.apache.nutch.net.URLNormalizer normalizers}.
21 | */
22 | package org.apache.nutch.net;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/net/protocols/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Helper classes related to the {@link org.apache.nutch.protocol.Protocol Protocol}
20 | * interface, sea also {@link org.apache.nutch.protocol}.
21 | */
22 | package org.apache.nutch.net.protocols;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/parse/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * The {@link org.apache.nutch.parse.Parse Parse} interface and related classes.
20 | */
21 | package org.apache.nutch.parse;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/protocol/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Classes related to the {@link org.apache.nutch.protocol.Protocol Protocol} interface,
20 | * see also {@link org.apache.nutch.net.protocols}.
21 | */
22 | package org.apache.nutch.protocol;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/scoring/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * The {@link org.apache.nutch.scoring.ScoringFilter ScoringFilter} interface.
20 | */
21 | package org.apache.nutch.scoring;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/scoring/webgraph/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Scoring implementation based on link analysis
20 | * ({@link org.apache.nutch.scoring.webgraph.LinkRank}),
21 | * see {@link org.apache.nutch.scoring.webgraph.WebGraph}.
22 | */
23 | package org.apache.nutch.scoring.webgraph;
24 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/segment/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * A segment stores all data from on generate/fetch/update cycle:
20 | * fetch list, protocol status, raw content, parsed content, and extracted outgoing links.
21 | */
22 | package org.apache.nutch.segment;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/tools/arc/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Tools to read the
20 | * Arc file format.
21 | */
22 | package org.apache.nutch.tools.arc;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/tools/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Miscellaneous tools.
20 | */
21 | package org.apache.nutch.tools;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/util/NutchJob.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.nutch.util;
19 |
20 | import org.apache.hadoop.conf.Configuration;
21 | import org.apache.hadoop.mapred.JobConf;
22 |
23 | /** A {@link JobConf} for Nutch jobs. */
24 | public class NutchJob extends JobConf {
25 |
26 | public NutchJob(Configuration conf) {
27 | super(conf, NutchJob.class);
28 | }
29 |
30 | }
31 |
32 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/util/domain/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Classes for domain name analysis.
4 |
5 | for information please refer to following urls :
6 |
12 |
13 |
14 |
15 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/org/apache/nutch/util/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Miscellaneous utility classes.
20 | */
21 | package org.apache.nutch.util;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/java/overview.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Apache Nutch
4 |
5 |
6 | Apache Nutch is a highly extensible and scalable open source web crawler software project.
7 | Nutch is a project of the Apache Software Foundation and is part of the larger Apache community of developers and users.
8 |
9 |
10 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/creativecommons/README.txt:
--------------------------------------------------------------------------------
1 | Support for crawling and searching Creative-Commons licensed content.
2 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/creativecommons/conf/crawl-urlfilter.txt:
--------------------------------------------------------------------------------
1 | # Creative Commnons crawl filter
2 |
3 | # Each non-comment, non-blank line contains a regular expression
4 | # prefixed by '+' or '-'. The first matching pattern in the file
5 | # determines whether a URL is included or ignored. If no pattern
6 | # matches, the URL is ignored.
7 |
8 | # skip file:, ftp:, & mailto: urls
9 | -^(file|ftp|mailto|https):
10 |
11 | # skip image and other suffixes we can't yet parse
12 | -\.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|rtf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|mp3|rss|xml|doc|pdf|txt|DOC|PDF|TXT)$
13 |
14 | # skip URLs containing certain characters as probable queries, etc.
15 | -[?*!@=]
16 |
17 | # accept anything else
18 | +.
19 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/creativecommons/data/anchor.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 | 
This file is licensed under a
7 | Creative Commons License.
8 |
9 |
10 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/creativecommons/data/rel.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | CC by-nc
5 |
6 |
7 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/creativecommons/src/java/org/creativecommons/nutch/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Sample plugins that parse and index Creative Commons medadata.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/feed/src/java/org/apache/nutch/indexer/feed/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Indexing filter to index meta data from RSS feeds.
20 | */
21 | package org.apache.nutch.indexer.feed;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/feed/src/java/org/apache/nutch/parse/feed/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Parse RSS feeds.
20 | */
21 | package org.apache.nutch.parse.feed;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/headings/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/headings/src/java/org/apache/nutch/parse/headings/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Parse filter to extract headings (h1, h2, etc.) from DOM parse tree.
20 | */
21 | package org.apache.nutch.parse.headings;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-anchor/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-anchor/src/java/org/apache/nutch/indexer/anchor/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | An indexing plugin for inbound anchor text.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-basic/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | A basic indexing plugin, adds basic fields: url, host, title, content, etc.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-metadata/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-metadata/src/java/org/apache/nutch/indexer/metadata/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Indexing filter to add document metadata to the index.
20 | * Metadata may come from CrawlDb, parse or content metadata.
21 | */
22 | package org.apache.nutch.indexer.metadata;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-more/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | A more indexing plugin, adds "more" index fields:
4 | last modified date, MIME type, content length.
5 |
6 |
7 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-static/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/index-static/src/java/org/apache/nutch/indexer/staticfield/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | A simple plugin called at indexing that adds fields with static data. You can specify a list of fieldname:fieldcontent per nutch job. It can be useful when collections can't be created by urlpatterns, like in subcollection, but on a job-basis.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/indexer-dummy/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/indexer-dummy/src/java/org/apache/nutch/indexwriter/dummy/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Index writer plugin for debugging, writes pairs of <action, url> to a
20 | * text file, action is one of "add", "update", or "delete".
21 | */
22 | package org.apache.nutch.indexwriter.dummy;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/indexer-elastic/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/indexer-elastic/src/java/org/apache/nutch/indexwriter/elastic/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Index writer plugin for Elasticsearch.
20 | */
21 | package org.apache.nutch.indexwriter.elastic;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/indexer-solr/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/indexer-solr/src/java/org/apache/nutch/indexwriter/solr/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Index writer plugin for Apache Solr.
20 | */
21 | package org.apache.nutch.indexwriter.solr;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/language-identifier/src/java/org/apache/nutch/analysis/lang/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Text document language identifier.
Language profiles are based on material from
4 | http://www.homepages.inf.ed.ac.uk/pkoehn/publications/europarl.ps/.
5 |
6 |
7 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/language-identifier/src/test/org/apache/nutch/analysis/lang/test-referencial.txt:
--------------------------------------------------------------------------------
1 | da.test;da
2 | de.test;de
3 | el.test;el
4 | en.test;en
5 | es.test;es
6 | fi.test;fi
7 | fr.test;fr
8 | it.test;it
9 | nl.test;nl
10 | pt.test;pt
11 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/lib-http/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/lib-http/plugin.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
21 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/BlockedException.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.nutch.protocol.http.api;
19 |
20 | public class BlockedException extends HttpException {
21 |
22 | public BlockedException(String msg) {
23 | super(msg);
24 | }
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Common API used by HTTP plugins ({@link org.apache.nutch.protocol.http http},
4 | {@link org.apache.nutch.protocol.httpclient httpclient})
5 |
6 |
7 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/lib-nekohtml/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
26 |
27 |
28 |
29 |
30 |
31 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/lib-regex-filter/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Generic {@link org.apache.nutch.net.URLFilter URL filter} library,
20 | * abstracting away from regular expression implementations.
21 | */
22 | package org.apache.nutch.urlfilter.api;
23 |
24 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/microformats-reltag/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | A microformats Rel-Tag
5 | Parser/Indexer/Querier plugin.
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/nutch-extensionpoints/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
26 |
27 |
28 |
29 |
30 |
31 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-ext/command:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | #
3 | # Sample bash script as external command invoked by parse-ext plugin
4 | #
5 | # 20040701, John Xing
6 |
7 | set -e
8 |
9 | if [ $# -ne 1 ]; then
10 | echo Usage:$0 mimeType >&2
11 | exit 1
12 | fi
13 |
14 | case $1 in
15 | "application/vnd.nutch.example.cat")
16 | cat
17 | ;;
18 | "application/vnd.nutch.example.md5sum")
19 | md5sum
20 | ;;
21 | *)
22 | echo "Can't parse mimeType $1" >&2
23 | exit 1
24 | esac
25 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Parse wrapper to run external command to do the parsing.
20 | */
21 | package org.apache.nutch.parse.ext;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | An HTML document parsing plugin.
This package relies on NekoHTML.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-js/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Parser and parse filter plugin to extract all (possible) links
20 | * from JavaScript files and embedded JavaScript code snippets.
21 | */
22 | package org.apache.nutch.parse.js;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-metatags/README.txt:
--------------------------------------------------------------------------------
1 | Parse-metatags plugin
2 |
3 | The parse-metatags plugin consists of a HTMLParserFilter which takes as parameter a list of metatag names with '*' as default value. The values are separated by ';'.
4 | In order to extract the values of the metatags description and keywords, you must specify in nutch-site.xml
5 |
6 |
7 | metatags.names
8 | description;keywords
9 |
10 |
11 | Prefixes the names with 'metatag.' in the parse-metadata. For instance to index description and keywords, you need to activate the plugin index-metadata and set the value of the parameter 'index.parse.md' to 'metatag.description;metatag.keywords'.
12 |
13 | This code has been developed by DigitalPebble Ltd and offered to the community by ANT.com
14 |
15 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-metatags/plugin.xml:
--------------------------------------------------------------------------------
1 |
2 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
17 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-metatags/sample/testMetatags.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 | text of the document
8 |
9 |
10 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-metatags/sample/testMultivalueMetatags.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 | A test for multi-valued metatags.
12 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-metatags/src/java/org/apache/nutch/parse/metatags/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Parse filter to extract meta tags: keywords, description, etc.
20 | * Used in combination with index-metadata plugin
21 | * (see {@link org.apache.nutch.indexer.metadata}).
22 | */
23 | package org.apache.nutch.parse.metatags;
24 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/lib/javaswf.jar:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/lib/javaswf.jar
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test1.swf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test1.swf
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test1.txt:
--------------------------------------------------------------------------------
1 |
2 | --------
3 | /go/gnav_cart
4 | /go/gnav_company
5 | /go/gnav_devnet
6 | /go/gnav_downloads
7 | /go/gnav_fl_minmessage
8 | /go/gnav_help
9 | /go/gnav_mm_home
10 | /go/gnav_products
11 | /go/gnav_search?loc=en_us
12 | /go/gnav_showcase
13 | /go/gnav_solutions
14 | /go/gnav_store
15 | /go/gnav_support
16 | /go/gnav_your_account
17 | Acquisition Info
18 | Adobe Home
19 | AppleGothic
20 | Array
21 | Company
22 | Developers
23 | Downloads
24 | Help
25 | Home
26 | International
27 | LocaleManager
28 | Macromedia Flash Player
29 | Macromedia Home
30 | MovieClip
31 | Products
32 | Showcase
33 | Solutions
34 | Store
35 | String
36 | Support
37 | TextFormat
38 | To ensure the best possible Internet Experience, please download the latest version of the free
39 | Verdana
40 | _sans
41 | active
42 | bluePill
43 | button
44 | color
45 | company
46 | devnet
47 | downloads
48 | en_us
49 | home
50 | javascript:openCrosslinkWindow('/go/adobeacquisition')
51 | javascript:openCrosslinkWindow('/go/gnav_adobe_home')
52 | products
53 | rollOut
54 | rollOver
55 | selected
56 | showcase
57 | solutions
58 | support
59 | tabHolder
60 | textColor
61 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test2.swf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test2.swf
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test2.txt:
--------------------------------------------------------------------------------
1 | Impact Impact Impact Arial Arial Arial Webdings Webdings Webdings Verdana Verdana Verdana CourierNew CourierNew CourierNew Bimini Bimini Bimini
2 | --------
3 | TextFormat
4 | color
5 | font
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test3.swf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test3.swf
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/sample/test3.txt:
--------------------------------------------------------------------------------
1 | Mix.
2 | Edit.
3 | Master.
4 | Compose.
5 | Animate.
6 | With a single suite of powerful tools
7 | that work together as one.
8 | World-class video and audio tools that bring
9 | new power and efficiency to your film, video,
10 | DVD, and web workflows.
11 | Learn more.
12 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Parse Flash SWF files.
20 | */
21 | package org.apache.nutch.parse.swf;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/howto_upgrade_tika.txt:
--------------------------------------------------------------------------------
1 | 1. Upgrade Tika depencency in trunk/ivy/ivy.xml
2 |
3 | 2. Upgrade Tika dependency in src/plugin/parse-tika/ivy.xml
4 |
5 | 3. Upgrade Tika's own dependencies in src/plugin/parse-tika/plugin.xml
6 | To get the list of dependencies and their versions execute:
7 | $ ant -f ./build-ivy.xml
8 | $ ls lib/
9 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/encrypted.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/encrypted.pdf
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/nutch_logo_tm.gif:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/nutch_logo_tm.gif
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/ootest.odt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/ootest.odt
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/ootest.sxw:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/ootest.sxw
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/ootest.txt:
--------------------------------------------------------------------------------
1 | Abcedfg ?????
2 | Abcdefg
3 | Abcdefg
4 | abcdefg
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 | http://www.openoffice.org
16 |
17 | Title
18 | Col1
19 | Col2
20 | Col3
21 | head
22 | Cell1
23 | Cell2
24 | Cel3
25 | total
26 | TOTAL
27 | TOTAL
28 | TOTAL
29 |
30 | Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Integer a leo in lacus malesuada ornare. Mauris sagittis. Nam vestibulum. Nunc gravida vestibulum augue. Praesent sed lectus quis lectus adipiscing bibendum. Sed nulla. Duis posuere justo eget urna. Proin lorem orci, vestibulum ut, consequat molestie, eleifend a, nibh. Mauris sed lacus. Etiam blandit tincidunt neque. Cras ac sapien. Duis erat.
31 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/pdftest.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/pdftest.pdf
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/word97.doc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/sample/word97.doc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Parse various document formats with help of
20 | * Apache Tika.
21 | */
22 | package org.apache.nutch.parse.tika;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-zip/sample/test.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/plugin/parse-zip/sample/test.zip
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Parse ZIP files: embedded files are recursively passed to appropriate parsers.
20 | */
21 | package org.apache.nutch.parse.zip;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-file/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-file/sample/testprotocolfile.txt:
--------------------------------------------------------------------------------
1 | Protocol File Test
2 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-file/sample/testprotocolfile_(encoded).txt:
--------------------------------------------------------------------------------
1 | Protocol File Test
2 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Protocol plugin which supports retrieving local file resources.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-ftp/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpExceptionBadSystResponse.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.nutch.protocol.ftp;
19 |
20 | /**
21 | * Exception indicating bad reply of SYST command.
22 | *
23 | * @author John Xing
24 | */
25 | public class FtpExceptionBadSystResponse extends FtpException {
26 | FtpExceptionBadSystResponse(String msg) {
27 | super(msg);
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Protocol plugin which supports retrieving documents via the ftp protocol.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Protocol plugin which supports retrieving documents via the http protocol.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Protocol plugin which supports retrieving documents via the HTTP and
4 | HTTPS protocols, optionally with Basic, Digest and NTLM authentication
5 | schemes for web server as well as proxy server. It handles cookies
6 | within a single fetch operation. This plugin is based on Jakarta
7 | Commons HttpClient library.
8 |
9 |
10 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/scoring-depth/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/scoring-depth/plugin.xml:
--------------------------------------------------------------------------------
1 |
2 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
21 |
23 |
24 |
25 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/scoring-depth/src/java/org/apache/nutch/scoring/depth/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Scoring filter to stop crawling at a configurable depth
20 | * (number of "hops" from seed URLs).
21 | */
22 | package org.apache.nutch.scoring.depth;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/scoring-link/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/scoring-link/src/java/org/apache/nutch/scoring/link/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Scoring filter used in conjunction with
20 | * {@link org.apache.nutch.scoring.webgraph.WebGraph}.
21 | */
22 | package org.apache.nutch.scoring.link;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/scoring-opic/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Scoring filter implementing a variant of the Online Page Importance Computation
20 | * (OPIC) algorithm.
21 | */
22 | package org.apache.nutch.scoring.opic;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/subcollection/README.txt:
--------------------------------------------------------------------------------
1 | For brief description about this plugin see
2 | src/java/org/apache/nutch/collection/package.html
3 |
4 | Basically:
5 | You need to enable this during indexing and during searching
6 |
7 | After indexing you can limit your searches to certain
8 | subcollection with keyword subcollection, eg.
9 |
10 | "subcollection:nutch hadoop"
11 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/subcollection/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/tld/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/tld/src/java/org/apache/nutch/indexer/tld/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Top Level Domain Indexing plugin.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/tld/src/java/org/apache/nutch/scoring/tld/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Top Level Domain Scoring plugin.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-automaton/sample/Benchmarks.rules:
--------------------------------------------------------------------------------
1 | # The url filter file used by the crawl command.
2 |
3 | # Better for intranet crawling.
4 | # Be sure to change MY.DOMAIN.NAME to your domain name.
5 |
6 | # Each non-comment, non-blank line contains a regular expression
7 | # prefixed by '+' or '-'. The first matching pattern in the file
8 | # determines whether a URL is included or ignored. If no pattern
9 | # matches, the URL is ignored.
10 |
11 | # skip file:, ftp:, & mailto: urls
12 | -(file|ftp|mailto):.*
13 |
14 | # skip image and other suffixes we can't yet parse
15 | -.*\.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|png)
16 |
17 | # skip URLs containing certain characters as probable queries, etc.
18 | -.*[?*!@=].*
19 |
20 | # skip .fr .org and .net domains
21 | -.*//.*\.fr/.*
22 | -.*//.*\.org/.*
23 | -.*//.*\.net/.*
24 |
25 | # skip everything else
26 | +.*
27 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-automaton/sample/IntranetCrawling.rules:
--------------------------------------------------------------------------------
1 | # The url filter file used by the crawl command.
2 |
3 | # Better for intranet crawling.
4 | # Be sure to change MY.DOMAIN.NAME to your domain name.
5 |
6 | # Each non-comment, non-blank line contains a regular expression
7 | # prefixed by '+' or '-'. The first matching pattern in the file
8 | # determines whether a URL is included or ignored. If no pattern
9 | # matches, the URL is ignored.
10 |
11 | # skip file:, ftp:, & mailto: urls
12 | -(file|ftp|mailto):.*
13 |
14 | # skip image and other suffixes we can't yet parse
15 | -.*\.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|png)
16 |
17 | # skip URLs containing certain characters as probable queries, etc.
18 | -.*[?*!@=].*
19 |
20 | # accept hosts in MY.DOMAIN.NAME
21 | +http://([a-z0-9]*\.)*MY.DOMAIN.NAME/.*
22 |
23 | # skip everything else
24 | -.*
25 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-automaton/sample/IntranetCrawling.urls:
--------------------------------------------------------------------------------
1 | -file://home/jc/nutch/index.html
2 | -ftp://ftp.apache.org/nutch.html
3 | -mailto:jerome.charron@gmail.com
4 | -news://any.news.server/comp.lang.java
5 | -whois:/nutch.org
6 | +http://MY.DOMAIN.NAME/
7 | +http://MY.DOMAIN.NAME/nutch
8 | +http://www.MY.DOMAIN.NAME/
9 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-automaton/sample/WholeWebCrawling.rules:
--------------------------------------------------------------------------------
1 | # The default url filter.
2 | # Better for whole-internet crawling.
3 |
4 | # Each non-comment, non-blank line contains a regular expression
5 | # prefixed by '+' or '-'. The first matching pattern in the file
6 | # determines whether a URL is included or ignored. If no pattern
7 | # matches, the URL is ignored.
8 |
9 | # skip file: ftp: and mailto: urls
10 | -(file|ftp|mailto):.*
11 |
12 | # skip image and other suffixes we can't yet parse
13 | -.*\.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe)
14 |
15 | # skip URLs containing certain characters as probable queries, etc.
16 | -.*[?*!@=].*
17 |
18 | # accept anything else
19 | +.*
20 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-automaton/sample/WholeWebCrawling.urls:
--------------------------------------------------------------------------------
1 | -file://home/jc/nutch/index.html
2 | -ftp://ftp.apache.org/nutch.html
3 | -mailto:jerome.charron@gmail.com
4 | +news://any.news.server/comp.lang.java
5 | +whois:/nutch.org
6 | -http://www.nutch.org/nutch.gif
7 | -http://www.nutch.org/nutch.eps
8 | -http://www.nutch.org/nutch?q=nutch
9 | +http://www.nutch.org/
10 | +http://www.nutch.org/abcd/foo/bar/foo/bar/foo/
11 | +http://www.nutch.org/abcd/foo/bar/xyz/foo/bar/foo/
12 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-automaton/src/java/org/apache/nutch/urlfilter/automaton/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | URL filter plugin based on
5 | dk.brics.automaton Finite-State
6 | Automata for JavaTM.
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-domain/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-domain/data/hosts.txt:
--------------------------------------------------------------------------------
1 | # comments start with the pound sign
2 | net
3 | apache.org
4 | be
5 | www.yahoo.com
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-domainblacklist/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-domainblacklist/data/hosts.txt:
--------------------------------------------------------------------------------
1 | # comments start with the pound sign
2 | net
3 | apache.org
4 | be
5 | www.yahoo.com
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-domainblacklist/src/java/org/apache/nutch/urlfilter/domainblacklist/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * URL filter plugin to exclude URLs by domain suffixes, domain names, and/or host names.
20 | * See {@link org.apache.nutch.urlfilter.domain} for the counterpart (include only URLs
21 | * matching host or domain).
22 | */
23 | package org.apache.nutch.urlfilter.domainblacklist;
24 |
25 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-prefix/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | URL filter plugin to include only URLs which match one of a given list of URL prefixes.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-regex/sample/Benchmarks.rules:
--------------------------------------------------------------------------------
1 | # The url filter file used by the crawl command.
2 |
3 | # Better for intranet crawling.
4 | # Be sure to change MY.DOMAIN.NAME to your domain name.
5 |
6 | # Each non-comment, non-blank line contains a regular expression
7 | # prefixed by '+' or '-'. The first matching pattern in the file
8 | # determines whether a URL is included or ignored. If no pattern
9 | # matches, the URL is ignored.
10 |
11 | # skip file:, ftp:, & mailto: urls
12 | -^(file|ftp|mailto):
13 |
14 | # skip image and other suffixes we can't yet parse
15 | -\.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|png)$
16 |
17 | # skip URLs containing certain characters as probable queries, etc.
18 | -[?*!@=]
19 |
20 | # skip .fr .org and .net domains
21 | -^.*//.*\.fr/
22 | -^.*//.*\.org/
23 | -^.*//.*\.net/
24 |
25 | # skip everything else
26 | +.
27 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-regex/sample/IntranetCrawling.rules:
--------------------------------------------------------------------------------
1 | # The url filter file used by the crawl command.
2 |
3 | # Better for intranet crawling.
4 | # Be sure to change MY.DOMAIN.NAME to your domain name.
5 |
6 | # Each non-comment, non-blank line contains a regular expression
7 | # prefixed by '+' or '-'. The first matching pattern in the file
8 | # determines whether a URL is included or ignored. If no pattern
9 | # matches, the URL is ignored.
10 |
11 | # skip file:, ftp:, & mailto: urls
12 | -^(file|ftp|mailto):
13 |
14 | # skip image and other suffixes we can't yet parse
15 | -\.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe|png)$
16 |
17 | # skip URLs containing certain characters as probable queries, etc.
18 | -[?*!@=]
19 |
20 | # skip URLs with slash-delimited segment that repeats 3+ times, to break loops
21 | -.*(/.+?)/.*?\1/.*?\1/
22 |
23 | # accept hosts in MY.DOMAIN.NAME
24 | +^http://([a-z0-9]*\.)*MY.DOMAIN.NAME/
25 |
26 | # skip everything else
27 | -.
28 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-regex/sample/IntranetCrawling.urls:
--------------------------------------------------------------------------------
1 | -file://home/jc/nutch/index.html
2 | -ftp://ftp.apache.org/nutch.html
3 | -mailto:jerome.charron@gmail.com
4 | -news://any.news.server/comp.lang.java
5 | -whois:/nutch.org
6 | +http://MY.DOMAIN.NAME/
7 | +http://MY.DOMAIN.NAME/nutch
8 | +http://www.MY.DOMAIN.NAME/
9 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-regex/sample/WholeWebCrawling.rules:
--------------------------------------------------------------------------------
1 | # The default url filter.
2 | # Better for whole-internet crawling.
3 |
4 | # Each non-comment, non-blank line contains a regular expression
5 | # prefixed by '+' or '-'. The first matching pattern in the file
6 | # determines whether a URL is included or ignored. If no pattern
7 | # matches, the URL is ignored.
8 |
9 | # skip file: ftp: and mailto: urls
10 | -^(file|ftp|mailto):
11 |
12 | # skip image and other suffixes we can't yet parse
13 | -\.(gif|GIF|jpg|JPG|ico|ICO|css|sit|eps|wmf|zip|ppt|mpg|xls|gz|rpm|tgz|mov|MOV|exe)$
14 |
15 | # skip URLs containing certain characters as probable queries, etc.
16 | -[?*!@=]
17 |
18 | # skip URLs with slash-delimited segment that repeats 3+ times, to break loops
19 | -.*(/.+?)/.*?\1/.*?\1/
20 |
21 | # accept anything else
22 | +.
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-regex/sample/WholeWebCrawling.urls:
--------------------------------------------------------------------------------
1 | -file://home/jc/nutch/index.html
2 | -ftp://ftp.apache.org/nutch.html
3 | -mailto:jerome.charron@gmail.com
4 | +news://any.news.server/comp.lang.java
5 | +whois:/nutch.org
6 | -http://www.nutch.org/nutch.gif
7 | -http://www.nutch.org/nutch.eps
8 | -http://www.nutch.org/nutch?q=nutch
9 | +http://www.nutch.org/
10 | -http://www.nutch.org/abcd/foo/bar/foo/bar/foo/
11 | -http://www.nutch.org/abcd/foo/bar/xyz/foo/bar/foo/
12 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-regex/src/java/org/apache/nutch/urlfilter/regex/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | URL filter plugin to include and/or exclude URLs matching Java regular expressions.
4 |
5 |
6 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-suffix/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * URL filter plugin to either exclude or include only URLs which match
20 | * one of the given (path) suffixes.
21 | */
22 | package org.apache.nutch.urlfilter.suffix;
23 |
24 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-validator/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlfilter-validator/src/java/org/apache/nutch/urlfilter/validator/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | URL filter plugin that validates given urls.
4 | This plugin runs a series of tests for the given url to make sure that given
5 | url is valid and 'fetchable'.
6 | Note: This plugin should only be used for web-related protocols such
7 | as http, https and ftp.
8 |
9 |
10 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlmeta/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlmeta/src/java/org/apache/nutch/indexer/urlmeta/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | URL Meta Tag Indexing Plugin
5 |
6 |
7 | Takes Meta Tags, injected alongside a URL (see NUTCH-655) and specified in the "urlmeta.tags" property,
8 | and inserts them into the document--which is then sent to the Indexer. If you specify these fields in
9 | the Nutch's schema (as well as the Indexer's), you can reasonably assume that they will be indexed.
10 |
11 |
12 |
13 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlmeta/src/java/org/apache/nutch/scoring/urlmeta/package.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | URL Meta Tag Scoring Plugin
5 |
6 |
7 | Propagates Meta Tags, injected alongside a URL (see NUTCH-655) and specified in the "urlmeta.tags" property,
8 | along to their outlinks. This does not actually perform scoring.
9 |
10 |
11 |
12 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-basic/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-basic/src/java/org/apache/nutch/net/urlnormalizer/basic/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * URL normalizer performing basic normalizations: remove default ports
20 | * and dot segments in path.
21 | */
22 | package org.apache.nutch.net.urlnormalizer.basic;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-host/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-host/data/hosts.txt:
--------------------------------------------------------------------------------
1 | # Force all sub domains to www.
2 | *.example.com example.com
3 |
4 | # Force no sub domain to www. URL's
5 | www.example.net example.net
6 |
7 | # Force www. sub domain when hitting link without sub domain
8 | example.org www.example.org
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-host/src/java/org/apache/nutch/net/urlnormalizer/host/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * URL normalizer renaming hosts to a canonical form listed in the
20 | * configuration file.
21 | */
22 | package org.apache.nutch.net.urlnormalizer.host;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-pass/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-pass/src/java/org/apache/nutch/net/urlnormalizer/pass/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * URL normalizer dummy which does not change URLs. Required because at least
20 | * one URL normalizer must be defined in any scope.
21 | */
22 | package org.apache.nutch.net.urlnormalizer.pass;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-querystring/build.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-querystring/src/java/org/apache/nutch/net/urlnormalizer/querystring/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * URL normalizer which sort the elements in the query part to avoid duplicates
20 | * by permutations.
21 | */
22 | package org.apache.nutch.net.urlnormalizer.querystring;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-regex/sample/regex-normalize-scope1.test:
--------------------------------------------------------------------------------
1 | # test removal of subdomains
2 | http://www.foo.bar.com/ http://bar.com/
3 |
4 | # test removal of url path
5 | http://www.foo.bar.com/foo.php?PHPSESSID=cdc993a493e899bed04f4d0c8a462a03 http://bar.com/
6 |
7 | # test removal of urls in arguments
8 | https://www.foo.bar.com/foo.php?url=http://www.example.com/test.php https://bar.com/
9 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-regex/sample/regex-normalize-scope1.xml:
--------------------------------------------------------------------------------
1 |
2 |
6 |
7 |
9 |
10 |
15 |
16 |
17 | (^[a-z]{3,5}://)([\w]+\.)*?(\w+\.\w+)[/$].*
18 | $1$3/
19 |
20 |
21 |
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/plugin/urlnormalizer-regex/src/java/org/apache/nutch/net/urlnormalizer/regex/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * URL normalizer with configurable rules based on regular expressions
20 | * ({@link java.util.regex.Pattern}).
21 | */
22 | package org.apache.nutch.net.urlnormalizer.regex;
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/test/domain-urlfilter.txt:
--------------------------------------------------------------------------------
1 | # Licensed to the Apache Software Foundation (ASF) under one or more
2 | # contributor license agreements. See the NOTICE file distributed with
3 | # this work for additional information regarding copyright ownership.
4 | # The ASF licenses this file to You under the Apache License, Version 2.0
5 | # (the "License"); you may not use this file except in compliance with
6 | # the License. You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 |
16 | # config file for urlfilter-domsin plugin
17 |
18 | com
19 | org
20 | net
21 | edu
22 | gov
23 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/test/filter-all.txt:
--------------------------------------------------------------------------------
1 | # Config file for urlfilter-suffix plugin
2 | # Filter away all urls
3 |
4 | # case-insensitive, disallow unknown suffixes
5 | -I
6 |
7 | # allow these
8 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/test/log4j.properties:
--------------------------------------------------------------------------------
1 | # log4j configuration used during build and unit tests
2 |
3 | log4j.rootLogger=info,stdout
4 | log4j.threshold=ALL
5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender
6 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
7 | log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
8 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/test/nutch-site.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 | plugin.includes
9 | .*
10 | Enable all plugins during unit testing.
11 |
12 |
13 |
14 | distributed.search.test.port
15 | 60000
16 | TCP port used during junit testing.
17 |
18 |
19 |
20 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/test/org/apache/nutch/crawl/DummyWritable.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | package org.apache.nutch.crawl;
19 |
20 | import org.apache.hadoop.io.IntWritable;
21 |
22 | public class DummyWritable extends IntWritable {
23 |
24 | public DummyWritable() {
25 |
26 | }
27 |
28 | public DummyWritable(int i) {
29 | super(i);
30 | }
31 |
32 | }
33 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/test/org/apache/nutch/plugin/ITestExtension.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 | package org.apache.nutch.plugin;
18 | /**
19 | * A Simple Test Extension Interface.
20 | *
21 | * @author joa23
22 | *
23 | */
24 | public interface ITestExtension {
25 | public String testGetExtension(String hello);
26 | }
27 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/test/org/apache/nutch/tools/proxy/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one or more
3 | * contributor license agreements. See the NOTICE file distributed with
4 | * this work for additional information regarding copyright ownership.
5 | * The ASF licenses this file to You under the Apache License, Version 2.0
6 | * (the "License"); you may not use this file except in compliance with
7 | * the License. You may obtain a copy of the License at
8 | *
9 | * http://www.apache.org/licenses/LICENSE-2.0
10 | *
11 | * Unless required by applicable law or agreed to in writing, software
12 | * distributed under the License is distributed on an "AS IS" BASIS,
13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | * See the License for the specific language governing permissions and
15 | * limitations under the License.
16 | */
17 |
18 | /**
19 | * Proxy to {@link org.apache.nutch.tools.Benchmark benchmark} the crawler.
20 | */
21 | package org.apache.nutch.tools.proxy;
22 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/fetch-test-site/dup_of_pagea.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | page a
4 |
5 |
6 | This is page a
7 | home
8 |
9 | Nutch fetcher test page
10 |
11 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/fetch-test-site/exception.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Exception
5 |
6 |
7 |
8 | !!Trying to parse this one will fail with a MalformedInputException!!
9 |
10 | Nutch fetcher test page.
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/fetch-test-site/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | front page
4 |
5 |
6 | This is front page.
7 | Page a
8 | Page b
9 | dup of Page a
10 |
11 | Nutch fetcher test page
12 |
13 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/fetch-test-site/pagea.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | page a
4 |
5 |
6 | This is page a
7 | home
8 |
9 | Nutch fetcher test page
10 |
11 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/fetch-test-site/pageb.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | bage b
4 |
5 |
6 | This is page b
7 | home
8 |
9 | Nutch fetcher test page
10 |
11 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/fetch-test-site/robots.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/fetch-test-site/robots.txt
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/test-mime-util/test.xlsx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/test-mime-util/test.xlsx
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/crawldb/current/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/crawldb/current/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/crawldb/current/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/crawldb/current/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/crawldb/current/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/crawldb/current/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/crawldb/current/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/crawldb/current/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.f0:
--------------------------------------------------------------------------------
1 | xwyvyuxyvxuxxxvxxxy
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.f1:
--------------------------------------------------------------------------------
1 | |||||||||||||||}|||
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.f2:
--------------------------------------------------------------------------------
1 | yxxxxxxyxxxxxxxzxxy
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.f3:
--------------------------------------------------------------------------------
1 | hhhhhhhhhhhhhhhihhh
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.f4:
--------------------------------------------------------------------------------
1 | srrrrrrsrrprrrrrrrr
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.f5:
--------------------------------------------------------------------------------
1 | vwzyzvxtzwvzwyvvzxt
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.fdx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.fdx
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.fnm:
--------------------------------------------------------------------------------
1 | titlesitehostcontenturlanchorsegment digest boost
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.frq:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.frq
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.prx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.prx
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.tii:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.tii
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.tis:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/_0.tis
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/deletable:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/segments:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/index/segments
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/.index.done.crc:
--------------------------------------------------------------------------------
1 | crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/.segments.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/.segments.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.f0:
--------------------------------------------------------------------------------
1 | xwyvyuxyvxuxxxvxxxy
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.f1:
--------------------------------------------------------------------------------
1 | |||||||||||||||}|||
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.f2:
--------------------------------------------------------------------------------
1 | yxxxxxxyxxxxxxxzxxy
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.f3:
--------------------------------------------------------------------------------
1 | hhhhhhhhhhhhhhhihhh
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.f4:
--------------------------------------------------------------------------------
1 | srrrrrrsrrprrrrrrrr
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.f5:
--------------------------------------------------------------------------------
1 | vwzyzvxtzwvzwyvvzxt
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.fdx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.fdx
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.fnm:
--------------------------------------------------------------------------------
1 | titlesitehostcontenturlanchorsegment digest boost
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.frq:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.frq
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.prx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.prx
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.tii:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.tii
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.tis:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/_j.tis
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/commit.lock:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/commit.lock
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/deletable:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/index.done:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/index.done
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/segments:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/segments
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/write.lock:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/indexes/part-00000/write.lock
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/linkdb/current/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/linkdb/current/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/linkdb/current/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/linkdb/current/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/linkdb/current/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/linkdb/current/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/linkdb/current/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/linkdb/current/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/content/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/content/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/content/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/content/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/content/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/content/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/content/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/content/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_fetch/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_fetch/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_fetch/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_fetch/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_fetch/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_fetch/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_fetch/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_fetch/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_generate/.part-00000.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_generate/.part-00000.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_generate/part-00000:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_generate/part-00000
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_parse/.part-00000.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_parse/.part-00000.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_parse/part-00000:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/crawl_parse/part-00000
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_data/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_data/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_data/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_data/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_data/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_data/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_data/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_data/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_text/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_text/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_text/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_text/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_text/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_text/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_text/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213635/parse_text/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/content/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/content/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/content/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/content/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/content/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/content/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/content/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/content/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_fetch/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_fetch/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_fetch/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_fetch/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_fetch/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_fetch/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_fetch/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_fetch/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_generate/.part-00000.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_generate/.part-00000.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_generate/part-00000:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_generate/part-00000
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_parse/.part-00000.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_parse/.part-00000.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_parse/part-00000:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/crawl_parse/part-00000
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_data/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_data/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_data/part-00000/.index.crc:
--------------------------------------------------------------------------------
1 | crc lkF
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_data/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_data/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_data/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_data/part-00000/index
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_text/part-00000/.data.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_text/part-00000/.data.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_text/part-00000/.index.crc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_text/part-00000/.index.crc
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_text/part-00000/data:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_text/part-00000/data
--------------------------------------------------------------------------------
/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_text/part-00000/index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CrawlScript/nutcher/085330f03955ae65ceae37967c5672bcc9a0a0dd/nutch-chinese/apache-nutch-1.9/src/testresources/testcrawl/segments/20060919213643/parse_text/part-00000/index
--------------------------------------------------------------------------------