├── .classpath ├── .gitignore ├── .project ├── .settings ├── org.eclipse.core.resources.prefs ├── org.eclipse.jdt.core.prefs └── org.eclipse.m2e.core.prefs ├── LICENSE ├── README.md ├── dependency-reduced-pom.xml ├── pom.xml └── src ├── main ├── java │ ├── edu │ │ └── cmu │ │ │ └── lemurproject │ │ │ ├── WarcFileInputFormat.java │ │ │ ├── WarcFileRecordReader.java │ │ │ ├── WarcRecord.java │ │ │ └── WritableWarcRecord.java │ └── org │ │ └── wikireverse │ │ └── commoncrawl │ │ ├── FileCountFilter.java │ │ ├── LinkArrayReducer.java │ │ ├── LinkArrayWritable.java │ │ ├── LinkWritable.java │ │ ├── MetadataParser.java │ │ ├── OutputToText.java │ │ ├── OutputToTextMapper.java │ │ ├── Page.java │ │ ├── SegmentCombiner.java │ │ ├── SegmentCombinerMapper.java │ │ ├── WikiArticle.java │ │ ├── WikiMetadata.java │ │ ├── WikiReverse.java │ │ └── WikiReverseMapper.java └── resources │ └── org │ └── wikireverse │ └── commoncrawl │ └── wiki_metadata.json └── test ├── java └── org │ └── wikireverse │ └── commoncrawl │ └── test │ ├── AllTests.java │ ├── LinkArrayReducerTest.java │ ├── LinkArrayWritableTest.java │ ├── LinkWritableTest.java │ ├── MetadataParserTest.java │ ├── SegmentCombinerMapperTest.java │ ├── TestHelper.java │ ├── WikiArticleTest.java │ ├── WikiMetadataTest.java │ └── WikiReverseMapperTest.java └── resources └── org └── wikireverse └── commoncrawl └── test ├── different_case_wiki_subdomains.json ├── edit_link.json ├── long_title_tag.json ├── no_wiki_articles.json ├── no_wiki_text.json ├── one_valid_one_invalid_wiki_links.json ├── one_valid_three_invalid_wiki_links.json ├── one_wiki_article.json ├── one_wiki_article_two_wiki_pages.json ├── three_wiki_articles.json ├── title_contains_whitespace.json ├── two_wiki_articles.json └── wiki_text_no_wiki_articles.json /.classpath: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/.classpath -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/.project -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/.settings/org.eclipse.core.resources.prefs -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/.settings/org.eclipse.jdt.core.prefs -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/.settings/org.eclipse.m2e.core.prefs -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/README.md -------------------------------------------------------------------------------- /dependency-reduced-pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/dependency-reduced-pom.xml -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/pom.xml -------------------------------------------------------------------------------- /src/main/java/edu/cmu/lemurproject/WarcFileInputFormat.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/edu/cmu/lemurproject/WarcFileInputFormat.java -------------------------------------------------------------------------------- /src/main/java/edu/cmu/lemurproject/WarcFileRecordReader.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/edu/cmu/lemurproject/WarcFileRecordReader.java -------------------------------------------------------------------------------- /src/main/java/edu/cmu/lemurproject/WarcRecord.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/edu/cmu/lemurproject/WarcRecord.java -------------------------------------------------------------------------------- /src/main/java/edu/cmu/lemurproject/WritableWarcRecord.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/edu/cmu/lemurproject/WritableWarcRecord.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/FileCountFilter.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/FileCountFilter.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/LinkArrayReducer.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/LinkArrayReducer.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/LinkArrayWritable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/LinkArrayWritable.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/LinkWritable.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/LinkWritable.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/MetadataParser.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/MetadataParser.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/OutputToText.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/OutputToText.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/OutputToTextMapper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/OutputToTextMapper.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/Page.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/Page.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/SegmentCombiner.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/SegmentCombiner.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/SegmentCombinerMapper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/SegmentCombinerMapper.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/WikiArticle.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/WikiArticle.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/WikiMetadata.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/WikiMetadata.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/WikiReverse.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/WikiReverse.java -------------------------------------------------------------------------------- /src/main/java/org/wikireverse/commoncrawl/WikiReverseMapper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/java/org/wikireverse/commoncrawl/WikiReverseMapper.java -------------------------------------------------------------------------------- /src/main/resources/org/wikireverse/commoncrawl/wiki_metadata.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/main/resources/org/wikireverse/commoncrawl/wiki_metadata.json -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/AllTests.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/AllTests.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/LinkArrayReducerTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/LinkArrayReducerTest.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/LinkArrayWritableTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/LinkArrayWritableTest.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/LinkWritableTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/LinkWritableTest.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/MetadataParserTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/MetadataParserTest.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/SegmentCombinerMapperTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/SegmentCombinerMapperTest.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/TestHelper.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/TestHelper.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/WikiArticleTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/WikiArticleTest.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/WikiMetadataTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/WikiMetadataTest.java -------------------------------------------------------------------------------- /src/test/java/org/wikireverse/commoncrawl/test/WikiReverseMapperTest.java: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/java/org/wikireverse/commoncrawl/test/WikiReverseMapperTest.java -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/different_case_wiki_subdomains.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/different_case_wiki_subdomains.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/edit_link.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/edit_link.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/long_title_tag.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/long_title_tag.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/no_wiki_articles.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/no_wiki_articles.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/no_wiki_text.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/no_wiki_text.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/one_valid_one_invalid_wiki_links.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/one_valid_one_invalid_wiki_links.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/one_valid_three_invalid_wiki_links.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/one_valid_three_invalid_wiki_links.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/one_wiki_article.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/one_wiki_article.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/one_wiki_article_two_wiki_pages.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/one_wiki_article_two_wiki_pages.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/three_wiki_articles.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/three_wiki_articles.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/title_contains_whitespace.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/title_contains_whitespace.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/two_wiki_articles.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/two_wiki_articles.json -------------------------------------------------------------------------------- /src/test/resources/org/wikireverse/commoncrawl/test/wiki_text_no_wiki_articles.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rossf7/wikireverse/HEAD/src/test/resources/org/wikireverse/commoncrawl/test/wiki_text_no_wiki_articles.json --------------------------------------------------------------------------------