results = List.of(
389 | new Result(0.5, "my snippet",
390 | new ResponseRecord("foo", "bar", null, null),
391 | new RequestRecord("https://webis.de", Instant.now())),
392 | new Result(0.25, "my second snippet",
393 | new ResponseRecord("foo2", "bar2", null, null),
394 | new RequestRecord("https://webis.de", Instant.now())));
395 | final int pageNumber = 1;
396 | final int numPages = 3;
397 | final UiPage page = new UiPage(
398 | "https://wasp.de", "mywasp",
399 | query, results, pageNumber, numPages,
400 | Locale.ENGLISH, TimeZone.getDefault());
401 | final StringWriter writer = new StringWriter();
402 | pageRenderer.execute(writer, page);
403 | System.out.println(writer.toString());
404 | }
405 |
406 | }
407 |
--------------------------------------------------------------------------------
/src/de/webis/wasp/warcs/ArchiveWatcher.java:
--------------------------------------------------------------------------------
1 | package de.webis.wasp.warcs;
2 |
3 | import java.io.File;
4 | import java.io.IOException;
5 | import java.nio.file.FileSystems;
6 | import java.nio.file.Path;
7 | import java.nio.file.StandardWatchEventKinds;
8 | import java.nio.file.WatchEvent;
9 | import java.nio.file.WatchKey;
10 | import java.nio.file.WatchService;
11 | import java.util.Arrays;
12 | import java.util.Comparator;
13 | import java.util.function.Consumer;
14 | import java.util.logging.Level;
15 | import java.util.logging.Logger;
16 |
17 | import edu.cmu.lemurproject.WarcRecord;
18 |
19 | /**
20 | * Creates a {@link Thread} that watches a web archives repository for new
21 | * content and passes the new records to a consumer.
22 | *
23 | * If archives exist already in the directory, they are read in order of their
24 | * last modified dates (if set so in the constructor). In this case, it will
25 | * monitor the latest archive for changes, but not the others!
26 | *
27 | * Currently, this treats every file (or directory) within the target directory
28 | * as an archive and tries to read from it.
29 | *
30 | *
31 | * @author johannes.kiesel@uni-weimar.de
32 | *
33 | */
34 | public class ArchiveWatcher
35 | extends Thread
36 | implements AutoCloseable {
37 |
38 | /////////////////////////////////////////////////////////////////////////////
39 | // LOGGING
40 | /////////////////////////////////////////////////////////////////////////////
41 |
42 | private static final Logger LOG =
43 | Logger.getLogger(ArchiveWatcher.class.getName());
44 |
45 | /////////////////////////////////////////////////////////////////////////////
46 | // MEMBERS
47 | /////////////////////////////////////////////////////////////////////////////
48 |
49 | private final Path directory;
50 |
51 | private final WatchService watchService;
52 |
53 | private final Consumer consumer;
54 |
55 | private WarcRecordReader reader;
56 |
57 | /////////////////////////////////////////////////////////////////////////////
58 | // CONSTRUCTION
59 | /////////////////////////////////////////////////////////////////////////////
60 |
61 | /**
62 | * Create a new watcher for given directory.
63 | * @param directory The directory that contains the archive files
64 | * @param readExistingRecords Whether records that already exist in the
65 | * archives in the directory should be read
66 | * @param consumer The consumer to which the records will be passed
67 | * @throws IOException On reading records
68 | */
69 | public ArchiveWatcher(
70 | final Path directory, final boolean readExistingRecords,
71 | final Consumer consumer)
72 | throws IOException {
73 | if (consumer == null) { throw new NullPointerException(); }
74 | this.directory = directory;
75 | this.consumer = consumer;
76 | this.reader = null;
77 |
78 | this.initForDirectory(readExistingRecords);
79 |
80 | this.watchService = FileSystems.getDefault().newWatchService();
81 | this.getDirectory().register(this.getWatchService(),
82 | StandardWatchEventKinds.ENTRY_CREATE);
83 | }
84 |
85 | private void initForDirectory(final boolean readExistingRecords)
86 | throws IOException {
87 | final File[] children = this.getDirectory().toFile().listFiles();
88 | Arrays.sort(children, new Comparator() {
89 | @Override
90 | public int compare(final File o1, final File o2) {
91 | return Long.compare(o1.lastModified(), o2.lastModified());
92 | }
93 | });
94 |
95 | if (readExistingRecords) {
96 | // Read what should be closed files
97 | if (children.length >= 2) {
98 | for (final File child
99 | : Arrays.copyOfRange(children, 0, children.length - 1)) {
100 | try (final WarcRecordReader reader = new WarcRecordReader(
101 | this.getDirectory().resolve(child.getName()),
102 | this.getConsumer())) {
103 | reader.run();
104 | }
105 | }
106 | }
107 | }
108 |
109 | // Read what may be the open file
110 | if (children.length >= 1) {
111 | this.openFile(this.getDirectory().resolve(
112 | children[children.length - 1].getName()), readExistingRecords);
113 | }
114 | }
115 |
116 | /////////////////////////////////////////////////////////////////////////////
117 | // GETTERS
118 | /////////////////////////////////////////////////////////////////////////////
119 |
120 | /**
121 | * Gets the directory being watched.
122 | * @return The directory
123 | */
124 | public Path getDirectory() {
125 | return this.directory;
126 | }
127 |
128 | /**
129 | * Gets the service watching for changes in the directory.
130 | * @return The service
131 | */
132 | protected WatchService getWatchService() {
133 | return this.watchService;
134 | }
135 |
136 | /**
137 | * Gets the consumer to which WARC records are passed to.
138 | * @return The consumer
139 | */
140 | public Consumer getConsumer() {
141 | return this.consumer;
142 | }
143 |
144 | /**
145 | * Gets the current WARC record reader.
146 | * @return The reader
147 | */
148 | protected WarcRecordReader getReader() {
149 | return this.reader;
150 | }
151 |
152 | /////////////////////////////////////////////////////////////////////////////
153 | // SETTER
154 | /////////////////////////////////////////////////////////////////////////////
155 |
156 | /**
157 | * Sets the WARC record reader.
158 | * @param reader The reader
159 | */
160 | public void setReader(final WarcRecordReader reader) {
161 | this.reader = reader;
162 | }
163 |
164 | /////////////////////////////////////////////////////////////////////////////
165 | // FUNCTIONALITY
166 | /////////////////////////////////////////////////////////////////////////////
167 |
168 | @Override
169 | public void run() {
170 | final Path directory = this.getDirectory();
171 | try {
172 | while (true) {
173 | final WatchKey key = this.getWatchService().take();
174 | for (final WatchEvent event : key.pollEvents()) {
175 | final WatchEvent.Kind kind = event.kind();
176 | if (kind == StandardWatchEventKinds.ENTRY_CREATE) {
177 | final Path inputFile = directory.resolve((Path) event.context());
178 | LOG.fine("New file created in " + directory + ": " + inputFile);
179 | this.openFile(inputFile, true);
180 | } else if (kind == StandardWatchEventKinds.OVERFLOW) {
181 | LOG.warning("Overflow detected when watching " + directory);
182 | } else {
183 | LOG.warning("Unknown watch event kind '" + kind + "' when watching "
184 | + directory);
185 | }
186 | }
187 |
188 | if (!key.reset()) {
189 | LOG.severe(
190 | "Directory " + directory + " can no longer be watched");
191 | break;
192 | }
193 | }
194 | } catch (final InterruptedException exception) {
195 | LOG.log(Level.SEVERE,
196 | "Interrupted watching " + directory, exception);
197 | } catch (final IOException exception) {
198 | LOG.log(Level.SEVERE, "Error watching " + directory, exception);
199 | }
200 | }
201 |
202 | @Override
203 | public void close() throws IOException {
204 | this.closeFile();
205 | }
206 |
207 | /**
208 | * Closes the currently opened file, if any.
209 | * @throws IOException On closing the file
210 | * @see {@link #openFile(Path, boolean)}
211 | */
212 | protected void closeFile() throws IOException {
213 | synchronized (this) {
214 | final WarcRecordReader reader = this.getReader();
215 | if (reader != null) {
216 | this.setReader(null);
217 | reader.close();
218 | }
219 | }
220 | }
221 |
222 | /**
223 | * Starts reading from a new file, keeping watch if records are appended.
224 | * @param inputFile The file to read
225 | * @param consumeExistingRecords Whether to also pass existing records to the
226 | * consumer
227 | * @throws IOException On opening the file
228 | */
229 | protected void openFile(
230 | final Path inputFile, final boolean consumeExistingRecords)
231 | throws IOException {
232 | synchronized (this) {
233 | this.closeFile();
234 | final WarcRecordReader reader = new ContinuousWarcRecordReader(
235 | inputFile, consumeExistingRecords, this.getConsumer(), 1000);
236 | this.setReader(reader);
237 | reader.start();
238 | }
239 | }
240 |
241 | }
242 |
--------------------------------------------------------------------------------
/src/de/webis/wasp/warcs/ContinuousWarcRecordReader.java:
--------------------------------------------------------------------------------
1 | package de.webis.wasp.warcs;
2 |
3 | import java.io.File;
4 | import java.io.FileInputStream;
5 | import java.io.FileNotFoundException;
6 | import java.io.IOException;
7 | import java.nio.file.Path;
8 | import java.util.function.Consumer;
9 | import java.util.logging.Level;
10 | import java.util.logging.Logger;
11 |
12 | import edu.cmu.lemurproject.WarcRecord;
13 |
14 | /**
15 | * A {@link WarcRecordReader} that will wait for new content even when it
16 | * reached the end of the archive.
17 | *
18 | * This class should be used for archives that are still filled. When you use
19 | * {@link #close()}, this reader will still continue to read until it
20 | * encounters the end of the file the next time.
21 | *
22 | *
23 | * @author johannes.kiesel@uni-weimar.de
24 | *
25 | */
26 | public class ContinuousWarcRecordReader extends WarcRecordReader {
27 |
28 | /////////////////////////////////////////////////////////////////////////////
29 | // LOGGING
30 | /////////////////////////////////////////////////////////////////////////////
31 |
32 | private static final Logger LOG =
33 | Logger.getLogger(ContinuousWarcRecordReader.class.getName());
34 |
35 | /////////////////////////////////////////////////////////////////////////////
36 | // MEMBERS
37 | /////////////////////////////////////////////////////////////////////////////
38 |
39 | protected final long pollIntervalMillis;
40 |
41 | protected boolean consume;
42 |
43 | protected boolean closed;
44 |
45 | /////////////////////////////////////////////////////////////////////////////
46 | // CONSTRUCTORS
47 | /////////////////////////////////////////////////////////////////////////////
48 |
49 | /**
50 | * Creates a new reader for an archive that is still being filled.
51 | * @param inputFile The archive file
52 | * @param consumeExistingRecords Whether records that are already in the file
53 | * should also be consumed
54 | * @param consumer Consumer for the WARC records that are read
55 | * @param pollIntervalMillis On encountering the end of archive, poll the file
56 | * in this interval to check when it has more content
57 | * @throws IOException When the file can not be opened
58 | */
59 | public ContinuousWarcRecordReader(
60 | final Path inputFile, final boolean consumeExistingRecords,
61 | final Consumer consumer,
62 | final long pollIntervalMillis)
63 | throws IOException {
64 | super(inputFile, consumer);
65 | this.pollIntervalMillis = pollIntervalMillis;
66 | this.consume = consumeExistingRecords;
67 | this.closed = false;
68 | }
69 |
70 | @Override
71 | protected FileInputStream openFileInputStream()
72 | throws IOException {
73 | final File file = this.getInputFile().toFile();
74 | LOG.fine("Open file: " + file);
75 | return new ContinuousFileInputStream(file);
76 | }
77 |
78 | /////////////////////////////////////////////////////////////////////////////
79 | // FUNCTIONALITY
80 | /////////////////////////////////////////////////////////////////////////////
81 |
82 | @Override
83 | public void close() throws IOException {
84 | LOG.fine("Closing " + this.getInputFile());
85 | this.closed = true;
86 | }
87 |
88 | @Override
89 | protected void consume(final WarcRecord record) {
90 | if (this.consume) {
91 | super.getConsumer().accept(record);
92 | }
93 | }
94 |
95 | protected void closeStream() throws IOException {
96 | super.close();
97 | }
98 |
99 | /////////////////////////////////////////////////////////////////////////////
100 | // HELPER CLASSES
101 | /////////////////////////////////////////////////////////////////////////////
102 |
103 | /**
104 | * Modification of {@link FileInputStream} that does waits at the end of the
105 | * file for more content to appear.
106 | *
107 | * @author johannes.kiesel@uni-weimar.de
108 | */
109 | protected class ContinuousFileInputStream
110 | extends FileInputStream {
111 |
112 | public ContinuousFileInputStream(final File file)
113 | throws FileNotFoundException {
114 | super(file);
115 | }
116 |
117 | @Override
118 | public int available() throws IOException {
119 | int available = super.available();
120 | try {
121 | while (available == 0 && !ContinuousWarcRecordReader.this.closed) {
122 | ContinuousWarcRecordReader.this.consume = true;
123 | Thread.sleep(ContinuousWarcRecordReader.this.pollIntervalMillis);
124 | available = super.available();
125 | }
126 | } catch (final InterruptedException exception) {
127 | LOG.log(Level.WARNING, "Interrupted " + this, exception);
128 | }
129 |
130 | if (ContinuousWarcRecordReader.this.closed) {
131 | ContinuousWarcRecordReader.this.closeStream();
132 | }
133 | return available;
134 | }
135 |
136 | @Override
137 | public int read() throws IOException {
138 | int read = super.read();
139 | try {
140 | while (read == -1 && !ContinuousWarcRecordReader.this.closed) {
141 | ContinuousWarcRecordReader.this.consume = true;
142 | Thread.sleep(ContinuousWarcRecordReader.this.pollIntervalMillis);
143 | read = super.read();
144 | }
145 | } catch (final InterruptedException exception) {
146 | LOG.log(Level.WARNING, "Interrupted " + this, exception);
147 | }
148 |
149 | if (ContinuousWarcRecordReader.this.closed) {
150 | ContinuousWarcRecordReader.this.closeStream();
151 | }
152 | return read;
153 | }
154 |
155 | @Override
156 | public int read(byte b[]) throws IOException {
157 | int read = super.read(b);
158 | try {
159 | while (read == -1 && !ContinuousWarcRecordReader.this.closed) {
160 | ContinuousWarcRecordReader.this.consume = true;
161 | Thread.sleep(ContinuousWarcRecordReader.this.pollIntervalMillis);
162 | read = super.read(b);
163 | }
164 | } catch (final InterruptedException exception) {
165 | LOG.log(Level.WARNING, "Interrupted " + this, exception);
166 | }
167 |
168 | if (ContinuousWarcRecordReader.this.closed) {
169 | ContinuousWarcRecordReader.this.closeStream();
170 | }
171 | return read;
172 | }
173 |
174 | @Override
175 | public int read(byte b[], int off, int len) throws IOException {
176 | int read = super.read(b, off, len);
177 | try {
178 | while (read == -1 && !ContinuousWarcRecordReader.this.closed) {
179 | ContinuousWarcRecordReader.this.consume = true;
180 | Thread.sleep(ContinuousWarcRecordReader.this.pollIntervalMillis);
181 | read = super.read(b, off, len);
182 | }
183 | } catch (final InterruptedException exception) {
184 | LOG.log(Level.WARNING, "Interrupted " + this, exception);
185 | }
186 |
187 | if (ContinuousWarcRecordReader.this.closed) {
188 | ContinuousWarcRecordReader.this.closeStream();
189 | }
190 | return read;
191 | }
192 |
193 | }
194 |
195 | }
196 |
--------------------------------------------------------------------------------
/src/de/webis/wasp/warcs/GenericHtmlWarcRecordConsumer.java:
--------------------------------------------------------------------------------
1 | package de.webis.wasp.warcs;
2 |
3 | import java.io.IOException;
4 | import java.time.Instant;
5 | import java.util.Objects;
6 | import java.util.function.Function;
7 |
8 | /**
9 | * Generic class for consuming HTML WARC records.
10 | *
11 | * @author johannes.kiesel@uni-weimar.de
12 | *
13 | */
14 | public abstract class GenericHtmlWarcRecordConsumer
15 | extends GenericWarcRecordConsumer {
16 |
17 | /////////////////////////////////////////////////////////////////////////////
18 | // CONSTANTS
19 | /////////////////////////////////////////////////////////////////////////////
20 |
21 | /**
22 | * Default function for extracting HTML from response records.
23 | */
24 | public static final Function DEFAULT_DOCUMENT_EXTRACTOR =
25 | JerichoDocumentExtractor.INSTANCE;
26 |
27 | /////////////////////////////////////////////////////////////////////////////
28 | // MEMBERS
29 | /////////////////////////////////////////////////////////////////////////////
30 |
31 | private Function documentExtractor;
32 |
33 | /////////////////////////////////////////////////////////////////////////////
34 | // CONSTRUCTION
35 | /////////////////////////////////////////////////////////////////////////////
36 |
37 | /**
38 | * Creates a new consumer using the default extractor for HTML responses.
39 | */
40 | public GenericHtmlWarcRecordConsumer() {
41 | this(DEFAULT_DOCUMENT_EXTRACTOR);
42 | }
43 |
44 | /**
45 | * Creates a new consumer using the specified extractor for HTML responses.
46 | * @param documentExtractor The extractor
47 | */
48 | public GenericHtmlWarcRecordConsumer(
49 | final Function documentExtractor) {
50 | this.setDocumentExtractor(documentExtractor);
51 | }
52 |
53 | /////////////////////////////////////////////////////////////////////////////
54 | // GETTERS
55 | /////////////////////////////////////////////////////////////////////////////
56 |
57 | /**
58 | * Gets the document extractor for HTML responses.
59 | * @return The extractor
60 | */
61 | public Function getDocumentExtractor() {
62 | return this.documentExtractor;
63 | }
64 |
65 | /////////////////////////////////////////////////////////////////////////////
66 | // SETTERS
67 | /////////////////////////////////////////////////////////////////////////////
68 |
69 | /**
70 | * Sets the document extractor for HTML responses.
71 | * @param documentExtractor The extractor
72 | */
73 | protected void setDocumentExtractor(
74 | final Function documentExtractor) {
75 | this.documentExtractor = Objects.requireNonNull(documentExtractor);
76 | }
77 |
78 | /////////////////////////////////////////////////////////////////////////////
79 | // FUNCTIONALITY
80 | /////////////////////////////////////////////////////////////////////////////
81 |
82 | @Override
83 | protected void acceptHtmlResponse(
84 | final String id, final String uri, final String html, final Instant time)
85 | throws IOException {
86 | final Document document = this.getDocumentExtractor().apply(html);
87 | this.acceptHtmlResponse(id, uri, document, time);
88 | }
89 |
90 | protected abstract void acceptHtmlResponse(
91 | final String id, final String uri, final Document document,
92 | final Instant time)
93 | throws IOException;
94 |
95 | /////////////////////////////////////////////////////////////////////////////
96 | // DOCUMENT
97 | /////////////////////////////////////////////////////////////////////////////
98 |
99 | /**
100 | * A processed document.
101 | *
102 | * @author johannes.kiesel@uni-weimar.de
103 | *
104 | */
105 | public static final class Document {
106 |
107 | private final String title;
108 |
109 | private final String content;
110 |
111 | /**
112 | * Creates a new document.
113 | * @param title The document's title (or null)
114 | * @param content The document's content (or null)
115 | */
116 | public Document(final String title, final String content) {
117 | this.title = title;
118 | this.content = content;
119 | }
120 |
121 | /**
122 | * Gets the title of the document.
123 | * @return The title (may be null or empty)
124 | */
125 | public String getTitle() {
126 | return this.title;
127 | }
128 |
129 | /**
130 | * Gets the text content of the document.
131 | * @return The content (may be null or empty)
132 | */
133 | public String getContent() {
134 | return this.content;
135 | }
136 |
137 | }
138 |
139 | }
140 |
--------------------------------------------------------------------------------
/src/de/webis/wasp/warcs/GenericWarcRecordConsumer.java:
--------------------------------------------------------------------------------
1 | package de.webis.wasp.warcs;
2 |
3 | import java.io.IOException;
4 | import java.util.function.Consumer;
5 | import java.util.logging.Level;
6 | import java.util.logging.Logger;
7 | import java.time.Instant;
8 |
9 | import org.apache.http.HttpResponse;
10 |
11 | import edu.cmu.lemurproject.WarcRecord;
12 |
13 | /**
14 | * Generic class for consuming WARC records with methods for different records.
15 | *
16 | * @author johannes.kiesel@uni-weimar.de
17 | *
18 | */
19 | public abstract class GenericWarcRecordConsumer
20 | implements Consumer {
21 |
22 | /////////////////////////////////////////////////////////////////////////////
23 | // LOGGING
24 | /////////////////////////////////////////////////////////////////////////////
25 |
26 | private static final Logger LOG =
27 | Logger.getLogger(GenericWarcRecordConsumer.class.getName());
28 |
29 | /////////////////////////////////////////////////////////////////////////////
30 | // FUNCTIONALITY
31 | /////////////////////////////////////////////////////////////////////////////
32 |
33 | @Override
34 | public void accept(final WarcRecord record) {
35 | final String type = Warcs.getType(record);
36 |
37 | final Instant time =
38 | Instant.ofEpochSecond(Warcs.getDate(record).getEpochSecond());
39 | try {
40 | switch (type) {
41 | case Warcs.HEADER_TYPE_RESPONSE:
42 | this.acceptResponse(record, time);
43 | break;
44 | case Warcs.HEADER_TYPE_REQUEST:
45 | this.acceptRequest(record, time);
46 | break;
47 | case Warcs.HEADER_TYPE_REVISIT:
48 | this.acceptRevisit(record, time);
49 | break;
50 | default:
51 | break;
52 | }
53 | } catch (final Throwable exception) {
54 | LOG.log(Level.WARNING, "Failed to index record " + Warcs.getId(record)
55 | + " of type " + Warcs.getType(record), exception);
56 | }
57 | }
58 |
59 | /////////////////////////////////////////////////////////////////////////////
60 | // Response
61 |
62 | protected void acceptResponse(final WarcRecord record, final Instant time)
63 | throws IOException {
64 | final String id = Warcs.getId(record);
65 | final String uri = Warcs.getTargetUri(record);
66 | final String html = this.getHtml(record);
67 | if (html != null) {
68 | LOG.fine("accept html response " + id + " -> " + uri);
69 | this.acceptHtmlResponse(id, uri, html, time);
70 | } else {
71 | LOG.fine("accept non-html response " + id + " -> " + uri);
72 | this.acceptNonHtmlResponse(id, uri, time);
73 | }
74 | }
75 |
76 | protected void acceptNonHtmlResponse(
77 | final String id, final String uri, final Instant time)
78 | throws IOException {
79 | // do nothing by default
80 | }
81 |
82 | protected void acceptHtmlResponse(
83 | final String id, final String uri, final String html, final Instant time)
84 | throws IOException {
85 | // do nothing by default
86 | }
87 |
88 | /////////////////////////////////////////////////////////////////////////////
89 | // Revisit
90 |
91 | protected void acceptRevisit(final WarcRecord record, final Instant time)
92 | throws IOException {
93 | this.acceptRevisit(
94 | Warcs.getId(record),
95 | Warcs.getReferedToTargetUri(record),
96 | Warcs.getReferedToDate(record),
97 | time);
98 | }
99 |
100 | protected void acceptRevisit(
101 | final String id, final String uri, final Instant originalTime,
102 | final Instant time)
103 | throws IOException {
104 | // do nothing by default
105 | }
106 |
107 | /////////////////////////////////////////////////////////////////////////////
108 | // Request
109 |
110 | protected void acceptRequest(final WarcRecord record, final Instant time)
111 | throws IOException {
112 | this.acceptRequest(
113 | Warcs.getConcurrentRecordId(record),
114 | Warcs.getTargetUri(record),
115 | time);
116 | }
117 |
118 | protected void acceptRequest(
119 | final String concurrentRecordId,
120 | final String targetUri,
121 | final Instant time)
122 | throws IOException {
123 | // do nothing by default
124 | }
125 |
126 | /////////////////////////////////////////////////////////////////////////////
127 | // HELPERS
128 | /////////////////////////////////////////////////////////////////////////////
129 |
130 | /**
131 | * Gets the HTML from a response WARC record.
132 | * @param record The record
133 | * @return The HTML if it exists, or null
134 | */
135 | protected String getHtml(final WarcRecord record) {
136 | try {
137 | final HttpResponse response = Warcs.toResponse(record);
138 | if (Warcs.isHtml(response)) {
139 | return Warcs.getHtml(record);
140 | }
141 | } catch (final Throwable exception) {
142 | LOG.log(Level.FINER,
143 | "Could not parse record " + Warcs.getId(record),
144 | exception);
145 | }
146 | return null;
147 | }
148 |
149 | }
150 |
--------------------------------------------------------------------------------
/src/de/webis/wasp/warcs/JerichoDocumentExtractor.java:
--------------------------------------------------------------------------------
1 | package de.webis.wasp.warcs;
2 |
3 | import java.util.Objects;
4 | import java.util.function.Function;
5 |
6 | import de.webis.wasp.warcs.GenericHtmlWarcRecordConsumer.Document;
7 | import net.htmlparser.jericho.CharacterReference;
8 | import net.htmlparser.jericho.Element;
9 | import net.htmlparser.jericho.HTMLElementName;
10 | import net.htmlparser.jericho.Renderer;
11 | import net.htmlparser.jericho.Source;
12 |
13 | /**
14 | * A document extractor using Jericho HTML parser.
15 | *
16 | * @author johannes.kiesel@uni-weimar.de
17 | *
18 | */
19 | public class JerichoDocumentExtractor
20 | implements Function {
21 |
22 | /**
23 | * The single instance of the extractor.
24 | */
25 | public static final JerichoDocumentExtractor INSTANCE =
26 | new JerichoDocumentExtractor();
27 |
28 | protected JerichoDocumentExtractor() { }
29 |
30 | @Override
31 | public Document apply(final String html) {
32 | final Source source = new Source(Objects.requireNonNull(html));
33 |
34 | final Renderer renderer = new Renderer(source);
35 | renderer.setMaxLineLength(0);
36 | renderer.setIncludeHyperlinkURLs(false);
37 | renderer.setIncludeAlternateText(true);
38 | final String content = renderer.toString();
39 |
40 | final Element titleElement =
41 | source.getFirstElement(HTMLElementName.TITLE);
42 | final String title = titleElement == null
43 | ? null
44 | : CharacterReference.decodeCollapseWhiteSpace(
45 | titleElement.getContent());
46 |
47 | return new Document(title, content);
48 | }
49 |
50 | }
51 |
--------------------------------------------------------------------------------
/src/de/webis/wasp/warcs/WarcRecordReader.java:
--------------------------------------------------------------------------------
1 | package de.webis.wasp.warcs;
2 |
3 | import java.io.DataInputStream;
4 | import java.io.File;
5 | import java.io.FileInputStream;
6 | import java.io.IOException;
7 | import java.io.InputStream;
8 | import java.nio.file.Path;
9 | import java.util.function.Consumer;
10 | import java.util.logging.Level;
11 | import java.util.logging.Logger;
12 | import java.util.zip.GZIPInputStream;
13 |
14 | import edu.cmu.lemurproject.WarcRecord;
15 |
16 | /**
17 | * Reader for WARC files that passes all read records to a consumer.
18 | *
19 | * Use the {@link #run()} or {@link #start()} methods to begin reading.
20 | *
21 | * If the archive is still being filled, use {@link ContinuousWarcRecordReader}
22 | * instead.
23 | *
24 | *
25 | * @author johannes.kiesel@uni-weimar.de
26 | *
27 | */
28 | public class WarcRecordReader
29 | extends Thread
30 | implements AutoCloseable {
31 |
32 | /////////////////////////////////////////////////////////////////////////////
33 | // LOGGING
34 | /////////////////////////////////////////////////////////////////////////////
35 |
36 | private static final Logger LOG =
37 | Logger.getLogger(WarcRecordReader.class.getName());
38 |
39 | /////////////////////////////////////////////////////////////////////////////
40 | // MEMBERS
41 | /////////////////////////////////////////////////////////////////////////////
42 |
43 | private final Consumer consumer;
44 |
45 | private final Path inputFile;
46 |
47 | private final DataInputStream input;
48 |
49 | /////////////////////////////////////////////////////////////////////////////
50 | // CONSTRUCTION
51 | /////////////////////////////////////////////////////////////////////////////
52 |
53 | /**
54 | * Creates a new reader for an archive.
55 | * @param inputFile The archive file
56 | * @param consumer Consumer for the WARC records that are read
57 | * @throws IOException When the file can not be opened
58 | */
59 | public WarcRecordReader(
60 | final Path inputFile, final Consumer consumer)
61 | throws IOException {
62 | if (consumer == null) { throw new NullPointerException(); }
63 | this.consumer = consumer;
64 | this.inputFile = inputFile;
65 | this.input = this.openDataInputStream();
66 | }
67 |
68 | /**
69 | * Opens a data input stream to the reader's file, applying GZip decompression
70 | * if the file ends on .gz.
71 | * @return The input stream
72 | * @throws IOException On opening the file
73 | */
74 | protected DataInputStream openDataInputStream()
75 | throws IOException {
76 | final InputStream inputStream = this.openFileInputStream();
77 | if (this.getInputFile().toString().toLowerCase().endsWith(".gz")) {
78 | return new DataInputStream(new GZIPInputStream(inputStream));
79 | } else {
80 | return new DataInputStream(inputStream);
81 | }
82 | }
83 |
84 | /**
85 | * Opens an input stream to the reader's file.
86 | * @return The input stream
87 | * @throws IOException On opening the file
88 | */
89 | protected FileInputStream openFileInputStream()
90 | throws IOException {
91 | final File file = this.getInputFile().toFile();
92 | LOG.fine("Open file: " + file);
93 | return new FileInputStream(file);
94 | }
95 |
96 | /////////////////////////////////////////////////////////////////////////////
97 | // CONSTRUCTION
98 | /////////////////////////////////////////////////////////////////////////////
99 |
100 | /**
101 | * Gets the file this reader reads from.
102 | * @return The file
103 | */
104 | public Path getInputFile() {
105 | return this.inputFile;
106 | }
107 |
108 | /**
109 | * Gets the consumer to which WARC records are passed to.
110 | * @return The consumer
111 | */
112 | public Consumer getConsumer() {
113 | return this.consumer;
114 | }
115 |
116 | /**
117 | * Gets the input stream.
118 | * @return The stream
119 | */
120 | protected DataInputStream getInput() {
121 | return this.input;
122 | }
123 |
124 | /////////////////////////////////////////////////////////////////////////////
125 | // FUNCTIONALITY
126 | /////////////////////////////////////////////////////////////////////////////
127 |
128 | @Override
129 | public void run() {
130 | final DataInputStream input = this.getInput();
131 | try {
132 | WarcRecord record = WarcRecord.readNextWarcRecord(input);
133 | while (record != null) {
134 | this.consume(record);
135 | record = WarcRecord.readNextWarcRecord(input);
136 | }
137 | LOG.fine("Finished " + this);
138 | this.close();
139 | } catch (final IOException exception) {
140 | LOG.log(Level.SEVERE,
141 | "Error while reading from " + this.getInputFile(), exception);
142 | }
143 | }
144 |
145 | @Override
146 | public void close() throws IOException {
147 | LOG.fine("Close file " + this.getInputFile());
148 | this.getInput().close();
149 | }
150 |
151 | @Override
152 | public String toString() {
153 | return this.getInputFile() + " -> " + this.getConsumer();
154 | }
155 |
156 | /////////////////////////////////////////////////////////////////////////////
157 | // HELPERS
158 | /////////////////////////////////////////////////////////////////////////////
159 |
160 | /**
161 | * Passes the record to the consumer.
162 | * @param record The record
163 | * @see #getConsumer()
164 | */
165 | protected void consume(final WarcRecord record) {
166 | this.getConsumer().accept(record);
167 | }
168 |
169 | }
170 |
--------------------------------------------------------------------------------
/src/de/webis/wasp/warcs/Warcs.java:
--------------------------------------------------------------------------------
1 | package de.webis.wasp.warcs;
2 |
3 | import java.io.ByteArrayInputStream;
4 | import java.io.IOException;
5 | import java.io.InputStream;
6 | import java.time.Instant;
7 | import java.time.format.DateTimeFormatter;
8 | import java.util.Locale;
9 | import java.util.regex.Pattern;
10 | import java.util.zip.GZIPInputStream;
11 |
12 | import org.apache.http.Header;
13 | import org.apache.http.HeaderElement;
14 | import org.apache.http.HttpEntity;
15 | import org.apache.http.HttpException;
16 | import org.apache.http.HttpResponse;
17 | import org.apache.http.ParseException;
18 | import org.apache.http.client.entity.DecompressingEntity;
19 | import org.apache.http.client.entity.DeflateInputStream;
20 | import org.apache.http.client.entity.InputStreamFactory;
21 | import org.apache.http.config.Lookup;
22 | import org.apache.http.config.MessageConstraints;
23 | import org.apache.http.config.RegistryBuilder;
24 | import org.apache.http.entity.BasicHttpEntity;
25 | import org.apache.http.entity.ContentLengthStrategy;
26 | import org.apache.http.impl.DefaultHttpResponseFactory;
27 | import org.apache.http.impl.entity.LaxContentLengthStrategy;
28 | import org.apache.http.impl.io.ChunkedInputStream;
29 | import org.apache.http.impl.io.ContentLengthInputStream;
30 | import org.apache.http.impl.io.DefaultHttpResponseParser;
31 | import org.apache.http.impl.io.EmptyInputStream;
32 | import org.apache.http.impl.io.HttpTransportMetricsImpl;
33 | import org.apache.http.impl.io.IdentityInputStream;
34 | import org.apache.http.impl.io.SessionInputBufferImpl;
35 | import org.apache.http.io.SessionInputBuffer;
36 | import org.apache.http.protocol.HTTP;
37 | import org.apache.http.util.EntityUtils;
38 |
39 | import edu.cmu.lemurproject.WarcRecord;
40 |
41 | /**
42 | * Utility class for working with WARC files.
43 | *
44 | * @author johannes.kiesel@uni-weimar.de
45 | *
46 | */
47 | public class Warcs {
48 |
49 | public static final String HEADER_ID = "WARC-Record-ID";
50 |
51 | public static final String HEADER_TYPE = "WARC-Type";
52 |
53 | public static final String HEADER_TYPE_INFO = "warcinfo";
54 |
55 | public static final String HEADER_TYPE_REQUEST = "request";
56 |
57 | public static final String HEADER_TYPE_RESPONSE = "response";
58 |
59 | public static final String HEADER_TYPE_REVISIT = "revisit";
60 |
61 | public static final String HEADER_REFERS_TO = "WARC-Refers-To";
62 |
63 | public static final String HEADER_REFERS_TO_TARGET_URI ="WARC-Refers-To-Target-URI";
64 |
65 | public static final String HEADER_REFERS_TO_DATE = "WARC-Refers-To-Date";
66 |
67 | public static final String HEADER_TARGET_URI = "WARC-Target-URI";
68 |
69 | public static final String HEADER_CONCURRENT = "WARC-Concurrent-To";
70 |
71 | public static final String HEADER_DATE = "WARC-Date";
72 |
73 | public static final DateTimeFormatter HEADER_DATE_FORMAT =
74 | DateTimeFormatter.ISO_INSTANT;
75 |
76 |
77 | public static final Pattern HTTP_HEADER_CONTENT_TYPE_HTML = Pattern.compile(
78 | "text/html.*");
79 |
80 | public static final String HTTP_HEADER_CONTENT_TYPE = "Content-Type";
81 |
82 | /////////////////////////////////////////////////////////////////////////////
83 | // STATIC HELPERS
84 | /////////////////////////////////////////////////////////////////////////////
85 |
86 | private final static InputStreamFactory GZIP = new InputStreamFactory() {
87 | @Override
88 | public InputStream create(final InputStream instream) throws IOException {
89 | return new GZIPInputStream(instream);
90 | }
91 | };
92 |
93 | private final static InputStreamFactory DEFLATE = new InputStreamFactory() {
94 | @Override
95 | public InputStream create(final InputStream instream) throws IOException {
96 | return new DeflateInputStream(instream);
97 | }
98 | };
99 |
100 | /////////////////////////////////////////////////////////////////////////////
101 | // CONSTRUCTORS
102 | /////////////////////////////////////////////////////////////////////////////
103 |
104 | // Utility class
105 | private Warcs() { }
106 |
107 | /////////////////////////////////////////////////////////////////////////////
108 | // FUNCTIONALITY
109 | /////////////////////////////////////////////////////////////////////////////
110 |
111 | /////////////////////////////////////////////////////////////////////////////
112 | // Access header fields
113 |
114 | public static String getHeader(
115 | final WarcRecord record, final String header) {
116 | return record.getHeaderMetadataItem(header);
117 | }
118 |
119 | public static String getId(final WarcRecord record) {
120 | return Warcs.getHeader(record, HEADER_ID);
121 | }
122 |
123 | public static String getType(final WarcRecord record) {
124 | return Warcs.getHeader(record, HEADER_TYPE);
125 | }
126 |
127 | public static Instant getDate(final WarcRecord record) {
128 | final String date = Warcs.getHeader(record, HEADER_DATE);
129 | return Instant.from(HEADER_DATE_FORMAT.parse(date));
130 | }
131 |
132 | public static String getTargetUri(final WarcRecord record) {
133 | return Warcs.getHeader(record, HEADER_TARGET_URI);
134 | }
135 |
136 | public static String getConcurrentRecordId(final WarcRecord record) {
137 | return Warcs.getHeader(record, HEADER_CONCURRENT);
138 | }
139 |
140 | public static String getReferedToRecordId(final WarcRecord record) {
141 | return Warcs.getHeader(record, HEADER_REFERS_TO);
142 | }
143 |
144 | public static String getReferedToTargetUri(final WarcRecord record) {
145 | return Warcs.getHeader(record, HEADER_REFERS_TO_TARGET_URI);
146 | }
147 |
148 | public static Instant getReferedToDate(final WarcRecord record) {
149 | final String date = Warcs.getHeader(record, HEADER_REFERS_TO_DATE);
150 | return Instant.from(HEADER_DATE_FORMAT.parse(date));
151 | }
152 |
153 | /////////////////////////////////////////////////////////////////////////////
154 | // HTML
155 |
156 | /**
157 | * Checks if this is a HTML response record.
158 | */
159 | public static boolean isHtml(final WarcRecord record)
160 | throws HttpException, IOException {
161 | if (record == null) { return false; }
162 | final HttpResponse response = Warcs.toResponse(record);
163 | return Warcs.isHtml(response);
164 | }
165 |
166 | /**
167 | * Checks if this is a HTML response.
168 | */
169 | public static boolean isHtml(final HttpResponse response) {
170 | if (response == null) { return false; }
171 |
172 | final String contentType =
173 | response.getLastHeader(HTTP_HEADER_CONTENT_TYPE).getValue();
174 | if (contentType == null) { return false; } // no content type
175 |
176 | if (!HTTP_HEADER_CONTENT_TYPE_HTML.matcher(contentType).matches()) {
177 | return false; // not HTML content type
178 | }
179 |
180 | return true;
181 | }
182 |
183 | /**
184 | * Gets the HTML part of a record or null if there is none or an
185 | * invalid one.
186 | */
187 | public static String getHtml(final WarcRecord record)
188 | throws ParseException, IOException, HttpException {
189 | final HttpResponse response = Warcs.toResponse(record);
190 | if (!Warcs.isHtml(response)) { return null; } // no HTML record
191 |
192 | final HttpEntity entity = response.getEntity();
193 | final String defaultCharset = null;
194 | return EntityUtils.toString(entity, defaultCharset);
195 | }
196 |
197 | /**
198 | * Gets an {@link HttpResponse} object from a WARC record of such a response.
199 | * @return The response or null when the record is not a response
200 | * record
201 | */
202 | public static HttpResponse toResponse(final WarcRecord record)
203 | throws IOException, HttpException {
204 | // based on http://stackoverflow.com/a/26586178
205 | if (!record.getHeaderRecordType().equals("response")) { return null; }
206 |
207 | final SessionInputBufferImpl sessionInputBuffer =
208 | new SessionInputBufferImpl(new HttpTransportMetricsImpl(), 2048);
209 | final InputStream inputStream =
210 | new ByteArrayInputStream(record.getByteContent());
211 | sessionInputBuffer.bind(inputStream);
212 | final MessageConstraints constraints = MessageConstraints.DEFAULT;
213 | final DefaultHttpResponseParser parser =
214 | new DefaultHttpResponseParser(
215 | sessionInputBuffer, null, new DefaultHttpResponseFactory(),
216 | constraints);
217 | final HttpResponse response = parser.parse();
218 | final HttpEntity entity = Warcs.getEntity(response, sessionInputBuffer);
219 | response.setEntity(entity);
220 | Warcs.encodeEntity(response);
221 | return response;
222 | }
223 |
224 |
225 | private static void encodeEntity(final HttpResponse response)
226 | throws HttpException, IOException {
227 | // Adapted from org.apache.http.client.protocol.ResponseContentEncoding
228 | final HttpEntity entity = response.getEntity();
229 |
230 | // entity can be null in case of 304 Not Modified, 204 No Content or similar
231 | // check for zero length entity.
232 | if (entity != null && entity.getContentLength() != 0) {
233 | final Header ceheader = entity.getContentEncoding();
234 | if (ceheader != null) {
235 | final HeaderElement[] codecs = ceheader.getElements();
236 | final Lookup decoderRegistry =
237 | RegistryBuilder.create()
238 | .register("gzip", GZIP)
239 | .register("x-gzip", GZIP)
240 | .register("deflate", DEFLATE)
241 | .build();
242 | for (final HeaderElement codec : codecs) {
243 | final String codecname = codec.getName().toLowerCase(Locale.ROOT);
244 | final InputStreamFactory decoderFactory =
245 | decoderRegistry.lookup(codecname);
246 | if (decoderFactory != null) {
247 | response.setEntity(new DecompressingEntity(
248 | response.getEntity(), decoderFactory));
249 | response.removeHeaders("Content-Length");
250 | response.removeHeaders("Content-Encoding");
251 | response.removeHeaders("Content-MD5");
252 | } else {
253 | if (!"identity".equals(codecname)) {
254 | throw new HttpException(
255 | "Unsupported Content-Encoding: " + codec.getName());
256 | }
257 | }
258 | }
259 | }
260 | }
261 | }
262 |
263 | private static InputStream createInputStream(
264 | final long len, final SessionInputBuffer input) {
265 | // Adapted from the org.apache.http.impl.BHttpConnectionBase
266 | if (len == ContentLengthStrategy.CHUNKED) {
267 | return new ChunkedInputStream(input);
268 | } else if (len == ContentLengthStrategy.IDENTITY) {
269 | return new IdentityInputStream(input);
270 | } else if (len == 0L) {
271 | return EmptyInputStream.INSTANCE;
272 | } else {
273 | return new ContentLengthInputStream(input, len);
274 | }
275 | }
276 |
277 | private static HttpEntity getEntity(
278 | final HttpResponse response, final SessionInputBuffer input)
279 | throws HttpException {
280 | // Adapted from the org.apache.http.impl.BHttpConnectionBase
281 | final BasicHttpEntity entity = new BasicHttpEntity();
282 |
283 | final long len =
284 | new LaxContentLengthStrategy().determineLength(response);
285 | final InputStream instream = Warcs.createInputStream(len, input);
286 | if (len == ContentLengthStrategy.CHUNKED) {
287 | entity.setChunked(true);
288 | entity.setContentLength(-1);
289 | entity.setContent(instream);
290 | } else if (len == ContentLengthStrategy.IDENTITY) {
291 | entity.setChunked(false);
292 | entity.setContentLength(-1);
293 | entity.setContent(instream);
294 | } else {
295 | entity.setChunked(false);
296 | entity.setContentLength(len);
297 | entity.setContent(instream);
298 | }
299 |
300 | final Header contentTypeHeader =
301 | response.getFirstHeader(HTTP.CONTENT_TYPE);
302 | if (contentTypeHeader != null) {
303 | entity.setContentType(contentTypeHeader);
304 | }
305 | final Header contentEncodingHeader =
306 | response.getFirstHeader(HTTP.CONTENT_ENCODING);
307 | if (contentEncodingHeader != null) {
308 | entity.setContentEncoding(contentEncodingHeader);
309 | }
310 | return entity;
311 | }
312 |
313 | }
314 |
--------------------------------------------------------------------------------
/src/edu/cmu/lemurproject/WarcRecord.java:
--------------------------------------------------------------------------------
1 | /*
2 | Lemur License Agreement
3 |
4 | Copyright (c) 2000-2011 The Lemur Project. All rights reserved.
5 |
6 | Redistribution and use in source and binary forms, with or without
7 | modification, are permitted provided that the following conditions
8 | are met:
9 |
10 | 1. Redistributions of source code must retain the above copyright
11 | notice, this list of conditions and the following disclaimer.
12 |
13 | 2. Redistributions in binary form must reproduce the above copyright
14 | notice, this list of conditions and the following disclaimer in
15 | the documentation and/or other materials provided with the
16 | distribution.
17 |
18 | 3. The names "Lemur", "Indri", "University of Massachusetts" and
19 | "Carnegie Mellon" must not be used to endorse or promote products
20 | derived from this software without prior written permission. To
21 | obtain permission, contact license@lemurproject.org
22 |
23 | 4. Products derived from this software may not be called "Lemur" or "Indri"
24 | nor may "Lemur" or "Indri" appear in their names without prior written
25 | permission of The Lemur Project. To obtain permission,
26 | contact license@lemurproject.org.
27 |
28 | THIS SOFTWARE IS PROVIDED BY THE LEMUR PROJECT AND OTHER
29 | CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING,
30 | BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
31 | FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
32 | COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
33 | INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
34 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
35 | OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
36 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
37 | TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
38 | USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
39 | DAMAGE.
40 |
41 | */
42 | /*
43 | * To change this template, choose Tools | Templates
44 | * and open the template in the editor.
45 | */
46 | package edu.cmu.lemurproject;
47 |
48 | import java.io.DataInput;
49 | import java.io.DataInputStream;
50 | import java.io.DataOutput;
51 | import java.io.EOFException;
52 | import java.io.IOException;
53 | import java.io.UnsupportedEncodingException;
54 | import java.util.HashMap;
55 | import java.util.Iterator;
56 | import java.util.Map.Entry;
57 | import java.util.Set;
58 | // import org.apache.commons.logging.Log;
59 | // import org.apache.commons.logging.LogFactory;
60 |
61 | /**
62 | *
63 | * @author mhoy
64 | */
65 | public class WarcRecord {
66 |
67 | // public static final Log LOG = LogFactory.getLog(WarcRecord.class);
68 |
69 | public static String WARC_VERSION = "WARC/";
70 | public static String WARC_VERSION_LINE = "WARC/0.18\n";
71 |
72 | ////public static String WARC_VERSION = "WARC/1.0";
73 | //public static String WARC_VERSION = "WARC/0.18";
74 | ////public static String WARC_VERSION_LINE = "WARC/1.0\n";
75 | //public static String WARC_VERSION_LINE = "WARC/0.18\n";
76 | private static String NEWLINE="\n";
77 | private static String CR_NEWLINE="\r\n";
78 |
79 | private static byte MASK_THREE_BYTE_CHAR=(byte)(0xE0);
80 | private static byte MASK_TWO_BYTE_CHAR=(byte)(0xC0);
81 | private static byte MASK_TOPMOST_BIT=(byte)(0x80);
82 | private static byte MASK_BOTTOM_SIX_BITS=(byte)(0x1F);
83 | private static byte MASK_BOTTOM_FIVE_BITS=(byte)(0x3F);
84 | private static byte MASK_BOTTOM_FOUR_BITS=(byte)(0x0F);
85 |
86 | private static String LINE_ENDING="\n";
87 |
88 | private static String readLineFromInputStream(DataInputStream in) throws IOException {
89 | StringBuilder retString=new StringBuilder();
90 | boolean found_cr = false;
91 | boolean keepReading=true;
92 | try {
93 | do {
94 | char thisChar=0;
95 | byte readByte=in.readByte();
96 | // check to see if it's a multibyte character
97 | if ((readByte & MASK_THREE_BYTE_CHAR) == MASK_THREE_BYTE_CHAR) {
98 | found_cr = false;
99 | // need to read the next 2 bytes
100 | if (in.available() < 2) {
101 | // treat these all as individual characters
102 | retString.append((char)readByte);
103 | int numAvailable=in.available();
104 | for (int i=0; i < numAvailable; i++) {
105 | retString.append((char)(in.readByte()));
106 | }
107 | continue;
108 | }
109 | byte secondByte=in.readByte();
110 | byte thirdByte=in.readByte();
111 | // ensure the topmost bit is set
112 | if (((secondByte & MASK_TOPMOST_BIT)!=MASK_TOPMOST_BIT) || ((thirdByte & MASK_TOPMOST_BIT)!=MASK_TOPMOST_BIT)) {
113 | //treat these as individual characters
114 | retString.append((char)readByte);
115 | retString.append((char)secondByte);
116 | retString.append((char)thirdByte);
117 | continue;
118 | }
119 | int finalVal=(thirdByte & MASK_BOTTOM_FIVE_BITS) + 64*(secondByte & MASK_BOTTOM_FIVE_BITS) + 4096*(readByte & MASK_BOTTOM_FOUR_BITS);
120 | thisChar=(char)finalVal;
121 | } else if ((readByte & MASK_TWO_BYTE_CHAR) == MASK_TWO_BYTE_CHAR) {
122 | found_cr = false;
123 |
124 | // need to read next byte
125 | if (in.available() < 1) {
126 | // treat this as individual characters
127 | retString.append((char)readByte);
128 | continue;
129 | }
130 | byte secondByte=in.readByte();
131 | if ((secondByte & MASK_TOPMOST_BIT)!=MASK_TOPMOST_BIT) {
132 | retString.append((char)readByte);
133 | retString.append((char)secondByte);
134 | continue;
135 | }
136 | int finalVal=(secondByte & MASK_BOTTOM_FIVE_BITS) + 64*(readByte & MASK_BOTTOM_SIX_BITS);
137 | thisChar=(char)finalVal;
138 | } else {
139 | // interpret it as a single byte
140 | thisChar=(char)readByte;
141 | }
142 | // Look for carriage return; if found set a flag
143 | if (thisChar=='\r') {
144 | found_cr = true;
145 | }
146 | if (thisChar=='\n') {
147 | // if the linefeed is the next character after the carriage return
148 | if (found_cr) {
149 | LINE_ENDING = CR_NEWLINE;
150 | } else {
151 | LINE_ENDING = NEWLINE;
152 | }
153 | keepReading=false;
154 | } else {
155 | retString.append(thisChar);
156 | }
157 | } while (keepReading);
158 | } catch (EOFException eofEx) {
159 | return null;
160 | }
161 |
162 | if (retString.length()==0) {
163 | return "";
164 | }
165 |
166 | return retString.toString();
167 | }
168 |
169 | private static byte[] readNextRecord(DataInputStream in, StringBuffer headerBuffer) throws IOException {
170 | if (in==null) { return null; }
171 | if (headerBuffer==null) { return null; }
172 |
173 | String line=null;
174 | boolean foundMark=false;
175 | byte[] retContent=null;
176 |
177 | // cannot be using a buffered reader here!!!!
178 | // just read the header
179 | // first - find our WARC header
180 | while ((!foundMark) && ((line=readLineFromInputStream(in))!=null)) {
181 | if (line.startsWith(WARC_VERSION)) {
182 | WARC_VERSION_LINE = line;
183 | foundMark=true;
184 | }
185 | }
186 |
187 | // no WARC mark?
188 | if (!foundMark) { return null; }
189 |
190 | // LOG.info("Found WARC_VERSION");
191 |
192 | int contentLength = -1;
193 | // read until we see contentLength then an empty line
194 | // (to handle malformed ClueWeb09 headers that have blank lines)
195 | // get the content length and set our retContent
196 | for (line = readLineFromInputStream(in).trim();
197 | line.length() > 0 || contentLength < 0;
198 | line = readLineFromInputStream(in).trim()) {
199 |
200 | if (line.length() > 0 ) {
201 | headerBuffer.append(line);
202 | headerBuffer.append(LINE_ENDING);
203 |
204 | // find the content length designated by Content-Length:
205 | String[] parts = line.split(":", 2);
206 | if (parts.length == 2 && parts[0].equals("Content-Length")) {
207 | try {
208 | contentLength=Integer.parseInt(parts[1].trim());
209 | // LOG.info("WARC record content length: " + contentLength);
210 | } catch (NumberFormatException nfEx) {
211 | contentLength=-1;
212 | }
213 | }
214 | }
215 | }
216 |
217 | // now read the bytes of the content
218 | retContent=new byte[contentLength];
219 | int totalWant=contentLength;
220 | int totalRead=0;
221 | //
222 | // LOOP TO REMOVE LEADING CR * LF
223 | // To prevent last few characters from being cut off of the content
224 | // when reading
225 | //
226 | while ((totalRead == 0) && (totalRead < contentLength)) {
227 | byte CR = in.readByte();
228 | byte LF = in.readByte();
229 | if ((CR != 13) && (LF != 10)) {
230 | retContent[0] = CR;
231 | retContent[1] = LF;
232 | totalRead = 2;
233 | totalWant = contentLength - totalRead;
234 | }
235 | }
236 | //
237 | //
238 | //
239 | while (totalRead < contentLength) {
240 | try {
241 | int numRead=in.read(retContent, totalRead, totalWant);
242 | if (numRead < 0) {
243 | return null;
244 | } else {
245 | totalRead += numRead;
246 | totalWant = contentLength-totalRead;
247 | } // end if (numRead < 0) / else
248 | } catch (EOFException eofEx) {
249 | // resize to what we have
250 | if (totalRead > 0) {
251 | byte[] newReturn=new byte[totalRead];
252 | System.arraycopy(retContent, 0, newReturn, 0, totalRead);
253 | return newReturn;
254 | } else {
255 | return null;
256 | }
257 | } // end try/catch (EOFException)
258 | } // end while (totalRead < contentLength)
259 |
260 | return retContent;
261 | }
262 |
263 | public static WarcRecord readNextWarcRecord(DataInputStream in) throws IOException {
264 | // LOG.info("Starting read of WARC record");
265 | StringBuffer recordHeader=new StringBuffer();
266 | byte[] recordContent=readNextRecord(in, recordHeader);
267 | if (recordContent==null) {
268 | // LOG.info("WARC content is null - file is complete");
269 | return null;
270 | }
271 |
272 | // extract out our header information
273 | String thisHeaderString=recordHeader.toString();
274 |
275 |
276 | String[] headerLines=thisHeaderString.split(LINE_ENDING);
277 |
278 | WarcRecord retRecord=new WarcRecord();
279 | for (int i=0; i < headerLines.length; i++) {
280 | String[] pieces=headerLines[i].split(":", 2);
281 | if (pieces.length!=2) {
282 | retRecord.addHeaderMetadata(pieces[0], "");
283 | continue;
284 | }
285 | String thisKey=pieces[0].trim();
286 | String thisValue=pieces[1].trim();
287 |
288 | // check for known keys
289 | if (thisKey.equals("WARC-Type")) {
290 | // LOG.info("Setting WARC record type: " + thisValue);
291 | retRecord.setWarcRecordType(thisValue);
292 | } else if (thisKey.equals("WARC-Date")) {
293 | retRecord.setWarcDate(thisValue);
294 | } else if (thisKey.equals("WARC-Record-ID")) {
295 | // LOG.info("Setting WARC record ID: " + thisValue);
296 | retRecord.setWarcUUID(thisValue);
297 | } else if (thisKey.equals("Content-Type")) {
298 | retRecord.setWarcContentType(thisValue);
299 | } else {
300 | retRecord.addHeaderMetadata(thisKey, thisValue);
301 | }
302 | }
303 |
304 | // set the content
305 | retRecord.setContent(recordContent);
306 |
307 | return retRecord;
308 | }
309 |
310 | public class WarcHeader {
311 | public String contentType="";
312 | public String UUID="";
313 | public String dateString="";
314 | public String recordType="";
315 | public HashMap metadata=new HashMap();
316 | public int contentLength=0;
317 |
318 | public WarcHeader() {
319 | }
320 |
321 | public WarcHeader(WarcHeader o) {
322 | this.contentType=o.contentType;
323 | this.UUID=o.UUID;
324 | this.dateString=o.dateString;
325 | this.recordType=o.recordType;
326 | this.metadata.putAll(o.metadata);
327 | this.contentLength=o.contentLength;
328 | }
329 |
330 | public void write(DataOutput out) throws IOException {
331 | out.writeUTF(contentType);
332 | out.writeUTF(UUID);
333 | out.writeUTF(dateString);
334 | out.writeUTF(recordType);
335 | out.writeInt(metadata.size());
336 | Iterator> metadataIterator=metadata.entrySet().iterator();
337 | while (metadataIterator.hasNext()) {
338 | Entry thisEntry=metadataIterator.next();
339 | out.writeUTF(thisEntry.getKey());
340 | out.writeUTF(thisEntry.getValue());
341 | }
342 | out.writeInt(contentLength);
343 | }
344 |
345 | public void readFields(DataInput in) throws IOException {
346 | contentType=in.readUTF();
347 | UUID=in.readUTF();
348 | dateString=in.readUTF();
349 | recordType=in.readUTF();
350 | metadata.clear();
351 | int numMetaItems=in.readInt();
352 | for (int i=0; i < numMetaItems; i++) {
353 | String thisKey=in.readUTF();
354 | String thisValue=in.readUTF();
355 | metadata.put(thisKey, thisValue);
356 | }
357 | contentLength=in.readInt();
358 | }
359 |
360 | @Override
361 | public String toString() {
362 | StringBuffer retBuffer=new StringBuffer();
363 |
364 | retBuffer.append(WARC_VERSION_LINE);
365 | retBuffer.append(LINE_ENDING);
366 |
367 | retBuffer.append("WARC-Type: " + recordType + LINE_ENDING);
368 | retBuffer.append("WARC-Date: " + dateString + LINE_ENDING);
369 |
370 | Iterator> metadataIterator=metadata.entrySet().iterator();
371 | while (metadataIterator.hasNext()) {
372 | Entry thisEntry=metadataIterator.next();
373 | retBuffer.append(thisEntry.getKey());
374 | retBuffer.append(": ");
375 | retBuffer.append(thisEntry.getValue());
376 | retBuffer.append(LINE_ENDING);
377 | }
378 | // Keep this as the last WARC-...
379 | retBuffer.append("WARC-Record-ID: " + UUID + LINE_ENDING);
380 |
381 | retBuffer.append("Content-Type: " + contentType + LINE_ENDING);
382 | retBuffer.append("Content-Length: " + contentLength + LINE_ENDING);
383 |
384 | return retBuffer.toString();
385 | }
386 | }
387 |
388 | private WarcHeader warcHeader=new WarcHeader();
389 | private byte[] warcContent=null;
390 | private String warcFilePath="";
391 |
392 | public WarcRecord() {
393 |
394 | }
395 |
396 | public WarcRecord(WarcRecord o) {
397 | this.warcHeader=new WarcHeader(o.warcHeader);
398 | this.warcContent=o.warcContent;
399 | }
400 |
401 | public int getTotalRecordLength() {
402 | int headerLength=warcHeader.toString().length();
403 | return (headerLength + warcContent.length);
404 | }
405 |
406 | public void set(WarcRecord o) {
407 | this.warcHeader=new WarcHeader(o.warcHeader);
408 | this.warcContent=o.warcContent;
409 | }
410 |
411 | public String getWarcFilePath() {
412 | return warcFilePath;
413 | }
414 |
415 | public void setWarcFilePath(String path) {
416 | warcFilePath=path;
417 | }
418 |
419 | public void setWarcRecordType(String recordType) {
420 | warcHeader.recordType=recordType;
421 | }
422 |
423 | public void setWarcContentType(String contentType) {
424 | warcHeader.contentType=contentType;
425 | }
426 |
427 | public void setWarcDate(String dateString) {
428 | warcHeader.dateString=dateString;
429 | }
430 |
431 | public void setWarcUUID(String UUID) {
432 | warcHeader.UUID=UUID;
433 | }
434 |
435 | public void addHeaderMetadata(String key, String value) {
436 | //System.out.println("+-- WarRecord.addHeaderMetadata key=" + key + " value=" + value);
437 | // don't allow addition of known keys
438 | if (key.equals("WARC-Type")) { return; }
439 | if (key.equals("WARC-Date")) { return; }
440 | if (key.equals("WARC-Record-ID")) { return; }
441 | if (key.equals("Content-Type")) { return; }
442 | if (key.equals("Content-Length")) { return; }
443 |
444 | warcHeader.metadata.put(key, value);
445 | }
446 |
447 |
448 | public void clearHeaderMetadata() {
449 | warcHeader.metadata.clear();
450 | }
451 |
452 | public Set> getHeaderMetadata() {
453 | return warcHeader.metadata.entrySet();
454 | }
455 |
456 | public String getHeaderMetadataItem(String key) {
457 | //System.out.println("+++ WarRecord.getHeaderMetadataItem key=" + key); // WARC-Target-URI
458 | if (key.equals("WARC-Type")) { return warcHeader.recordType; }
459 | if (key.equals("WARC-Date")) { return warcHeader.dateString; }
460 | if (key.equals("WARC-Record-ID")) { return warcHeader.UUID; }
461 | if (key.equals("Content-Type")) { return warcHeader.contentType; }
462 | if (key.equals("Content-Length")) { return Integer.toString(warcHeader.contentLength); }
463 |
464 | return warcHeader.metadata.get(key);
465 | }
466 |
467 | public void setContent(byte[] content) {
468 | warcContent=content;
469 | warcHeader.contentLength=content.length;
470 | }
471 |
472 | public void setContent(String content) {
473 | setContent(content.getBytes());
474 | }
475 | public void setContentLength(int len) {
476 | warcHeader.contentLength=len;
477 | }
478 |
479 | public byte[] getContent() {
480 | return warcContent;
481 | }
482 | public byte[] getByteContent() {
483 | return warcContent;
484 | }
485 |
486 | public String getContentUTF8() {
487 | String retString=null;
488 | try {
489 | retString = new String(warcContent, "UTF-8");
490 | } catch (UnsupportedEncodingException ex) {
491 | retString=new String(warcContent);
492 | }
493 | return retString;
494 | }
495 |
496 | public String getHeaderRecordType() {
497 | return warcHeader.recordType;
498 | }
499 |
500 | @Override
501 | public String toString() {
502 | StringBuffer retBuffer=new StringBuffer();
503 | retBuffer.append(warcHeader.toString());
504 | retBuffer.append(LINE_ENDING);
505 | retBuffer.append(new String(warcContent));
506 | return retBuffer.toString();
507 | }
508 |
509 | public String getHeaderString() {
510 | return warcHeader.toString();
511 | }
512 |
513 | public void write(DataOutput out) throws IOException {
514 | warcHeader.write(out);
515 | out.write(warcContent);
516 | }
517 |
518 | public void readFields(DataInput in) throws IOException {
519 | warcHeader.readFields(in);
520 | int contentLengthBytes=warcHeader.contentLength;
521 | warcContent=new byte[contentLengthBytes];
522 | in.readFully(warcContent);
523 | }
524 |
525 | }
526 |
527 |
--------------------------------------------------------------------------------