This is the default SolrWayback about us text.
2 | 3 | The text is intended to have information about the archive. 4 | 5 | Change the property about.text.file in solrwaybackweb.properties to point on a local file with the 6 | full absolute file path. Use HTML markup but with the html/body start and end tag. 7 | 8 | -------------------------------------------------------------------------------- /src/main/webapp/images/schedule-24dp.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/example_rewrite/inline_css.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |Inline CSS
12 | 13 | 14 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "files.exclude": { 4 | "**/.classpath": true, 5 | "**/.project": true, 6 | "**/.settings": true, 7 | "**/.factorypath": true 8 | }, 9 | "editor.codeActionsOnSave": { 10 | "source.fixAll.eslint": true 11 | }, 12 | "eslint.validate": ["javascript", "vue"], 13 | "eslint.workingDirectories": [ 14 | "./src/js" 15 | ] 16 | } -------------------------------------------------------------------------------- /src/js/src/assets/icons/web.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/solr_9/netarchivebuilder/conf/lang/contractions_it.txt: -------------------------------------------------------------------------------- 1 | # Set of Italian contractions for ElisionFilter 2 | # TODO: load this as a resource from the analyzer and sync it in build.xml 3 | c 4 | l 5 | all 6 | dall 7 | dell 8 | nell 9 | sull 10 | coll 11 | pell 12 | gl 13 | agl 14 | dagl 15 | degl 16 | negl 17 | sugl 18 | un 19 | m 20 | t 21 | s 22 | v 23 | d 24 | -------------------------------------------------------------------------------- /src/test/resources/example_rewrite/css_import.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |Multiple CSS includes
11 | 12 | 13 | -------------------------------------------------------------------------------- /src/test/resources/example_rewrite/url_escape_expected.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |
3 | Less
4 |
5 |
6 |
7 | More
8 |
9 |
10 |
11 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/src/test/resources/example_rewrite/cdata.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | CDATA is not needed in HTML script-elements, but browsers are browsers...
7 |Processing this page should not result in any changes.
8 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /src/js/src/assets/icons/location.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/resources/example_rewrite/cdata_expected.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |CDATA is not needed in HTML script-elements, but browsers are browsers...
7 |Processing this page should not result in any changes.
8 | 13 | 14 | -------------------------------------------------------------------------------- /src/js/README.md: -------------------------------------------------------------------------------- 1 | # solrwayback_vue 2 | 3 | ## Project setup 4 | ``` 5 | npm install 6 | ``` 7 | 8 | ### Compiles and hot-reloads for development 9 | ``` 10 | npm run serve 11 | ``` 12 | 13 | ### Compiles and minifies for production 14 | ``` 15 | npm run build 16 | ``` 17 | 18 | ### Lints and fixes files 19 | ``` 20 | npm run lint 21 | ``` 22 | 23 | ### Customize configuration 24 | See [Configuration Reference](https://cli.vuejs.org/config/). 25 | -------------------------------------------------------------------------------- /src/bundle/solr_config/conf/lang/stopwords_hy.txt: -------------------------------------------------------------------------------- 1 | # example set of Armenian stopwords. 2 | այդ 3 | այլ 4 | այն 5 | այս 6 | դու 7 | դուք 8 | եմ 9 | են 10 | ենք 11 | ես 12 | եք 13 | է 14 | էի 15 | էին 16 | էինք 17 | էիր 18 | էիք 19 | էր 20 | ըստ 21 | թ 22 | ի 23 | ին 24 | իսկ 25 | իր 26 | կամ 27 | համար 28 | հետ 29 | հետո 30 | մենք 31 | մեջ 32 | մի 33 | ն 34 | նա 35 | նաև 36 | նրա 37 | նրանք 38 | որ 39 | որը 40 | որոնք 41 | որպես 42 | ու 43 | ում 44 | պիտի 45 | վրա 46 | և 47 | -------------------------------------------------------------------------------- /src/js/src/assets/icons/warc.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/js/src/App.vue: -------------------------------------------------------------------------------- 1 | 2 |
3 |
4 |
6 |
7 |
19 |
20 |
--------------------------------------------------------------------------------
/src/test/java/dk/kb/netarchivesuite/solrwayback/parsers/CssParserReplacerTest.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.parsers;
2 |
3 | public class CssParserReplacerTest {
4 |
5 |
6 | //private static String cssExample=
7 |
8 |
9 | //CSS på følgende side: http://belinda.statsbiblioteket.dk:9721/solrwayback/services/view?source_file_path=/netarkiv-stage/0001/PLIGT/filedir/5065-215-20131114083855-00000-kb-test-har-003.kb.dk.warc.gz&offset=310685392
10 |
11 |
12 |
13 | }
14 |
--------------------------------------------------------------------------------
/src/test/resources/solr_9/netarchivebuilder/conf/lang/stopwords_hy.txt:
--------------------------------------------------------------------------------
1 | # example set of Armenian stopwords.
2 | այդ
3 | այլ
4 | այն
5 | այս
6 | դու
7 | դուք
8 | եմ
9 | են
10 | ենք
11 | ես
12 | եք
13 | է
14 | էի
15 | էին
16 | էինք
17 | էիր
18 | էիք
19 | էր
20 | ըստ
21 | թ
22 | ի
23 | ին
24 | իսկ
25 | իր
26 | կամ
27 | համար
28 | հետ
29 | հետո
30 | մենք
31 | մեջ
32 | մի
33 | ն
34 | նա
35 | նաև
36 | նրա
37 | նրանք
38 | որ
39 | որը
40 | որոնք
41 | որպես
42 | ու
43 | ում
44 | պիտի
45 | վրա
46 | և
47 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/UrlWrapper.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.service.dto;
2 |
3 | import javax.xml.bind.annotation.XmlRootElement;
4 |
5 | @XmlRootElement
6 | public class UrlWrapper {
7 |
8 | private String url;
9 |
10 | public UrlWrapper(){
11 |
12 | }
13 |
14 | public String getUrl() {
15 | return url;
16 | }
17 |
18 | public void setUrl(String url) {
19 | this.url = url;
20 | }
21 |
22 |
23 | }
24 |
--------------------------------------------------------------------------------
/src/js/src/store/modal.store.js:
--------------------------------------------------------------------------------
1 | // Global modal state
2 | import { defineStore } from 'pinia'
3 |
4 | export const useModalStore = defineStore('modal', {
5 |
6 | state: () => ({
7 | showModal: false,
8 | currentModal: ''
9 | }),
10 |
11 | actions: {
12 | updateShowModal(shown) {
13 | this.showModal = shown
14 | },
15 | updateCurrentModal(modal){
16 | this.currentModal = modal
17 | },
18 | resetState(){
19 | this.$reset()
20 | }
21 | }
22 | })
23 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/parsers/LineAndByteCount.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.parsers;
2 |
3 | public class LineAndByteCount {
4 | private String line;
5 | private int byteCount;
6 |
7 | public String getLine() {
8 | return line;
9 | }
10 | public void setLine(String line) {
11 | this.line = line;
12 | }
13 | public int getByteCount() {
14 | return byteCount;
15 | }
16 | public void setByteCount(int byteCount) {
17 | this.byteCount = byteCount;
18 | }
19 |
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/src/test/resources/example_rewrite/style_element.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Some styling
7 | 8 |Hello
9 | World
10 |
11 | paragraph
12 | 13 | -------------------------------------------------------------------------------- /src/test/resources/example_rewrite/css_import_expected.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |Multiple CSS includes
12 | 13 | -------------------------------------------------------------------------------- /src/test/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 |Less han/greater than problems
7 | 19 |Click me!
20 | 21 | -------------------------------------------------------------------------------- /src/test/resources/example_rewrite/script_escape.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |Less han/greater than problems
7 | 8 | 20 |Click me!
21 | -------------------------------------------------------------------------------- /src/js/src/assets/icons/Font_Awesome_5_regular_clipboard.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/js/src/components/ToolboxComponents/NgramNetarchive.vue: -------------------------------------------------------------------------------- 1 | 2 |
3 |
11 |
12 |
13 |
29 |
--------------------------------------------------------------------------------
/src/js/src/store/notifier.store.js:
--------------------------------------------------------------------------------
1 | // Global notifier state
2 | import { defineStore } from 'pinia'
3 | import { useSearchStore } from '../store/search.store'
4 |
5 | export const useNotifierStore = defineStore('notifier', {
6 |
7 | state: () => ({
8 | notifications: []
9 | }),
10 |
11 |
12 | actions: {
13 | setNotification( notification) {
14 | this.notifications.push(notification)
15 | },
16 | dismissNotification ( notification) {
17 | this.notifications.pop()
18 | const search = useSearchStore()
19 | search.setLoadingStatus(false)
20 | },
21 | resetState(){
22 | this.$reset()
23 | }
24 | }
25 |
26 | })
27 |
28 |
--------------------------------------------------------------------------------
/src/main/webapp/WEB-INF/rewrite.config:
--------------------------------------------------------------------------------
1 | # Add this line to tomcat/conf/logging.properties to get debug log for the rewritevalve
2 | # org.apache.catalina.core.ContainerBase.[Catalina].[localhost].level = FINE
3 |
4 | RewriteCond %{REQUEST_URI} !^/solrwayback/static/.*
5 | RewriteCond %{REQUEST_URI} !^/solrwayback/services/.*
6 | RewriteRule ^. /solrwayback_index_page.html [L]
7 | RewriteRule ^.calendar*$ /solrwayback_index_page.html [L]
8 | RewriteRule ^.pageharvestdata*$ /solrwayback_index_page.html [L]
9 | RewriteRule ^.pwid*$ /solrwayback_index_page.html [L]
10 | RewriteRule ^.search*$ /solrwayback_index_page.html [L]
11 | RewriteRule ^.linkgraph*$ /solrwayback_index_page.html [L]
12 |
13 |
14 |
--------------------------------------------------------------------------------
/src/test/java/dk/kb/netarchivesuite/solrwayback/util/SolrUtilsTest.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.util;
2 |
3 | import org.junit.Test;
4 |
5 | import static org.junit.Assert.assertEquals;
6 |
7 | public class SolrUtilsTest {
8 |
9 | @Test
10 | public void combineFilterQueriesTest(){
11 | String[] filtersFromFrontend = new String[]{"filter1:value1", "filter2:value2", "foo:bar OR bar:zoo"};
12 | String filterquery = SolrUtils.combineFilterQueries("content_type", "text/html", filtersFromFrontend);
13 |
14 | assertEquals("(content_type:text/html) AND (filter1:value1) AND (filter2:value2) AND (foo:bar OR bar:zoo)", filterquery);
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/src/test/resources/compressions_warc/README.md:
--------------------------------------------------------------------------------
1 | # Brotli and GZip compression sample WARCs
2 |
3 | The WARCs in this folder are used to test compression support.
4 | Each WARC contains a single HTML-page with the text
5 | _"Extremely simple webpage used for testing GZip and Brotli transmission compression."_.
6 |
7 | The WARCs has been harvested using wget, specifying either none, GZip or Brotli as transmission compression. For completeness this was done with and without WARC-GZip compression. A sample call is
8 | ```
9 | ./wget_latest --delete-after --no-warc-keep-log --header="accept-encoding: br" --warc-file="transfer_compression_brotli" 'http://tokemon.sb.statsbiblioteket.dk'
10 | ```
11 |
12 |
--------------------------------------------------------------------------------
/src/main/webapp/META-INF/context.xml:
--------------------------------------------------------------------------------
1 | 4 | Ngram Netarchive 5 |
6 |
7 |
10 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/src/js/src/configs/index.js:
--------------------------------------------------------------------------------
1 | export default {
2 | playbackConfig: {
3 | alternativePlaybackBaseURL: '',
4 | solrwaybackBaseURL:'',
5 | playbackDisabled:false
6 | },
7 | exportOptions: {
8 | warcAllowed:false,
9 | csvAllowed:false,
10 | csvFields:''
11 | },
12 |
13 | visualizations:{
14 | ngram:{
15 | startYear:''
16 | }
17 | },
18 |
19 | logo:{
20 | url: ''
21 | },
22 |
23 | collection:{
24 | playback: new Map()
25 | },
26 |
27 | search:{
28 | uploadedFileDisabled:false,
29 | pagination:20
30 | },
31 |
32 | leaflet: {
33 | attribution:'',
34 | source:'',
35 | map: {
36 | latitude:'',
37 | longitude:'',
38 | radius:''
39 | }
40 | },
41 | }
--------------------------------------------------------------------------------
/src/js/src/components/modalComponents/SearchGuidelines.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/graph/D3Graph.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.service.dto.graph;
2 |
3 |
4 | import java.util.ArrayList;
5 | import java.util.List;
6 |
7 | import javax.xml.bind.annotation.XmlRootElement;
8 |
9 | @XmlRootElement
10 | public class D3Graph {
11 |
12 | public ListNothing here
14 | 15 | " -------------------------------------------------------------------------------- /src/bundle/solrwayback_bundle.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Check if an argument was provided 3 | if [ -z "$1" ]; then 4 | echo "Usage: $0 {start|stop}" 5 | exit 1 6 | fi 7 | 8 | case "$1" in 9 | start) 10 | echo "Starting solr..." 11 | ./solr-9/bin/solr start -c -m 4g 12 | 13 | echo "Starting SolrWayback in tomcat..." 14 | ./tomcat-9/bin/startup.sh 15 | 16 | printf "\n\nStarted SolrWayback\n" 17 | ;; 18 | stop) 19 | echo "Stopping SolrWayback in tomcat..." 20 | ./tomcat-9/bin/shutdown.sh 21 | 22 | echo "Stopping solr..." 23 | ./solr-9/bin/solr stop 24 | 25 | printf "\n\nStopped SolrWayback\n" 26 | ;; 27 | *) 28 | echo "Invalid option: $1" 29 | echo "Usage: $0 {start|stop}" 30 | exit 2 31 | ;; 32 | esac 33 | -------------------------------------------------------------------------------- /src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/smurf/DateCount.java: -------------------------------------------------------------------------------- 1 | package dk.kb.netarchivesuite.solrwayback.service.dto.smurf; 2 | 3 | import javax.xml.bind.annotation.XmlRootElement; 4 | 5 | @XmlRootElement 6 | public class DateCount { 7 | 8 | private String date; 9 | private long count; 10 | private long total; 11 | 12 | public DateCount(){ 13 | } 14 | 15 | public long getCount() { 16 | return count; 17 | } 18 | 19 | public void setCount(long count) { 20 | this.count = count; 21 | } 22 | 23 | public long getTotal() { 24 | return total; 25 | } 26 | 27 | public void setTotal(long total) { 28 | this.total = total; 29 | } 30 | 31 | public String getDate() { 32 | return date; 33 | } 34 | 35 | public void setDate(String date) { 36 | this.date = date; 37 | } 38 | 39 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .DS_Store 3 | solrwayback.iml 4 | /solrwayback.properties 5 | target 6 | /.classpath 7 | /deployLocalTomcat.sh 8 | /.settings/ 9 | /.project 10 | /deployBelindaTomcat.sh 11 | /deployTeg.sh 12 | /solrwayback.log 13 | /temp/ 14 | /bugs.txt 15 | /deployWindows.bat 16 | 17 | 18 | # Webapp 19 | 20 | src/js/.DS_Store 21 | src/js/node_modules 22 | src/js/dist 23 | 24 | # local env files 25 | src/js/.env.local 26 | src/js/.env.*.local 27 | 28 | # Log files 29 | src/js/npm-debug.log* 30 | src/js/yarn-debug.log* 31 | src/js/yarn-error.log* 32 | 33 | 34 | #deployToTest 35 | src/js/deployToTest.sh 36 | 37 | # Editor directories and files 38 | src/js/.idea 39 | src/js/.vscode 40 | src/js/*.suo 41 | src/js/*.ntvs* 42 | src/js/*.njsproj 43 | src/js/*.sln 44 | src/js/*.sw? 45 | 46 | 47 | /deployTegLocal.sh 48 | -------------------------------------------------------------------------------- /src/bundle/solr_config/conf/stopwords.txt: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one or more 2 | # contributor license agreements. See the NOTICE file distributed with 3 | # this work for additional information regarding copyright ownership. 4 | # The ASF licenses this file to You under the Apache License, Version 2.0 5 | # (the "License"); you may not use this file except in compliance with 6 | # the License. You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | -------------------------------------------------------------------------------- /src/test/resources/example_rewrite/script2.html: -------------------------------------------------------------------------------- 1 | 2 | 3 |Modified from a specific page in the Danish net Archive
7 | 8 | 17 | -------------------------------------------------------------------------------- /src/main/java/dk/kb/netarchivesuite/solrwayback/service/SolrWaybackApplication.java: -------------------------------------------------------------------------------- 1 | package dk.kb.netarchivesuite.solrwayback.service; 2 | 3 | import java.util.Arrays; 4 | import java.util.HashSet; 5 | import java.util.Set; 6 | 7 | import javax.ws.rs.core.Application; 8 | 9 | import com.fasterxml.jackson.jaxrs.json.JacksonJsonProvider; 10 | 11 | import dk.kb.netarchivesuite.solrwayback.service.exception.ServiceExceptionMapper; 12 | 13 | 14 | public class SolrWaybackApplication extends Application { 15 | 16 | public Set
3 |
4 |
19 |
20 |
21 |
35 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/parsers/json/TweetMedia.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.parsers.json;
2 |
3 | public class TweetMedia {
4 | private String mediaUrl;
5 |
6 | private String type;
7 |
8 | private TweetVideoInfo videoInfo;
9 |
10 | public TweetMedia() {
11 | }
12 |
13 | public String getMediaUrl() {
14 | return mediaUrl;
15 | }
16 |
17 | public void setMediaUrl(String mediaUrl) {
18 | this.mediaUrl = mediaUrl;
19 | }
20 |
21 | public String getType() {
22 | return type;
23 | }
24 |
25 | public void setType(String type) {
26 | this.type = type;
27 | }
28 |
29 | public TweetVideoInfo getVideoInfo() {
30 | return videoInfo;
31 | }
32 |
33 | public void setVideoInfo(TweetVideoInfo videoInfo) {
34 | this.videoInfo = videoInfo;
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/bundle/solr_config/conf/lang/stopwords_el.txt:
--------------------------------------------------------------------------------
1 | # Lucene Greek Stopwords list
2 | # Note: by default this file is used after GreekLowerCaseFilter,
3 | # so when modifying this file use 'σ' instead of 'ς'
4 | ο
5 | η
6 | το
7 | οι
8 | τα
9 | του
10 | τησ
11 | των
12 | τον
13 | την
14 | και
15 | κι
16 | κ
17 | ειμαι
18 | εισαι
19 | ειναι
20 | ειμαστε
21 | ειστε
22 | στο
23 | στον
24 | στη
25 | στην
26 | μα
27 | αλλα
28 | απο
29 | για
30 | προσ
31 | με
32 | σε
33 | ωσ
34 | παρα
35 | αντι
36 | κατα
37 | μετα
38 | θα
39 | να
40 | δε
41 | δεν
42 | μη
43 | μην
44 | επι
45 | ενω
46 | εαν
47 | αν
48 | τοτε
49 | που
50 | πωσ
51 | ποιοσ
52 | ποια
53 | ποιο
54 | ποιοι
55 | ποιεσ
56 | ποιων
57 | ποιουσ
58 | αυτοσ
59 | αυτη
60 | αυτο
61 | αυτοι
62 | αυτων
63 | αυτουσ
64 | αυτεσ
65 | αυτα
66 | εκεινοσ
67 | εκεινη
68 | εκεινο
69 | εκεινοι
70 | εκεινεσ
71 | εκεινα
72 | εκεινων
73 | εκεινουσ
74 | οπωσ
75 | ομωσ
76 | ισωσ
77 | οσο
78 | οτι
79 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/service/exception/NotFoundServiceException.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.service.exception;
2 |
3 | import javax.ws.rs.core.Response;
4 |
5 | public class NotFoundServiceException extends SolrWaybackServiceException {
6 |
7 | private static final long serialVersionUID = 27182818L;
8 | private static final Response.Status responseStatus = Response.Status.NOT_FOUND;
9 |
10 | public NotFoundServiceException() {
11 | super(responseStatus);
12 | }
13 |
14 | public NotFoundServiceException(String message) {
15 | super(message, responseStatus);
16 | }
17 |
18 | public NotFoundServiceException(String message, Throwable cause) {
19 | super(message, cause, responseStatus);
20 | }
21 |
22 | public NotFoundServiceException(Throwable cause) {
23 | super(cause, responseStatus);
24 | }
25 | }
26 |
27 |
--------------------------------------------------------------------------------
/src/test/resources/solr_9/netarchivebuilder/conf/lang/stopwords_el.txt:
--------------------------------------------------------------------------------
1 | # Lucene Greek Stopwords list
2 | # Note: by default this file is used after GreekLowerCaseFilter,
3 | # so when modifying this file use 'σ' instead of 'ς'
4 | ο
5 | η
6 | το
7 | οι
8 | τα
9 | του
10 | τησ
11 | των
12 | τον
13 | την
14 | και
15 | κι
16 | κ
17 | ειμαι
18 | εισαι
19 | ειναι
20 | ειμαστε
21 | ειστε
22 | στο
23 | στον
24 | στη
25 | στην
26 | μα
27 | αλλα
28 | απο
29 | για
30 | προσ
31 | με
32 | σε
33 | ωσ
34 | παρα
35 | αντι
36 | κατα
37 | μετα
38 | θα
39 | να
40 | δε
41 | δεν
42 | μη
43 | μην
44 | επι
45 | ενω
46 | εαν
47 | αν
48 | τοτε
49 | που
50 | πωσ
51 | ποιοσ
52 | ποια
53 | ποιο
54 | ποιοι
55 | ποιεσ
56 | ποιων
57 | ποιουσ
58 | αυτοσ
59 | αυτη
60 | αυτο
61 | αυτοι
62 | αυτων
63 | αυτουσ
64 | αυτεσ
65 | αυτα
66 | εκεινοσ
67 | εκεινη
68 | εκεινο
69 | εκεινοι
70 | εκεινεσ
71 | εκεινα
72 | εκεινων
73 | εκεινουσ
74 | οπωσ
75 | ομωσ
76 | ισωσ
77 | οσο
78 | οτι
79 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/PagePreview.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.service.dto;
2 |
3 | public class PagePreview {
4 |
5 | private String pagePreviewUrl;
6 | private Long crawlDate;
7 | private String solrWaybackUrl;
8 |
9 | public PagePreview(){
10 |
11 | }
12 |
13 | public String getPagePreviewUrl() {
14 | return pagePreviewUrl;
15 | }
16 |
17 | public void setPagePreviewUrl(String pagePreviewUrl) {
18 | this.pagePreviewUrl = pagePreviewUrl;
19 | }
20 |
21 | public Long getCrawlDate() {
22 | return crawlDate;
23 | }
24 |
25 | public void setCrawlDate(Long crawlDate) {
26 | this.crawlDate = crawlDate;
27 | }
28 |
29 | public String getSolrWaybackUrl() {
30 | return solrWaybackUrl;
31 | }
32 |
33 | public void setSolrWaybackUrl(String solrWaybackUrl) {
34 | this.solrWaybackUrl = solrWaybackUrl;
35 | }
36 |
37 |
38 | }
39 |
--------------------------------------------------------------------------------
/src/js/src/components/ngrams/chartsCore/chartHelpers/index.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Helpers for charting.
3 | *
4 | */
5 |
6 | import ChartOptionsCore from '../../chartsCore/chartConfigs/chartOptionsCore'
7 |
8 | export default {
9 |
10 | /**
11 | * Generate labels for chart.
12 | * - Override with own label config as you see fit -
13 | */
14 | getChartLabels: (labels, scale) => {
15 | return ChartOptionsCore.getChartLabels(labels, scale)
16 | },
17 |
18 | /**
19 | * Generate options for chart.
20 | * - Override with own options config as you see fit -
21 | */
22 | getChartOptions: (searchType, scale) => {
23 | return ChartOptionsCore.getChartOptions(searchType, scale)
24 | },
25 |
26 | /**
27 | * Generate dataset for chart.
28 | * - Override with own data set config as you see fit -
29 | */
30 | getChartDataSet(rawData) {
31 | return ChartOptionsCore.getChartDataSet(rawData)
32 | }
33 |
34 | }
35 |
36 |
37 |
38 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/interfaces/IdentityArcFileResolver.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.interfaces;
2 |
3 |
4 | import java.util.Map;
5 |
6 | /**
7 | * @deprecated use {@link RewriteLocationResolver} instead. Its default behaviour works 100% as IdentityArcFileResolver.
8 | */
9 | public class IdentityArcFileResolver implements ArcFileLocationResolverInterface {
10 | /*
11 | * This implementation just returns the same file location as output. Can be used if path to the arc-files is the same as
12 | * the index field: source_file_path
13 | */
14 | @Override
15 | public ArcSource resolveArcFileLocation(String source_file_path) {
16 | return ArcSource.fromFile(source_file_path);
17 | }
18 | @Override
19 | public void setParameters(Map5 | Missing resources 6 |
7 | 8 |9 | No missing resources registered for this harvest 10 |
11 |
12 |
18 | -
13 |
- 14 | {{ resource }} 15 | 16 |
3 |
4 |
17 |
18 |
19 |
32 |
--------------------------------------------------------------------------------
/src/js/src/components/searchSingleItemComponents/SearchSingleItemFocusImage.vue:
--------------------------------------------------------------------------------
1 |
2 | 5 | Maximum time difference for ressources 6 |
7 | 8 |
9 |
16 |
10 | Backward ← {{ harvestTimesData.maximumTimeDifferenceBackward }}
11 |
12 |
13 | Forward → {{ harvestTimesData.maximumTimeDifferenceForward }}
14 |
15 |
7 | ![]()
8 |
11 |
12 |
13 |
14 |
46 |
47 |
48 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/HarvestDates.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.service.dto;
2 |
3 | import java.util.ArrayList;
4 | import java.util.List;
5 |
6 | import javax.xml.bind.annotation.XmlRootElement;
7 |
8 | @XmlRootElement
9 | public class HarvestDates {
10 |
11 | private String url;
12 | private long numberOfHarvests=0;
13 | private ListNothing here
" 14 | 15 | -------------------------------------------------------------------------------- /src/test/resources/solr_9/netarchivebuilder/conf/protwords.txt: -------------------------------------------------------------------------------- 1 | # The ASF licenses this file to You under the Apache License, Version 2.0 2 | # (the "License"); you may not use this file except in compliance with 3 | # the License. You may obtain a copy of the License at 4 | # 5 | # http://www.apache.org/licenses/LICENSE-2.0 6 | # 7 | # Unless required by applicable law or agreed to in writing, software 8 | # distributed under the License is distributed on an "AS IS" BASIS, 9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | # See the License for the specific language governing permissions and 11 | # limitations under the License. 12 | 13 | #----------------------------------------------------------------------- 14 | # Use a protected word file to protect against the stemmer reducing two 15 | # unrelated words to the same base word. 16 | 17 | # Some non-words that normally won't be encountered, 18 | # just to test that they won't be stemmed. 19 | dontstems 20 | zwhacky 21 | 22 | -------------------------------------------------------------------------------- /src/main/java/dk/kb/netarchivesuite/solrwayback/parsers/json/TweetHashtag.java: -------------------------------------------------------------------------------- 1 | package dk.kb.netarchivesuite.solrwayback.parsers.json; 2 | 3 | import com.fasterxml.jackson.annotation.JsonProperty; 4 | import org.apache.commons.lang3.tuple.Pair; 5 | 6 | public class TweetHashtag implements TweetEntity { 7 | private PairModified from a specific page in the Danish net Archive
7 | 16 | 17 | -------------------------------------------------------------------------------- /src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/graph/Link.java: -------------------------------------------------------------------------------- 1 | package dk.kb.netarchivesuite.solrwayback.service.dto.graph; 2 | 3 | import javax.xml.bind.annotation.XmlRootElement; 4 | 5 | @XmlRootElement 6 | public class Link { 7 | 8 | private int source; 9 | private int target; 10 | private int weight = 1; 11 | 12 | public Link(){ 13 | } 14 | 15 | public Link(int source, int target, int weight){ 16 | this.source=source; 17 | this.target=target; 18 | this.weight=weight; 19 | } 20 | 21 | public int getSource() { 22 | return source; 23 | } 24 | 25 | public void setSource(int source) { 26 | this.source = source; 27 | } 28 | 29 | public int getTarget() { 30 | return target; 31 | } 32 | 33 | public void setTarget(int target) { 34 | this.target = target; 35 | } 36 | 37 | public int getWeight() { 38 | return weight; 39 | } 40 | 41 | public void setWeight(int weight) { 42 | this.weight = weight; 43 | } 44 | 45 | 46 | 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/js/src/main.js: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import { createPinia } from 'pinia' 3 | import App from './App.vue' 4 | import router from './router' 5 | import Axios from 'axios' 6 | import VTooltip from 'v-tooltip' 7 | import { setServerConfigInApp } from './configs/configHelper' 8 | 9 | import './assets/styles/main.scss' 10 | 11 | const baseURL = import.meta.env.DEV 12 | ? '/' 13 | : import.meta.env.BASE_URL; 14 | 15 | if (import.meta.env.DEV) { 16 | Axios.defaults.baseURL = '/'; 17 | } 18 | 19 | Axios.get(baseURL + 'services/frontend/properties/solrwaybackweb/') 20 | .then(response => { 21 | setServerConfigInApp(response.data) 22 | 23 | const pinia = createPinia() 24 | const app = createApp(App) 25 | 26 | app.use(pinia) 27 | app.use(router) 28 | app.use(VTooltip) 29 | 30 | app.mount('#app') 31 | }) 32 | .catch(error => { 33 | // TODO - unsure what best to do here (Ben) 34 | console.error("Failed to load server config", error) 35 | }) 36 | 37 | -------------------------------------------------------------------------------- /src/js/src/assets/icons/twitter.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 15 | -------------------------------------------------------------------------------- /src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/statistics/QueryPercentilesStatistics.java: -------------------------------------------------------------------------------- 1 | package dk.kb.netarchivesuite.solrwayback.service.dto.statistics; 2 | 3 | import org.apache.solr.client.solrj.response.FieldStatsInfo; 4 | 5 | import javax.xml.bind.annotation.XmlRootElement; 6 | import java.util.Map; 7 | 8 | @XmlRootElement 9 | public class QueryPercentilesStatistics { 10 | private String name; 11 | private Map
3 |
4 |
7 |
8 |
9 |
10 |
39 |
40 |
41 |
--------------------------------------------------------------------------------
/src/bundle/solrwayback_bundle.bat:
--------------------------------------------------------------------------------
1 | @echo off
2 | REM Check if an argument was provided
3 | IF "%~1"=="" (
4 | echo Usage: %~nx0 ^{start^|stop^}
5 | exit /b 1
6 | )
7 |
8 | REM Main case handling
9 | IF /I "%~1"=="start" (
10 | echo Starting solr...
11 | call solr-9\bin\solr.cmd start -c -m 4g
12 |
13 | echo Starting SolrWayback in tomcat...
14 |
15 | REM Set CATALINA_HOME to the "tomcat-9" folder inside the current directory
16 | set "CATALINA_HOME=%cd%\tomcat-9"
17 |
18 | call tomcat-9\bin\startup.bat
19 |
20 | echo.
21 | echo Started SolrWayback
22 | GOTO :eof
23 | )
24 |
25 | IF /I "%~1"=="stop" (
26 | echo Stopping SolrWayback in tomcat...
27 |
28 | REM Set CATALINA_HOME to the "tomcat-9" folder inside the current directory
29 | set "CATALINA_HOME=%cd%\tomcat-9"
30 | call tomcat-9\bin\shutdown.bat
31 |
32 | echo Stopping solr...
33 | call solr-9\bin\solr.cmd stop -all
34 |
35 | echo.
36 | echo Stopped SolrWayback
37 | GOTO :eof
38 | )
39 |
40 | REM Invalid option
41 | echo Invalid option: %~1
42 | echo Usage: %~nx0 ^{start^|stop^}
43 | exit /b 2
44 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/interfaces/ArcHTTPResolver.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | *
14 | */
15 | package dk.kb.netarchivesuite.solrwayback.interfaces;
16 |
17 | import org.slf4j.Logger;
18 | import org.slf4j.LoggerFactory;
19 |
20 | /**
21 | * This class is only kept for backwards compatibility.
22 | * @deprecated use {@link RewriteLocationResolver} instead.
23 | */
24 | public class ArcHTTPResolver extends RewriteLocationResolver {
25 | private static final Logger log = LoggerFactory.getLogger(ArcHTTPResolver.class);
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/src/js/src/mixins/StringManipulationUtils.js:
--------------------------------------------------------------------------------
1 | export default {
2 | methods: {
3 |
4 | $_displayFacetName(facet) {
5 | return facet.replace('&fq=','').split(':')[0] + ': '
6 | },
7 | $_displayFacetValue(facet) {
8 | let s = facet.split(':')
9 | return s.slice(1, s.length).join(' ').replace(/"/g,'')
10 | },
11 | $_checkDomain(domain) {
12 | // Matches at least 1 dot in the string, and no spaces.
13 | return domain.match(/^[^\s]+\.[^\s]+$/)
14 | },
15 | $_checkDate(date) {
16 | // Matches format YYYY-MM-DD
17 | return date.match(/^([12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01]))$/)
18 | },
19 | $_displayDate(date, timeScale){
20 | // Display only the scale
21 | let end = 0
22 | if (date.length > 0) {
23 | switch (timeScale) {
24 | case 'YEAR':
25 | case 'null':
26 | end = 4
27 | break
28 | case 'MONTH':
29 | end = 7
30 | break
31 | case 'WEEK':
32 | case 'DAY':
33 | default:
34 | end = 10
35 | }}
36 | return date.slice(0, end)
37 | }
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/test/resources/example_rewrite/style_element_expected.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Some styling
8 |
9 | Hello
10 |
11 |
12 | World
13 |
14 |
15 | paragraph
16 | 17 | -------------------------------------------------------------------------------- /src/js/src/mixins/ImageSearchUtils.js: -------------------------------------------------------------------------------- 1 | import HistoryRoutingUtils from './HistoryRoutingUtils' 2 | import { mapStores, mapActions } from 'pinia' 3 | import { useSearchStore } from '../store/search.store' 4 | 5 | export default { 6 | mixins: [HistoryRoutingUtils], 7 | computed: { 8 | // ...mapState({ 9 | // searchAppliedFacets: state => state.Search.searchAppliedFacets, 10 | // solrSettings: state => state.Search.solrSettings, 11 | // }), 12 | ...mapStores(useSearchStore) 13 | }, 14 | methods: { 15 | ...mapActions(useSearchStore, { 16 | updateSolrSettingImgSearch:'updateSolrSettingImgSearch', 17 | }), 18 | $_startPageSearchFromImage(searchItem) { 19 | return '/search?query=' + 'links_images:"' + encodeURIComponent(searchItem) + '"' + '&offset=0&grouping=' + this.searchStore.solrSettings.grouping + '&imgSearch=false&urlSearch=false&facets=' 20 | }, 21 | $_startImageSearchFromImage(searchItem) { 22 | return '/search?query=' + 'hash:"' + encodeURIComponent(searchItem) + '"' + '&offset=0&grouping=' + this.searchStore.solrSettings.grouping + '&imgSearch=false&urlSearch=false&facets=' 23 | }, 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/bundle/solr_config/conf/lang/stopwords_ga.txt: -------------------------------------------------------------------------------- 1 | 2 | a 3 | ach 4 | ag 5 | agus 6 | an 7 | aon 8 | ar 9 | arna 10 | as 11 | b' 12 | ba 13 | beirt 14 | bhúr 15 | caoga 16 | ceathair 17 | ceathrar 18 | chomh 19 | chtó 20 | chuig 21 | chun 22 | cois 23 | céad 24 | cúig 25 | cúigear 26 | d' 27 | daichead 28 | dar 29 | de 30 | deich 31 | deichniúr 32 | den 33 | dhá 34 | do 35 | don 36 | dtí 37 | dá 38 | dár 39 | dó 40 | faoi 41 | faoin 42 | faoina 43 | faoinár 44 | fara 45 | fiche 46 | gach 47 | gan 48 | go 49 | gur 50 | haon 51 | hocht 52 | i 53 | iad 54 | idir 55 | in 56 | ina 57 | ins 58 | inár 59 | is 60 | le 61 | leis 62 | lena 63 | lenár 64 | m' 65 | mar 66 | mo 67 | mé 68 | na 69 | nach 70 | naoi 71 | naonúr 72 | ná 73 | ní 74 | níor 75 | nó 76 | nócha 77 | ocht 78 | ochtar 79 | os 80 | roimh 81 | sa 82 | seacht 83 | seachtar 84 | seachtó 85 | seasca 86 | seisear 87 | siad 88 | sibh 89 | sinn 90 | sna 91 | sé 92 | sí 93 | tar 94 | thar 95 | thú 96 | triúr 97 | trí 98 | trína 99 | trínár 100 | tríocha 101 | tú 102 | um 103 | ár 104 | é 105 | éis 106 | í 107 | ó 108 | ón 109 | óna 110 | ónár 111 | -------------------------------------------------------------------------------- /src/main/resources/about_collection.txt: -------------------------------------------------------------------------------- 1 |About the collection
2 |General Information
3 |
4 | This is the default example template for the 'About The Collection' text.
5 | Change the property collection.text.file in solrwaybackweb.properties to point on a local file with the
6 | full absolute file path. Use HTML markup but with the html/body start and end tag.
7 |
8 |
11 | Our collection has been collected through four primary strategies for collecting. 12 |
13 |14 |
Harvest strategy example
15 |-
16 |
- Cross-sectional collection which takes a snapshot of all Danish domains up to four times a year 17 |
- Selective collection from the following types of websites: all Danish news media (ranging from 12 times daily to weekly), political parties, organisations and associations, ministries and agencies, selected profiles from social media, YouTube videos (for example weekly) 18 |
- Event collection of two or three events annually (for example parliamentary elections or the Corona pandemic) 19 |
- Special collections for example based on research requests 20 |
22 | -------------------------------------------------------------------------------- /src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/smurf/SmurfBuckets.java: -------------------------------------------------------------------------------- 1 | package dk.kb.netarchivesuite.solrwayback.service.dto.smurf; 2 | 3 | 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | import javax.xml.bind.annotation.XmlRootElement; 8 | 9 | @XmlRootElement 10 | public class SmurfBuckets { 11 | 12 | 13 | private List
3 | ![webpage preview]()
15 |
16 |
17 |
18 |
44 |
--------------------------------------------------------------------------------
/src/bundle/solr_config/conf/lang/stopwords_eu.txt:
--------------------------------------------------------------------------------
1 | # example set of basque stopwords
2 | al
3 | anitz
4 | arabera
5 | asko
6 | baina
7 | bat
8 | batean
9 | batek
10 | bati
11 | batzuei
12 | batzuek
13 | batzuetan
14 | batzuk
15 | bera
16 | beraiek
17 | berau
18 | berauek
19 | bere
20 | berori
21 | beroriek
22 | beste
23 | bezala
24 | da
25 | dago
26 | dira
27 | ditu
28 | du
29 | dute
30 | edo
31 | egin
32 | ere
33 | eta
34 | eurak
35 | ez
36 | gainera
37 | gu
38 | gutxi
39 | guzti
40 | haiei
41 | haiek
42 | haietan
43 | hainbeste
44 | hala
45 | han
46 | handik
47 | hango
48 | hara
49 | hari
50 | hark
51 | hartan
52 | hau
53 | hauei
54 | hauek
55 | hauetan
56 | hemen
57 | hemendik
58 | hemengo
59 | hi
60 | hona
61 | honek
62 | honela
63 | honetan
64 | honi
65 | hor
66 | hori
67 | horiei
68 | horiek
69 | horietan
70 | horko
71 | horra
72 | horrek
73 | horrela
74 | horretan
75 | horri
76 | hortik
77 | hura
78 | izan
79 | ni
80 | noiz
81 | nola
82 | non
83 | nondik
84 | nongo
85 | nor
86 | nora
87 | ze
88 | zein
89 | zen
90 | zenbait
91 | zenbat
92 | zer
93 | zergatik
94 | ziren
95 | zituen
96 | zu
97 | zuek
98 | zuen
99 | zuten
100 |
--------------------------------------------------------------------------------
/src/bundle/solr_config/conf/lang/userdict_ja.txt:
--------------------------------------------------------------------------------
1 | #
2 | # This is a sample user dictionary for Kuromoji (JapaneseTokenizer)
3 | #
4 | # Add entries to this file in order to override the statistical model in terms
5 | # of segmentation, readings and part-of-speech tags. Notice that entries do
6 | # not have weights since they are always used when found. This is by-design
7 | # in order to maximize ease-of-use.
8 | #
9 | # Entries are defined using the following CSV format:
10 | # 4 | Preview 5 |
6 | 7 | 8 | Preview of web page could not be loaded - use direct this link 9 |
3 |
13 |
14 |
41 |
--------------------------------------------------------------------------------
/src/js/src/components/ngrams/searchHelper.js:
--------------------------------------------------------------------------------
1 | import Configs from './netarchive/configs'
2 | export default {
3 | handleSearch(queryFromClick, dateFromClick, searchType, scale) {
4 | let url
5 | let gap
6 | let filter
7 | // search on crawl_year if scale is YEAR
8 | if (scale == 'YEAR') {
9 | filter = '&fq=crawl_year:' + dateFromClick.slice(0,4)
10 | } else {
11 | // search on crawl_date otherwise
12 | if (scale == 'WEEK') {
13 | gap = '+7DAYS'
14 | } else {
15 | gap = '+1' + scale
16 | }
17 | filter = '&fq=crawl_date:[' + dateFromClick + 'T00:00:00Z TO ' + dateFromClick + 'T00:00:00Z'+ gap +']'
18 | }
19 | let param = '&grouping=false&imgSearch=false&offset=0&urlSearch=false'
20 | let facets = filter + '&fq=content_type_norm:"html"'
21 | if (searchType === 'tags'){
22 | url = `${Configs.BASE_SEARCH_URL()}?query=${encodeURIComponent('elements_used:"'+ queryFromClick +'"')}${param}&facets=${encodeURIComponent(facets)}`
23 | } else {
24 | url = `${Configs.BASE_SEARCH_URL()}?query=${encodeURIComponent(queryFromClick)}${param}&facets=${encodeURIComponent(facets)}`
25 | }
26 | window.open(url, '_blank')
27 | }
28 | }
--------------------------------------------------------------------------------
/src/bundle/solr_config/conf/synonyms.txt:
--------------------------------------------------------------------------------
1 | # The ASF licenses this file to You under the Apache License, Version 2.0
2 | # (the "License"); you may not use this file except in compliance with
3 | # the License. You may obtain a copy of the License at
4 | #
5 | # http://www.apache.org/licenses/LICENSE-2.0
6 | #
7 | # Unless required by applicable law or agreed to in writing, software
8 | # distributed under the License is distributed on an "AS IS" BASIS,
9 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 | # See the License for the specific language governing permissions and
11 | # limitations under the License.
12 |
13 | #-----------------------------------------------------------------------
14 | ##some test synonym mappings unlikely to appear in real input text
15 | #aaafoo => aaabar
16 | #bbbfoo => bbbfoo bbbbar
17 | #cccfoo => cccbar cccbaz
18 | #fooaaa,baraaa,bazaaa
19 |
20 | ## Some synonym groups specific to this example
21 | #GB,gib,gigabyte,gigabytes
22 | #MB,mib,megabyte,megabytes
23 | #Television, Televisions, TV, TVs
24 | ##notice we use "gib" instead of "GiB" so any WordDelimiterFilter coming
25 | ##after us won't split it into two words.
26 |
27 | ## Synonym mappings can be used for spelling correction too
28 | #pixima => pixma
29 |
30 |
--------------------------------------------------------------------------------
/src/js/src/components/notifications/Notifications.vue:
--------------------------------------------------------------------------------
1 |
2 |
4 |
5 |
12 |
6 |
10 |
11 |
3 |
9 |
10 |
11 |
43 |
44 |
45 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/FacetCount.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.service.dto;
2 |
3 | public class FacetCount {
4 | private long count;
5 | private String value;
6 |
7 |
8 | public FacetCount(){
9 | }
10 |
11 |
12 | public long getCount() {
13 | return count;
14 | }
15 |
16 |
17 | public void setCount(long count) {
18 | this.count = count;
19 | }
20 |
21 |
22 | public String getValue() {
23 | return value;
24 | }
25 |
26 |
27 | public void setValue(String value) {
28 | this.value = value;
29 | }
30 |
31 |
32 | @Override
33 | public int hashCode() {
34 | final int prime = 31;
35 | int result = 1;
36 | result = prime * result + ((value == null) ? 0 : value.hashCode());
37 | return result;
38 | }
39 |
40 |
41 | @Override
42 | public boolean equals(Object obj) {
43 | if (this == obj)
44 | return true;
45 | if (obj == null)
46 | return false;
47 | if (getClass() != obj.getClass())
48 | return false;
49 | FacetCount other = (FacetCount) obj;
50 | if (value == null) {
51 | if (other.value != null)
52 | return false;
53 | } else if (!value.equals(other.value))
54 | return false;
55 | return true;
56 | }
57 |
58 |
59 |
60 | }
61 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/parsers/json/TweetURL.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.parsers.json;
2 |
3 | import com.fasterxml.jackson.annotation.JsonProperty;
4 | import org.apache.commons.lang3.tuple.Pair;
5 |
6 | public class TweetURL implements TweetEntity {
7 | private PairA very simple page
9 | 10 |An absolute link leading to example.com
11 |A relative link leading to subfoldersubfolder
12 |A root-based link leading to a parallel page otherpage.html.
13 | 14 |An absolute referenced image: 
A relatively referenced image: 
A root referenced image (same as the absolute one): 
An image with delayed loading as per JQuery convention: 
| Single cell |
3 |
13 |
14 |
15 |
41 |
42 |
43 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/service/dto/IndexDocShort.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.service.dto;
2 |
3 |
4 | import javax.xml.bind.annotation.XmlRootElement;
5 |
6 |
7 | @XmlRootElement
8 | public class IndexDocShort {
9 |
10 | private long offset;
11 | private String source_file_path;
12 | private String crawlDate; // format 2009-12-09T05:32:50Z
13 | private String url;
14 | private String url_norm;
15 |
16 | public IndexDocShort(){
17 | }
18 |
19 | public long getOffset() {
20 | return offset;
21 | }
22 |
23 | public void setOffset(long offset) {
24 | this.offset = offset;
25 | }
26 |
27 | public String getSource_file_path() {
28 | return source_file_path;
29 | }
30 |
31 | public void setSource_file_path(String source_file_path) {
32 | this.source_file_path = source_file_path;
33 | }
34 |
35 | public String getUrl() {
36 | return url;
37 | }
38 |
39 | public void setUrl(String url) {
40 | this.url = url;
41 | }
42 |
43 | public String getUrl_norm() {
44 | return url_norm;
45 | }
46 |
47 | public void setUrl_norm(String url_norm) {
48 | this.url_norm = url_norm;
49 | }
50 |
51 | public String getCrawlDate() {
52 | return crawlDate;
53 | }
54 |
55 | public void setCrawlDate(String crawlDate) {
56 | this.crawlDate = crawlDate;
57 | }
58 |
59 |
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/src/bundle/solr_config/conf/elevate.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
26 | srcset testing
7 | 8 |An image with absolute srcset: 
An image with mixed casing and spacing:
An image with absolute srcset and post-space:
An image with absolute srcset and substring trickery:
An image with relative srcset: 
An image with relative srcset and substring trickery: 
An image with root srcset: 
Reproduction of specific problem:
3 |
13 |
14 |
15 |
52 |
53 |
54 |
--------------------------------------------------------------------------------
/src/js/src/components/searchSingleItemComponents/searchSingleItemTypes/SearchSingleItemWeb.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
13 |
14 |
15 |
52 |
53 |
54 |
--------------------------------------------------------------------------------
/src/js/src/components/searchSingleItemComponents/searchSingleItemTypes/SearchSingleItemDefault.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
13 |
14 |
15 |
52 |
53 |
54 |
--------------------------------------------------------------------------------
/src/js/src/components/harvestCalendar/HarvestsDay.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
10 |
11 |
12 |
60 |
61 |
62 |
--------------------------------------------------------------------------------
/src/js/src/components/harvestCalendar/plugins/tranformationHelpers.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Calculate activity level linearly between 0 and 4.
3 | * 0 is no activity level at all, 4 is the max level.
4 | */
5 | export function calculateLinearActivityLevel(harvestsInMonth, maximumHarvests) {
6 | if (harvestsInMonth > maximumHarvests * 0.75 && harvestsInMonth <= maximumHarvests) {
7 | return 4
8 | } else if (harvestsInMonth > maximumHarvests * 0.50 && harvestsInMonth <= maximumHarvests * 0.75) {
9 | return 3
10 | } else if (harvestsInMonth > maximumHarvests * 0.25 && harvestsInMonth <= maximumHarvests * 0.50) {
11 | return 2
12 | } else if (harvestsInMonth > 0 && harvestsInMonth <= maximumHarvests * 0.25) {
13 | return 1
14 | }
15 |
16 | return 0
17 | }
18 |
19 |
20 | /**
21 | * Calculate activity level logarithmically.
22 | */
23 | export function calculateLogarithmicActivityLevel(harvestsInMonth, maximumHarvests) {
24 |
25 | const logarithmicResult = getBaseLog(maximumHarvests, harvestsInMonth)
26 |
27 | if (logarithmicResult > 0.75 && logarithmicResult <= 1) {
28 | return 4
29 | } else if (logarithmicResult > 0.50 && logarithmicResult <= 0.75) {
30 | return 3
31 | } else if (logarithmicResult > 0.25 && logarithmicResult <= 0.50) {
32 | return 2
33 | } else if (logarithmicResult > 0 && logarithmicResult <= 0.25) {
34 | return 1
35 | }
36 |
37 | return 0
38 | }
39 |
40 |
41 | /**
42 | * The following function returns the logarithm of y with base x, ie. logx(y):
43 | */
44 | function getBaseLog(x, y) {
45 | return Math.log(y) / Math.log(x)
46 | }
47 |
48 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/util/CountingMap.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | *
14 | */
15 | package dk.kb.netarchivesuite.solrwayback.util;
16 |
17 | import org.apache.commons.logging.LogFactory;
18 | import org.apache.commons.logging.Log;
19 |
20 | import java.util.HashMap;
21 |
22 | /**
23 | * Map that keeps track of the number of successfull and unsuccessfull {@link #get(Object)} calls.
24 | */
25 | public class CountingMapHarvests for {{ formatHumanDate(date) }}
4 |-
5 |
- 6 | {{ formatHumanDateTime(harvest) }} 7 | 8 |
3 |
22 |
23 |
24 |
55 |
56 |
57 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/util/LimitedReader.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed under the Apache License, Version 2.0 (the "License");
3 | * you may not use this file except in compliance with the License.
4 | * You may obtain a copy of the License at
5 | *
6 | * http://www.apache.org/licenses/LICENSE-2.0
7 | *
8 | * Unless required by applicable law or agreed to in writing, software
9 | * distributed under the License is distributed on an "AS IS" BASIS,
10 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 | * See the License for the specific language governing permissions and
12 | * limitations under the License.
13 | *
14 | */
15 | package dk.kb.netarchivesuite.solrwayback.util;
16 |
17 | import java.io.IOException;
18 | import java.io.Reader;
19 |
20 | /**
21 | * Wrapper for a {@link Reader} that limits the amount of characters that are delivered.
22 | * Excess characters are ignored.
23 | */
24 | public class LimitedReader extends Reader {
25 | private final Reader source;
26 | private long charactersLeft;
27 |
28 | public LimitedReader(Reader source, long maxCharacters) {
29 | this.source = source;
30 | charactersLeft = maxCharacters;
31 | }
32 |
33 | @Override
34 | public int read(char[] cbuf, int off, int len) throws IOException {
35 | if (charactersLeft == 0) {
36 | return -1;
37 | }
38 | int newLen = (int) Math.min(charactersLeft, len);
39 | int read = source.read(cbuf, off, newLen);
40 | if (read != -1) {
41 | charactersLeft -= read;
42 | }
43 | return read;
44 | }
45 |
46 | @Override
47 | public void close() throws IOException {
48 | source.close();
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/test/resources/solr_9/netarchivebuilder/conf/lang/stopwords_gl.txt:
--------------------------------------------------------------------------------
1 | # galican stopwords
2 | a
3 | aínda
4 | alí
5 | aquel
6 | aquela
7 | aquelas
8 | aqueles
9 | aquilo
10 | aquí
11 | ao
12 | aos
13 | as
14 | así
15 | á
16 | ben
17 | cando
18 | che
19 | co
20 | coa
21 | comigo
22 | con
23 | connosco
24 | contigo
25 | convosco
26 | coas
27 | cos
28 | cun
29 | cuns
30 | cunha
31 | cunhas
32 | da
33 | dalgunha
34 | dalgunhas
35 | dalgún
36 | dalgúns
37 | das
38 | de
39 | del
40 | dela
41 | delas
42 | deles
43 | desde
44 | deste
45 | do
46 | dos
47 | dun
48 | duns
49 | dunha
50 | dunhas
51 | e
52 | el
53 | ela
54 | elas
55 | eles
56 | en
57 | era
58 | eran
59 | esa
60 | esas
61 | ese
62 | eses
63 | esta
64 | estar
65 | estaba
66 | está
67 | están
68 | este
69 | estes
70 | estiven
71 | estou
72 | eu
73 | é
74 | facer
75 | foi
76 | foron
77 | fun
78 | había
79 | hai
80 | iso
81 | isto
82 | la
83 | las
84 | lle
85 | lles
86 | lo
87 | los
88 | mais
89 | me
90 | meu
91 | meus
92 | min
93 | miña
94 | miñas
95 | moi
96 | na
97 | nas
98 | neste
99 | nin
100 | no
101 | non
102 | nos
103 | nosa
104 | nosas
105 | noso
106 | nosos
107 | nós
108 | nun
109 | nunha
110 | nuns
111 | nunhas
112 | o
113 | os
114 | ou
115 | ó
116 | ós
117 | para
118 | pero
119 | pode
120 | pois
121 | pola
122 | polas
123 | polo
124 | polos
125 | por
126 | que
127 | se
128 | senón
129 | ser
130 | seu
131 | seus
132 | sexa
133 | sido
134 | sobre
135 | súa
136 | súas
137 | tamén
138 | tan
139 | te
140 | ten
141 | teñen
142 | teño
143 | ter
144 | teu
145 | teus
146 | ti
147 | tido
148 | tiña
149 | tiven
150 | túa
151 | túas
152 | un
153 | unha
154 | unhas
155 | uns
156 | vos
157 | vosa
158 | vosas
159 | voso
160 | vosos
161 | vós
162 |
--------------------------------------------------------------------------------
/src/bundle/solr_config/conf/lang/stopwords_cz.txt:
--------------------------------------------------------------------------------
1 | a
2 | s
3 | k
4 | o
5 | i
6 | u
7 | v
8 | z
9 | dnes
10 | cz
11 | tímto
12 | budeš
13 | budem
14 | byli
15 | jseš
16 | můj
17 | svým
18 | ta
19 | tomto
20 | tohle
21 | tuto
22 | tyto
23 | jej
24 | zda
25 | proč
26 | máte
27 | tato
28 | kam
29 | tohoto
30 | kdo
31 | kteří
32 | mi
33 | nám
34 | tom
35 | tomuto
36 | mít
37 | nic
38 | proto
39 | kterou
40 | byla
41 | toho
42 | protože
43 | asi
44 | ho
45 | naši
46 | napište
47 | re
48 | což
49 | tím
50 | takže
51 | svých
52 | její
53 | svými
54 | jste
55 | aj
56 | tu
57 | tedy
58 | teto
59 | bylo
60 | kde
61 | ke
62 | pravé
63 | ji
64 | nad
65 | nejsou
66 | či
67 | pod
68 | téma
69 | mezi
70 | přes
71 | ty
72 | pak
73 | vám
74 | ani
75 | když
76 | však
77 | neg
78 | jsem
79 | tento
80 | článku
81 | články
82 | aby
83 | jsme
84 | před
85 | pta
86 | jejich
87 | byl
88 | ještě
89 | až
90 | bez
91 | také
92 | pouze
93 | první
94 | vaše
95 | která
96 | nás
97 | nový
98 | tipy
99 | pokud
100 | může
101 | strana
102 | jeho
103 | své
104 | jiné
105 | zprávy
106 | nové
107 | není
108 | vás
109 | jen
110 | podle
111 | zde
112 | už
113 | být
114 | více
115 | bude
116 | již
117 | než
118 | který
119 | by
120 | které
121 | co
122 | nebo
123 | ten
124 | tak
125 | má
126 | při
127 | od
128 | po
129 | jsou
130 | jak
131 | další
132 | ale
133 | si
134 | se
135 | ve
136 | to
137 | jako
138 | za
139 | zpět
140 | ze
141 | do
142 | pro
143 | je
144 | na
145 | atd
146 | atp
147 | jakmile
148 | přičemž
149 | já
150 | on
151 | ona
152 | ono
153 | oni
154 | ony
155 | my
156 | vy
157 | jí
158 | ji
159 | mě
160 | mne
161 | jemu
162 | tomu
163 | těm
164 | těmu
165 | němu
166 | němuž
167 | jehož
168 | jíž
169 | jelikož
170 | jež
171 | jakož
172 | načež
173 |
--------------------------------------------------------------------------------
/src/js/src/components/harvestCalendar/harvestDateHelper.js:
--------------------------------------------------------------------------------
1 | /**
2 | * Returns an array of the months in the year (0-11)
3 | */
4 | export function getArrayOfMonths() {
5 | return [...Array(12).keys()] // [0, 1, 2, ..., 11]
6 | }
7 |
8 |
9 | /**
10 | * Returns an array of the harvest for a given month and year.
11 | *
12 | * @param {number} year
13 | * @param {number} month
14 | * @param {Array
4 |
8 |
9 |
20 |
21 | 5 | {{ year }} 6 |
7 || 14 | | 15 | 16 | | 17 |
3 |
10 |
11 |
12 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/src/test/java/dk/kb/netarchivesuite/solrwayback/parsers/TestExportArc.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.parsers;
2 |
3 | import java.nio.file.Files;
4 | import java.nio.file.Path;
5 | import java.nio.file.Paths;
6 | import java.nio.file.StandardOpenOption;
7 |
8 | import dk.kb.netarchivesuite.solrwayback.UnitTestUtils;
9 | import dk.kb.netarchivesuite.solrwayback.interfaces.ArcSource;
10 | import dk.kb.netarchivesuite.solrwayback.properties.PropertiesLoader;
11 | import dk.kb.netarchivesuite.solrwayback.service.dto.ArcEntry;
12 |
13 | public class TestExportArc {
14 |
15 | public static void main (String[] args) throws Exception{
16 |
17 | PropertiesLoader.initProperties(UnitTestUtils.getFile("properties/solrwayback_unittest.properties").getPath());
18 |
19 |
20 | String arcFile="/media/teg/1200GB_SSD/netarkiv/0205/filedir/27119-33-20080401194737-00004-kb-prod-har-001.kb.dk.arc.gz";
21 | long offset=10776284;
22 |
23 |
24 |
25 | ArcEntry arcEntry = ArcParser.getArcEntry(ArcSource.fromFile(arcFile), offset);
26 |
27 | String warcHeader = ArcHeader2WarcHeader.arcHeader2WarcHeader(arcEntry);
28 |
29 |
30 | Path exportPath = Paths.get("arc2warc-pdf-error.warc");
31 |
32 | try{
33 | Files.delete(exportPath);
34 | }
35 | catch(Exception e){
36 |
37 | }
38 | Files.createFile(exportPath);
39 | System.out.println(arcEntry.getHeader());
40 | System.out.println("-----");
41 | System.out.println(warcHeader);
42 |
43 | Files.write(exportPath, warcHeader.getBytes(WarcParser.WARC_HEADER_ENCODING), StandardOpenOption.APPEND);
44 | Files.write(exportPath, arcEntry.getBinaryDecodedBytes(), StandardOpenOption.APPEND);
45 | Files.write(exportPath, "\r\n\r\n".getBytes(WarcParser.WARC_HEADER_ENCODING), StandardOpenOption.APPEND); // separator
46 |
47 | }
48 |
49 | }
50 |
--------------------------------------------------------------------------------
/src/js/src/components/modalComponents/PrimaryModal.vue:
--------------------------------------------------------------------------------
1 |
2 | Applied facets
4 |
5 | {{ $_displayFacetName(item) }}{{ $_displayFacetValue(item) }}
8 |
9 |
6 |
9 |
16 |
17 |
18 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/playback/HtmlPlayback.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.playback;
2 |
3 | import org.slf4j.Logger;
4 | import org.slf4j.LoggerFactory;
5 |
6 | import dk.kb.netarchivesuite.solrwayback.parsers.ParseResult;
7 | import dk.kb.netarchivesuite.solrwayback.parsers.HtmlParserUrlRewriter;
8 | import dk.kb.netarchivesuite.solrwayback.parsers.WaybackToolbarInjecter;
9 | import dk.kb.netarchivesuite.solrwayback.service.dto.ArcEntry;
10 | import dk.kb.netarchivesuite.solrwayback.service.dto.IndexDoc;
11 |
12 | public class HtmlPlayback extends PlaybackHandler{
13 |
14 | private static final Logger log = LoggerFactory.getLogger(HtmlPlayback.class);
15 |
16 | public HtmlPlayback(ArcEntry arc, IndexDoc doc, boolean showToolbar){
17 | super(arc,doc,showToolbar);
18 | }
19 |
20 | @Override
21 | public ArcEntry playback(boolean lenient) throws Exception{
22 | log.debug(" Generate webpage from FilePath:{} offset:{} content encoding:{} lenient:{}",
23 | doc.getSource_file_path(), doc.getOffset(), arc.getContentEncoding(), lenient);
24 | long start = System.currentTimeMillis();
25 |
26 |
27 | ParseResult htmlReplaced = HtmlParserUrlRewriter.replaceLinks(arc, lenient);
28 | String textReplaced=htmlReplaced.getReplaced();
29 |
30 | boolean xhtml = doc.getContentType().toLowerCase().contains("application/xhtml");
31 | //Inject tooolbar
32 | if (showToolbar ){ //If true or null.
33 | textReplaced = WaybackToolbarInjecter.injectWaybacktoolBar(doc.getSource_file_path(),doc.getOffset(),htmlReplaced , xhtml);
34 | }
35 |
36 | arc.setStringContent(textReplaced);
37 |
38 | log.info("Generating webpage total processing:"+(System.currentTimeMillis()-start) + " "+doc.getSource_file_path()+ " "+ doc.getOffset() +" "+arc.getUrl());
39 | arc.setHasBeenDecompressed(true);
40 | return arc;
41 | }
42 |
43 | }
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/encoders/Sha1Hash.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.encoders;
2 |
3 | import java.io.File;
4 | import java.io.FileInputStream;
5 | import java.io.InputStream;
6 | import java.security.MessageDigest;
7 | public class Sha1Hash {
8 |
9 | public static String createSha1(File file) throws Exception {
10 | InputStream fis = null;
11 | try{
12 | fis = new FileInputStream(file);
13 | MessageDigest digest = MessageDigest.getInstance("SHA-1");
14 | int n = 0;
15 | byte[] buffer = new byte[8192];
16 | while (n != -1) {
17 | n = fis.read(buffer);
18 | if (n > 0) {
19 | digest.update(buffer, 0, n);
20 | }
21 | }
22 | String hash = "sha1:" + Base32Encoder.encode( digest.digest());
23 | return hash;
24 | }
25 | catch(Exception e){
26 | throw e;
27 | }
28 | finally{
29 | if (fis!=null){
30 | fis.close();
31 | }
32 | }
33 | }
34 |
35 | public static String createSha1(InputStream fis) throws Exception {
36 | try{
37 | MessageDigest digest = MessageDigest.getInstance("SHA-1");
38 | int n = 0;
39 | byte[] buffer = new byte[8192];
40 | while (n != -1) {
41 | n = fis.read(buffer);
42 | if (n > 0) {
43 | digest.update(buffer, 0, n);
44 | }
45 | }
46 | String hash = "sha1:" + Base32Encoder.encode( digest.digest());
47 | return hash;
48 | }
49 | catch(Exception e){
50 | throw e;
51 | }
52 | finally{
53 | if (fis!=null){
54 | fis.close();
55 | }
56 | }
57 | }
58 | }
59 |
--------------------------------------------------------------------------------
/src/test/java/dk/kb/netarchivesuite/solrwayback/parsers/warc/ArcGzParserTest.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.parsers.warc;
2 |
3 | import static org.junit.Assert.*;
4 |
5 | import java.io.File;
6 |
7 | import org.junit.Test;
8 |
9 | import dk.kb.netarchivesuite.solrwayback.UnitTestUtils;
10 | import dk.kb.netarchivesuite.solrwayback.facade.Facade;
11 | import dk.kb.netarchivesuite.solrwayback.service.dto.ArcEntry;
12 |
13 |
14 | public class ArcGzParserTest extends UnitTestUtils {
15 |
16 | @Test
17 | public void testArcGzParserHtml() throws Exception {
18 |
19 | File file = getFile("src/test/resources/example_arc/IAH-20080430204825-00000-blackbook.arc.gz");
20 |
21 | ArcEntry arcEntry = Facade.getArcEntry(file.getCanonicalPath(), 1306); //HTML entry
22 | assertEquals("text/html", arcEntry.getContentType());
23 | assertEquals("www.archive.org", arcEntry.getFileName());
24 | assertEquals(366, arcEntry.getContentLength()); //From header
25 | assertEquals(366,arcEntry.getBinaryDecodedBytes().length); //Actually loaded in binary
26 | assertEquals(200,arcEntry.getStatus_code());
27 | //System.out.println(new String(arcEntry.getBinary())); //from to
28 | }
29 |
30 |
31 | @Test
32 | public void testArcGzParserImage() throws Exception {
33 |
34 | File file = getFile("src/test/resources/example_arc/IAH-20080430204825-00000-blackbook.arc.gz");
35 |
36 | ArcEntry arcEntry = Facade.getArcEntry(file.getCanonicalPath(), 7733); //Image entry (or 9699)
37 | assertEquals("image/jpeg", arcEntry.getContentType());
38 | assertEquals("logoc.jpg", arcEntry.getFileName());
39 | assertEquals(1662, arcEntry.getContentLength()); //From header
40 | assertEquals(1662,arcEntry.getBinaryDecodedBytes().length); //Actually loaded in binary
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/test/java/dk/kb/netarchivesuite/solrwayback/interfaces/FileMovedMappingResolverTest.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.interfaces;
2 |
3 | import static org.junit.Assert.assertEquals;
4 |
5 | import java.io.File;
6 | import org.junit.Test;
7 |
8 | import dk.kb.netarchivesuite.solrwayback.UnitTestUtils;
9 |
10 | public class FileMovedMappingResolverTest extends UnitTestUtils {
11 |
12 | @Test
13 | public void testFileMovedMappingResolver() throws Exception {
14 | File file = getFile("src/test/resources/arc_resolvers/FileMovedMappingTest.txt");
15 |
16 | FileMovedMappingResolver resolver = new FileMovedMappingResolver();
17 | resolver.setMappingFile(file.getCanonicalPath());
18 | resolver.initialize();
19 |
20 | //Some warc-files not defined in the moved list
21 | String warc1="/abc/test/example.warc";
22 | String warc2="/netarchivemount/warcs/111/example111.warc";
23 | assertEquals(warc1, resolver.resolveArcFileLocation(warc1).getSource());
24 | assertEquals(warc2, resolver.resolveArcFileLocation(warc2).getSource());
25 |
26 | //These two has been moved
27 | String warc3="/home/xxx/solrwayback_package_4.2.1/indexing/warcs1/356548-347-20210201093000132-00000-sb-prod-har-001.statsbiblioteket.dk.warc.gz";
28 | String warc4="/mount/netarchive/test-00000.warc.gz";
29 | String warc3_moved= resolver.resolveArcFileLocation("/home/old/location/356548-347-20210201093000132-00000-sb-prod-har-001.statsbiblioteket.dk.warc.gz").getSource();
30 | warc3_moved = warc3_moved.replace("\\", "/"); //When running unittest on windows...
31 | assertEquals(warc3, warc3_moved);
32 | String warc4_moved= resolver.resolveArcFileLocation("/oldlocation/test-00000.warc.gz").getSource();
33 | warc4_moved = warc4_moved.replace("\\", "/"); //When running unittest on windows...
34 | assertEquals(warc4,warc4_moved);
35 |
36 |
37 |
38 | }
39 |
40 | }
41 |
--------------------------------------------------------------------------------
/src/main/java/dk/kb/netarchivesuite/solrwayback/playback/JodelPlayback.java:
--------------------------------------------------------------------------------
1 | package dk.kb.netarchivesuite.solrwayback.playback;
2 |
3 | import org.slf4j.Logger;
4 | import org.slf4j.LoggerFactory;
5 |
6 |
7 | import dk.kb.netarchivesuite.solrwayback.parsers.ParseResult;
8 | import dk.kb.netarchivesuite.solrwayback.parsers.WaybackToolbarInjecter;
9 | import dk.kb.netarchivesuite.solrwayback.service.dto.ArcEntry;
10 | import dk.kb.netarchivesuite.solrwayback.service.dto.IndexDoc;
11 |
12 | /*
13 | * This is for JSON harvesting from Jodel. (There is no HTML harvest for Jodel)
14 | */
15 | public class JodelPlayback extends PlaybackHandler{
16 |
17 | private static final Logger log = LoggerFactory.getLogger(JodelPlayback.class);
18 |
19 | public JodelPlayback(ArcEntry arc, IndexDoc doc, boolean showToolbar){
20 | super(arc,doc,showToolbar);
21 | }
22 |
23 | @Override
24 | public ArcEntry playback(boolean lenient) throws Exception{
25 | log.debug(" Generate Jodel post from FilePath:" + doc.getSource_file_path() + " offset:" + doc.getOffset());
26 | //Fake html into arc.
27 |
28 | String json = arc.getStringContentAsStringSafe();
29 | String html = "TODO";//Jodel2Html.render(json, arc.getCrawlDate());
30 | arc.setStringContent(html);
31 | arc.setContentType("text/html");
32 | ParseResult htmlReplaced = new ParseResult(); //Do not parse.
33 | htmlReplaced.setReplaced(html);
34 | String textReplaced=htmlReplaced.getReplaced(); //TODO count linkes found, replaced
35 |
36 | //Inject tooolbar
37 | if (showToolbar){ //If true or null.
38 | textReplaced = WaybackToolbarInjecter.injectWaybacktoolBar(doc,htmlReplaced, false);
39 | }
40 | String encoding="UTF-8"; // hack, since the HTML was generated as UTF-8.
41 | arc.setContentEncoding(encoding);
42 | arc.setStringContent(textReplaced); //can give error. uses UTF-8 (from index) instead of ISO-8859-1
43 |
44 | return arc;
45 | }
46 |
47 | }
48 |
--------------------------------------------------------------------------------
10 |
15 |