[a-wA-Z0-9_\-]+)(?P=quote)",
16 | re.ASCII,
17 | )
18 |
19 | DEFAULT_ENCODING_ALIASES = {
20 | "ansi": "windows-1252",
21 | "65001": "utf-8",
22 | "iso-utf-8": "utf-8",
23 | "u": "utf-8",
24 | "unicode": "utf-8",
25 | "utf-8": "utf-8",
26 | "utf-08": "utf-8",
27 | "utf-f": "utf-8",
28 | "utp-8": "utf-8",
29 | "windows-8859-1": "iso-8859-1",
30 | "iso88591": "iso-8859-1",
31 | }
32 |
33 | ENCODING_ALIASES = {}
34 |
35 |
36 | def set_encoding_aliases(aliases: dict[str, str]):
37 | """Set the encoding aliases to use to decode"""
38 | ENCODING_ALIASES.clear()
39 | ENCODING_ALIASES.update({**DEFAULT_ENCODING_ALIASES, **aliases})
40 |
41 |
42 | def get_encoding_by_alias(alias: str) -> str:
43 | """Get the encoding method for alias."""
44 | key = alias.lower().strip()
45 | return ENCODING_ALIASES.get(key, key)
46 |
47 |
48 | def get_version():
49 | return __version__
50 |
51 |
52 | def get_record_url(record) -> str:
53 | """Check if record has url converted from POST/PUT, and if so, use that
54 | otherwise return the target url"""
55 | if hasattr(record, "urlkey"):
56 | return record.urlkey
57 | return str(record.rec_headers["WARC-Target-URI"])
58 |
59 |
60 | def get_status_code(record: ArcWarcRecord) -> HTTPStatus | int | None:
61 | """Get the HTTP status of a given ArcWarcRecord
62 |
63 | Returns HTTPStatus value or None if status code is not found / supported
64 | """
65 | if record.rec_type == "response":
66 | status_code = record.http_headers.get_statuscode()
67 | else:
68 | status_code = record.rec_headers.get_statuscode()
69 |
70 | if status_code is None or status_code.strip() == "":
71 | # null / missing http status found, ignore it
72 | return None
73 |
74 | status_code = int(status_code)
75 |
76 | try:
77 | status_code = HTTPStatus(status_code)
78 | except ValueError:
79 | # invalid http status found, ignore it (happens when bad http status is
80 | # returned, e.g 0, 306)
81 | return status_code
82 |
83 | return status_code
84 |
85 |
86 | def can_process_status_code(status_code: HTTPStatus | int | None) -> bool:
87 | """Return a boolean indicating if this status code is a processable redirect"""
88 | return isinstance(status_code, HTTPStatus) and not (
89 | status_code.is_informational # not supposed to exist in WARC files
90 | or status_code.is_client_error
91 | or status_code.is_server_error
92 | or (
93 | status_code.is_success
94 | and status_code
95 | not in [
96 | HTTPStatus.OK,
97 | HTTPStatus.CREATED,
98 | HTTPStatus.ACCEPTED,
99 | HTTPStatus.NON_AUTHORITATIVE_INFORMATION,
100 | ]
101 | )
102 | or (
103 | status_code.is_redirection
104 | and status_code
105 | not in [
106 | HTTPStatus.MOVED_PERMANENTLY,
107 | HTTPStatus.FOUND,
108 | HTTPStatus.TEMPORARY_REDIRECT,
109 | HTTPStatus.PERMANENT_REDIRECT,
110 | ]
111 | )
112 | )
113 |
114 |
115 | def status_code_is_processable_redirect(status_code: HTTPStatus | int | None) -> bool:
116 | """Return a boolean indicating if this status code is processable redirect"""
117 | return isinstance(status_code, HTTPStatus) and status_code in [
118 | HTTPStatus.MOVED_PERMANENTLY,
119 | HTTPStatus.FOUND,
120 | HTTPStatus.TEMPORARY_REDIRECT,
121 | HTTPStatus.PERMANENT_REDIRECT,
122 | ]
123 |
124 |
125 | def get_record_content_type(record: ArcWarcRecord) -> str:
126 | if record.http_headers:
127 | # if the record has HTTP headers, use the Content-Type from those
128 | # (eg. 'response' record)
129 | content_type = record.http_headers["Content-Type"]
130 | else:
131 | # otherwise, use the Content-Type from WARC headers
132 | content_type = record.rec_headers["Content-Type"]
133 | return content_type or ""
134 |
135 |
136 | def get_record_mime_type(record: ArcWarcRecord) -> str:
137 | content_type = get_record_content_type(record)
138 | return content_type.split(";")[0]
139 |
140 |
141 | def parse_title(content):
142 | try:
143 | soup = BeautifulSoup(content, "html.parser")
144 | return soup.title.text or "" # pyright: ignore[reportOptionalMemberAccess]
145 | except Exception:
146 | return ""
147 |
148 |
149 | def get_record_encoding(record: ArcWarcRecord) -> str | None:
150 | content_type = get_record_content_type(record)
151 | if m := ENCODING_RE.search(content_type):
152 | return m.group("encoding")
153 |
154 |
155 | def to_string(
156 | input_: str | bytes,
157 | http_encoding: str | None,
158 | charsets_to_try: list[str],
159 | content_header_bytes_length: int,
160 | *,
161 | ignore_content_header_charsets: bool,
162 | ignore_http_header_charsets: bool,
163 | ) -> str:
164 | """
165 | Decode content to string based on charset declared in content or fallback.
166 |
167 | This method tries to not be smarter than necessary.
168 |
169 | First, it tries to find an charset declaration inside the first bytes of the content
170 | (hopping that content first bytes can be losely decoded using few known encoding to
171 | something usable). If found, it is used to decode and any bad character is
172 | automatically replaced, assuming document editor is right.
173 |
174 | Second, if no charset declaration has been found in content, it uses the charset
175 | declared in HTTP `Content-Type` header. This is passed to this method as
176 | `http_encoding` argument. If present, it is used to decode and any bad character is
177 | automatically replaced, assuming web server is right.
178 |
179 | Finally, we fallback to use `charsets_to_try` argument, which is a list of charsets
180 | to try. Each charset is tried in order, but any bad character found is raising an
181 | error. If none of these charsets achieves to decode the content, an exception is
182 | raised.
183 |
184 | Returns the decoded content.
185 |
186 | """
187 |
188 | if isinstance(input_, str):
189 | return input_
190 |
191 | if not input_:
192 | # Empty bytes are easy to decode
193 | return ""
194 |
195 | # Search for encoding from content first bytes based on regexp
196 | if not ignore_content_header_charsets:
197 | for encoding in ["ascii", "utf-16", "utf-32"]:
198 | content_start = input_[:content_header_bytes_length].decode(
199 | encoding, errors="replace"
200 | )
201 | if m := ENCODING_RE.search(content_start):
202 | head_encoding = m.group("encoding")
203 | return input_.decode(
204 | get_encoding_by_alias(head_encoding),
205 | errors="replace",
206 | )
207 |
208 | # Search for encofing in HTTP `Content-Type` header
209 | if not ignore_http_header_charsets and http_encoding:
210 | return input_.decode(get_encoding_by_alias(http_encoding), errors="replace")
211 |
212 | # Try all charsets_to_try passed
213 | for charset_to_try in charsets_to_try:
214 | try:
215 | return input_.decode(get_encoding_by_alias(charset_to_try))
216 | except (ValueError, LookupError):
217 | pass
218 |
219 | raise ValueError(f"No suitable charset found to decode content {input_[:200]}")
220 |
221 |
222 | def get_record_content(record: ArcWarcRecord) -> bytes:
223 | if hasattr(record, "buffered_stream"):
224 | stream = (
225 | record.buffered_stream # pyright: ignore [reportGeneralTypeIssues, reportAttributeAccessIssue]
226 | )
227 | stream.seek(0)
228 | return stream.read()
229 | else:
230 | return record.content_stream().read()
231 |
--------------------------------------------------------------------------------
/tasks.py:
--------------------------------------------------------------------------------
1 | # pyright: strict, reportUntypedFunctionDecorator=false
2 | import os
3 |
4 | from invoke.context import Context
5 | from invoke.tasks import task # pyright: ignore [reportUnknownVariableType]
6 |
7 | use_pty = not os.getenv("CI", "")
8 |
9 |
10 | @task(optional=["args"], help={"args": "pytest additional arguments"})
11 | def test(ctx: Context, args: str = ""):
12 | """run tests (without coverage)"""
13 | ctx.run(f"pytest {args}", pty=use_pty)
14 |
15 |
16 | @task(optional=["args"], help={"args": "pytest additional arguments"})
17 | def test_cov(ctx: Context, args: str = ""):
18 | """run test vith coverage"""
19 | ctx.run(f"coverage run -m pytest {args}", pty=use_pty)
20 |
21 |
22 | @task(optional=["html"], help={"html": "flag to export html report"})
23 | def report_cov(ctx: Context, *, html: bool = False):
24 | """report coverage"""
25 | ctx.run("coverage combine", warn=True, pty=use_pty)
26 | ctx.run("coverage report --show-missing", pty=use_pty)
27 | if html:
28 | ctx.run("coverage html", pty=use_pty)
29 |
30 |
31 | @task(
32 | optional=["args", "html"],
33 | help={
34 | "args": "pytest additional arguments",
35 | "html": "flag to export html report",
36 | },
37 | )
38 | def coverage(ctx: Context, args: str = "", *, html: bool = False):
39 | """run tests and report coverage"""
40 | test_cov(ctx, args=args)
41 | report_cov(ctx, html=html)
42 |
43 |
44 | @task(optional=["args"], help={"args": "black additional arguments"})
45 | def lint_black(ctx: Context, args: str = "."):
46 | args = args or "." # needed for hatch script
47 | ctx.run("black --version", pty=use_pty)
48 | ctx.run(f"black --check --diff {args}", pty=use_pty)
49 |
50 |
51 | @task(optional=["args"], help={"args": "ruff additional arguments"})
52 | def lint_ruff(ctx: Context, args: str = "."):
53 | args = args or "." # needed for hatch script
54 | ctx.run("ruff --version", pty=use_pty)
55 | ctx.run(f"ruff check {args}", pty=use_pty)
56 |
57 |
58 | @task(
59 | optional=["args"],
60 | help={
61 | "args": "linting tools (black, ruff) additional arguments, typically a path",
62 | },
63 | )
64 | def lintall(ctx: Context, args: str = "."):
65 | """Check linting"""
66 | args = args or "." # needed for hatch script
67 | lint_black(ctx, args)
68 | lint_ruff(ctx, args)
69 |
70 |
71 | @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"})
72 | def check_pyright(ctx: Context, args: str = ""):
73 | """check static types with pyright"""
74 | ctx.run("pyright --version")
75 | ctx.run(f"pyright {args}", pty=use_pty)
76 |
77 |
78 | @task(optional=["args"], help={"args": "check tools (pyright) additional arguments"})
79 | def checkall(ctx: Context, args: str = ""):
80 | """check static types"""
81 | check_pyright(ctx, args)
82 |
83 |
84 | @task(optional=["args"], help={"args": "black additional arguments"})
85 | def fix_black(ctx: Context, args: str = "."):
86 | """fix black formatting"""
87 | args = args or "." # needed for hatch script
88 | ctx.run(f"black {args}", pty=use_pty)
89 |
90 |
91 | @task(optional=["args"], help={"args": "ruff additional arguments"})
92 | def fix_ruff(ctx: Context, args: str = "."):
93 | """fix all ruff rules"""
94 | args = args or "." # needed for hatch script
95 | ctx.run(f"ruff check --fix {args}", pty=use_pty)
96 |
97 |
98 | @task(
99 | optional=["args"],
100 | help={
101 | "args": "linting tools (black, ruff) additional arguments, typically a path",
102 | },
103 | )
104 | def fixall(ctx: Context, args: str = "."):
105 | """Fix everything automatically"""
106 | args = args or "." # needed for hatch script
107 | fix_black(ctx, args)
108 | fix_ruff(ctx, args)
109 | lintall(ctx, args)
110 |
--------------------------------------------------------------------------------
/test-website/Caddyfile:
--------------------------------------------------------------------------------
1 | {$SITE_ADDRESS} {
2 | header {
3 | Access-Control-Allow-Origin *
4 | }
5 |
6 | file_server {
7 | root /var/www/html
8 | }
9 |
10 | handle /dynimages {
11 | @hasPathParam query "path=*"
12 | handle @hasPathParam {
13 | rewrite * /images/{http.request.uri.query.path}
14 | }
15 | handle {
16 | respond 500 "Bad Request"
17 | }
18 | }
19 |
20 | handle /dynform-rewrite {
21 | @hasIdParam query "id=*"
22 | handle @hasIdParam {
23 | rewrite * /form-get-{http.request.uri.query.id}.html
24 | }
25 | handle {
26 | respond 500 "Bad Request"
27 | }
28 | }
29 |
30 | handle /dynform-redirect {
31 | @hasIdParam query "id=*"
32 | handle @hasIdParam {
33 | redir * /form-get-{http.request.uri.query.id}.html 302
34 | }
35 | handle {
36 | respond 500 "Bad Request"
37 | }
38 | }
39 |
40 | respond /200-response "Never mind" 200
41 |
42 | respond /201-response "Never mind" 201
43 |
44 | respond /202-response "Never mind" 202
45 |
46 | respond /204-response 204
47 |
48 | respond /206-response "Never mind" 206
49 |
50 | respond /300-response "Never mind" 300
51 |
52 | respond /303-response "Never mind" 303
53 |
54 | respond /304-response "Never mind" 304
55 |
56 | respond /305-response "Never mind" 305
57 |
58 | respond /306-response "Never mind" 306
59 |
60 | respond /400-response 400
61 |
62 | respond /401-response 401
63 |
64 | respond /402-response 402
65 |
66 | respond /403-response 403
67 |
68 | respond /404-response 404
69 |
70 | respond /500-response 500
71 |
72 | respond /501-response 501
73 |
74 | respond /502-response 502
75 |
76 | respond // "Hello you" 400
77 |
78 | respond /double-slash/test1 "Hello you" 200
79 | respond /double-slash//test1 400
80 |
81 | respond /double-slash/test2 "Hello you v1" 200
82 | respond /double-slash//test2 "Hello you v2" 200
83 |
84 | redir /301-internal-redirect-ok /internal_redirect_target.html 301
85 | redir /301-external-redirect-ok https://www.example.com 301
86 | redir /302-internal-redirect-ok /internal_redirect_target.html 302
87 | redir /302-external-redirect-ok https://www.example.com 302
88 | redir /307-internal-redirect-ok /internal_redirect_target.html 307
89 | redir /307-external-redirect-ok https://www.example.com 307
90 | redir /308-internal-redirect-ok /internal_redirect_target.html 308
91 | redir /308-external-redirect-ok https://www.example.com 308
92 |
93 | redir /301-internal-redirect-ko /internal_redirect_missing.html 301
94 | redir /301-external-redirect-ko https://www.example.invalid 301
95 | redir /302-internal-redirect-ko /internal_redirect_missing.html 302
96 | redir /302-external-redirect-ko https://www.example.invalid 302
97 | redir /307-internal-redirect-ko /internal_redirect_missing.html 307
98 | redir /307-external-redirect-ko https://www.example.invalid 307
99 | redir /308-internal-redirect-ko /internal_redirect_missing.html 308
100 | redir /308-external-redirect-ko https://www.example.invalid 308
101 |
102 | redir /bad-redir-loop-A /bad-redir-loop-B 307
103 | redir /bad-redir-loop-B /bad-redir-loop-C 307
104 | redir /bad-redir-loop-C /bad-redir-loop-D 307
105 | redir /bad-redir-loop-D /bad-redir-loop-B 307
106 | redir /bad-redir-target-A https://I%20mNotAhostname 307
107 | redir /bad-redir-target-B intent://example.com/path#Intent;scheme=http;package=com.example.myapp;component=com.example.myapp/.MainActivity;end 307
108 |
109 | header /content-types/script1.js Content-Type application/javascript
110 | header /content-types/script2.js Content-Type text/javascript
111 | header /content-types/style1.css Content-Type text/css
112 | header /content-types/style2.css Content-Type idontexist
113 | header /content-types/image1.png Content-Type idontexist
114 | header /content-types/image2.png Content-Type text/html
115 | header /content-types/content2.json Content-Type idontexist
116 |
117 | }
118 |
--------------------------------------------------------------------------------
/test-website/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM caddy:2.6.1-alpine
2 | LABEL org.opencontainers.image.source=https://github.com/openzim/warc2zim
3 |
4 | COPY Caddyfile /etc/caddy/Caddyfile
5 |
6 | COPY content/ /var/www/html/
7 |
8 | RUN apk add --no-cache bash
9 |
10 | COPY entrypoint.sh /root/entrypoint.sh
11 |
12 | ENTRYPOINT [ "/root/entrypoint.sh" ]
13 |
14 | CMD [ "caddy", "run", "--config", "/etc/caddy/Caddyfile", "--adapter", "caddyfile"]
15 |
--------------------------------------------------------------------------------
/test-website/README.md:
--------------------------------------------------------------------------------
1 | # Test website
2 |
3 | This is a test website for zimit / warc2zim tests.
4 |
5 | It contains all kind of situations we currently cope with:
6 | - a youtube player
7 | - a vimeo player
8 | - an X/twitter post
9 | - an X/twitter video
10 | - a facebook video
11 | - a facebook post
12 | - an instagram embed
13 |
14 | - invalid inline CSS
15 | - invalid rule in a CSS file
16 |
17 |
18 | ## How to use locally
19 |
20 | Build the docker image
21 |
22 | ```
23 | docker build -t test-website .
24 | ```
25 |
26 | Start the test website with appropriate environment variables.
27 |
28 | ```
29 | docker run -p 8888:80 --rm --name test-website -e SITE_ADDRESS="test-website.local.oviles.info:80, xn--jdacents-v0aqb.local.oviles.info:80" -e STANDARD_NETLOC="http:\/\/test-website.local.oviles.info:8888" -e NOT_STANDARD_NETLOC_NOT_ENCODED="http:\/\/jédéacçents.local.oviles.info:8888" -e NOT_STANDARD_NETLOC_PUNNY_ENCODED="http:\/\/xn--jdacents-v0aqb.local.oviles.info:8888" test-website
30 | ```
31 |
32 | In the example above, the trick is that we have the following DNS records in place : `local.oviles.info A 127.0.0.1` and `*.local.oviles.info CNAME local.oviles.info`, meaning that any request to local.oviles.info or one of its subdomain will resolve to localhost IP 127.0.0.1 ; we use local ports 8080 for HTTP and 8443 for HTTPS.
33 |
34 | You can then open https://test-website.local.oviles.info:8888 in your favorite browser and run manual tests on this website (which uses the other one as sub-site for few resources on special domains with special characters).
35 |
36 | If you wanna develop the test-website locally, you might want as well to mount the `content` folder inside the container
37 |
38 | ```
39 | docker run -v $PWD/content:/var/www/html -p 8888:80 --rm --name test-website -e SITE_ADDRESS="test-website.local.oviles.info:80, xn--jdacents-v0aqb.local.oviles.info:80" -e STANDARD_NETLOC="http:\/\/test-website.local.oviles.info:8888" -e NOT_STANDARD_NETLOC_NOT_ENCODED="http:\/\/jédéacçents.local.oviles.info:8888" -e NOT_STANDARD_NETLOC_PUNNY_ENCODED="http:\/\/xn--jdacents-v0aqb.local.oviles.info:8888" test-website
40 | ```
41 |
42 | This will have the adverse effect that local files will be modified as well by the `entrypoint.sh` to replace placeholders by environment variables value. And it means that you have to use "real" netloc from the environment in your modifications for test.
43 |
44 | Once done, there is a utility script at `entrypoint-reverse.sh` which can be used to reverse these modifications once you are about to commit to Github (this will break the test-website inside Docker container, but you will be able to commit with proper modifications and then just restart the container to reapply needed modification).
45 |
46 | ## Environments variables needed in Docker image
47 |
48 | |Environment variable | Usage | Comment | Sample value |
49 | |--|--|--|--|
50 | | `SITE_ADDRESS` | Caddyfile | The site address for Caddy operation ; should contain a standard and a not-standard punny-encoded hostnames for proper testing | `test-website.local.oviles.info:80, xn--jdacents-v0aqb.local.oviles.info:80` |
51 | | `STANDARD_NETLOC` | sed in HTML/JS/... files ^1 | The URL to the standard netloc (no special characters) | `http:\/\/test-website.local.oviles.info:8888` |
52 | | `NOT_STANDARD_NETLOC_NOT_ENCODED` | sed in HTML/JS/... files ^1 | The URL to the not standard netloc (with special characters) but not encoded | `http:\/\/jédéacçents.local.oviles.info:8888` |
53 | | `NOT_STANDARD_NETLOC_PUNNY_ENCODED` | sed in HTML/JS/... files ^1 | The URL to the not standard netloc (with special characters) punny encoded | `http:\/\/xn--jdacents-v0aqb.local.oviles.info:8888` |
54 |
55 | 1. variables that will be used by `sed` as replacement have to be be escaped for proper sed operations ; since they will be interpreted by the browser, they should contain the "user-visible" FQDN and port
56 |
--------------------------------------------------------------------------------
/test-website/content/audio.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Audio tag
17 |
18 | Test website
7 | Bad redirections
17 |
18 | Links below are indefinitely redirecting and hence not working
19 |
20 | Redirect to self through loop
21 |
22 | Redirect to inner-loop
23 |
24 | Links below is targeting something which is not working
25 |
26 | Redirect to silly HTTP URL
27 |
28 | Redirect to an intent (not working inside ZIM)
29 |
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/test-website/content/base-href.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Base href
20 |
21 | You should check that icon (meaning favicon) / webmanifest are properly loaded as well
22 |
23 | Something on relative link
24 | Something on absolute link
25 | A picture with relative link:
26 | Another picture with relative link:
27 | A picture with absolute link:
28 |
29 |
30 |
31 |
32 |
--------------------------------------------------------------------------------
/test-website/content/base-href/image1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/base-href/image1.png
--------------------------------------------------------------------------------
/test-website/content/base-href/something.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | I'm located here
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/test-website/content/chinese-encoding.html:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/chinese-encoding.html
--------------------------------------------------------------------------------
/test-website/content/content-types/content1.json:
--------------------------------------------------------------------------------
1 | {
2 | "property": "I'm OK"
3 | }
4 |
--------------------------------------------------------------------------------
/test-website/content/content-types/content2.json:
--------------------------------------------------------------------------------
1 | {
2 | "property": "I'm OK"
3 | }
4 |
--------------------------------------------------------------------------------
/test-website/content/content-types/image1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/content-types/image1.png
--------------------------------------------------------------------------------
/test-website/content/content-types/image2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/content-types/image2.png
--------------------------------------------------------------------------------
/test-website/content/content-types/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Tests of records with some missing or bad content-type headers
27 |
28 |
29 | An image should be displayed here:
30 |
32 |
33 | And mostly same image should be displayed here:
34 |
36 | Open a PDF
37 | This text must be in green.
38 | This text must also be in green.
39 | Something should be loaded here from JS: I'm not OK
40 | Something also should be loaded here from JS: I'm not OK
41 |
42 |
43 |
44 |
45 |
--------------------------------------------------------------------------------
/test-website/content/content-types/script1.js:
--------------------------------------------------------------------------------
1 | async function fetchContent1() {
2 | const response = await fetch('./content1.json')
3 | const json = await response.json();
4 | const span = document.getElementById("span01");
5 | span.innerHTML=json.property;
6 | span.classList.add('green');
7 | }
8 | fetchContent1()
9 |
--------------------------------------------------------------------------------
/test-website/content/content-types/script2.js:
--------------------------------------------------------------------------------
1 | async function fetchContent2() {
2 | const response = await fetch('./content2.json')
3 | const json = await response.json();
4 | const span = document.getElementById("span02");
5 | span.innerHTML=json.property;
6 | span.classList.add('green');
7 | }
8 | fetchContent2()
9 |
--------------------------------------------------------------------------------
/test-website/content/content-types/small.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/content-types/small.pdf
--------------------------------------------------------------------------------
/test-website/content/content-types/style1.css:
--------------------------------------------------------------------------------
1 | .green1 {
2 | color: green;
3 | }
4 |
--------------------------------------------------------------------------------
/test-website/content/content-types/style2.css:
--------------------------------------------------------------------------------
1 | .green2 {
2 | color: green;
3 | }
4 |
--------------------------------------------------------------------------------
/test-website/content/css-broken-file.css:
--------------------------------------------------------------------------------
1 |
2 | .great {
3 | not_exits_prop: red; /* property does not exist */
4 | background-image: url('http://www.mediaelementjs.com/images/big_buck_bunny.jpg');
5 | width: 150px;
6 | font-size: ; /* missing value */
7 | display= 12; /* equal sign */
8 | height: 100px;
9 | margin: 2rem;
10 | padding: 1rem;
11 | font-weight: bolder;
12 | color: white;
13 | text-align: center;
14 | }
15 |
16 | "broken
17 |
--------------------------------------------------------------------------------
/test-website/content/css-broken-file.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Broken CSS file
18 |
19 |
20 | CSS file is broken but background image is displayed
21 |
22 |
23 |
24 |
25 |
26 |
--------------------------------------------------------------------------------
/test-website/content/css-inline-broken.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Broken inline CSS
17 |
18 |
20 | CSS not broken, background image displayed
21 |
22 |
24 | CSS broken, background image displayed as well
25 |
26 |
27 |
28 |
29 |
30 |
--------------------------------------------------------------------------------
/test-website/content/dailymotion.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Dailymotion
17 |
18 |
19 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/test-website/content/double-slash.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Double slash in URLs
17 |
18 | .//
19 | ./double-slash//test1
20 | ./double-slash/test1
21 | ./double-slash/test2
22 | ./double-slash//test2
23 |
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/test-website/content/facebook1.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Facebook video embed from Feb. 2024
18 |
19 |
23 |
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/test-website/content/facebook2.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Facebook post embed from Feb. 2024
17 |
18 |
22 |
23 |
24 |
25 |
26 |
--------------------------------------------------------------------------------
/test-website/content/form-get-1.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Form get subpage 1
17 |
18 | Congratulations, you found subpage 1
19 |
20 |
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/test-website/content/form-get-2.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Form get subpage 2
17 |
18 | Congratulations, you found subpage 2
19 |
20 |
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/test-website/content/form-get-3.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Form get subpage 3
17 |
18 | Congratulations, you found subpage 3
19 |
20 |
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/test-website/content/form-get-4.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Form get subpage 4
17 |
18 | Congratulations, you found subpage 4
19 |
20 |
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/test-website/content/form-get.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Test website
6 | Form GET
29 |
30 | This page showcase a situation where someone used a form with a combobox to enable navigation on the website.
31 |
32 | Nota: this has been seen in the wild on https://chopin.lib.uchicago.edu/ (open any title scores and the combobox will appear in top right corner) and on https://medecine-integree.com/ (for which we - Kiwix - do not have any rights to copy but have been approached by a user).
33 |
34 | Rewrite
35 | Server configuration will intercept the queryparameter and rewrite internal to proper HTML file, invisible to the end user.
36 |
47 |
48 | Redirect
49 | Server configuration will intercept the queryparameter and redirect with a 302 to proper URL, hence visible to the end user.
50 |
61 |
62 |
63 |
64 |
--------------------------------------------------------------------------------
/test-website/content/href-to-folder/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Tests of links to folder instead of document
17 |
18 | ./
19 | ../href-to-folder/
20 | ../../href-to-folder/ (too deep)
21 | #section1
22 | #section2
23 | ...
24 | ...
25 | ...
26 | ...
27 | ...
28 | ...
29 | ...
30 | ...
31 | ...
32 | ...
33 | ...
34 | ...
35 | ...
36 | ...
37 | ...
38 | Section1
39 | ...
40 | ...
41 | ...
42 | ...
43 | ...
44 | ...
45 | ...
46 | ...
47 | ...
48 | ...
49 | Section2
50 | ...
51 | ...
52 | ...
53 | ...
54 | ...
55 | ...
56 | ...
57 | ...
58 | ...
59 | ...
60 |
61 |
62 |
63 |
64 |
--------------------------------------------------------------------------------
/test-website/content/http-equiv-redirect.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Redirect with http-equiv meta directive
18 |
19 | You should be redirected to home page in 3 seconds
20 |
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/test-website/content/http-return-codes.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Various redirects
17 |
18 | 200 response
19 | 201 response
20 | 202 response
21 | 204 response
22 | 206 response
23 | 300 response
24 | 303 response
25 | 304 response
26 | 305 response
27 | 306 response
28 | 400 response
29 | 401 response
30 | 402 response
31 | 403 response
32 | 404 response
33 | 500 response
34 | 501 response
35 | 502 response
36 | 301 Internal redirect OK
37 | 301 External redirect OK
38 | 302 Internal redirect OK
39 | 302 External redirect OK
40 | 307 Internal redirect OK
41 | 307 External redirect OK
42 | 308 Internal redirect OK
43 | 308 External redirect OK
44 | 301 Internal redirect KO
45 | 301 External redirect KO
46 | 302 Internal redirect KO
47 | 302 External redirect KO
48 | 307 Internal redirect KO
49 | 307 External redirect KO
50 | 308 Internal redirect KO
51 | 308 External redirect KO
52 |
53 |
54 |
55 |
--------------------------------------------------------------------------------
/test-website/content/icons/android-chrome-192x192.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/icons/android-chrome-192x192.png
--------------------------------------------------------------------------------
/test-website/content/icons/android-chrome-512x512.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/icons/android-chrome-512x512.png
--------------------------------------------------------------------------------
/test-website/content/icons/apple-touch-icon.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/icons/apple-touch-icon.png
--------------------------------------------------------------------------------
/test-website/content/icons/favicon-16x16.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/icons/favicon-16x16.png
--------------------------------------------------------------------------------
/test-website/content/icons/favicon-32x32.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/icons/favicon-32x32.png
--------------------------------------------------------------------------------
/test-website/content/icons/favicon.ico:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/icons/favicon.ico
--------------------------------------------------------------------------------
/test-website/content/icons/site.webmanifest:
--------------------------------------------------------------------------------
1 | {
2 | "name": "RFG",
3 | "short_name": "RFG",
4 | "icons": [
5 | {
6 | "src": "./android-chrome-192x192.png",
7 | "sizes": "192x192",
8 | "type": "image/png"
9 | },
10 | {
11 | "src": "./android-chrome-512x512.png",
12 | "sizes": "512x512",
13 | "type": "image/png"
14 | }
15 | ],
16 | "theme_color": "#377dba",
17 | "background_color": "#377dba",
18 | "display": "standalone"
19 | }
20 |
--------------------------------------------------------------------------------
/test-website/content/image-srcset.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Test website
6 |
29 | This page contains tests around varying images based on screen / device.
30 |
31 |
32 | Image srcset
33 |
34 |
35 | An image should be displayed below at all screen sizes and pixel ratios.
36 |
37 |
38 |
49 | Another image should be displayed below at all screen sizes and pixel
50 | ratios.
51 |
52 |
53 | Picture sources - with srcset pixel ratio
60 |
61 |
62 | An image should be displayed below at all screen sizes and pixel ratios.
63 |
64 |
65 |
66 |
76 |
77 | Picture sources - with media queries
78 |
79 |
80 | An image should be displayed below at all screen sizes and pixel ratios.
81 |
82 |
83 |
84 |
101 |
102 |
103 |
--------------------------------------------------------------------------------
/test-website/content/images/image1-1x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image1-1x.png
--------------------------------------------------------------------------------
/test-website/content/images/image1-2x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image1-2x.png
--------------------------------------------------------------------------------
/test-website/content/images/image1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image1.png
--------------------------------------------------------------------------------
/test-website/content/images/image2-1x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image2-1x.png
--------------------------------------------------------------------------------
/test-website/content/images/image2-2x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image2-2x.png
--------------------------------------------------------------------------------
/test-website/content/images/image2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image2.png
--------------------------------------------------------------------------------
/test-website/content/images/image3-high.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image3-high.png
--------------------------------------------------------------------------------
/test-website/content/images/image3-medium.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image3-medium.png
--------------------------------------------------------------------------------
/test-website/content/images/image3-small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image3-small.png
--------------------------------------------------------------------------------
/test-website/content/images/image3.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image3.png
--------------------------------------------------------------------------------
/test-website/content/images/image4-1.5x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image4-1.5x.png
--------------------------------------------------------------------------------
/test-website/content/images/image4.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/image4.png
--------------------------------------------------------------------------------
/test-website/content/images/urlencoding1_icône-débuter-Solidarité-Numérique_1@300x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/urlencoding1_icône-débuter-Solidarité-Numérique_1@300x.png
--------------------------------------------------------------------------------
/test-website/content/images/urlencoding2_ico%CC%82ne-de%CC%81buter-Solidarite%CC%81-Nume%CC%81rique_1%40300x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/urlencoding2_ico%CC%82ne-de%CC%81buter-Solidarite%CC%81-Nume%CC%81rique_1%40300x.png
--------------------------------------------------------------------------------
/test-website/content/images/urlencoding3_icône-débuter-Solidarite%CC%81-Nume%CC%81rique_1@300x.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/urlencoding3_icône-débuter-Solidarite%CC%81-Nume%CC%81rique_1@300x.png
--------------------------------------------------------------------------------
/test-website/content/images/urlencoding4_(_[_|_]_).png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/urlencoding4_(_[_|_]_).png
--------------------------------------------------------------------------------
/test-website/content/images/urlencoding4_ؼ.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/urlencoding4_ؼ.png
--------------------------------------------------------------------------------
/test-website/content/images/urlencoding5_(_[_|_ؼ_]_).png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/urlencoding5_(_[_|_ؼ_]_).png
--------------------------------------------------------------------------------
/test-website/content/images/urlencoding5_(_[_|_ؼ_]_)_1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/urlencoding5_(_[_|_ؼ_]_)_1.png
--------------------------------------------------------------------------------
/test-website/content/images/urlencoding5_(_[_|_ؼ_]_)_2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/images/urlencoding5_(_[_|_ؼ_]_)_2.png
--------------------------------------------------------------------------------
/test-website/content/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | This is a test website for zimit/warc2zim.
17 |
18 | Caution! Opening any of the subpages will likely activate all kind of trackers from the source websites
19 |
20 |
21 |
57 |
58 |
59 |
60 |
--------------------------------------------------------------------------------
/test-website/content/instagram.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | This page contains Instagram test(s).
17 |
18 | Instagram embed from Feb. 2024
19 |
20 |
24 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
--------------------------------------------------------------------------------
/test-website/content/internal_redirect_target.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | You've been redirect to me
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/test-website/content/javascript/cont!nt.txt:
--------------------------------------------------------------------------------
1 | This is working OK
2 |
--------------------------------------------------------------------------------
/test-website/content/javascript/cont?nt.txt:
--------------------------------------------------------------------------------
1 | This is working OK
2 |
--------------------------------------------------------------------------------
/test-website/content/javascript/content.txt:
--------------------------------------------------------------------------------
1 | This is working OK
2 |
--------------------------------------------------------------------------------
/test-website/content/javascript/contént.txt:
--------------------------------------------------------------------------------
1 | This is working OK
2 |
--------------------------------------------------------------------------------
/test-website/content/javascript/cont🎁nt.txt:
--------------------------------------------------------------------------------
1 | This is working OK
2 |
--------------------------------------------------------------------------------
/test-website/content/javascript/not_working.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/javascript/not_working.png
--------------------------------------------------------------------------------
/test-website/content/javascript/r?sources.js:
--------------------------------------------------------------------------------
1 | const okValue = 'This is working OK';
2 | const otherValue = 'This is another value';
3 |
4 | const okClass = 'green';
5 |
6 | export {okClass, otherValue};
7 | export default okValue;
8 |
--------------------------------------------------------------------------------
/test-website/content/javascript/resources.js:
--------------------------------------------------------------------------------
1 | const okValue = 'This is working OK';
2 | const otherValue = 'This is another value';
3 |
4 | const okClass = 'green';
5 |
6 | export {okClass, otherValue};
7 | export default okValue;
8 |
--------------------------------------------------------------------------------
/test-website/content/javascript/résources.js:
--------------------------------------------------------------------------------
1 | const okValue = 'This is working OK';
2 | const otherValue = 'This is another value';
3 |
4 | const okClass = 'green';
5 |
6 | export {okClass, otherValue};
7 | export default okValue;
8 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script02!b.js:
--------------------------------------------------------------------------------
1 | const span02b = document.getElementById('span02b');
2 | span02b.innerHTML = 'This is working OK';
3 | span02b.classList.add('green');
4 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script02a.js:
--------------------------------------------------------------------------------
1 | const span02a = document.getElementById('span02a');
2 | span02a.innerHTML = 'This is working OK';
3 | span02a.classList.add('green');
4 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script03.js:
--------------------------------------------------------------------------------
1 | const span03 = document.getElementById("span03");
2 | span03.innerHTML="This is working OK";
3 | span03.classList.add("green");
4 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script04.js:
--------------------------------------------------------------------------------
1 | window.onload = function() {
2 | const span04 = document.getElementById("span04");
3 | span04.innerHTML="This is working OK";
4 | span04.classList.add("green");
5 | }
6 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script05.js:
--------------------------------------------------------------------------------
1 | const span05 = document.getElementById("span05");
2 | span05.innerHTML="This is working OK";
3 | span05.classList.add("green");
4 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06a.js:
--------------------------------------------------------------------------------
1 | import okValue from './resources.js';
2 | import { okClass } from './resources.js';
3 |
4 | const span = document.getElementById("span06a");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06b.js:
--------------------------------------------------------------------------------
1 | import okValue from './resources.js';
2 | import { otherValue, okClass as myClass } from './resources.js';
3 |
4 | const span = document.getElementById("span06b");
5 | span.innerHTML=okValue;
6 | span.classList.add(myClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06c.js:
--------------------------------------------------------------------------------
1 | import okValue from '/javascript/resources.js';
2 | import { okClass } from '/javascript/resources.js';
3 |
4 | const span = document.getElementById("span06c");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06d.js:
--------------------------------------------------------------------------------
1 | import okValue from '../javascript/resources.js';
2 | import { okClass } from '../javascript/resources.js';
3 |
4 | const span = document.getElementById("span06d");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06e.js:
--------------------------------------------------------------------------------
1 | import okValue from './résources.js';
2 | import { okClass } from './résources.js';
3 |
4 | const span = document.getElementById("span06e");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06f.js:
--------------------------------------------------------------------------------
1 | import okValue from './resources.js?query=value';
2 | import { okClass } from './resources.js?query=value';
3 |
4 | const span = document.getElementById("span06f");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06g.js:
--------------------------------------------------------------------------------
1 | import okValue from './resources.js?query=valu%3De';
2 | import { okClass } from './resources.js?query=valu%3De';
3 |
4 | const span = document.getElementById("span06g");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06h.js:
--------------------------------------------------------------------------------
1 | import okValue from './resources.js?query=valu%25e';
2 | import { okClass } from './resources.js?query=valu%25e';
3 |
4 | const span = document.getElementById("span06h");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06i.js:
--------------------------------------------------------------------------------
1 | import okValue from './r%3Fsources.js';
2 | import { okClass } from './r%3Fsources.js';
3 |
4 | const span = document.getElementById("span06i");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script06j.js:
--------------------------------------------------------------------------------
1 | import okValue from './r%3Fsources.js?query=value';
2 | import { okClass } from './r%3Fsources.js?query=value';
3 |
4 | const span = document.getElementById("span06j");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script07.js:
--------------------------------------------------------------------------------
1 | import okValue from './resources.js';
2 | import { okClass } from './resources.js';
3 |
4 | const span = document.getElementById("span07");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script08a.js:
--------------------------------------------------------------------------------
1 | import okValue from 'resources';
2 | import { okClass } from 'resources';
3 |
4 | const span = document.getElementById("span08a");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script08b.js:
--------------------------------------------------------------------------------
1 | import okValue from 'javascript/resources';
2 | import { okClass } from 'javascript/resources';
3 |
4 | const span = document.getElementById("span08b");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script08c.js:
--------------------------------------------------------------------------------
1 | import okValue from '../javascript/resources';
2 | import { okClass } from '../javascript/resources';
3 |
4 | const span = document.getElementById("span08c");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script09a.js:
--------------------------------------------------------------------------------
1 | import okValue from 'https://example.com/resources';
2 | import { okClass } from 'https://example.com/resources';
3 |
4 | const span = document.getElementById("span09a");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script09b.js:
--------------------------------------------------------------------------------
1 | import okValue from 'https://standard_netloc/resources';
2 | import { okClass } from 'https://standard_netloc/resources';
3 |
4 | const span = document.getElementById("span09b");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script09c.js:
--------------------------------------------------------------------------------
1 | import okValue from 'https://not_standard_netloc_punny_encoded/resources';
2 | import { okClass } from 'https://not_standard_netloc_punny_encoded/resources';
3 |
4 | const span = document.getElementById("span09c");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script09d.js:
--------------------------------------------------------------------------------
1 | import okValue from 'https://not_standard_netloc_not_encoded/resources';
2 | import { okClass } from 'https://not_standard_netloc_not_encoded/resources';
3 |
4 | const span = document.getElementById("span09d");
5 | span.innerHTML=okValue;
6 | span.classList.add(okClass);
7 |
--------------------------------------------------------------------------------
/test-website/content/javascript/script10.js:
--------------------------------------------------------------------------------
1 | const img10 = document.getElementById('img10');
2 | const origSrc = img10.getAttribute('src')
3 | const newSrc = origSrc.replace('not_working', 'working')
4 | console.debug('Replacing ' + origSrc + ' with ' + newSrc)
5 | img10.src = newSrc;
--------------------------------------------------------------------------------
/test-website/content/javascript/working.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/test-website/content/javascript/working.png
--------------------------------------------------------------------------------
/test-website/content/jàvàscrïpt.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Javascript acting on file with special chars in its path
25 |
26 | On every lines below, you should see written "This is working OK"
27 |
28 |
29 | Test: This is not working
30 |
31 |
32 |
43 |
44 |
45 |
46 |
47 |
--------------------------------------------------------------------------------
/test-website/content/mediaelement.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | MediaElement.js player
17 |
18 |
19 |
27 |
29 |
30 |
31 |
32 |
33 |
34 |
36 |
38 |
39 |
40 |
41 |
49 |
50 |
51 |
52 |
53 |
--------------------------------------------------------------------------------
/test-website/content/onxxx.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | onxxx HTML events
25 |
26 | When clicking images below, it should redirect your browser to the another page
27 |
28 | Relative (this site home page ):
29 |
30 | Absolute internal 1 (this site home page ):
31 |
32 | Absolute internal 2 (another site home page ):
33 |
34 | Absolute external (kiwix homepage):
35 |
36 |
37 |
38 |
39 |
--------------------------------------------------------------------------------
/test-website/content/soundcloud.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Soundcloud
17 |
18 |
20 |
26 |
27 |
28 |
29 |
30 |
--------------------------------------------------------------------------------
/test-website/content/spotify.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Spotify
17 |
18 |
22 |
23 |
24 |
25 |
26 |
--------------------------------------------------------------------------------
/test-website/content/tiktok.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | TikTok
17 |
18 |
20 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
--------------------------------------------------------------------------------
/test-website/content/twitch1.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Twitch iframe
17 |
18 |
20 |
21 |
22 |
23 |
24 |
--------------------------------------------------------------------------------
/test-website/content/twitch2.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Twitch JS
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
30 |
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/test-website/content/twitter1.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Twitter post embed from Feb. 2024
17 |
18 |
24 |
25 |
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/test-website/content/twitter2.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | Twitter video embed from Feb. 2024
17 |
18 |
24 |
25 |
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/test-website/content/url-encoding.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | This page contains URL encoding test(s).
17 |
18 | Image with non-encoded chars in name with non-encoded special chars in link
19 |
20 | An image should be displayed below
21 | Image with non-encoded chars in name and already encoded special chars
24 |
25 | An image should be displayed below
26 | Image with non-encoded chars in name and a mix of non-encoded and already encoded special chars
29 |
30 | An image should be displayed below
31 | Image with non-encoded chars in name with non-encoded special chars in link and a query parameter and fragment with reserved chars
34 |
35 | An image should be displayed below as well
36 | Image with already encoded chars in name with non-encoded special chars in link
41 |
42 | This is not working on source website, so should not work in ZIM
43 | Image with already encoded chars in name and already encoded special chars in link
46 |
47 | This is not working on source website, so should not work in ZIM
48 | Image with already encoded chars in name and a mix of non-encoded and already encoded special chars in link
51 |
52 | This is not working on source website, so should not work in ZIM
53 | Image with a mix of non-encoded and already encoded special chars in name with non-encoded special chars in link
58 |
59 | This is not working on source website, so should not work in ZIM
60 | Image with a mix of non-encoded and already encoded special chars in name and already encoded special chars in link
63 |
64 | This is not working on source website, so should not work in ZIM
65 | Image with a mix of non-encoded and already encoded special chars in name and a mix of non-encoded and already encoded special chars in link
68 |
69 | This is not working on source website, so should not work in ZIM
70 | Image with many weird characters
75 |
76 | An image should be displayed below
77 | Image with UTF-8 characters
80 |
81 | An image should be displayed below
82 | Hostname with UTF-8 chars and simple image
87 |
88 | An image should be displayed below
89 | Hostname with UTF-8 chars and image with UTF-8 chars
92 |
93 | An image should be displayed below
94 | Hostname with UTF-8 chars and query parameter 1
97 |
98 | An image should be displayed below with a 1 in the right
99 | Hostname with UTF-8 chars and query parameter 2
102 |
103 | An image should be displayed below with a 2 in the right
104 | Test website
7 | Video tag
17 |
18 |
19 |
20 |
21 | Your browser does not support the video tag.
22 |
23 |
24 |
25 |
26 |
27 |
--------------------------------------------------------------------------------
/test-website/content/vimeo.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | This page contains Vimeo test(s).
17 |
18 | Vimeo video
19 |
20 |
22 |
23 |
24 |
25 |
26 |
--------------------------------------------------------------------------------
/test-website/content/youtube.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Test website
7 | This page contains Youtube test(s).
17 |
18 | Youtube video
19 |
20 | VIDEO
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/test-website/entrypoint-reverse.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Reverse the modifications done by entrypoint.sh ; usefull for local development when local directory is mounted inside the container
4 | # for rapid tests but the modifications have to be reversed so that proper stuff is commited in Github
5 |
6 | find /var/www/html -type f -exec sed -i 's/http:\/\/jédéacçents.local.oviles.info:8888/https:\/\/not_standard_netloc_not_encoded/g' {} \;
7 | find /var/www/html -type f -exec sed -i 's/http:\/\/xn--jdacents-v0aqb.local.oviles.info:8888/https:\/\/not_standard_netloc_punny_encoded/g' {} \;
8 | find /var/www/html -type f -exec sed -i 's/http:\/\/test-website.local.oviles.info:8888/https:\/\/standard_netloc/g' {} \;
9 |
--------------------------------------------------------------------------------
/test-website/entrypoint.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Replace all occurences of hostnames in html files
4 |
5 | find /var/www/html -type f -exec sed -i "s/https:\/\/not_standard_netloc_not_encoded/$NOT_STANDARD_NETLOC_NOT_ENCODED/g" {} \;
6 | find /var/www/html -type f -exec sed -i "s/https:\/\/not_standard_netloc_punny_encoded/$NOT_STANDARD_NETLOC_PUNNY_ENCODED/g" {} \;
7 | find /var/www/html -type f -exec sed -i "s/https:\/\/standard_netloc/$STANDARD_NETLOC/g" {} \;
8 |
9 | caddy run --config /etc/caddy/Caddyfile --adapter caddyfile
10 |
--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/__init__.py
--------------------------------------------------------------------------------
/tests/cdxj_indexer/test_postappend.py:
--------------------------------------------------------------------------------
1 | from io import BytesIO
2 |
3 | from warc2zim.cdxj_indexer.postquery import append_method_query
4 |
5 |
6 | # ============================================================================
7 | class MethodQueryCanonicalizer:
8 | def __init__(self, method, content_type, req_len, req_stream):
9 | self.method = method
10 | self.content_type = content_type
11 | self.req_len = req_len
12 | self.req_stream = req_stream
13 |
14 | def append_query(self, url):
15 | self.req_stream.seek(0)
16 | query_only, full_string = append_method_query(
17 | self.method, self.content_type, self.req_len, self.req_stream, url
18 | )
19 | return url + full_string
20 |
21 |
22 | # ============================================================================
23 | class TestPostQueryExtract:
24 | @classmethod
25 | def setup_class(cls):
26 | cls.post_data = b"foo=bar&dir=%2Fbaz"
27 | cls.binary_post_data = (
28 | b"\x816l`L\xa04P\x0e\xe0r\x02\xb5\x89\x19\x00fP\xdb\x0e\xb0\x02,"
29 | )
30 |
31 | def test_post_extract_1(self):
32 | mq = MethodQueryCanonicalizer(
33 | "POST",
34 | "application/x-www-form-urlencoded",
35 | len(self.post_data),
36 | BytesIO(self.post_data),
37 | )
38 |
39 | assert (
40 | mq.append_query("http://example.com/")
41 | == "http://example.com/?__wb_method=POST&foo=bar&dir=/baz"
42 | )
43 |
44 | assert (
45 | mq.append_query("http://example.com/?123=ABC")
46 | == "http://example.com/?123=ABC&__wb_method=POST&foo=bar&dir=/baz"
47 | )
48 |
49 | def test_post_extract_json(self):
50 | post_data = b'{"a": "b", "c": {"a": 2}, "d": "e"}'
51 | mq = MethodQueryCanonicalizer(
52 | "POST", "application/json", len(post_data), BytesIO(post_data)
53 | )
54 |
55 | assert (
56 | mq.append_query("http://example.com/")
57 | == "http://example.com/?__wb_method=POST&a=b&a.2_=2&d=e"
58 | )
59 |
60 | def test_post_extract_json_top_list(self):
61 | post_data = (
62 | b'[{"a": "b", "c": {"a": 2}}, {"d": "e"}, "ignored", false, null, 0]'
63 | )
64 | mq = MethodQueryCanonicalizer(
65 | "POST", "application/json", len(post_data), BytesIO(post_data)
66 | )
67 |
68 | assert (
69 | mq.append_query("http://example.com/")
70 | == "http://example.com/?__wb_method=POST&a=b&a.2_=2&d=e"
71 | )
72 |
73 | def test_post_extract_json_lines(self):
74 | post_data = b'{"a": "b"}\n{"c": {"a": 2}, "d": "e"}'
75 | mq = MethodQueryCanonicalizer(
76 | "POST", "application/json", len(post_data), BytesIO(post_data)
77 | )
78 |
79 | assert (
80 | mq.append_query("http://example.com/")
81 | == "http://example.com/?__wb_method=POST&a=b&a.2_=2&d=e"
82 | )
83 |
84 | def test_put_extract_method(self):
85 | mq = MethodQueryCanonicalizer(
86 | "PUT",
87 | "application/x-www-form-urlencoded",
88 | len(self.post_data),
89 | BytesIO(self.post_data),
90 | )
91 |
92 | assert (
93 | mq.append_query("http://example.com/")
94 | == "http://example.com/?__wb_method=PUT&foo=bar&dir=/baz"
95 | )
96 |
97 | def test_post_extract_non_form_data_1(self):
98 | mq = MethodQueryCanonicalizer(
99 | "POST",
100 | "application/octet-stream",
101 | len(self.post_data),
102 | BytesIO(self.post_data),
103 | )
104 |
105 | # base64 encoded data
106 | assert (
107 | mq.append_query("http://example.com/")
108 | == "http://example.com/?__wb_method=POST&__wb_post_data=Zm9vPWJhciZkaXI9JTJGYmF6"
109 | )
110 |
111 | def test_post_extract_non_form_data_2(self):
112 | mq = MethodQueryCanonicalizer(
113 | "POST", "text/plain", len(self.post_data), BytesIO(self.post_data)
114 | )
115 |
116 | # base64 encoded data
117 | assert (
118 | mq.append_query("http://example.com/pathbar?id=123")
119 | == "http://example.com/pathbar?id=123&__wb_method=POST&__wb_post_data=Zm9vPWJhciZkaXI9JTJGYmF6"
120 | )
121 |
122 | def test_post_extract_length_invalid_ignore(self):
123 | mq = MethodQueryCanonicalizer(
124 | "POST", "application/x-www-form-urlencoded", 0, BytesIO(self.post_data)
125 | )
126 |
127 | assert (
128 | mq.append_query("http://example.com/")
129 | == "http://example.com/?__wb_method=POST"
130 | )
131 |
132 | mq = MethodQueryCanonicalizer(
133 | "POST", "application/x-www-form-urlencoded", "abc", BytesIO(self.post_data)
134 | )
135 |
136 | assert (
137 | mq.append_query("http://example.com/")
138 | == "http://example.com/?__wb_method=POST"
139 | )
140 |
141 | def test_post_extract_length_too_short(self):
142 | mq = MethodQueryCanonicalizer(
143 | "POST",
144 | "application/x-www-form-urlencoded",
145 | len(self.post_data) - 4,
146 | BytesIO(self.post_data),
147 | )
148 |
149 | assert (
150 | mq.append_query("http://example.com/")
151 | == "http://example.com/?__wb_method=POST&foo=bar&dir=%2"
152 | )
153 |
154 | def test_post_extract_length_too_long(self):
155 | mq = MethodQueryCanonicalizer(
156 | "POST",
157 | "application/x-www-form-urlencoded",
158 | len(self.post_data) + 4,
159 | BytesIO(self.post_data),
160 | )
161 |
162 | assert (
163 | mq.append_query("http://example.com/")
164 | == "http://example.com/?__wb_method=POST&foo=bar&dir=/baz"
165 | )
166 |
167 | def test_post_extract_malformed_form_data(self):
168 | mq = MethodQueryCanonicalizer(
169 | "POST",
170 | "application/x-www-form-urlencoded",
171 | len(self.binary_post_data),
172 | BytesIO(self.binary_post_data),
173 | )
174 |
175 | # base64 encoded data
176 | assert (
177 | mq.append_query("http://example.com/")
178 | == "http://example.com/?__wb_method=POST&__wb_post_data=gTZsYEygNFAO4HICtYkZAGZQ2w6wAiw="
179 | )
180 |
181 | def test_post_extract_no_boundary_in_multipart_form_mimetype(self):
182 | mq = MethodQueryCanonicalizer(
183 | "POST", "multipart/form-data", len(self.post_data), BytesIO(self.post_data)
184 | )
185 |
186 | assert (
187 | mq.append_query("http://example.com/")
188 | == "http://example.com/?__wb_method=POST&__wb_post_data=Zm9vPWJhciZkaXI9JTJGYmF6"
189 | )
190 |
191 | def test_options(self):
192 | mq = MethodQueryCanonicalizer("OPTIONS", "", 0, BytesIO())
193 | assert (
194 | mq.append_query("http://example.com/")
195 | == "http://example.com/?__wb_method=OPTIONS"
196 | )
197 |
198 | def test_head(self):
199 | mq = MethodQueryCanonicalizer("HEAD", "", 0, BytesIO())
200 | assert (
201 | mq.append_query("http://example.com/")
202 | == "http://example.com/?__wb_method=HEAD"
203 | )
204 |
--------------------------------------------------------------------------------
/tests/data-special/qsl.net-encoding-alias.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data-special/qsl.net-encoding-alias.warc.gz
--------------------------------------------------------------------------------
/tests/data/bad-redirections.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/bad-redirections.warc.gz
--------------------------------------------------------------------------------
/tests/data/content-resource-types.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/content-resource-types.warc.gz
--------------------------------------------------------------------------------
/tests/data/empty-file:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/empty-file
--------------------------------------------------------------------------------
/tests/data/example-response.warc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/example-response.warc
--------------------------------------------------------------------------------
/tests/data/example-revisit.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/example-revisit.warc.gz
--------------------------------------------------------------------------------
/tests/data/example-utf8.warc:
--------------------------------------------------------------------------------
1 | WARC/1.0
2 | WARC-IP-Address: 3.220.112.94
3 | WARC-Type: response
4 | WARC-Record-ID:
5 | WARC-Target-URI: https://httpbin.org/anything/utf8=%E2%9C%93?query=test&a=b&1=%E2%9C%93
6 | WARC-Date: 2020-07-09T20:41:58Z
7 | WARC-Payload-Digest: sha1:APPCHQ3MPXOSGO3CDWKQD2TIOJGCLD7B
8 | WARC-Block-Digest: sha1:6X2SE5VSJI4EQHXZ7C5AWTU5H2HXKXMW
9 | Content-Type: application/http; msgtype=response
10 | Content-Length: 818
11 |
12 | HTTP/1.1 200 OK
13 | Date: Thu, 09 Jul 2020 20:41:58 GMT
14 | Content-Type: application/json
15 | Content-Length: 588
16 | Connection: keep-alive
17 | Server: gunicorn/19.9.0
18 | Access-Control-Allow-Origin: *
19 | Access-Control-Allow-Credentials: true
20 |
21 | {
22 | "args": {
23 | "1": "\u2713",
24 | "a": "b",
25 | "query": "test"
26 | },
27 | "data": "",
28 | "files": {},
29 | "form": {},
30 | "headers": {
31 | "Accept": "*/*",
32 | "Accept-Encoding": "gzip, deflate",
33 | "Host": "httpbin.org",
34 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
35 | "X-Amzn-Trace-Id": "Root=1-5f078116-58ed90b44cbd1e86b92d2c19"
36 | },
37 | "json": null,
38 | "method": "GET",
39 | "origin": "161.35.72.192",
40 | "url": "https://httpbin.org/anything/utf8=\u2713?query=test&a=b&1=\u2713"
41 | }
42 |
43 |
44 | WARC/1.0
45 | WARC-IP-Address: 3.220.112.94
46 | WARC-Type: request
47 | WARC-Record-ID:
48 | WARC-Target-URI: https://httpbin.org/anything/utf8=%E2%9C%93?query=test&a=b&1=%E2%9C%93
49 | WARC-Date: 2020-07-09T20:41:58Z
50 | WARC-Payload-Digest: sha1:3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ
51 | WARC-Concurrent-To:
52 | WARC-Block-Digest: sha1:CJ2JE5XZQ6K2BY2ILND5LBBKSWJWFG44
53 | Content-Type: application/http; msgtype=request
54 | Content-Length: 291
55 |
56 | GET /anything/utf8=%E2%9C%93?query=test&a=b&1=%E2%9C%93 HTTP/1.1
57 | Host: httpbin.org
58 | User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36
59 | Accept-Encoding: gzip, deflate
60 | Accept: */*
61 | Connection: keep-alive
62 |
63 |
64 |
65 | WARC/1.0
66 | WARC-IP-Address: 3.220.112.94
67 | WARC-Type: response
68 | WARC-Record-ID:
69 | WARC-Target-URI: https://httpbin.org/anything/utf8=%E2%9C%93?query=test&a=b&1=%E2%9C%93
70 | WARC-Date: 2020-07-12T04:55:45Z
71 | WARC-Payload-Digest: sha1:3C2SYPEZC3KIGXFSCKVWEC3NE3WIQOAI
72 | WARC-Block-Digest: sha1:VFMTQFAU7C4XNASY5LAXE2JH3PEDJNHW
73 | Content-Type: application/http; msgtype=response
74 | Content-Length: 818
75 |
76 | HTTP/1.1 200 OK
77 | Date: Sun, 12 Jul 2020 04:55:45 GMT
78 | Content-Type: application/json
79 | Content-Length: 588
80 | Connection: keep-alive
81 | Server: gunicorn/19.9.0
82 | Access-Control-Allow-Origin: *
83 | Access-Control-Allow-Credentials: true
84 |
85 | {
86 | "args": {
87 | "1": "\u2713",
88 | "a": "b",
89 | "query": "test"
90 | },
91 | "data": "",
92 | "files": {},
93 | "form": {},
94 | "headers": {
95 | "Accept": "*/*",
96 | "Accept-Encoding": "gzip, deflate",
97 | "Host": "httpbin.org",
98 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
99 | "X-Amzn-Trace-Id": "Root=1-5f0a97d1-68b1d51c705ee29c610f8d2b"
100 | },
101 | "json": null,
102 | "method": "GET",
103 | "origin": "161.35.72.192",
104 | "url": "https://httpbin.org/anything/utf8=\u2713?query=test&a=b&1=\u2713"
105 | }
106 |
107 |
108 | WARC/1.0
109 | WARC-IP-Address: 3.220.112.94
110 | WARC-Type: request
111 | WARC-Record-ID:
112 | WARC-Target-URI: https://httpbin.org/anything/utf8=%E2%9C%93?query=test&a=b&1=%E2%9C%93
113 | WARC-Date: 2020-07-12T04:55:45Z
114 | WARC-Payload-Digest: sha1:3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ
115 | WARC-Concurrent-To:
116 | WARC-Block-Digest: sha1:CJ2JE5XZQ6K2BY2ILND5LBBKSWJWFG44
117 | Content-Type: application/http; msgtype=request
118 | Content-Length: 291
119 |
120 | GET /anything/utf8=%E2%9C%93?query=test&a=b&1=%E2%9C%93 HTTP/1.1
121 | Host: httpbin.org
122 | User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36
123 | Accept-Encoding: gzip, deflate
124 | Accept: */*
125 | Connection: keep-alive
126 |
127 |
128 |
129 | WARC/1.0
130 | WARC-IP-Address: 54.236.246.173
131 | WARC-Type: response
132 | WARC-Record-ID:
133 | WARC-Target-URI: https://httpbin.org/anything/Saint-Exup%C3%A9ry?foo=bar
134 | WARC-Date: 2020-07-12T04:55:45Z
135 | WARC-Payload-Digest: sha1:AH45AISVSL5WPLO7KFHMYYQUGHU3QM7I
136 | WARC-Block-Digest: sha1:LAYKJWASNW67SCJNBSPRCLHA57WR5B3M
137 | Content-Type: application/http; msgtype=response
138 | Content-Length: 771
139 |
140 | HTTP/1.1 200 OK
141 | Date: Sun, 12 Jul 2020 04:55:45 GMT
142 | Content-Type: application/json
143 | Content-Length: 541
144 | Connection: keep-alive
145 | Server: gunicorn/19.9.0
146 | Access-Control-Allow-Origin: *
147 | Access-Control-Allow-Credentials: true
148 |
149 | {
150 | "args": {
151 | "foo": "bar"
152 | },
153 | "data": "",
154 | "files": {},
155 | "form": {},
156 | "headers": {
157 | "Accept": "*/*",
158 | "Accept-Encoding": "gzip, deflate",
159 | "Host": "httpbin.org",
160 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
161 | "X-Amzn-Trace-Id": "Root=1-5f0a97d1-3e7c0264c617439a850ca5a5"
162 | },
163 | "json": null,
164 | "method": "GET",
165 | "origin": "161.35.72.192",
166 | "url": "https://httpbin.org/anything/Saint-Exup\u00e9ry?foo=bar"
167 | }
168 |
169 |
170 | WARC/1.0
171 | WARC-IP-Address: 54.236.246.173
172 | WARC-Type: request
173 | WARC-Record-ID:
174 | WARC-Target-URI: https://httpbin.org/anything/Saint-Exup%C3%A9ry?foo=bar
175 | WARC-Date: 2020-07-12T04:55:45Z
176 | WARC-Payload-Digest: sha1:3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ
177 | WARC-Concurrent-To:
178 | WARC-Block-Digest: sha1:PZFUDJ3ZKH5JZQ66M7E47TD2GP4ZZABR
179 | Content-Type: application/http; msgtype=request
180 | Content-Length: 276
181 |
182 | GET /anything/Saint-Exup%C3%A9ry?foo=bar HTTP/1.1
183 | Host: httpbin.org
184 | User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36
185 | Accept-Encoding: gzip, deflate
186 | Accept: */*
187 | Connection: keep-alive
188 |
189 |
190 |
191 | WARC/1.0
192 | WARC-IP-Address: 54.236.246.173
193 | WARC-Type: response
194 | WARC-Record-ID:
195 | WARC-Target-URI: https://httpbin.org/anything/test?foo=%C3%A9
196 | WARC-Date: 2020-07-12T04:55:46Z
197 | WARC-Payload-Digest: sha1:UWV3XF55NFX2YU23BUEY52H6LK54LV7G
198 | WARC-Block-Digest: sha1:EDHXZSWGEJUKE6WMJZYKZHC2SLQEM5U5
199 | Content-Type: application/http; msgtype=response
200 | Content-Length: 763
201 |
202 | HTTP/1.1 200 OK
203 | Date: Sun, 12 Jul 2020 04:55:45 GMT
204 | Content-Type: application/json
205 | Content-Length: 533
206 | Connection: keep-alive
207 | Server: gunicorn/19.9.0
208 | Access-Control-Allow-Origin: *
209 | Access-Control-Allow-Credentials: true
210 |
211 | {
212 | "args": {
213 | "foo": "\u00e9"
214 | },
215 | "data": "",
216 | "files": {},
217 | "form": {},
218 | "headers": {
219 | "Accept": "*/*",
220 | "Accept-Encoding": "gzip, deflate",
221 | "Host": "httpbin.org",
222 | "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
223 | "X-Amzn-Trace-Id": "Root=1-5f0a97d1-61e09380e5ba93c0c33af8c0"
224 | },
225 | "json": null,
226 | "method": "GET",
227 | "origin": "161.35.72.192",
228 | "url": "https://httpbin.org/anything/test?foo=\u00e9"
229 | }
230 |
231 |
232 | WARC/1.0
233 | WARC-IP-Address: 54.236.246.173
234 | WARC-Type: request
235 | WARC-Record-ID:
236 | WARC-Target-URI: https://httpbin.org/anything/test?foo=%C3%A9
237 | WARC-Date: 2020-07-12T04:55:46Z
238 | WARC-Payload-Digest: sha1:3I42H3S6NNFQ2MSVX7XZKYAYSCX5QBYJ
239 | WARC-Concurrent-To:
240 | WARC-Block-Digest: sha1:R5RVON3BIZ6XHBHC2CNZACZXOQU2TKSA
241 | Content-Type: application/http; msgtype=request
242 | Content-Length: 265
243 |
244 | GET /anything/test?foo=%C3%A9 HTTP/1.1
245 | Host: httpbin.org
246 | User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36
247 | Accept-Encoding: gzip, deflate
248 | Accept: */*
249 | Connection: keep-alive
250 |
251 |
252 |
253 |
--------------------------------------------------------------------------------
/tests/data/example-with-timestamp.warc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/example-with-timestamp.warc
--------------------------------------------------------------------------------
/tests/data/http-return-codes.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/http-return-codes.warc.gz
--------------------------------------------------------------------------------
/tests/data/kiwix-with-redirects.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/kiwix-with-redirects.warc.gz
--------------------------------------------------------------------------------
/tests/data/main-entry-403.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/main-entry-403.warc.gz
--------------------------------------------------------------------------------
/tests/data/self-redirect.warc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/self-redirect.warc
--------------------------------------------------------------------------------
/tests/data/single-page-test.warc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/single-page-test.warc
--------------------------------------------------------------------------------
/tests/data/solidaritenum.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/solidaritenum.warc.gz
--------------------------------------------------------------------------------
/tests/data/video-vimeo.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/video-vimeo.warc.gz
--------------------------------------------------------------------------------
/tests/data/video-yt-2.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/video-yt-2.warc.gz
--------------------------------------------------------------------------------
/tests/data/video-yt.warc.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/openzim/warc2zim/270f5dbaaed6455ccb1dc05e4f10cf460f41b992/tests/data/video-yt.warc.gz
--------------------------------------------------------------------------------
/tests/encodings/definition.json:
--------------------------------------------------------------------------------
1 | {
2 | "files": [
3 | {
4 | "filename": "file01.js",
5 | "source": "https://www.marxists.org/espanol/menu.js",
6 | "date": "2024-06",
7 | "probable_charset": "ISO-8859-1",
8 | "expected_strings": [
9 | "Afanásiev, Víktor",
10 | "Andrópov, Yuri",
11 | "Amaguaña, Tránsito",
12 | "Cunhal, Álvaro",
13 | "De la Cruz, Juana Inés",
14 | "Faure, Sèbastien"
15 | ]
16 | },
17 | {
18 | "filename": "file02.js",
19 | "source": "https://www.cloudflare.com/vendor/onetrust/scripttemplates/202308.2.0/otBannerSdk.js",
20 | "date": "2024-06",
21 | "probable_charset": "UTF-8",
22 | "expected_strings": [
23 | "_Container:\"#ot-ven-lst\",P_Ven_Bx:\"ot-ven-box\",P_Ven_Name:\".ot-ven-name\"",
24 | "ist,IabType:e.IabType,InactiveText:e.InactiveText,IsConsentLoggingEnabled:e.IsConsentLoggingEnabl",
25 | "0;\\n transition: visibility 0s \"+e+\"ms, opacity \"+e+\"ms linear;\\n \",!0);var",
26 | "r.prototype.escapeRegExp=function(e){return e.replace(/[-/\\\\^$*+?.()|[\\]{}]/g,\"\\\\$&\")}"
27 | ]
28 | },
29 | {
30 | "filename": "file03.html",
31 | "source": "https://www.solidarite-numerique.fr/tutoriels/comprendre-les-cookies/?thematique=internet",
32 | "date": "2024-06",
33 | "probable_charset": "UTF-8",
34 | "contains_bad_chars": true,
35 | "expected_strings": [
36 | "Vous souhaitez changer de navigateur et utiliser Firefox ? Ce tutoriel vous détaille la procédure d'installation et la configuration pour une premi�..."
37 | ]
38 | },
39 | {
40 | "filename": "file04.js",
41 | "source": "https://static.mailerlite.com/js/w/ml_jQuery.inputmask.bundle.min.js?v3.3.1",
42 | "date": "2024-06",
43 | "probable_charset": "ascii",
44 | "expected_strings": [
45 | "1,this.isOptional=b||!1,this.isQuantifier=c||!1,this.isAlterna",
46 | "is;if(na=!1,g.clearMaskOnLostFocus&&document.activeElement!==b){var c=x().slice(),d=b.inputmask._v"
47 | ]
48 | },
49 | {
50 | "filename": "file05.js",
51 | "source": "https://static.sketchfab.com/static/builds/web/dist/ac0f732c4fc1a30c77920d75c1a9be83-v2.js",
52 | "date": "2024-06",
53 | "probable_charset": "ascii",
54 | "expected_strings": [
55 | "isTickUsed||(this._isTickUsed=!0,this._schedule(this.drainQueues))},s.prototype._reset=function(){this._is"
56 | ]
57 | },
58 | {
59 | "filename": "file06.html",
60 | "source": "https://website.test.openzim.org/chinese-encoding.html",
61 | "date": "2024-06",
62 | "known_charset": "gb2312",
63 | "expected_strings": [
64 | "simplified chinese characters: 汉字"
65 | ]
66 | },
67 | {
68 | "filename": "file07.html",
69 | "source": "https://website.test.openzim.org/chinese-encoding.html without 
35 | 
47 |
48 | 
58 |
59 | 
75 | 
100 | 
22 |
23 | 
27 |
28 | 
32 |
33 | 
37 |
38 | 
44 |
45 | 
49 |
50 | 
54 |
55 | 
61 |
62 | 
66 |
67 | 
71 |
72 | .png)
78 |
79 | 
83 |
84 | 
90 |
91 | .png)
95 |
96 | _1.png)
100 |
101 | _2.png)
105 |
106 |
107 |
108 |
109 |
--------------------------------------------------------------------------------
/test-website/content/video.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |