25 |
26 | {% endblock %}
27 |
--------------------------------------------------------------------------------
/lib/tagdef.html:
--------------------------------------------------------------------------------
1 | {% set tag_table = {
2 | 'none': ('no warnings', 'No content warnings'),
3 | 'text-porn': ('pornographic text', 'Sexual content meant principly to arouse'),
4 | 'visual-porn': ('pornographic images', 'Sexual content meant principly to arouse'),
5 | 'visual-gore': ('violent or gory images', 'Violent images or depictions of gore'),
6 | 'self-harm': ('suicide or self-injury', 'Content which encourages or provides instructions for suicide or self-injury'),
7 | 'eating-disorder': ('disordered eating behavior', 'Content which encourages or provides instructions for an eating disorder or behavior associated with same'),
8 | 'unknown': ('not reviewed', 'Content has not been reviewed; any warnings may apply'),
9 | } %}
10 |
--------------------------------------------------------------------------------
/lib/subdirentry.html:
--------------------------------------------------------------------------------
1 |
{{ subdir.name }}
2 |
3 | ◆
4 |
5 | {%- if subdir.islink %}
6 | [link to {{ subdir.linkdir }}]
7 | {%- endif %}
8 |
9 | {%- if subdir._metadata %}
10 | {% set dat = subdir._metadata %} {% set datel = 'dd' %}
11 | {% include "metadata.html" %}
12 | {%- endif %}
13 |
14 | {%- if subdir.hasdesc %}
15 |
{% autoescape false %}{{ subdir.desc }}{% endautoescape %}
16 | {%- endif %}
17 |
--------------------------------------------------------------------------------
/lib/no-index-entry:
--------------------------------------------------------------------------------
1 | if-archive/README
2 | if-archive/directory-tree
3 | if-archive/new-since-last-post
4 | if-archive/Master-Index
5 | if-archive/md5.txt
6 | if-archive/ls-lR
7 | if-archive/games/appleII/eamon
8 | if-archive/games/competition
9 | if-archive/info/archive-stats
10 | if-archive/info/ifdb
11 | if-archive/info/ifwiki
12 | if-archive/info/intficforum
13 | if-archive/infocom/patches
14 | if-archive/infocom/compilers/zilf/old
15 | if-archive/programming/glk/old
16 | if-archive/programming/glk/implementations/old
17 | if-archive/programming/glulx/interpreters/glulxe/old
18 | if-archive/programming/glulx/interpreters/quixe/old
19 | if-archive/programming/glulx/interpreters/git/old
20 | if-archive/rec.arts.int-fiction
21 | if-archive/rec.games.int-fiction
22 | if-archive/unprocessed
23 |
--------------------------------------------------------------------------------
/testdata/if-archive/games/Index:
--------------------------------------------------------------------------------
1 |
2 | Hey: these are some game files.
3 |
4 | - A directory link:
5 | - And a file link:
6 | - And a file index link: or
7 |
8 | # game1
9 | tuid: xyz111
10 | ifwiki: Adventure
11 | safety: text-porn
12 |
13 | Game number 1.
14 |
15 | # game2
16 | tuid: xyz222
17 |
18 | # bothgames
19 | tuid: xyz111
20 | xyz222
21 |
22 | # all.zip
23 |
24 | Linked to comp/files.
25 |
26 | # old
27 | tuid: abc12345
28 |
29 | The directory of old stuff.
30 |
31 | # old/older
32 | tuid: def56789
33 |
34 | A double-depth directory.
35 |
36 | # old/older/game1-pre
37 | tuid: ghi98765
38 | safety: self-harm
39 |
40 | Triple-depth file reference.
41 |
--------------------------------------------------------------------------------
/lib/blocktag-head.txt:
--------------------------------------------------------------------------------
1 | # File blocking tags for the archiveblock plugin
2 | # See: https://github.com/iftechfoundation/ifarchive-archiveblock
3 |
4 | # This file is generated from the "safety" tags in the Index files.
5 | # Last generated: {{ date|isodate }}
6 |
7 | # Each line represents a file or directory which is blocked in the UK.
8 | # A pathname that ends with "/*" tags all the files in that directory.
9 | # A pathname that ends with "/**" tags all the files in that directory *and*
10 | # its subdirectories.
11 |
12 | # The next field is separated from the pathname by a tab. The field has
13 | # the form "FLAGS:TAGS". The FLAGS is empty or "u" (meaning "block this
14 | # file in the UK"). The TAGS are a comma-separated list of values;
15 | # see https://ifarchive.org/misc/org-procedures.html.
16 |
17 |
--------------------------------------------------------------------------------
/lib/rss.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | IFArchive Recent Additions
6 | https://ifarchive.org
7 | New additions to the Interactive Fiction Archive
8 | en-us
9 | tech@ifarchive.org (Archive Support)
10 | {{ changedate |isodate }}
11 | {{ curdate |isodate }}
12 |
13 | {% for file in _files %}
14 |
15 | {{ file.name }}
16 | https://ifarchive.org/{{ file.path |urlencode }}
17 | {{ file.path |urlencode }}::{{ file.date }}
18 | {{ file.date |isodate }}
19 |
20 | {%- if file.hasxmldesc %}
21 | {{ file.xmldesc }}
22 | {%- endif %}
23 | [at: {{ file.path |urlencode }}]
24 |
25 |
26 | {% endfor %}
27 |
28 |
29 |
30 |
--------------------------------------------------------------------------------
/nightly-backup:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # Sync the files tree (not indexes or other material) to an AWS bucket.
4 | # The bucket is configured with versioning and a 7-day expiration for
5 | # old file versions, so we get that much rolling history.
6 |
7 | # As of late 2024, this takes about 90 seconds to run for a no-change
8 | # sync.
9 |
10 | # This requires secret credentials, so it must be run as root.
11 |
12 | # Requires rclone 1.53 or later.
13 | # /var/ifarchive/lib/rclone.conf: rclone configuration (names AWS as the provider)
14 | # /var/ifarchive/lib/aws.config: AWS access keys (must be root to read)
15 |
16 | /usr/local/bin/rclone --verbose --config /var/ifarchive/lib/rclone.conf --s3-shared-credentials-file /var/ifarchive/lib/aws.config sync /var/ifarchive/htdocs/if-archive s3:iftf-ifarchive-backup --checksum --links --checkers 50 --transfers 10 &>> /var/ifarchive/logs/nightly-backup.log
17 |
18 | # Docs suggest a fast "top-up" of recent files, and then a full sync less often. I haven't set that up yet. See: https://rclone.org/s3/
19 |
--------------------------------------------------------------------------------
/testdata/if-archive/Index:
--------------------------------------------------------------------------------
1 |
2 | > _You feel somewhat disoriented as you pass through..._
3 |
4 | This is the Interactive Fiction Archive at
5 | [ifarchive.org](/).
6 |
7 | You may be interested in...
8 |
9 | - The [README file for the Archive](/if-archive/README).
10 | - The [list of directories](/indexes/dirmap.html) in the Archive.
11 | - New arrivals in the [past week](/indexes/datev_1.html), [past month](/indexes/datev_2.html), [past three months](/indexes/datev_3.html), or [past year](/indexes/datev_4.html).
12 | - An [RSS feed](/indexes/archive.rss) of recent arrivals.
13 | - A list of [every file in the archive](/indexes/date.html) (tens of thousands!)
14 | - [Master-Index.xml](/indexes/Master-Index.xml), which describes every file in the Archive (very large XML file!)
15 |
16 | Or, go delving into the hierarchical tree shown here.
17 |
18 | # art
19 |
20 | Art stuff.
21 |
22 | # comp
23 |
24 | Competitions. This is a long line of text so we can see how the column width is going to work out in real life.
25 |
26 | # games
27 |
28 | Games stuff.
29 |
--------------------------------------------------------------------------------
/lib/dirmap.html:
--------------------------------------------------------------------------------
1 | {% extends "page.html" %}
2 | {% from 'macros.html' import wbrslash %}
3 |
4 | {% block description %}
5 |
6 | This is a somewhat abbreviated list of the directories on the Archive.
7 | Directories for individual competition games and outdated game versions
8 | have been omitted.
9 | For a really absolutely complete list of directories,
10 | see here.
11 |
6 | {% if interval %}
7 | This page lists the most recent {{ interval }} of files in the Archive,
8 | sorted by date.
9 | {% else %}
10 | This page lists every file in the Archive, sorted by date.
11 | {% endif %}
12 | (The last update of the file, not the date it first entered the
13 | Archive.) Note that this list is based on filesystem timestamps,
14 | which may be inaccurate – or just plain wrong –
15 | for various reasons.
16 |
29 |
30 | This page lists the most recent week of files in the Archive,
31 | sorted by date.
32 |
33 | (The last update of the file, not the date it first entered the
34 | Archive.) Note that this list is based on filesystem timestamps,
35 | which may be inaccurate – or just plain wrong –
36 | for various reasons.
37 |
29 |
30 | This page lists the most recent week of files in the Archive,
31 | sorted by date.
32 |
33 | (The last update of the file, not the date it first entered the
34 | Archive.) Note that this list is based on filesystem timestamps,
35 | which may be inaccurate – or just plain wrong –
36 | for various reasons.
37 |
130 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # ifmap.py -- the index generator tool for the IF Archive
2 |
3 | - Copyright 2017-23 by the Interactive Fiction Technology Foundation
4 | - Distributed under the MIT license
5 | - Created by Andrew Plotkin
6 |
7 | This program has one core task: to look through all the files in the IF Archive, combine that with the contents of the Master-Index file, and generate all the index.html files in the indexes subdirectory.
8 |
9 | (The Master-Index file is created by sewing together all the Index files in all the directories of the Archive. That's done by the make-master-index.py script, also included in this repo.)
10 |
11 | Related files in this repo:
12 |
13 | - `make-master-index.py`: Sew together all the Index files to create Master-Index.
14 | - `make-master-index`: Shell script wrapper to run `make-master-index.py` on the Archive.
15 | - `build-indexes`: Shell script wrapper to run `ifmap.py` on the Archive.
16 | - `build-indexes-bg`: Another shell script wrapper to run both `make-master-index` and `build-indexes` sequentially in the background. (This is used by the admin web interface.)
17 | - `uncache.py`: Cloudflare API request to purge an Archive file from the CDN cache.
18 | - `nightly-backup`: Shell script wrapper for the `rclone` nightly backup to AWS.
19 |
20 | (Okay, some of these are only tangentially related. Miscellaneous admin scripts have accumulated here.)
21 |
22 | ## Setup
23 |
24 | The generator uses the Python Markdown and Jinja packages. To install these:
25 |
26 | ```
27 | pip3 install -r requirements.txt
28 | ```
29 |
30 | ## Arguments
31 |
32 | In normal Archive operation, this is invoked from the `build-indexes` script.
33 |
34 | - --index FILE: pathname of Master-Index. (Normally /var/ifarchive/htdocs/if-archive/Master-Index.)
35 | - --src DIR: Pathname of the directory full of HTML templates which control the appearance of the index files. (Normally /var/ifarchive/lib/ifmap.)
36 | - --tree DIR: Pathname of the root directory which the Archive serves. (Normally /var/ifarchive/htdocs.)
37 | - --v: If set, print verbose output.
38 | - --exclude: If set, files without index entries are excluded from index listings. (Normally *not* set.)
39 | - --dest DIR: Firectory to write index files (relative to --tree; default "indexes")
40 | - --meta DIR: Firectory to write metadata files (relative to --tree; default "metadata")
41 |
42 | The `--dest` and `--meta` arguments exist only for development testing. If you use any value other than the default ("indexes", "metadata"), the generated indexes won't properly link to anything.
43 |
44 | ## Testing
45 |
46 | Type `python3 tests.py` to run tests on the low-level string-escaping and templating code.
47 |
48 | For an end-to-end test, try:
49 |
50 | ./end-to-end-test
51 |
52 | If everything works, the generated files in testdata/indexes should match what's in the Git repository. (`git status` should show no changes.)
53 |
54 | ## History
55 |
56 | I wrote the first version of this program in 1999-ish. It was built around the original Index files, which were hand-written by Volker Blasius (the original Archive curator) for human consumption. Their format was not particularly convenient for parsing, but I parsed them anyway.
57 |
58 | I wrote the original program in C because it was portable and I didn't know Python or Perl yet. C is a terrible language for this sort of thing, of course -- I started by implementing my own hash tables. And escaping strings for HTML? Yuck.
59 |
60 | I finally ported it all to Python in July of 2017. It's now got less than half the lines of code, it's infinitely more readable, and it's faster. (Not because Python is faster, but because I added an MD5-caching feature.)
61 |
62 | December 2018: Added SHA512 checksums to the output (and to md5-cache.txt, which is now misnamed). Updated the script to generate foo/bar/index.html indexes as well as fooXbar.html indexes.
63 |
64 | December 2019: Renamed md5-cache.txt to checksum-cache.txt and moved it to the archive root. Simplified the arguments a bit (--dest is usually not needed). Updated everything to use the new Markdown-based Index format.
65 |
66 | October 2023: Directories can now have metadata, just like files. Support the "ifwiki" metadata line.
67 |
68 | November 2023: Generate the RSS feed along with all the other index files.
69 |
70 |
--------------------------------------------------------------------------------
/testdata/indexes/if-archive/comp/files/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | Index: if-archive/comp/files
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
29 | This is a somewhat abbreviated list of the directories on the Archive.
30 | Directories for individual competition games and outdated game versions
31 | have been omitted.
32 | For a really absolutely complete list of directories,
33 | see here.
34 |
29 |
30 | This page lists the most recent month of files in the Archive,
31 | sorted by date.
32 |
33 | (The last update of the file, not the date it first entered the
34 | Archive.) Note that this list is based on filesystem timestamps,
35 | which may be inaccurate – or just plain wrong –
36 | for various reasons.
37 |
29 |
30 | This page lists the most recent year of files in the Archive,
31 | sorted by date.
32 |
33 | (The last update of the file, not the date it first entered the
34 | Archive.) Note that this list is based on filesystem timestamps,
35 | which may be inaccurate – or just plain wrong –
36 | for various reasons.
37 |
29 |
30 | This page lists the most recent three months of files in the Archive,
31 | sorted by date.
32 |
33 | (The last update of the file, not the date it first entered the
34 | Archive.) Note that this list is based on filesystem timestamps,
35 | which may be inaccurate – or just plain wrong –
36 | for various reasons.
37 |
29 |
30 | This page lists the most recent month of files in the Archive,
31 | sorted by date.
32 |
33 | (The last update of the file, not the date it first entered the
34 | Archive.) Note that this list is based on filesystem timestamps,
35 | which may be inaccurate – or just plain wrong –
36 | for various reasons.
37 |
29 |
30 | This page lists every file in the Archive, sorted by date.
31 |
32 | (The last update of the file, not the date it first entered the
33 | Archive.) Note that this list is based on filesystem timestamps,
34 | which may be inaccurate – or just plain wrong –
35 | for various reasons.
36 |
29 |
30 | This page lists the most recent year of files in the Archive,
31 | sorted by date.
32 |
33 | (The last update of the file, not the date it first entered the
34 | Archive.) Note that this list is based on filesystem timestamps,
35 | which may be inaccurate – or just plain wrong –
36 | for various reasons.
37 |
29 |
30 | This page lists the most recent three months of files in the Archive,
31 | sorted by date.
32 |
33 | (The last update of the file, not the date it first entered the
34 | Archive.) Note that this list is based on filesystem timestamps,
35 | which may be inaccurate – or just plain wrong –
36 | for various reasons.
37 |