├── script ├── test-uncache-bg ├── create-var-ifarchive.sh └── testframe-start.sh ├── config ├── sudoers_ifarch ├── 000-ifarchive.conf └── ifarch.config ├── .gitmodules ├── Dockerfile ├── docker-compose.yml └── README.md /script/test-uncache-bg: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo 'uncache is not available in test mode' 4 | 5 | exit 1 6 | -------------------------------------------------------------------------------- /config/sudoers_ifarch: -------------------------------------------------------------------------------- 1 | 2 | Cmnd_Alias BUILDINDEX = /var/ifarchive/bin/build-indexes-bg 3 | Cmnd_Alias UNCACHE = /var/ifarchive/bin/uncache-bg 4 | 5 | www-data ALL = NOPASSWD: BUILDINDEX 6 | ifarchive ALL = NOPASSWD: BUILDINDEX 7 | www-data ALL = NOPASSWD: UNCACHE 8 | ifarchive ALL = NOPASSWD: UNCACHE 9 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ifarchive-static"] 2 | path = ifarchive-static 3 | url = https://github.com/iftechfoundation/ifarchive-static 4 | [submodule "ifarchive-ifmap-py"] 5 | path = ifarchive-ifmap-py 6 | url = https://github.com/iftechfoundation/ifarchive-ifmap-py 7 | [submodule "ifarchive-upload-py"] 8 | path = ifarchive-upload-py 9 | url = https://github.com/iftechfoundation/ifarchive-upload-py 10 | [submodule "ifarchive-admintool"] 11 | path = ifarchive-admintool 12 | url = https://github.com/iftechfoundation/ifarchive-admintool 13 | [submodule "ifarchive-search"] 14 | path = ifarchive-search 15 | url = https://github.com/iftechfoundation/ifarchive-search 16 | -------------------------------------------------------------------------------- /script/create-var-ifarchive.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Initial Archive setup. 4 | # This runs when the Docker container is built. 5 | 6 | # Create two users. 7 | adduser --gecos INFO --disabled-password ifarchive 8 | adduser --gecos INFO --disabled-password uploaders 9 | adduser ifarchive uploaders 10 | adduser www-data uploaders 11 | 12 | # Create the directory tree that Archive files will live in. 13 | 14 | mkdir /var/ifarchive 15 | cd /var/ifarchive 16 | mkdir bin doc lib logs htdocs incoming trash cgi-bin wsgi-bin 17 | mkdir lib/sql lib/searchindex 18 | 19 | chown ifarchive:ifarchive bin doc lib htdocs cgi-bin wsgi-bin 20 | chown www-data:uploaders incoming trash lib/sql 21 | chown www-data:ifarchive logs 22 | 23 | chmod 775 bin doc lib htdocs incoming trash cgi-bin wsgi-bin 24 | chmod 770 logs lib/sql 25 | 26 | mkdir htdocs/misc htdocs/if-archive htdocs/indexes htdocs/metadata 27 | chown ifarchive:ifarchive htdocs/* 28 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # The Archive machine is on a pretty old version of Debian because I don't think about updating very often. 2 | FROM debian:10.13 3 | 4 | EXPOSE 80 5 | 6 | # Install Apache and Python. 7 | RUN apt-get update -y 8 | RUN apt-get install -y sudo apache2 apache2-dev python3 python3-pip libapache2-mod-wsgi-py3 9 | 10 | RUN pip3 install markdown Jinja2 pytz mod-wsgi whoosh 11 | 12 | # Copy over a script that creates the basic directory structure. 13 | COPY --chmod=755 script/create-var-ifarchive.sh /tmp 14 | RUN /tmp/create-var-ifarchive.sh 15 | 16 | # Set up sudoers config. (Needed for the admin tool to run build-indexes-bg.) 17 | COPY --chmod=440 config/sudoers_ifarch /etc/sudoers.d 18 | 19 | # Set up Apache config. 20 | COPY config/000-ifarchive.conf /etc/apache2/sites-available 21 | RUN a2dissite 000-default 22 | RUN a2ensite 000-ifarchive 23 | RUN a2enmod headers 24 | RUN a2enmod rewrite 25 | RUN a2enmod cgid 26 | 27 | # Copy over config file for our scripts. 28 | COPY --chown=ifarchive:uploaders --chmod=640 config/ifarch.config /var/ifarchive/lib 29 | 30 | # Copy over test data for the web server. (Maybe this should be a mount.) 31 | COPY --chown=ifarchive ifarchive-ifmap-py/testdata/if-archive /var/ifarchive/htdocs/if-archive 32 | COPY ifarchive-ifmap-py/testdata/set-timestamps.py /var/ifarchive/bin/testdata-set-timestamps.py 33 | 34 | # Bang the timestamps on the (test) if-archive files. 35 | RUN python3 /var/ifarchive/bin/testdata-set-timestamps.py /var/ifarchive/htdocs/if-archive 36 | 37 | COPY --chmod=755 script/testframe-start.sh /var/ifarchive/bin 38 | 39 | # This script does launch-time setup and then runs Apache (non-backgrounded). 40 | CMD [ "/var/ifarchive/bin/testframe-start.sh" ] 41 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | name: ifarchive-testframe 2 | 3 | services: 4 | ifarch: 5 | build: . 6 | ports: 7 | - 8888:80 8 | volumes: 9 | # We will copy index-orig.html to index.html, with tweaks, at launch 10 | # time. (Sadly this means that index.html cannot be edited on 11 | # the fly, even though it is a mount.) 12 | - ./ifarchive-static/index.html:/var/ifarchive/htdocs/index-orig.html 13 | - ./ifarchive-static/misc:/var/ifarchive/htdocs/misc 14 | - ./ifarchive-ifmap-py/make-master-index:/var/ifarchive/bin/make-master-index 15 | - ./ifarchive-ifmap-py/make-master-index.py:/var/ifarchive/bin/make-master-index.py 16 | - ./ifarchive-ifmap-py/build-indexes:/var/ifarchive/bin/build-indexes 17 | - ./ifarchive-ifmap-py/build-indexes-bg:/var/ifarchive/bin/build-indexes-bg 18 | - ./ifarchive-ifmap-py/configparse.py:/var/ifarchive/bin/configparse.py 19 | - ./script/test-uncache-bg:/var/ifarchive/bin/uncache-bg 20 | # for ifmap 21 | - ./ifarchive-ifmap-py/ifmap.py:/var/ifarchive/bin/ifmap.py 22 | - ./ifarchive-ifmap-py/lib:/var/ifarchive/lib/ifmap 23 | # for upload 24 | - ./ifarchive-upload-py/upload.py:/var/ifarchive/cgi-bin/upload.py 25 | - ./ifarchive-upload-py/lib:/var/ifarchive/lib/uploader 26 | # for admintool 27 | - ./ifarchive-admintool/admin.wsgi:/var/ifarchive/wsgi-bin/admin.wsgi 28 | - ./ifarchive-admintool/tinyapp:/var/ifarchive/wsgi-bin/tinyapp 29 | - ./ifarchive-admintool/adminlib:/var/ifarchive/wsgi-bin/adminlib 30 | - ./ifarchive-admintool/templates:/var/ifarchive/lib/admintool 31 | # for search 32 | - ./ifarchive-search/search.wsgi:/var/ifarchive/wsgi-bin/search.wsgi 33 | - ./ifarchive-search/searchlib:/var/ifarchive/wsgi-bin/searchlib 34 | - ./ifarchive-search/templates:/var/ifarchive/lib/searchtpl 35 | -------------------------------------------------------------------------------- /script/testframe-start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # This runs when the Docker container is launched. If it's the first 4 | # time, we do some initial setup work. 5 | 6 | # (We can't do this work when the container is built, because it relies 7 | # on scripts and config which are mounted via docker-compose. That stuff 8 | # doesn't exist when the container is being built.) 9 | 10 | if [ -e /var/ifarchive/lib/testframe-started ]; then 11 | 12 | echo 'Testframe already inited' 13 | 14 | else 15 | 16 | echo 'Initing testframe...' 17 | 18 | # Adjust front-page links to be server-relative rather than pointing at 19 | # production domains. 20 | cp /var/ifarchive/htdocs/index-orig.html /var/ifarchive/htdocs/index.html 21 | sed -i 's,https://search.ifarchive.org,,g' /var/ifarchive/htdocs/index.html 22 | sed -i 's,https://upload.ifarchive.org,,g' /var/ifarchive/htdocs/index.html 23 | 24 | # Initial setup for Archive index files 25 | /var/ifarchive/bin/make-master-index 26 | /var/ifarchive/bin/build-indexes 27 | 28 | # Initial setup for the admin tool and uploader 29 | python3 /var/ifarchive/wsgi-bin/admin.wsgi createdb 30 | python3 /var/ifarchive/wsgi-bin/admin.wsgi adduser fred fred@example.com password --roles admin 31 | 32 | # Initial setup for the search library 33 | python3 /var/ifarchive/wsgi-bin/search.wsgi build --create 34 | chown www-data:uploaders /var/ifarchive/lib/sql/* 35 | chmod 660 /var/ifarchive/lib/sql/* 36 | 37 | # Those setup commands created some logs; make sure they have the right owner. 38 | chown www-data:www-data /var/ifarchive/logs/search.log 39 | chown www-data:www-data /var/ifarchive/logs/admintool.log 40 | 41 | # Note so that the above code is once-only 42 | touch /var/ifarchive/lib/testframe-started 43 | 44 | fi 45 | 46 | echo 'Starting testframe...' 47 | 48 | # Run Apache (with logs on stdout) as our front container process. 49 | apachectl -D FOREGROUND 50 | -------------------------------------------------------------------------------- /config/000-ifarchive.conf: -------------------------------------------------------------------------------- 1 | # This is the Apache configuration for running the tool in Docker. 2 | # 3 | # This is a simplified version of the production Apache config file. 4 | # The main difference is that everything runs on a single virtual host. 5 | # (Not divided up into ifarchive.org, search.ifarchive.org, 6 | # upload.ifarchive.org, etc.) 7 | 8 | ServerName localhost 9 | 10 | 11 | ServerAdmin webmaster@localhost 12 | 13 | DocumentRoot /var/ifarchive/htdocs 14 | 15 | ErrorLog /dev/stdout 16 | CustomLog /dev/stdout combined 17 | 18 | ScriptAlias /cgi-bin/ /var/ifarchive/cgi-bin/ 19 | WSGIScriptAlias /admin /var/ifarchive/wsgi-bin/admin.wsgi 20 | WSGIScriptAlias /search /var/ifarchive/wsgi-bin/search.wsgi 21 | 22 | 23 | Order allow,deny 24 | Allow from all 25 | Require all granted 26 | Options +Indexes +FollowSymLinks +MultiViews 27 | DirectoryIndex index.html index.htm 28 | ReadmeName /misc/autoindex-footer.html 29 | 30 | 31 | # The Archive content directory (and subdirs) have custom auto-indexing. 32 | 33 | HeaderName Index 34 | ReadmeName /misc/autoindex-footer.html 35 | IndexOptions FancyIndexing HTMLTable SuppressDescription SuppressIcon IgnoreCase FoldersFirst 36 | 37 | 38 | # The "Index" files are presumed to be UTF-8. 39 | 40 | ForceType text/plain;charset="UTF-8" 41 | 42 | 43 | 44 | Options +ExecCGI 45 | SetHandler cgi-script 46 | Order allow,deny 47 | Allow from all 48 | Require all granted 49 | SetEnv LANG en_US.UTF-8 50 | 51 | 52 | 53 | Require all granted 54 | SetEnv LANG en_US.UTF-8 55 | 56 | 57 | 58 | # This line must be at the top level, even though it is only used 59 | # by WSGIScriptAlias directives in the VirtualHost above. 60 | WSGIPythonPath /var/ifarchive/wsgi-bin 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # IF Archive test framework 2 | 3 | This repository agglomerates all (most) of the repos involved in running the [IF Archive][ifarch]. It allows you to build a test-mode Archive as a Docker container. 4 | 5 | Note: I don't have the container published anywhere. This is meant for testing and development, not real-world use. But you can build it yourself, using the Dockerfile in this repo. See below. 6 | 7 | [ifarch]: https://ifarchive.org 8 | 9 | ## How does this differ from the real IF Archive? 10 | 11 | The real Archive is not built on Docker! Maybe it should be, but the current configuration was set up in 2017 and I didn't use Docker then. This setup is only for testing. 12 | 13 | The real Archive service is split across several subdomains: [ifarchive.org][ifarch], [search.ifarchive.org][if-search], [upload.ifarchive.org][if-upload], and so on. They all currently run on the same Linux instance; the subdomain split is just futureproofing. But it means that the pages have some absolute-URL links to each other. In test mode, we adjust (some of) those links to be server-relative. 14 | 15 | Another Archive subdomain is [unbox.ifarchive.org][if-unbox], which runs on a second Linux instance. Unbox is not yet included in this test framework. 16 | 17 | [if-search]: https://search.ifarchive.org/search 18 | [if-upload]: https://upload.ifarchive.org/cgi-bin/upload.py 19 | [if-unbox]: https://unbox.ifarchive.org/ 20 | 21 | The real Archive hosts several other domains (babel.ifarchive.org, spagmag.org, inform-fiction.org) which are unrelated to the core Archive service. These are not represented in this test framework either. 22 | 23 | On the real Archive, the file directory (`/var/ifarchive/htdocs/if-archive`) is a mount point for a separate storage volume. 24 | 25 | ## To run 26 | 27 | # If the submodules were not filled in by your git client 28 | git submodule init; git submodule update 29 | 30 | # Construct and launch the image (takes a while the first time) 31 | docker compose up 32 | 33 | Then visit `http://localhost:8888/` in your browser. 34 | 35 | The admin interface is at `http://localhost:8888/admin`. Log in with username `fred`, password `password`. 36 | 37 | To get a root shell inside the container: 38 | 39 | docker compose exec -i -t ifarch bash 40 | 41 | 42 | ## To develop 43 | 44 | Most of the scripts and templates used by the various repos are mounted into the container. (See the `volumes` list of `docker-compose.yml`.) This means you can edit files and the updated versions will be visible (and active) immediately inside the container. 45 | 46 | Some changes require you to restart Apache, which means `docker compose restart`. Changes to files in `tinyapp`, `searchlib`, and `adminlib` are in this category. 47 | 48 | A few files are copied into the Docker image and cannot be changed on the fly. Currently these are `ifarch.config` and the test files that populate the Archive. If you change these, you'll need to do `docker compose down; docker compose build; docker compose up`. 49 | -------------------------------------------------------------------------------- /config/ifarch.config: -------------------------------------------------------------------------------- 1 | [DEFAULT] 2 | 3 | # This is the test configuration for the IF Archive. It is similar 4 | # to the production ifarch.config, but secrets are stripped out. 5 | # It's also configured for running all services on http://localhost 6 | # (rather than https: and several ifarchive.org domains). 7 | 8 | # Domain to refer results to (in /indexes). No closing slash. 9 | # Omit this key if search is on the same domain as the results. 10 | # ArchiveDomain = 11 | 12 | # Full URL of the search form. This is not used by the search tool itself 13 | # (that uses server-relative URLs internally). It's used by the index 14 | # generator and other templates containing search forms. 15 | # Omit this key if search is on the same domain as everything else. 16 | # SearchURL = 17 | 18 | # Path to Master-Index.xml. 19 | MasterIndexXML = /var/ifarchive/htdocs/indexes/Master-Index.xml 20 | 21 | # The directories of interest. 22 | IncomingDir = /var/ifarchive/incoming 23 | TrashDir = /var/ifarchive/trash 24 | ArchiveDir = /var/ifarchive/htdocs/if-archive 25 | 26 | # SQLite database for admin tasks. 27 | DBFile = /var/ifarchive/lib/sql/admin.db 28 | 29 | # Max total size of files in the incoming directory 30 | # Currently: one gigabyte 31 | MaxIncomingDirSize = 1073741824 32 | 33 | # This is a secret shared with IFDB 34 | IFDBCommitKey = XXXX 35 | 36 | # If true, the admin page can only be logged into via https, not http. 37 | # This is a safeguard against cookie-snooping attacks. 38 | SecureSite = false 39 | 40 | 41 | [Upload] 42 | 43 | # Email address to receive reports of uploaded files. 44 | # Omit this key to skip emailing. 45 | # ReportEmail = AAAA@ifarchive.org 46 | 47 | 48 | [AdminTool] 49 | 50 | # URI (on the web domain) where the admin script runs. 51 | AppRoot = /admin 52 | # URI (on the web domain) where the admin page's CSS file lives. 53 | AppCSSURI = /misc/admintool.css 54 | 55 | # Jinja template dir. 56 | TemplateDir = /var/ifarchive/lib/admintool 57 | 58 | # Log file. 59 | LogFile = /var/ifarchive/logs/admintool.log 60 | 61 | # Path for build-indexes script. This must take zero time; that is, it must 62 | # background the real work. 63 | BuildScriptFile = /var/ifarchive/bin/build-indexes-bg 64 | 65 | # Path for build-indexes lock file. 66 | BuildLockFile = /var/ifarchive/htdocs/build.lock 67 | 68 | # Path for the build-indexes output file. 69 | BuildOutputFile = /var/ifarchive/htdocs/build.out 70 | 71 | # Path for uncache script. This must take zero time; that is, it must 72 | # background the real work. 73 | UncacheScriptFile = /var/ifarchive/bin/uncache-bg 74 | 75 | # If true, we "sudo" to run BuildScriptFile and UncacheScriptFile. 76 | SudoScripts = true 77 | 78 | # Duration of a log-in session, unless extended. 79 | # Currently: ten days (in seconds) 80 | MaxSessionAge = 864000 81 | 82 | # Age at which to delete files in /trash. 83 | # Currently: thirty days (in seconds). Note that Index-* files are deleted 84 | # in a quarter of this time. 85 | MaxTrashAge = 2592000 86 | 87 | 88 | [Search] 89 | 90 | # URI (on the web domain) where the search script runs. 91 | AppRoot = /search 92 | 93 | # Max results to display on a page. 94 | ResultsPerPage = 10 95 | 96 | # Maximum time (in seconds) to spend on a query. 97 | QueryTimeout = 0.5 98 | 99 | # Search index directory. This should be readable (not writable) by 100 | # www-data. 101 | SearchIndexDir = /var/ifarchive/lib/searchindex 102 | 103 | # Jinja template dir. 104 | TemplateDir = /var/ifarchive/lib/searchtpl 105 | 106 | # Log file. 107 | LogFile = /var/ifarchive/logs/search.log 108 | --------------------------------------------------------------------------------