├── .env ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── README.md ├── bin └── console ├── composer.json ├── composer.lock ├── config ├── bundles.php ├── packages │ ├── cache.yaml │ ├── doctrine.yaml │ ├── framework.yaml │ ├── prod │ │ └── doctrine.yaml │ ├── routing.yaml │ ├── toolforge.yaml │ └── twig.yaml ├── preload.php ├── routes │ ├── annotations.yaml │ └── framework.yaml └── services.yaml ├── data └── works.json ├── i18n ├── bn.json ├── de.json ├── en.json ├── pl.json └── sr.json ├── phpcs.xml ├── public ├── img │ └── loading.gif ├── index.php ├── scripts.js ├── style.css └── toolinfo.json ├── src ├── Command │ └── BuildCommand.php ├── Controller │ └── HomeController.php ├── Kernel.php └── WsCatBrowser.php ├── symfony.lock └── templates └── base.html.twig /.env: -------------------------------------------------------------------------------- 1 | APP_ENV=prod 2 | APP_SECRET=changethis 3 | 4 | REPLICAS_HOST_S1=s1.analytics.db.svc.wikimedia.cloud 5 | REPLICAS_HOST_S2=s2.analytics.db.svc.wikimedia.cloud 6 | REPLICAS_HOST_S3=s3.analytics.db.svc.wikimedia.cloud 7 | REPLICAS_HOST_S4=s4.analytics.db.svc.wikimedia.cloud 8 | REPLICAS_HOST_S5=s5.analytics.db.svc.wikimedia.cloud 9 | REPLICAS_HOST_S6=s6.analytics.db.svc.wikimedia.cloud 10 | REPLICAS_HOST_S7=s7.analytics.db.svc.wikimedia.cloud 11 | REPLICAS_HOST_S8=s8.analytics.db.svc.wikimedia.cloud 12 | REPLICAS_PORT_S1=3306 13 | REPLICAS_PORT_S2=3306 14 | REPLICAS_PORT_S3=3306 15 | REPLICAS_PORT_S4=3306 16 | REPLICAS_PORT_S5=3306 17 | REPLICAS_PORT_S6=3306 18 | REPLICAS_PORT_S7=3306 19 | REPLICAS_PORT_S8=3306 20 | 21 | REPLICAS_USERNAME=uxxxx 22 | REPLICAS_PASSWORD= 23 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | 8 | env: 9 | APP_ENV: test 10 | 11 | strategy: 12 | matrix: 13 | php: [ '7.2', '7.3', '7.4', '8.0' ] 14 | 15 | runs-on: ubuntu-latest 16 | 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v1 20 | 21 | - name: Set up PHP 22 | uses: shivammathur/setup-php@v2 23 | with: 24 | php-version: ${{matrix.php}} 25 | coverage: none 26 | extensions: ast 27 | 28 | - name: Install 29 | run: | 30 | composer install 31 | 32 | - name: Test 33 | run: | 34 | composer test 35 | git status 36 | git status | grep "nothing to commit, working tree clean" 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /nbproject 2 | /.idea 3 | /.env.local 4 | /vendor 5 | /var 6 | /public/categories*.json 7 | /public/works*.json 8 | /public/sites.json 9 | 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Wikisource category browser 2 | =========================== 3 | 4 | This script is running at https://ws-cat-browser.toolforge.org/ 5 | and is updated weekly. 6 | 7 | ## Missing languages? 8 | 9 | If your Wikisource is missing, please add relevant sitelinks to the following items on Wikdiata: 10 | 11 | 1. the [root category (Q1281)](https://www.wikidata.org/wiki/Q1281), and 12 | 2. the category for [validated indices Q15634466](https://www.wikidata.org/wiki/Q15634466) 13 | (e.g. `Index_Validated`, `Pagine_indice_SAL_100%`). 14 | 15 | Alternatively, you can [create a task on Phabricator](https://phabricator.wikimedia.org/maniphest/task/edit/form/1/?project=wikisource) 16 | and just tell us the names of the above categories. 17 | 18 | ## Tracking 19 | At some point, the git repository of this tool will track changes of (or at least additions to) validated works. 20 | For now, the `data/works.json` is manually updated with a [formatted](http://jsonformatter.curiousconcept.com/) 21 | version of the generated `works.json` file, so added and removed entries can be tracked. 22 | 23 | ## Development 24 | 25 | To work with this tool locally, edit the list of languages specified in `download_dumps.sh`, 26 | then run this script to download the relevant database dumps. 27 | Import these with something like the following: 28 | 29 | Then set `$dbs`, `$dbuser`, and `$dbpass` in `config.php` (copy it from `config_dist.php`) 30 | and run `php build.php`. 31 | 32 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env php 2 | =7.2.5", 18 | "ext-ctype": "*", 19 | "ext-iconv": "*", 20 | "ext-json": "*", 21 | "symfony/console": "5.3.*", 22 | "symfony/dotenv": "5.3.*", 23 | "symfony/flex": "^1.3.1", 24 | "symfony/framework-bundle": "5.3.*", 25 | "symfony/runtime": "5.3.*", 26 | "symfony/twig-bundle": "5.3.*", 27 | "symfony/yaml": "5.3.*", 28 | "wikimedia/toolforge-bundle": "^1.4" 29 | }, 30 | "require-dev": { 31 | "mediawiki/mediawiki-codesniffer": "^37.0", 32 | "mediawiki/minus-x": "^1.0" 33 | }, 34 | "config": { 35 | "optimize-autoloader": true, 36 | "preferred-install": { 37 | "*": "dist" 38 | }, 39 | "sort-packages": true, 40 | "platform": { 41 | "php": "7.2.31" 42 | } 43 | }, 44 | "autoload": { 45 | "psr-4": { 46 | "App\\": "src/" 47 | } 48 | }, 49 | "autoload-dev": { 50 | "psr-4": { 51 | "App\\Tests\\": "tests/" 52 | } 53 | }, 54 | "replace": { 55 | "symfony/polyfill-ctype": "*", 56 | "symfony/polyfill-iconv": "*", 57 | "symfony/polyfill-php72": "*" 58 | }, 59 | "scripts": { 60 | "auto-scripts": { 61 | "cache:clear": "symfony-cmd", 62 | "assets:install %PUBLIC_DIR%": "symfony-cmd" 63 | }, 64 | "post-install-cmd": [ 65 | "@auto-scripts" 66 | ], 67 | "post-update-cmd": [ 68 | "@auto-scripts" 69 | ], 70 | "test": [ 71 | "composer validate", 72 | "phpcs -s .", 73 | "./bin/console lint:twig ./templates", 74 | "./bin/console lint:yaml ./config", 75 | "minus-x check ." 76 | ] 77 | }, 78 | "conflict": { 79 | "symfony/symfony": "*" 80 | }, 81 | "extra": { 82 | "symfony": { 83 | "allow-contrib": false, 84 | "require": "5.3.*" 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /config/bundles.php: -------------------------------------------------------------------------------- 1 | [ 'all' => true ], 5 | Symfony\Bundle\TwigBundle\TwigBundle::class => [ 'all' => true ], 6 | Doctrine\Bundle\DoctrineBundle\DoctrineBundle::class => [ 'all' => true ], 7 | Wikimedia\ToolforgeBundle\ToolforgeBundle::class => [ 'all' => true ], 8 | ]; 9 | -------------------------------------------------------------------------------- /config/packages/cache.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | cache: 3 | prefix_seed: ws-cat-browser 4 | pools: 5 | cache.replicas: 6 | adapter: cache.adapter.filesystem 7 | default_lifetime: 600 8 | -------------------------------------------------------------------------------- /config/packages/doctrine.yaml: -------------------------------------------------------------------------------- 1 | doctrine: 2 | dbal: 3 | connections: 4 | toolforge_s1: 5 | host: '%env(REPLICAS_HOST_S1)%' 6 | port: '%env(REPLICAS_PORT_S1)%' 7 | user: '%env(REPLICAS_USERNAME)%' 8 | password: '%env(REPLICAS_PASSWORD)%' 9 | toolforge_s2: 10 | host: '%env(REPLICAS_HOST_S2)%' 11 | port: '%env(REPLICAS_PORT_S2)%' 12 | user: '%env(REPLICAS_USERNAME)%' 13 | password: '%env(REPLICAS_PASSWORD)%' 14 | toolforge_s3: 15 | host: '%env(REPLICAS_HOST_S3)%' 16 | port: '%env(REPLICAS_PORT_S3)%' 17 | user: '%env(REPLICAS_USERNAME)%' 18 | password: '%env(REPLICAS_PASSWORD)%' 19 | toolforge_s4: 20 | host: '%env(REPLICAS_HOST_S4)%' 21 | port: '%env(REPLICAS_PORT_S4)%' 22 | user: '%env(REPLICAS_USERNAME)%' 23 | password: '%env(REPLICAS_PASSWORD)%' 24 | toolforge_s5: 25 | host: '%env(REPLICAS_HOST_S5)%' 26 | port: '%env(REPLICAS_PORT_S5)%' 27 | user: '%env(REPLICAS_USERNAME)%' 28 | password: '%env(REPLICAS_PASSWORD)%' 29 | toolforge_s6: 30 | host: '%env(REPLICAS_HOST_S6)%' 31 | port: '%env(REPLICAS_PORT_S6)%' 32 | user: '%env(REPLICAS_USERNAME)%' 33 | password: '%env(REPLICAS_PASSWORD)%' 34 | toolforge_s7: 35 | host: '%env(REPLICAS_HOST_S7)%' 36 | port: '%env(REPLICAS_PORT_S7)%' 37 | user: '%env(REPLICAS_USERNAME)%' 38 | password: '%env(REPLICAS_PASSWORD)%' 39 | toolforge_s8: 40 | host: '%env(REPLICAS_HOST_S8)%' 41 | port: '%env(REPLICAS_PORT_S8)%' 42 | user: '%env(REPLICAS_USERNAME)%' 43 | password: '%env(REPLICAS_PASSWORD)%' 44 | -------------------------------------------------------------------------------- /config/packages/framework.yaml: -------------------------------------------------------------------------------- 1 | # see https://symfony.com/doc/current/reference/configuration/framework.html 2 | framework: 3 | secret: '%env(APP_SECRET)%' 4 | http_method_override: false 5 | 6 | session: 7 | handler_id: null 8 | cookie_secure: auto 9 | cookie_samesite: lax 10 | storage_factory_id: session.storage.factory.native 11 | 12 | php_errors: 13 | log: true 14 | 15 | when@test: 16 | framework: 17 | test: true 18 | -------------------------------------------------------------------------------- /config/packages/prod/doctrine.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | cache: 3 | pools: 4 | doctrine.result_cache_pool: 5 | adapter: cache.app 6 | doctrine.system_cache_pool: 7 | adapter: cache.system 8 | -------------------------------------------------------------------------------- /config/packages/routing.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | router: 3 | utf8: true 4 | 5 | # Configure how to generate URLs in non-HTTP contexts, such as CLI commands. 6 | # See https://symfony.com/doc/current/routing.html#generating-urls-in-commands 7 | #default_uri: http://localhost 8 | 9 | when@prod: 10 | framework: 11 | router: 12 | strict_requirements: null 13 | -------------------------------------------------------------------------------- /config/packages/toolforge.yaml: -------------------------------------------------------------------------------- 1 | toolforge: 2 | intuition: 3 | domain: 'ws-cat-browser' 4 | -------------------------------------------------------------------------------- /config/packages/twig.yaml: -------------------------------------------------------------------------------- 1 | twig: 2 | default_path: '%kernel.project_dir%/templates' 3 | 4 | when@test: 5 | twig: 6 | strict_variables: true 7 | -------------------------------------------------------------------------------- /config/preload.php: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | ./src/Controller/ 8 | 9 | 10 | . 11 | ./vendor/ 12 | ./var/ 13 | ./bin/.phpunit/ 14 | ./.phan/ 15 | 16 | -------------------------------------------------------------------------------- /public/img/loading.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wikisource/ws-cat-browser/ff043c8cb98a0c2641c9a3a31685101758b2d28b/public/img/loading.gif -------------------------------------------------------------------------------- /public/index.php: -------------------------------------------------------------------------------- 1 | ') 36 | .addClass('alert-box info radius hide') 37 | .text('Error: this Wikisource language (' + lang + ') appears to not have any validated works.'); 38 | $catlist.replaceWith($notice); 39 | $notice.removeClass("hide").fadeIn(400); 40 | }); 41 | } 42 | }); 43 | }); 44 | 45 | function addCats(allCats, $parent, cat, catLabel) { 46 | $.each(cat, function(i, subcat){ 47 | var title = subcat.replace(/_/g, " "); 48 | if (subcat.substr(0, catLabel.length + 1) === catLabel + ":") { 49 | title = '' + title.substr(catLabel.length + 1) + ''; 50 | } else { 51 | var encodedCat = encodeURIComponent(subcat); 52 | title = "" + title + "" 53 | + "" 54 | + " " 56 | + ""; 57 | } 58 | var $newItem = $("
  • " + title + "
      1. "); 59 | $parent.append($newItem); 60 | $newItem.find("span").on("click", function(){ 61 | var $sublist = $(this).next("ol"); 62 | if ($sublist.is(":empty")) { 63 | $(this).addClass("open").removeClass("closed"); 64 | addCats(allCats, $sublist, allCats[subcat], catLabel); 65 | } else { 66 | $(this).addClass("closed").removeClass("open"); 67 | $sublist.children().remove(); 68 | } 69 | }) 70 | }); 71 | } -------------------------------------------------------------------------------- /public/style.css: -------------------------------------------------------------------------------- 1 | ol.c { 2 | border-left: thin solid lightblue; 3 | list-style-type: none; 4 | padding-left: 0.3em; 5 | } 6 | ol.c span { cursor:s-resize; } 7 | ol.c span.open { cursor:n-resize; } 8 | ol.c span:hover { background-color:#efefef; } 9 | ol.c a:hover { text-decoration:underline; } 10 | ol.c a.epub:hover { text-decoration:none; } 11 | 12 | p.loading { color:#999; } 13 | -------------------------------------------------------------------------------- /public/toolinfo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "ws-cat-browser", 3 | "title" : "Wikisource category browser", 4 | "description" : "A generator for a tree browser for categories of validated Wikisource works, in multiple languages.", 5 | "url" : "https://ws-cat-browser.toolforge.org/", 6 | "keywords" : "wikisource, categories, ebooks, epub", 7 | "author" : "Sam Wilson", 8 | "repository" : "https://github.com/wikisource/ws-cat-browser.git" 9 | } 10 | -------------------------------------------------------------------------------- /src/Command/BuildCommand.php: -------------------------------------------------------------------------------- 1 | replicasClient = $replicasClient; 39 | $this->cache = $cache; 40 | $this->wsCatBrowser = $wsCatBrowser; 41 | } 42 | 43 | public function configure() { 44 | $this->addOption( 45 | 'lang', 'l', InputOption::VALUE_IS_ARRAY | InputOption::VALUE_REQUIRED, 46 | 'Wikisource language code.' 47 | ); 48 | } 49 | 50 | /** 51 | * @param InputInterface $input 52 | * @param OutputInterface $output 53 | * @return int 54 | */ 55 | public function execute( InputInterface $input, OutputInterface $output ) { 56 | $timeStart = microtime( true ); 57 | $this->out = $output; 58 | 59 | $siteInfo = $this->getSiteInfo(); 60 | $this->out->writeln( 'Writing sites.json' ); 61 | file_put_contents( $this->wsCatBrowser->getSitesFilename(), json_encode( $siteInfo ) ); 62 | 63 | // For each site, build categories.json and works.json 64 | $langs = $input->getOption( 'lang' ); 65 | foreach ( $siteInfo as $lang => $info ) { 66 | if ( $langs && !in_array( $lang, $langs ) ) { 67 | $this->out->writeln( 'Skipping ' . $lang ); 68 | continue; 69 | } 70 | $db = $this->replicasClient->getConnection( $lang === 'www' ? 'sourceswiki' : $lang . 'wikisource' ); 71 | $this->buildOneLang( $db, $lang, $info['index_cat'], $info['cat_label'], $info['index_ns'] ); 72 | } 73 | 74 | // Report completion. 75 | $minutes = round( ( microtime( true ) - $timeStart ) / 60, 1 ); 76 | $this->out->writeln( "Done. Total time: $minutes minutes." ); 77 | return Command::SUCCESS; 78 | } 79 | 80 | /** 81 | * @return array 82 | */ 83 | private function getSiteInfo() { 84 | $this->out->writeln( 'Getting site information . . . ' ); 85 | $rootCatItem = 'Q1281'; 86 | $validatedCatItem = 'Q15634466'; 87 | $rootCats = $this->siteLinks( $rootCatItem ); 88 | $validatedCats = $this->siteLinks( $validatedCatItem ); 89 | $out = []; 90 | foreach ( $rootCats as $site => $rootCat ) { 91 | // If both a root cat and an Index cat exist. 92 | if ( isset( $validatedCats[$site] ) ) { 93 | $lang = $site === 'sourceswiki' 94 | ? $lang = 'www' 95 | : substr( $site, 0, -strlen( 'wikisource' ) ); 96 | $nsInfo = $this->getNamespaceInfo( $lang ); 97 | $catLabel = $nsInfo['Category']['*']; 98 | // Strip cat label from cats 99 | $rootCat = substr( $rootCat, strlen( $catLabel ) + 1 ); 100 | $indexCat = substr( $validatedCats[$site], strlen( $catLabel ) + 1 ); 101 | // Put it all together, replacing spaces with underscores. 102 | $out[$lang] = [ 103 | 'cat_label' => $catLabel, 104 | 'cat_root' => str_replace( ' ', '_', $rootCat ), 105 | 'index_ns' => $nsInfo['Index']['id'], 106 | 'index_cat' => str_replace( ' ', '_', $indexCat ), 107 | ]; 108 | } 109 | } 110 | $this->out->writeln( 'done' ); 111 | return $out; 112 | } 113 | 114 | /** 115 | * @param string $lang 116 | * @return mixed 117 | */ 118 | private function getNamespaceInfo( string $lang ) { 119 | return $this->cache->get( 'namespaces_' . $lang, function ( CacheItemInterface $cacheItem ) use ( $lang ) { 120 | $cacheItem->expiresAfter( new DateInterval( 'P7D' ) ); 121 | $this->out->writeln( "Getting namespaces for $lang" ); 122 | $url = "https://$lang.wikisource.org/w/api.php?action=query&meta=siteinfo&siprop=namespaces&format=json"; 123 | $data = json_decode( file_get_contents( $url ), true ); 124 | if ( !isset( $data['query']['namespaces'] ) ) { 125 | return false; 126 | } 127 | $desired = [ 'Index', 'Category' ]; 128 | $out = []; 129 | foreach ( $data['query']['namespaces'] as $ns ) { 130 | if ( isset( $ns['canonical'] ) && in_array( $ns['canonical'], $desired ) ) { 131 | $out[$ns['canonical']] = $ns; 132 | } 133 | } 134 | return $out; 135 | } ); 136 | } 137 | 138 | /** 139 | * Get sitelinks for the given item ID. 140 | * @param string $item Q-number. 141 | * @return string[] Page names, keyed by site name 142 | */ 143 | private function siteLinks( $item ) { 144 | return $this->cache->get( 'site_links_' . $item, function ( ItemInterface $cacheItem ) use ( $item ) { 145 | $cacheItem->expiresAfter( new DateInterval( 'P7D' ) ); 146 | $params = [ 147 | 'action' => 'wbgetentities', 148 | 'format' => 'json', 149 | 'ids' => $item, 150 | 'props' => 'sitelinks', 151 | ]; 152 | $url = 'https://www.wikidata.org/w/api.php?' . http_build_query( $params ); 153 | $this->out->write( "Getting site links from Wikidata for $item . . . " ); 154 | $data = json_decode( file_get_contents( $url ), true ); 155 | $cats = []; 156 | if ( isset( $data['entities'][$item]['sitelinks'] ) ) { 157 | foreach ( $data['entities'][$item]['sitelinks'] as $sitelink ) { 158 | $cats[$sitelink['site']] = $sitelink['title']; 159 | } 160 | } 161 | $this->out->writeln( 'found ' . count( $cats ) ); 162 | return $cats; 163 | } ); 164 | } 165 | 166 | /** 167 | * @param Connection $db 168 | * @param string $lang 169 | * @param string $indexRoot 170 | * @param string $catLabel 171 | * @param string $indexNs 172 | * @return bool 173 | */ 174 | private function buildOneLang( Connection $db, $lang, $indexRoot, $catLabel, $indexNs ) { 175 | echo "Getting list of validated works for '$lang' . . . "; 176 | $validatedWorks = $this->getValidatedWorks( $db, $indexRoot, $indexNs ); 177 | file_put_contents( $this->wsCatBrowser->getDataFilename( 'works', $lang ), json_encode( $validatedWorks ) ); 178 | echo "done\n"; 179 | 180 | echo "Getting category data for '$lang' . . . "; 181 | $allCats = []; 182 | foreach ( $validatedWorks as $indexTitle => $workTitle ) { 183 | $catTree = $this->getAllCats( $db, $lang, $workTitle, 0, [], [], $catLabel ); 184 | $allCats = array_map( 'array_unique', array_merge_recursive( $allCats, $catTree ) ); 185 | } 186 | echo "done\n"; 187 | 188 | echo "Sorting categories for '$lang' . . . "; 189 | foreach ( $allCats as $cat => $cats ) { 190 | sort( $allCats[$cat] ); 191 | } 192 | echo "done\n"; 193 | 194 | // Make sure the category list was successfully built before replacing the old JSON file. 195 | if ( count( $allCats ) > 0 ) { 196 | $catFile = $this->wsCatBrowser->getDataFilename( 'categories', $lang ); 197 | echo "Writing $catFile\n"; 198 | file_put_contents( $catFile, json_encode( $allCats ) ); 199 | return true; 200 | } else { 201 | echo "No validated works found for $lang!\n"; 202 | return false; 203 | } 204 | } 205 | 206 | /** 207 | * Get a list of validated works. 208 | * @param Connection $db 209 | * @param string $indexRoot 210 | * @param string $indexNs 211 | * @return array 212 | */ 213 | private function getValidatedWorks( Connection $db, string $indexRoot, string $indexNs ) { 214 | $sql = 'SELECT ' 215 | . ' indexpage.page_title AS indextitle,' 216 | . ' workpage.page_title AS worktitle' 217 | . ' FROM page AS indexpage ' 218 | . ' JOIN categorylinks ON cl_from=indexpage.page_id ' 219 | . ' JOIN pagelinks ON pl_from=indexpage.page_id ' 220 | . ' JOIN page AS workpage ON workpage.page_title=pl_title ' 221 | . ' WHERE ' 222 | . ' workpage.page_title NOT LIKE "%/%" ' 223 | . ' AND pl_namespace = 0 ' 224 | . ' AND workpage.page_namespace = 0' 225 | . ' AND indexpage.page_namespace = :index_ns' 226 | . ' AND cl_to = :index_root'; 227 | $stmt = $db->executeQuery( $sql, [ 228 | 'index_ns' => $indexNs, 229 | 'index_root' => $indexRoot, 230 | ] ); 231 | $out = []; 232 | foreach ( $stmt->fetchAllAssociative() as $res ) { 233 | $out[$res['indextitle']] = $res['worktitle']; 234 | } 235 | return $out; 236 | } 237 | 238 | /** 239 | * @param Connection $db 240 | * @param string $lang 241 | * @param string $baseCat 242 | * @param string $ns 243 | * @param array $catList 244 | * @param array $tracker 245 | * @param string $catLabel 246 | * @return array|mixed 247 | */ 248 | private function getAllCats( Connection $db, $lang, $baseCat, $ns, $catList, $tracker, $catLabel ) { 249 | $cats = $this->getCats( $db, $baseCat, $ns, $catLabel ); 250 | if ( empty( $cats ) ) { 251 | return $catList; 252 | } 253 | if ( $ns == 0 ) { 254 | $tracker = []; 255 | } 256 | 257 | // For each cat, create an element in the output array. 258 | foreach ( $cats as $cat ) { 259 | // echo "Getting supercats of $baseCat via $cat.\n"; 260 | $tracker_tag = [ $baseCat, $cat ]; 261 | if ( in_array( $tracker_tag, $tracker ) ) { 262 | echo "A category loop has been detected in $lang:\n\ndigraph G {\n"; 263 | foreach ( $tracker as $trackerItem ) { 264 | echo '"' . str_replace( '"', '\"', $trackerItem[0] ) 265 | . '" -> "' . str_replace( '"', '\"', $trackerItem[1] ) . '"' . "\n"; 266 | } 267 | echo "}\n"; 268 | continue; 269 | } 270 | array_push( $tracker, $tracker_tag ); 271 | // Add all of $cat's parents to the $catList. 272 | $superCats = $this->getAllCats( $db, $lang, $cat, 14, $catList, $tracker, $catLabel ); 273 | $catList = array_merge_recursive( $catList, $superCats ); 274 | // Initialise $cat as a parent if it's not there yet. 275 | if ( !isset( $catList[$cat] ) ) { 276 | $catList[$cat] = []; 277 | } 278 | // Then add the $baseCat as a child. 279 | if ( !in_array( $baseCat, $catList[$cat] ) ) { 280 | array_push( $catList[$cat], $baseCat ); 281 | } 282 | $catList = array_map( 'array_unique', $catList ); 283 | } 284 | return array_map( 'array_unique', $catList ); 285 | } 286 | 287 | /** 288 | * @param Connection $db 289 | * @param string $baseCat 290 | * @param string $ns 291 | * @param string $catLabel 292 | * @return array 293 | */ 294 | private function getCats( Connection $db, $baseCat, $ns, $catLabel = 'Category' ) { 295 | // Get the starting categories. 296 | $sql = 'SELECT cl_to AS catname FROM page ' 297 | . ' JOIN categorylinks ON cl_from=page_id' 298 | . ' WHERE page_title = :page_title' 299 | . ' AND page_namespace = :page_namespace '; 300 | $result = $db->executeQuery( $sql, [ 301 | 'page_title' => str_replace( $catLabel . ':', '', $baseCat ), 302 | 'page_namespace' => $ns, 303 | ] ); 304 | $cats = []; 305 | foreach ( $result->fetchAllAssociative() as $cat ) { 306 | $cats[] = $catLabel . ':' . $cat['catname']; 307 | } 308 | return $cats; 309 | } 310 | 311 | } 312 | -------------------------------------------------------------------------------- /src/Controller/HomeController.php: -------------------------------------------------------------------------------- 1 | wsCatBrowser = $wsCatBrowser; 23 | } 24 | 25 | /** 26 | * @Route("/", name="home") 27 | * @param Request $request 28 | * @return Response 29 | */ 30 | public function home( Request $request ) { 31 | $lang = $request->get( 'lang' ); 32 | if ( !$lang ) { 33 | $lang = 'en'; 34 | } 35 | 36 | $siteInfo = file_exists( $this->wsCatBrowser->getSitesFilename() ) 37 | ? json_decode( file_get_contents( $this->wsCatBrowser->getSitesFilename() ), true ) 38 | : []; 39 | 40 | $err = null; 41 | if ( !array_key_exists( $lang, $siteInfo ) ) { 42 | $err = [ 'language-not-found', [ $lang ] ]; 43 | $lang = 'en'; 44 | } 45 | 46 | return $this->render( 'base.html.twig', [ 47 | 'site_info' => $siteInfo, 48 | 'lang' => $lang, 49 | 'suffix' => $lang === 'en' ? '' : '_' . $lang, 50 | 'err' => $err, 51 | ] ); 52 | } 53 | 54 | /** 55 | * @Route("/meta.json", name="meta"); 56 | * @param Request $request 57 | * @return JsonResponse 58 | */ 59 | public function meta( Request $request ) { 60 | $siteInfo = json_decode( file_get_contents( $this->wsCatBrowser->getSitesFilename() ), true ); 61 | $lang = $request->get( 'lang' ); 62 | if ( !array_key_exists( $lang, $siteInfo ) ) { 63 | $lang = 'en'; 64 | } 65 | 66 | $worksFile = $this->wsCatBrowser->getDataFilename( 'works', $lang ); 67 | $categoriesFile = $this->wsCatBrowser->getDataFilename( 'categories', $lang ); 68 | $metadata = [ 69 | 'works_count' => count( (array)json_decode( file_get_contents( $worksFile ) ) ), 70 | 'last_modified' => date( 'Y-m-d H:i', filemtime( $categoriesFile ) ), 71 | 'category_label' => $siteInfo[$lang]['cat_label'], 72 | 'category_root' => $siteInfo[$lang]['cat_root'], 73 | ]; 74 | return new JsonResponse( $metadata ); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/Kernel.php: -------------------------------------------------------------------------------- 1 | import( '../config/{packages}/*.yaml' ); 18 | $container->import( '../config/{packages}/' . $this->environment . '/*.yaml' ); 19 | 20 | if ( is_file( \dirname( __DIR__ ) . '/config/services.yaml' ) ) { 21 | $container->import( '../config/services.yaml' ); 22 | $container->import( '../config/{services}_' . $this->environment . '.yaml' ); 23 | } else { 24 | $container->import( '../config/{services}.php' ); 25 | } 26 | } 27 | 28 | /** 29 | * @param RoutingConfigurator $routes 30 | */ 31 | protected function configureRoutes( RoutingConfigurator $routes ): void { 32 | $routes->import( '../config/{routes}/' . $this->environment . '/*.yaml' ); 33 | $routes->import( '../config/{routes}/*.yaml' ); 34 | 35 | if ( is_file( \dirname( __DIR__ ) . '/config/routes.yaml' ) ) { 36 | $routes->import( '../config/routes.yaml' ); 37 | } else { 38 | $routes->import( '../config/{routes}.php' ); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/WsCatBrowser.php: -------------------------------------------------------------------------------- 1 | publicDir = $projectDir . '/public'; 15 | } 16 | 17 | /** 18 | * Get the name of a xxx_xx.json file. 19 | * @param string $name 20 | * @param string $lang 21 | * @return string 22 | */ 23 | public function getDataFilename( $name, $lang ): string { 24 | $suffix = ( $lang == 'en' ) ? '' : '_' . $lang; 25 | return $this->publicDir . '/' . $name . $suffix . '.json'; 26 | } 27 | 28 | /** 29 | * Get the filesystem path to the sites.json file. 30 | * @return string 31 | */ 32 | public function getSitesFilename(): string { 33 | return $this->publicDir . '/sites.json'; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /symfony.lock: -------------------------------------------------------------------------------- 1 | { 2 | "composer/semver": { 3 | "version": "3.2.5" 4 | }, 5 | "composer/spdx-licenses": { 6 | "version": "1.5.5" 7 | }, 8 | "doctrine/annotations": { 9 | "version": "1.0", 10 | "recipe": { 11 | "repo": "github.com/symfony/recipes", 12 | "branch": "master", 13 | "version": "1.0", 14 | "ref": "a2759dd6123694c8d901d0ec80006e044c2e6457" 15 | }, 16 | "files": [ 17 | "config/routes/annotations.yaml" 18 | ] 19 | }, 20 | "doctrine/cache": { 21 | "version": "2.1.1" 22 | }, 23 | "doctrine/collections": { 24 | "version": "1.6.8" 25 | }, 26 | "doctrine/dbal": { 27 | "version": "3.1.2" 28 | }, 29 | "doctrine/deprecations": { 30 | "version": "v0.5.3" 31 | }, 32 | "doctrine/doctrine-bundle": { 33 | "version": "2.4", 34 | "recipe": { 35 | "repo": "github.com/symfony/recipes", 36 | "branch": "master", 37 | "version": "2.4", 38 | "ref": "bac5c852ff628886de2753215fe5eb1f9ce980fb" 39 | }, 40 | "files": [ 41 | "config/packages/doctrine.yaml", 42 | "config/packages/prod/doctrine.yaml", 43 | "config/packages/test/doctrine.yaml", 44 | "src/Entity/.gitignore", 45 | "src/Repository/.gitignore" 46 | ] 47 | }, 48 | "doctrine/event-manager": { 49 | "version": "1.1.1" 50 | }, 51 | "doctrine/lexer": { 52 | "version": "1.2.1" 53 | }, 54 | "doctrine/persistence": { 55 | "version": "2.2.2" 56 | }, 57 | "doctrine/sql-formatter": { 58 | "version": "1.1.1" 59 | }, 60 | "krinkle/intuition": { 61 | "version": "v1.2.0" 62 | }, 63 | "mediawiki/mediawiki-codesniffer": { 64 | "version": "v34.0.0" 65 | }, 66 | "mediawiki/minus-x": { 67 | "version": "1.1.1" 68 | }, 69 | "mediawiki/oauthclient": { 70 | "version": "1.1.0" 71 | }, 72 | "psr/cache": { 73 | "version": "1.0.1" 74 | }, 75 | "psr/container": { 76 | "version": "1.1.1" 77 | }, 78 | "psr/event-dispatcher": { 79 | "version": "1.0.0" 80 | }, 81 | "psr/log": { 82 | "version": "1.1.4" 83 | }, 84 | "sebastian/diff": { 85 | "version": "3.0.3" 86 | }, 87 | "squizlabs/php_codesniffer": { 88 | "version": "3.0", 89 | "recipe": { 90 | "repo": "github.com/symfony/recipes-contrib", 91 | "branch": "master", 92 | "version": "3.0", 93 | "ref": "0dc9cceda799fd3a08b96987e176a261028a3709" 94 | } 95 | }, 96 | "symfony/cache": { 97 | "version": "v5.3.7" 98 | }, 99 | "symfony/cache-contracts": { 100 | "version": "v2.4.0" 101 | }, 102 | "symfony/config": { 103 | "version": "v5.3.4" 104 | }, 105 | "symfony/console": { 106 | "version": "5.3", 107 | "recipe": { 108 | "repo": "github.com/symfony/recipes", 109 | "branch": "master", 110 | "version": "5.3", 111 | "ref": "da0c8be8157600ad34f10ff0c9cc91232522e047" 112 | }, 113 | "files": [ 114 | "bin/console" 115 | ] 116 | }, 117 | "symfony/dependency-injection": { 118 | "version": "v5.3.7" 119 | }, 120 | "symfony/deprecation-contracts": { 121 | "version": "v2.4.0" 122 | }, 123 | "symfony/doctrine-bridge": { 124 | "version": "v5.3.7" 125 | }, 126 | "symfony/dotenv": { 127 | "version": "v5.3.7" 128 | }, 129 | "symfony/error-handler": { 130 | "version": "v5.3.7" 131 | }, 132 | "symfony/event-dispatcher": { 133 | "version": "v5.3.7" 134 | }, 135 | "symfony/event-dispatcher-contracts": { 136 | "version": "v2.4.0" 137 | }, 138 | "symfony/filesystem": { 139 | "version": "v5.3.4" 140 | }, 141 | "symfony/finder": { 142 | "version": "v5.3.7" 143 | }, 144 | "symfony/flex": { 145 | "version": "1.0", 146 | "recipe": { 147 | "repo": "github.com/symfony/recipes", 148 | "branch": "master", 149 | "version": "1.0", 150 | "ref": "c0eeb50665f0f77226616b6038a9b06c03752d8e" 151 | }, 152 | "files": [ 153 | ".env" 154 | ] 155 | }, 156 | "symfony/framework-bundle": { 157 | "version": "5.3", 158 | "recipe": { 159 | "repo": "github.com/symfony/recipes", 160 | "branch": "master", 161 | "version": "5.3", 162 | "ref": "414ba00ad43fa71be42c7906a551f1831716b03c" 163 | }, 164 | "files": [ 165 | "config/packages/cache.yaml", 166 | "config/packages/framework.yaml", 167 | "config/preload.php", 168 | "config/routes/framework.yaml", 169 | "config/services.yaml", 170 | "public/index.php", 171 | "src/Controller/.gitignore", 172 | "src/Kernel.php" 173 | ] 174 | }, 175 | "symfony/http-client": { 176 | "version": "v5.3.7" 177 | }, 178 | "symfony/http-client-contracts": { 179 | "version": "v2.4.0" 180 | }, 181 | "symfony/http-foundation": { 182 | "version": "v5.3.7" 183 | }, 184 | "symfony/http-kernel": { 185 | "version": "v5.3.7" 186 | }, 187 | "symfony/polyfill-intl-grapheme": { 188 | "version": "v1.23.1" 189 | }, 190 | "symfony/polyfill-intl-normalizer": { 191 | "version": "v1.23.0" 192 | }, 193 | "symfony/polyfill-mbstring": { 194 | "version": "v1.23.1" 195 | }, 196 | "symfony/polyfill-php73": { 197 | "version": "v1.23.0" 198 | }, 199 | "symfony/polyfill-php80": { 200 | "version": "v1.23.1" 201 | }, 202 | "symfony/polyfill-php81": { 203 | "version": "v1.23.0" 204 | }, 205 | "symfony/process": { 206 | "version": "v5.3.7" 207 | }, 208 | "symfony/routing": { 209 | "version": "5.3", 210 | "recipe": { 211 | "repo": "github.com/symfony/recipes", 212 | "branch": "master", 213 | "version": "5.3", 214 | "ref": "44633353926a0382d7dfb0530922c5c0b30fae11" 215 | }, 216 | "files": [ 217 | "config/packages/routing.yaml", 218 | "config/routes.yaml" 219 | ] 220 | }, 221 | "symfony/runtime": { 222 | "version": "v5.3.4" 223 | }, 224 | "symfony/service-contracts": { 225 | "version": "v2.4.0" 226 | }, 227 | "symfony/string": { 228 | "version": "v5.3.7" 229 | }, 230 | "symfony/translation-contracts": { 231 | "version": "v2.4.0" 232 | }, 233 | "symfony/twig-bridge": { 234 | "version": "v5.3.7" 235 | }, 236 | "symfony/twig-bundle": { 237 | "version": "5.3", 238 | "recipe": { 239 | "repo": "github.com/symfony/recipes", 240 | "branch": "master", 241 | "version": "5.3", 242 | "ref": "3dd530739a4284e3272274c128dbb7a8140a66f1" 243 | }, 244 | "files": [ 245 | "config/packages/twig.yaml", 246 | "templates/base.html.twig" 247 | ] 248 | }, 249 | "symfony/var-dumper": { 250 | "version": "v5.3.7" 251 | }, 252 | "symfony/var-exporter": { 253 | "version": "v5.3.7" 254 | }, 255 | "symfony/yaml": { 256 | "version": "v5.3.6" 257 | }, 258 | "twig/twig": { 259 | "version": "v3.3.2" 260 | }, 261 | "wikimedia/toolforge-bundle": { 262 | "version": "1.4.1" 263 | } 264 | } 265 | -------------------------------------------------------------------------------- /templates/base.html.twig: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | {{ msg('wikisource') }} 8 | ({{ msg('all-validated-works') }}) 9 | 10 | 11 | 12 | 13 | 14 |
      15 | 16 |
      17 |
      18 |

      19 | {{ msg('wikisource') }} 20 | {{ msg('all-validated-works') }} 21 |

      22 |
      23 |
      24 | 25 |
      26 |
      27 | 28 |
        29 |
      • {{ msg('languages') }}
        • 30 | {% for l,info in site_info %} 31 |
      • 32 | {% if lang == l %} 33 | {{ l }} 34 | {% else %} 35 | {{ l }} 36 | {% endif %} 37 |
        • 38 | {% endfor %} 39 |
      40 | 41 | {% if err is defined and err %} 42 |

      {{ msg(err[0], err[1]) }}

      43 | {% endif %} 44 | 45 |

      46 | {{ msg('introduction', [ 47 | 'x', 48 | ''~lang~' Wikisource' 49 | ])}} 50 |

      51 |

      52 | {{ msg('loading') }} 53 |

      54 |
        55 |

        56 | This list was last updated at: 57 | y UTC. 58 | The above data is available in 59 | works{{ suffix }}.json 60 | and categories{{ suffix }}.json. 61 |

        62 |

        63 | If you don't see your language's Wikisource listed above, 64 | please make sure it is present as a sitelink on Wikidata for 65 | the root category (Q1281) and 66 | the category for validated indexes (Q15634466). 67 |

        68 |

        69 | For more information please see 70 | the code on Github 71 | or contact User:Samwilson. 72 |

        73 |
        74 |
        75 |
        76 | 77 |
        78 | 79 |
        80 | 81 | 82 | 83 | 86 | 87 | 88 | --------------------------------------------------------------------------------