├── .gitignore ├── LICENSE ├── README.md ├── cgicc-configure.patch ├── cgicc-package.scm ├── data ├── customFrontPage │ ├── authors.json │ ├── batch.json │ └── pages │ │ └── __front-page │ │ ├── data.json │ │ ├── files │ │ ├── designLayout.png │ │ ├── designLayout.svg │ │ ├── graphPreview.png │ │ ├── parserPipeline.png │ │ └── parserPipeline.svg │ │ └── frontpage.txt ├── exampleNginx.conf └── webroot │ ├── __static │ ├── linkPreview.js │ ├── logo.png │ ├── logo.svg │ ├── pdfStyle.css │ ├── sigma9Backup.css │ └── style.css │ └── robots.txt ├── scipnet-package.scm ├── scp-archive-package.scm ├── scp-scraper-package.scm ├── soci-package.scm └── src ├── Scipnet ├── Scipnet │ ├── Cli.cpp │ ├── Cli.hpp │ ├── ColorTable.cpp │ ├── ColorTable.hpp │ ├── Converter.cpp │ ├── Converter.hpp │ ├── Database.cpp │ ├── Database.hpp │ ├── Emscript.cpp │ ├── Helpers.hpp │ ├── Ncurses.cpp │ ├── Ncurses.hpp │ ├── layout │ │ ├── Layout.cpp │ │ └── Layout.hpp │ └── main.cpp ├── emscriptenCompile.sh ├── meson.build ├── nlohmann │ └── json.hpp └── scipnet.html ├── ScpArchive ├── ScpArchive │ ├── Config.cpp │ ├── Config.hpp │ ├── Database │ │ ├── Batch.cpp │ │ ├── Batch.hpp │ │ ├── Database.cpp │ │ ├── Database.hpp │ │ ├── Importer.cpp │ │ ├── Importer.hpp │ │ ├── Json.cpp │ │ └── Json.hpp │ ├── HTTP │ │ ├── HtmlEntity.cpp │ │ ├── HtmlEntity.hpp │ │ ├── MarkupOutStream.cpp │ │ ├── MarkupOutStream.hpp │ │ └── entities │ │ │ ├── entities.cpp │ │ │ └── entities.h │ ├── PDF │ │ ├── PDFConverter.cpp │ │ └── PDFConverter.hpp │ ├── Parser │ │ ├── DatabaseUtil.cpp │ │ ├── DatabaseUtil.hpp │ │ ├── HTMLConverter.cpp │ │ ├── HTMLConverter.hpp │ │ ├── Parser.cpp │ │ ├── Parser.hpp │ │ ├── Rules │ │ │ ├── RuleSet.cpp │ │ │ ├── RuleSet.hpp │ │ │ ├── RuleSetUtil.hpp │ │ │ └── RuleSets │ │ │ │ ├── ARuleSet.cpp │ │ │ │ ├── ARuleSet.hpp │ │ │ │ ├── AlignRuleSet.cpp │ │ │ │ ├── AlignRuleSet.hpp │ │ │ │ ├── AnchorRuleSet.cpp │ │ │ │ ├── AnchorRuleSet.hpp │ │ │ │ ├── BasicTextRuleSet.cpp │ │ │ │ ├── BasicTextRuleSet.hpp │ │ │ │ ├── CSSRuleSet.cpp │ │ │ │ ├── CSSRuleSet.hpp │ │ │ │ ├── CenterTextRuleSet.cpp │ │ │ │ ├── CenterTextRuleSet.hpp │ │ │ │ ├── CodeRuleSet.cpp │ │ │ │ ├── CodeRuleSet.hpp │ │ │ │ ├── CollapsibleRuleSet.cpp │ │ │ │ ├── CollapsibleRuleSet.hpp │ │ │ │ ├── CommentRuleSet.cpp │ │ │ │ ├── CommentRuleSet.hpp │ │ │ │ ├── DivRuleSet.cpp │ │ │ │ ├── DivRuleSet.hpp │ │ │ │ ├── DividerRuleSet.cpp │ │ │ │ ├── DividerRuleSet.hpp │ │ │ │ ├── EntityEscapeRuleSet.cpp │ │ │ │ ├── EntityEscapeRuleSet.hpp │ │ │ │ ├── FootNoteRuleSet.cpp │ │ │ │ ├── FootNoteRuleSet.hpp │ │ │ │ ├── ForumRuleSet.cpp │ │ │ │ ├── ForumRuleSet.hpp │ │ │ │ ├── HTMLRuleSet.cpp │ │ │ │ ├── HTMLRuleSet.hpp │ │ │ │ ├── HeadingRuleSet.cpp │ │ │ │ ├── HeadingRuleSet.hpp │ │ │ │ ├── HyperLinkRuleSet.cpp │ │ │ │ ├── HyperLinkRuleSet.hpp │ │ │ │ ├── IFrameRuleSet.cpp │ │ │ │ ├── IFrameRuleSet.hpp │ │ │ │ ├── IftagsRuleSet.cpp │ │ │ │ ├── IftagsRuleSet.hpp │ │ │ │ ├── ImageRuleSet.cpp │ │ │ │ ├── ImageRuleSet.hpp │ │ │ │ ├── IncludeRuleSet.cpp │ │ │ │ ├── IncludeRuleSet.hpp │ │ │ │ ├── InlineFormatRuleSet.cpp │ │ │ │ ├── InlineFormatRuleSet.hpp │ │ │ │ ├── ListPagesRuleSet.cpp │ │ │ │ ├── ListPagesRuleSet.hpp │ │ │ │ ├── ListRuleSet.cpp │ │ │ │ ├── ListRuleSet.hpp │ │ │ │ ├── LiteralTextRuleSet.cpp │ │ │ │ ├── LiteralTextRuleSet.hpp │ │ │ │ ├── NullRuleSet.cpp │ │ │ │ ├── NullRuleSet.hpp │ │ │ │ ├── QuoteBoxRuleSet.cpp │ │ │ │ ├── QuoteBoxRuleSet.hpp │ │ │ │ ├── RateRuleSet.cpp │ │ │ │ ├── RateRuleSet.hpp │ │ │ │ ├── RedirectRuleSet.cpp │ │ │ │ ├── RedirectRuleSet.hpp │ │ │ │ ├── SCPConversionProjectInfoBoxRuleSet.cpp │ │ │ │ ├── SCPConversionProjectInfoBoxRuleSet.hpp │ │ │ │ ├── SectionRuleSet.cpp │ │ │ │ ├── SectionRuleSet.hpp │ │ │ │ ├── SizeRuleSet.cpp │ │ │ │ ├── SizeRuleSet.hpp │ │ │ │ ├── SpanRuleSet.cpp │ │ │ │ ├── SpanRuleSet.hpp │ │ │ │ ├── TabViewRuleSet.cpp │ │ │ │ ├── TabViewRuleSet.hpp │ │ │ │ ├── TableOfContentsRuleSet.cpp │ │ │ │ ├── TableOfContentsRuleSet.hpp │ │ │ │ ├── TableRuleSet.cpp │ │ │ │ ├── TableRuleSet.hpp │ │ │ │ ├── UserRuleSet.cpp │ │ │ │ └── UserRuleSet.hpp │ │ ├── Templater.cpp │ │ ├── Templater.hpp │ │ ├── Treer.cpp │ │ └── Treer.hpp │ ├── Tests │ │ ├── AutomatedTester.cpp │ │ ├── AutomatedTester.hpp │ │ ├── Database │ │ │ ├── DatabaseTests.cpp │ │ │ ├── DatabaseTests.hpp │ │ │ ├── ImporterTests.cpp │ │ │ ├── ImporterTests.hpp │ │ │ ├── JsonTests.cpp │ │ │ └── JsonTests.hpp │ │ ├── HTTP │ │ │ ├── HtmlEntityTests.cpp │ │ │ ├── HtmlEntityTests.hpp │ │ │ ├── MarkupOutStreamTests.cpp │ │ │ └── MarkupOutStreamTests.hpp │ │ ├── Parser │ │ │ ├── HTMLConverterTests.cpp │ │ │ ├── HTMLConverterTests.hpp │ │ │ ├── ParserTests.cpp │ │ │ ├── ParserTests.hpp │ │ │ ├── TemplaterTests.cpp │ │ │ ├── TemplaterTests.hpp │ │ │ ├── TreerTests.cpp │ │ │ └── TreerTests.hpp │ │ ├── Tests.cpp │ │ └── Tests.hpp │ ├── UntestedUtils │ │ ├── UntestedUtils.cpp │ │ └── UntestedUtils.hpp │ ├── Website │ │ ├── Gateway.cpp │ │ ├── Gateway.hpp │ │ ├── Website.cpp │ │ └── Website.hpp │ └── main.cpp └── meson.build ├── ScpScraper ├── ScpScraper │ ├── Scraper │ │ ├── Helpers.cpp │ │ ├── Helpers.hpp │ │ ├── Scraper.cpp │ │ ├── Scraper.hpp │ │ └── entities │ │ │ ├── Entites.hpp │ │ │ ├── entities.c │ │ │ └── entities.h │ └── main.cpp └── meson.build └── dataLayout.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | src/*/bin/ 35 | src/*/obj/ 36 | src/*/*.depend 37 | src/*/*.layout 38 | 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # SCP Conversion Project 3 | 4 | This project is focused on creating a free, open-source reimplementation of the SCP Wiki. Documentation about the programming is a little scarce at the moment, but more can be found on the [website](https://scp.karagory.com/) and I am always open for question on the project's [discord server](https://discord.gg/7GjqMJj). 5 | 6 | -------------------------------------------------------------------------------- /cgicc-configure.patch: -------------------------------------------------------------------------------- 1 | --- cgicc-3.2.19/configure 2017-06-27 15:38:18.000000000 -0400 2 | +++ cgicc-3.2.19/configure-new 2021-08-04 23:12:01.918533135 -0400 3 | @@ -15979,7 +15979,7 @@ 4 | done 5 | IFS=$as_save_IFS 6 | 7 | - test -z "$ac_cv_prog_DOXYGEN" && ac_cv_prog_DOXYGEN="/bin/echo" 8 | + test -z "$ac_cv_prog_DOXYGEN" && ac_cv_prog_DOXYGEN="echo" 9 | fi 10 | fi 11 | DOXYGEN=$ac_cv_prog_DOXYGEN 12 | -------------------------------------------------------------------------------- /cgicc-package.scm: -------------------------------------------------------------------------------- 1 | (define-module (gnu packages daniel) 2 | #:use-module (guix packages) 3 | #:use-module (guix gexp) 4 | #:use-module ((guix licenses) #:prefix license:) 5 | #:use-module (guix build-system gnu) 6 | #:use-module (guix download) 7 | ;#:use-module (guix utils) 8 | #:use-module (gnu packages autotools)) 9 | 10 | (package 11 | (name "cgicc") 12 | (version "3.2.19") 13 | (inputs 14 | `(("autoconf" ,autoconf))) 15 | (native-inputs '()) 16 | (propagated-inputs '()) 17 | (source (origin 18 | (method url-fetch) 19 | (uri "https://ftp.gnu.org/gnu/cgicc/cgicc-3.2.19.tar.gz") 20 | (sha256 21 | (base32 22 | "1w7pczmi7i09xbz2sgjlv289jx40ibhnqvg3z53k9q4d4ivbj5ra")) 23 | (patches 24 | `(,(local-file "cgicc-configure.patch"))))) 25 | (build-system gnu-build-system) 26 | (synopsis "Cgicc: GNU Cgicc is a C++ api for creating CGI applications") 27 | (description 28 | "GNU Cgicc is an ANSI C++ compliant class library that greatly simplifies the creation of CGI applications for the World Wide Web.") 29 | (home-page "https://www.gnu.org/software/cgicc/") 30 | (license license:lgpl3)) 31 | 32 | -------------------------------------------------------------------------------- /data/customFrontPage/authors.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /data/customFrontPage/batch.json: -------------------------------------------------------------------------------- 1 | { 2 | "pageList": [ 3 | "__front-page" 4 | ], 5 | "threadList": [], 6 | "type": "user" 7 | } 8 | -------------------------------------------------------------------------------- /data/customFrontPage/pages/__front-page/files/designLayout.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DanielBatteryStapler/SCP_Conversion_Project/1b6faccc62a552a2b7d1d0020910e630759b6a21/data/customFrontPage/pages/__front-page/files/designLayout.png -------------------------------------------------------------------------------- /data/customFrontPage/pages/__front-page/files/graphPreview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DanielBatteryStapler/SCP_Conversion_Project/1b6faccc62a552a2b7d1d0020910e630759b6a21/data/customFrontPage/pages/__front-page/files/graphPreview.png -------------------------------------------------------------------------------- /data/customFrontPage/pages/__front-page/files/parserPipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DanielBatteryStapler/SCP_Conversion_Project/1b6faccc62a552a2b7d1d0020910e630759b6a21/data/customFrontPage/pages/__front-page/files/parserPipeline.png -------------------------------------------------------------------------------- /data/exampleNginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | server_tokens off; 3 | 4 | listen 443 ssl http2; 5 | 6 | ssl_certificate /certs/yourdomain.com/fullchain.pem; 7 | 8 | ssl_certificate_key /certs/yourdomain.com/privkey.pem; 9 | 10 | server_name yourdomain.com; 11 | 12 | ssi on; 13 | 14 | root /yourWebRoot/webRoot; 15 | 16 | #max upload size 17 | client_max_body_size 1m; 18 | 19 | location = /robots.txt { 20 | 21 | } 22 | 23 | location /__static { 24 | 25 | } 26 | 27 | location /__pdfs { 28 | 29 | } 30 | 31 | location / { 32 | fastcgi_param GATEWAY_INTERFACE CGI/1.1; 33 | fastcgi_param SERVER_SOFTWARE nginx; 34 | #fastcgi_param QUERY_STRING $query_string; 35 | fastcgi_param QUERY_STRING $document_uri; 36 | fastcgi_param REQUEST_METHOD $request_method; 37 | fastcgi_param CONTENT_TYPE $content_type; 38 | fastcgi_param CONTENT_LENGTH $content_length; 39 | fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name; 40 | fastcgi_param SCRIPT_NAME $fastcgi_script_name; 41 | fastcgi_param REQUEST_URI $request_uri; 42 | fastcgi_param DOCUMENT_URI $document_uri; 43 | fastcgi_param DOCUMENT_ROOT $document_root; 44 | fastcgi_param SERVER_PROTOCOL $server_protocol; 45 | fastcgi_param REMOTE_ADDR $remote_addr; 46 | fastcgi_param REMOTE_PORT $remote_port; 47 | fastcgi_param SERVER_ADDR $server_addr; 48 | fastcgi_param SERVER_PORT $server_port; 49 | fastcgi_param SERVER_NAME $server_name; 50 | 51 | fastcgi_pass_request_body on; 52 | 53 | fastcgi_pass 127.0.0.1:8222; 54 | } 55 | } 56 | 57 | -------------------------------------------------------------------------------- /data/webroot/__static/linkPreview.js: -------------------------------------------------------------------------------- 1 | 2 | window.onload = function(){ 3 | var allLinks = document.getElementsByClassName("previewableLink"); 4 | 5 | var anchorExists = function(element){ 6 | var next = element.nextSibling; 7 | if(next !== null){ 8 | if(typeof next.getAttribute !== "undefined" && next.getAttribute("class") == "linkPreviewAnchor"){ 9 | return true; 10 | } 11 | } 12 | return false; 13 | } 14 | 15 | var initFunction = function(element){ 16 | if(!anchorExists(element)){ 17 | var source = element.getAttribute("href"); 18 | if(source.length > 0 && source.charAt(0) == '/' && source.indexOf('.') == -1){ 19 | source = source.substring(1, source.length); 20 | if(source.length > 0 && source.indexOf('/') == -1){ 21 | var previewAnchor = document.createElement('div'); 22 | previewAnchor.setAttribute("class", "linkPreviewAnchor"); 23 | var preview = document.createElement('div'); 24 | preview.setAttribute("class", "linkPreview"); 25 | var frame = document.createElement('iframe'); 26 | frame.setAttribute("class", "linkPreviewFrame"); 27 | frame.setAttribute("src", "about:blank"); 28 | frame.setAttribute("data-src", "/" + source + "/previewArticle"); 29 | frame.setAttribute("data-timer", "none"); 30 | frame.setAttribute("scrolling", "no"); 31 | preview.appendChild(frame); 32 | previewAnchor.appendChild(preview); 33 | element.parentNode.insertBefore(previewAnchor, element.nextSibling); 34 | } 35 | } 36 | } 37 | } 38 | 39 | var enterFunction = function(){ 40 | if(anchorExists(this)){ 41 | var frame = this.nextSibling.firstElementChild.firstElementChild; 42 | if(frame.getAttribute("data-src") != "loaded"){ 43 | var timer = setTimeout(function(){ 44 | frame.setAttribute("src", frame.getAttribute("data-src")); 45 | frame.setAttribute("data-src", "loaded"); 46 | }, 500); 47 | frame.setAttribute("data-timer", timer); 48 | } 49 | } 50 | }; 51 | 52 | var leaveFunction = function(){ 53 | if(anchorExists(this)){ 54 | var frame = this.nextSibling.firstElementChild.firstElementChild; 55 | if(frame.getAttribute("data-src") != "loaded"){ 56 | clearTimeout(+frame.getAttribute("data-timer")); 57 | frame.setAttribute("data-timer", "loaded"); 58 | } 59 | } 60 | } 61 | 62 | for(var i = 0; i < allLinks.length; ++i){ 63 | var element = allLinks[i]; 64 | initFunction(element); 65 | element.addEventListener('mouseenter', enterFunction); 66 | element.addEventListener('mouseleave', leaveFunction); 67 | } 68 | }; 69 | -------------------------------------------------------------------------------- /data/webroot/__static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DanielBatteryStapler/SCP_Conversion_Project/1b6faccc62a552a2b7d1d0020910e630759b6a21/data/webroot/__static/logo.png -------------------------------------------------------------------------------- /data/webroot/__static/pdfStyle.css: -------------------------------------------------------------------------------- 1 | .footer-wikiwalk-nav{display:none;} 2 | a{text-decoration:inherit;color:inherit;} 3 | .CollapsibleContent{display:block;} 4 | .CollapsibleHiddenText{display:block;} 5 | .CollapsibleShownText{display:block;} 6 | .heritage-rating-module{display:none;} 7 | .page-rate-widget-box{display:none;} 8 | .info-container{display:none;} 9 | 10 | body{font-size:20px;} 11 | *{ 12 | page-break-inside: avoid; 13 | page-break-after: avoid; 14 | page-break-before: avoid; 15 | } 16 | 17 | .TabViewButtons{display:none;} 18 | .TabSectionTitle{margin-bottom:0.5rem;font-size:1.3rem;} 19 | .TabSection{display:block;padding:0;border-style:none;} 20 | 21 | h1{color:black; 22 | blockquote{background-color:white !important;} 23 | .scp-image-caption{background-color:white !important;} 24 | -------------------------------------------------------------------------------- /data/webroot/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: /__system 3 | Disallow: /__pdfs 4 | 5 | -------------------------------------------------------------------------------- /scipnet-package.scm: -------------------------------------------------------------------------------- 1 | (use-modules (guix packages) 2 | (guix gexp) 3 | ((guix licenses) #:prefix license:) 4 | (guix build-system meson) 5 | (gnu packages pkg-config) 6 | (gnu packages cmake) 7 | (gnu packages imagemagick) 8 | (gnu packages curl) 9 | (gnu packages ncurses) 10 | (gnu packages tls)) 11 | 12 | (package 13 | (name "Scipnet") 14 | (version "0.0") 15 | (inputs 16 | `(("imagemagick" ,imagemagick) 17 | ("curl" ,curl) 18 | ("curlpp" ,curlpp) 19 | ("ncurses" ,ncurses) 20 | ("openssl" ,openssl))) 21 | (native-inputs 22 | `(("pkg-config" ,pkg-config) 23 | ("cmake" ,cmake))) 24 | (propagated-inputs '()) 25 | (source (local-file "./src/Scipnet" #:recursive? #t)) 26 | (build-system meson-build-system) 27 | (synopsis "Scipnet: Terminal-based client for the SCP Conversion Project") 28 | (description 29 | "Terminal-based viewer client for the SCP Conversion Project. Works both in-browser and in-terminal featuring text-reflow support and character-based image rendering.") 30 | (home-page "https://github.com/danielbatterystapler/SCP_Conversion_Project") 31 | (license license:gpl3)) 32 | 33 | -------------------------------------------------------------------------------- /scp-archive-package.scm: -------------------------------------------------------------------------------- 1 | (use-modules (guix packages) 2 | (guix gexp) 3 | ((guix licenses) #:prefix license:) 4 | (guix build-system meson) 5 | (gnu packages pkg-config) 6 | (gnu packages cmake) 7 | (gnu packages cpp) 8 | (gnu packages web) 9 | (gnu packages boost) 10 | (gnu packages databases) 11 | (gnu packages compression)) 12 | 13 | (package 14 | (name "ScpArchive") 15 | (version "0.0") 16 | (inputs 17 | `(("json-modern-cxx" ,json-modern-cxx) 18 | ("fcgi" ,fcgi) 19 | ("cgicc" ,(load "cgicc-package.scm")) 20 | ("boost" ,boost) 21 | ("soci" ,(load "soci-package.scm")) 22 | ("mariadb:dev" ,mariadb "dev") 23 | ("zlib" ,zlib))) 24 | (native-inputs 25 | `(("pkg-config" ,pkg-config) 26 | ("cmake" ,cmake))) 27 | (propagated-inputs '()) 28 | (source (local-file "./src/ScpArchive" #:recursive? #t)) 29 | (build-system meson-build-system) 30 | (synopsis "ScpArchive: Web-Backend for the SCP Conversion Project") 31 | (description 32 | "Web-Backend for the SCP Conversion Project configured as an FCGI-gateway") 33 | (home-page "https://github.com/danielbatterystapler/SCP_Conversion_Project") 34 | (license license:gpl3)) 35 | 36 | -------------------------------------------------------------------------------- /scp-scraper-package.scm: -------------------------------------------------------------------------------- 1 | (use-modules (guix packages) 2 | (guix gexp) 3 | ((guix licenses) #:prefix license:) 4 | (guix build-system meson) 5 | (gnu packages pkg-config) 6 | (gnu packages cmake) 7 | (gnu packages cpp) 8 | (gnu packages boost) 9 | (gnu packages curl) 10 | (gnu packages tls) 11 | (gnu packages certs)) 12 | 13 | (package 14 | (name "ScpScraper") 15 | (version "0.0") 16 | (inputs 17 | `(("json-modern-cxx" ,json-modern-cxx) 18 | ("boost" ,boost) 19 | ("curl" ,curl) 20 | ("curlpp" ,curlpp) 21 | ("openssl" ,openssl))) 22 | (native-inputs 23 | `(("pkg-config" ,pkg-config) 24 | ("cmake" ,cmake))) 25 | (propagated-inputs 26 | `(("nss-certs" ,nss-certs))) 27 | (source (local-file "./src/ScpScraper" #:recursive? #t)) 28 | (build-system meson-build-system) 29 | (synopsis "ScpScraper: Scraper utility for the SCP-Wiki") 30 | (description 31 | "Scraper Utility for the SCP-Wiki that collects and packages all information for importation using the ScpArchive program") 32 | (home-page "https://github.com/danielbatterystapler/SCP_Conversion_Project") 33 | (license license:gpl3)) 34 | 35 | -------------------------------------------------------------------------------- /soci-package.scm: -------------------------------------------------------------------------------- 1 | (use-modules (guix packages) 2 | (guix gexp) 3 | ((guix licenses) #:prefix license:) 4 | (srfi srfi-1) 5 | (gnu packages databases)) 6 | 7 | ;we need to modify the soci package in Giux's gnu/packages/database.scm 8 | ;because it is built with mariadb, not mysql, and soci doesn't like that 9 | ;I believe this needs to either be upstreamed to Guix or soci needs to be modified to build with mariadb 10 | ;for now, we're going to modify it for us 11 | 12 | (package 13 | (inherit soci) 14 | (name "soci-pero-mysql") 15 | (propagated-inputs 16 | (cons `("mysql" ,mysql) 17 | (alist-delete "mariadb" 18 | (package-propagated-inputs soci))))) 19 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/Cli.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CLI_HPP 2 | #define CLI_HPP 3 | 4 | #include "nlohmann/json.hpp" 5 | 6 | namespace Cli{ 7 | nlohmann::json processCliProxy(nlohmann::json input); 8 | nlohmann::json processCli(nlohmann::json input); 9 | } 10 | 11 | #endif // CLI_HPP 12 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/ColorTable.cpp: -------------------------------------------------------------------------------- 1 | #include "ColorTable.hpp" 2 | 3 | std::map colorTable = { 4 | {"aliceblue", "#f0f8ff"}, 5 | {"antiquewhite", "#faebd7"}, 6 | {"aqua", "#00ffff"}, 7 | {"aquamarine", "#7fffd4"}, 8 | {"azure", "#f0ffff"}, 9 | {"beige", "#f5f5dc"}, 10 | {"bisque", "#ffe4c4"}, 11 | {"black", "#000000"}, 12 | {"blanchedalmond", "#ffebcd"}, 13 | {"blue", "#0000ff"}, 14 | {"blueviolet", "#8a2be2"}, 15 | {"brown", "#a52a2a"}, 16 | {"burlywood", "#deb887"}, 17 | {"cadetblue", "#5f9ea0"}, 18 | {"chartreuse", "#7fff00"}, 19 | {"chocolate", "#d2691e"}, 20 | {"coral", "#ff7f50"}, 21 | {"cornflowerblue", "#6495ed"}, 22 | {"cornsilk", "#fff8dc"}, 23 | {"crimson", "#dc143c"}, 24 | {"cyan", "#00ffff"}, 25 | {"darkblue", "#00008b"}, 26 | {"darkcyan", "#008b8b"}, 27 | {"darkgoldenrod", "#b8860b"}, 28 | {"darkgray", "#a9a9a9"}, 29 | {"darkgreen", "#006400"}, 30 | {"darkgrey", "#a9a9a9"}, 31 | {"darkkhaki", "#bdb76b"}, 32 | {"darkmagenta", "#8b008b"}, 33 | {"darkolivegreen", "#556b2f"}, 34 | {"darkorange", "#ff8c00"}, 35 | {"darkorchid", "#9932cc"}, 36 | {"darkred", "#8b0000"}, 37 | {"darksalmon", "#e9967a"}, 38 | {"darkseagreen", "#8fbc8f"}, 39 | {"darkslateblue", "#483d8b"}, 40 | {"darkslategray", "#2f4f4f"}, 41 | {"darkslategrey", "#2f4f4f"}, 42 | {"darkturquoise", "#00ced1"}, 43 | {"darkviolet", "#9400d3"}, 44 | {"deeppink", "#ff1493"}, 45 | {"deepskyblue", "#00bfff"}, 46 | {"dimgray", "#696969"}, 47 | {"dimgrey", "#696969"}, 48 | {"dodgerblue", "#1e90ff"}, 49 | {"firebrick", "#b22222"}, 50 | {"floralwhite", "#fffaf0"}, 51 | {"forestgreen", "#228b22"}, 52 | {"fuchsia", "#ff00ff"}, 53 | {"gainsboro", "#dcdcdc"}, 54 | {"ghostwhite", "#f8f8ff"}, 55 | {"gold", "#ffd700"}, 56 | {"goldenrod", "#daa520"}, 57 | {"gray", "#808080"}, 58 | {"green", "#008000"}, 59 | {"greenyellow", "#adff2f"}, 60 | {"grey", "#808080"}, 61 | {"honeydew", "#f0fff0"}, 62 | {"hotpink", "#ff69b4"}, 63 | {"indianred", "#cd5c5c"}, 64 | {"indigo", "#4b0082"}, 65 | {"ivory", "#fffff0"}, 66 | {"khaki", "#f0e68c"}, 67 | {"lavender", "#e6e6fa"}, 68 | {"lavenderblush", "#fff0f5"}, 69 | {"lawngreen", "#7cfc00"}, 70 | {"lemonchiffon", "#fffacd"}, 71 | {"lightblue", "#add8e6"}, 72 | {"lightcoral", "#f08080"}, 73 | {"lightcyan", "#e0ffff"}, 74 | {"lightgoldenrodyellow", "#fafad2"}, 75 | {"lightgray", "#d3d3d3"}, 76 | {"lightgreen", "#90ee90"}, 77 | {"lightgrey", "#d3d3d3"}, 78 | {"lightpink", "#ffb6c1"}, 79 | {"lightsalmon", "#ffa07a"}, 80 | {"lightseagreen", "#20b2aa"}, 81 | {"lightskyblue", "#87cefa"}, 82 | {"lightslategray", "#778899"}, 83 | {"lightslategrey", "#778899"}, 84 | {"lightsteelblue", "#b0c4de"}, 85 | {"lightyellow", "#ffffe0"}, 86 | {"lime", "#00ff00"}, 87 | {"limegreen", "#32cd32"}, 88 | {"linen", "#faf0e6"}, 89 | {"magenta", "#ff00ff"}, 90 | {"maroon", "#800000"}, 91 | {"mediumaquamarine", "#66cdaa"}, 92 | {"mediumblue", "#0000cd"}, 93 | {"mediumorchid", "#ba55d3"}, 94 | {"mediumpurple", "#9370db"}, 95 | {"mediumseagreen", "#3cb371"}, 96 | {"mediumslateblue", "#7b68ee"}, 97 | {"mediumspringgreen", "#00fa9a"}, 98 | {"mediumturquoise", "#48d1cc"}, 99 | {"mediumvioletred", "#c71585"}, 100 | {"midnightblue", "#191970"}, 101 | {"mintcream", "#f5fffa"}, 102 | {"mistyrose", "#ffe4e1"}, 103 | {"moccasin", "#ffe4b5"}, 104 | {"navajowhite", "#ffdead"}, 105 | {"navy", "#000080"}, 106 | {"oldlace", "#fdf5e6"}, 107 | {"olive", "#808000"}, 108 | {"olivedrab", "#6b8e23"}, 109 | {"orange", "#ffa500"}, 110 | {"orangered", "#ff4500"}, 111 | {"orchid", "#da70d6"}, 112 | {"palegoldenrod", "#eee8aa"}, 113 | {"palegreen", "#98fb98"}, 114 | {"paleturquoise", "#afeeee"}, 115 | {"palevioletred", "#db7093"}, 116 | {"papayawhip", "#ffefd5"}, 117 | {"peachpuff", "#ffdab9"}, 118 | {"peru", "#cd853f"}, 119 | {"pink", "#ffc0cb"}, 120 | {"plum", "#dda0dd"}, 121 | {"powderblue", "#b0e0e6"}, 122 | {"purple", "#800080"}, 123 | {"red", "#ff0000"}, 124 | {"rosybrown", "#bc8f8f"}, 125 | {"royalblue", "#4169e1"}, 126 | {"saddlebrown", "#8b4513"}, 127 | {"salmon", "#fa8072"}, 128 | {"sandybrown", "#f4a460"}, 129 | {"seagreen", "#2e8b57"}, 130 | {"seashell", "#fff5ee"}, 131 | {"sienna", "#a0522d"}, 132 | {"silver", "#c0c0c0"}, 133 | {"skyblue", "#87ceeb"}, 134 | {"slateblue", "#6a5acd"}, 135 | {"slategray", "#708090"}, 136 | {"slategrey", "#708090"}, 137 | {"snow", "#fffafa"}, 138 | {"springgreen", "#00ff7f"}, 139 | {"steelblue", "#4682b4"}, 140 | {"tan", "#d2b48c"}, 141 | {"teal", "#008080"}, 142 | {"thistle", "#d8bfd8"}, 143 | {"tomato", "#ff6347"}, 144 | {"turquoise", "#40e0d0"}, 145 | {"violet", "#ee82ee"}, 146 | {"wheat", "#f5deb3"}, 147 | {"white", "#ffffff"}, 148 | {"whitesmoke", "#f5f5f5"}, 149 | {"yellow", "#ffff00"}, 150 | {"yellowgreen", "#9acd32"} 151 | }; 152 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/ColorTable.hpp: -------------------------------------------------------------------------------- 1 | #ifndef COLORTABLE_HPP 2 | #define COLORTABLE_HPP 3 | 4 | #include 5 | #include 6 | 7 | extern std::map colorTable; 8 | 9 | #endif // COLORTABLE_HPP 10 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/Converter.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CONVERTER_HPP 2 | #define CONVERTER_HPP 3 | 4 | #include "layout/Layout.hpp" 5 | 6 | #include "nlohmann/json.hpp" 7 | 8 | namespace Converter{ 9 | std::shared_ptr makeLayoutFromPageJson(const nlohmann::json& page); 10 | 11 | inline int maxPageWidth = 140; 12 | inline int imageMaxWidthExpandCutoff = 100; 13 | }; 14 | 15 | #endif // CONVERTER_HPP 16 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/Database.cpp: -------------------------------------------------------------------------------- 1 | #include "Database.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include "Helpers.hpp" 14 | 15 | constexpr unsigned int MaxRGBValue = 65535; 16 | //should really be defined off of the "QuantumRange" macro exported by ImageMagick, but doing so gives a compiler error 17 | //for some reaosn, and this number is really just not going to change much 18 | 19 | namespace Database{ 20 | nlohmann::json getPageTree(std::string name){ 21 | curlpp::Cleanup myCleanup; 22 | curlpp::Easy request; 23 | request.setOpt("https://scp.karagory.com/" + name + "/showTreeJSON"); 24 | std::ostringstream os; 25 | curlpp::options::WriteStream ws(&os); 26 | request.setOpt(ws); 27 | request.perform(); 28 | std::string data = os.str(); 29 | return nlohmann::json::parse(data); 30 | } 31 | 32 | std::string downloadImage(std::string url){ 33 | curlpp::Cleanup myCleanup; 34 | curlpp::Easy request; 35 | if(url.find("/") == 0){ 36 | request.setOpt("https://scp.karagory.com/" + url); 37 | } 38 | else{ 39 | request.setOpt(url); 40 | } 41 | std::ostringstream os; 42 | curlpp::options::WriteStream ws(&os); 43 | request.setOpt(ws); 44 | request.perform(); 45 | std::string data = os.str(); 46 | return data; 47 | } 48 | 49 | nlohmann::json getPage(std::string name){ 50 | nlohmann::json output; 51 | 52 | try{ 53 | nlohmann::json pageTree = getPageTree(name); 54 | loadImagesIntoPageTree(pageTree); 55 | output["pageTree"] = pageTree; 56 | } 57 | catch(std::exception& e){ 58 | output["pageTree"] = { 59 | {"type", "RootPage"}, 60 | {"data", nullptr}, 61 | {"branches", { 62 | { 63 | {"type", "Paragraph"}, 64 | {"data", nullptr}, 65 | {"branches", { 66 | { 67 | {"type", "PlainText"}, 68 | {"data", "Error while loading page '" + name + "'"}, 69 | {"branches", nlohmann::json::array()} 70 | } 71 | }} 72 | } 73 | }} 74 | }; 75 | } 76 | 77 | return output; 78 | } 79 | 80 | namespace{ 81 | 82 | void visitPageTree(nlohmann::json& pageTree, std::function visitor){ 83 | visitor(pageTree); 84 | for(auto& branch : pageTree["branches"]){ 85 | visitPageTree(branch, visitor); 86 | } 87 | } 88 | 89 | } 90 | 91 | void loadImagesIntoPageTree(nlohmann::json& pageTree){ 92 | 93 | visitPageTree(pageTree, [&](nlohmann::json& node){ 94 | if(node["type"].get() == "Image"){ 95 | 96 | std::string rawImageFile = downloadImage(redirectLink(node["data"]["source"].get())); 97 | 98 | try{ 99 | Magick::Blob blob{rawImageFile.c_str(), rawImageFile.size()}; 100 | Magick::Image image{blob}; 101 | 102 | //image.quantizeColorSpace(Magick::RGBColorspace); 103 | //image.quantizeColors(256); 104 | //image.quantize(); 105 | 106 | unsigned int rows = image.rows(); 107 | unsigned int columns = image.columns(); 108 | 109 | constexpr unsigned int maximumDimension = 256; 110 | 111 | if(std::max(rows, columns) > maximumDimension){ 112 | if(rows > columns){ 113 | columns = columns * static_cast(maximumDimension)/rows; 114 | rows = maximumDimension; 115 | } 116 | else{ 117 | rows = rows * static_cast(maximumDimension)/columns; 118 | columns = maximumDimension; 119 | } 120 | } 121 | 122 | image.resize(Magick::Geometry{columns, rows}); 123 | columns = image.columns(); 124 | rows = image.rows(); 125 | 126 | std::stringstream hexData; 127 | 128 | constexpr auto toHex = [](int num)->std::string{ 129 | std::stringstream out; 130 | out << std::hex << (num / 16) << (num % 16); 131 | return out.str(); 132 | }; 133 | 134 | const Magick::PixelPacket* pixel_cache = image.getPixels(0, 0, columns, rows); 135 | 136 | for(unsigned int y = 0; y < rows; y++){ 137 | for(unsigned int x = 0; x < columns; x++){ 138 | const Magick::PixelPacket* pixel = (pixel_cache+y*columns+x); 139 | unsigned int depthCorrection = MaxRGBValue / 255; 140 | std::stringstream singlePixel; 141 | singlePixel << toHex(pixel->red / depthCorrection) << toHex(pixel->green / depthCorrection) << toHex(pixel->blue / depthCorrection); 142 | if(singlePixel.str().size() != 6){ 143 | throw std::runtime_error{"invalid pixel: " + singlePixel.str()}; 144 | } 145 | hexData << singlePixel.str(); 146 | } 147 | } 148 | 149 | nlohmann::json imageJson; 150 | imageJson["valid"] = true; 151 | imageJson["pixels"] = hexData.str(); 152 | imageJson["height"] = rows; 153 | imageJson["width"] = columns; 154 | node["imageData"] = imageJson; 155 | } 156 | catch(std::exception& e){ 157 | 158 | nlohmann::json imageJson; 159 | imageJson["valid"] = false; 160 | node["imageData"] = imageJson; 161 | } 162 | } 163 | }); 164 | 165 | } 166 | } 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/Database.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATABASE_HPP 2 | #define DATABASE_HPP 3 | 4 | #include "nlohmann/json.hpp" 5 | 6 | namespace Database{ 7 | nlohmann::json getPageTree(std::string name); 8 | 9 | nlohmann::json getPage(std::string name); 10 | void loadImagesIntoPageTree(nlohmann::json& pageTree); 11 | } 12 | 13 | #endif // DATABASE_HPP 14 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/Emscript.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include "Cli.hpp" 5 | #include "Helpers.hpp" 6 | 7 | #include 8 | using namespace emscripten; 9 | 10 | extern "C" int main(int argc, char** argv){ 11 | printf("Hello from SCiPNET wasm!\n"); 12 | return 0; 13 | } 14 | 15 | std::string processCliString(std::string jsonInput){ 16 | return Cli::processCli(nlohmann::json::parse(jsonInput)).dump(); 17 | } 18 | 19 | EMSCRIPTEN_BINDINGS(scipnet_module){ 20 | function("processCliString", &processCliString); 21 | function("redirectLink", &redirectLink); 22 | } -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/Helpers.hpp: -------------------------------------------------------------------------------- 1 | #ifndef HELPERS_HPP_INCLUDED 2 | #define HELPERS_HPP_INCLUDED 3 | 4 | #include 5 | 6 | static std::string redirectLink(std::string rawLink){ 7 | std::string output; 8 | const auto doRedirect = [&rawLink, &output](std::string oldPrefix, std::string newPrefix){ 9 | if(output == "" && rawLink.rfind(oldPrefix, 0) == 0){ 10 | output = newPrefix + rawLink.substr(oldPrefix.size(), rawLink.size() - oldPrefix.size()); 11 | } 12 | }; 13 | doRedirect("http://www.scp-wiki.net/local--files/", "/__system/pageFile/"); 14 | doRedirect("https://www.scp-wiki.net/local--files/", "/__system/pageFile/"); 15 | doRedirect("http://www.scp-wiki.net/", "/"); 16 | doRedirect("https://www.scp-wiki.net/", "/"); 17 | doRedirect("http://scp-wiki.wikidot.com/", "/"); 18 | doRedirect("https://scp-wiki.wikidot.com/", "/"); 19 | doRedirect("http://scp-wiki.wdfiles.com/local--files/", "/__system/pageFile/"); 20 | doRedirect("https://scp-wiki.wdfiles.com/local--files/", "/__system/pageFile/"); 21 | if(output == ""){ 22 | return rawLink; 23 | } 24 | else{ 25 | return output; 26 | } 27 | } 28 | 29 | #endif // HELPERS_HPP_INCLUDED 30 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/Ncurses.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NCURSES_HPP 2 | #define NCURSES_HPP 3 | 4 | void runNcursesTerminal(); 5 | 6 | #endif // NCURSES_HPP 7 | -------------------------------------------------------------------------------- /src/Scipnet/Scipnet/main.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include "Ncurses.hpp" 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | static void handler(int signum){ 11 | printf("\n"); 12 | exit(0); 13 | } 14 | 15 | int main(){ 16 | Magick::InitializeMagick(nullptr); 17 | 18 | struct sigaction sa = {}; 19 | sa.sa_handler = handler; 20 | sigemptyset(&sa.sa_mask); 21 | sa.sa_flags = SA_RESTART; 22 | 23 | sigaction(SIGINT, &sa, NULL); 24 | 25 | try{ 26 | runNcursesTerminal(); 27 | } 28 | catch(std::exception& e){ 29 | std::cout << "exception: " << e.what() << "\n"; 30 | while(true){ 31 | 32 | } 33 | } 34 | 35 | return 0; 36 | } 37 | -------------------------------------------------------------------------------- /src/Scipnet/emscriptenCompile.sh: -------------------------------------------------------------------------------- 1 | mkdir bin/Emscripten 2 | source ~/Projects/emscripten/emsdk/emsdk_env.sh 3 | 4 | em++ --std=c++17 -s TOTAL_MEMORY=256MB --bind -O3 -o bin/Emscripten/scipnet.js -I./ Scipnet/layout/Layout.cpp Scipnet/Cli.cpp Scipnet/ColorTable.cpp Scipnet/Converter.cpp Scipnet/Emscript.cpp 5 | #cp scipnet.html bin/Emscripten/scipnet.html 6 | -------------------------------------------------------------------------------- /src/Scipnet/meson.build: -------------------------------------------------------------------------------- 1 | project('Scipnet', 'cpp') 2 | 3 | magick_config = run_command('Magick++-config', '--cppflags', '--cxxflags') 4 | magick_arguments = magick_config.stdout().strip().split() 5 | add_project_arguments(magick_arguments, language: 'cpp') 6 | 7 | magick_link_config = run_command('Magick++-config', '--ldflags', '--libs') 8 | magick_link_arguments = magick_link_config.stdout().strip().split() 9 | add_project_link_arguments(magick_link_arguments, language: 'cpp') 10 | 11 | cpp = meson.get_compiler('cpp') 12 | 13 | dep = [ 14 | dependency('ncurses'), 15 | cpp.find_library('ssl'), 16 | cpp.find_library('crypto'), 17 | cpp.find_library('curlpp'), 18 | cpp.find_library('curl') 19 | ] 20 | 21 | inc = [ 22 | '.' 23 | ] 24 | 25 | src = [ 26 | 'Scipnet/main.cpp', 27 | 'Scipnet/Cli.cpp', 28 | 'Scipnet/ColorTable.cpp', 29 | 'Scipnet/Converter.cpp', 30 | 'Scipnet/Database.cpp', 31 | 'Scipnet/Ncurses.cpp', 32 | 'Scipnet/layout/Layout.cpp' 33 | ] 34 | 35 | executable( 36 | 'Scipnet', 37 | sources : src, 38 | dependencies : dep, 39 | include_directories : inc, 40 | install : true, 41 | override_options : ['cpp_std=c++17'] 42 | ) 43 | 44 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Config.cpp: -------------------------------------------------------------------------------- 1 | #include "Config.hpp" 2 | 3 | #include 4 | 5 | namespace Config{ 6 | std::string getTestingDatabaseName(){ 7 | return "Temporary_SCP_Testing_Database"; 8 | } 9 | 10 | unsigned int getNumberOfThreadsForGateway(){ 11 | return 16; 12 | } 13 | 14 | std::string getProductionDatabaseName(){ 15 | return "SCPArchive"; 16 | } 17 | 18 | std::string getFastCGISocket(){ 19 | return ":8222"; 20 | } 21 | 22 | std::string getWebsiteDomainName(){ 23 | return "scp.karagory.com"; 24 | } 25 | 26 | std::string getDatabaseHost(){ 27 | return "scp-database"; 28 | } 29 | 30 | std::string getDatabaseUser(){ 31 | return "scparchive"; 32 | } 33 | 34 | std::string getDatabasePassword(){ 35 | return "password"; 36 | } 37 | 38 | std::string getScraperFolder(){ 39 | return "/scp_conversion_project/batchArchive/"; 40 | } 41 | 42 | std::string getPDFFolder(){ 43 | return "/scp_conversion_project/webRoot/__pdfs/"; 44 | } 45 | 46 | std::string getWebRootStaticFolder(){ 47 | return "/scp_conversion_project/webRoot/__static/"; 48 | } 49 | } 50 | 51 | 52 | 53 | 54 | TimeStamp getCurrentTime(){ 55 | return static_cast(std::time(nullptr)); 56 | } -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Config.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_HPP 2 | #define CONFIG_HPP 3 | 4 | #include 5 | 6 | namespace Config{ 7 | std::string getTestingDatabaseName(); 8 | 9 | std::string getImportMapFileName(); 10 | 11 | unsigned int getNumberOfThreadsForGateway(); 12 | std::string getProductionDatabaseName(); 13 | std::string getFastCGISocket(); 14 | std::string getWebsiteDomainName(); 15 | 16 | std::string getDatabaseHost(); 17 | std::string getDatabaseUser(); 18 | std::string getDatabasePassword(); 19 | 20 | std::string getScraperFolder(); 21 | std::string getPDFFolder(); 22 | std::string getWebRootStaticFolder(); 23 | } 24 | 25 | using TimeStamp = std::int64_t; 26 | TimeStamp getCurrentTime(); 27 | 28 | #endif // CONFIG_HPP 29 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Database/Batch.hpp: -------------------------------------------------------------------------------- 1 | #ifndef BATCH_HPP 2 | #define BATCH_HPP 3 | 4 | #include "Importer.hpp" 5 | 6 | namespace Importer{ 7 | void handleBatches(std::string batchesFolder, std::string batchDataFile, bool automatic = false); 8 | void importBatch(std::string batchFolder); 9 | 10 | TimeStamp lastImportedBatchTimeStamp(std::string batchesFolder, std::string batchDataFile); 11 | } 12 | 13 | #endif // BATCH_HPP 14 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Database/Importer.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IMPORTER_HPP 2 | #define IMPORTER_HPP 3 | 4 | #include 5 | #include 6 | 7 | #include "Database.hpp" 8 | 9 | #include 10 | 11 | namespace Importer{ 12 | 13 | struct ImportMap{ 14 | public: 15 | 16 | ImportMap(Database* database); 17 | 18 | void setPageMap(std::string raw, Database::ID id); 19 | Database::ID getPageMap(std::string raw); 20 | std::string getPageMapRaw(Database::ID id); 21 | bool pageMapExists(std::string raw); 22 | 23 | void setFileMap(std::string raw, Database::ID id); 24 | Database::ID getFileMap(std::string raw); 25 | std::string getFileMapRaw(Database::ID id); 26 | bool fileMapExists(std::string raw); 27 | 28 | void setAuthorMap(std::string raw, Database::ID id); 29 | Database::ID getAuthorMap(std::string raw); 30 | std::string getAuthorMapRaw(Database::ID id); 31 | bool authorMapExists(std::string raw); 32 | 33 | private: 34 | Database* db; 35 | }; 36 | 37 | void importAuthors(Database* database, ImportMap& map, const nlohmann::json& authorsData); 38 | void importAuthor(Database* database, ImportMap& map, const nlohmann::json& authorData); 39 | 40 | void importForumGroups(Database* database, ImportMap& map, const nlohmann::json& forumGroups); 41 | void importThreadsFromFolder(Database* database, ImportMap& map, std::string threadsDirectory, std::vector threads); 42 | void importPosts(Database* database, ImportMap& map, const nlohmann::json& posts, Database::ID parentThread, std::optional parentPost); 43 | void importThread(Database* database, ImportMap& map, const nlohmann::json& threadData); 44 | 45 | void importPagesFromFolder(Database* database, ImportMap& map, std::string pagesDirectory, std::vector pages); 46 | void importPage(Database* database, ImportMap& map, const nlohmann::json& pageData); 47 | 48 | void uploadPageFilesFromFolder(Database* database, ImportMap& map, std::string pagesDirectory, std::vector pages); 49 | void uploadPageFiles(Database* database, ImportMap& map, const nlohmann::json& pageData, std::string pageDirectory); 50 | } 51 | 52 | #endif // IMPORTER_HPP 53 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Database/Json.cpp: -------------------------------------------------------------------------------- 1 | #include "Json.hpp" 2 | 3 | #include 4 | 5 | namespace Json{ 6 | nlohmann::json loadJsonFromFile(std::string fileName){ 7 | std::ifstream file(fileName); 8 | if(!file){ 9 | throw std::runtime_error("Attempted to JSON from file \"" + fileName + "\", but could not read from that file"); 10 | } 11 | 12 | std::vector fileContents((std::istreambuf_iterator(file)), std::istreambuf_iterator()); 13 | std::string fileContentsStr(fileContents.begin(), fileContents.end()); 14 | return nlohmann::json::parse(fileContentsStr); 15 | } 16 | 17 | void saveJsonToFile(std::string fileName, nlohmann::json data){ 18 | std::ofstream(fileName) << data.dump(4); 19 | } 20 | } 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Database/Json.hpp: -------------------------------------------------------------------------------- 1 | #ifndef JSON_HPP 2 | #define JSON_HPP 3 | 4 | #include 5 | 6 | namespace Json{ 7 | nlohmann::json loadJsonFromFile(std::string fileName); 8 | void saveJsonToFile(std::string fileName, nlohmann::json data); 9 | } 10 | 11 | #endif // JSON_HPP 12 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/HTTP/HtmlEntity.cpp: -------------------------------------------------------------------------------- 1 | #include "HtmlEntity.hpp" 2 | 3 | #include "entities/entities.h" 4 | 5 | std::string decodeHtmlEntities(std::string input){ 6 | char* outputBuffer = new char[input.size() + 1]; 7 | std::size_t outputSize = decode_html_entities_utf8(outputBuffer, input.c_str()); 8 | std::string output{outputBuffer, outputSize}; 9 | return output; 10 | } 11 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/HTTP/HtmlEntity.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ENTITIES_HPP 2 | #define ENTITIES_HPP 3 | 4 | #include 5 | 6 | std::string decodeHtmlEntities(std::string input); 7 | 8 | #endif // ENTITIES_HPP 9 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/HTTP/MarkupOutStream.cpp: -------------------------------------------------------------------------------- 1 | #include "MarkupOutStream.hpp" 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | std::string urlEncode(const std::string& text){ 9 | std::ostringstream escaped; 10 | escaped.fill('0'); 11 | escaped << std::hex; 12 | 13 | for (char c : text) { 14 | //Keep alphanumeric and other accepted characters intact 15 | if(isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~'){ 16 | escaped << c; 17 | continue; 18 | } 19 | 20 | //special case for spaces 21 | if(c == ' '){ 22 | escaped << '+'; 23 | continue; 24 | } 25 | 26 | //Any other characters are percent-encoded 27 | escaped << std::uppercase; 28 | escaped << '%' << std::setw(2) << static_cast(static_cast(c)); 29 | escaped << std::nouppercase; 30 | } 31 | 32 | return escaped.str(); 33 | } 34 | 35 | std::string urlDecode(const std::string& text){ 36 | //for now we're just going to rely on cgicc's implementation for this 37 | return cgicc::form_urldecode(text); 38 | } 39 | 40 | namespace{ 41 | std::string escapeMarkup(const std::string& str){ 42 | std::stringstream output; 43 | for(char c : str){ 44 | switch(c){ 45 | case '<': 46 | output << "<"; 47 | break; 48 | case '>': 49 | output << ">"; 50 | break; 51 | case '&': 52 | output << "&"; 53 | break; 54 | case ';': 55 | output << ";"; 56 | break; 57 | case '"': 58 | output << """; 59 | break; 60 | case '\'': 61 | output << "'"; 62 | break; 63 | default: 64 | output << c; 65 | break; 66 | } 67 | } 68 | return output.str(); 69 | } 70 | } 71 | 72 | MarkupOutStream::MarkupOutStream(): 73 | outStream(nullptr){ 74 | 75 | } 76 | 77 | MarkupOutStream::MarkupOutStream(std::ostream* outputStream): 78 | outStream(outputStream){ 79 | 80 | } 81 | 82 | MarkupOutStream& MarkupOutStream::operator<<(char c){ 83 | *outStream << escapeMarkup(std::string(1, c)); 84 | return *this; 85 | } 86 | 87 | MarkupOutStream& MarkupOutStream::operator<<(std::string str){ 88 | *outStream << escapeMarkup(str); 89 | return *this; 90 | } 91 | 92 | MarkupOutStream& MarkupOutStream::operator<<(const MarkupOutStream::MarkupOutString& str){ 93 | *outStream << str.buffer; 94 | return *this; 95 | } 96 | 97 | MarkupOutStream& MarkupOutStream::operator<<(MarkupOutStream::MarkupOutString&& str){ 98 | *outStream << str.buffer; 99 | return *this; 100 | } 101 | 102 | std::ostream* MarkupOutStream::getUnsafeRawOutputStream(){ 103 | return outStream; 104 | } 105 | 106 | MarkupOutStream::MarkupOutString::MarkupOutString(std::string str): 107 | buffer(str){ 108 | 109 | } 110 | 111 | MarkupOutStream::MarkupOutString allowMarkup(std::string str){ 112 | return MarkupOutStream::MarkupOutString{str}; 113 | } 114 | 115 | MarkupOutStream::MarkupOutString operator""_AM(const char* str, long unsigned int length){ 116 | return MarkupOutStream::MarkupOutString{std::string{str, length}}; 117 | } 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/HTTP/MarkupOutStream.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MARKUPOUTSTREAM_HPP 2 | #define MARKUPOUTSTREAM_HPP 3 | 4 | #include 5 | #include 6 | 7 | std::string urlEncode(const std::string& text); 8 | std::string urlDecode(const std::string& text); 9 | 10 | class MarkupOutStream{ 11 | public: 12 | class MarkupOutString; 13 | 14 | MarkupOutStream(); 15 | MarkupOutStream(std::ostream* outputStream); 16 | 17 | MarkupOutStream& operator<<(char c); 18 | MarkupOutStream& operator<<(std::string str); 19 | MarkupOutStream& operator<<(const MarkupOutStream::MarkupOutString& str); 20 | MarkupOutStream& operator<<(MarkupOutStream::MarkupOutString&& str); 21 | 22 | std::ostream* getUnsafeRawOutputStream(); 23 | 24 | private: 25 | std::ostream* outStream; 26 | 27 | public: 28 | class MarkupOutString{ 29 | public: 30 | friend class MarkupOutStream; 31 | friend MarkupOutString allowMarkup(std::string str); 32 | friend MarkupOutString operator""_AM(const char* str, long unsigned int length); 33 | 34 | private: 35 | MarkupOutString(std::string str); 36 | 37 | std::string buffer; 38 | }; 39 | }; 40 | 41 | MarkupOutStream::MarkupOutString allowMarkup(std::string str); 42 | MarkupOutStream::MarkupOutString operator""_AM(const char* str, long unsigned int length); 43 | 44 | #endif // MARKUPOUTSTREAM_HPP 45 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/HTTP/entities/entities.h: -------------------------------------------------------------------------------- 1 | extern "C"{ 2 | /* Copyright 2012 Christoph Gärtner 3 | Distributed under the Boost Software License, Version 1.0 4 | */ 5 | 6 | #ifndef DECODE_HTML_ENTITIES_UTF8_ 7 | #define DECODE_HTML_ENTITIES_UTF8_ 8 | 9 | #include 10 | 11 | extern size_t decode_html_entities_utf8(char *dest, const char *src); 12 | /* Takes input from and decodes into , which should be a buffer 13 | large enough to hold characters. 14 | 15 | If is , input will be taken from , decoding 16 | the entities in-place. 17 | 18 | The function returns the length of the decoded string. 19 | */ 20 | 21 | #endif 22 | } 23 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/PDF/PDFConverter.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PDFCONVERTER_HPP 2 | #define PDFCONVERTER_HPP 3 | 4 | #include 5 | #include "../Database/Database.hpp" 6 | 7 | namespace PDF{ 8 | void doPDFConversionForPage(Database* db, std::string pageName, std::string tempFolder, std::string pdfCollectionFolder); 9 | void createZIPArchive(std::string pdfCollectionFolder, std::string tempFolder, std::string zipArchiveFile); 10 | void updatePDFs(); 11 | } 12 | 13 | #endif // PDFCONVERTER_HPP 14 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/DatabaseUtil.cpp: -------------------------------------------------------------------------------- 1 | #include "DatabaseUtil.hpp" 2 | 3 | namespace Parser{ 4 | ShownAuthor getShownAuthor(Database* db, std::optional authorId){ 5 | ShownAuthor out; 6 | if(authorId){ 7 | Database::Author author = db->getAuthor(authorId.value()); 8 | switch(author.type){ 9 | default: 10 | throw std::runtime_error("Got Author from Database with invalid Author::Type"); 11 | break; 12 | case Database::Author::Type::System: 13 | out.type = ShownAuthor::Type::System; 14 | break; 15 | case Database::Author::Type::User: 16 | out.type = ShownAuthor::Type::User; 17 | break; 18 | } 19 | if(author.name == ""){//if the author has a blank name, that means it should be treated as deleted 20 | out.type = ShownAuthor::Type::Deleted; 21 | } 22 | else{ 23 | out.shownName = author.name; 24 | out.linkName = normalizePageName(out.shownName); 25 | } 26 | } 27 | else{ 28 | out.type = ShownAuthor::Type::Deleted; 29 | } 30 | return out; 31 | } 32 | 33 | PageInfo getPageInfo(Database* db, Database::ID pageId){ 34 | PageInfo info; 35 | { 36 | Database::PageRevision latest = db->getLatestPageRevision(pageId); 37 | info.title = latest.title; 38 | } 39 | { 40 | Database::PageRevision oldest = db->getPageRevision(db->getPageRevisions(pageId).back()); 41 | if(oldest.authorId){ 42 | Database::Author author = db->getAuthor(*oldest.authorId); 43 | if(author.type == Database::Author::Type::User && author.name != ""){ 44 | info.author = author.name; 45 | } 46 | } 47 | info.creationTimeStamp = oldest.timeStamp; 48 | } 49 | info.name = db->getPageName(pageId); 50 | info.parent = db->getPageParent(pageId); 51 | info.tags = db->getPageTags(pageId); 52 | info.rating = db->getPageRating(pageId); 53 | info.votes = db->getPageVotesCount(pageId); 54 | return info; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/DatabaseUtil.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATABASEUTIL_HPP 2 | #define DATABASEUTIL_HPP 3 | 4 | #include "Treer.hpp" 5 | #include "../Database/Database.hpp" 6 | 7 | namespace Parser{ 8 | ShownAuthor getShownAuthor(Database* db, std::optional authorId); 9 | PageInfo getPageInfo(Database* db, Database::ID pageId); 10 | } 11 | 12 | #endif // DATABASEUTIL_HPP 13 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/HTMLConverter.hpp: -------------------------------------------------------------------------------- 1 | #ifndef HTMLCONVERTER_HPP 2 | #define HTMLCONVERTER_HPP 3 | 4 | #include "../HTTP/MarkupOutStream.hpp" 5 | #include "Treer.hpp" 6 | 7 | namespace Parser{ 8 | struct HtmlContext{ 9 | MarkupOutStream& out; 10 | mutable std::size_t uniqueId; 11 | bool linkImagesLocally = false; 12 | }; 13 | 14 | void convertNodeToHtml(const HtmlContext& con, const Node& nod); 15 | void delegateNodeBranches(const HtmlContext& con, const Node& nod); 16 | std::string getUniqueHtmlId(const HtmlContext& con); 17 | void convertPageTreeToHtml(MarkupOutStream& out, const PageTree& tree, bool linkImagesLocally = false); 18 | 19 | void convertTokenedPageToHtmlAnnotations(MarkupOutStream& out, const TokenedPage& page); 20 | void convertNodeToHtmlAnnotations(MarkupOutStream& out, const Node& nod); 21 | void convertPageTreeToHtmlAnnotations(MarkupOutStream& out, const PageTree& page); 22 | 23 | void toHtmlShownAuthor(MarkupOutStream& out, const ShownAuthor& author); 24 | std::string formatTimeStamp(TimeStamp timeStamp); 25 | std::string redirectLink(std::string rawLink); 26 | } 27 | 28 | #endif // HTMLCONVERTER_HPP 29 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Parser.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PARSER_HPP 2 | #define PARSER_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "../Config.hpp" 13 | 14 | class Database; 15 | 16 | namespace Parser{ 17 | std::string& trimLeft(std::string& s); 18 | std::string& trimRight(std::string& s); 19 | std::string& trimString(std::string& s); 20 | std::string normalizePageName(std::string link); 21 | 22 | enum class SectionType{Unknown, Module, FootNote, FootNoteBlock, TableOfContents, AdvTable, AdvTableRow, AdvTableElement, 23 | HTML, IFrame, Iftags, Include, Image, TabView, Tab, Collapsible, Span, Size, Anchor, Align, Div, Code, A, 24 | AdvList, AdvListElement, User, SCPConversionProjectInfoBox}; 25 | enum class ModuleType{Unknown, CSS, Rate, ForumStart, ForumCategory, ForumThread, ListPages, Redirect}; 26 | 27 | struct Section{ 28 | SectionType type = SectionType::Unknown; 29 | std::string typeString; 30 | ModuleType moduleType = ModuleType::Unknown; 31 | std::string mainParameter; 32 | std::map parameters; 33 | }; 34 | 35 | struct SectionStart : public Section{ 36 | 37 | }; 38 | 39 | struct SectionEnd{ 40 | using Type = SectionType; 41 | Type type = Type::Unknown; 42 | std::string typeString; 43 | }; 44 | 45 | struct SectionComplete : public SectionStart{ 46 | std::string contents; 47 | }; 48 | 49 | enum class DividerType{Unknown, Line, ClearBoth, ClearLeft, ClearRight, Separator}; 50 | 51 | struct Divider{ 52 | using Type = DividerType; 53 | Type type = Type::Unknown; 54 | }; 55 | 56 | struct Heading{ 57 | unsigned int degree; 58 | bool hidden; 59 | 60 | std::optional tocNumber;//only used when in a Node tree and not hidden 61 | }; 62 | 63 | struct QuoteBoxPrefix{ 64 | unsigned int degree; 65 | }; 66 | 67 | enum class ListPrefixType{Unknown, Bullet, Number}; 68 | struct ListPrefix{ 69 | using Type = ListPrefixType; 70 | Type type = Type::Unknown; 71 | unsigned int degree; 72 | }; 73 | 74 | enum class InlineFormatType{Unknown, Strike, Italics, Bold, Underline, Super, Sub, Monospace, Color}; 75 | 76 | struct InlineFormat{ 77 | using Type = InlineFormatType; 78 | Type type; 79 | bool begin; 80 | bool end; 81 | 82 | std::string color;//optional 83 | }; 84 | 85 | struct TableMarker{ 86 | enum class Type{Start, Middle, End, StartEnd}; 87 | Type type; 88 | enum class AlignmentType{Default, Header, Left, Right, Center}; 89 | AlignmentType alignment; 90 | unsigned int span; 91 | }; 92 | 93 | struct HyperLink{ 94 | std::string shownText; 95 | std::string url; 96 | bool newWindow; 97 | }; 98 | 99 | struct LiteralText{ 100 | std::string text; 101 | }; 102 | 103 | struct PlainText{ 104 | std::string text; 105 | }; 106 | 107 | struct CenterText{ 108 | }; 109 | 110 | struct LineBreak{ 111 | }; 112 | 113 | struct NewLine{ 114 | }; 115 | 116 | const inline std::vector TokenTypeNames = 117 | {"Unknown", "Section", "SectionStart", "SectionEnd", "SectionComplete", "Divider", "Heading", "CenterText", "QuoteBoxPrefix", 118 | "ListPrefix", "TableMarker", "InlineFormat", "HyperLink", "LiteralText", "PlainText", "LineBreak", "NewLine"}; 119 | 120 | enum class TokenType 121 | {Unknown = 0, Section, SectionStart, SectionEnd, SectionComplete, Divider, Heading, CenterText, QuoteBoxPrefix, 122 | ListPrefix, TableMarker, InlineFormat, HyperLink, LiteralText, PlainText, LineBreak, NewLine}; 123 | 124 | using TokenVariant = std::variant 125 | ; 127 | 128 | struct Token{ 129 | using Type = TokenType; 130 | using Variant = TokenVariant; 131 | Variant token; 132 | 133 | Type getType()const; 134 | 135 | std::size_t sourceStart; 136 | std::size_t sourceEnd; 137 | std::string source; 138 | 139 | bool operator==(const Token& tok)const; 140 | }; 141 | 142 | std::string getTokenTypeName(Token::Type type); 143 | nlohmann::json printTokenVariant(const Token& token); 144 | nlohmann::json printToken(const Token& token); 145 | std::ostream& operator<<(std::ostream& out, const Token& tok); 146 | 147 | struct TokenedPage{ 148 | std::vector tokens; 149 | std::string originalPage; 150 | }; 151 | 152 | struct PageInfo{ 153 | std::string name; 154 | std::string title; 155 | std::optional parent; 156 | std::optional author; 157 | std::vector tags; 158 | std::int64_t rating; 159 | std::int64_t votes; 160 | TimeStamp creationTimeStamp; 161 | }; 162 | 163 | struct ParserParameters{ 164 | PageInfo page; 165 | std::map includeParameters; 166 | std::map urlParameters; 167 | Database* database = nullptr; 168 | int includeDepth = 0; 169 | }; 170 | 171 | struct TokenRuleContext{ 172 | std::string page; 173 | std::size_t pagePos; 174 | std::vector tokens; 175 | bool wasNewLine; 176 | }; 177 | 178 | struct TokenRuleResult{ 179 | std::size_t newPos; 180 | bool nowNewline = false; 181 | std::vector newTokens; 182 | }; 183 | 184 | TokenedPage tokenizePage(std::string page, ParserParameters parameters = {}); 185 | std::vector getPageLinks(std::string page); 186 | } 187 | 188 | #endif // PARSER_HPP 189 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "RuleSet.hpp" 2 | 3 | #include "RuleSets/SCPConversionProjectInfoBoxRuleSet.hpp" 4 | 5 | #include "RuleSets/SpanRuleSet.hpp" 6 | #include "RuleSets/SizeRuleSet.hpp" 7 | #include "RuleSets/DivRuleSet.hpp" 8 | #include "RuleSets/AnchorRuleSet.hpp" 9 | #include "RuleSets/AlignRuleSet.hpp" 10 | #include "RuleSets/CSSRuleSet.hpp" 11 | #include "RuleSets/IncludeRuleSet.hpp" 12 | #include "RuleSets/CodeRuleSet.hpp" 13 | #include "RuleSets/CollapsibleRuleSet.hpp" 14 | #include "RuleSets/ImageRuleSet.hpp" 15 | #include "RuleSets/IftagsRuleSet.hpp" 16 | #include "RuleSets/IFrameRuleSet.hpp" 17 | #include "RuleSets/HTMLRuleSet.hpp" 18 | #include "RuleSets/TabViewRuleSet.hpp" 19 | #include "RuleSets/TableOfContentsRuleSet.hpp" 20 | #include "RuleSets/FootNoteRuleSet.hpp" 21 | #include "RuleSets/RateRuleSet.hpp" 22 | #include "RuleSets/UserRuleSet.hpp" 23 | #include "RuleSets/ARuleSet.hpp" 24 | #include "RuleSets/ForumRuleSet.hpp" 25 | #include "RuleSets/ListPagesRuleSet.hpp" 26 | #include "RuleSets/RedirectRuleSet.hpp" 27 | 28 | #include "RuleSets/SectionRuleSet.hpp" 29 | #include "RuleSets/BasicTextRuleSet.hpp" 30 | #include "RuleSets/LiteralTextRuleSet.hpp" 31 | #include "RuleSets/EntityEscapeRuleSet.hpp" 32 | #include "RuleSets/HyperLinkRuleSet.hpp" 33 | #include "RuleSets/InlineFormatRuleSet.hpp" 34 | #include "RuleSets/ListRuleSet.hpp" 35 | #include "RuleSets/QuoteBoxRuleSet.hpp" 36 | #include "RuleSets/DividerRuleSet.hpp" 37 | #include "RuleSets/CenterTextRuleSet.hpp" 38 | #include "RuleSets/TableRuleSet.hpp" 39 | #include "RuleSets/HeadingRuleSet.hpp" 40 | #include "RuleSets/CommentRuleSet.hpp" 41 | #include "RuleSets/NullRuleSet.hpp" 42 | 43 | namespace Parser{ 44 | const inline std::vector ruleSets = { 45 | scpConversionProjectInfoBoxRuleSet, 46 | 47 | spanRuleSet, 48 | sizeRuleSet, 49 | divRuleSet, 50 | anchorRuleSet, 51 | alignRuleSet, 52 | cssRuleSet, 53 | includeRuleSet, 54 | codeRuleSet, 55 | collapsibleRuleSet, 56 | imageRuleSet, 57 | iftagsRuleSet, 58 | iframeRuleSet, 59 | htmlRuleSet, 60 | tabViewRuleSet, 61 | tableOfContentsRuleSet, 62 | footNoteRuleSet, 63 | rateRuleSet, 64 | userRuleSet, 65 | aRuleSet, 66 | forumRuleSet, 67 | listPagesRuleSet, 68 | redirectRuleSet, 69 | 70 | commentRuleSet, 71 | headingRuleSet, 72 | centerTextRuleSet, 73 | dividerRuleSet, 74 | quoteBoxRuleSet, 75 | listRuleSet, 76 | tableRuleSet, 77 | inlineFormatRuleSet, 78 | tripleHyperLinkRuleSet, 79 | sectionRuleSet, 80 | hyperLinkRuleSet, 81 | entityEscapeRuleSet, 82 | literalTextRuleSet, 83 | basicTextRuleSet 84 | //, nullRuleSet 85 | }; 86 | 87 | namespace{ 88 | inline RuleType getType(const RuleVariant& var){ 89 | return static_cast(var.index()); 90 | } 91 | 92 | template 93 | inline std::vector getRules(){ 94 | std::vector output; 95 | for(const RuleSet& ruleSet : ruleSets){ 96 | for(const RuleVariant& ruleVariant : ruleSet.rules){ 97 | if(getType(ruleVariant) == type){ 98 | Type rule = std::get(ruleVariant); 99 | rule.parentRuleSet = ruleSet.name; 100 | output.push_back(std::move(rule)); 101 | } 102 | } 103 | } 104 | return output; 105 | } 106 | } 107 | 108 | std::vector getTokenPrintRules(){ 109 | return getRules(); 110 | } 111 | 112 | std::vector getNodePrintRules(){ 113 | return getRules(); 114 | } 115 | 116 | std::vector getTokenRules(){ 117 | return getRules(); 118 | } 119 | 120 | std::vector getSectionRules(){ 121 | return getRules(); 122 | } 123 | 124 | std::vector getTreeRules(){ 125 | return getRules(); 126 | } 127 | 128 | std::vector getPostTreeRules(){ 129 | return getRules(); 130 | } 131 | 132 | std::vector getHtmlRules(){ 133 | return getRules(); 134 | } 135 | 136 | void printFullRuleSetList(){ 137 | for(const RuleSet& ruleSet : ruleSets){ 138 | std::cout << ruleSet.name << ":\n"; 139 | for(const RuleVariant& rule : ruleSet.rules){ 140 | std::cout << "\t"; 141 | switch(getType(rule)){ 142 | case RuleType::HtmlRule: 143 | std::cout << "HtmlRule"; 144 | break; 145 | case RuleType::NodePrintRule: 146 | std::cout << "NodePrintRule"; 147 | break; 148 | case RuleType::SectionRule: 149 | std::cout << "SectionRule"; 150 | break; 151 | case RuleType::TokenPrintRule: 152 | std::cout << "TokenPrintRule"; 153 | break; 154 | case RuleType::TokenRule: 155 | std::cout << "TokenRule"; 156 | break; 157 | case RuleType::TreeRule: 158 | std::cout << "TreeRule"; 159 | break; 160 | case RuleType::PostTreeRule: 161 | std::cout << "PostTreeRule"; 162 | break; 163 | } 164 | std::cout << "\n"; 165 | } 166 | } 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef RULESET_HPP 2 | #define RULESET_HPP 3 | 4 | #include 5 | 6 | #include "../Parser.hpp" 7 | #include "../Treer.hpp" 8 | #include "../HTMLConverter.hpp" 9 | 10 | #include "RuleSetUtil.hpp" 11 | 12 | namespace Parser{ 13 | struct TokenPrintRule{ 14 | Token::Type type; 15 | std::function print; 16 | 17 | std::string parentRuleSet; 18 | }; 19 | 20 | struct NodePrintRule{ 21 | Node::Type type; 22 | std::function print; 23 | 24 | std::string parentRuleSet; 25 | }; 26 | 27 | struct TokenRule{ 28 | std::function tryRule; 29 | std::function doRule; 30 | 31 | std::string parentRuleSet; 32 | }; 33 | 34 | enum class SubnameType{None, Parameter, Module}; 35 | enum class ContentType{None, Surround, Contain}; 36 | enum class ParameterType{None, Quoted, Lined}; 37 | 38 | struct SectionRule{ 39 | SectionType type; 40 | std::vector matchingNames; 41 | SubnameType subnameType; 42 | ModuleType moduleType; 43 | std::vector matchingModules; 44 | ContentType contentType; 45 | ParameterType parameterType; 46 | bool allowInline; 47 | 48 | std::string parentRuleSet; 49 | }; 50 | 51 | struct TreeRule{ 52 | struct TreeRuleType{ 53 | Token::Type mainType; 54 | SectionType sectionType = SectionType::Unknown; 55 | ModuleType moduleType = ModuleType::Unknown; 56 | }; 57 | TreeRuleType type; 58 | std::function handleRule; 59 | 60 | std::string parentRuleSet; 61 | }; 62 | 63 | struct PostTreeRule{ 64 | std::function rule; 65 | 66 | std::string parentRuleSet; 67 | }; 68 | 69 | struct HtmlRule{ 70 | Node::Type type; 71 | std::function handleRule; 72 | 73 | std::string parentRuleSet; 74 | }; 75 | 76 | using RuleVariant = std::variant; 77 | enum class RuleType{TokenPrintRule = 0, NodePrintRule, TokenRule, SectionRule, TreeRule, PostTreeRule, HtmlRule}; 78 | 79 | struct RuleSet{ 80 | std::string name; 81 | std::vector rules; 82 | }; 83 | 84 | std::vector getTokenPrintRules(); 85 | std::vector getNodePrintRules(); 86 | std::vector getTokenRules(); 87 | std::vector getSectionRules(); 88 | std::vector getTreeRules(); 89 | std::vector getPostTreeRules(); 90 | std::vector getHtmlRules(); 91 | 92 | void printFullRuleSetList(); 93 | } 94 | 95 | #endif // RULESET_HPP 96 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSetUtil.hpp: -------------------------------------------------------------------------------- 1 | #ifndef RULEFUNCTIONS_HPP 2 | #define RULEFUNCTIONS_HPP 3 | 4 | #include 5 | #include 6 | 7 | #include "RuleSet.hpp" 8 | 9 | namespace Parser{ 10 | inline static bool check(const std::string& buffer, std::size_t pos, std::string text){ 11 | if(pos + text.size() > buffer.size()){ 12 | return false; 13 | } 14 | std::string temp = buffer.substr(pos, text.size()); 15 | return text == temp; 16 | } 17 | 18 | inline static bool checkWithoutCase(const std::string& buffer, std::size_t pos, std::string text){ 19 | if(pos + text.size() > buffer.size()){ 20 | return false; 21 | } 22 | std::string temp = buffer.substr(pos, text.size()); 23 | std::transform(temp.begin(), temp.end(), temp.begin(), ::tolower); 24 | std::transform(text.begin(), text.end(), text.begin(), ::tolower); 25 | return text == temp; 26 | } 27 | 28 | inline static bool checkLine(const std::string& buffer, std::size_t pos, std::string text){ 29 | while(pos < buffer.size()){ 30 | if(check(buffer, pos, text)){ 31 | return true; 32 | } 33 | if(check(buffer, pos, "\n")){ 34 | return false; 35 | } 36 | pos++; 37 | } 38 | return false; 39 | } 40 | 41 | inline static bool checkParagraph(const std::string& buffer, std::size_t pos, std::string text){ 42 | while(pos < buffer.size()){ 43 | if(check(buffer, pos, text)){ 44 | return true; 45 | } 46 | if(check(buffer, pos, "\n\n")){ 47 | return false; 48 | } 49 | pos++; 50 | } 51 | return false; 52 | } 53 | 54 | inline static bool checkParagraphBack(const std::string& buffer, std::size_t pos, std::string text){ 55 | while(true){ 56 | if(check(buffer, pos, text)){ 57 | return true; 58 | } 59 | if(check(buffer, pos, "\n\n")){ 60 | return false; 61 | } 62 | if(pos == 0){ 63 | break; 64 | } 65 | pos--; 66 | } 67 | return false; 68 | } 69 | 70 | inline static void handleSectionStartEnd(const Token& token, std::function startFunc, std::function endFunc){ 71 | if(token.getType() == Token::Type::SectionStart){ 72 | startFunc(std::get(token.token)); 73 | } 74 | else{ 75 | endFunc(std::get(token.token)); 76 | } 77 | } 78 | 79 | inline static void travelPageTreeNodes(Node& root, std::function func){ 80 | if(func(root)){ 81 | for(Node& branch : root.branches){ 82 | travelPageTreeNodes(branch, func); 83 | } 84 | } 85 | } 86 | } 87 | 88 | #endif // RULEFUNCTIONS_HPP 89 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/ARuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "ARuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeA(const NodeVariant& nod){ 5 | const A& a = std::get(nod); 6 | nlohmann::json out = nlohmann::json::object(); 7 | for(auto i = a.parameters.begin(); i != a.parameters.end(); i++){ 8 | out[i->first] = i->second; 9 | } 10 | return out; 11 | } 12 | 13 | void handleA(TreeContext& context, const Token& token){ 14 | handleSectionStartEnd(token, 15 | [&](const SectionStart& section){ 16 | makeTextAddable(context); 17 | pushStack(context, Node{A{section.parameters}}); 18 | }, [&](const SectionEnd& section){ 19 | if(isInside(context, Node::Type::A)){ 20 | popSingle(context, Node::Type::A); 21 | } 22 | }); 23 | } 24 | 25 | void toHtmlNodeA(const HtmlContext& con, const Node& nod){ 26 | const A& node = std::get(nod.node); 27 | con.out << "first == "id" && check(i->second, 0, "u-") == false){ 30 | con.out << " "_AM << i->first << "='u-"_AM << i->second << "'"_AM; 31 | } 32 | else{ 33 | con.out << " "_AM << i->first << "='"_AM << i->second << "'"_AM; 34 | } 35 | } 36 | con.out << ">"_AM; 37 | delegateNodeBranches(con, nod); 38 | con.out << ""_AM; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/ARuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ARULESET_HPP 2 | #define ARULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeA(const NodeVariant& nod); 8 | 9 | void handleA(TreeContext& context, const Token& token); 10 | 11 | void toHtmlNodeA(const HtmlContext& con, const Node& nod); 12 | 13 | const inline auto aRuleSet = RuleSet{"A", { 14 | NodePrintRule{Node::Type::A, printNodeA}, 15 | 16 | SectionRule{SectionType::A, {"a", "a_"}, SubnameType::None, ModuleType::Unknown, {}, 17 | ContentType::Surround, ParameterType::Quoted, true}, 18 | 19 | TreeRule{{Token::Type::Section, SectionType::A}, handleA}, 20 | 21 | HtmlRule{Node::Type::A, toHtmlNodeA} 22 | }}; 23 | 24 | } 25 | 26 | #endif // ARULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/AlignRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "AlignRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeAlign(const NodeVariant& nod){ 5 | const Align& align = std::get(nod); 6 | switch(align.type){ 7 | default: 8 | return "Unknown"; 9 | case Align::Type::Center: 10 | return "Center"; 11 | case Align::Type::Left: 12 | return "Left"; 13 | case Align::Type::Right: 14 | return "Right"; 15 | case Align::Type::Justify: 16 | return "Justify"; 17 | } 18 | } 19 | 20 | void handleAlign(TreeContext& context, const Token& token){ 21 | handleSectionStartEnd(token, 22 | [&](const SectionStart& section){ 23 | Align::Type type; 24 | if(section.typeString == "="){ 25 | type = Align::Type::Center; 26 | } 27 | else if(section.typeString == "<"){ 28 | type = Align::Type::Left; 29 | } 30 | else if(section.typeString == ">"){ 31 | type = Align::Type::Right; 32 | } 33 | else if(section.typeString == "=="){ 34 | type = Align::Type::Justify; 35 | } 36 | makeDivPushable(context); 37 | pushStack(context, Node{Align{type}}); 38 | }, [&](const SectionEnd& section){ 39 | if(isInside(context, Node::Type::Align)){//not actually correct, this should keep track of what kind of align it is talking about 40 | //but who's nesting different kind of aligns anyways? this work just fine, I'm sure... 41 | popSingle(context, Node::Type::Align); 42 | } 43 | }); 44 | } 45 | 46 | void toHtmlNodeAlign(const HtmlContext& con, const Node& nod){ 47 | const Align& node = std::get(nod.node); 48 | std::string textAlign; 49 | switch(node.type){ 50 | case Align::Type::Center: 51 | textAlign = "center"; 52 | break; 53 | case Align::Type::Left: 54 | textAlign = "left"; 55 | break; 56 | case Align::Type::Right: 57 | textAlign = "right"; 58 | break; 59 | case Align::Type::Justify: 60 | textAlign = "justify"; 61 | break; 62 | default: 63 | throw std::runtime_error("Encountered invalid Align::Type"); 64 | } 65 | con.out << "
"_AM; 66 | delegateNodeBranches(con, nod); 67 | con.out << "
"_AM; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/AlignRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ALIGNRULESET_HPP 2 | #define ALIGNRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | nlohmann::json printNodeAlign(const NodeVariant& nod); 9 | 10 | void handleAlign(TreeContext& context, const Token& token); 11 | 12 | void toHtmlNodeAlign(const HtmlContext& con, const Node& nod); 13 | 14 | const inline auto alignRuleSet = RuleSet{"Align", { 15 | NodePrintRule{Node::Type::Align, printNodeAlign}, 16 | 17 | SectionRule{SectionType::Align, {"<", ">", "=", "=="}, SubnameType::None, ModuleType::Unknown, {}, 18 | ContentType::Surround, ParameterType::None, false}, 19 | 20 | TreeRule{{Token::Type::Section, SectionType::Align}, handleAlign}, 21 | 22 | HtmlRule{Node::Type::Align, toHtmlNodeAlign} 23 | }}; 24 | 25 | } 26 | 27 | #endif // ALIGNRULESET_HPP 28 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/AnchorRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "AnchorRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeAnchor(const NodeVariant& nod){ 5 | const Anchor& anchor = std::get(nod); 6 | return anchor.name; 7 | } 8 | 9 | void handleAnchor(TreeContext& context, const Token& token){ 10 | const Section& section = std::get
(token.token); 11 | addAsText(context, Node{Anchor{section.mainParameter}}); 12 | } 13 | 14 | void toHtmlNodeAnchor(const HtmlContext& con, const Node& nod){ 15 | const Anchor& node = std::get(nod.node); 16 | con.out << ""_AM; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/AnchorRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ANCHORRULESET_HPP 2 | #define ANCHORRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | nlohmann::json printNodeAnchor(const NodeVariant& nod); 9 | 10 | void handleAnchor(TreeContext& context, const Token& token); 11 | 12 | void toHtmlNodeAnchor(const HtmlContext& con, const Node& nod); 13 | 14 | const inline auto anchorRuleSet = RuleSet{"Anchor", { 15 | NodePrintRule{Node::Type::Anchor, printNodeAnchor}, 16 | 17 | SectionRule{SectionType::Anchor, {"#"}, SubnameType::Parameter, ModuleType::Unknown, {}, 18 | ContentType::None, ParameterType::None, true}, 19 | 20 | TreeRule{{Token::Type::Section, SectionType::Anchor}, handleAnchor}, 21 | 22 | HtmlRule{Node::Type::Anchor, toHtmlNodeAnchor} 23 | }}; 24 | 25 | } 26 | 27 | #endif // ANCHORRULESET_HPP 28 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/BasicTextRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef BASICTEXTRULESET_HPP 2 | #define BASICTEXTRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenPlainText(const TokenVariant& tok); 8 | nlohmann::json printTokenNewLine(const TokenVariant& tok); 9 | nlohmann::json printTokenLineBreak(const TokenVariant& tok); 10 | 11 | nlohmann::json printNodePlainText(const NodeVariant& nod); 12 | nlohmann::json printNodeParagraph(const NodeVariant& nod); 13 | nlohmann::json printNodeLineBreak(const NodeVariant& nod); 14 | nlohmann::json printNodeRootPage(const NodeVariant& nod); 15 | 16 | bool tryCarriageReturn(const TokenRuleContext& context); 17 | TokenRuleResult doCarriageReturn(const TokenRuleContext& context); 18 | 19 | bool tryLineBreakRule(const TokenRuleContext& context); 20 | TokenRuleResult doLineBreakRule(const TokenRuleContext& context); 21 | 22 | bool tryEscapedNewLineRule(const TokenRuleContext& context); 23 | TokenRuleResult doEscapedNewLineRule(const TokenRuleContext& context); 24 | 25 | bool tryNewLineRule(const TokenRuleContext& context); 26 | TokenRuleResult doNewLineRule(const TokenRuleContext& context); 27 | 28 | bool tryTypographyRule(const TokenRuleContext& context); 29 | TokenRuleResult doTypographyRule(const TokenRuleContext& context); 30 | 31 | bool tryPrelineSpaceRule(const TokenRuleContext& context); 32 | TokenRuleResult doPrelineSpaceRule(const TokenRuleContext& context); 33 | 34 | bool tryPlainTextRule(const TokenRuleContext& context); 35 | TokenRuleResult doPlainTextRule(const TokenRuleContext& context); 36 | 37 | void handleLineBreak(TreeContext& context, const Token& token); 38 | void handlePlainText(TreeContext& context, const Token& token); 39 | 40 | void toHtmlNodeRootPage(const HtmlContext& con, const Node& nod); 41 | void toHtmlNodeParagraph(const HtmlContext& con, const Node& nod); 42 | void toHtmlNodeLineBreak(const HtmlContext& con, const Node& nod); 43 | void toHtmlNodePlainText(const HtmlContext& con, const Node& nod); 44 | 45 | const inline auto basicTextRuleSet = RuleSet{"BasicText", { 46 | TokenPrintRule{Token::Type::PlainText, printTokenPlainText}, 47 | TokenPrintRule{Token::Type::NewLine, printTokenNewLine}, 48 | TokenPrintRule{Token::Type::LineBreak, printTokenLineBreak}, 49 | 50 | NodePrintRule{Node::Type::PlainText, printNodePlainText}, 51 | NodePrintRule{Node::Type::Paragraph, printNodeParagraph}, 52 | NodePrintRule{Node::Type::LineBreak, printNodeLineBreak}, 53 | NodePrintRule{Node::Type::RootPage, printNodeRootPage}, 54 | 55 | TokenRule{tryCarriageReturn, doCarriageReturn}, 56 | TokenRule{tryLineBreakRule, doLineBreakRule}, 57 | TokenRule{tryEscapedNewLineRule, doEscapedNewLineRule}, 58 | TokenRule{tryNewLineRule, doNewLineRule}, 59 | TokenRule{tryTypographyRule, doTypographyRule}, 60 | TokenRule{tryPrelineSpaceRule, doPrelineSpaceRule}, 61 | TokenRule{tryPlainTextRule, doPlainTextRule}, 62 | 63 | TreeRule{{Token::Type::PlainText}, handlePlainText}, 64 | TreeRule{{Token::Type::LineBreak}, handleLineBreak}, 65 | 66 | HtmlRule{Node::Type::RootPage, toHtmlNodeRootPage}, 67 | HtmlRule{Node::Type::Paragraph, toHtmlNodeParagraph}, 68 | HtmlRule{Node::Type::LineBreak, toHtmlNodeLineBreak}, 69 | HtmlRule{Node::Type::PlainText, toHtmlNodePlainText} 70 | }}; 71 | } 72 | 73 | #endif // BASICTEXTRULESET_HPP 74 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CSSRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "CSSRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | void handleCSS(TreeContext& context, const Token& token){ 5 | const SectionComplete& sectionComplete = std::get(token.token); 6 | //just add the css to the cssData, don't even touch the tree 7 | context.cssData.push_back(CSS{sectionComplete.contents}); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CSSRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CSSRULESET_HPP 2 | #define CSSRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | void handleCSS(TreeContext& context, const Token& token); 9 | 10 | const inline auto cssRuleSet = RuleSet{"CSS", { 11 | SectionRule{SectionType::Module, {"module"}, SubnameType::Module, ModuleType::CSS, {"css"}, 12 | ContentType::Contain, ParameterType::None, false}, 13 | 14 | TreeRule{{Token::Type::Section, SectionType::Module, ModuleType::CSS}, handleCSS} 15 | }}; 16 | 17 | } 18 | 19 | #endif // CSSRULESET_HPP 20 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CenterTextRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "CenterTextRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printTokenCenterText(const TokenVariant& tok){ 5 | return {}; 6 | } 7 | 8 | nlohmann::json printNodeCenterText(const NodeVariant& nod){ 9 | return {}; 10 | } 11 | 12 | bool tryCenterTextRule(const TokenRuleContext& context){ 13 | return context.wasNewLine && check(context.page, context.pagePos, "= "); 14 | } 15 | 16 | TokenRuleResult doCenterTextRule(const TokenRuleContext& context){ 17 | TokenRuleResult result; 18 | result.newPos = context.pagePos + 2; 19 | result.newTokens.push_back(Token{CenterText{}, context.pagePos, result.newPos, context.page.substr(context.pagePos, result.newPos - context.pagePos)}); 20 | return result; 21 | } 22 | 23 | void handleCenterText(TreeContext& context, const Token& token){ 24 | makeDivPushable(context); 25 | pushStack(context, Node{CenterText{}}); 26 | } 27 | 28 | void toHtmlNodeCenterText(const HtmlContext& con, const Node& nod){ 29 | con.out << "

"_AM; 30 | delegateNodeBranches(con, nod); 31 | con.out << "

"_AM; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CenterTextRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CENTERTEXTRULESET_HPP 2 | #define CENTERTEXTRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenCenterText(const TokenVariant& tok); 8 | nlohmann::json printNodeCenterText(const NodeVariant& nod); 9 | 10 | bool tryCenterTextRule(const TokenRuleContext& context); 11 | TokenRuleResult doCenterTextRule(const TokenRuleContext& context); 12 | 13 | void handleCenterText(TreeContext& context, const Token& token); 14 | 15 | void toHtmlNodeCenterText(const HtmlContext& con, const Node& nod); 16 | 17 | const inline auto centerTextRuleSet = RuleSet{"CenterText", { 18 | TokenPrintRule{Token::Type::CenterText, printTokenCenterText}, 19 | NodePrintRule{Node::Type::CenterText, printNodeCenterText}, 20 | 21 | TokenRule{tryCenterTextRule, doCenterTextRule}, 22 | 23 | TreeRule{{Token::Type::CenterText}, handleCenterText}, 24 | 25 | HtmlRule{Node::Type::CenterText, toHtmlNodeCenterText} 26 | }}; 27 | 28 | } 29 | 30 | #endif // CENTERTEXTRULESET_HPP 31 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CodeRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "CodeRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeCode(const NodeVariant& nod){ 5 | return std::get(nod).contents; 6 | } 7 | 8 | void handleCode(TreeContext& context, const Token& token){ 9 | const SectionComplete& section = std::get(token.token); 10 | 11 | Code code; 12 | if(section.parameters.find("type") != section.parameters.end()){ 13 | code.type = section.parameters.find("type")->second; 14 | std::transform(code.type.begin(), code.type.end(), code.type.begin(), ::tolower); 15 | } 16 | code.contents = section.contents; 17 | addAsDiv(context, Node{code}); 18 | context.codeData.push_back(code); 19 | } 20 | 21 | void toHtmlNodeCode(const HtmlContext& con, const Node& nod){ 22 | const Code& code = std::get(nod.node); 23 | 24 | con.out << "
"_AM; 25 | for(const char c : code.contents){ 26 | switch(c){ 27 | default: 28 | con.out << c; 29 | break; 30 | case '\n': 31 | con.out << "
"_AM; 32 | break; 33 | case ' ': 34 | con.out << " "_AM; 35 | break; 36 | } 37 | } 38 | con.out << "
"_AM; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CodeRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef CODERULESET_HPP 2 | #define CODERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | nlohmann::json printNodeCode(const NodeVariant& nod); 9 | 10 | void handleCode(TreeContext& context, const Token& token); 11 | 12 | void toHtmlNodeCode(const HtmlContext& con, const Node& nod); 13 | 14 | const inline auto codeRuleSet = RuleSet{"Code", { 15 | NodePrintRule{Node::Type::Code, printNodeCode}, 16 | 17 | SectionRule{SectionType::Code, {"code"}, SubnameType::None, ModuleType::Unknown, {}, 18 | ContentType::Contain, ParameterType::Quoted, false}, 19 | 20 | TreeRule{{Token::Type::Section, SectionType::Code}, handleCode}, 21 | 22 | HtmlRule{Node::Type::Code, toHtmlNodeCode} 23 | }}; 24 | 25 | } 26 | 27 | #endif // CODERULESET_HPP 28 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CollapsibleRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "CollapsibleRuleSet.hpp" 2 | 3 | #include 4 | 5 | namespace Parser{ 6 | nlohmann::json printNodeCollapsible(const NodeVariant& nod){ 7 | const Collapsible& collapsible = std::get(nod); 8 | nlohmann::json out; 9 | out["openedText"] = collapsible.openedText; 10 | out["closedText"] = collapsible.closedText; 11 | out["defaultShow"] = collapsible.defaultShow; 12 | return out; 13 | } 14 | 15 | void handleCollapsible(TreeContext& context, const Token& token){ 16 | handleSectionStartEnd(token, 17 | [&](const SectionStart& section){ 18 | Collapsible collapsible; 19 | if(section.parameters.find("show") != section.parameters.end()){ 20 | collapsible.closedText = section.parameters.find("show")->second; 21 | } 22 | else{ 23 | collapsible.closedText = "+ show block"; 24 | } 25 | 26 | if(section.parameters.find("hide") != section.parameters.end()){ 27 | collapsible.openedText = section.parameters.find("hide")->second; 28 | } 29 | else{ 30 | collapsible.openedText = "- hide block"; 31 | } 32 | 33 | if(section.parameters.find("folded") != section.parameters.end()){ 34 | if(section.parameters.find("folded")->second == "no"){ 35 | collapsible.defaultShow = true; 36 | } 37 | else{ 38 | collapsible.defaultShow = false; 39 | } 40 | } 41 | else{ 42 | collapsible.defaultShow = false; 43 | } 44 | 45 | makeDivPushable(context); 46 | pushStack(context, Node{collapsible}); 47 | }, [&](const SectionEnd& section){ 48 | if(isInside(context, Node::Type::Collapsible)){ 49 | popSingle(context, Node::Type::Collapsible); 50 | } 51 | }); 52 | } 53 | 54 | void toHtmlNodeCollapsible(const HtmlContext& con, const Node& nod){ 55 | const Collapsible& collapsible = std::get(nod.node); 56 | 57 | std::string uniqueId = getUniqueHtmlId(con); 58 | 59 | con.out << "
"_AM 60 | << ""_AM 61 | << ""_AM 64 | << ""_AM 67 | << "
"_AM; 68 | 69 | delegateNodeBranches(con, nod); 70 | 71 | con.out << "
"_AM; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CollapsibleRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef COLLAPSIBLERULESET_HPP 2 | #define COLLAPSIBLERULESET_HPP 3 | 4 | 5 | #include "../RuleSet.hpp" 6 | 7 | namespace Parser{ 8 | nlohmann::json printNodeCollapsible(const NodeVariant& nod); 9 | 10 | void handleCollapsible(TreeContext& context, const Token& token); 11 | 12 | void toHtmlNodeCollapsible(const HtmlContext& con, const Node& nod); 13 | 14 | const inline auto collapsibleRuleSet = RuleSet{"Collapsible", { 15 | NodePrintRule{Node::Type::Collapsible, printNodeCollapsible}, 16 | 17 | SectionRule{SectionType::Collapsible, {"collapsible"}, SubnameType::None, ModuleType::Unknown, {}, 18 | ContentType::Surround, ParameterType::Quoted, false}, 19 | 20 | TreeRule{{Token::Type::Section, SectionType::Collapsible}, handleCollapsible}, 21 | 22 | HtmlRule{Node::Type::Collapsible, toHtmlNodeCollapsible} 23 | }}; 24 | 25 | } 26 | 27 | #endif // COLLAPSIBLERULESET_HPP 28 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CommentRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "CommentRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | bool tryCommentRule(const TokenRuleContext& context){ 5 | if(check(context.page, context.pagePos, "[!--")){ 6 | return true; 7 | } 8 | return false; 9 | } 10 | 11 | TokenRuleResult doCommentRule(const TokenRuleContext& context){ 12 | std::size_t pos = context.pagePos; 13 | bool literal = false; 14 | while(pos < context.page.size()){ 15 | if(!literal && check(context.page, pos, "--]")){ 16 | pos += 3; 17 | break; 18 | } 19 | if(!literal && check(context.page, pos, "@@") && checkParagraph(context.page, pos + 2, "@@")){ 20 | literal = true; 21 | pos += 2; 22 | } 23 | if(literal && check(context.page, pos, "@@")){ 24 | literal = false; 25 | pos += 2; 26 | } 27 | pos++; 28 | } 29 | 30 | TokenRuleResult result; 31 | result.newPos = pos; 32 | result.nowNewline = context.wasNewLine;//preserve newline status 33 | return result; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/CommentRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef COMMENTRULESET_HPP 2 | #define COMMENTRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | bool tryCommentRule(const TokenRuleContext& context); 8 | TokenRuleResult doCommentRule(const TokenRuleContext& context); 9 | 10 | const inline auto commentRuleSet = RuleSet{"Comment", { 11 | TokenRule{tryCommentRule, doCommentRule} 12 | }}; 13 | } 14 | 15 | #endif // COMMENTRULESET_HPP 16 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/DivRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "DivRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeDiv(const NodeVariant& nod){ 5 | const Div& div = std::get
(nod); 6 | nlohmann::json out = nlohmann::json::object(); 7 | for(auto i = div.parameters.begin(); i != div.parameters.end(); i++){ 8 | out[i->first] = i->second; 9 | } 10 | return out; 11 | } 12 | 13 | void handleDiv(TreeContext& context, const Token& token){ 14 | handleSectionStartEnd(token, 15 | [&](const SectionStart& section){ 16 | makeDivPushable(context); 17 | pushStack(context, Node{Div{section.parameters}}); 18 | }, [&](const SectionEnd& section){ 19 | if(isInside(context, Node::Type::Div)){ 20 | popSingle(context, Node::Type::Div); 21 | } 22 | }); 23 | } 24 | 25 | void toHtmlNodeDiv(const HtmlContext& con, const Node& nod){ 26 | const Div& node = std::get
(nod.node); 27 | con.out << "first == "id" && check(i->second, 0, "u-") == false){ 30 | con.out << " "_AM << i->first << "='u-"_AM << i->second << "'"_AM; 31 | } 32 | else{ 33 | con.out << " "_AM << i->first << "='"_AM << i->second << "'"_AM; 34 | } 35 | } 36 | con.out << ">"_AM; 37 | delegateNodeBranches(con, nod); 38 | con.out << "
"_AM; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/DivRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DIVRULESET_HPP 2 | #define DIVRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeDiv(const NodeVariant& nod); 8 | 9 | void handleDiv(TreeContext& context, const Token& token); 10 | 11 | void toHtmlNodeDiv(const HtmlContext& con, const Node& nod); 12 | 13 | const inline auto divRuleSet = RuleSet{"Div", { 14 | NodePrintRule{Node::Type::Div, printNodeDiv}, 15 | 16 | SectionRule{SectionType::Div, {"div", "div_"}, SubnameType::None, ModuleType::Unknown, {}, 17 | ContentType::Surround, ParameterType::Quoted, false}, 18 | 19 | TreeRule{{Token::Type::Section, SectionType::Div}, handleDiv}, 20 | 21 | HtmlRule{Node::Type::Div, toHtmlNodeDiv} 22 | }}; 23 | 24 | } 25 | 26 | #endif // DIVRULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/DividerRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "DividerRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printTokenDivider(const TokenVariant& tok){ 5 | const Divider& divider = std::get(tok); 6 | switch(divider.type){ 7 | default: 8 | return "Unknown"; 9 | case Divider::Type::Line: 10 | return "Line"; 11 | case Divider::Type::ClearBoth: 12 | return "ClearBoth"; 13 | case Divider::Type::ClearLeft: 14 | return "ClearLeft"; 15 | case Divider::Type::ClearRight: 16 | return "ClearRight"; 17 | case Divider::Type::Separator: 18 | return "Separator"; 19 | } 20 | } 21 | 22 | nlohmann::json printNodeDivider(const NodeVariant& nod){ 23 | return printTokenDivider(std::get(nod)); 24 | } 25 | 26 | bool tryDividerRule(const TokenRuleContext& context){ 27 | if(context.wasNewLine){ 28 | const auto checkFunction = [&context](char c)->bool{ 29 | std::size_t pos = context.pagePos; 30 | while(pos < context.page.size()){ 31 | if(context.page[pos] == '\n'){ 32 | break; 33 | } 34 | else if(c == '~' && (context.page[pos] == '<' || context.page[pos] == '>') && (pos + 1 == context.page.size() || context.page[pos + 1] == '\n')){ 35 | break; 36 | } 37 | else if(context.page[pos] != c){ 38 | return false; 39 | } 40 | pos++; 41 | } 42 | return true; 43 | }; 44 | 45 | if(check(context.page, context.pagePos, "----")){ 46 | return checkFunction('-'); 47 | } 48 | else if(check(context.page, context.pagePos, "~~~~")){ 49 | return checkFunction('~'); 50 | } 51 | else if(check(context.page, context.pagePos, "====")){ 52 | return checkFunction('='); 53 | } 54 | } 55 | return false; 56 | } 57 | 58 | TokenRuleResult doDividerRule(const TokenRuleContext& context){ 59 | Divider divider; 60 | 61 | if(context.page[context.pagePos] == '-'){ 62 | divider.type = Divider::Type::Line; 63 | } 64 | else if(context.page[context.pagePos] == '~'){ 65 | divider.type = Divider::Type::ClearBoth; 66 | } 67 | else if(context.page[context.pagePos] == '='){ 68 | divider.type = Divider::Type::Separator; 69 | } 70 | 71 | std::size_t pos = context.pagePos; 72 | while(pos < context.page.size()){ 73 | if(context.page[pos] == '\n'){ 74 | break; 75 | } 76 | else if(context.page[pos] == '<'){ 77 | divider.type = Divider::Type::ClearLeft; 78 | } 79 | else if(context.page[pos] == '>'){ 80 | divider.type = Divider::Type::ClearRight; 81 | } 82 | pos++; 83 | } 84 | 85 | TokenRuleResult result; 86 | result.newPos = pos; 87 | result.newTokens.push_back(Token{divider, context.pagePos, pos, context.page.substr(context.pagePos, pos - context.pagePos)}); 88 | return result; 89 | } 90 | 91 | void handleDivider(TreeContext& context, const Token& token){ 92 | addAsDiv(context, Node{std::get(token.token)}); 93 | } 94 | 95 | void toHtmlNodeDivider(const HtmlContext& con, const Node& nod){ 96 | const Divider& node = std::get(nod.node); 97 | switch(node.type){ 98 | case Divider::Type::Line: 99 | con.out << "
"_AM; 100 | break; 101 | case Divider::Type::ClearBoth: 102 | con.out << "
"_AM; 103 | break; 104 | case Divider::Type::ClearLeft: 105 | con.out << "
"_AM; 106 | break; 107 | case Divider::Type::ClearRight: 108 | con.out << "
"_AM; 109 | break; 110 | case Divider::Type::Separator: 111 | //has no effect on html 112 | break; 113 | default: 114 | throw std::runtime_error("Invalid Divider type"); 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/DividerRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DIVIDERRULESET_HPP 2 | #define DIVIDERRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenDivider(const TokenVariant& tok); 8 | nlohmann::json printNodeDivider(const NodeVariant& nod); 9 | 10 | bool tryDividerRule(const TokenRuleContext& context); 11 | TokenRuleResult doDividerRule(const TokenRuleContext& context); 12 | 13 | void handleDivider(TreeContext& context, const Token& token); 14 | 15 | void toHtmlNodeDivider(const HtmlContext& con, const Node& nod); 16 | 17 | const inline auto dividerRuleSet = RuleSet{"Divider", { 18 | TokenPrintRule{Token::Type::Divider, printTokenDivider}, 19 | NodePrintRule{Node::Type::Divider, printNodeDivider}, 20 | 21 | TokenRule{tryDividerRule, doDividerRule}, 22 | 23 | TreeRule{{Token::Type::Divider}, handleDivider}, 24 | 25 | HtmlRule{Node::Type::Divider, toHtmlNodeDivider} 26 | }}; 27 | 28 | } 29 | 30 | #endif // DIVIDERRULESET_HPP 31 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/EntityEscapeRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "EntityEscapeRuleSet.hpp" 2 | 3 | #include "../../../HTTP/HtmlEntity.hpp" 4 | 5 | namespace Parser{ 6 | bool tryEntityEscapeRule(const TokenRuleContext& context){ 7 | if(check(context.page, context.pagePos, "@<") && checkLine(context.page, context.pagePos + 2, ">@")){ 8 | return true; 9 | } 10 | return false; 11 | } 12 | 13 | TokenRuleResult doEntityEscapeRule(const TokenRuleContext& context){ 14 | std::size_t pos = context.pagePos + 2; 15 | std::size_t startPos = pos; 16 | std::size_t endPos = 0; 17 | while(pos < context.page.size()){ 18 | if(check(context.page, pos, ">@")){ 19 | endPos = pos; 20 | pos += 2; 21 | break; 22 | } 23 | pos++; 24 | } 25 | 26 | std::string rawSource = context.page.substr(context.pagePos, pos - context.pagePos); 27 | std::string content = decodeHtmlEntities(context.page.substr(startPos, endPos - startPos)); 28 | 29 | TokenRuleResult result; 30 | result.newPos = pos; 31 | result.newTokens.push_back(Token{LiteralText{content}, context.pagePos, pos, rawSource}); 32 | return result; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/EntityEscapeRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ENTITYESCAPERULESET_HPP 2 | #define ENTITYESCAPERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | bool tryEntityEscapeRule(const TokenRuleContext& context); 9 | TokenRuleResult doEntityEscapeRule(const TokenRuleContext& context); 10 | 11 | const inline auto entityEscapeRuleSet = RuleSet{"EntityEscape", { 12 | TokenRule{tryEntityEscapeRule, doEntityEscapeRule} 13 | }}; 14 | 15 | } 16 | 17 | #endif // ENTITYESCAPERULESET_HPP 18 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/FootNoteRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "FootNoteRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeFootNote(const NodeVariant& nod){ 5 | return std::get(nod).number; 6 | } 7 | 8 | nlohmann::json printNodeFootNoteBlock(const NodeVariant& nod){ 9 | return std::get(nod).title; 10 | } 11 | 12 | void handleFootNote(TreeContext& context, const Token& token){ 13 | handleSectionStartEnd(token, 14 | [&](const SectionStart& section){ 15 | makeTextAddable(context); 16 | pushStack(context, Node{FootNote{}}); 17 | }, [&](const SectionEnd& section){ 18 | if(isInside(context, Node::Type::FootNote)){ 19 | popSingle(context, Node::Type::FootNote); 20 | } 21 | }); 22 | } 23 | 24 | void handleFootNoteBlock(TreeContext& context, const Token& token){ 25 | const Section& section = std::get
(token.token); 26 | FootNoteBlock block; 27 | if(section.parameters.find("title") != section.parameters.end()){ 28 | block.title = section.parameters.find("title")->second; 29 | } 30 | else{ 31 | block.title = "Footnotes"; 32 | } 33 | addAsDiv(context, Node{block}); 34 | } 35 | 36 | void postTreeRuleFootNotes(TreeContext& context){ 37 | std::vector footnotes; 38 | bool foundBlock = false; 39 | travelPageTreeNodes(context.stack.back(), [&foundBlock, &footnotes](Node& node)->bool{ 40 | Node::Type type = node.getType(); 41 | if(type == Node::Type::TableOfContents){ 42 | return false; 43 | } 44 | else if(type == Node::Type::FootNoteBlock){ 45 | foundBlock = true; 46 | return false; 47 | } 48 | else if(type == Node::Type::FootNote){ 49 | FootNote& footnote = std::get(node.node); 50 | footnote.number = footnotes.size() + 1; 51 | footnotes.push_back(node); 52 | } 53 | return true; 54 | }); 55 | if(!foundBlock && footnotes.size() > 0){ 56 | addAsDiv(context, Node{FootNoteBlock{"Footnotes"}}); 57 | } 58 | travelPageTreeNodes(context.stack.back(), [&footnotes](Node& node)->bool{ 59 | Node::Type type = node.getType(); 60 | if(type == Node::Type::FootNoteBlock){ 61 | node.branches = footnotes; 62 | return false; 63 | } 64 | return true; 65 | }); 66 | } 67 | 68 | void toHtmlNodeFootNote(const HtmlContext& con, const Node& nod){ 69 | const FootNote& footnote = std::get(nod.node); 70 | con.out << ""_AM 71 | << "["_AM << std::to_string(footnote.number) << "]"_AM 72 | << "
"_AM; 73 | delegateNodeBranches(con, nod); 74 | con.out << "
"_AM; 75 | 76 | } 77 | 78 | void toHtmlNodeFootNoteBlock(const HtmlContext& con, const Node& nod){ 79 | if(nod.branches.size() > 0){ 80 | const FootNoteBlock& block = std::get(nod.node); 81 | con.out << "

"_AM << block.title << "

"_AM 82 | << "
    "_AM; 83 | std::size_t i = 1; 84 | for(const Node& branch : nod.branches){ 85 | con.out << "
  1. "_AM << std::to_string(i) << ". "_AM; 86 | delegateNodeBranches(con, branch); 87 | con.out << "
    2. "_AM; 88 | i++; 89 | } 90 | con.out << "
"_AM; 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/FootNoteRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef FOOTNOTERULESET_HPP 2 | #define FOOTNOTERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeFootNote(const NodeVariant& nod); 8 | nlohmann::json printNodeFootNoteBlock(const NodeVariant& nod); 9 | 10 | void handleFootNote(TreeContext& context, const Token& token); 11 | void handleFootNoteBlock(TreeContext& context, const Token& token); 12 | 13 | void postTreeRuleFootNotes(TreeContext& context); 14 | 15 | void toHtmlNodeFootNote(const HtmlContext& con, const Node& nod); 16 | void toHtmlNodeFootNoteBlock(const HtmlContext& con, const Node& nod); 17 | 18 | const inline auto footNoteRuleSet = RuleSet{"FootNote", { 19 | NodePrintRule{Node::Type::FootNote, printNodeFootNote}, 20 | NodePrintRule{Node::Type::FootNoteBlock, printNodeFootNoteBlock}, 21 | 22 | SectionRule{SectionType::FootNote, {"footnote"}, SubnameType::None, ModuleType::Unknown, {}, 23 | ContentType::Surround, ParameterType::None, true}, 24 | SectionRule{SectionType::FootNoteBlock, {"footnoteblock"}, SubnameType::None, ModuleType::Unknown, {}, 25 | ContentType::None, ParameterType::Quoted, false}, 26 | 27 | TreeRule{{Token::Type::Section, SectionType::FootNote}, handleFootNote}, 28 | TreeRule{{Token::Type::Section, SectionType::FootNoteBlock}, handleFootNoteBlock}, 29 | 30 | PostTreeRule{postTreeRuleFootNotes}, 31 | 32 | HtmlRule{Node::Type::FootNote, toHtmlNodeFootNote}, 33 | HtmlRule{Node::Type::FootNoteBlock, toHtmlNodeFootNoteBlock} 34 | }}; 35 | 36 | } 37 | 38 | #endif // FOOTNOTERULESET_HPP 39 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/ForumRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef FORUMRULESET_HPP 2 | #define FORUMRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeForumStart(const NodeVariant& nod); 8 | nlohmann::json printNodeForumCategory(const NodeVariant& nod); 9 | nlohmann::json printNodeForumThread(const NodeVariant& nod); 10 | nlohmann::json printNodeForumPost(const NodeVariant& nod); 11 | 12 | void handleForumStart(TreeContext& context, const Token& token); 13 | void handleForumCategory(TreeContext& context, const Token& token); 14 | void handleForumThread(TreeContext& context, const Token& token); 15 | 16 | void toHtmlNodeForumStart(const HtmlContext& con, const Node& nod); 17 | void toHtmlNodeForumCategory(const HtmlContext& con, const Node& nod); 18 | void toHtmlNodeForumThread(const HtmlContext& con, const Node& nod); 19 | void toHtmlNodeForumPost(const HtmlContext& con, const Node& nod); 20 | 21 | const inline auto forumRuleSet = RuleSet{"Forum", { 22 | NodePrintRule{Node::Type::ForumStart, printNodeForumStart}, 23 | NodePrintRule{Node::Type::ForumCategory, printNodeForumCategory}, 24 | NodePrintRule{Node::Type::ForumThread, printNodeForumThread}, 25 | NodePrintRule{Node::Type::ForumPost, printNodeForumPost}, 26 | 27 | SectionRule{SectionType::Module, {"module"}, SubnameType::Module, ModuleType::ForumStart, {"forumstart"}, 28 | ContentType::None, ParameterType::None, false}, 29 | SectionRule{SectionType::Module, {"module"}, SubnameType::Module, ModuleType::ForumCategory, {"forumcategory"}, 30 | ContentType::None, ParameterType::None, false}, 31 | SectionRule{SectionType::Module, {"module"}, SubnameType::Module, ModuleType::ForumThread, {"forumthread"}, 32 | ContentType::None, ParameterType::None, false}, 33 | 34 | TreeRule{{Token::Type::Section, SectionType::Module, ModuleType::ForumStart}, handleForumStart}, 35 | TreeRule{{Token::Type::Section, SectionType::Module, ModuleType::ForumCategory}, handleForumCategory}, 36 | TreeRule{{Token::Type::Section, SectionType::Module, ModuleType::ForumThread}, handleForumThread}, 37 | 38 | HtmlRule{Node::Type::ForumStart, toHtmlNodeForumStart}, 39 | HtmlRule{Node::Type::ForumCategory, toHtmlNodeForumCategory}, 40 | HtmlRule{Node::Type::ForumThread, toHtmlNodeForumThread}, 41 | HtmlRule{Node::Type::ForumPost, toHtmlNodeForumPost} 42 | }}; 43 | } 44 | 45 | #endif // FORUMRULESET_HPP 46 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/HTMLRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "HTMLRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeHTML(const NodeVariant& nod){ 5 | const HTML& html = std::get(nod); 6 | return html.contents; 7 | } 8 | 9 | void handleHTML(TreeContext& context, const Token& token){ 10 | const SectionComplete& section = std::get(token.token); 11 | HTML html; 12 | html.contents = section.contents; 13 | addAsDiv(context, Node{html}); 14 | } 15 | 16 | void toHtmlNodeHTML(const HtmlContext& con, const Node& nod){ 17 | const HTML& html = std::get(nod.node); 18 | con.out << ""_AM; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/HTMLRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef HTMLRULESET_HPP 2 | #define HTMLRULESET_HPP 3 | 4 | 5 | #include "../RuleSet.hpp" 6 | 7 | namespace Parser{ 8 | nlohmann::json printNodeHTML(const NodeVariant& nod); 9 | 10 | void handleHTML(TreeContext& context, const Token& token); 11 | 12 | void toHtmlNodeHTML(const HtmlContext& con, const Node& nod); 13 | 14 | const inline auto htmlRuleSet = RuleSet{"HTML", { 15 | NodePrintRule{Node::Type::HTML, printNodeHTML}, 16 | 17 | SectionRule{SectionType::HTML, {"html", "embed", "embedvideo", "embedaudio"}, SubnameType::None, ModuleType::Unknown, {}, 18 | ContentType::Contain, ParameterType::None, false}, 19 | 20 | TreeRule{{Token::Type::Section, SectionType::HTML}, handleHTML}, 21 | 22 | HtmlRule{Node::Type::HTML, toHtmlNodeHTML} 23 | }}; 24 | 25 | } 26 | 27 | #endif // HTMLRULESET_HPP 28 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/HeadingRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "HeadingRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printTokenHeading(const TokenVariant& tok){ 5 | const Heading& heading = std::get(tok); 6 | nlohmann::json out; 7 | out["degree"] = heading.degree; 8 | out["hidden"] = heading.hidden; 9 | return out; 10 | } 11 | 12 | nlohmann::json printNodeHeading(const NodeVariant& nod){ 13 | const Heading& heading = std::get(nod); 14 | nlohmann::json out; 15 | out["degree"] = heading.degree; 16 | out["hidden"] = heading.hidden; 17 | if(heading.hidden == false){ 18 | out["tocNumber"] = *heading.tocNumber; 19 | } 20 | else{ 21 | out["tocNumber"] = nlohmann::json(); 22 | } 23 | return out; 24 | } 25 | 26 | bool tryHeadingRule(const TokenRuleContext& context){ 27 | if(context.wasNewLine && check(context.page, context.pagePos, "+")){ 28 | std::size_t pos = context.pagePos; 29 | while(pos < context.page.size()){ 30 | if(check(context.page, pos, " ") || check(context.page, pos, "* ")){ 31 | return true; 32 | } 33 | else if(context.page[pos] != '+'){ 34 | return false; 35 | } 36 | pos++; 37 | } 38 | return false; 39 | } 40 | return false; 41 | } 42 | 43 | TokenRuleResult doHeadingRule(const TokenRuleContext& context){ 44 | Heading heading; 45 | heading.degree = 0; 46 | 47 | std::size_t pos = context.pagePos; 48 | while(pos < context.page.size()){ 49 | if(check(context.page, pos, " ")){ 50 | pos++; 51 | heading.hidden = false; 52 | break; 53 | } 54 | else if(check(context.page, pos, "* ")){ 55 | pos += 2; 56 | heading.hidden = true; 57 | break; 58 | } 59 | pos++; 60 | heading.degree++; 61 | } 62 | 63 | TokenRuleResult result; 64 | result.newPos = pos; 65 | result.newTokens.push_back(Token{heading, context.pagePos, pos, context.page.substr(context.pagePos, pos - context.pagePos)}); 66 | return result; 67 | } 68 | 69 | void handleHeading(TreeContext& context, const Token& token){ 70 | makeDivPushable(context); 71 | pushStack(context, Node{std::get(token.token)}); 72 | } 73 | 74 | void toHtmlNodeHeading(const HtmlContext& con, const Node& nod){ 75 | const Heading& node = std::get(nod.node); 76 | 77 | con.out << ""_AM; 82 | delegateNodeBranches(con, nod); 83 | con.out << ""_AM; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/HeadingRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef HEADINGRULESET_HPP 2 | #define HEADINGRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenHeading(const TokenVariant& tok); 8 | nlohmann::json printNodeHeading(const NodeVariant& nod); 9 | 10 | bool tryHeadingRule(const TokenRuleContext& context); 11 | TokenRuleResult doHeadingRule(const TokenRuleContext& context); 12 | 13 | void handleHeading(TreeContext& context, const Token& token); 14 | 15 | void toHtmlNodeHeading(const HtmlContext& con, const Node& nod); 16 | 17 | const inline auto headingRuleSet = RuleSet{"Heading", { 18 | TokenPrintRule{Token::Type::Heading, printTokenHeading}, 19 | NodePrintRule{Node::Type::Heading, printNodeHeading}, 20 | 21 | TokenRule{tryHeadingRule, doHeadingRule}, 22 | 23 | TreeRule{{Token::Type::Heading}, handleHeading}, 24 | 25 | HtmlRule{Node::Type::Heading, toHtmlNodeHeading} 26 | }}; 27 | } 28 | 29 | #endif // HEADINGRULESET_HPP 30 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/HyperLinkRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef HYPERLINKRULESET_HPP 2 | #define HYPERLINKRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenHyperLink(const TokenVariant& tok); 8 | nlohmann::json printNodeHyperLink(const NodeVariant& nod); 9 | 10 | bool tryTripleLinkRule(const TokenRuleContext& context); 11 | TokenRuleResult doTripleLinkRule(const TokenRuleContext& context); 12 | 13 | const inline auto tripleHyperLinkRuleSet = RuleSet{"TripleHyperLink", { 14 | TokenRule{tryTripleLinkRule, doTripleLinkRule} 15 | }}; 16 | 17 | bool trySingleLinkRule(const TokenRuleContext& context); 18 | TokenRuleResult doSingleLinkRule(const TokenRuleContext& context); 19 | 20 | bool tryBareLinkRule(const TokenRuleContext& context); 21 | TokenRuleResult doBareLinkRule(const TokenRuleContext& context); 22 | 23 | void handleHyperLink(TreeContext& context, const Token& token); 24 | 25 | void toHtmlNodeHyperLink(const HtmlContext& con, const Node& nod); 26 | 27 | const inline auto hyperLinkRuleSet = RuleSet{"HyperLink", { 28 | TokenPrintRule{Token::Type::HyperLink, printTokenHyperLink}, 29 | NodePrintRule{Node::Type::HyperLink, printNodeHyperLink}, 30 | 31 | TokenRule{trySingleLinkRule, doSingleLinkRule}, 32 | TokenRule{tryBareLinkRule, doBareLinkRule}, 33 | 34 | TreeRule{{Token::Type::HyperLink}, handleHyperLink}, 35 | 36 | HtmlRule{Node::Type::HyperLink, toHtmlNodeHyperLink} 37 | }}; 38 | 39 | } 40 | 41 | #endif // HYPERLINKRULESET_HPP 42 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/IFrameRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "IFrameRuleSet.hpp" 2 | 3 | #include 4 | 5 | namespace Parser{ 6 | nlohmann::json printNodeIFrame(const NodeVariant& nod){ 7 | const IFrame& iframe = std::get"_AM; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/IFrameRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IFRAMERULESET_HPP 2 | #define IFRAMERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeIFrame(const NodeVariant& nod); 8 | 9 | void handleIFrame(TreeContext& context, const Token& token); 10 | 11 | void toHtmlNodeIFrame(const HtmlContext& con, const Node& nod); 12 | 13 | const inline auto iframeRuleSet = RuleSet{"IFrame", { 14 | NodePrintRule{Node::Type::IFrame, printNodeIFrame}, 15 | 16 | SectionRule{SectionType::IFrame, {"iframe"}, SubnameType::Parameter, ModuleType::Unknown, {}, 17 | ContentType::None, ParameterType::Quoted, false}, 18 | 19 | TreeRule{{Token::Type::Section, SectionType::IFrame}, handleIFrame}, 20 | 21 | HtmlRule{Node::Type::IFrame, toHtmlNodeIFrame} 22 | }}; 23 | 24 | } 25 | 26 | #endif // IFRAMERULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/IftagsRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "IftagsRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | void handleIftags(TreeContext& context, const Token& token){ 5 | handleSectionStartEnd(token, 6 | [&](const SectionStart& section){ 7 | 8 | //this is a really weird tag 9 | bool condition = true; 10 | for(auto i = section.parameters.begin(); i != section.parameters.end(); i++){ 11 | std::string tag = i->first; 12 | if(tag.size() > 0){ 13 | switch(tag[0]){ 14 | case '-': 15 | tag = tag.substr(1, tag.size() - 1); 16 | if(std::find(context.parameters.page.tags.begin(), context.parameters.page.tags.end(), tag) != context.parameters.page.tags.end()){ 17 | condition = false; 18 | } 19 | break; 20 | case '+': 21 | tag = tag.substr(1, tag.size() - 1); 22 | default: 23 | if(std::find(context.parameters.page.tags.begin(), context.parameters.page.tags.end(), tag) == context.parameters.page.tags.end()){ 24 | condition = false; 25 | } 26 | break; 27 | } 28 | } 29 | } 30 | 31 | if(!condition){ 32 | std::size_t pos = context.tokenPos; 33 | int depth = 0; 34 | while(pos < context.tokenedPage.tokens.size()){ 35 | if(context.tokenedPage.tokens[pos].getType() == Token::Type::SectionEnd){ 36 | const SectionEnd& section = std::get(context.tokenedPage.tokens[pos].token); 37 | if(section.type == SectionType::Iftags){ 38 | depth--; 39 | if(depth == 0){ 40 | context.tokenPos = pos; 41 | break; 42 | } 43 | } 44 | } 45 | else if(context.tokenedPage.tokens[pos].getType() == Token::Type::SectionStart){ 46 | const SectionStart& section = std::get(context.tokenedPage.tokens[pos].token); 47 | if(section.type == SectionType::Iftags){ 48 | depth++; 49 | } 50 | } 51 | pos++; 52 | } 53 | } 54 | 55 | }, [&](const SectionEnd& section){ 56 | //do absolutely nothing 57 | }); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/IftagsRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IFTAGSRULESET_HPP 2 | #define IFTAGSRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | void handleIftags(TreeContext& context, const Token& token); 9 | 10 | const inline auto iftagsRuleSet = RuleSet{"Iftags", { 11 | SectionRule{SectionType::Iftags, {"iftags"}, SubnameType::None, ModuleType::Unknown, {}, 12 | ContentType::Surround, ParameterType::Quoted, false}, 13 | 14 | TreeRule{{Token::Type::Section, SectionType::Iftags}, handleIftags} 15 | }}; 16 | 17 | } 18 | 19 | #endif // IFTAGSRULESET_HPP 20 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/ImageRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef IMAGERULESET_HPP 2 | #define IMAGERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | nlohmann::json printNodeImage(const NodeVariant& nod); 9 | 10 | void handleImage(TreeContext& context, const Token& token); 11 | 12 | void toHtmlNodeImage(const HtmlContext& con, const Node& nod); 13 | 14 | const inline auto imageRuleSet = RuleSet{"Image", { 15 | NodePrintRule{Node::Type::Image, printNodeImage}, 16 | 17 | SectionRule{SectionType::Image, {"image", "=image", "image", "fimage"}, SubnameType::Parameter, ModuleType::Unknown, {}, 18 | ContentType::None, ParameterType::Quoted, false}, 19 | 20 | TreeRule{{Token::Type::Section, SectionType::Image}, handleImage}, 21 | 22 | HtmlRule{Node::Type::Image, toHtmlNodeImage} 23 | }}; 24 | 25 | } 26 | 27 | #endif // IMAGERULESET_HPP 28 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/IncludeRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "IncludeRuleSet.hpp" 2 | 3 | #include "../../../Database/Database.hpp" 4 | 5 | namespace Parser{ 6 | void handleInclude(TreeContext& context, const Token& token){ 7 | if(context.parameters.includeDepth > 5){//if we have gone too many layers deep, then we need to stop doing this 8 | makeDivPushable(context); 9 | addAsText(context, Node{PlainText{"Include failed: maximum include depth reached"}}); 10 | makeDivPushable(context); 11 | return; 12 | } 13 | 14 | const Section& section = std::get
(token.token); 15 | auto pageName = section.mainParameter; 16 | auto db = context.parameters.database; 17 | if(db == nullptr){ 18 | makeDivPushable(context); 19 | addAsText(context, Node{PlainText{"Include failed: database connection is invalid"}}); 20 | makeDivPushable(context); 21 | return; 22 | } 23 | if(check(pageName, 0, ":scp-wiki:")){//if you're requesting a page on the current site, there's no need for a prefix 24 | const std::size_t length = std::string(":scp-wiki:").size(); 25 | pageName = pageName.substr(length, pageName.size() - length); 26 | } 27 | if(pageName == context.parameters.page.name){ 28 | makeDivPushable(context); 29 | addAsText(context, Node{PlainText{"Include failed: recursive include is not allowed"}}); 30 | makeDivPushable(context); 31 | return; 32 | } 33 | std::optional pageId = db->getPageId(pageName); 34 | if(!pageId){ 35 | makeDivPushable(context); 36 | addAsText(context, Node{PlainText{"Include failed: cannot find page \"" + pageName + "\""}}); 37 | makeDivPushable(context); 38 | return; 39 | } 40 | Database::PageRevision pageRevision = db->getLatestPageRevision(*pageId); 41 | ParserParameters parameters = context.parameters; 42 | parameters.includeParameters = section.parameters; 43 | parameters.includeDepth = context.parameters.includeDepth + 1; 44 | PageTree pageTree = Parser::makeTreeFromPage(pageRevision.sourceCode, parameters); 45 | 46 | makeDivPushable(context); 47 | //so what this line does is get all of the nodes from the pageTree and insert them into the current page 48 | //appending vectors is really ugly in C++ so this is the beautiful line we get 49 | context.stack.back().branches.insert(context.stack.back().branches.end(), pageTree.pageRoot.branches.begin(), pageTree.pageRoot.branches.end()); 50 | context.cssData.insert(context.cssData.begin(), pageTree.cssData.begin(), pageTree.cssData.end()); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/IncludeRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef INCLUDERULESET_HPP 2 | #define INCLUDERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | void handleInclude(TreeContext& context, const Token& token); 9 | 10 | const inline auto includeRuleSet = RuleSet{"Include", { 11 | SectionRule{SectionType::Include, {"include"}, SubnameType::Parameter, ModuleType::Unknown, {}, 12 | ContentType::None, ParameterType::Lined, false}, 13 | 14 | TreeRule{{Token::Type::Section, SectionType::Include}, handleInclude} 15 | }}; 16 | 17 | } 18 | 19 | #endif // INCLUDERULESET_HPP 20 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/InlineFormatRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef INLINEFORMATRULESET_HPP 2 | #define INLINEFORMATRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenInlineFormat(const TokenVariant& tok); 8 | nlohmann::json printNodeStyleFormat(const NodeVariant& nod); 9 | 10 | bool tryStrikeRule(const TokenRuleContext& context); 11 | TokenRuleResult doStrikeRule(const TokenRuleContext& context); 12 | bool tryItalicsRule(const TokenRuleContext& context); 13 | TokenRuleResult doItalicsRule(const TokenRuleContext& context); 14 | bool tryBoldRule(const TokenRuleContext& context); 15 | TokenRuleResult doBoldRule(const TokenRuleContext& context); 16 | bool tryUnderlineRule(const TokenRuleContext& context); 17 | TokenRuleResult doUnderlineRule(const TokenRuleContext& context); 18 | bool trySuperRule(const TokenRuleContext& context); 19 | TokenRuleResult doSuperRule(const TokenRuleContext& context); 20 | bool trySubRule(const TokenRuleContext& context); 21 | TokenRuleResult doSubRule(const TokenRuleContext& context); 22 | bool tryMonospaceRule(const TokenRuleContext& context); 23 | TokenRuleResult doMonospaceRule(const TokenRuleContext& context); 24 | bool tryColorRule(const TokenRuleContext& context); 25 | TokenRuleResult doColorRule(const TokenRuleContext& context); 26 | 27 | void handleInlineFormat(TreeContext& context, const Token& token); 28 | 29 | void toHtmlNodeStyleFormat(const HtmlContext& con, const Node& nod); 30 | 31 | const inline auto inlineFormatRuleSet = RuleSet{"InlineFormat", { 32 | TokenPrintRule{Token::Type::InlineFormat, printTokenInlineFormat}, 33 | NodePrintRule{Node::Type::StyleFormat, printNodeStyleFormat}, 34 | 35 | TokenRule{tryStrikeRule, doStrikeRule}, 36 | TokenRule{tryItalicsRule, doItalicsRule}, 37 | TokenRule{tryBoldRule, doBoldRule}, 38 | TokenRule{tryUnderlineRule, doUnderlineRule}, 39 | TokenRule{trySuperRule, doSuperRule}, 40 | TokenRule{trySubRule, doSubRule}, 41 | TokenRule{tryMonospaceRule, doMonospaceRule}, 42 | TokenRule{tryColorRule, doColorRule}, 43 | 44 | TreeRule{{Token::Type::InlineFormat}, handleInlineFormat}, 45 | 46 | HtmlRule{Node::Type::StyleFormat, toHtmlNodeStyleFormat} 47 | }}; 48 | 49 | } 50 | 51 | #endif // INLINEFORMATRULESET_HPP 52 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/ListPagesRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef LISTPAGESRULESET_HPP 2 | #define LISTPAGESRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | 8 | void handleListPages(TreeContext& context, const Token& token); 9 | 10 | const inline auto listPagesRuleSet = RuleSet{"ListPages", { 11 | SectionRule{SectionType::Module, {"module"}, SubnameType::Module, ModuleType::ListPages, {"listpages"}, 12 | ContentType::Contain, ParameterType::Quoted, false}, 13 | 14 | TreeRule{{Token::Type::Section, SectionType::Module, ModuleType::ListPages}, handleListPages} 15 | }}; 16 | 17 | } 18 | 19 | #endif // LISTPAGESRULESET_HPP 20 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/ListRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef LISTRULESET_HPP 2 | #define LISTRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenListPrefix(const TokenVariant& tok); 8 | 9 | nlohmann::json printNodeList(const NodeVariant& nod); 10 | nlohmann::json printNodeListElement(const NodeVariant& nod); 11 | 12 | nlohmann::json printNodeAdvList(const NodeVariant& nod); 13 | nlohmann::json printNodeAdvListElement(const NodeVariant& nod); 14 | 15 | bool tryListPrefixRule(const TokenRuleContext& context); 16 | TokenRuleResult doListPrefixRule(const TokenRuleContext& context); 17 | 18 | void toHtmlNodeList(const HtmlContext& con, const Node& nod); 19 | void toHtmlNodeListElement(const HtmlContext& con, const Node& nod); 20 | 21 | void handleAdvList(TreeContext& context, const Token& token); 22 | void handleAdvListElement(TreeContext& context, const Token& token); 23 | 24 | void toHtmlNodeAdvList(const HtmlContext& con, const Node& nod); 25 | void toHtmlNodeAdvListElement(const HtmlContext& con, const Node& nod); 26 | 27 | const inline auto listRuleSet = RuleSet{"List", { 28 | TokenPrintRule{Token::Type::ListPrefix, printTokenListPrefix}, 29 | NodePrintRule{Node::Type::List, printNodeList}, 30 | NodePrintRule{Node::Type::ListElement, printNodeListElement}, 31 | NodePrintRule{Node::Type::AdvList, printNodeAdvList}, 32 | NodePrintRule{Node::Type::AdvListElement, printNodeAdvListElement}, 33 | 34 | TokenRule{tryListPrefixRule, doListPrefixRule}, 35 | 36 | HtmlRule{Node::Type::List, toHtmlNodeList}, 37 | HtmlRule{Node::Type::ListElement, toHtmlNodeListElement}, 38 | 39 | SectionRule{SectionType::AdvList, {"ol", "ol_", "ul", "ul_"}, SubnameType::None, ModuleType::Unknown, {}, 40 | ContentType::Surround, ParameterType::Quoted, true}, 41 | SectionRule{SectionType::AdvListElement, {"li", "li_"}, SubnameType::None, ModuleType::Unknown, {}, 42 | ContentType::Surround, ParameterType::Quoted, true}, 43 | 44 | TreeRule{{Token::Type::Section, SectionType::AdvList}, handleAdvList}, 45 | TreeRule{{Token::Type::Section, SectionType::AdvListElement}, handleAdvListElement}, 46 | 47 | HtmlRule{Node::Type::AdvList, toHtmlNodeAdvList}, 48 | HtmlRule{Node::Type::AdvListElement, toHtmlNodeAdvListElement} 49 | }}; 50 | 51 | } 52 | 53 | #endif // LISTRULESET_HPP 54 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/LiteralTextRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "LiteralTextRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printTokenLiteralText(const TokenVariant& tok){ 5 | return std::get(tok).text; 6 | } 7 | 8 | nlohmann::json printNodeLiteralText(const NodeVariant& nod){ 9 | return std::get(nod).text; 10 | } 11 | 12 | bool tryLiteralTextRule(const TokenRuleContext& context){ 13 | if(check(context.page, context.pagePos, "@@") 14 | && checkLine(context.page, context.pagePos + 2, "@@")){ 15 | return true; 16 | } 17 | return false; 18 | } 19 | 20 | TokenRuleResult doLiteralTextRule(const TokenRuleContext& context){ 21 | std::size_t pos = context.pagePos + 2; 22 | std::size_t startPos = pos; 23 | std::size_t endPos = 0; 24 | while(pos < context.page.size()){ 25 | if(check(context.page, pos, "@@")){ 26 | endPos = pos; 27 | pos += 2; 28 | break; 29 | } 30 | pos++; 31 | } 32 | 33 | std::string rawSource = context.page.substr(context.pagePos, pos - context.pagePos); 34 | std::string content = context.page.substr(startPos, endPos - startPos); 35 | 36 | TokenRuleResult result; 37 | result.newPos = pos; 38 | result.newTokens.push_back(Token{LiteralText{content}, context.pagePos, pos, rawSource}); 39 | return result; 40 | } 41 | 42 | void handleLiteralText(TreeContext& context, const Token& token){ 43 | addAsText(context, Node{std::get(token.token)}); 44 | } 45 | 46 | void toHtmlNodeLiteralText(const HtmlContext& con, const Node& nod){ 47 | const LiteralText& node = std::get(nod.node); 48 | con.out << node.text;///TODO: this isn't correct, it should have a style that makes some whitespace oddities 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/LiteralTextRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef LITERALTEXTRULESET_HPP 2 | #define LITERALTEXTRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenLiteralText(const TokenVariant& tok); 8 | nlohmann::json printNodeLiteralText(const NodeVariant& nod); 9 | 10 | bool tryLiteralTextRule(const TokenRuleContext& context); 11 | TokenRuleResult doLiteralTextRule(const TokenRuleContext& context); 12 | 13 | void handleLiteralText(TreeContext& context, const Token& token); 14 | 15 | void toHtmlNodeLiteralText(const HtmlContext& con, const Node& nod); 16 | 17 | const inline auto literalTextRuleSet = RuleSet{"LiteralText", { 18 | TokenPrintRule{Token::Type::LiteralText, printTokenLiteralText}, 19 | NodePrintRule{Node::Type::LiteralText, printNodeLiteralText}, 20 | 21 | TokenRule{tryLiteralTextRule, doLiteralTextRule}, 22 | 23 | TreeRule{{Token::Type::LiteralText}, handleLiteralText}, 24 | 25 | HtmlRule{Node::Type::LiteralText, toHtmlNodeLiteralText} 26 | }}; 27 | 28 | } 29 | 30 | #endif // LITERALTEXTRULESET_HPP 31 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/NullRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "NullRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | bool tryNullRule(const TokenRuleContext& context){ 5 | return true; 6 | } 7 | 8 | TokenRuleResult doNullRule(const TokenRuleContext& context){ 9 | TokenRuleResult result; 10 | result.newPos = context.pagePos + 1; 11 | return result; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/NullRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef NULLRULESET_HPP 2 | #define NULLRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | bool tryNullRule(const TokenRuleContext& context); 8 | TokenRuleResult doNullRule(const TokenRuleContext& context); 9 | 10 | const inline auto nullRuleSet = RuleSet{"Null", { 11 | TokenRule{tryNullRule, doNullRule} 12 | }}; 13 | } 14 | 15 | #endif // NULLRULESET_HPP 16 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/QuoteBoxRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "QuoteBoxRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printTokenQuoteBoxPrefix(const TokenVariant& tok){ 5 | const QuoteBoxPrefix& quoteBoxPrefix = std::get(tok); 6 | return quoteBoxPrefix.degree; 7 | } 8 | 9 | nlohmann::json printNodeQuoteBox(const NodeVariant& nod){ 10 | return {}; 11 | } 12 | 13 | bool tryQuoteBoxPrefixRule(const TokenRuleContext& context){ 14 | if(context.tokens.size() > 0 && context.tokens.back().getType() == Token::Type::QuoteBoxPrefix){ 15 | return false;//a quote box prefix cannot directly follow another quote box 16 | } 17 | if(context.wasNewLine && check(context.page, context.pagePos, ">")){ 18 | std::size_t pos = context.pagePos; 19 | while(pos < context.page.size()){ 20 | if(check(context.page, pos, ">")){ 21 | //keep going 22 | } 23 | else if(check(context.page, pos, " ") || check(context.page, pos, "\n")){ 24 | return true; 25 | } 26 | else{ 27 | return false;//fail if the quote box prefix ends with anything other than a space 28 | } 29 | pos++; 30 | } 31 | } 32 | return false; 33 | } 34 | 35 | TokenRuleResult doQuoteBoxPrefixRule(const TokenRuleContext& context){ 36 | std::size_t pos = context.pagePos; 37 | QuoteBoxPrefix prefix; 38 | while(pos < context.page.size()){ 39 | if(check(context.page, pos, " ")){ 40 | prefix.degree = pos - context.pagePos; 41 | pos++;//skip the space too 42 | break; 43 | } 44 | else if(check(context.page, pos, "\n")){ 45 | prefix.degree = pos - context.pagePos; 46 | break; 47 | } 48 | pos++; 49 | } 50 | 51 | 52 | TokenRuleResult result; 53 | result.newPos = pos; 54 | result.nowNewline = true;//preserve newline status 55 | result.newTokens.push_back(Token{prefix, context.pagePos, pos, context.page.substr(context.pagePos, pos - context.pagePos)}); 56 | return result; 57 | } 58 | 59 | void toHtmlNodeQuoteBox(const HtmlContext& con, const Node& nod){ 60 | con.out << "
"_AM; 61 | delegateNodeBranches(con, nod); 62 | con.out << "
"_AM; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/QuoteBoxRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef QUOTEBOXRULESET_HPP 2 | #define QUOTEBOXRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenQuoteBoxPrefix(const TokenVariant& tok); 8 | nlohmann::json printNodeQuoteBox(const NodeVariant& nod); 9 | 10 | bool tryQuoteBoxPrefixRule(const TokenRuleContext& context); 11 | TokenRuleResult doQuoteBoxPrefixRule(const TokenRuleContext& context); 12 | 13 | void toHtmlNodeQuoteBox(const HtmlContext& con, const Node& nod); 14 | 15 | const inline auto quoteBoxRuleSet = RuleSet{"QuoteBox", { 16 | TokenPrintRule{Token::Type::QuoteBoxPrefix, printTokenQuoteBoxPrefix}, 17 | NodePrintRule{Node::Type::QuoteBox, printNodeQuoteBox}, 18 | 19 | TokenRule{tryQuoteBoxPrefixRule, doQuoteBoxPrefixRule}, 20 | 21 | HtmlRule{Node::Type::QuoteBox, toHtmlNodeQuoteBox} 22 | }}; 23 | 24 | } 25 | 26 | #endif // QUOTEBOXRULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/RateRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "RateRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeRate(const NodeVariant& nod){ 5 | const Rate& rate = std::get(nod); 6 | return rate.rating; 7 | } 8 | 9 | void handleRate(TreeContext& context, const Token& token){ 10 | addAsDiv(context, Node{Rate{context.parameters.page.rating}}); 11 | } 12 | 13 | void toHtmlNodeRate(const HtmlContext& con, const Node& nod){ 14 | const Rate& rate = std::get(nod.node); 15 | con.out 16 | << "
"_AM 17 | << ""_AM 18 | << "rating: " << ((rate.rating > 0)?"+":"") << std::to_string(rate.rating) 19 | << ""_AM 20 | << "
"_AM; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/RateRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef RATERULESET_HPP 2 | #define RATERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeRate(const NodeVariant& nod); 8 | 9 | void handleRate(TreeContext& context, const Token& token); 10 | 11 | void toHtmlNodeRate(const HtmlContext& con, const Node& nod); 12 | 13 | const inline auto rateRuleSet = RuleSet{"Rate", { 14 | NodePrintRule{Node::Type::Rate, printNodeRate}, 15 | 16 | SectionRule{SectionType::Module, {"module"}, SubnameType::Module, ModuleType::Rate, {"rate"}, 17 | ContentType::None, ParameterType::None, false}, 18 | 19 | TreeRule{{Token::Type::Section, SectionType::Module, ModuleType::Rate}, handleRate}, 20 | 21 | HtmlRule{Node::Type::Rate, toHtmlNodeRate} 22 | }}; 23 | 24 | } 25 | 26 | #endif // RATERULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/RedirectRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "RedirectRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | void handleRedirect(TreeContext& context, const Token& token){ 5 | ///TODO: make this do something 6 | //for now it's just going to be left as a "null" rule and just throw them away, they don't really *need* to be there, right? 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/RedirectRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef REDIRECTRULESET_HPP 2 | #define REDIRECTRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | void handleRedirect(TreeContext& context, const Token& token); 8 | 9 | const inline auto redirectRuleSet = RuleSet{"Redirect", { 10 | SectionRule{SectionType::Module, {"module"}, SubnameType::Module, ModuleType::Redirect, {"redirect"}, 11 | ContentType::None, ParameterType::Quoted, false}, 12 | 13 | TreeRule{{Token::Type::Section, SectionType::Module, ModuleType::Redirect}, handleRedirect}, 14 | }}; 15 | 16 | } 17 | 18 | #endif // REDIRECTRULESET_HPP 19 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/SCPConversionProjectInfoBoxRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "SCPConversionProjectInfoBoxRuleSet.hpp" 2 | 3 | #include "../../../Database/Database.hpp" 4 | #include "../../../Database/Batch.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeSCPConversionProjectInfoBox(const NodeVariant& nod){ 8 | const SCPConversionProjectInfoBox& infoBox = std::get(nod); 9 | nlohmann::json out; 10 | out["lastUpdate"] = infoBox.lastUpdate; 11 | out["pageCount"] = infoBox.pageCount; 12 | out["revisionCount"] = infoBox.revisionCount; 13 | out["threadCount"] = infoBox.threadCount; 14 | out["postCount"] = infoBox.postCount; 15 | return out; 16 | } 17 | 18 | void handleSCPConversionProjectInfoBox(TreeContext& context, const Token& token){ 19 | auto db = context.parameters.database; 20 | if(db == nullptr){ 21 | makeDivPushable(context); 22 | addAsText(context, Node{PlainText{"SCPConversionProjectInfoBox Failed: database connection is invalid"}}); 23 | makeDivPushable(context); 24 | return; 25 | } 26 | SCPConversionProjectInfoBox infoBox; 27 | { 28 | std::string scraperFolder = Config::getScraperFolder(); 29 | infoBox.lastUpdate = Importer::lastImportedBatchTimeStamp(scraperFolder + "batches/", scraperFolder + "batchData.json"); 30 | } 31 | infoBox.pageCount = db->getNumberOfPages(); 32 | infoBox.revisionCount = db->getNumberOfPageRevisions(); 33 | infoBox.threadCount = db->getNumberOfForumThreads(); 34 | infoBox.postCount = db->getNumberOfForumPosts(); 35 | addAsDiv(context, Node{infoBox}); 36 | } 37 | 38 | void toHtmlNodeSCPConversionProjectInfoBox(const HtmlContext& con, const Node& nod){ 39 | const SCPConversionProjectInfoBox& infoBox = std::get(nod.node); 40 | con.out << ""_AM 41 | << ""_AM 42 | << ""_AM 43 | << ""_AM 44 | << ""_AM 45 | << ""_AM 46 | << "
Last Update"_AM << formatTimeStamp(infoBox.lastUpdate) << "
Page Count"_AM << std::to_string(infoBox.pageCount) << "
Revision Count"_AM << std::to_string(infoBox.revisionCount) << "
Thread Count"_AM << std::to_string(infoBox.threadCount) << "
Post Count"_AM << std::to_string(infoBox.postCount) << "
"_AM; 47 | } 48 | } -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/SCPConversionProjectInfoBoxRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SCPCONVERSIONPROJECTINFOBOXRULESET_HPP 2 | #define SCPCONVERSIONPROJECTINFOBOXRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | //The stupidly long name is on purpose, I want to make it obvious that this isn't something added by wikidot; 7 | //this is honest to goodness a completely new feature that would never work with the old wiki. 8 | //I also want to make it impossible to accidentally use in some other code without realizing that this isn't a real formatting element. 9 | 10 | namespace Parser{ 11 | nlohmann::json printNodeSCPConversionProjectInfoBox(const NodeVariant& nod); 12 | 13 | void handleSCPConversionProjectInfoBox(TreeContext& context, const Token& token); 14 | 15 | void toHtmlNodeSCPConversionProjectInfoBox(const HtmlContext& con, const Node& nod); 16 | 17 | const inline auto scpConversionProjectInfoBoxRuleSet = RuleSet{"SCPConversionProjectInfoBox", { 18 | NodePrintRule{Node::Type::SCPConversionProjectInfoBox, printNodeSCPConversionProjectInfoBox}, 19 | 20 | SectionRule{SectionType::SCPConversionProjectInfoBox, {"scp_conversion_project_info_box"}, SubnameType::None, ModuleType::Unknown, {}, 21 | ContentType::None, ParameterType::None, false}, 22 | 23 | TreeRule{{Token::Type::Section, SectionType::SCPConversionProjectInfoBox}, handleSCPConversionProjectInfoBox}, 24 | 25 | HtmlRule{Node::Type::SCPConversionProjectInfoBox, toHtmlNodeSCPConversionProjectInfoBox} 26 | }}; 27 | 28 | } 29 | 30 | #endif // SCPCONVERSIONPROJECTINFOBOXRULESET_HPP 31 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/SectionRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SECTIONRULESET_HPP 2 | #define SECTIONRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenSection(const TokenVariant& tok); 8 | nlohmann::json printTokenSectionStart(const TokenVariant& tok); 9 | nlohmann::json printTokenSectionEnd(const TokenVariant& tok); 10 | nlohmann::json printTokenSectionComplete(const TokenVariant& tok); 11 | 12 | bool trySectionRule(const TokenRuleContext& context); 13 | TokenRuleResult doSectionRule(const TokenRuleContext& context); 14 | 15 | const inline auto sectionRuleSet = RuleSet{"Section", { 16 | TokenPrintRule{Token::Type::Section, printTokenSection}, 17 | TokenPrintRule{Token::Type::SectionStart, printTokenSectionStart}, 18 | TokenPrintRule{Token::Type::SectionEnd, printTokenSectionEnd}, 19 | TokenPrintRule{Token::Type::SectionComplete, printTokenSectionComplete}, 20 | 21 | TokenRule{trySectionRule, doSectionRule} 22 | }}; 23 | 24 | } 25 | 26 | #endif // SECTIONRULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/SizeRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "SizeRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeSize(const NodeVariant& nod){ 5 | const Size& size = std::get(nod); 6 | return size.size; 7 | } 8 | 9 | void handleSize(TreeContext& context, const Token& token){ 10 | handleSectionStartEnd(token, 11 | [&](const SectionStart& section){ 12 | makeTextAddable(context); 13 | pushStack(context, Node{Size{section.mainParameter}}); 14 | }, [&](const SectionEnd& section){ 15 | if(isInside(context, Node::Type::Size)){ 16 | popSingle(context, Node::Type::Size); 17 | } 18 | }); 19 | } 20 | 21 | void toHtmlNodeSize(const HtmlContext& con, const Node& nod){ 22 | const Size& node = std::get(nod.node); 23 | con.out << ""_AM; 24 | delegateNodeBranches(con, nod); 25 | con.out << ""_AM; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/SizeRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SIZERULESET_HPP 2 | #define SIZERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeSize(const NodeVariant& nod); 8 | 9 | void handleSize(TreeContext& context, const Token& token); 10 | 11 | void toHtmlNodeSize(const HtmlContext& con, const Node& nod); 12 | 13 | const inline auto sizeRuleSet = RuleSet{"Size", { 14 | NodePrintRule{Node::Type::Size, printNodeSize}, 15 | 16 | TreeRule{{Token::Type::Section, SectionType::Size}, handleSize}, 17 | 18 | SectionRule{SectionType::Size, {"size"}, SubnameType::Parameter, ModuleType::Unknown, {}, 19 | ContentType::Surround, ParameterType::None, true}, 20 | 21 | HtmlRule{Node::Type::Size, toHtmlNodeSize} 22 | }}; 23 | 24 | } 25 | 26 | #endif // SIZERULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/SpanRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "SpanRuleSet.hpp" 2 | 3 | #include 4 | 5 | namespace Parser{ 6 | nlohmann::json printNodeSpan(const NodeVariant& nod){ 7 | const Span& span = std::get(nod); 8 | nlohmann::json out = nlohmann::json::object(); 9 | for(auto i = span.parameters.begin(); i != span.parameters.end(); i++){ 10 | out[i->first] = i->second; 11 | } 12 | return out; 13 | } 14 | 15 | void handleSpan(TreeContext& context, const Token& token){ 16 | handleSectionStartEnd(token, 17 | [&](const SectionStart& section){ 18 | makeTextAddable(context); 19 | pushStack(context, Node{Span{section.parameters}}); 20 | }, [&](const SectionEnd& section){ 21 | if(isInside(context, Node::Type::Span)){ 22 | popSingle(context, Node::Type::Span); 23 | } 24 | }); 25 | } 26 | 27 | void toHtmlNodeSpan(const HtmlContext& con, const Node& nod){ 28 | const Span& node = std::get(nod.node); 29 | con.out << "first == "id" && check(i->second, 0, "u-") == false){ 32 | con.out << " "_AM << i->first << "='u-"_AM << i->second << "'"_AM; 33 | } 34 | else{ 35 | con.out << " "_AM << i->first << "='"_AM << i->second << "'"_AM; 36 | } 37 | } 38 | con.out << ">"_AM; 39 | delegateNodeBranches(con, nod); 40 | con.out << ""_AM; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/SpanRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SPANRULESET_HPP 2 | #define SPANRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeSpan(const NodeVariant& nod); 8 | 9 | void handleSpan(TreeContext& context, const Token& token); 10 | 11 | void toHtmlNodeSpan(const HtmlContext& con, const Node& nod); 12 | 13 | const inline auto spanRuleSet = RuleSet{"Span", { 14 | NodePrintRule{Node::Type::Span, printNodeSpan}, 15 | 16 | SectionRule{SectionType::Span, {"span", "span_"}, SubnameType::None, ModuleType::Unknown, {}, 17 | ContentType::Surround, ParameterType::Quoted, true}, 18 | 19 | TreeRule{{Token::Type::Section, SectionType::Span}, handleSpan}, 20 | 21 | HtmlRule{Node::Type::Span, toHtmlNodeSpan} 22 | }}; 23 | 24 | } 25 | 26 | #endif // SPANRULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/TabViewRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "TabViewRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeTabView(const NodeVariant& nod){ 5 | return {}; 6 | } 7 | 8 | nlohmann::json printNodeTab(const NodeVariant& nod){ 9 | return std::get(nod).title; 10 | } 11 | 12 | void handleTabView(TreeContext& context, const Token& token){ 13 | handleSectionStartEnd(token, 14 | [&](const SectionStart& section){ 15 | makeDivPushable(context); 16 | pushStack(context, Node{TabView{}}); 17 | }, [&](const SectionEnd& section){ 18 | if(isInside(context, Node::Type::TabView)){ 19 | popSingle(context, Node::Type::TabView); 20 | } 21 | }); 22 | } 23 | 24 | void handleTab(TreeContext& context, const Token& token){ 25 | handleSectionStartEnd(token, 26 | [&](const SectionStart& section){ 27 | if(isInside(context, Node::Type::TabView)){ 28 | while(context.stack.back().getType() != Node::Type::TabView){ 29 | popStackWithCarry(context); 30 | } 31 | pushStack(context, Node{Tab{section.mainParameter}}); 32 | } 33 | }, [&](const SectionEnd& section){ 34 | if(isInside(context, Node::Type::Tab)){ 35 | popSingle(context, Node::Type::Tab); 36 | } 37 | }); 38 | } 39 | 40 | void toHtmlNodeTabView(const HtmlContext& con, const Node& nod){ 41 | std::string formId = getUniqueHtmlId(con); 42 | con.out << "
"_AM 43 | << "
"_AM 44 | << "
"_AM; 45 | std::vector radioIds; 46 | for(auto i = nod.branches.begin(); i != nod.branches.end(); i++){ 47 | const Tab& tab = std::get(i->node); 48 | std::string radioId = getUniqueHtmlId(con); 49 | con.out << ""_AM; 50 | radioIds.push_back(radioId); 51 | } 52 | con.out << "
"_AM; 53 | { 54 | std::size_t num = 0; 55 | for(auto i = nod.branches.begin(); i != nod.branches.end(); i++, num++){ 56 | con.out << ""_AM; 57 | convertNodeToHtml(con, *i); 58 | } 59 | } 60 | con.out << "
"_AM; 61 | } 62 | 63 | void toHtmlNodeTab(const HtmlContext& con, const Node& nod){ 64 | const Tab& tab = std::get(nod.node); 65 | con.out << "
"_AM 66 | << "
"_AM << tab.title << "
"_AM; 67 | delegateNodeBranches(con, nod); 68 | con.out << "
"_AM; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/TabViewRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TABVIEWRULESET_HPP 2 | #define TABVIEWRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeTabView(const NodeVariant& nod); 8 | nlohmann::json printNodeTab(const NodeVariant& nod); 9 | 10 | void handleTabView(TreeContext& context, const Token& token); 11 | void handleTab(TreeContext& context, const Token& token); 12 | 13 | void toHtmlNodeTabView(const HtmlContext& con, const Node& nod); 14 | void toHtmlNodeTab(const HtmlContext& con, const Node& nod); 15 | 16 | const inline auto tabViewRuleSet = RuleSet{"TabView", { 17 | NodePrintRule{Node::Type::TabView, printNodeTabView}, 18 | NodePrintRule{Node::Type::Tab, printNodeTab}, 19 | 20 | SectionRule{SectionType::TabView, {"tabview"}, SubnameType::None, ModuleType::Unknown, {}, 21 | ContentType::Surround, ParameterType::None, false}, 22 | SectionRule{SectionType::Tab, {"tab"}, SubnameType::Parameter, ModuleType::Unknown, {}, 23 | ContentType::Surround, ParameterType::None, false}, 24 | 25 | TreeRule{{Token::Type::Section, SectionType::TabView}, handleTabView}, 26 | TreeRule{{Token::Type::Section, SectionType::Tab}, handleTab}, 27 | 28 | HtmlRule{Node::Type::TabView, toHtmlNodeTabView}, 29 | HtmlRule{Node::Type::Tab, toHtmlNodeTab} 30 | }}; 31 | 32 | } 33 | 34 | #endif // TABVIEWRULESET_HPP 35 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/TableOfContentsRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "TableOfContentsRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeTableOfContents(const NodeVariant& nod){ 5 | const TableOfContents& toc = std::get(nod); 6 | switch(toc.alignment){ 7 | case TableOfContents::AlignmentType::Default: 8 | return "Default"; 9 | case TableOfContents::AlignmentType::FloatLeft: 10 | return "FloatLeft"; 11 | case TableOfContents::AlignmentType::FloatRight: 12 | return "FloatRight"; 13 | default: 14 | return "Unknown"; 15 | } 16 | } 17 | 18 | void handleTableOfContents(TreeContext& context, const Token& token){ 19 | const Section& section = std::get
(token.token); 20 | 21 | TableOfContents toc; 22 | if(check(section.typeString, 0, "f<")){ 23 | toc.alignment = TableOfContents::AlignmentType::FloatLeft; 24 | } 25 | else if(check(section.typeString, 0, "f>")){ 26 | toc.alignment = TableOfContents::AlignmentType::FloatRight; 27 | } 28 | else{ 29 | toc.alignment = TableOfContents::AlignmentType::Default; 30 | } 31 | 32 | addAsDiv(context, Node{toc}); 33 | } 34 | 35 | void postTreeRuleTableOfContents(TreeContext& context){ 36 | std::vector headings; 37 | travelPageTreeNodes(context.stack.back(), [&headings](Node& node)->bool{ 38 | Node::Type type = node.getType(); 39 | if(type == Node::Type::TableOfContents || type == Node::Type::FootNoteBlock){ 40 | return false; 41 | } 42 | if(type == Node::Type::Heading){ 43 | Heading& heading = std::get(node.node); 44 | if(heading.hidden == false){ 45 | heading.tocNumber = headings.size(); 46 | headings.push_back(node); 47 | } 48 | } 49 | return true; 50 | }); 51 | travelPageTreeNodes(context.stack.back(), [&headings](Node& node)->bool{ 52 | Node::Type type = node.getType(); 53 | if(type == Node::Type::TableOfContents){ 54 | node.branches = headings; 55 | return false; 56 | } 57 | return true; 58 | }); 59 | } 60 | 61 | void toHtmlNodeTableOfContents(const HtmlContext& con, const Node& nod){ 62 | const TableOfContents& toc = std::get(nod.node); 63 | con.out << "
"_AM; 67 | break; 68 | case TableOfContents::AlignmentType::FloatLeft: 69 | con.out << " style='float:left;'>"_AM; 70 | break; 71 | case TableOfContents::AlignmentType::FloatRight: 72 | con.out << " style='float:right;'>"_AM; 73 | break; 74 | } 75 | 76 | con.out << "
Table of Contents
"_AM; 77 | for(const Node& headingNode : nod.branches){ 78 | const Heading& heading = std::get(headingNode.node); 79 | con.out << ""_AM; 83 | } 84 | con.out << "
"_AM; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/TableOfContentsRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TABLEOFCONTENTSRULESET_HPP 2 | #define TABLEOFCONTENTSRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeTableOfContents(const NodeVariant& nod); 8 | 9 | void handleTableOfContents(TreeContext& context, const Token& token); 10 | 11 | void postTreeRuleTableOfContents(TreeContext& context); 12 | 13 | void toHtmlNodeTableOfContents(const HtmlContext& con, const Node& nod); 14 | 15 | const inline auto tableOfContentsRuleSet = RuleSet{"TableOfContents", { 16 | NodePrintRule{Node::Type::TableOfContents, printNodeTableOfContents}, 17 | 18 | SectionRule{SectionType::TableOfContents, {"toc", "ftoc"}, SubnameType::None, ModuleType::Unknown, {}, 19 | ContentType::None, ParameterType::None, false}, 20 | 21 | TreeRule{{Token::Type::Section, SectionType::TableOfContents}, handleTableOfContents}, 22 | 23 | PostTreeRule{postTreeRuleTableOfContents}, 24 | 25 | HtmlRule{Node::Type::TableOfContents, toHtmlNodeTableOfContents} 26 | }}; 27 | 28 | } 29 | 30 | 31 | #endif // TABLEOFCONTENTSRULESET_HPP 32 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/TableRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TABLERULESET_HPP 2 | #define TABLERULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printTokenTableMarker(const TokenVariant& tok); 8 | nlohmann::json printNodeTable(const NodeVariant& nod); 9 | nlohmann::json printNodeTableRow(const NodeVariant& nod); 10 | nlohmann::json printNodeTableElement(const NodeVariant& nod); 11 | 12 | nlohmann::json printNodeAdvTable(const NodeVariant& nod); 13 | nlohmann::json printNodeAdvTableRow(const NodeVariant& nod); 14 | nlohmann::json printNodeAdvTableElement(const NodeVariant& nod); 15 | 16 | bool tryTableMarkerRule(const TokenRuleContext& context); 17 | TokenRuleResult doTableMarkerRule(const TokenRuleContext& context); 18 | 19 | void handleTableMarker(TreeContext& context, const Token& token); 20 | 21 | void handleAdvTable(TreeContext& context, const Token& token); 22 | void handleAdvTableRow(TreeContext& context, const Token& token); 23 | void handleAdvTableElement(TreeContext& context, const Token& token); 24 | 25 | void toHtmlNodeTable(const HtmlContext& con, const Node& nod); 26 | void toHtmlNodeTableRow(const HtmlContext& con, const Node& nod); 27 | void toHtmlNodeTableElement(const HtmlContext& con, const Node& nod); 28 | 29 | void toHtmlNodeAdvTable(const HtmlContext& con, const Node& nod); 30 | void toHtmlNodeAdvTableRow(const HtmlContext& con, const Node& nod); 31 | void toHtmlNodeAdvTableElement(const HtmlContext& con, const Node& nod); 32 | 33 | const inline auto tableRuleSet = RuleSet{"Table", { 34 | TokenPrintRule{Token::Type::TableMarker, printTokenTableMarker}, 35 | NodePrintRule{Node::Type::Table, printNodeTable}, 36 | NodePrintRule{Node::Type::TableRow, printNodeTableRow}, 37 | NodePrintRule{Node::Type::TableElement, printNodeTableElement}, 38 | 39 | NodePrintRule{Node::Type::AdvTable, printNodeAdvTable}, 40 | NodePrintRule{Node::Type::AdvTableRow, printNodeAdvTableRow}, 41 | NodePrintRule{Node::Type::AdvTableElement, printNodeAdvTableElement}, 42 | 43 | TokenRule{tryTableMarkerRule, doTableMarkerRule}, 44 | 45 | SectionRule{SectionType::AdvTable, {"table"}, SubnameType::None, ModuleType::Unknown, {}, 46 | ContentType::Surround, ParameterType::Quoted, false}, 47 | SectionRule{SectionType::AdvTableRow, {"row"}, SubnameType::None, ModuleType::Unknown, {}, 48 | ContentType::Surround, ParameterType::Quoted, false}, 49 | SectionRule{SectionType::AdvTableElement, {"cell", "hcell"}, SubnameType::None, ModuleType::Unknown, {}, 50 | ContentType::Surround, ParameterType::Quoted, false}, 51 | 52 | TreeRule{{Token::Type::TableMarker}, handleTableMarker}, 53 | 54 | TreeRule{{Token::Type::Section, SectionType::AdvTable}, handleAdvTable}, 55 | TreeRule{{Token::Type::Section, SectionType::AdvTableRow}, handleAdvTableRow}, 56 | TreeRule{{Token::Type::Section, SectionType::AdvTableElement}, handleAdvTableElement}, 57 | 58 | HtmlRule{Node::Type::Table, toHtmlNodeTable}, 59 | HtmlRule{Node::Type::TableRow, toHtmlNodeTableRow}, 60 | HtmlRule{Node::Type::TableElement, toHtmlNodeTableElement}, 61 | 62 | HtmlRule{Node::Type::AdvTable, toHtmlNodeAdvTable}, 63 | HtmlRule{Node::Type::AdvTableRow, toHtmlNodeAdvTableRow}, 64 | HtmlRule{Node::Type::AdvTableElement, toHtmlNodeAdvTableElement} 65 | }}; 66 | } 67 | 68 | #endif // TABLERULESET_HPP 69 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/UserRuleSet.cpp: -------------------------------------------------------------------------------- 1 | #include "UserRuleSet.hpp" 2 | 3 | namespace Parser{ 4 | nlohmann::json printNodeUser(const NodeVariant& nod){ 5 | const User& user = std::get(nod); 6 | nlohmann::json out; 7 | out["shownName"] = user.shownName; 8 | out["linkName"] = user.linkName; 9 | out["newWindow"] = user.newWindow; 10 | return out; 11 | } 12 | 13 | void handleUser(TreeContext& context, const Token& token){ 14 | const Section& section = std::get
(token.token); 15 | User user; 16 | user.shownName = section.mainParameter; 17 | user.linkName = normalizePageName(user.shownName); 18 | user.newWindow = (section.typeString == "*user"); 19 | addAsText(context, Node{user}); 20 | } 21 | 22 | void toHtmlNodeUser(const HtmlContext& con, const Node& nod){ 23 | const User& user = std::get(nod.node); 24 | con.out << ""_AM << user.shownName << ""_AM; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Rules/RuleSets/UserRuleSet.hpp: -------------------------------------------------------------------------------- 1 | #ifndef USERRULESET_HPP 2 | #define USERRULESET_HPP 3 | 4 | #include "../RuleSet.hpp" 5 | 6 | namespace Parser{ 7 | nlohmann::json printNodeUser(const NodeVariant& nod); 8 | 9 | void handleUser(TreeContext& context, const Token& token); 10 | 11 | void toHtmlNodeUser(const HtmlContext& con, const Node& nod); 12 | 13 | const inline auto userRuleSet = RuleSet{"User", { 14 | NodePrintRule{Node::Type::User, printNodeUser}, 15 | 16 | SectionRule{SectionType::User, {"user", "*user"}, SubnameType::Parameter, ModuleType::Unknown, {}, 17 | ContentType::None, ParameterType::None, true}, 18 | 19 | TreeRule{{Token::Type::Section, SectionType::User}, handleUser}, 20 | 21 | HtmlRule{Node::Type::User, toHtmlNodeUser} 22 | }}; 23 | 24 | } 25 | 26 | #endif // USERRULESET_HPP 27 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Templater.cpp: -------------------------------------------------------------------------------- 1 | #include "Templater.hpp" 2 | 3 | #include 4 | 5 | #include "Rules/RuleSetUtil.hpp" 6 | 7 | namespace Parser{ 8 | namespace{ 9 | struct TemplateRule{ 10 | std::string from; 11 | std::string to; 12 | }; 13 | 14 | std::string applyTemplateRules(std::string pageContent, const std::vector& rules){ 15 | std::stringstream output; 16 | 17 | std::size_t pos = 0; 18 | while(pos < pageContent.size()){ 19 | bool ruleMatched = false; 20 | for(const TemplateRule& rule : rules){ 21 | if(checkWithoutCase(pageContent, pos, rule.from)){///TODO: make this case in-sensitive 22 | ruleMatched = true; 23 | output << rule.to; 24 | pos += rule.from.size(); 25 | } 26 | } 27 | if(ruleMatched == false){ 28 | output << pageContent[pos]; 29 | pos++; 30 | } 31 | } 32 | 33 | return output.str(); 34 | } 35 | } 36 | 37 | std::string applyPageTemplate(std::string format, std::string pageContent, PageInfo pageInfo){ 38 | std::vector rules; 39 | rules.push_back({"%%content%%", pageContent}); 40 | rules.push_back({"%%name%%", pageInfo.name});//technically incorrect by the standard 41 | rules.push_back({"%%fullname%%", pageInfo.name}); 42 | rules.push_back({"%%title%%", pageInfo.name}); 43 | rules.push_back({"%%title_linked%%", "[/" + pageInfo.name + " " + pageInfo.title + "]"}); 44 | rules.push_back({"%%rating%%", std::to_string(pageInfo.rating)}); 45 | 46 | //The follow is commented out because it is extremely slow and I have never seen it used on the Wiki 47 | //I'm sure it has been, but I've never seen it. AFAIK it works perfectly fine, it's just really slow 48 | /* 49 | { 50 | std::vector contents; 51 | TokenedPage tokens = tokenizePage(pageContent, {pageInfo}); 52 | std::size_t pos = 0; 53 | for(const Token& token : tokens.tokens){ 54 | if(token.getType() == Token::Type::Divider){ 55 | const Divider& divider = std::get(token.token); 56 | if(divider.type == Divider::Type::Separator){ 57 | contents.push_back(tokens.originalPage.substr(pos, token.sourceStart - pos)); 58 | pos = token.sourceEnd; 59 | } 60 | } 61 | } 62 | contents.push_back(tokens.originalPage.substr(pos, tokens.originalPage.size() - pos)); 63 | for(std::size_t i = 0; i < contents.size(); i++){ 64 | rules.push_back({"%%content(" + std::to_string(i + 1) + ")%%", contents[i]}); 65 | } 66 | } 67 | */ 68 | 69 | return applyTemplateRules(format, rules); 70 | } 71 | 72 | std::string applyIncludeParameters(std::string pageContent, const std::map& parameters){ 73 | std::vector rules; 74 | for(auto i = parameters.begin(); i != parameters.end(); i++){ 75 | rules.push_back({"{$" + i->first + "}", i->second}); 76 | } 77 | return applyTemplateRules(pageContent, rules); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Parser/Templater.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TEMPLATER_HPP 2 | #define TEMPLATER_HPP 3 | 4 | #include "Parser.hpp" 5 | 6 | namespace Parser{ 7 | std::string applyPageTemplate(std::string format, std::string pageContent, PageInfo pageInfo = {}); 8 | std::string applyIncludeParameters(std::string pageContent, const std::map& parameters); 9 | } 10 | 11 | #endif // TEMPLATER_HPP 12 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/AutomatedTester.cpp: -------------------------------------------------------------------------------- 1 | #include "AutomatedTester.hpp" 2 | 3 | namespace Tests{ 4 | void Tester::add(std::string name, std::function func){ 5 | tests.push_back(std::make_pair(name, func)); 6 | } 7 | 8 | void Tester::runTests(){ 9 | unsigned int passedTests = 0; 10 | for(unsigned int i = 0; i < tests.size(); i++){ 11 | std::string name = tests[i].first; 12 | std::function func = tests[i].second; 13 | std::cout << "\nRunning test \"" << name << "\"...\n"; 14 | bool passed = true; 15 | try{ 16 | func(); 17 | } 18 | catch(std::exception& e){ 19 | std::cout << "Test \"" << name << "\" failed, Error:\"" << e.what() << "\"\n"; 20 | passed = false; 21 | } 22 | if(passed){ 23 | std::cout << "Test \"" << name << "\" passed\n"; 24 | passedTests++; 25 | } 26 | } 27 | std::cout << "\n\n||=====================================||\n" 28 | << " Summary:\n" 29 | << " " << passedTests << " out of " << tests.size() << " tests passed.\n" 30 | ; 31 | if(passedTests == tests.size()){ 32 | std::cout << "\n All tests passed!\n"; 33 | } 34 | else{ 35 | throw std::runtime_error("Not all tests passed"); 36 | } 37 | } 38 | 39 | void shouldThrowException(std::function func){ 40 | try{ 41 | func(); 42 | } 43 | catch(std::exception& e){ 44 | return;//return with no error 45 | } 46 | throw std::runtime_error("Expected a thrown exception, but got none"); 47 | } 48 | 49 | void assertTrue(bool assertion){ 50 | if(assertion == false){ 51 | throw std::runtime_error("Assertion Error"); 52 | } 53 | } 54 | 55 | void addTesterTests(Tester& tester){ 56 | tester.add("Tests::shouldThrowException", [](){ 57 | shouldThrowException([](){shouldThrowException([](){});}); 58 | try{ 59 | shouldThrowException([](){}); 60 | } 61 | catch(std::exception& e){ 62 | return; 63 | } 64 | throw std::runtime_error("Expected \"shouldThrowException\" to throw an exception, but it didn't"); 65 | }); 66 | 67 | tester.add("Tests::assertEquals", [](){ 68 | assertEquals(1, 1); 69 | assertEquals("hello", "hello"); 70 | assertEqualsVec({}, {}); 71 | assertEqualsVec({1, 2, 3}, {1, 2, 3}); 72 | 73 | shouldThrowException([](){ 74 | assertEquals(1, 2); 75 | }); 76 | shouldThrowException([](){ 77 | assertEquals("hello", "bye"); 78 | }); 79 | shouldThrowException([](){ 80 | assertEqualsVec({1, 2}, {1, 2, 3}); 81 | }); 82 | shouldThrowException([](){ 83 | assertEqualsVec({1, 2}, {2, 1}); 84 | }); 85 | }); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/AutomatedTester.hpp: -------------------------------------------------------------------------------- 1 | #ifndef AUTOMATEDTESTER_H 2 | #define AUTOMATEDTESTER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace Tests{ 12 | class Tester{ 13 | public: 14 | void add(std::string name, std::function func); 15 | void runTests(); 16 | 17 | private: 18 | std::vector>> tests; 19 | }; 20 | 21 | void shouldThrowException(std::function func); 22 | 23 | void assertTrue(bool assertion); 24 | 25 | template 26 | inline void assertEquals(A expected, B actual){ 27 | if(!(expected == actual)){ 28 | std::stringstream ss; 29 | ss << "Assert Fail, expected the following to be equal:\n" 30 | << "\tExpected: {\n" << expected << "\n}\n" 31 | << "\tActual: {\n" << actual << "\n}\n"; 32 | throw std::runtime_error(ss.str()); 33 | } 34 | } 35 | 36 | template 37 | inline void assertEqualsVec(std::vector expected, std::vector actual){ 38 | bool areEqual = true; 39 | if(expected.size() == actual.size()){ 40 | for(std::size_t i = 0; i < expected.size(); i++){ 41 | if(!(expected[i] == actual[i])){ 42 | areEqual = false; 43 | break; 44 | } 45 | } 46 | } 47 | else{ 48 | areEqual = false; 49 | } 50 | if(!areEqual){ 51 | std::stringstream ss; 52 | ss << "Assert Fail, expected the following std::vectors to be equal:\n" 53 | << "\tExpected: {\n"; 54 | for(std::size_t i = 0; i < expected.size(); i++){ 55 | ss << "\t\t" << expected[i]; 56 | if(i != expected.size() - 1){ 57 | ss << ", "; 58 | } 59 | ss << "\n"; 60 | } 61 | ss << "\t}\n\tActual: {\n"; 62 | for(std::size_t i = 0; i < actual.size(); i++){ 63 | ss << "\t\t" << actual[i]; 64 | if(i != actual.size() - 1){ 65 | ss << ", "; 66 | } 67 | ss << "\n"; 68 | } 69 | ss << "\t}\n"; 70 | throw std::runtime_error(ss.str()); 71 | } 72 | } 73 | 74 | void addTesterTests(Tester& tester); 75 | } 76 | 77 | #endif // AUTOMATEDTESTER_H 78 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Database/DatabaseTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATABASETESTS_HPP 2 | #define DATABASETESTS_HPP 3 | 4 | #include "../AutomatedTester.hpp" 5 | 6 | namespace Tests{ 7 | 8 | void addDatabaseTests(Tester& tester); 9 | } 10 | 11 | #endif // DATABASETESTS_HPP 12 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Database/ImporterTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef DATABASETESTS_H 2 | #define DATABASETESTS_H 3 | 4 | #include "../AutomatedTester.hpp" 5 | 6 | namespace Tests{ 7 | void addImporterTests(Tester& tester); 8 | } 9 | 10 | #endif // DATABASETESTS_H 11 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Database/JsonTests.cpp: -------------------------------------------------------------------------------- 1 | #include "JsonTests.hpp" 2 | 3 | #include "../../Database/Json.hpp" 4 | 5 | namespace Tests{ 6 | void addJsonTests(Tester& tester){ 7 | //there actually aren't any tests to put here anymore... 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Database/JsonTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef JSONTESTS_HPP 2 | #define JSONTESTS_HPP 3 | 4 | 5 | #include "../AutomatedTester.hpp" 6 | 7 | namespace Tests{ 8 | void addJsonTests(Tester& tester); 9 | } 10 | 11 | #endif // JSONTESTS_HPP 12 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/HTTP/HtmlEntityTests.cpp: -------------------------------------------------------------------------------- 1 | #include "HtmlEntityTests.hpp" 2 | 3 | #include "../../HTTP/HtmlEntity.hpp" 4 | 5 | namespace Tests{ 6 | void addHtmlEntityTests(Tester& tester){ 7 | tester.add("decodeHtmlEntities", [](){ 8 | //this function was obtained from an outside source so we're just gonna hope that it works correctly 9 | //but we'll have a couple basic tests for good measure 10 | assertEquals("hello, world!", decodeHtmlEntities("hello, world!")); 11 | assertEquals("", decodeHtmlEntities("<a>")); 12 | }); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/HTTP/HtmlEntityTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef HTMLENTITYTESTS_HPP 2 | #define HTMLENTITYTESTS_HPP 3 | 4 | #include "../AutomatedTester.hpp" 5 | 6 | namespace Tests{ 7 | void addHtmlEntityTests(Tester& tester); 8 | } 9 | 10 | #endif // HTMLENTITYTESTS_HPP 11 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/HTTP/MarkupOutStreamTests.cpp: -------------------------------------------------------------------------------- 1 | #include "MarkupOutStreamTests.hpp" 2 | 3 | #include "../../HTTP/MarkupOutStream.hpp" 4 | 5 | #include 6 | 7 | namespace Tests{ 8 | void addMarkupOutStreamTests(Tester& tester){ 9 | tester.add("MarkupOutStream", [](){ 10 | const auto escape = [](std::string testInput){ 11 | std::stringstream output; 12 | MarkupOutStream stream(&output); 13 | stream << testInput; 14 | return output.str(); 15 | }; 16 | 17 | assertEquals("hello, test text", escape("hello, test text")); 18 | assertEquals("<html>is escaped</html>", escape("is escaped")); 19 | assertEquals("&amp;", escape("&")); 20 | assertEquals("<a href='google.com'>link</a>", escape("link")); 21 | assertEquals("<a href="google.com">link</a>", escape("link")); 22 | 23 | std::stringstream output; 24 | MarkupOutStream stream(&output); 25 | stream << "<"_AM << "<" << allowMarkup(">"); 26 | assertEquals("<<>", output.str()); 27 | }); 28 | 29 | tester.add("percentEncode", [](){ 30 | assertEquals("hello+world%21", urlEncode("hello world!")); 31 | assertEquals("", urlEncode("")); 32 | assertEquals("hello", urlEncode("hello")); 33 | assertEquals("hey%2C+what%27s+going+on%3F", urlEncode("hey, what's going on?")); 34 | assertEquals("this+is%2Fwas+madness", urlEncode("this is/was madness")); 35 | assertEquals("first%0D%0Asecond", urlEncode("first\r\nsecond")); 36 | }); 37 | 38 | tester.add("percentDecode", [](){ 39 | assertEquals("hello world!", urlDecode("hello+world%21")); 40 | assertEquals("", urlDecode("")); 41 | assertEquals("hello", urlDecode("hello")); 42 | assertEquals("hey, what's going on?", urlDecode("hey%2C+what%27s+going+on%3F")); 43 | assertEquals("this is/was madness", urlDecode("this+is%2Fwas+madness")); 44 | assertEquals("first\r\nsecond", urlDecode("first%0D%0Asecond")); 45 | }); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/HTTP/MarkupOutStreamTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef MARKUPOUTSTREAMTESTS_HPP 2 | #define MARKUPOUTSTREAMTESTS_HPP 3 | 4 | #include "../AutomatedTester.hpp" 5 | 6 | namespace Tests{ 7 | void addMarkupOutStreamTests(Tester& tester); 8 | } 9 | 10 | #endif // MARKUPOUTSTREAMTESTS_HPP 11 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Parser/HTMLConverterTests.cpp: -------------------------------------------------------------------------------- 1 | #include "HTMLConverterTests.hpp" 2 | 3 | #include "../../Parser/HTMLConverter.hpp" 4 | 5 | namespace Tests{ 6 | using namespace Parser; 7 | 8 | void addHTMLConverterTests(Tester& tester){ 9 | tester.add("Parser::redirectLink", [](){ 10 | assertEquals("/scp-173", redirectLink("http://www.scp-wiki.net/scp-173")); 11 | assertEquals("/scp-173", redirectLink("http://scp-wiki.wikidot.com/scp-173")); 12 | assertEquals("/__system/pageFile/scp-173/SCP-173.jpg", redirectLink("http://scp-wiki.wdfiles.com/local--files/scp-173/SCP-173.jpg")); 13 | assertEquals("/__system/pageFile/page/image.jpg", redirectLink("http://www.scp-wiki.net/local--files/page/image.jpg")); 14 | assertEquals("/scp-173", redirectLink("/scp-173")); 15 | assertEquals("https://google.com/", redirectLink("https://google.com/")); 16 | }); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Parser/HTMLConverterTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef HTMLCONVERTERTESTS_HPP 2 | #define HTMLCONVERTERTESTS_HPP 3 | 4 | #include "../AutomatedTester.hpp" 5 | 6 | namespace Tests{ 7 | void addHTMLConverterTests(Tester& tester); 8 | } 9 | 10 | #endif // HTMLCONVERTERTESTS_HPP 11 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Parser/ParserTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef PARSERTESTS_HPP 2 | #define PARSERTESTS_HPP 3 | 4 | #include "../AutomatedTester.hpp" 5 | 6 | namespace Tests{ 7 | void addParserTests(Tester& tester); 8 | } 9 | 10 | #endif // PARSERTESTS_HPP 11 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Parser/TemplaterTests.cpp: -------------------------------------------------------------------------------- 1 | #include "TemplaterTests.hpp" 2 | 3 | #include "../../Parser/Templater.hpp" 4 | 5 | namespace Tests{ 6 | using namespace Parser; 7 | 8 | void addTemplaterTests(Tester& tester){ 9 | tester.add("Parser::applyIncludeParameters", [](){ 10 | assertEquals( 11 | "this was replaced" 12 | , applyIncludeParameters("this {$varA} replaced", {{"varA", "was"}})); 13 | }); 14 | 15 | tester.add("Parser::applyPageTemplate", [](){ 16 | assertEquals( 17 | "just ignored" 18 | , applyPageTemplate("just ignored", "page content")); 19 | 20 | assertEquals( 21 | "page content" 22 | , applyPageTemplate("%%content%%", "page content")); 23 | 24 | assertEquals( 25 | "page:\n" 26 | "page content" 27 | , applyPageTemplate("page:\n%%content%%", "page content")); 28 | 29 | //due to performance issues the following has been disabled 30 | /* 31 | assertEquals( 32 | "page A:\n" 33 | "page content A\n" 34 | "page B:\n" 35 | "page content B" 36 | , applyPageTemplate("page A:\n%%content(1)%%page B:%%content(2)%%", "page content A\n=======\npage content B")); 37 | */ 38 | }); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Parser/TemplaterTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TEMPLATERTESTS_HPP 2 | #define TEMPLATERTESTS_HPP 3 | 4 | #include "../AutomatedTester.hpp" 5 | 6 | namespace Tests{ 7 | void addTemplaterTests(Tester& tester); 8 | } 9 | 10 | #endif // TEMPLATERTESTS_HPP 11 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Parser/TreerTests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TREERTESTS_HPP 2 | #define TREERTESTS_HPP 3 | 4 | #include "../AutomatedTester.hpp" 5 | 6 | namespace Tests{ 7 | void addTreerTests(Tester& tester); 8 | } 9 | 10 | #endif // TREERTESTS_HPP 11 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Tests.cpp: -------------------------------------------------------------------------------- 1 | #include "AutomatedTester.hpp" 2 | 3 | #include "Database/ImporterTests.hpp" 4 | #include "Database/DatabaseTests.hpp" 5 | #include "Database/JsonTests.hpp" 6 | 7 | #include "HTTP/HtmlEntityTests.hpp" 8 | #include "HTTP/MarkupOutStreamTests.hpp" 9 | 10 | #include "Parser/ParserTests.hpp" 11 | #include "Parser/TreerTests.hpp" 12 | #include "Parser/TemplaterTests.hpp" 13 | #include "Parser/HTMLConverterTests.hpp" 14 | 15 | namespace Tests{ 16 | void runAllTests(){ 17 | Tester tester; 18 | 19 | addParserTests(tester); 20 | addTreerTests(tester); 21 | addTemplaterTests(tester); 22 | addHTMLConverterTests(tester); 23 | 24 | addHtmlEntityTests(tester); 25 | addMarkupOutStreamTests(tester); 26 | 27 | addJsonTests(tester); 28 | addDatabaseTests(tester); 29 | addImporterTests(tester); 30 | 31 | addTesterTests(tester); 32 | 33 | tester.runTests(); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Tests/Tests.hpp: -------------------------------------------------------------------------------- 1 | #ifndef TESTS_H 2 | #define TESTS_H 3 | 4 | namespace Tests{ 5 | void runAllTests(); 6 | } 7 | 8 | #endif // TESTS_H 9 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/UntestedUtils/UntestedUtils.cpp: -------------------------------------------------------------------------------- 1 | #include "UntestedUtils.hpp" 2 | 3 | #include "../Database/Database.hpp" 4 | #include "../Parser/Parser.hpp" 5 | #include "../Parser/HTMLConverter.hpp" 6 | #include "../Config.hpp" 7 | 8 | #include "../Database/Json.hpp" 9 | 10 | #include 11 | #include 12 | 13 | namespace UntestedUtils{ 14 | namespace{ 15 | 16 | struct TimelineUpdate{ 17 | enum class Type{Creation, Update}; 18 | Type type; 19 | std::string name; 20 | int64_t revisionIndex; 21 | std::string title; 22 | TimeStamp timeStamp; 23 | std::vector linkedNames; 24 | }; 25 | } 26 | 27 | void exportNodeDiagramData(){ 28 | std::unique_ptr database = Database::connectToDatabase(Config::getProductionDatabaseName()); 29 | std::vector pageList = database->getPageList(); 30 | 31 | std::vector timeline; 32 | 33 | int num = 0; 34 | for(Database::ID pageId : pageList){ 35 | std::string name = database->getPageName(pageId); 36 | 37 | num++; 38 | std::cout << num << "/" << pageList.size() << " " << (num * 100.0 / pageList.size()) << "%: " << name << "...\n"; 39 | 40 | std::vector revisionList = database->getPageRevisions(pageId); 41 | if(revisionList.size() == 0){ 42 | std::cout << "Error on page " << name << "\n"; 43 | } 44 | bool creation = true; 45 | int64_t revisionIndex = 0; 46 | for(Database::ID revisionId : revisionList){ 47 | std::cout << " " << revisionIndex << "/" << revisionList.size() << "\r" << std::flush; 48 | 49 | TimelineUpdate update; 50 | if(creation){ 51 | update.type = TimelineUpdate::Type::Creation; 52 | creation = false; 53 | } 54 | else{ 55 | update.type = TimelineUpdate::Type::Update; 56 | } 57 | update.name = name; 58 | 59 | Database::PageRevision pageRevision = database->getPageRevision(revisionId); 60 | update.title = pageRevision.title; 61 | update.timeStamp = pageRevision.timeStamp; 62 | update.linkedNames = Parser::getPageLinks(pageRevision.sourceCode); 63 | update.revisionIndex = revisionIndex; 64 | revisionIndex++; 65 | 66 | timeline.push_back(update); 67 | } 68 | } 69 | 70 | std::sort(timeline.begin(), timeline.end(), [](const TimelineUpdate& a, const TimelineUpdate& b){return a.timeStamp < b.timeStamp;}); 71 | nlohmann::json output = nlohmann::json::array(); 72 | for(const TimelineUpdate& update : timeline){ 73 | nlohmann::json uj; 74 | uj["type"] = (update.type == TimelineUpdate::Type::Creation)?"creation":"update"; 75 | uj["name"] = update.name; 76 | uj["title"] = update.title; 77 | uj["timeStamp"] = update.timeStamp; 78 | uj["linkedNames"] = update.linkedNames; 79 | uj["revisionIndex"] = update.revisionIndex; 80 | output.push_back(uj); 81 | } 82 | try{ 83 | std::ofstream("/scp_conversion_project/nodeDiagramDataExportFinal.json") << output.dump(4); 84 | } 85 | catch(std::exception& e){ 86 | while(true){ 87 | std::cout << "Error Exporting!\nType out new directory:"; 88 | std::string file; 89 | std::getline(std::cin, file); 90 | try{ 91 | std::ofstream(file) << output.dump(4); 92 | break; 93 | } 94 | catch(std::exception& e){ 95 | 96 | } 97 | } 98 | } 99 | } 100 | 101 | void benchmarkPageLoad(std::string pageName){ 102 | std::unique_ptr database = Database::connectToDatabase(Config::getProductionDatabaseName()); 103 | std::optional pageId = database->getPageId(pageName); 104 | Database::PageRevision revision = database->getLatestPageRevision(pageId.value()); 105 | 106 | std::stringstream outputString; 107 | MarkupOutStream outStream{&outputString}; 108 | 109 | using TimePoint = std::chrono::time_point; 110 | const auto getTime = [](){return std::chrono::high_resolution_clock::now();}; 111 | const auto timeString = [](TimePoint before, TimePoint after)->std::string{return std::to_string(std::chrono::duration_cast(after - before).count() / 1000000000.0);}; 112 | 113 | TimePoint start = getTime(); 114 | Parser::TokenedPage pageTokens = Parser::tokenizePage(revision.sourceCode); 115 | TimePoint endToken = getTime(); 116 | Parser::PageTree pageTree = Parser::makeTreeFromTokenedPage(pageTokens); 117 | TimePoint endTree = getTime(); 118 | Parser::convertPageTreeToHtml(outStream, pageTree); 119 | TimePoint endHtml = getTime(); 120 | 121 | 122 | std::cout << "Token Duration: " << timeString(start, endToken) << "\n"; 123 | std::cout << "Tree Duration: " << timeString(endToken, endTree) << "\n"; 124 | std::cout << "HTML Duration: " << timeString(endTree, endHtml) << "\n"; 125 | std::cout << "Total Duration: " << timeString(start, endHtml) << "\n"; 126 | } 127 | } 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/UntestedUtils/UntestedUtils.hpp: -------------------------------------------------------------------------------- 1 | #ifndef UNTESTEDUTILS_HPP 2 | #define UNTESTEDUTILS_HPP 3 | 4 | #include 5 | 6 | namespace UntestedUtils{ 7 | void exportNodeDiagramData(); 8 | void benchmarkPageLoad(std::string pageName); 9 | } 10 | 11 | #endif // UNTESTEDUTILS_HPP 12 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Website/Gateway.cpp: -------------------------------------------------------------------------------- 1 | #include "Gateway.hpp" 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | #include "../Config.hpp" 9 | 10 | void Gateway::setup(std::string domainName){ 11 | domain = domainName; 12 | } 13 | 14 | void Gateway::run(std::string socket, std::function threadFunc){ 15 | shutdownFlag = false; 16 | fcgiListenSocket = FCGX_OpenSocket(socket.c_str(), 500); 17 | FCGX_Init(); 18 | 19 | auto signalHandler = [](int signum){ 20 | std::cout << "\nSIGNAL INPUT:Start Shutdown Procedure\n"; 21 | Gateway::shutdown(); 22 | }; 23 | signal(SIGINT, signalHandler); 24 | signal(SIGTERM, signalHandler); 25 | 26 | unsigned int numberOfGatewayThreads = Config::getNumberOfThreadsForGateway(); 27 | 28 | std::vector threadPool; 29 | for(unsigned int i = 0; i < numberOfGatewayThreads; i++){ 30 | ThreadContext threadContext; 31 | threadContext.threadIndex = i; 32 | threadContext.domainName = domain; 33 | FCGX_InitRequest(&threadContext.fcgiRequest, fcgiListenSocket, 0); 34 | 35 | threadPool.push_back(std::thread(threadFunc, threadContext)); 36 | } 37 | 38 | for(unsigned int i = 0; i < numberOfGatewayThreads; i++){ 39 | threadPool[i].join(); 40 | } 41 | } 42 | 43 | void Gateway::shutdown(){ 44 | shutdownFlag = true; 45 | FCGX_ShutdownPending(); 46 | ::shutdown(fcgiListenSocket, SHUT_RDWR); 47 | } 48 | 49 | 50 | 51 | Gateway::RequestContext Gateway::ThreadContext::getNextRequest(){ 52 | if(FCGX_Accept_r(&fcgiRequest) == 0){ 53 | RequestContext output; 54 | 55 | output.shouldShutdown = false; 56 | output.fcgiOutBuffer = std::make_unique(fcgiRequest.out); 57 | output.rawOutputStream = std::make_unique(output.fcgiOutBuffer.get()); 58 | 59 | output.fcgiInputWrapper.fcgiRequest = &fcgiRequest; 60 | output.cgi = std::make_unique(&output.fcgiInputWrapper); 61 | output.env = &output.cgi->getEnvironment(); 62 | 63 | output.out = MarkupOutStream(output.rawOutputStream.get()); 64 | 65 | return output; 66 | } 67 | else{ 68 | RequestContext output; 69 | output.shouldShutdown = true; 70 | return output; 71 | } 72 | } 73 | 74 | void Gateway::ThreadContext::finishRequest(Gateway::RequestContext requestContext){ 75 | FCGX_Finish_r(&fcgiRequest); 76 | } 77 | 78 | std::size_t Gateway::FcgiInputWrapper::read(char* data, std::size_t length){ 79 | return FCGX_GetStr(data, length, fcgiRequest->in); 80 | } 81 | 82 | std::string Gateway::FcgiInputWrapper::getenv(const char* varName){ 83 | const char* temp = FCGX_GetParam(varName, fcgiRequest->envp); 84 | if(temp == nullptr){ 85 | return std::string(); 86 | } 87 | else{ 88 | return std::string(temp); 89 | } 90 | } 91 | 92 | std::string Gateway::RequestContext::getUriString(){ 93 | return fcgiInputWrapper.getenv("DOCUMENT_URI"); 94 | } 95 | 96 | 97 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Website/Gateway.hpp: -------------------------------------------------------------------------------- 1 | #ifndef GATEWAY_HPP 2 | #define GATEWAY_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "../HTTP/MarkupOutStream.hpp" 19 | 20 | class Gateway{ 21 | private: 22 | struct FcgiInputWrapper : public cgicc::CgiInput{ 23 | FCGX_Request* fcgiRequest; 24 | virtual std::size_t read(char* data, std::size_t length); 25 | virtual std::string getenv(const char* varName); 26 | }; 27 | public: 28 | struct RequestContext{ 29 | friend class Gateway; 30 | 31 | public: 32 | std::string getUriString(); 33 | 34 | bool shouldShutdown; 35 | std::unique_ptr cgi; 36 | const cgicc::CgiEnvironment* env; 37 | MarkupOutStream out; 38 | 39 | private: 40 | FcgiInputWrapper fcgiInputWrapper; 41 | std::unique_ptr fcgiOutBuffer; 42 | std::unique_ptr rawOutputStream; 43 | }; 44 | 45 | struct ThreadContext{ 46 | friend class Gateway; 47 | 48 | public: 49 | Gateway::RequestContext getNextRequest(); 50 | void finishRequest(Gateway::RequestContext request); 51 | 52 | std::string domainName; 53 | unsigned int threadIndex; 54 | private: 55 | FCGX_Request fcgiRequest; 56 | }; 57 | 58 | Gateway() = delete; 59 | 60 | static void setup(std::string domainName); 61 | 62 | static void run(std::string socket, std::function threadFunc); 63 | static void shutdown(); 64 | 65 | private: 66 | static inline std::atomic shutdownFlag; 67 | static inline std::string domain; 68 | 69 | static inline int fcgiListenSocket; 70 | }; 71 | 72 | #endif // GATEWAY_HPP 73 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/Website/Website.hpp: -------------------------------------------------------------------------------- 1 | #ifndef WEBSITE_HPP 2 | #define WEBSITE_HPP 3 | 4 | #include 5 | 6 | #include "Gateway.hpp" 7 | #include "../Database/Database.hpp" 8 | 9 | class Website{ 10 | public: 11 | struct Context{ 12 | std::unique_ptr db; 13 | }; 14 | 15 | Website() = delete; 16 | 17 | static void run(); 18 | 19 | private: 20 | static void threadProcess(Gateway::ThreadContext threadContext); 21 | static std::vector splitUri(std::string uri); 22 | static void handleUri(Gateway::RequestContext& reqCon, Website::Context& webCon, std::vector uri); 23 | static bool handlePage(Gateway::RequestContext& reqCon, Website::Context& webCon, std::string pageName, Database::ID pageId, std::map parameters); 24 | static bool handleFormattedArticle(Gateway::RequestContext& reqCon, Website::Context& webCon, std::string pageName, Database::ID pageId, std::map parameters, Database::PageRevision& revision, std::optional revisionIndex); 25 | }; 26 | 27 | #endif // WEBSITE_HPP 28 | -------------------------------------------------------------------------------- /src/ScpArchive/ScpArchive/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "Tests/Tests.hpp" 4 | 5 | #include "Website/Website.hpp" 6 | #include "Database/Database.hpp" 7 | #include "Database/Batch.hpp" 8 | 9 | #include "Config.hpp" 10 | 11 | #include "UntestedUtils/UntestedUtils.hpp" 12 | 13 | #include "Parser/Parser.hpp" 14 | #include "Parser/Treer.hpp" 15 | #include "Parser/HTMLConverter.hpp" 16 | 17 | #include "Parser/Rules/RuleSet.hpp" 18 | 19 | #include "Database/Json.hpp" 20 | 21 | #include "PDF/PDFConverter.hpp" 22 | 23 | int main(int argc, char** argv){ 24 | 25 | if(argc == 2 && std::string(argv[1]) == "--printRuleSet"){ 26 | Parser::printFullRuleSetList(); 27 | } 28 | else if(argc == 2 && std::string(argv[1]) == "--importFrontPage"){ 29 | nlohmann::json frontPage = Json::loadJsonFromFile("/home/daniel/Projects/SCP_Conversion_Project/data/customFrontPage/pages/__front-page/data.json"); 30 | std::ifstream pageFile("/home/daniel/Projects/SCP_Conversion_Project/data/customFrontPage/pages/__front-page/frontpage.txt"); 31 | std::string str{std::istreambuf_iterator(pageFile), std::istreambuf_iterator()}; 32 | frontPage["revisions"][0]["sourceCode"] = str; 33 | Json::saveJsonToFile("/home/daniel/Projects/SCP_Conversion_Project/data/customFrontPage/pages/__front-page/data.json", frontPage); 34 | 35 | Importer::importBatch("/home/daniel/Projects/SCP_Conversion_Project/data/customFrontPage/"); 36 | 37 | Website::run(); 38 | } 39 | else if(argc == 2 && std::string(argv[1]) == "--runTests"){ 40 | Tests::runAllTests(); 41 | } 42 | else if(argc == 2 && std::string(argv[1]) == "--clearData"){ 43 | std::cout << "Are you sure you want to CLEAR database \"" << Config::getProductionDatabaseName() << "\"?(y/n):\n"; 44 | std::string temp; 45 | std::getline(std::cin, temp); 46 | if(temp != "y"){ 47 | std::cout << "Aborting.\n"; 48 | return 0; 49 | } 50 | 51 | std::unique_ptr database = Database::connectToDatabase(Config::getProductionDatabaseName()); 52 | database->cleanAndInitDatabase(); 53 | } 54 | else if(argc == 2 && std::string(argv[1]) == "--doBatches"){ 55 | std::string batchesFolder = Config::getScraperFolder() + "batches/"; 56 | std::string batchDataFile = Config::getScraperFolder() + "batchData.json"; 57 | Importer::handleBatches(batchesFolder, batchDataFile); 58 | } 59 | else if(argc == 2 && std::string(argv[1]) == "--autoDoBatches"){ 60 | std::string batchesFolder = Config::getScraperFolder() + "batches/"; 61 | std::string batchDataFile = Config::getScraperFolder() + "batchData.json"; 62 | Importer::handleBatches(batchesFolder, batchDataFile, true); 63 | } 64 | else if(argc == 3 && std::string(argv[1]) == "--doCustomBatch"){ 65 | Importer::importBatch(argv[2]); 66 | } 67 | else if(argc == 2 && std::string(argv[1]) == "--runWebsite"){ 68 | Website::run(); 69 | } 70 | else if(argc == 2 && std::string(argv[1]) == "--exportNodeDiagramData"){ 71 | UntestedUtils::exportNodeDiagramData(); 72 | } 73 | else if(argc == 2 && std::string(argv[1]) == "--runBenchmark"){ 74 | UntestedUtils::benchmarkPageLoad("attribution-metadata"); 75 | } 76 | else if(argc == 2 && std::string(argv[1]) == "--updatePDFs"){ 77 | PDF::updatePDFs(); 78 | } 79 | else{ 80 | std::cout << "Invalid/No Options Specified\n"; 81 | } 82 | 83 | return 0; 84 | } 85 | -------------------------------------------------------------------------------- /src/ScpArchive/meson.build: -------------------------------------------------------------------------------- 1 | project('ScpArchive', 'cpp') 2 | 3 | cpp = meson.get_compiler('cpp') 4 | 5 | dep = [ 6 | dependency('boost', modules : ['filesystem']), 7 | dependency('nlohmann_json'), 8 | cpp.find_library('cgicc'), 9 | dependency('fcgi'), 10 | cpp.find_library('fcgi++'), 11 | cpp.find_library('soci_core'), 12 | cpp.find_library('soci_mysql'), 13 | dependency('mariadb') 14 | ] 15 | 16 | inc = [ 17 | '.' 18 | ] 19 | 20 | src = [ 21 | 'ScpArchive/main.cpp', 22 | 'ScpArchive/Config.cpp', 23 | 'ScpArchive/Database/Batch.cpp', 24 | 'ScpArchive/Database/Database.cpp', 25 | 'ScpArchive/Database/Importer.cpp', 26 | 'ScpArchive/Database/Json.cpp', 27 | 'ScpArchive/HTTP/entities/entities.cpp', 28 | 'ScpArchive/HTTP/HtmlEntity.cpp', 29 | 'ScpArchive/HTTP/MarkupOutStream.cpp', 30 | 'ScpArchive/Parser/DatabaseUtil.cpp', 31 | 'ScpArchive/Parser/HTMLConverter.cpp', 32 | 'ScpArchive/Parser/Parser.cpp', 33 | 'ScpArchive/Parser/Rules/RuleSet.cpp', 34 | 'ScpArchive/Parser/Rules/RuleSets/AlignRuleSet.cpp', 35 | 'ScpArchive/Parser/Rules/RuleSets/AnchorRuleSet.cpp', 36 | 'ScpArchive/Parser/Rules/RuleSets/ARuleSet.cpp', 37 | 'ScpArchive/Parser/Rules/RuleSets/BasicTextRuleSet.cpp', 38 | 'ScpArchive/Parser/Rules/RuleSets/CenterTextRuleSet.cpp', 39 | 'ScpArchive/Parser/Rules/RuleSets/CodeRuleSet.cpp', 40 | 'ScpArchive/Parser/Rules/RuleSets/CollapsibleRuleSet.cpp', 41 | 'ScpArchive/Parser/Rules/RuleSets/CommentRuleSet.cpp', 42 | 'ScpArchive/Parser/Rules/RuleSets/CSSRuleSet.cpp', 43 | 'ScpArchive/Parser/Rules/RuleSets/DividerRuleSet.cpp', 44 | 'ScpArchive/Parser/Rules/RuleSets/DivRuleSet.cpp', 45 | 'ScpArchive/Parser/Rules/RuleSets/EntityEscapeRuleSet.cpp', 46 | 'ScpArchive/Parser/Rules/RuleSets/FootNoteRuleSet.cpp', 47 | 'ScpArchive/Parser/Rules/RuleSets/ForumRuleSet.cpp', 48 | 'ScpArchive/Parser/Rules/RuleSets/HeadingRuleSet.cpp', 49 | 'ScpArchive/Parser/Rules/RuleSets/HTMLRuleSet.cpp', 50 | 'ScpArchive/Parser/Rules/RuleSets/HyperLinkRuleSet.cpp', 51 | 'ScpArchive/Parser/Rules/RuleSets/IFrameRuleSet.cpp', 52 | 'ScpArchive/Parser/Rules/RuleSets/IftagsRuleSet.cpp', 53 | 'ScpArchive/Parser/Rules/RuleSets/ImageRuleSet.cpp', 54 | 'ScpArchive/Parser/Rules/RuleSets/IncludeRuleSet.cpp', 55 | 'ScpArchive/Parser/Rules/RuleSets/InlineFormatRuleSet.cpp', 56 | 'ScpArchive/Parser/Rules/RuleSets/ListPagesRuleSet.cpp', 57 | 'ScpArchive/Parser/Rules/RuleSets/ListRuleSet.cpp', 58 | 'ScpArchive/Parser/Rules/RuleSets/LiteralTextRuleSet.cpp', 59 | 'ScpArchive/Parser/Rules/RuleSets/NullRuleSet.cpp', 60 | 'ScpArchive/Parser/Rules/RuleSets/QuoteBoxRuleSet.cpp', 61 | 'ScpArchive/Parser/Rules/RuleSets/RateRuleSet.cpp', 62 | 'ScpArchive/Parser/Rules/RuleSets/RedirectRuleSet.cpp', 63 | 'ScpArchive/Parser/Rules/RuleSets/SCPConversionProjectInfoBoxRuleSet.cpp', 64 | 'ScpArchive/Parser/Rules/RuleSets/SectionRuleSet.cpp', 65 | 'ScpArchive/Parser/Rules/RuleSets/SizeRuleSet.cpp', 66 | 'ScpArchive/Parser/Rules/RuleSets/SpanRuleSet.cpp', 67 | 'ScpArchive/Parser/Rules/RuleSets/TableOfContentsRuleSet.cpp', 68 | 'ScpArchive/Parser/Rules/RuleSets/TableRuleSet.cpp', 69 | 'ScpArchive/Parser/Rules/RuleSets/TabViewRuleSet.cpp', 70 | 'ScpArchive/Parser/Rules/RuleSets/UserRuleSet.cpp', 71 | 'ScpArchive/Parser/Templater.cpp', 72 | 'ScpArchive/Parser/Treer.cpp', 73 | 'ScpArchive/PDF/PDFConverter.cpp', 74 | 'ScpArchive/Tests/AutomatedTester.cpp', 75 | 'ScpArchive/Tests/Database/DatabaseTests.cpp', 76 | 'ScpArchive/Tests/Database/ImporterTests.cpp', 77 | 'ScpArchive/Tests/Database/JsonTests.cpp', 78 | 'ScpArchive/Tests/HTTP/HtmlEntityTests.cpp', 79 | 'ScpArchive/Tests/HTTP/MarkupOutStreamTests.cpp', 80 | 'ScpArchive/Tests/Parser/HTMLConverterTests.cpp', 81 | 'ScpArchive/Tests/Parser/ParserTests.cpp', 82 | 'ScpArchive/Tests/Parser/TemplaterTests.cpp', 83 | 'ScpArchive/Tests/Parser/TreerTests.cpp', 84 | 'ScpArchive/Tests/Tests.cpp', 85 | 'ScpArchive/UntestedUtils/UntestedUtils.cpp', 86 | 'ScpArchive/Website/Gateway.cpp', 87 | 'ScpArchive/Website/Website.cpp' 88 | ] 89 | 90 | exe = executable( 91 | 'ScpArchive', 92 | sources : src, 93 | dependencies : dep, 94 | include_directories : inc, 95 | install : true, 96 | override_options : ['cpp_std=c++17'] 97 | ) 98 | 99 | #test('ScpArchive Automated Test', exe, args : ['--runTests']) 100 | 101 | -------------------------------------------------------------------------------- /src/ScpScraper/ScpScraper/Scraper/Helpers.cpp: -------------------------------------------------------------------------------- 1 | #include "Helpers.hpp" 2 | 3 | #include 4 | 5 | nlohmann::json loadJsonFromFile(std::string fileName){ 6 | std::ifstream file(fileName); 7 | std::vector fileContents((std::istreambuf_iterator(file)), std::istreambuf_iterator()); 8 | std::string fileContentsStr(fileContents.begin(), fileContents.end()); 9 | nlohmann::json data = nlohmann::json::parse(fileContentsStr); 10 | return data; 11 | } 12 | 13 | void saveJsonToFile(std::string fileName, nlohmann::json data){ 14 | std::ofstream(fileName) << data.dump(4); 15 | } 16 | 17 | std::size_t getData(const std::string& data, std::string start, std::string end, std::size_t findPos, std::string& output){ 18 | std::size_t startPos = data.find(start, findPos); 19 | if(startPos == std::string::npos){ 20 | return std::string::npos; 21 | } 22 | startPos += start.size(); 23 | std::size_t endPos = data.find(end, startPos); 24 | if(endPos == std::string::npos){ 25 | output = data.substr(startPos, output.size() - 1 - startPos); 26 | return std::string::npos; 27 | } 28 | output = data.substr(startPos, endPos - startPos); 29 | return endPos + end.size(); 30 | } 31 | 32 | std::string& trimLeft(std::string& s) { 33 | s.erase(s.begin(), std::find_if(s.begin(), s.end(), std::not1(std::ptr_fun(std::isspace)))); 34 | return s; 35 | } 36 | // trim from end 37 | std::string& trimRight(std::string& s) { 38 | s.erase(std::find_if(s.rbegin(), s.rend(), std::not1(std::ptr_fun(std::isspace))).base(), s.end()); 39 | return s; 40 | } 41 | // trim from both ends 42 | std::string& trimString(std::string& s) { 43 | return trimLeft(trimRight(s)); 44 | } 45 | 46 | void replaceAll(std::string& str, const std::string& from, const std::string& to) { 47 | if(from.empty()) 48 | return; 49 | std::size_t start_pos = 0; 50 | while((start_pos = str.find(from, start_pos)) != std::string::npos) { 51 | str.replace(start_pos, from.length(), to); 52 | start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx' 53 | } 54 | } 55 | 56 | std::string percentDecode(const std::string& SRC){//works with utf8 57 | std::string ret; 58 | char ch; 59 | unsigned int i, ii; 60 | for (i = 0; i < SRC.length(); i++) { 61 | if (int(SRC[i]) == 37) { 62 | sscanf(SRC.substr(i + 1, 2).c_str(), "%x", &ii); 63 | ch=static_cast(ii); 64 | ret+=ch; 65 | i=i+2; 66 | } else { 67 | ret+=SRC[i]; 68 | } 69 | } 70 | return ret; 71 | } 72 | 73 | std::string percentEncode(const std::string &value) {//works with utf8 74 | std::ostringstream escaped; 75 | escaped.fill('0'); 76 | escaped << std::hex; 77 | 78 | for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) { 79 | std::string::value_type c = (*i); 80 | 81 | // Keep alphanumeric and other accepted characters intact 82 | if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') { 83 | escaped << c; 84 | continue; 85 | } 86 | 87 | // Any other characters are percent-encoded 88 | escaped << std::uppercase; 89 | escaped << '%' << std::setw(2) << int((unsigned char) c); 90 | escaped << std::nouppercase; 91 | } 92 | 93 | return escaped.str(); 94 | } 95 | 96 | -------------------------------------------------------------------------------- /src/ScpScraper/ScpScraper/Scraper/Helpers.hpp: -------------------------------------------------------------------------------- 1 | #ifndef HELPERS_HPP 2 | #define HELPERS_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | nlohmann::json loadJsonFromFile(std::string fileName); 11 | void saveJsonToFile(std::string fileName, nlohmann::json data); 12 | 13 | std::size_t getData(const std::string& data, std::string start, std::string end, std::size_t findPos, std::string& output); 14 | 15 | std::string& trimLeft(std::string& s); 16 | std::string& trimRight(std::string& s); 17 | std::string& trimString(std::string& s); 18 | 19 | void replaceAll(std::string& str, const std::string& from, const std::string& to); 20 | 21 | std::string percentDecode(const std::string& SRC); 22 | std::string percentEncode(const std::string &value); 23 | 24 | #endif // HELPERS_HPP 25 | -------------------------------------------------------------------------------- /src/ScpScraper/ScpScraper/Scraper/Scraper.hpp: -------------------------------------------------------------------------------- 1 | #ifndef SCAPER_HPP 2 | #define SCAPER_HPP 3 | 4 | #include 5 | #include 6 | 7 | #include 8 | 9 | #include "Helpers.hpp" 10 | 11 | namespace Scraper{ 12 | std::string generateInitialBatch(std::string batchesFolder, std::string batchDataFile); 13 | std::string generateDiffBatch(std::string batchesFolder, std::string batchDataFile); 14 | 15 | void downloadBatchData(std::string batchFolder); 16 | void checkBatchDownloads(std::string batchesFolder, std::string batchDataFile, std::string batchFolder); 17 | void checkAllBatchDownloads(std::string batchesFolder, std::string batchDataFile); 18 | void checkLastBatchDownload(std::string batchesFolder, std::string batchDataFile); 19 | 20 | std::vector getFullPageList(); 21 | std::vector getUpdatedPageList(std::int64_t startTime); 22 | 23 | void downloadPageList(std::string pagesFolder, std::vector pageList); 24 | void downloadFullPageArchive(std::string pagesFolder, std::string pageName); 25 | 26 | bool pageExists(std::string pageName); 27 | std::string getPageId(std::string pageName); 28 | std::string getPageParent(std::string pageName); 29 | std::string getPageDiscussionId(std::string pageName); 30 | nlohmann::json getPageRevisions(std::string pageId); 31 | nlohmann::json getRevisionData(std::string revisionId); 32 | std::string getRevisionSource(std::string revisionId); 33 | nlohmann::json getPageFiles(std::string pageId); 34 | nlohmann::json getFileData(std::string fileId); 35 | nlohmann::json getPageTags(std::string pageName); 36 | nlohmann::json getPageVotes(std::string pageId); 37 | 38 | void downloadImage(std::string fileName, std::string fileUrl); 39 | 40 | nlohmann::json getForumGroups(); 41 | std::vector getThreadListForForumCategory(std::string categoryId); 42 | std::vector getThreadListForAllCategories(nlohmann::json forumGroups); 43 | std::vector getUpdatedThreadList(std::int64_t startTime); 44 | 45 | bool threadExists(std::string threadId); 46 | void downloadThreadList(std::string threadsFolder, std::vector threadList); 47 | void downloadFullThreadArchive(std::string threadsFolder, std::string threadId); 48 | nlohmann::json getThreadPostReplies(const nlohmann::json& divTree, int offset); 49 | nlohmann::json getThreadPost(const nlohmann::json& divTree); 50 | 51 | std::vector getAuthorList(std::string pagesFolder, std::vector pageList, std::string threadsFolder, std::vector threadList); 52 | void downloadAuthorList(std::string authorsFile, std::vector authorList); 53 | nlohmann::json getAuthorData(std::string authorId); 54 | } 55 | 56 | #endif // SCAPER_HPP 57 | -------------------------------------------------------------------------------- /src/ScpScraper/ScpScraper/Scraper/entities/Entites.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ENTITIES_HPP 2 | #define ENTITIES_HPP 3 | 4 | extern "C"{ 5 | #include "entities.h" 6 | } 7 | 8 | static std::string decodeHtmlEntities(std::string input){ 9 | char* outputBuffer = new char[input.size() + 1]; 10 | std::size_t outputSize = decode_html_entities_utf8(outputBuffer, input.c_str()); 11 | std::string output{outputBuffer, outputSize}; 12 | return output; 13 | } 14 | 15 | #endif // ENTITIES_HPP 16 | -------------------------------------------------------------------------------- /src/ScpScraper/ScpScraper/Scraper/entities/entities.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2012 Christoph Gärtner 2 | Distributed under the Boost Software License, Version 1.0 3 | */ 4 | 5 | #ifndef DECODE_HTML_ENTITIES_UTF8_ 6 | #define DECODE_HTML_ENTITIES_UTF8_ 7 | 8 | #include 9 | 10 | extern size_t decode_html_entities_utf8(char *dest, const char *src); 11 | /* Takes input from and decodes into , which should be a buffer 12 | large enough to hold characters. 13 | 14 | If is , input will be taken from , decoding 15 | the entities in-place. 16 | 17 | The function returns the length of the decoded string. 18 | */ 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /src/ScpScraper/ScpScraper/main.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "Scraper/Scraper.hpp" 3 | 4 | #include 5 | #include 6 | 7 | int main(int argc, char** argv){ 8 | std::string dataFolder = "/scp_conversion_project/batchArchive/"; 9 | std::string batchDataFile = dataFolder + "batchData.json"; 10 | std::string batchesFolder = dataFolder + "batches/"; 11 | 12 | if(argc == 2 && std::string(argv[1]) == "--makeInitialBatch"){ 13 | std::string batchId = Scraper::generateInitialBatch(batchesFolder, batchDataFile); 14 | Scraper::downloadBatchData(batchesFolder + batchId + "/"); 15 | } 16 | else if(argc == 2 && std::string(argv[1]) == "--makeDiffBatch"){ 17 | std::string batchId = Scraper::generateDiffBatch(batchesFolder, batchDataFile); 18 | if(batchId == ""){ 19 | std::cout << "Aborting new batch.\n"; 20 | } 21 | else{ 22 | Scraper::downloadBatchData(batchesFolder + batchId + "/"); 23 | Scraper::checkBatchDownloads(batchesFolder, batchDataFile, batchesFolder + batchId + "/"); 24 | } 25 | } 26 | else if(argc == 3 && std::string(argv[1]) == "--checkBatchDownloads"){ 27 | Scraper::checkBatchDownloads(batchesFolder, batchDataFile, argv[2]); 28 | } 29 | else if(argc == 2 && std::string(argv[1]) == "--checkAllBatchDownloads"){ 30 | Scraper::checkAllBatchDownloads(batchesFolder, batchDataFile); 31 | } 32 | else if(argc == 2 && std::string(argv[1]) == "--checkLastBatchDownload"){ 33 | Scraper::checkLastBatchDownload(batchesFolder, batchDataFile); 34 | } 35 | else{ 36 | std::cout << "Invalid/No Options Specified\n"; 37 | } 38 | 39 | return 0; 40 | } 41 | -------------------------------------------------------------------------------- /src/ScpScraper/meson.build: -------------------------------------------------------------------------------- 1 | project('ScpScraper', ['cpp', 'c']) 2 | 3 | cpp = meson.get_compiler('cpp') 4 | 5 | dep = [ 6 | dependency('boost', modules : ['filesystem']), 7 | dependency('nlohmann_json'), 8 | dependency('threads'), 9 | cpp.find_library('ssl'), 10 | cpp.find_library('crypto'), 11 | cpp.find_library('curlpp'), 12 | cpp.find_library('curl') 13 | ] 14 | 15 | inc = [ 16 | '.' 17 | ] 18 | 19 | src = [ 20 | 'ScpScraper/main.cpp', 21 | 'ScpScraper/Scraper/Scraper.cpp', 22 | 'ScpScraper/Scraper/Helpers.cpp', 23 | 'ScpScraper/Scraper/entities/entities.c', 24 | ] 25 | 26 | executable( 27 | 'ScpScraper', 28 | sources : src, 29 | dependencies : dep, 30 | include_directories : inc, 31 | install : true, 32 | override_options : ['cpp_std=c++17'] 33 | ) 34 | 35 | -------------------------------------------------------------------------------- /src/dataLayout.md: -------------------------------------------------------------------------------- 1 | 2 | # PDF File Map: 3 | - pdfData.json 4 | - collection 5 | - .pdf 6 | - ... 7 | - temp 8 | - ... 9 | # PDF File Layouts: 10 | ## pdfData.json 11 | - pages 12 | - 0 13 | - pageName: 14 | - revisionIndex: 15 | - ... 16 | 17 | # MySQL Database Map: 18 | full table definitions can be found in src/ScpArchive/Database/Database.cpp 19 | 20 | # Scraper Data File Map: 21 | - batchData.json 22 | - authors.json 23 | - batches 24 | - 25 | - batch.json 26 | - pages 27 | - 28 | - data.json 29 | - files 30 | - ... 31 | - ... 32 | - threads 33 | - 34 | - data.json 35 | - ... 36 | - ... 37 | - threads 38 | - 39 | - data.json 40 | - ... 41 | 42 | # Scraper Data File Layouts: 43 | ## batchData.json 44 | - availableBatches 45 | - 0: 46 | - appliedBatches 47 | - 0: 48 | - ... 49 | - batchErrors 50 | - 0 51 | - error: 52 | - batchId: 53 | - ... 54 | 55 | ## batches//batch.json 56 | ### initial-type 57 | used for initial database creation, generated by ScpScraper 58 | - type: "initial" 59 | - timeStamp: 60 | - pageList 61 | - 0: 62 | - ... 63 | - forumGroups 64 | - 0 65 | - title: 66 | - description: 67 | - categories 68 | - 0 69 | - id: 70 | - title: 71 | - description: 72 | - ... 73 | - ... 74 | - threadList 75 | - 0: 76 | - ... 77 | 78 | ### diff-type 79 | used to keep the database up to date with the actual wiki, generated by ScpScraper 80 | - type: "diff" 81 | - timeStamp: 82 | - pageList 83 | - 0: 84 | - ... 85 | - threadList 86 | - 0: 87 | - ... 88 | 89 | ### user-type 90 | used by a user attempted to insert special data into the thread that is not on the actual wiki 91 | notice the lack of timestamp, as this does not have an impact on the timeline of the batch diffs 92 | - type: "user" 93 | - pageList 94 | - 0: 95 | - ... 96 | - threadList 97 | - 0: 98 | - ... 99 | 100 | ## pages//data.json when page does not exist 101 | - nonExistent: true 102 | 103 | ## pages//data.json 104 | - id: 105 | - name: 106 | - parent: ("" if no parent) 107 | - discussionId: ("" if no discussion) 108 | - tags 109 | - 0: 110 | - ... 111 | - votes 112 | - 0 113 | - authorId: ("deleted" if account is deleted) 114 | - voteType: (true if upvote, false if downvote) 115 | - ... 116 | - files 117 | - 0 118 | - id: 119 | - name: 120 | - description: 121 | - url: 122 | - timeStamp: 123 | - authorId: ("deleted" if account is deleted) 124 | - ... 125 | - revisions 126 | - 0 127 | - id: 128 | - title: 129 | - timeStamp: 130 | - authorId: ("deleted" if account is deleted) 131 | - changeMessage: 132 | - changeType: (possible values are defined by Wikidot) 133 | - sourceCode: 134 | - ... 135 | 136 | 137 | ## threads//data.json when thread does not exist 138 | - nonExistent: true 139 | 140 | ## threads//data.json 141 | - id: 142 | - categoryId: 143 | - authorId: ("deleted" if the account is deleted, "wikidot" if created by wikidot) 144 | - timeStamp: 145 | - title: 146 | - description: 147 | - posts: 148 | - 0 149 | - postId: 150 | - authorId: ("deleted" if account is deleted) 151 | - timeStamp: 152 | - title: 153 | - content: 154 | - posts: 155 | - 0: ... 156 | - ... 157 | - ... 158 | 159 | ## authors.json 160 | - 0 161 | - id: 162 | - name: 163 | - ... 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | --------------------------------------------------------------------------------