├── .gitattributes
├── .gitignore
├── LICENSE
├── README.md
├── arbital.lisp
├── color-scheme-convert.php
├── color-usage-stats.php
├── compute_FontAwesome_subset.php
├── config-example.lisp
├── data
├── bigrams.lisp
├── hyphenation-patterns
│ └── hyph_en_US.txt
└── webcolors.json
├── html-clean-regexps.js
├── js-foreign-lib
├── .gitignore
├── convert.js
├── mathjax.js
├── package-lock.json
├── package.json
└── web-push.js
├── logs
└── .gitignore
├── lw2-viewer.asd
├── lw2.lisp
├── pre-commit.hook
├── rts-content
├── A-Technical-Explanation-Of-Technical-Explanation
├── An-Intuitive-Explanation-Of-Bayess-Theorem
├── The-Twelve-Virtues-Of-Rationality
└── rts.css
├── src
├── admin.lisp
├── backend-modules.lisp
├── backend.lisp
├── backends
│ └── accordius.lisp
├── background-loader.lisp
├── backlinks.lisp
├── clean-html.lisp
├── client-script.lisp
├── colors.lisp
├── comment-threads.lisp
├── components.lisp
├── conditions.lisp
├── config-package.lisp
├── context.lisp
├── csrf.lisp
├── data-viewers
│ ├── comment.lisp
│ ├── post.lisp
│ └── tag.lisp
├── dnsbl.lisp
├── elicit-predictions.lisp
├── fonts-modules.lisp
├── fonts.lisp
├── graphql.lisp
├── hash-utils.lisp
├── html-reader.lisp
├── images.lisp
├── interface-utils.lisp
├── legacy-archive.lisp
├── links.lisp
├── lmdb.lisp
├── lw2-login.lisp
├── macro-utils.lisp
├── push-notifications.lisp
├── raw-memory-streams.lisp
├── resources.lisp
├── response.lisp
├── routes.lisp
├── rwlock.lisp
├── schema-type.lisp
├── sites.lisp
├── user-context.lisp
├── utils.lisp
└── web-push.lisp
├── templates
├── conversation.html
├── edit-post.html
└── reset-password.html
├── text-clean-regexps.js
└── www
├── about.html
├── accordius
├── style.css.php
├── theme-brutalist.css.php
├── theme-classic.css.php
├── theme-default.css.php
├── theme-grey.css.php
├── theme-less.css.php
├── theme-rts.css.php
├── theme-ultramodern.css.php
└── theme-zero.css.php
├── arbital.css
├── assets
├── A.gif
├── A_minus.gif
├── A_plus.gif
├── about-page-images
│ ├── archive-browser.png
│ ├── comments-view-selector.png
│ ├── compact-comments-list-view.png
│ ├── customize-appearance.png
│ ├── expanded-comment-in-compact-view.png
│ ├── fixed-width-narrow.png
│ ├── fixed-width-wide.png
│ ├── fluid-width.png
│ ├── markdown-editor.png
│ ├── new-comments-quick-nav.png
│ ├── next-new-comment-button.png
│ ├── previous-new-comment-button.png
│ ├── rss-feeds.png
│ ├── special-linking-options.png
│ ├── text-size-adjustment.png
│ ├── theme-selector.png
│ └── width-adjustment.png
├── anchor-blue-on-white.gif
├── anchor-white-on-blue.gif
├── basilisk.png
├── checkerboard2_1px.gif
├── checkerboard2_1px_gray.gif
├── checkerboard_1px.gif
├── compact_1x.gif
├── compact_2x.gif
├── expanded_1x.gif
├── expanded_2x.gif
├── favicon.ico
├── fluid.gif
├── lw-blue-on-white.gif
├── lw-white-on-blue.gif
├── markdown.png
├── minimize_button_icon.gif
├── normal.gif
├── one_pixel_DDD.gif
├── popup.svg
├── rss.svg
├── telegraph.jpg
├── theme_A.gif
├── theme_B.gif
├── theme_C.gif
├── theme_D.gif
├── theme_E.gif
├── theme_F.gif
├── theme_G.gif
├── theme_H.gif
├── up-arrow-blue-on-white.gif
├── up-arrow-white-on-blue.gif
├── vote_button_icons
│ ├── agree-black-square-check.svg
│ ├── agree-disabled-grey-circle-check.svg
│ ├── agree-disabled-grey-square-check.svg
│ ├── agree-green-circle-check.svg
│ ├── disagree-black-square-x.svg
│ ├── disagree-disabled-grey-circle-x.svg
│ ├── disagree-disabled-grey-square-x.svg
│ ├── disagree-red-circle-x.svg
│ ├── downvote-black-square-minus.svg
│ ├── downvote-disabled-grey-circle-minus.svg
│ ├── downvote-disabled-grey-square-minus.svg
│ ├── downvote-red-circle-minus.svg
│ ├── upvote-black-square-plus.svg
│ ├── upvote-disabled-grey-circle-plus.svg
│ ├── upvote-disabled-grey-square-plus.svg
│ └── upvote-green-circle-plus.svg
├── wide.gif
├── win95_checkmark.gif
├── win95_close_widget.gif
├── win95_help_icon.gif
├── win95_help_widget.gif
├── win95_maximize_widget.gif
├── win95_minimize_widget.gif
├── win95_scrollbar_track.gif
└── win95_themes_icon.gif
├── css
├── style-brutalist.linux.css
├── style-brutalist.mac.css
├── style-brutalist.windows.css
├── style-classic.linux.css
├── style-classic.mac.css
├── style-classic.windows.css
├── style-grey.linux.css
├── style-grey.mac.css
├── style-grey.windows.css
├── style-less.linux.css
├── style-less.mac.css
├── style-less.windows.css
├── style-rts.linux.css
├── style-rts.mac.css
├── style-rts.windows.css
├── style-ultramodern.linux.css
├── style-ultramodern.mac.css
├── style-ultramodern.windows.css
├── style-zero.linux.css
├── style-zero.mac.css
├── style-zero.windows.css
├── style.linux.css
├── style.mac.css
├── style.windows.css
└── theme_tweaker.css
├── ea
└── assets
│ ├── ea-blue-on-white.gif
│ ├── ea-white-on-blue.gif
│ └── favicon.ico
├── head.js
├── proxy-assets
└── .gitignore
├── script.js
├── service-worker.js
├── style.css.php
├── style_mobile_additions.css.php
├── theme-brutalist.css.php
├── theme-classic.css.php
├── theme-default.css.php
├── theme-grey.css.php
├── theme-less.css.php
├── theme-rts.css.php
├── theme-ultramodern.css.php
├── theme-zero.css.php
└── theme_tweaker.css.php
/.gitattributes:
--------------------------------------------------------------------------------
1 | * linguist-vendored=true
2 | *.css linguist-generated=true
3 | *.lisp linguist-vendored=false
4 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | cache*
2 | local
3 | .*
4 | *~
5 | *#
6 | post-commit.hook
7 | config.lisp
8 | webpush.vapid.key
9 | www/robots.txt
10 | www/fonts.css
11 | www/generated-css
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 kronusaturn
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # lw2-viewer
2 | An alternative frontend for [LessWrong 2.0](https://www.lesserwrong.com/), with a focus on speed and usability.
3 |
4 | * [Issue tracker](https://saturn.obormot.net/LW2Reader/Issues)
5 |
6 | ## Installation
7 | ### Linux/Unix
8 |
9 | *This is the bare minimum to get the server running. To set up a full Common Lisp development environment, see [here](http://lisp-lang.org/learn/getting-started/)*
10 |
11 | * Install `sbcl` and `lmdb` from apt or using your favorite method.
12 | e.g. `sudo apt install sbcl liblmdb0`
13 |
14 | * Install [quicklisp](https://beta.quicklisp.org/)
15 | `curl -O https://beta.quicklisp.org/quicklisp.lisp`
16 | `sbcl --load quicklisp.lisp --eval '(quicklisp-quickstart:install)' --eval '(ql:add-to-init-file)' --eval '(exit)'`
17 |
18 | * Clone required git repositories to the quicklisp `local-projects` directory. (You can also use symlinks if you want to put them elsewhere.)
19 | `cd ~/quicklisp/local-projects`
20 | `git clone https://github.com/kronusaturn/liblmdb.git`
21 | `git clone https://github.com/kronusaturn/lmdb.git`
22 | `git clone https://github.com/kronusaturn/plump.git`
23 | `git clone https://github.com/kronusaturn/lw2-viewer.git`
24 |
25 | * Start the server
26 | `cd ~/quicklisp/local-projects/lw2-viewer`
27 | `sbcl --eval '(ql:quickload :lw2-viewer)' --eval '(hunchentoot:start (make-instance (quote hunchentoot:easy-acceptor) :port 4242 :document-root "./www/"))'`
28 |
29 | * Open `http://localhost:4242` in your browser
30 |
31 | ### Mac
32 | Similar to above, but:
33 | * `sbcl` and `lmdb` are available from MacPorts.
34 |
35 | * Make sure to run system updates, using an outdated OS version can cause installation errors.
36 |
37 | * As Mac OS does not support sparse files, the system will create an empty 16 GB file on startup.
38 | If you want to reduce disk space usage, before starting the server, copy config-example.lisp to config.lisp and change the line `(defparameter *lmdb-mapsize* (expt 2 34))` to, for example, `(defparameter *lmdb-mapsize* (expt 2 28))`
39 |
40 | * You will need to set the `DYLD_LIBRARY_PATH` environment variable to the location where `liblmdb.dylib` is installed, for example `export DYLD_LIBRARY_PATH=/opt/local/lib/`
41 |
--------------------------------------------------------------------------------
/color-scheme-convert.php:
--------------------------------------------------------------------------------
1 | $v) {
31 | $rgba[$k] = round($v);
32 | }
33 | $rgba[] = $m[4];
34 |
35 | return "rgba(" . implode(", ", $rgba) . ")";
36 | }
37 |
38 | /***********/
39 | /* HELPERS */
40 | /***********/
41 |
42 | function debug_log($string) {
43 | global $debug_enabled;
44 | if ($debug_enabled)
45 | error_log($string);
46 | }
47 |
48 | /******************/
49 | /* TRANSFORMATION */
50 | /******************/
51 |
52 | ## CVT = "Color Value Transform"
53 | function CVT($value, $color_space) {
54 | global $mode;
55 | ## The mode is a bit field; set binary flags indicate specific transformations.
56 | ## Flags are checked, and applied, in order from lowest bit position to highest.
57 | ##
58 | ## 0x0001: lightness inversion (in Lab).
59 | ## 0x0002: hue inversion (in Lab).
60 | ##
61 | ## The following six flags are mutually exclusive:
62 | ##
63 | ## 0x0004: maps whites to reds (in HSV; keeps V constant, sets H to 0°, S to maximum)
64 | ## 0x0008: maps whites to yellows (in HSV; keeps V constant, sets H to 60°, S to maximum)
65 | ## 0x0010: maps whites to greens (in HSV; keeps V constant, sets H to 120°, S to maximum)
66 | ## 0x0020: maps whites to teal/turquoise (in HSV; keeps V constant, sets H to 180°, S to maximum)
67 | ## 0x0040: maps whites to blue (in HSV; keeps V constant, sets H to 240°, S to maximum)
68 | ## 0x0080: maps whites to magenta (in HSV; keeps V constant, sets H to 300°, S to maximum)
69 |
70 | if ($mode & 0x0001) {
71 | $value[0] = 100 - $value[0];
72 | }
73 | if ($mode & 0x0002) {
74 | $value[1] *= -1;
75 | $value[2] *= -1;
76 | }
77 |
78 | $hsv_value = HSVFromRGB(RGBFromXYZ(XYZFromLab($value)));
79 | if ($mode & 0x0004) {
80 | $hsv_value[0] = 0.0 / 360.0;
81 | $hsv_value[1] = 1.0;
82 | } else if ($mode & 0x0008) {
83 | $hsv_value[0] = 60.0 / 360.0;
84 | $hsv_value[1] = 1.0;
85 | } else if ($mode & 0x0010) {
86 | $hsv_value[0] = 120.0 / 360.0;
87 | $hsv_value[1] = 1.0;
88 | } else if ($mode & 0x0020) {
89 | $hsv_value[0] = 180.0 / 360.0;
90 | $hsv_value[1] = 1.0;
91 | } else if ($mode & 0x0040) {
92 | $hsv_value[0] = 240.0 / 360.0;
93 | $hsv_value[1] = 1.0;
94 | } else if ($mode & 0x0080) {
95 | $hsv_value[0] = 300.0 / 360.0;
96 | $hsv_value[1] = 1.0;
97 | }
98 | $value = LabFromXYZ(XYZFromRGB(RGBFromHSV($hsv_value)));
99 | debug_log(" → {$color_space} ".PCC($value));
100 |
101 | return $value;
102 | }
103 |
104 | /*********************/
105 | /* FORMAT CONVERSION */
106 | /*********************/
107 |
108 | function RGBFromHex($hexColorString) {
109 | if ($hexColorString[0] == '#')
110 | $hexColorString = substr($hexColorString,1);
111 | if (strlen($hexColorString) == 3)
112 | $hexColorString = preg_replace("/./","$0$0",$hexColorString);
113 | $components = str_split($hexColorString,2);
114 | foreach ($components as $i => $hexColor)
115 | $components[$i] = hexdec($hexColor);
116 | debug_log(" → RGB ".PCC($components));
117 | return $components;
118 | }
119 |
120 | function HexFromRGB($rgb_components) {
121 | foreach ($rgb_components as $i => $component) {
122 | $hex_value = dechex(round($component));
123 | if (strlen($hex_value) == 1)
124 | $hex_value = "0".$hex_value;
125 | $rgb_components[$i] = $hex_value;
126 | }
127 | $hexColorString = "#" . implode($rgb_components);
128 | $hexColorString = preg_replace("/([0-9abcdef])\\1([0-9abcdef])\\2([0-9abcdef])\\3/", "$1$2$3", $hexColorString);
129 | debug_log(" → ".$hexColorString);
130 | return $hexColorString;
131 | }
132 |
133 | ## PCC = "Print Color Components"
134 | function PCC($components) {
135 | foreach ($components as $k => $v) {
136 | $components[$k] = round($v, 2);
137 | }
138 | return "( " . implode(", ", $components) . " )";
139 | }
140 |
141 | /**************************/
142 | /* COLOR SPACE CONVERSION */
143 | /**************************/
144 |
145 | function HSVFromRGB($rgb_components) {
146 | $var_R = $rgb_components[0] / 255.0;
147 | $var_G = $rgb_components[1] / 255.0;
148 | $var_B = $rgb_components[2] / 255.0;
149 |
150 | $var_Min = min($var_R, $var_G, $var_B);
151 | $var_Max = max($var_R, $var_G, $var_B);
152 | $del_Max = $var_Max - $var_Min;
153 |
154 | $V = $var_Max;
155 | $H = 0;
156 | $S = 0;
157 |
158 | if ($del_Max != 0) {
159 | $S = $del_Max / $var_Max;
160 |
161 | $del_R = ((($var_Max - $var_R) / 6) + ($del_Max / 2)) / $del_Max;
162 | $del_G = ((($var_Max - $var_G) / 6) + ($del_Max / 2)) / $del_Max;
163 | $del_B = ((($var_Max - $var_B) / 6) + ($del_Max / 2)) / $del_Max;
164 |
165 | if ($var_R == $var_Max) $H = $del_B - $del_G;
166 | else if ($var_G == $var_Max) $H = (1.0/3.0) + $del_R - $del_B;
167 | else if ($var_B == $var_Max) $H = (2.0/3.0) + $del_G - $del_R;
168 |
169 | if ($H < 0) $H += 1;
170 | else if ($H > 1) $H -= 1;
171 | }
172 |
173 | debug_log(" → HSV ".PCC([ $H, $S, $V ]));
174 | return [ $H, $S, $V ];
175 | }
176 |
177 | function RGBFromHSV($hsv_components) {
178 | $H = $hsv_components[0];
179 | $S = $hsv_components[1];
180 | $V = $hsv_components[2];
181 |
182 | $R = $G = $B = $V * 255.0;
183 |
184 | if ($S != 0) {
185 | $var_h = $H * 6.0;
186 | if ($var_h == 6.0)
187 | $var_h = 0;
188 | $var_i = floor($var_h);
189 | $var_1 = $V * (1 - $S);
190 | $var_2 = $V * (1 - $S * ($var_h - $var_i));
191 | $var_3 = $V * (1 - $S * (1 - ($var_h - $var_i)));
192 |
193 | $var_r = $var_g = $var_b = 0.0;
194 |
195 | if ($var_i == 0) { $var_r = $V; $var_g = $var_3; $var_b = $var_1; }
196 | else if ($var_i == 1) { $var_r = $var_2; $var_g = $V; $var_b = $var_1; }
197 | else if ($var_i == 2) { $var_r = $var_1; $var_g = $V; $var_b = $var_3; }
198 | else if ($var_i == 3) { $var_r = $var_1; $var_g = $var_2; $var_b = $V; }
199 | else if ($var_i == 4) { $var_r = $var_3; $var_g = $var_1; $var_b = $V ; }
200 | else { $var_r = $V; $var_g = $var_1 ; $var_b = $var_2; }
201 |
202 | $R = $var_r * 255.0;
203 | $G = $var_g * 255.0;
204 | $B = $var_b * 255.0;
205 | }
206 |
207 | debug_log(" → RGB ".PCC([ $R, $G, $B ]));
208 | return [ $R, $G, $B ];
209 | }
210 |
211 | function XYZFromRGB($rgb_components) {
212 | foreach ($rgb_components as $i => $component) {
213 | $component /= 255.0;
214 | $rgb_components[$i] = ($component > 0.04045) ?
215 | (pow((($component + 0.055) / 1.055), 2.4)) :
216 | ($component / 12.92);
217 | }
218 |
219 | $var_R = $rgb_components[0] * 100.0;
220 | $var_G = $rgb_components[1] * 100.0;
221 | $var_B = $rgb_components[2] * 100.0;
222 |
223 | $X = $var_R * 0.4124 + $var_G * 0.3576 + $var_B * 0.1805;
224 | $Y = $var_R * 0.2126 + $var_G * 0.7152 + $var_B * 0.0722;
225 | $Z = $var_R * 0.0193 + $var_G * 0.1192 + $var_B * 0.9505;
226 |
227 | debug_log(" → XYZ ".PCC([ $X, $Y, $Z ]));
228 | return [ $X, $Y, $Z ];
229 | }
230 |
231 | function LabFromXYZ($xyz_components) {
232 | $xyz_components[0] /= 95.047;
233 | $xyz_components[1] /= 100.000;
234 | $xyz_components[2] /= 108.883;
235 |
236 | foreach ($xyz_components as $i => $component) {
237 | $xyz_components[$i] = ($component > 0.008856) ?
238 | (pow($component, (1.0/3.0))) :
239 | ((7.787 * $component) + (16.0/116.0));
240 | }
241 |
242 | $var_X = $xyz_components[0];
243 | $var_Y = $xyz_components[1];
244 | $var_Z = $xyz_components[2];
245 |
246 | $L = (116.0 * $var_Y) - 16.0;
247 | $a = 500.0 * ($var_X - $var_Y);
248 | $b = 200.0 * ($var_Y - $var_Z);
249 |
250 | debug_log(" → Lab ".PCC([ $L, $a, $b ]));
251 | return [ $L, $a, $b ];
252 | }
253 |
254 | function XYZFromLab($lab_components) {
255 |
256 | $var_Y = ($lab_components[0] + 16.0) / 116.0;
257 | $var_X = $lab_components[1] / 500.0 + $var_Y;
258 | $var_Z = $var_Y - $lab_components[2] / 200.0;
259 | $xyz_components = [ $var_X, $var_Y, $var_Z ];
260 |
261 | foreach ($xyz_components as $i => $component) {
262 | $xyz_components[$i] = (pow($component, 3) > 0.008856) ?
263 | (pow($component, 3)) :
264 | (($component - 16.0/116.0) / 7.787);
265 | }
266 |
267 | $xyz_components[0] *= 95.047;
268 | $xyz_components[1] *= 100.000;
269 | $xyz_components[2] *= 108.883;
270 |
271 | debug_log(" → XYZ ".PCC($xyz_components));
272 | return $xyz_components;
273 | }
274 |
275 | function RGBFromXYZ($xyz_components) {
276 | $var_X = $xyz_components[0] / 100.0;
277 | $var_Y = $xyz_components[1] / 100.0;
278 | $var_Z = $xyz_components[2] / 100.0;
279 |
280 | $var_R = $var_X * 3.2406 + $var_Y * -1.5372 + $var_Z * -0.4986;
281 | $var_G = $var_X * -0.9689 + $var_Y * 1.8758 + $var_Z * 0.0415;
282 | $var_B = $var_X * 0.0557 + $var_Y * -0.2040 + $var_Z * 1.0570;
283 |
284 | $rgb_components = [ $var_R, $var_G, $var_B ];
285 | foreach ($rgb_components as $i => $component) {
286 | $component = ($component > 0.0031308) ?
287 | (1.055 * pow($component, (1.0/2.4)) - 0.055) :
288 | (12.92 * $component);
289 | $rgb_components[$i] = min(max($component, 0.0), 1.0) * 255.0;
290 | }
291 |
292 | debug_log(" → RGB ".PCC($rgb_components));
293 | return $rgb_components;
294 | }
295 |
296 | ?>
--------------------------------------------------------------------------------
/color-usage-stats.php:
--------------------------------------------------------------------------------
1 |
--------------------------------------------------------------------------------
/compute_FontAwesome_subset.php:
--------------------------------------------------------------------------------
1 | $value) {
57 | $characters[$key] = strtoupper($value);
58 | }
59 | $characters = array_unique($characters);
60 | sort($characters);
61 | echo implode(",",$characters);
62 | echo "\n";
63 |
64 | ?>
--------------------------------------------------------------------------------
/config-example.lisp:
--------------------------------------------------------------------------------
1 | (in-package #:lw2-viewer.config)
2 |
3 | (reset-site-definitions)
4 |
5 | (define-site
6 | :uri "http://localhost:4242/"
7 | :title "Example Site"
8 | :description "This is an example."
9 | :class lesswrong-viewer-site
10 | :main-site-title "LessWrong"
11 | :main-site-abbreviation "LW"
12 | :main-site-uri "https://www.lesswrong.com/"
13 |
14 | ; The following line will enable use of fonts.obormot.net.
15 | ; This will not work unless you have permission to use these fonts.
16 |
17 | ; :fonts-source (make-instance 'obormot-fonts-source)
18 |
19 | :backend ("lw2" ; Supported backends: lw2 ea-forum lw2-legacy arbital accordius
20 | :graphql-uri "https://www.lesswrong.com/graphql"
21 | ;:magnum-crosspost-site "ea.example.com" ; set this to the hostname of another defined site to enable crosspost retrieval.
22 |
23 | ; Uncomment the following for EA Forum OAuth 2.0 support.
24 | ;:oauth2.0-login-uri "https://login.effectivealtruism.org/"
25 | ;:oauth2.0-client-id "foo"
26 | ;:oauth2.0-client-secret "bar"
27 |
28 | :algolia-search-uri "https://www.lesswrong.com/api/search"
29 | :cache-db-path "./cache/")) ; Location of the cache database. Be sure to include the trailing slash.
30 |
31 | ; (You can add more than one define-site directive.)
32 |
33 | ; Maximum size of the cache database.
34 | ; On platforms that don't support sparse files, you may want to reduce this
35 | ; to conserve disk space. Default is 2^34 or 16GB.
36 | (defparameter *lmdb-mapsize* (expt 2 34))
37 |
38 | ; List of DNSBLs to check before allowing users to log in.
39 | ;(defparameter *dnsbl-list* (list "dnsbl.example.com"))
40 |
41 | ; List of extra resources to include on every page on every site.
42 | ;(defparameter *html-global-resources* '())
43 |
44 | ; Limit how many requests to handle in parallel, as a last ditch rate limit.
45 | (defparameter *max-requests-in-progress* 16)
46 |
--------------------------------------------------------------------------------
/html-clean-regexps.js:
--------------------------------------------------------------------------------
1 | [
2 | // fractions
3 | [/(^| )(?!9\/11)([0-9,]+(?:,[0-9]+)*)\/(?![12][0-9]{3})([0-9]+(?:,[0-9]+)*)(?=\s|[])}.,;"']|$)/ug, '$1$2⁄$3'],
4 | [/(^|[^\u0336])([^\u0336])\u0336/ug, '$1$2'],
5 | [/\u0336([^\u0336])([^\u0336]|$)/ug, '$1$2'],
6 | [/([^\u0336])\u0336/ug, '$1']
7 | ]
8 |
--------------------------------------------------------------------------------
/js-foreign-lib/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules/
2 |
--------------------------------------------------------------------------------
/js-foreign-lib/convert.js:
--------------------------------------------------------------------------------
1 | var showdown = require('showdown'),
2 | converter = new showdown.Converter(),
3 | markdown = '';
4 |
5 | process.stdin.on('data', (chunk) => { markdown += chunk });
6 |
7 | process.stdin.on('end', () => {
8 | process.stdout.write(converter.makeHtml(markdown));
9 | });
10 |
--------------------------------------------------------------------------------
/js-foreign-lib/mathjax.js:
--------------------------------------------------------------------------------
1 | var mathjax = require('mathjax-node'),
2 | chunks = [],
3 | size = null;
4 |
5 | mathjax.config({ MathJax: { loader: {load: ['ui/safe']},
6 | extensions: ["Safe.js"] } });
7 |
8 | process.stdin.on('data', (chunk) => {
9 | if(size === null) {
10 | size = chunk.readUInt32LE();
11 | onDataChunk(chunk.subarray(4));
12 | } else {
13 | onDataChunk(chunk);
14 | }
15 | });
16 |
17 | function onDataChunk(chunk) {
18 | if(chunk.length > size) {
19 | var excessChunk = chunk.subarray(size);
20 | process.stdin.unshift(excessChunk);
21 | chunk = chunk.subarray(0, size);
22 | }
23 |
24 | chunks.push(chunk);
25 |
26 | if(chunk.length == size) {
27 | onMessage(Buffer.concat(chunks).toString());
28 | size = null;
29 | chunks = [];
30 | }
31 | else {
32 | size -= remainingChunk.length;
33 | }
34 | }
35 |
36 | function onMessage(input) {
37 | mathjax.typeset(
38 | { math: input,
39 | format: "inline-TeX",
40 | html: true,
41 | css: true
42 | },
43 | (data) => {
44 | var sizeBuf = Buffer.alloc(4);
45 | if(data.errors) {
46 | sizeBuf.writeUInt32LE(0);
47 | process.stdout.write(sizeBuf);
48 | }
49 | else {
50 | var dataBuf = Buffer.from(""+data.html);
51 | sizeBuf.writeUInt32LE(dataBuf.length);
52 | process.stdout.write(sizeBuf);
53 | process.stdout.write(dataBuf);
54 | }
55 | }
56 | );
57 | }
58 |
--------------------------------------------------------------------------------
/js-foreign-lib/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "dependencies": {
3 | "mathjax-node": "^2.1.1",
4 | "showdown": "^1.9.1",
5 | "web-push": "^3.4.5"
6 | }
7 | }
8 |
--------------------------------------------------------------------------------
/js-foreign-lib/web-push.js:
--------------------------------------------------------------------------------
1 | const webPush = require('web-push');
2 | var inputText = "";
3 |
4 | process.stdin.on('data', (chunk) => { inputText += chunk });
5 |
6 | process.stdin.on('end', () => {
7 | process.stdout.write(JSON.stringify(eval(inputText)));
8 | });
9 |
--------------------------------------------------------------------------------
/logs/.gitignore:
--------------------------------------------------------------------------------
1 | *
2 | !.gitignore
3 |
--------------------------------------------------------------------------------
/lw2-viewer.asd:
--------------------------------------------------------------------------------
1 | (in-package :asdf)
2 |
3 | (asdf:defsystem :lw2-viewer
4 | :depends-on ("uiop" "flexi-streams" "hunchentoot" "dexador" "cl-json" "lmdb" "local-time" "plump" "clss" "cl-ppcre" "xml-emitter" "city-hash" "bit-smasher" "cl-unicode" "parse-js" "cl-markdown" "websocket-driver-client" "ironclad" "cl-base64" "djula" "split-sequence" "cl-typesetting" "named-readtables" "collectors" "closer-mop" "chronicity" "parenscript" "trivial-gray-streams" "trivia" "iterate" "introspect-environment" "trivial-macroexpand-all" "trivial-cltl2" "dufy/core" "parse-float" "global-vars" "cl-grnm")
5 | :components ((:module "src"
6 | :components ((:file "utils" :depends-on ("macro-utils"))
7 | (:file "macro-utils")
8 | (:file "raw-memory-streams")
9 | (:file "rwlock")
10 | (:file "graphql" :depends-on ("macro-utils"))
11 | (:file "hash-utils")
12 | (:file "context")
13 | (:file "html-reader")
14 | (:file "client-script" :depends-on ("html-reader"))
15 | (:file "interface-utils" :depends-on ("links" "html-reader"))
16 | (:file "user-context")
17 | (:file "conditions" :depends-on ("utils" "html-reader"))
18 | (:file "schema-type" :depends-on ("utils" "backend-modules"))
19 | (:file "dnsbl" :depends-on ("../config"))
20 | (:file "backend-modules")
21 | (:module "backends"
22 | :components ((:file "accordius"))
23 | :depends-on ("backend-modules" "backend" "lw2-login"))
24 | (:file "routes")
25 | (:file "sites" :depends-on ("utils" "routes" "backend-modules" "fonts-modules"))
26 | (:file "resources" :depends-on ("config-package" "utils" "sites" "context" "colors"))
27 | (:file "response" :depends-on ("utils" "conditions" "sites" "routes" "html-reader"))
28 | (:file "fonts-modules")
29 | (:file "fonts" :depends-on ("html-reader" "utils" "sites" "fonts-modules" "backend" "resources"))
30 | (:file "config-package" :depends-on ("sites" "backend-modules" "fonts-modules"))
31 | (:module "config-copy"
32 | :pathname "../"
33 | :output-files (compile-op (o c) (if (file-exists-p "config.lisp") nil (list "config.lisp")))
34 | :perform (compile-op :before (o c)
35 | (if (file-exists-p "config.lisp")
36 | (mark-operation-done o c)
37 | (copy-file "config-example.lisp" "config.lisp"))))
38 | (:file "../config" :depends-on ("config-copy" "config-package"))
39 | (:file "lmdb" :depends-on ("rwlock" "conditions" "raw-memory-streams" "hash-utils" "sites" "context" "../config"))
40 | (:file "backend" :depends-on ("utils" "hash-utils" "backend-modules" "lmdb" "graphql" "context" "user-context" "sites" "schema-type" "conditions" "web-push"))
41 | (:file "csrf" :depends-on ("conditions" "client-script"))
42 | (:file "components" :depends-on ("utils" "csrf"))
43 | (:file "links" :depends-on ("utils" "lmdb" "backend" "sites" "context"))
44 | (:file "legacy-archive" :depends-on ("utils" "backend"))
45 | (:static-file "../text-clean-regexps.js")
46 | (:static-file "../html-clean-regexps.js")
47 | (:file "colors" :depends-on ("utils"))
48 | (:file "images" :depends-on ("conditions" "html-reader" "utils" "lmdb" "backend" "legacy-archive" "resources"))
49 | (:file "elicit-predictions" :depends-on ("utils" "html-reader" "backend" "graphql"))
50 | (:file "clean-html" :depends-on ("utils" "links" "lmdb" "backend" "context" "sites" "conditions" "colors" "images" "elicit-predictions" "../text-clean-regexps.js" "../html-clean-regexps.js"))
51 | (:file "lw2-login" :depends-on ("utils" "backend" "backend-modules" "context"))
52 | (:file "backlinks" :depends-on ("html-reader" "lmdb" "backend" "backend-modules" "sites" "links" "context" "clean-html" "conditions" "utils" "interface-utils"))
53 | (:file "web-push" :depends-on ("utils" "conditions"))
54 | (:file "push-notifications" :depends-on ("backend"))
55 | (:file "background-loader" :depends-on ("backend" "push-notifications" "clean-html"))
56 | (:file "admin" :depends-on ("lmdb" "clean-html" "backend" "backlinks"))
57 | (:file "comment-threads" :depends-on ("utils" "context" "user-context" "conditions" "html-reader" "data-viewers/comment"))
58 | (:file "data-viewers/post" :depends-on ("schema-type" "utils" "backend" "comment-threads" "context" "user-context" "sites" "clean-html" "html-reader" "interface-utils" "links" "lmdb" "backlinks"))
59 | (:file "data-viewers/comment" :depends-on ("schema-type" "utils" "backend" "context" "user-context" "sites" "clean-html" "html-reader" "interface-utils" "links" "lmdb" "backlinks"))
60 | (:file "data-viewers/tag" :depends-on ("schema-type" "backend-modules")))
61 | :depends-on ())
62 | (:module "templates"
63 | :components ((:static-file "conversation.html")
64 | (:static-file "edit-post.html")
65 | (:static-file "reset-password.html")))
66 | (:static-file "www/head.js")
67 | (:file "lw2" :depends-on ("src" "www/head.js" "templates"))
68 | (:file "arbital" :depends-on ("lw2"))))
69 |
--------------------------------------------------------------------------------
/pre-commit.hook:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | set -e
4 |
5 | SYSTEMS=(Mac Windows Linux)
6 | SUFFIXES=(.mac.css .windows.css .linux.css)
7 |
8 | if [[ "$1" = "--force" ]] || ! git diff-index --quiet --cached HEAD -- www/style.css.php www/style_mobile_additions.css.php www/theme\* www/ea/\*.css.php www/accordius/\*.css.php ; then
9 | git ls-files --cached HEAD 'www/theme-*' |while read F; do
10 | IN="${F#www/}"
11 | OUTBASE="css/style${IN#theme}"
12 | OUTBASE="${OUTBASE%.php}"
13 | OUTBASE="${OUTBASE%.css}"
14 | OUTBASE="${OUTBASE%-default}"
15 | for N in ${!SYSTEMS[*]}; do
16 | OUT="${OUTBASE}${SUFFIXES[N]}"
17 | git show :www/style.css.php | (cd www; php -- ${SYSTEMS[N]} "$IN" >"$OUT")
18 | git add www/"$OUT"
19 | done
20 | done
21 | for N in ${!SYSTEMS[*]}; do
22 | git add www/css/style${SUFFIXES[N]}
23 | done
24 | git show :www/theme_tweaker.css.php | (cd www; php >css/theme_tweaker.css)
25 | git add www/css/theme_tweaker.css
26 | fi
27 |
--------------------------------------------------------------------------------
/src/admin.lisp:
--------------------------------------------------------------------------------
1 | (uiop:define-package #:lw2.admin
2 | (:use #:cl #:alexandria #:lw2.lmdb #:lw2.clean-html #:lw2.backend #:lw2.backlinks)
3 | (:export #:reclean-html))
4 |
5 | (in-package #:lw2.admin)
6 |
7 | (defun check-valid-alist (object)
8 | (and (consp object)
9 | (every (lambda (pair)
10 | (and (consp pair)
11 | (let ((key (car pair)))
12 | (or (symbolp key) (stringp key)))))
13 | object)))
14 |
15 | (defun map-posts-and-comments (fn &key skip-comments skip-posts)
16 | (let ((total-count (count-database-entries "post-body-json"))
17 | (done-count 0)
18 | (last-done nil))
19 | (format *error-output* "Press Enter to abort.~%")
20 | (labels ((report-progress ()
21 | (when (= 0 (mod done-count 10))
22 | (format *error-output* "~AFinished ~A of ~A posts." (string #\Return) done-count total-count)
23 | (force-output *error-output*))))
24 | (loop
25 | for (post post-id) = (with-cache-readonly-transaction
26 | (call-with-cursor "post-body-json"
27 | (lambda (db cursor)
28 | (declare (ignore db))
29 | (multiple-value-bind (post post-id)
30 | (if last-done
31 | (progn
32 | (cursor-get cursor :set-range :key last-done :return-type 'existence)
33 | (cursor-get cursor :next :value-type :json))
34 | (cursor-get cursor :first :value-type :json))
35 | (list (ignore-errors (postprocess-query-result post)) post-id)))))
36 | while post-id
37 | do (when (read-char-no-hang)
38 | (format *error-output* "Aborted.~%")
39 | (return-from map-posts-and-comments (values)))
40 | do (report-progress)
41 | do (progn
42 | (unless skip-posts
43 | (with-simple-restart (continue "Ignore this post and continue.")
44 | (funcall fn post post-id)))
45 | (unless skip-comments
46 | (ignore-errors
47 | (let ((comments (if (cdr (assoc :question post))
48 | (append (get-post-comments post-id :revalidate nil)
49 | (get-post-answers post-id :revalidate nil))
50 | (get-post-comments post-id :revalidate nil))))
51 | (loop for comment in comments
52 | for comment-id = (cdr (assoc :--id comment))
53 | do (with-simple-restart (continue "Ignore this comment and continue.")
54 | (funcall fn comment post-id comment-id))))))
55 | (incf done-count)
56 | (setf last-done post-id)))
57 | (report-progress)
58 | (format *error-output* "~%Done.~%")
59 | (values))))
60 |
61 | (defun reclean-html ()
62 | (map-posts-and-comments
63 | (lambda (item post-id &optional comment-id)
64 | (if (not comment-id)
65 | (ignore-errors
66 | (let ((*before-clean-hook* (lambda () (clear-backlinks post-id)))
67 | (*link-hook* (lambda (link)
68 | (add-backlink link post-id))))
69 | (clean-html (or (cdr (assoc :html-body item)) "") :with-toc t :post-id post-id)))
70 | (ignore-errors
71 | (let ((*before-clean-hook* (lambda () (clear-backlinks post-id comment-id)))
72 | (*link-hook* (lambda (link)
73 | (add-backlink link post-id comment-id))))
74 | (clean-html (or (cdr (assoc :html-body item)) ""))))))))
75 |
76 | (defun grep-posts-and-comments (regex &key skip-comments print-ids)
77 | (let* ((scanner (ppcre:create-scanner regex))
78 | (printer (if print-ids
79 | (lambda (item post-id &optional comment-id)
80 | (when (ppcre:scan scanner (or (cdr (assoc :html-body item)) ""))
81 | (format t "~A~@[/~A~]~%" post-id comment-id)))
82 | (lambda (item post-id &optional comment-id)
83 | (declare (ignore post-id comment-id))
84 | (ppcre:do-matches-as-strings (match scanner (or (cdr (assoc :html-body item)) ""))
85 | (write-line match))))))
86 | (map-posts-and-comments
87 | printer
88 | :skip-comments skip-comments)))
89 |
90 | (defun call-with-compressed-output-stream (fn output)
91 | (let ((compressor
92 | (uiop:launch-program '("zstd" "-19")
93 | :output output
94 | :input :stream))
95 | (abnormal-exit t))
96 | (unwind-protect
97 | (progn (funcall fn (uiop:process-info-input compressor))
98 | (setf abnormal-exit nil))
99 | (when abnormal-exit
100 | (uiop:terminate-process compressor))
101 | (uiop:close-streams compressor)
102 | (uiop:wait-process compressor))))
103 |
104 | (defmacro with-compressed-output-stream ((stream filespec) &body body)
105 | `(let ((fn (lambda (,stream) ,@body)))
106 | (declare (dynamic-extent fn))
107 | (call-with-compressed-output-stream fn ,filespec)))
108 |
109 | (defun write-user-comments-to-stream (identifier-type identifier stream)
110 | (let* ((user-id (ccase identifier-type
111 | (:user-id identifier)
112 | (:user-slug (get-slug-userid identifier))))
113 | (first t)
114 | (fn (lambda (comment post-id comment-id)
115 | (declare (ignore post-id comment-id))
116 | (when (and (check-valid-alist comment)
117 | (string= (cdr (assoc :user-id comment)) user-id))
118 | (if first
119 | (setf first nil)
120 | (format stream ",~%"))
121 | (json:encode-json comment stream)))))
122 | (format stream "[~%")
123 | (map-posts-and-comments fn :skip-posts t)
124 | (format stream "~%]~%")))
125 |
126 | (defun write-all-posts-to-stream (stream)
127 | (let* ((first t)
128 | (fn (lambda (post post-id &optional comment-id)
129 | (declare (ignore post-id comment-id))
130 | (when (check-valid-alist post)
131 | (if first
132 | (setf first nil)
133 | (format stream ",~%"))
134 | (json:encode-json post stream)))))
135 | (format stream "[~%")
136 | (map-posts-and-comments fn :skip-comments t)
137 | (format stream "]~%")))
138 |
--------------------------------------------------------------------------------
/src/backends/accordius.lisp:
--------------------------------------------------------------------------------
1 | (in-package #:lw2.backend)
2 |
3 | ;;; REST API
4 |
5 | (defun do-wl-rest-query (endpoint filters &key auth-token)
6 | (lw2-graphql-query (lambda () (values endpoint filters)) :auth-token auth-token))
7 |
8 | (defclass accordius-query (closer-mop:funcallable-standard-object) ()
9 | (:metaclass closer-mop:funcallable-standard-class))
10 |
11 | (defmethod run-query ((query accordius-query))
12 | (lw2-graphql-query query :return-type :string))
13 |
14 | (define-backend-operation lw2-query-string backend-accordius (query-type return-type args &rest rest)
15 | (declare (ignore rest))
16 | (let ((obj (make-instance 'accordius-query)))
17 | (closer-mop:set-funcallable-instance-function
18 | obj
19 | (lambda ()
20 | (values
21 | (format nil "~(~A~)s/~@[~A/~]" query-type (if (eq return-type :single) (cdr (assoc :document-id args))))
22 | (loop for arg in args
23 | unless (member (car arg) '(:document-id :limit :view))
24 | collect (cons (json:lisp-to-camel-case (string (car arg))) (cdr arg))))))
25 | obj))
26 |
27 | (define-backend-operation postprocess-query-result backend-accordius (result &optional cache-fn)
28 | (when cache-fn
29 | (error "Not implemented!"))
30 | (if-let (data (assoc :data result))
31 | (cdadr data)
32 | result))
33 |
34 | (define-backend-operation call-with-backend-response backend-accordius (fn query &key return-type auth-token)
35 | (multiple-value-bind (endpoint filters)
36 | (funcall query)
37 | (call-with-http-response
38 | fn
39 | (quri:render-uri (quri:merge-uris (quri:make-uri :path endpoint :query filters) (quri:uri (rest-api-uri *current-backend*))))
40 | :additional-headers (if auth-token `(("authorization" . ,auth-token)) nil)
41 | :want-stream (not return-type))))
42 |
43 | (define-backend-operation get-post-body backend-accordius (post-id &key &allow-other-keys)
44 | (acons :tags (lw2-graphql-query (lambda () (values "tags/" `(("document_id" . ,post-id))))) (call-next-method)))
45 |
46 | (define-backend-operation lw2-search-query backend-accordius (query &key &allow-other-keys)
47 | (values
48 | (do-wl-rest-query "post_search/" `(("query" . ,query)))
49 | (do-wl-rest-query "comment_search/" `(("query" . ,query)))))
50 |
51 | (defun do-wl-rest-mutate (mutation-type endpoint post-params auth-token)
52 | (call-with-http-response
53 | #'identity
54 | (quri:render-uri (quri:merge-uris (quri:make-uri :path endpoint :query "") (quri:uri (rest-api-uri *current-backend*))))
55 | :method mutation-type
56 | :content post-params
57 | :headers (alist "authorization" auth-token)))
58 |
59 | (defun do-wl-create-tag (document-id text auth-token)
60 | (do-wl-rest-mutate :post "tags/" `((:DOCUMENT-ID . ,document-id) (:TEXT . ,text)) auth-token))
61 |
62 |
63 | ;;;; BACKEND SPECIFIC GRAPHQL
64 |
65 | (define-backend-operation get-user-page-items backend-accordius (user-id request-type &key &allow-other-keys)
66 | (declare (ignore user-id request-type))
67 | (let ((*graphql-correct* t))
68 | (declare (special *graphql-correct*))
69 | (call-next-method)))
70 |
71 | (define-backend-operation get-conversation-messages backend-accordius (conversation-id auth-token)
72 | (declare (ignore conversation-id auth-token))
73 | (let ((*messages-index-fields* (cons :html-body (remove :content *messages-index-fields*))))
74 | (call-next-method)))
75 |
76 | (define-backend-operation user-fields backend-accordius ()
77 | (remove :groups (call-next-method)))
78 |
79 | ;;;; LOGIN
80 |
81 | (in-package #:lw2.login)
82 |
83 | (define-backend-operation do-lw2-mutation backend-accordius (auth-token target-type mutation-type terms fields)
84 | (let ((endpoint
85 | (case target-type
86 | (:post "posts")
87 | (:comment "comments")
88 | )))
89 | (cond
90 | ((eq mutation-type :delete) (do-wl-rest-mutate mutation-type
91 | (concatenate 'string endpoint "/"
92 | (cdr (assoc :DOCUMENT-ID terms)))
93 | nil
94 | auth-token))
95 | (t (call-next-method)))))
96 |
97 |
98 | (define-backend-operation do-login backend-accordius (user-designator password &key &allow-other-keys)
99 | (let* ((response
100 | (do-lw2-post-query nil `(("query" . "mutation Login($username: String, $password: String) { Login(username: $username, password: $password) {userId, sessionKey, expiration}}")
101 | ("variables" .
102 | (("username" . ,user-designator)
103 | ("password" . ,password))))))
104 | (user-id (format nil "~A" (cdr (assoc :user-id response))))
105 | (auth-token (cdr (assoc :session-key response)))
106 | (expiration (truncate (* 1000 (cdr (assoc :expiration response))))))
107 | (values user-id auth-token nil expiration)))
108 |
109 | (define-backend-operation do-lw2-create-user backend-accordius (username email password)
110 | ;; TODO: Add actual code
111 | (let (user-id auth-token error-message expiration)
112 | (values user-id auth-token error-message expiration)))
113 |
114 | (define-backend-operation do-lw2-forgot-password backend-accordius (email)
115 | ;; TODO: Add actual code
116 | (let (successfulp error-message)
117 | (values successfulp error-message)))
118 |
119 | (define-backend-operation do-lw2-reset-password backend-accordius (auth-token password)
120 | ;; TODO: Add actual code
121 | (let (user-id auth-token error-message expiration)
122 | (values user-id auth-token error-message expiration)))
123 |
--------------------------------------------------------------------------------
/src/background-loader.lisp:
--------------------------------------------------------------------------------
1 | (in-package #:lw2.backend)
2 |
3 | (defvar *background-loader-thread* nil)
4 | (defvar *background-loader-semaphore* (make-semaphore :count 1))
5 | (defvar *background-loader-ready* nil)
6 |
7 | (defun background-loader-running-p ()
8 | (case (semaphore-count *background-loader-semaphore*)
9 | (0 t)
10 | (1 nil)))
11 |
12 | (defun background-loader-ready-p ()
13 | (and (background-loader-running-p)
14 | (background-loader-enabled *current-site*)
15 | *background-loader-ready*))
16 |
17 | (defun make-site-background-loader-fn (site)
18 | (let (last-post-processed last-comment-processed)
19 | (lambda ()
20 | (with-site-context (site :request nil)
21 | (log-and-ignore-errors
22 | (let* ((posts-json (sb-sys:with-deadline (:seconds 120) (get-posts-json)))
23 | (posts-list (decode-query-result posts-json)))
24 | (when posts-list
25 | (with-cache-transaction
26 | (cache-update "index-json" "new-not-meta" posts-json #'decode-query-result)
27 | (dolist (post posts-list)
28 | (cache-put "postid-to-title" (cdr (assoc :--id post)) (cdr (assoc :title post))))
29 | (dolist (post posts-list)
30 | (cache-put "postid-to-slug" (cdr (assoc :--id post)) (cdr (assoc :slug post)))))
31 | (loop for post in posts-list
32 | as post-id = (cdr (assoc :--id post))
33 | if (string= post-id last-post-processed) return nil
34 | do (log-and-ignore-errors
35 | (lw2.clean-html:clean-html (cdr (assoc :html-body (get-post-body post-id :revalidate nil))) :with-toc t :post-id post-id)))
36 | (setf last-post-processed (cdr (assoc :--id (first posts-list)))))))
37 | (log-and-ignore-errors
38 | (let ((recent-comments-json (sb-sys:with-deadline (:seconds 120) (get-recent-comments-json))))
39 | (when-let ((recent-comments (ignore-errors (decode-query-result recent-comments-json))))
40 | (cache-update "index-json" "recent-comments" recent-comments-json #'decode-query-result)
41 | (loop for comment in recent-comments
42 | as comment-id = (cdr (assoc :--id comment))
43 | as cache-database = (if (or (cdr (assoc :answer comment)) (cdr (assoc :parent-answer-id comment)))
44 | "post-answers-json"
45 | "post-comments-json")
46 | if (string= comment-id last-comment-processed) return nil
47 | do (log-and-ignore-errors
48 | (with-cache-transaction
49 | (when-let ((post-id (cdr (assoc :post-id comment))))
50 | (let* ((post-comments (when-let ((x (cache-get cache-database post-id :return-type 'binary-stream))) (decode-query-result x)))
51 | (new-post-comments (sort (cons comment (delete-if (lambda (c) (string= comment-id (cdr (assoc :--id c)))) post-comments))
52 | #'> :key (lambda (c) (cdr (assoc :base-score c))))))
53 | (cache-update cache-database post-id (make-graphql-json :results new-post-comments) #'decode-query-result)))
54 | (when-let ((user-id (cdr (assoc :user-id comment))))
55 | (cache-mark-stale "user-page-items" user-id))
56 | (mark-comment-replied comment)))
57 | do (log-and-ignore-errors
58 | (lw2.clean-html:clean-html (cdr (assoc :html-body comment)))))
59 | (setf last-comment-processed (cdr (assoc :--id (first recent-comments)))))))
60 | (send-all-notifications)))))
61 |
62 | (defun background-loader ()
63 | (let (sites loader-functions)
64 | (loop
65 | (unless (eq sites *sites*)
66 | (setf sites *sites*
67 | loader-functions (loop for site in sites
68 | when (background-loader-enabled site)
69 | collect (make-site-background-loader-fn site))))
70 | (dolist (loader-fn loader-functions)
71 | (funcall loader-fn))
72 | (setf *background-loader-ready* t)
73 | (if (wait-on-semaphore *background-loader-semaphore* :timeout 60)
74 | (return)))))
75 |
76 | (defun start-background-loader ()
77 | (if (background-loader-running-p)
78 | (warn "Background loader already running.")
79 | (progn
80 | (wait-on-semaphore *background-loader-semaphore*)
81 | (setf *background-loader-thread* (sb-thread:make-thread #'background-loader :name "background loader")))))
82 |
83 | (defun stop-background-loader ()
84 | (if (background-loader-running-p)
85 | (progn
86 | (signal-semaphore *background-loader-semaphore*)
87 | (join-thread *background-loader-thread*)
88 | (setf *background-loader-thread* nil
89 | *background-loader-ready* nil)
90 | (signal-semaphore *background-loader-semaphore*))
91 | (warn "Background loader not running.")))
92 |
--------------------------------------------------------------------------------
/src/backlinks.lisp:
--------------------------------------------------------------------------------
1 | (uiop:define-package #:lw2.backlinks
2 | (:use #:cl #:alexandria #:split-sequence
3 | #:lw2.html-reader #:lw2.lmdb #:lw2.backend-modules #:lw2.backend #:lw2.sites #:lw2.links #:lw2.context #:lw2.clean-html #:lw2.conditions #:lw2.utils #:lw2.interface-utils)
4 | (:import-from #:collectors #:with-collector)
5 | (:export #:clear-backlinks #:add-backlink #:get-backlinks #:backlinks-to-html))
6 |
7 | (in-package #:lw2.backlinks)
8 |
9 | (named-readtables:in-readtable html-reader)
10 |
11 | (define-cache-database 'backend-backlinks
12 | (list "backlinks" :flags liblmdb:+dupsort+)
13 | (list "frontlinks" :flags liblmdb:+dupsort+)
14 | "backlinks-cache")
15 |
16 | (declaim (ftype (function (string (or null string) &optional string) string) item-reference-string))
17 |
18 | (defun item-reference-string (post-id comment-id &optional host)
19 | (format nil "~@[~A ~]~A~@[ ~A~]" host post-id comment-id))
20 |
21 | (define-backend-function clear-backlinks (post-id &optional comment-id)
22 | (backend-backlinks
23 | (cache-del "frontlinks" (item-reference-string post-id comment-id))))
24 |
25 | (define-backend-function add-backlink (link post-id &optional comment-id)
26 | (backend-backlinks
27 | (let* ((link-host (or (quri:uri-host (quri:uri link)) (site-host *current-site*)))
28 | (link-site (and link-host (find-site link-host)))
29 | (current-host (site-host *current-site*)))
30 | (when link-site
31 | (multiple-value-bind (link-post-id link-comment-id) (match-lw2-link link)
32 | (when link-post-id
33 | (ignore-errors
34 | (cache-put "frontlinks"
35 | (item-reference-string post-id comment-id)
36 | (item-reference-string link-post-id link-comment-id link-host))
37 | (with-site-context (link-site)
38 | (cache-put "backlinks"
39 | (item-reference-string link-post-id link-comment-id)
40 | (item-reference-string post-id comment-id current-host)))))))))
41 | (backend-base
42 | (declare (ignore link post-id comment-id))
43 | nil))
44 |
45 | (define-backend-function link-exists-p (source-post-id source-comment-id target-host target-post-id target-comment-id)
46 | (backend-backlinks
47 | (call-with-cursor "frontlinks"
48 | (lambda (db cursor)
49 | (declare (ignore db))
50 | (cursor-get cursor :get-both
51 | :key (item-reference-string source-post-id source-comment-id)
52 | :value (item-reference-string target-post-id target-comment-id target-host)
53 | :return-type 'existence)))))
54 |
55 | (define-backend-function get-backlink-pointers (post-id &optional comment-id)
56 | (backend-backlinks
57 | (call-with-cursor "backlinks"
58 | (lambda (db cursor)
59 | (declare (ignore db))
60 | (loop for backlink-data = (cursor-get cursor :set :key (item-reference-string post-id comment-id))
61 | then (cursor-get cursor :next-dup)
62 | while backlink-data
63 | collect (split-sequence #\Space backlink-data)))
64 | :read-only t))
65 | (backend-base
66 | (declare (ignore post-id comment-id))
67 | nil))
68 |
69 | (define-backend-function process-backlink (current-post-id current-comment-id source-site-host source-post-id &optional source-comment-id)
70 | (backend-backlinks
71 | (let* ((source-db (if source-comment-id "post-comments-json-meta" "post-body-json-meta"))
72 | (metadata (cache-get source-db source-post-id :value-type :lisp))
73 | (cache-key (format nil "~@{~S~^ ~}" current-post-id current-comment-id source-site-host source-post-id source-comment-id))
74 | (cached-data (cache-get "backlinks-cache" cache-key :value-type :lisp))
75 | (last-modified (cdr (assoc :last-modified metadata)))
76 | (if-modified-since (cdr (assoc :if-modified-since cached-data))))
77 | (if (and last-modified if-modified-since (= last-modified if-modified-since))
78 | cached-data
79 | (log-and-ignore-errors
80 | (let ((current-site-host (site-host *current-site*)))
81 | (labels ((cleanup-stale-backlink ()
82 | (with-cache-transaction
83 | (cache-del "backlinks-cache" cache-key)
84 | (cache-del "backlinks"
85 | (item-reference-string current-post-id current-comment-id)
86 | :value (item-reference-string source-post-id source-comment-id source-site-host)))
87 | nil))
88 | (handler-case
89 | (with-site-context ((find-site source-site-host))
90 | (if (not (link-exists-p source-post-id source-comment-id current-site-host current-post-id current-comment-id))
91 | (cleanup-stale-backlink)
92 | (let* ((source-post (get-post-body source-post-id :revalidate nil))
93 | (source-comment (when source-comment-id
94 | (find-if (lambda (c) (string= source-comment-id (cdr (assoc :--id c))))
95 | (get-post-comments source-post-id :revalidate nil))))
96 | (result
97 | (alist :if-modified-since last-modified
98 | :site-host source-site-host
99 | :link (generate-item-link :post source-post :comment-id source-comment-id :absolute t)
100 | :post-title (cdr (assoc :title source-post))
101 | :post-user-id (cdr (assoc :user-id source-post))
102 | :comment-user-id (cdr (assoc :user-id source-comment))
103 | :posted-at (cdr (assoc :posted-at (or source-comment source-post)))
104 | :score (cdr (assoc :base-score (or source-comment source-post))))))
105 | (cache-put "backlinks-cache" cache-key (prin1-to-string result))
106 | result)))
107 | (lw2-client-error ()
108 | (cleanup-stale-backlink))))))))))
109 |
110 | (define-backend-function get-backlinks (post-id &optional comment-id)
111 | (backend-backlinks
112 | (loop
113 | for bp in (get-backlink-pointers post-id comment-id)
114 | for backlink-data = (apply 'process-backlink post-id comment-id bp)
115 | when backlink-data collect backlink-data))
116 | (backend-base
117 | (declare (ignore post-id comment-id))
118 | nil))
119 |
120 | (defun backlinks-to-html (backlinks id)
121 | (when backlinks
122 |
40 | (with-output-to-string (outstream)
41 | (sb-debug:print-backtrace :stream outstream :from :interrupted-frame :print-frame-source t))
42 |
)
43 | (princ-to-string condition)
)
47 |
48 | (define-condition lw2-error (error) ((http-return-code :allocation :class :reader condition-http-return-code :initform 503)))
49 |
50 | (defmethod error-to-html ((condition lw2-error))
51 | (princ-to-string condition)
) 52 | 53 | (define-condition lw2-client-error (lw2-error) ((http-return-code :allocation :class :initform 400))) 54 | 55 | (define-condition csrf-check-failed (lw2-error) () 56 | (:report "CSRF check failed.")) 57 | 58 | (defmethod error-to-html ((condition csrf-check-failed)) 59 |CSRF check failed.
60 |You may need to adjust your browser settings to allow cookies.
) 61 | 62 | (define-condition lw2-not-found-error (lw2-client-error) ((http-return-code :allocation :class :initform 404)) 63 | (:report "Document not found.")) 64 | 65 | (define-condition lw2-user-not-found-error (lw2-not-found-error) () 66 | (:report "User not found.")) 67 | 68 | (define-condition lw2-not-allowed-error (lw2-client-error) ((http-return-code :allocation :class :initform 403)) 69 | (:report "LW server reports: not allowed.")) 70 | 71 | (define-condition lw2-login-required-error (lw2-client-error) ((http-return-code :allocation :class :initform 403)) 72 | (:report "This document is only visible to logged-in users.")) 73 | 74 | (define-condition lw2-rate-limit-exceeded (lw2-client-error) ((http-return-code :allocation :class :initform 429)) 75 | (:report "Rate limit exceeded. Try again later.")) 76 | 77 | (define-condition lw2-server-error (lw2-error) 78 | ((message :initarg :message :reader lw2-server-error-message) 79 | (introduction :allocation :class :reader condition-introduction)) 80 | (:report (lambda (c s) 81 | (format s "~A:~%~A" (condition-introduction c) (lw2-server-error-message c))))) 82 | 83 | (define-condition lw2-connection-error (lw2-server-error) 84 | ((introduction :allocation :class :initform "Unable to connect to LW server"))) 85 | 86 | (define-condition lw2-unknown-error (lw2-server-error) 87 | ((introduction :allocation :class :initform "Unrecognized LW server error"))) 88 | 89 | (defmethod error-to-html ((condition lw2-server-error)) 90 |(condition-introduction condition):
91 |(lw2-server-error-message condition)
)
92 |
93 | (defmacro error-explanation-case (expression &rest clauses)
94 | (with-gensyms (condition)
95 | `(let ((*error-explanation-hook* (lambda (,condition)
96 | (typecase ,condition ,@clauses))))
97 | (declare (dynamic-extent *error-explanation-hook*))
98 | ,expression)))
99 |
100 | (defun html-output-stream-error-p (condition)
101 | (and (typep condition 'stream-error)
102 | *html-output*
103 | (compare-streams (stream-error-stream condition) *html-output*)))
104 |
105 | (defun interesting-condition-p (condition)
106 | (not (or (typep condition 'lw2-client-error)
107 | (html-output-stream-error-p condition))))
108 |
109 | (defun log-condition (condition)
110 | (handler-case
111 | (with-open-file (outstream "./logs/error.log" :direction :output :if-exists :append :if-does-not-exist :create)
112 | (format outstream "~%~A: ~S ~A~%" (local-time:format-timestring nil (local-time:now)) condition condition)
113 | (sb-debug:print-backtrace :stream outstream :from :interrupted-frame :print-frame-source t))
114 | (serious-condition ()
115 | nil)))
116 |
117 | (defmacro log-conditions (&body body)
118 | `(block log-conditions
119 | (handler-bind
120 | (((or warning serious-condition) (lambda (c) (when (interesting-condition-p c) (log-condition c)))))
121 | ,@body)))
122 |
123 | (defmacro log-and-ignore-errors (&body body)
124 | `(block log-and-ignore-errors
125 | (handler-bind
126 | ((fatal-error
127 | (lambda (c)
128 | (when (interesting-condition-p c) (log-condition c))
129 | (return-from log-and-ignore-errors (values nil c)))))
130 | ,@body)))
131 |
132 | (defun abort-response ()
133 | (throw 'abort-response nil))
134 |
135 | (defun abort-response-if-unrecoverable (condition)
136 | (when (html-output-stream-error-p condition)
137 | (abort-response)))
138 |
139 | (defmacro with-error-html-block (() &body body)
140 | "If an error occurs within BODY, write an HTML representation of the
141 | signaled condition to *HTML-OUTPUT*."
142 | `(block with-error-html-block
143 | (handler-bind ((serious-condition (lambda (c)
144 | (abort-response-if-unrecoverable c)
145 | (error-to-html c)
146 | (return-from with-error-html-block nil))))
147 | (log-conditions (progn ,@body)))))
148 |
--------------------------------------------------------------------------------
/src/config-package.lisp:
--------------------------------------------------------------------------------
1 | (uiop:define-package #:lw2-viewer.config
2 | (:use #:cl #:lw2.sites #:lw2.backend-modules #:lw2.fonts-modules)
3 | (:export #:*lmdb-mapsize* #:*dnsbl-list* #:*html-global-resources* #:*max-requests-in-progress*)
4 | (:unintern #:*site-uri* #:*graphql-uri* #:*websocket-uri* #:*backend-type* #:*secure-cookies* #:*cache-db*))
5 |
6 | (in-package #:lw2-viewer.config)
7 |
8 | (defvar *dnsbl-list* nil)
9 | (defvar *html-global-resources* nil)
10 | (defvar *max-requests-in-progress* nil)
11 |
--------------------------------------------------------------------------------
/src/context.lisp:
--------------------------------------------------------------------------------
1 | (uiop:define-package #:lw2.context
2 | (:use #:cl)
3 | (:export #:*current-site* #:*current-backend* #:*default-last-modified* #:*preview* #:*enable-voting*)
4 | (:recycle #:lw2.context #:lw2.backend #:lw2-viewer))
5 |
6 | (in-package #:lw2.context)
7 |
8 | (defvar *current-site*)
9 |
10 | (defvar *current-backend*)
11 |
12 | (defvar *default-last-modified*)
13 |
14 | (defparameter *preview* nil)
15 |
16 | (defparameter *enable-voting* nil)
17 |
--------------------------------------------------------------------------------
/src/csrf.lisp:
--------------------------------------------------------------------------------
1 | (uiop:define-package #:lw2.csrf
2 | (:use #:cl #:lw2.conditions #:lw2.client-script)
3 | (:export #:make-csrf-token #:check-csrf-token #:check-csrf)
4 | (:recycle #:lw2-viewer))
5 |
6 | (in-package #:lw2.csrf)
7 |
8 | (client-defun make-csrf-token (&optional (session-token (when-server (hunchentoot:cookie-in "session-token"))) (nonce (when-server (ironclad:make-random-salt))))
9 | (if-client
10 | (ps:chain -g-w csrf-token)
11 | (progn
12 | (if (typep session-token 'string) (setf session-token (base64:base64-string-to-usb8-array session-token)))
13 | (let ((csrf-token (concatenate '(vector (unsigned-byte 8)) nonce (ironclad:digest-sequence :sha256 (concatenate '(vector (unsigned-byte 8)) nonce session-token)))))
14 | (values (base64:usb8-array-to-base64-string csrf-token) csrf-token)))))
15 |
16 | (defun check-csrf-token (csrf-token &optional (session-token (hunchentoot:cookie-in "session-token")))
17 | (unless (and (> (length csrf-token) 0)
18 | (> (length session-token) 0))
19 | (error 'csrf-check-failed))
20 | (let* ((session-token (base64:base64-string-to-usb8-array session-token))
21 | (csrf-token (base64:base64-string-to-usb8-array csrf-token))
22 | (correct-token (nth-value 1 (make-csrf-token session-token (subseq csrf-token 0 16)))))
23 | (unless (ironclad:constant-time-equal csrf-token correct-token)
24 | (error 'csrf-check-failed))
25 | t))
26 |
27 | (defun check-csrf ()
28 | (unless (member (hunchentoot:request-method*) '(:get :head))
29 | (check-csrf-token (hunchentoot:post-parameter "csrf-token"))))
30 |
--------------------------------------------------------------------------------
/src/data-viewers/comment.lisp:
--------------------------------------------------------------------------------
1 | (uiop:define-package #:lw2.data-viewers.comment
2 | (:use #:cl #:lw2.html-reader #:lw2.utils #:lw2.schema-type #:lw2.context #:lw2.user-context #:lw2.backend #:lw2.links #:lw2.interface-utils #:lw2.sites #:lw2.clean-html #:lw2.lmdb #:lw2.backlinks)
3 | (:export #:*comment-individual-link* #:comment-to-html))
4 |
5 | (in-package #:lw2.data-viewers.comment)
6 |
7 | (named-readtables:in-readtable html-reader)
8 |
9 | (defparameter *comment-individual-link* nil)
10 |
11 | (define-schema-type :comment ()
12 | ((comment-id string :alias :--id)
13 | (user-id string)
14 | (posted-at string)
15 | (highlight-new boolean :graphql-ignore t)
16 | (replied list :graphql-ignore t)
17 | (post-id (or null simple-string))
18 | (tag list :backend-type backend-lw2-tags-comments :subfields (:--id :name :slug))
19 | (base-score (or null fixnum))
20 | (af-base-score (or null fixnum))
21 | (vote-count (or null fixnum))
22 | (extended-score list)
23 | (page-url (or null string) :context-not :user-index) ; page-url sometimes causes "Cannot read property '_id' of undefined" error
24 | (parent-comment list :backend-type backend-lw2-tags-comments :context :index :subfields (:--id :user-id :post-id (:tag :--id :name :slug)))
25 | (parent-comment list :context :index :subfields (:--id :user-id :post-id))
26 | (parent-comment-id (or null string))
27 | (child-count (or null fixnum) :graphql-ignore t)
28 | (children list :graphql-ignore t)
29 | (af boolean :backend-type backend-alignment-forum)
30 | (retracted boolean)
31 | (deleted-public boolean)
32 | (answer boolean :backend-type backend-q-and-a)
33 | (debate-response boolean :backend-type backend-debates)
34 | (parent-answer-id (or null string) :backend-type backend-q-and-a)
35 | (nominated-for-review t :backend-type backend-lw2)
36 | (reviewing-for-review t :backend-type backend-lw2)
37 | (top-level-comment list :backend-type backend-lw2 :subfields (:nominated-for-review :reviewing-for-review))
38 | (latest-children list
39 | :backend-type backend-shortform
40 | :context :shortform
41 | :subfields (:--id :user-id :posted-at :post-id :base-score :af-base-score :page-url
42 | :parent-comment-id :af :vote-count :retracted :deleted-public :html-body))
43 | (html-body (or null string memoized-reference))))
44 |
45 | (defun comment-link (post-id tag &optional comment-id)
46 | (when (or post-id tag)
47 | (generate-item-link (if post-id :post :tag) (or post-id (cdr (assoc :slug tag))) :comment-id comment-id)))
48 |
49 | (defun comment-to-html (out-stream comment &key with-post-title)
50 | (if (or (cdr (assoc :deleted comment)) (cdr (assoc :deleted-public comment)) (not (cdr (assoc :html-body comment))))
51 | (format out-stream "Could not retrieve ~A link.
You may wish to try ~:*~A" site-name (concatenate 'string base-uri link)))) 109 | 110 | (defun convert-redirect-link (link match-fn get-fn base-uri) 111 | (if-let (matched-link (funcall match-fn link)) 112 | (with-direct-link-restart ((concatenate 'string base-uri matched-link)) 113 | (merge-uris (funcall get-fn matched-link) 114 | (site-uri (find-link-site base-uri)))))) 115 | 116 | (simple-cacheable ("lw1-link" 'backend-lmdb-cache "lw1-link" link :catch-errors nil) 117 | (process-redirect-link link "https://www.lesswrong.com" "LessWrong 1.0")) 118 | 119 | (defun convert-lw1-link (link) 120 | (convert-redirect-link link #'match-lw1-link #'get-lw1-link "https://www.lesswrong.com")) 121 | 122 | (simple-cacheable ("ea1-link" 'backend-lmdb-cache "ea1-link" link :catch-errors nil) 123 | (process-redirect-link link "https://forum.effectivealtruism.org" "EA Forum 1.0")) 124 | 125 | (defun convert-ea1-link (link) 126 | (convert-redirect-link link #'match-ea1-link #'get-ea1-link "https://forum.effectivealtruism.org")) 127 | 128 | (defun match-overcomingbias-link (link) 129 | (if (ppcre:scan "^https?://(?:www\\.)?overcomingbias\\.com/" link) 130 | link 131 | nil)) 132 | 133 | (simple-cacheable ("overcomingbias-link" 'backend-lmdb-cache "overcomingbias-link" link :catch-errors nil) 134 | (if-let ((location (get-redirect link))) 135 | (match-lw1-link location) 136 | "")) 137 | 138 | (defun convert-overcomingbias-link (link) 139 | (when (match-overcomingbias-link link) 140 | (with-direct-link-restart (link) 141 | (let ((lw1-link (get-overcomingbias-link link))) 142 | (if (string= lw1-link "") 143 | nil 144 | (convert-lw1-link lw1-link)))))) 145 | 146 | (simple-cacheable ("agentfoundations-link" 'backend-lmdb-cache "agentfoundations-link" link :catch-errors nil) 147 | (process-redirect-link link "https://www.lesswrong.com" "Agent Foundations")) 148 | 149 | (defun convert-agentfoundations-link (link) 150 | (convert-redirect-link link #'match-agentfoundations-link #'get-agentfoundations-link "https://www.lesswrong.com")) 151 | 152 | (defun gen-internal (post-id slug comment-id &optional absolute-uri stream item-subtype) 153 | (format stream "~A~As/~A/~A~:[~@[#~A~]~;~@[#comment-~A~]~]" (or absolute-uri "/") (or item-subtype "post") post-id (or slug (get-post-slug post-id) "-") (and comment-id (= (length comment-id) 17)) comment-id)) 154 | 155 | (defun convert-lw2-slug-link (link) 156 | (multiple-value-bind (slug comment-id) (match-lw2-slug-link link) 157 | (when slug 158 | (gen-internal (get-slug-postid slug) slug comment-id)))) 159 | 160 | (defun convert-lw2-sequence-link (link) 161 | (if-let (site (find-link-site link)) 162 | (multiple-value-bind (sequence-id post-id comment-id) (match-lw2-sequence-link link) 163 | (cond 164 | (post-id (gen-internal post-id (get-post-slug post-id) comment-id (site-link-prefix site))) 165 | (sequence-id (format nil "~As/~A" (site-link-prefix site) sequence-id)))))) 166 | 167 | (defun convert-lw2-link (link) 168 | (multiple-value-bind (post-id comment-id slug) (match-lw2-link link) 169 | (when post-id 170 | (if-let (site (find-link-site link)) 171 | (gen-internal post-id slug comment-id (site-link-prefix site)))))) 172 | 173 | (defun generate-item-link (item-type item-designator &key comment-id absolute stream item-subtype) 174 | (let ((absolute (if (eq absolute t) (site-uri *current-site*) absolute))) 175 | (ecase item-type 176 | (:post 177 | (typecase item-designator 178 | (string 179 | (gen-internal item-designator (get-post-slug item-designator) comment-id absolute stream (or item-subtype "post"))) 180 | (cons 181 | (let ((post-id (cdr (assoc :--id item-designator)))) 182 | (gen-internal post-id (or (cdr (assoc :slug item-designator)) (get-post-slug post-id)) comment-id absolute stream (or item-subtype "post")))))) 183 | (:tag 184 | (with-output-to-designator (out stream) 185 | (format out "~Atag/~A~@[#comment-~A~]" (or absolute "/") item-designator comment-id)))))) 186 | 187 | (defun convert-any-link* (url) 188 | (let ((url (sanitize-link url))) 189 | (or (convert-lw2-link url) 190 | (convert-lw2-slug-link url) 191 | (convert-lw2-sequence-link url) 192 | (convert-lw1-link url) 193 | (convert-ea1-link url) 194 | (convert-agentfoundations-link url) 195 | (convert-overcomingbias-link url) 196 | (convert-lw2-misc-link url) 197 | (convert-arbital-link url)))) 198 | 199 | (defun convert-any-link (url) 200 | (or (convert-any-link* url) url)) 201 | 202 | (defun presentable-link (link &optional context) 203 | (or (and (ppcre:scan "^#" link) link) 204 | (and (not (eq context :image)) (convert-any-link* link)) 205 | (and (not (eq context :search)) 206 | (let ((sanitized-link (sanitize-link link))) 207 | (handler-case 208 | (merge-uris 209 | sanitized-link 210 | (site-link-base *current-site*)) 211 | (error () sanitized-link)))))) 212 | -------------------------------------------------------------------------------- /src/macro-utils.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.macro-utils 2 | (:documentation "Facilities for working with macros.") 3 | (:use #:cl #:iterate #:trivial-cltl2) 4 | (:import-from #:introspect-environment #:compiler-macroexpand #:compiler-macroexpand-1) 5 | (:export #:compiler-constantp #:macroexpand-both-1 #:macroexpand-both 6 | #:eval-in-environment 7 | #:augment-macros #:macro-as-lambda #:macro-list-as-lambdas)) 8 | 9 | (in-package #:lw2.macro-utils) 10 | 11 | (defun compiler-constantp (form &optional environment) 12 | "Like CONSTANTP, but also try expanding compiler macros." 13 | (or (constantp form environment) 14 | (constantp (compiler-macroexpand form environment) environment))) 15 | 16 | (defun macroexpand-both-1 (form &optional environment) 17 | (multiple-value-bind (result expandedp) (macroexpand-1 form environment) 18 | (multiple-value-bind (result c-expandedp) (compiler-macroexpand-1 result environment) 19 | (values result (or expandedp c-expandedp))))) 20 | 21 | (defun macroexpand-both (form &optional environment) 22 | (let (result expandedp any-expandedp) 23 | (iter 24 | (multiple-value-setq (result expandedp) (macroexpand-both-1 form environment)) 25 | (while expandedp) 26 | (setf any-expandedp t)) 27 | (values result any-expandedp))) 28 | 29 | (defun eval-in-environment (form &optional environment) 30 | (funcall (enclose `(lambda () ,form) environment))) 31 | 32 | (defun augment-macros (environment macro-bindings) 33 | "Add a set of MACROLET-style macro bindings to an environment." 34 | (let ((macro-list (iter (for (name args . body) in macro-bindings) 35 | (for macro-lambda = (parse-macro name args body)) 36 | (for macro-fn = (enclose macro-lambda environment)) 37 | (collect (list name macro-fn))))) 38 | (augment-environment environment :macro macro-list))) 39 | 40 | (defmacro macro-as-lambda (name args &body body) 41 | "Create a macro expander function as a lambda form." 42 | (parse-macro name args body)) 43 | 44 | (defmacro macro-list-as-lambdas (&rest clauses) 45 | "Create a list of macro expander functions as lambda forms. This is 46 | suitable for passing to AUGMENT-ENVIRONMENT." 47 | `(list 48 | ,@(iter (for (name args . body) in clauses) 49 | (collect `(list ',name ,(parse-macro name args body)))))) 50 | -------------------------------------------------------------------------------- /src/push-notifications.lisp: -------------------------------------------------------------------------------- 1 | (in-package #:lw2.backend) 2 | 3 | (define-cache-database 'backend-push-notifications "push-subscriptions") 4 | 5 | (export 'make-subscription) 6 | (defun make-subscription (auth-token endpoint expires) 7 | (cache-put "push-subscriptions" 8 | auth-token 9 | (alist :endpoint endpoint 10 | :expires expires) 11 | :value-type :json)) 12 | 13 | (export 'find-subscription) 14 | (defun find-subscription (auth-token) 15 | (cache-get "push-subscriptions" auth-token :value-type :json)) 16 | 17 | (export 'delete-subscription) 18 | (defun delete-subscription (auth-token) 19 | (cache-del "push-subscriptions" auth-token)) 20 | 21 | (export 'send-all-notifications) 22 | (define-backend-function send-all-notifications () 23 | (backend-push-notifications 24 | (let* ((all-subscriptions 25 | (with-collector (col) 26 | (call-with-cursor "push-subscriptions" 27 | (lambda (db cursor) 28 | (declare (ignore db)) 29 | (multiple-value-bind (value key) (cursor-get cursor :first) 30 | (loop while key do 31 | (col (cons key value)) 32 | (multiple-value-setq (value key) (cursor-get cursor :next))))) 33 | :read-only t) 34 | (col))) 35 | (current-time (local-time:now)) 36 | (current-time-unix (local-time:timestamp-to-unix current-time))) 37 | (loop for (auth-token . subscription-json) in all-subscriptions 38 | do (log-and-ignore-errors 39 | (let* ((subscription (json:decode-json-from-string subscription-json)) 40 | (last-check-cons (or (assoc :last-check subscription) (cons :last-check nil))) 41 | (since (if-let (unix (cdr last-check-cons)) (local-time:unix-to-timestamp unix)))) 42 | (cond 43 | ((let ((expires (cdr (assoc :expires subscription)))) (and expires (> current-time-unix expires))) 44 | (delete-subscription auth-token)) 45 | ((sb-sys:with-deadline (:seconds 30) 46 | (check-notifications (cache-get "auth-token-to-userid" auth-token) auth-token :since since)) 47 | (handler-case 48 | (sb-sys:with-deadline (:seconds 30) 49 | (send-notification (cdr (assoc :endpoint subscription)))) 50 | (dex:http-request-gone () 51 | (delete-subscription auth-token)) 52 | (:no-error (&rest args) 53 | (declare (ignore args)) 54 | (setf (cdr last-check-cons) (local-time:timestamp-to-unix current-time)) 55 | (cache-put "push-subscriptions" auth-token (adjoin last-check-cons subscription) :value-type :json)))))))))) 56 | (backend-base nil)) 57 | -------------------------------------------------------------------------------- /src/raw-memory-streams.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.raw-memory-streams 2 | (:use #:cl #:trivial-gray-streams) 3 | (:export #:raw-memory-stream)) 4 | 5 | (in-package #:lw2.raw-memory-streams) 6 | 7 | (defclass raw-memory-stream (fundamental-binary-input-stream) 8 | ((pointer :initarg :pointer) 9 | (length :initarg :length :initform (error "No length parameter supplied when creating raw-memory-stream") :type (and fixnum (integer 0))) 10 | (position :initform 0 :type fixnum :accessor stream-file-position))) 11 | 12 | (defmethod stream-element-type ((self raw-memory-stream)) 13 | '(unsigned-byte 8)) 14 | 15 | (defmethod stream-read-byte ((self raw-memory-stream)) 16 | (declare (optimize (safety 0) (debug 0))) 17 | (with-slots (pointer length position) self 18 | (declare (type (and fixnum (integer 0)) length position) 19 | (type cffi:foreign-pointer pointer)) 20 | (if (>= position length) 21 | :eof 22 | (prog1 23 | (cffi:mem-aref pointer :unsigned-char position) 24 | (setf position (the fixnum (+ 1 position))))))) 25 | 26 | (defmethod stream-read-sequence ((self raw-memory-stream) sequence start end &key) 27 | (declare (optimize (safety 0) (debug 0)) 28 | (type (and fixnum (integer 0)) start end)) 29 | (with-slots (pointer length position) self 30 | (declare (type (and fixnum (integer 0)) length position) 31 | (type cffi:foreign-pointer pointer)) 32 | (let* ((remaining-length (- length position)) 33 | (requested-length (- end start)) 34 | (actual-length (min remaining-length requested-length))) 35 | (declare (type (and fixnum (integer 0)) remaining-length requested-length actual-length)) 36 | (macrolet ((inner () 37 | `(loop for mem-index from position to (+ position actual-length) 38 | for sequence-index from start 39 | do (setf (elt sequence sequence-index) 40 | (cffi:mem-aref pointer :unsigned-char mem-index))))) 41 | (etypecase sequence 42 | ((simple-array (unsigned-byte 8) (*)) (inner)) 43 | ((array (unsigned-byte 8) (*)) (inner)) 44 | (sequence (inner)))) 45 | (setf position (the fixnum (+ position actual-length))) 46 | (the fixnum (+ start actual-length))))) 47 | -------------------------------------------------------------------------------- /src/response.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.response 2 | (:use #:cl #:lw2.utils #:lw2.conditions #:lw2.sites #:lw2.routes) 3 | (:import-from #:lw2.html-reader #:*html-output*) 4 | (:export #:with-response-context #:with-response-stream #:define-json-endpoint) 5 | (:recycle #:lw2-viewer)) 6 | 7 | (in-package #:lw2.response) 8 | 9 | (defun call-with-response-context (fn) 10 | (with-site-context ((let ((host (or (hunchentoot:header-in* :x-forwarded-host) (hunchentoot:header-in* :host)))) 11 | (or (find-site host) 12 | (error "Unknown site: ~A" host)))) 13 | (funcall fn))) 14 | 15 | (defmacro with-response-context (() &body body) 16 | `(dynamic-flet ((fn () ,@body)) (call-with-response-context #'fn))) 17 | 18 | (defun call-with-response-stream (fn) 19 | (unless (eq (hunchentoot:request-method*) :head) 20 | (let ((*html-output* (flex:make-flexi-stream (hunchentoot:send-headers) :external-format :utf-8))) 21 | (handler-case 22 | (funcall fn *html-output*) 23 | (serious-condition () (close *html-output*)) 24 | (:no-error (&rest x) (declare (ignore x)) (finish-output *html-output*)))))) 25 | 26 | (defmacro with-response-stream ((out-stream) &body body) `(dynamic-flet ((fn (,out-stream) ,@body)) (call-with-response-stream #'fn))) 27 | 28 | (defun serve-json-request (fn) 29 | (with-response-context () 30 | (let ((result 31 | (handler-case (funcall fn) 32 | (fatal-error (condition) 33 | (setf (hunchentoot:return-code*) (condition-http-return-code condition)) 34 | (list-cond (t :error (princ-to-string condition)) 35 | #|todo (*debug-mode* :backtrace ...)|#))))) 36 | (setf (hunchentoot:content-type*) "application/json") 37 | (with-response-stream (out-stream) 38 | (json:encode-json result out-stream))))) 39 | 40 | (defmacro define-json-endpoint ((name site-class uri) &body body) 41 | `(define-route ',site-class 'standard-route :name ',name :uri ,uri 42 | :handler (lambda () (dynamic-flet ((fn () ,@body)) (serve-json-request #'fn))))) 43 | -------------------------------------------------------------------------------- /src/routes.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.routes 2 | (:use #:cl #:lw2.utils) 3 | (:export #:route #:route-name #:standard-route #:execute-route 4 | #:function-route #:no-match 5 | #:regex-route)) 6 | 7 | (in-package #:lw2.routes) 8 | 9 | (defclass route () 10 | ((name :initarg :name :accessor route-name :type symbol) 11 | (handler :initarg :handler :type function))) 12 | 13 | (defclass standard-route (route) 14 | ((uri :initarg :uri :type string))) 15 | 16 | (defmethod execute-route ((r standard-route) request-uri) 17 | (with-slots (uri handler) r 18 | (if (string= uri request-uri) 19 | (progn 20 | (funcall handler) 21 | t) 22 | nil))) 23 | 24 | (defclass function-route (route) 25 | ((function :initarg :function :type function))) 26 | 27 | (define-condition no-match () ()) 28 | 29 | (defmethod execute-route ((r function-route) request-uri) 30 | (with-slots (function handler) r 31 | (handler-case 32 | (progn 33 | (multiple-value-call handler (funcall function request-uri)) 34 | t) 35 | (no-match () nil)))) 36 | 37 | (defclass regex-route (function-route) ()) 38 | 39 | (defmethod initialize-instance :around ((r regex-route) &rest args &key regex &allow-other-keys) 40 | (let* ((scanner (ppcre:create-scanner regex)) 41 | (function 42 | (lambda (request-uri) 43 | (multiple-value-bind (match? strings) 44 | (ppcre:scan-to-strings scanner request-uri) 45 | (if match? 46 | (values-list (coerce strings 'list)) 47 | (signal (load-time-value (make-condition 'no-match)))))))) 48 | (apply #'call-next-method 49 | r 50 | :function function 51 | (map-plist 52 | (lambda (k v) (unless (eq k :regex) (list k v))) 53 | args)))) 54 | -------------------------------------------------------------------------------- /src/rwlock.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.rwlock 2 | (:use #:cl) 3 | (:import-from #:sb-thread 4 | #:make-mutex #:with-mutex #:grab-mutex #:release-mutex 5 | #:make-waitqueue #:condition-notify #:condition-broadcast #:condition-wait) 6 | (:import-from #:sb-ext 7 | #:atomic-incf #:atomic-decf) 8 | (:import-from #:sb-sys 9 | #:without-interrupts #:allow-with-interrupts #:with-interrupts) 10 | (:import-from #:alexandria 11 | #:with-gensyms 12 | #:once-only) 13 | (:export #:rwlock #:make-rwlock #:read-lock #:read-unlock #:write-lock #:write-unlock #:with-read-lock #:with-write-lock #:with-rwlock-protect)) 14 | 15 | (in-package #:lw2.rwlock) 16 | 17 | (declaim (inline make-rwlock rwlock-readers rwlock-draining-readers)) 18 | 19 | (defstruct rwlock 20 | (readers 0 :type sb-ext:word) 21 | (draining-readers 0 :type (signed-byte 64)) 22 | (write-mutex (make-mutex)) 23 | (read-waitqueue-mutex (make-mutex)) 24 | (read-waitqueue (make-waitqueue)) 25 | (write-waitqueue-mutex (make-mutex)) 26 | (write-waitqueue (make-waitqueue))) 27 | 28 | (defmacro with-rwlock-accessors ((rwlock) &body body) 29 | `(with-accessors ,(loop for var in '(readers draining-readers write-mutex read-waitqueue-mutex read-waitqueue write-waitqueue-mutex write-waitqueue) 30 | collect `(,var ,(find-symbol (format nil "~A-~A" 'rwlock var) '#:lw2.rwlock))) 31 | ,rwlock ,@body)) 32 | 33 | ;;; States: 34 | ;;; Readers running 35 | ;;; Readers draining 36 | ;;; Writer running 37 | 38 | (declaim (inline read-lock read-unlock)) 39 | 40 | (defun read-lock-slowpath (rwlock) 41 | (with-rwlock-accessors (rwlock) 42 | (with-mutex (read-waitqueue-mutex) 43 | (loop until (evenp (atomic-incf readers 0)) 44 | do (or (condition-wait read-waitqueue read-waitqueue-mutex) (error "Waitqueue error")))) 45 | (values nil))) 46 | 47 | (defun read-lock (rwlock) 48 | (with-rwlock-accessors (rwlock) 49 | (let ((orig-readers (atomic-incf readers 2))) 50 | (when (oddp orig-readers) 51 | (read-lock-slowpath rwlock))) 52 | (values nil))) 53 | 54 | (defun read-unlock-slowpath (rwlock) 55 | (with-rwlock-accessors (rwlock) 56 | (with-mutex (write-waitqueue-mutex) 57 | (decf (the (signed-byte 61) draining-readers)) 58 | (when (= draining-readers 0) 59 | (condition-notify write-waitqueue))) 60 | (values nil))) 61 | 62 | (defun read-unlock (rwlock) 63 | (with-rwlock-accessors (rwlock) 64 | (let ((orig-readers (atomic-decf readers 2))) 65 | (when (oddp orig-readers) 66 | (read-unlock-slowpath rwlock))) 67 | (values nil))) 68 | 69 | (defun write-lock (rwlock) 70 | (with-rwlock-accessors (rwlock) 71 | (grab-mutex write-mutex) 72 | (let ((orig-readers (atomic-incf readers 1))) 73 | (unless (= orig-readers 0) 74 | (with-mutex (write-waitqueue-mutex) 75 | (incf (the (signed-byte 61) draining-readers) (the (signed-byte 61) (ash orig-readers -1))) 76 | (loop until (= draining-readers 0) 77 | do (or (condition-wait write-waitqueue write-waitqueue-mutex) (error "Waitqueue error")))))) 78 | (values nil))) 79 | 80 | (defun write-unlock (rwlock) 81 | (with-rwlock-accessors (rwlock) 82 | (with-mutex (read-waitqueue-mutex) 83 | (atomic-decf readers 1) 84 | (condition-broadcast read-waitqueue)) 85 | (release-mutex write-mutex) 86 | (values nil))) 87 | 88 | (defmacro with-rwlock ((rwlock disposition) &body body) 89 | (multiple-value-bind (lock unlock) (ecase disposition 90 | (:read (values 'read-lock 'read-unlock)) 91 | (:write (values 'write-lock 'write-unlock))) 92 | `(without-interrupts 93 | (allow-with-interrupts 94 | (,lock ,rwlock) 95 | (unwind-protect 96 | (with-interrupts ,@body) 97 | (,unlock ,rwlock)))))) 98 | 99 | (defmacro with-read-lock ((rwlock &key upgrade-fn) &body body) 100 | (if upgrade-fn 101 | (with-gensyms (upgraded) 102 | `(let ((,upgraded nil)) 103 | (flet ((,upgrade-fn () 104 | (without-interrupts 105 | (allow-with-interrupts 106 | (read-unlock ,rwlock) 107 | (write-lock ,rwlock) 108 | (setf ,upgraded t))))) 109 | (without-interrupts 110 | (allow-with-interrupts 111 | (read-lock ,rwlock) 112 | (unwind-protect 113 | (with-interrupts ,@body) 114 | (if (not ,upgraded) 115 | (read-unlock ,rwlock) 116 | (write-unlock ,rwlock)))))))) 117 | `(with-rwlock (,rwlock :read) ,@body))) 118 | 119 | (defmacro with-write-lock ((rwlock) &body body) 120 | `(with-rwlock (,rwlock :write) ,@body)) 121 | 122 | (defmacro with-rwlock-protect (rwlock predicate-form write-form &body read-forms) 123 | " 124 | Protect READ-FORMS from being evaluated when PREDICATE-FORM returns false. 125 | RWLOCK will be locked in read mode. If PREDICATE-FORM returns false, RWLOCK will 126 | be upgraded to write mode and WRITE-FORM will be evaluated. WRITE-FORM should 127 | ensure that PREDICATE-FORM will return true. PREDICATE-FORM may be evaluated 128 | more than once. Returns the values returned by READ-FORMS." 129 | (once-only (rwlock) 130 | (with-gensyms (predicate-fn write-fn read-fn) 131 | `(flet ((,predicate-fn () ,predicate-form) 132 | (,write-fn () ,write-form) 133 | (,read-fn () ,@read-forms)) 134 | (declare (dynamic-extent #',predicate-fn #',write-fn #',read-fn)) 135 | (with-read-lock (,rwlock :upgrade-fn upgrade-lock) 136 | (unless (,predicate-fn) 137 | (upgrade-lock) 138 | (unless (,predicate-fn) 139 | (,write-fn))) 140 | (,read-fn)))))) 141 | -------------------------------------------------------------------------------- /src/schema-type.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.schema-type 2 | (:use #:cl #:lw2.utils) 3 | (:export #:define-schema-type #:undefine-schema-type #:find-schema-type #:schema-bind)) 4 | 5 | (in-package #:lw2.schema-type) 6 | 7 | (defvar *schema-types* nil) 8 | 9 | (defun delete-schema-type (name) 10 | (setf *schema-types* (delete name *schema-types* :key #'car))) 11 | 12 | (defmacro undefine-schema-type (name) 13 | `(eval-when (:compile-toplevel :load-toplevel :execute) 14 | (delete-schema-type ,name))) 15 | 16 | (defmacro define-schema-type (name options fields) 17 | (declare (ignore options)) 18 | `(eval-when (:compile-toplevel :load-toplevel :execute) 19 | (delete-schema-type ,name) 20 | (setf *schema-types* 21 | (acons ,name (alist :fields ',fields) 22 | *schema-types*)))) 23 | 24 | (defun find-schema-type (schema-type-name) 25 | (let ((c (assoc schema-type-name *schema-types*))) 26 | (if c 27 | (cdr c) 28 | (error "Undefined schema-type: ~A" schema-type-name)))) 29 | 30 | (defmacro schema-bind ((schema-type-name datum bindings &key context) &body body) 31 | (let* ((schema-type (find-schema-type schema-type-name)) 32 | (fields (cdr (assoc :fields schema-type)))) 33 | `(alist-bind 34 | ,(loop with added = (make-hash-table :test 'eq) 35 | for type-field in fields 36 | nconc (destructuring-bind (binding-sym type &key alias ((:context field-context)) &allow-other-keys) type-field 37 | (when (and (not (gethash binding-sym added)) 38 | (if (eq bindings :auto) 39 | (or (not field-context) (eq field-context context)) 40 | (member binding-sym bindings :test #'string=))) 41 | (setf (gethash binding-sym added) t) 42 | (list (list* (intern (string binding-sym) *package*) 43 | (if (eq type 'string) 'simple-string type) ; Optimization, assuming strings coming from the backend 44 | ; can never be displaced etc. 45 | (if alias (list alias))))))) 46 | ,datum 47 | ,@body))) 48 | -------------------------------------------------------------------------------- /src/sites.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.sites 2 | (:use #:cl #:lw2.utils #:lw2.context #:lw2.routes #:lw2.backend-modules #:lw2.fonts-modules) 3 | (:import-from #:sb-ext #:defglobal) 4 | (:export 5 | #:*sites* 6 | #:site #:forum-site #:wiki-site 7 | #:login-site #:basic-login-site #:oidc-login-site 8 | #:shortform-site #:ignore-list-site 9 | #:alternate-frontend-site #:lw2-frontend-site #:lesswrong-viewer-site #:ea-forum-viewer-site 10 | #:progress-forum-viewer-site 11 | #:arbital-site 12 | #:site-extended-vote-style 13 | #:site-class #:call-route-handler #:site-class-routes 14 | #:site-uri #:site-host #:site-domain #:site-link-base #:site-secure #:site-backend #:site-title #:site-description #:background-loader-enabled #:site-fonts-source 15 | #:main-site-title #:main-site-abbreviation #:main-site-uri #:always-canonical 16 | #:host-matches #:find-site 17 | #:call-with-site-context #:with-site-context 18 | #:reset-site-definitions 19 | #:define-site 20 | #:define-route)) 21 | 22 | (in-package #:lw2.sites) 23 | 24 | (defglobal *sites* nil) 25 | 26 | (defclass site-class (standard-class) 27 | ((routes :accessor site-class-routes :initform nil))) 28 | 29 | (defmethod closer-mop:validate-superclass ((c site-class) (sc standard-class)) 30 | t) 31 | 32 | (defmethod site-class-routes ((c t)) 33 | nil) 34 | 35 | (defmethod call-route-handler ((original-class site-class) request-uri) 36 | (dolist (class (closer-mop:class-precedence-list original-class)) 37 | (dolist (route (site-class-routes class)) 38 | (when (execute-route route request-uri) 39 | (return-from call-route-handler t))))) 40 | 41 | (defclass site () 42 | ((uri :accessor site-uri :initarg :uri :type simple-string) 43 | (host :accessor site-host :initarg :host :type simple-string) 44 | (domain :accessor site-domain :initarg :domain :initform nil) 45 | (secure :accessor site-secure :initarg :secure) 46 | (title :accessor site-title :initarg :title :type simple-string) 47 | (description :accessor site-description :initarg :description :type simple-string) 48 | (fonts-source :accessor site-fonts-source :initarg :fonts-source :initform (make-instance 'google-fonts-source) :type fonts-source)) 49 | (:metaclass site-class)) 50 | 51 | (defmethod main-site-title ((s site)) nil) 52 | 53 | (defmethod main-site-abbreviation ((s site)) nil) 54 | 55 | (defmethod site-link-base ((s site)) (site-uri s)) 56 | 57 | (defmethod always-canonical ((s site)) nil) 58 | 59 | (defmethod call-route-handler ((s site) request-uri) 60 | (call-route-handler (class-of s) request-uri)) 61 | 62 | (defclass backend-site (site) 63 | ((backend :accessor site-backend :initarg :backend :type backend-base) 64 | (background-loader-enabled :accessor background-loader-enabled :initarg :use-background-loader :initform nil :type boolean)) 65 | (:metaclass site-class)) 66 | 67 | (defclass login-site (backend-site) () 68 | (:metaclass site-class)) 69 | 70 | (defclass forum-site (backend-site) () 71 | (:metaclass site-class)) 72 | 73 | (defclass wiki-site (backend-site) () 74 | (:metaclass site-class)) 75 | 76 | (defclass shortform-site (backend-site) () 77 | (:metaclass site-class)) 78 | 79 | (defclass ignore-list-site (login-site) () 80 | (:metaclass site-class)) 81 | 82 | (defclass alternate-frontend-site (backend-site) 83 | ((main-site-title :accessor main-site-title :initarg :main-site-title :type simple-string) 84 | (main-site-abbreviation :accessor main-site-abbreviation :initarg :main-site-abbreviation :type simple-string) 85 | (main-site-uri :accessor main-site-uri :initarg :main-site-uri :type simple-string) 86 | (always-canonical :accessor always-canonical :initarg :always-canonical :initform nil :type boolean)) 87 | (:metaclass site-class)) 88 | 89 | (defmethod site-link-base ((s alternate-frontend-site)) (main-site-uri s)) 90 | 91 | (defclass lw2-frontend-site (alternate-frontend-site) () 92 | (:metaclass site-class)) 93 | 94 | (defclass lesswrong-viewer-site (forum-site ignore-list-site login-site lw2-frontend-site shortform-site) () 95 | (:metaclass site-class)) 96 | 97 | (defclass ea-forum-viewer-site (forum-site ignore-list-site login-site lw2-frontend-site shortform-site) () 98 | (:metaclass site-class)) 99 | 100 | (defclass progress-forum-viewer-site (forum-site ignore-list-site login-site lw2-frontend-site shortform-site) () 101 | (:metaclass site-class)) 102 | 103 | (defclass arbital-site (wiki-site alternate-frontend-site) () 104 | (:metaclass site-class)) 105 | 106 | (defmethod host-matches ((site site) host) 107 | (let ((site-host (site-host site))) 108 | (and site-host (string-equal site-host host)))) 109 | 110 | (defun find-site (host) 111 | (find-if (lambda (site) (host-matches site host)) 112 | *sites*)) 113 | 114 | (defgeneric site-extended-vote-style (site) 115 | (:method ((site lesswrong-viewer-site)) :lw) 116 | (:method ((site ea-forum-viewer-site)) :ea) 117 | (:method ((site t)) nil)) 118 | 119 | (defgeneric call-with-site-context (site request fn) 120 | (:method :around ((site site) (request t) fn) 121 | (let ((*current-site* site) 122 | (*default-last-modified* (load-time-value (get-universal-time)))) 123 | (call-next-method))) 124 | (:method ((site site) (request t) fn) 125 | (funcall fn)) 126 | (:method ((site backend-site) request fn) 127 | (let* ((backend (site-backend site)) 128 | (*current-backend* backend)) 129 | (call-with-backend-context backend request #'call-next-method)))) 130 | 131 | (defmacro with-site-context ((site &key (request t)) &body body) 132 | `(call-with-site-context ,site ,request (lambda () ,@body))) 133 | 134 | (defun reset-site-definitions () 135 | (setf *sites* nil)) 136 | 137 | (defmacro define-site (&rest args) 138 | (let* ((class 'site) 139 | (args2 140 | (map-plist (lambda (key val) 141 | (cond 142 | ((eq key :class) 143 | (setf class val) 144 | nil) 145 | ((eq key :backend) 146 | (list key `(make-backend ,@val))) 147 | ((eq key :uri) 148 | (let* ((uri (quri:uri val)) 149 | (scheme (quri:uri-scheme uri)) 150 | (host (quri:uri-host uri)) 151 | (port (quri:uri-port uri)) 152 | (default-port (quri.port:scheme-default-port scheme))) 153 | (list key val 154 | :host (format nil "~A~@[:~A~]" 155 | host 156 | (if (/= default-port port) port)) 157 | :secure (string-equal "https" scheme)))) 158 | (t (list key val)))) 159 | args))) 160 | `(push (make-instance ',class ,.args2) *sites*))) 161 | 162 | (defun define-route (site-class route-class &rest args) 163 | (let ((new-route (apply #'make-instance route-class args))) 164 | (setf (site-class-routes (find-class site-class)) 165 | (cons 166 | new-route 167 | (remove (route-name new-route) (site-class-routes (find-class site-class)) :key #'route-name))))) 168 | -------------------------------------------------------------------------------- /src/user-context.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.user-context 2 | (:use #:cl) 3 | (:export #:*current-auth-token* #:*current-auth-status* #:*current-userid* #:*current-username* #:*current-user-slug* #:*current-ignore-hash* 4 | #:logged-in-userid #:logged-in-username #:logged-in-user-slug)) 5 | 6 | (in-package #:lw2.user-context) 7 | 8 | (defvar *current-auth-token*) 9 | (defvar *current-auth-status*) 10 | (defvar *current-userid*) 11 | (defvar *current-username*) 12 | (defvar *current-user-slug*) 13 | (defvar *current-ignore-hash*) 14 | 15 | (defun logged-in-userid (&optional is-userid) 16 | (let ((current-userid *current-userid*)) 17 | (if is-userid 18 | (string= current-userid is-userid) 19 | current-userid))) 20 | 21 | (defun logged-in-username () 22 | *current-username*) 23 | 24 | (defun logged-in-user-slug () 25 | *current-user-slug*) 26 | -------------------------------------------------------------------------------- /src/web-push.lisp: -------------------------------------------------------------------------------- 1 | (uiop:define-package #:lw2.web-push 2 | (:use #:cl #:alexandria #:lw2.utils #:lw2.conditions) 3 | (:export #:get-vapid-public-key #:send-notification)) 4 | 5 | (in-package #:lw2.web-push) 6 | 7 | (defparameter *vapid-key-filename* (asdf:system-relative-pathname :lw2-viewer "webpush.vapid.key")) 8 | (sb-ext:defglobal *vapid-key* nil) 9 | (sb-ext:defglobal *vapid-header-cache* (make-hash-table :test 'equal)) 10 | (sb-ext:defglobal *vapid-header-cache-lock* (sb-thread:make-mutex :name "vapid header cache lock")) 11 | 12 | (defun call-with-private-file (fn name) 13 | (let ((fd nil)) 14 | (unwind-protect 15 | (progn 16 | (setf fd (sb-posix:open name (logior sb-posix:o-creat sb-posix:o-rdwr) #o600)) 17 | (funcall fn (sb-sys:make-fd-stream fd :input t :output t))) 18 | (when fd (sb-posix:close fd))))) 19 | 20 | (defun invoke-node-process (command &optional (output #'json:decode-json)) 21 | (uiop:run-program "node js-foreign-lib/web-push.js" 22 | :input (list command) 23 | :output output 24 | :error-output *error-output*)) 25 | 26 | (defun ensure-vapid-key () 27 | (unless *vapid-key* 28 | (labels ((read-vapid-key (stream) 29 | (setf *vapid-key* (json:decode-json stream)))) 30 | (log-and-ignore-errors 31 | (with-open-file (stream *vapid-key-filename* :direction :input :if-does-not-exist nil) 32 | (if stream 33 | (read-vapid-key stream) 34 | (call-with-private-file (lambda (stream) 35 | (invoke-node-process "webPush.generateVAPIDKeys()" stream) 36 | (file-position stream 0) 37 | (read-vapid-key stream)) 38 | *vapid-key-filename*))))))) 39 | 40 | (ensure-vapid-key) 41 | 42 | (defun get-vapid-public-key () 43 | (cdr (assoc :public-key *vapid-key*))) 44 | 45 | (defun generate-vapid-headers (origin) 46 | (let* ((result-json 47 | (invoke-node-process 48 | (format nil "webPush.getVapidHeaders(~{~A~^,~});" 49 | (mapcar #'json:encode-json-to-string 50 | (list origin "mailto:test@example.com" 51 | (cdr (assoc :public-key *vapid-key*)) (cdr (assoc :private-key *vapid-key*)) "aes128gcm"))))) 52 | (result-string (cdar result-json)) 53 | (result-parts (nth-value 1 54 | (ppcre:scan-to-strings "vapid t=([^,]+), k=([^,]+)" result-string)))) 55 | (assert (= (length result-parts) 2)) 56 | (alist :authorization (format nil "WebPush ~A" (aref result-parts 0)) 57 | :crypto-key (format nil "p256ecdsa=~A" (aref result-parts 1))))) 58 | 59 | (defun get-vapid-headers (origin) 60 | (let ((unlocked-value 61 | (sb-thread:with-mutex (*vapid-header-cache-lock*) 62 | (let ((value (gethash origin *vapid-header-cache*)) 63 | (current-time (get-unix-time))) 64 | (if (and value (< current-time (+ (car value) (* 60 60 12)))) 65 | (cdr value) 66 | (setf (gethash origin *vapid-header-cache*) 67 | (sb-thread:make-thread (lambda () 68 | (let ((vapid (generate-vapid-headers origin))) 69 | (sb-thread:with-mutex (*vapid-header-cache-lock*) 70 | (setf (gethash origin *vapid-header-cache*) (cons current-time vapid))) 71 | vapid)) 72 | :name "generate vapid headers"))))))) 73 | (typecase unlocked-value 74 | (sb-thread:thread (sb-thread:join-thread unlocked-value)) 75 | (t unlocked-value)))) 76 | 77 | (defun send-notification (endpoint &key (ttl (* 60 60 24))) 78 | ;; we don't support content yet since it requires encryption 79 | (let* ((endpoint-uri (quri:uri endpoint)) 80 | (origin (concatenate 'string (quri:uri-scheme endpoint-uri) "://" (quri:uri-authority endpoint-uri))) 81 | (headers (alist* :ttl ttl 82 | :content-encoding "identity" 83 | (get-vapid-headers origin)))) 84 | (dex:request endpoint 85 | :method :post 86 | :headers headers 87 | :keep-alive nil))) 88 | -------------------------------------------------------------------------------- /templates/conversation.html: -------------------------------------------------------------------------------- 1 | {% if conversation %} 2 |
Request a password reset link.
6 | 12 |If your browser is capable of viewing LW 2.0, you can simply follow the instructions in the email.
15 | Otherwise, copy the password reset link from the email and paste it below, and choose a new password: