├── .arc ├── .eslintrc.js ├── .github └── FUNDING.yml ├── .gitignore ├── .nvmrc ├── .prettierrc.yaml ├── README.md ├── __tests__ ├── atom.test.js ├── endpoint.test.js ├── fixtures │ ├── adactio-link.html │ ├── alt-but.html │ ├── brucel.xml │ ├── but.html │ ├── fsis.xml │ ├── jeremy.xml │ ├── marc.xml │ ├── mf-missing.html │ ├── pingback-first.html │ ├── rcp.xml │ ├── rem.xml │ ├── sample.html │ ├── simon-all-atom.xml │ ├── simon-links-atom.xml │ ├── snarfed.html │ └── summary.atom ├── h-entry.test.js ├── index.test.js ├── mf.test.js ├── pingback.test.js └── request.test.js ├── app ├── api │ ├── auth.mjs │ ├── auth │ │ └── $$.mjs │ ├── check.mjs │ ├── docs │ │ └── index.mjs │ ├── index.mjs │ ├── stats.mjs │ └── token.mjs ├── browser │ ├── $.mjs │ ├── check-mention.mjs │ ├── getting-started.mjs │ ├── started-docs.mjs │ └── web-mention.mjs ├── elements │ ├── app-footer.mjs │ ├── app-layout.mjs │ ├── app-logo.mjs │ ├── app-navigation.mjs │ ├── check-mention.mjs │ ├── getting-started.mjs │ ├── started-docs.mjs │ └── web-mention.mjs ├── head.mjs └── pages │ ├── about.html │ ├── check.mjs │ ├── docs │ ├── index.html │ └── todo.html │ ├── index.mjs │ ├── stats.mjs │ ├── token-failure.html │ └── token.mjs ├── bin └── wm.js ├── index.d.ts ├── jsconfig.json ├── package-lock.json ├── package.json ├── prefs.arc ├── public ├── copy.svg ├── favicon.png ├── logo.svg ├── netlify.png ├── style.css └── webmention-app-card.jpg └── shared ├── enhance-styles ├── lib ├── db.js ├── endpoint.js ├── get-wm-endpoints.js ├── html │ └── dom.js ├── ignored-endpoints.js ├── links.js ├── microformat │ └── dom.js ├── passport.js ├── query-string.js ├── request.js ├── rss │ ├── dom.js │ └── is.js ├── send │ ├── index.js │ ├── pingback.js │ └── webmention.js ├── uuid.js └── webmention.js └── static.json /.arc: -------------------------------------------------------------------------------- 1 | @app 2 | webmention-dot-app 3 | 4 | @static 5 | prune true 6 | fingerprint true # or false 7 | 8 | @plugins 9 | enhance/arc-plugin-enhance 10 | 11 | @aws 12 | timeout 30 13 | profile arc.code 14 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | env: { 4 | node: true, 5 | es6: true, 6 | }, 7 | extends: 'eslint:recommended', 8 | rules: { 9 | indent: ['error', 2], 10 | }, 11 | ignorePatterns: [], 12 | parserOptions: { 13 | sourceType: 'module', 14 | ecmaVersion: 'latest', 15 | }, 16 | }; 17 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: remy 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .vscode 3 | dist 4 | .nuxt 5 | .nyc_output 6 | .env 7 | .env.* 8 | tmp 9 | .now 10 | .tap 11 | # Enhance temp files 12 | .enhance/ 13 | 14 | coverage/ 15 | 16 | # Generated assets 17 | public/static.json 18 | public/browser/ 19 | public/bundles/ 20 | public/pages/ 21 | 22 | # Architect CloudFormation 23 | sam.json 24 | sam.yaml 25 | 26 | .DS_Store 27 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 20 2 | -------------------------------------------------------------------------------- /.prettierrc.yaml: -------------------------------------------------------------------------------- 1 | singleQuote: true 2 | trailingComma: 'es5' 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # webmention.app 2 | 3 | ## Automate your outgoing webmentions 4 | 5 | [webmention.app](https://webmention.app) is a platform agnostic service that will check a given URL for links to other sites, discover if they support webmentions, then send a webmention to the target. 6 | 7 | This repository also includes a stand alone command line tool that doesn't rely on [webmention.app](https://webmention.app) at all and doesn't require a token - so you can run it locally with the knowledge that if your site outlives this one, the tool will still work. 8 | 9 | ### Installation 10 | 11 | The tool uses nodejs and once nodejs is installed, you can install the tool using: 12 | 13 | ``` 14 | $ npm install @remy/webmention 15 | ``` 16 | 17 | This provides an executable under the command webmention (also available as wm). Default usage allows you to pass a filename (like a newly generated RSS feed) or a specific URL. It will default to the 10 most recent entries found (using item for RSS and `h-entry` for HTML). 18 | 19 | ### Usage 20 | 21 | By default, the command will perform a dry-run/discovery only. To complete the notification of webmentions use the `--send` flag. 22 | 23 | The options available are: 24 | 25 | - `--send` (default: false) send the webmention to all valid endpoints 26 | - `--limit n` (default: 10) limit to n entries found 27 | - `--debug` (default: false) print internal debugging 28 | Using npx you can invoke the tool to read the latest entry in your RSS feed: 29 | 30 | ``` 31 | $ npx webmention https://yoursite.com/feed.xml --limit 1 --send 32 | ``` 33 | 34 | Alternatively, you can make the tool part of your build workflow and have it execute during a postbuild phase: 35 | 36 | ```json 37 | { 38 | "scripts": { 39 | "postbuild": "webmention dist/feed.xml --limit 1 --send" 40 | } 41 | } 42 | ``` 43 | 44 | ## Misc 45 | 46 | - Further documentation found on [webmention.app/docs](https://webmention.app/docs) 47 | - Built by [@rem](https://remysharp.com) 48 | - MIT / [rem.mit-license.org](https://rem.mit-license.org/) 49 | -------------------------------------------------------------------------------- /__tests__/atom.test.js: -------------------------------------------------------------------------------- 1 | const tap = require('tap'); 2 | const fs = require('fs'); 3 | const read = (f) => fs.readFileSync(__dirname + f, 'utf8'); 4 | const parse = require('../shared/lib/rss/dom'); 5 | const { links } = require('../shared/lib/links'); 6 | 7 | tap.test('compile dom for atom', async (t) => { 8 | t.plan(1); 9 | 10 | const xml = read('/fixtures/summary.atom'); 11 | links(await parse(xml, 10)); 12 | t.pass('worked'); 13 | }); 14 | 15 | tap.test('detected escaped links in atom', async (t) => { 16 | t.plan(2); 17 | 18 | const xml = read('/fixtures/summary.atom'); 19 | const dom = await parse(xml, 10); 20 | 21 | const [res] = links(dom); 22 | t.same(res.links.length, 1, 'finds example.com'); 23 | t.same(res.links[0], 'https://example.com/marker', 'finds example.com'); 24 | }); 25 | -------------------------------------------------------------------------------- /__tests__/endpoint.test.js: -------------------------------------------------------------------------------- 1 | const tap = require('tap'); 2 | const fs = require('fs'); 3 | const read = (f) => fs.readFileSync(__dirname + f, 'utf8'); 4 | const endpoint = require('../shared/lib/endpoint'); 5 | 6 | tap.test('atom', (t) => { 7 | const source = read('/fixtures/pingback-first.html'); 8 | 9 | const res = endpoint.findEndpoints(source, 'https://www.w3.org/TR/websub/'); 10 | t.equal(res.type, 'webmention'); 11 | t.end(); 12 | }); 13 | 14 | tap.test('endpoint main', async (t) => { 15 | const res = await endpoint({ url: 'https://remysharp.com/' }); 16 | 17 | t.equal(res.endpoint.type, 'webmention'); 18 | t.end(); 19 | }); 20 | -------------------------------------------------------------------------------- /__tests__/fixtures/adactio-link.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Adactio: Links—How I failed the <a> 9 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 80 | 81 |
82 | 83 | 84 | 85 |
86 |
87 |
88 | 89 | 90 | 91 |
92 | 93 | 102 | 103 |
104 | 105 |
106 | 107 | 108 | 109 |
110 |
111 | 112 |
113 | 114 |

115 | 116 | How I failed the <a> 117 | 118 |

119 |

120 | 123 |

124 | 125 |

I think the situation that Remy outlines here is quite common (in client-rehydrated server-rendered pages), but what’s less common is Remy’s questioning and iteration.

126 | 127 |
128 |

So I now have a simple rule of thumb: if there’s an onClick, there’s got to be an anchor around the component.

129 |
130 | 131 |
132 | 133 | How I failed the <a> 134 |

135 | 138 |

139 |

140 | Tagged with 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 |

154 | 155 |

156 | 157 | 158 |

159 | 160 | 161 |
162 |
163 | 164 | 231 | 232 | 233 | 234 | 235 | 238 | 239 | 240 | 256 | 281 | 282 | 283 | 284 | -------------------------------------------------------------------------------- /__tests__/fixtures/but.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | But… 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 41 | 42 | 43 | 55 |
56 |
57 |

But…

58 |

59 |

I've always had a curious fascination with the english language, and words and meaning in particular. I think it stems from my early misunderstands and confusion around the damn thing. And probably my own dyslexia. I never understood why the word "woman" needed to have the word "man" as part of it. It seemed unfair…and stupid.

60 |

But in an especially interesting one. It's like the word exist entirely to cancel out everything before it. The word but should be defined as something like "the exact opposite of the preposition is true". Or as Jon Snow puts it:

61 |
62 |

Sansa Stark: They respect you, they really do, but you have to... Why are you laughing?

63 |

Jon Snow: What did father used to say? Everything before the word "but" is horse shit.

64 |
65 | 66 |

I've read and heard it first hand: I'm not sexists, but…

67 |

I'm totally guilty of this, but I didn't really notice until I had kids. I find that I might overreact to a situation and try to apologise to my child. I'd say:

68 |
69 |

I'm sorry that I shouted at you, but…

70 |
71 |

…then I'd catch myself. "But". But…somehow it's their fault? Is that what I want to pass on? It's like I'm faking being sorry.

72 |

So I think this is a word I'd like to retire now, certainly retire in the context of apologies. When I see it included after something like "I'm not racist, but—" I'm always going to think the opposite.

73 |

Originally posted on remy.blog

74 | 75 |

Posted under personal.

76 | 118 |
119 |
120 |
121 | 179 | 186 | 187 | 188 | 189 | 237 | 238 | -------------------------------------------------------------------------------- /__tests__/fixtures/fsis.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | fsis.site 🐴 5 | https://fsis.site/blog/ 6 | Recent content on fsis.site 🐴 7 | Hugo -- gohugo.io 8 | en-US 9 | Copyright © 2020, FSi 10 | Tue, 13 Oct 2020 00:00:00 +0000 11 | 12 | Links for 2020-10-13 13 | https://fsis.site/blog/20201013173555/ 14 | Tue, 13 Oct 2020 00:00:00 +0000 15 | 16 | https://fsis.site/blog/20201013173555/ 17 | plug-n-pwn Unfortunately Intel has placed large amounts of information about the Thunderbolt controllers and protocol under NDA, meaning that it has not been properly researched leading to a string of vulnerabilities over the years. 18 | Vt Cup Of Water, No Water, Wasp 19 | 20 | 21 | 22 | Migrated to Hugo 23 | https://fsis.site/blog/2020-10-06/ 24 | Tue, 06 Oct 2020 00:00:00 +0000 25 | 26 | https://fsis.site/blog/2020-10-06/ 27 | This is something I&rsquo;ve wanted to do for a long time (11 months? omg), but here it is at last. This site/blog is now powered by Hugo. I tried to keep stuff similar to how it was before, while adding my own flair. 28 | The feed URL has been changed. I guess you know that if you&rsquo;re reading this, but make sure to update your RSS reader otherwise. 29 | Also, as the sources for this site is now in plain markdown, I&rsquo;ve merged them with my (previously private) notes, so now you can enjoy such masterpieces as my meat marinades and add-srt-to-mp4. 30 | 31 | 32 | 33 | Letter from Italy 34 | https://fsis.site/blog/2020-06-19-letter-from-italy/ 35 | Tue, 09 Jun 2020 14:34:00 +0300 36 | 37 | https://fsis.site/blog/2020-06-19-letter-from-italy/ 38 | Today I&rsquo;ve finally got a letter from a friend in Italy that took more than a month to arrive. Understandable, I guess, with the quarantine affecting the post office and all that. 39 | 40 | 41 | 42 | Middle of November 43 | https://fsis.site/blog/2019-11-16/ 44 | Sat, 16 Nov 2019 12:37:59 +0000 45 | 46 | https://fsis.site/blog/2019-11-16/ 47 | Programming Laser-Based Audio Injection on Voice-Controllable Systems 48 | TL;DR: 49 | By shining the laser through the window at microphones inside smart speakers, tablets, or phones, a far away attacker can remotely send inaudible and potentially invisible commands which are then acted upon by Alexa, Portal, Google assistant or Siri. 50 | or, as they say in Russia, РЕШЕТО 51 | Text Rendering Hates You 52 | Fun quote: 53 | Also have you heard of Animated SVG Fonts? 54 | 55 | 56 | 57 | End of October Spooky Post 58 | https://fsis.site/blog/2019-11-01/ 59 | Fri, 01 Nov 2019 00:00:00 +0000 60 | 61 | https://fsis.site/blog/2019-11-01/ 62 | Real life: happens. 63 | Me: 64 | Programming XML is almost always misused Articles Exfiltrating information using the light sensor 65 | Might not sound like much, but they mean, like, in a browser context. From JavaScript. 66 | Why Renaissance Paintings Aren&rsquo;t as Green as They Used to Be 67 | TL;DR: the green pigment they used is getting less green with time. 68 | Nonviolent communication 69 | A neat idea, should probably give the book a go. 70 | 71 | 72 | 73 | Untitled Goose Post 74 | https://fsis.site/blog/2019-10-13/ 75 | Sun, 13 Oct 2019 20:33:39 +0000 76 | 77 | https://fsis.site/blog/2019-10-13/ 78 | Programming CollapseOS 79 | TL;DR: Someone is making an 8-bit OS just in case to be useful in post-collapse scenarios. 80 | With a copy of this project, a capable and creative person should be able to manage to build and install Collapse OS without external resources (i.e. internet) on a machine of her design, built from scavenged parts with low-tech tools. 81 | Don&rsquo;t Look Into The Light 82 | TL;DR: 83 | 84 | 85 | 86 | Are plants intelligent enough to use PHP? 87 | https://fsis.site/blog/2019-09-30/ 88 | Mon, 30 Sep 2019 19:48:10 +0000 89 | 90 | https://fsis.site/blog/2019-09-30/ 91 | Programming No, PHP doesn&rsquo;t have closures Articles WeWork and Counterfeit Capitalism 92 | What if Planet 9 is a Primordial Black Hole? 93 | What killed me was the &ldquo;exact scale illustration of a 5 Earth mass black hole&rdquo; 94 | The Octopus: An Alien Among Us 95 | Clickbaity quote: 96 | Yes, an octopus is objectively aware of itself and of the objects around it. It contains the information. 97 | 98 | 99 | 100 | Folding paper globes and simulated universe 101 | https://fsis.site/blog/2019-09-23/ 102 | Mon, 23 Sep 2019 19:38:54 +0000 103 | 104 | https://fsis.site/blog/2019-09-23/ 105 | &hellip;and this time the post is small-ish (and a week delayed) because I suddenly got sick (well, not suddenly, it was all because I spent a night at that The Comet is Coming concert, probably) and just started to return to the rhythm of things. 106 | Programming POS software written in QBasic, used worldwide Articles The Termination Risks of Simulation Science 107 | TL;DR: a scientist argues that it&rsquo;s quite unreasanable to expect the beings running our universe&rsquo;s simulation (if it indeed is one) to stop simulating it just because we&rsquo;ve found out it is a simulation. 108 | 109 | 110 | 111 | Moscow Music Week report and some links 112 | https://fsis.site/blog/2019-09-08/ 113 | Sun, 08 Sep 2019 21:52:05 +0000 114 | 115 | https://fsis.site/blog/2019-09-08/ 116 | Shorter one this time, for I was attending Moscow Music Week. 117 | It&rsquo;s a festival here in Moscow, with quite a few places and performances (that, sadly, happen all at the same time) for four days straight. 118 | Due to the aforementioned happening-at-the-same-time, I was only able to attend a couple of the showcases, namely Free Jazz and Dark Jazz ones. Quite liked these (even though the Dark Jazz showcase there were three free jazz performances and only one dark-jazz one). 119 | 120 | 121 | 122 | Another digest 123 | https://fsis.site/blog/2019-09-01/ 124 | Sun, 01 Sep 2019 09:24:39 +0000 125 | 126 | https://fsis.site/blog/2019-09-01/ 127 | Aaand back to our regularly scheduled program. Speaking of programming&hellip; 128 | Programming Rabbit holes 129 | TL;DR: how ubuntu generates its motd is weird (and pings their server once every 12 hours) 130 | Funny quote 131 | It&rsquo;s quite simple. PHP was not designed. It evolved, without a predator to remove bad mutations. &ndash; The_Sly_Marbo on reddit 132 | Articles Black hole &ldquo;so big it shouldn&rsquo;t exist&rdquo; 133 | Note: the title is quite click-baity. 134 | 135 | 136 | 137 | August travels 138 | https://fsis.site/blog/2019-08-26/ 139 | Mon, 26 Aug 2019 17:43:50 +0000 140 | 141 | https://fsis.site/blog/2019-08-26/ 142 | Here&rsquo;s a brief log of my travels to make up for all the missed posts (as I, in fact, pledged to post once a week or so). 143 | Brutal Assault (August 7 to August 11) Brutal Assault is a yearly heavy metal festival held in fortress Josefov, near Jaromerz in Czech Republic. 144 | It is held in an old fortress that has seen WWII, WWI and many wars before that, presumably (being medieval and all that). 145 | 146 | 147 | 148 | Light Sail, OH GOD WHY and some grindcore 149 | https://fsis.site/blog/2019-08-04/ 150 | Sun, 04 Aug 2019 10:31:11 +0000 151 | 152 | https://fsis.site/blog/2019-08-04/ 153 | Articles LightSail2 is working! 154 | A nuclear war might make crops stop growing, so we need a backup plan. 155 | TL;DR: prepping for the apocalypse but scientifically! 156 | (Won&rsquo;t read the book because fuck elsevier, but it does sound interesting) 157 | Why are the prices so damn high? 158 | Mostly linking it for this tangent that almost hurts to read: 159 | The feeling is “subjective” in the sense that it occurs inside your mind, but it is “objective” in the sense that you cannot get it arbitrarily by wishing; some things produce it and some do not. 160 | 161 | 162 | 163 | Marble, dungeons and quantum darwinism 164 | https://fsis.site/blog/2019-07-27/ 165 | Sat, 27 Jul 2019 14:41:29 +0000 166 | 167 | https://fsis.site/blog/2019-07-27/ 168 | First off, some updates: 169 | I&rsquo;ve removed the Google Fonts dependency. Uses a couple of self-hosted fonts with system fallback now. Small visual updates as well. Now on to business as usual. 170 | Programming The Mutable Web, via hackernews 171 | SVG PORN 172 | NB: not actually porn, just a collection of hand-crafted SVG icons 173 | (via hackernews) 174 | Dungeon Generation in Diablo 1 175 | TL;DR: some were heavily inspired by Rogue and the likes. 176 | 177 | 178 | 179 | The best medieval snails rendering bicycle supernests in human skin 180 | https://fsis.site/blog/2019-07-21/ 181 | Sun, 21 Jul 2019 13:27:41 +0000 182 | 183 | https://fsis.site/blog/2019-07-21/ 184 | Programming The Best Refactoring You&rsquo;ve Never Heard Of 185 | TL;DR: defunctionalize the continuation! 186 | The PGP Problem 187 | TL;DR: no solution in sight (just &ldquo;use signal&rdquo; is not a solution). 188 | Neat quote: 189 | Take AEAD ciphers: the Rust-language Sequoia PGP defaulted to the AES-EAX AEAD mode, which is great, and nobody can read those messages because most PGP installs don’t know what EAX mode is, which is not great 190 | 191 | 192 | 193 | Ottoman floating point zip bombs on the moon 194 | https://fsis.site/blog/2019-07-13/ 195 | Sat, 13 Jul 2019 11:55:19 +0000 196 | 197 | https://fsis.site/blog/2019-07-13/ 198 | Programming Floating point routines for 6502 by Woz 199 | (direct link) 200 | A better zip bomb; NON-RECURSIVE zip-bombs that extract from 42 kB to 5.5 GB, from 10MB to 281TB, or from 46MB to 4.5PB (last one is Zip64 and not as compatible). Non-recursive means you get that amount of date in a single extract operation. 201 | Not quite useful information, but interesting nonetheless. 202 | Articles Designing the First Full-Time Human Habitat on the Moon, original press-release 203 | 204 | 205 | 206 | Live-coding long-term lewd asshole torture storage 207 | https://fsis.site/blog/2019-07-09/ 208 | Tue, 09 Jul 2019 18:35:52 +0000 209 | 210 | https://fsis.site/blog/2019-07-09/ 211 | Programming Live coding a vi for CP/M, from scratch 212 | Somewhat related (and ancient), Linux on an 8-bit micro by emulating an ARM processor, no less! (slower than molasses, obviously, like, literally, hours to boot) 213 | Also related, Why 80s BASIC still matters! - this made me want to buy the book, and I probably will. 214 | Articles Inside the World&rsquo;s First Long-term Storage Facility for Highly Radioactive Nuclear Waste 215 | 216 | 217 | 218 | Don't take fractal fireball notes written in FORTH 219 | https://fsis.site/blog/2019-06-29/ 220 | Sat, 29 Jun 2019 11:00:00 +0000 221 | 222 | https://fsis.site/blog/2019-06-29/ 223 | Programming 1991: A SERVER-SIDE WEB FRAMEWORK WRITTEN IN FORTH, github 224 | BECAUSE OF COURSE 225 | Articles Don&rsquo;t take notes with a laptop 226 | TL;DR: This has been told again and again, but hand-writing lecture notes makes you remember them better (and makes your hands hurt a lot) 227 | Floppycasts! 228 | TL;DR: if satisfied with absolutely terrifying quality, you can fit about 22 minutes of MP3 speech on a floppy disk, or about 30 minutes of Opus (approximately as terrifying but in a different way) 229 | 230 | 231 | 232 | Next level mead and honey outcome of amateur opossum romance 233 | https://fsis.site/blog/2019-06-16/ 234 | Sun, 16 Jun 2019 21:15:00 +0000 235 | 236 | https://fsis.site/blog/2019-06-16/ 237 | Programming Next level fork bomb 238 | WARNING: it&rsquo;s a fork bomb, don&rsquo;t paste it anywhere! 239 | TL;DR: a fork bomb with a couple extra layers of deviousness (via bici) 240 | Articles FunKey 241 | TL;DR: TL;DR: A keychain with a bunch of emulators, some buttons and an LCD screen; like a really tiny nintendo ds 242 | Stardock and Star Control creators settle lawsuits—with mead and honey 243 | TL;DR: The kind of news headline you read first thing in the morning and do a double take 244 | 245 | 246 | 247 | Russian independence day special 248 | https://fsis.site/blog/2019-06-12/ 249 | Wed, 12 Jun 2019 10:11:00 +0000 250 | 251 | https://fsis.site/blog/2019-06-12/ 252 | Programming On Dat:// 253 | TL;DR: Dat is a distributed web network, and this post talks about building a clone of an old (and dead) mixtape-sharing website muxtape with it, called duxtape (github). 254 | Suffice it to say, I am very interested. 255 | Articles You (probably) don&rsquo;t need ReCAPTCHA 256 | TL;DR: (probably) yes 257 | I myself quite liked the proof-of-work captcha concept (i.e. &ldquo;mine some hashes for me if you want to comment; if you want to spam a lot, it&rsquo;s ok, just mine a lot of hashes&rdquo;) 258 | 259 | 260 | 261 | Horrifying Satanic Space Colonies: Odyssey 262 | https://fsis.site/blog/2019-06-08/ 263 | Sat, 08 Jun 2019 11:27:00 +0000 264 | 265 | https://fsis.site/blog/2019-06-08/ 266 | Programming Horrifying PDF experiments 267 | e.g. breakout Articles How to Join a Social Network in 1998 268 | TL;DR you mail them CSV of your friends in response 269 | Ancient Egyptian Bread (unrolled here) 270 | Paradigms and priors 271 | TL;DR how science works 272 | I found two identical packs of Skittles among 468 packs 273 | TL;DR why 274 | Can we all please stop using medium? 275 | 276 | 277 | 278 | Bizzarre Japanese sorting and exploding things 279 | https://fsis.site/blog/2019-04-14/ 280 | Sun, 14 Apr 2019 16:39:43 +0000 281 | 282 | https://fsis.site/blog/2019-04-14/ 283 | Programming Sorting in Japanese is hard 284 | There are four Japanese women whose names you have to sort: Junko, Atsuko, Kiyoko, and Akiko. This does not seem difficult, until they each show you how they write their names in kanji: 285 | 淳子 (Junko) 淳子 (Atsuko) 淳子 (Kiyoko) 淳子 (Akiko) AMP for email is a terrible idea 286 | TL;DR: yes. 287 | Articles Israel&rsquo;s Moon lander crashed, and that&rsquo;s OK 288 | 289 | 290 | 291 | Can't unsee these insecure Wordpress plugins 292 | https://fsis.site/blog/2019-04-09/ 293 | Tue, 09 Apr 2019 12:00:00 +0000 294 | 295 | https://fsis.site/blog/2019-04-09/ 296 | Programming Water is still wet. In other news, wordpress plugin ecosystem is still trash a ball of slimy writhing unspeakable horrors. 297 | one two hackernews discussion TL;DR: If you ever used anything from these &ldquo;pipdig&rdquo; guys on your wordpress or blogger site, please stop asap. 298 | I personally recommend switching to Hugo instead (do as I say, not as I do; you generally don&rsquo;t need the complexity of Gatsby). 299 | 300 | 301 | 302 | Garfield phones, x-ray records and phone tombs 303 | https://fsis.site/blog/2019-03-31/ 304 | Sun, 31 Mar 2019 12:00:00 +0000 305 | 306 | https://fsis.site/blog/2019-03-31/ 307 | Programming Exercises in Emulation: Xbox 360’s FMA Instruction Articles (quote) Algorithmic recommendations [&hellip;] are failing users by encouraging them to take their interests to the farthest maximum: a you like coffee; have you tried cocaine? kind of effect. Are these features necessary? 308 | What Achilles should have said to the Tortoise 309 | Garfield phones beach mystery finally solved after 35 years 310 | Procrastination Sucks — So Here’s The “Eat That Frog” Way to Powerful Productivity 311 | 312 | 313 | 314 | Doom, Fire and Wombats 315 | https://fsis.site/blog/2019-03-17/ 316 | Sun, 17 Mar 2019 12:00:00 +0000 317 | 318 | https://fsis.site/blog/2019-03-17/ 319 | Programming How Doom fire was done 320 | A short and to the point tutorial. (&ldquo;Doom&rdquo; here refers to PSX and N64 Doom ports) 321 | Articles Going old school: how I replaced Facebook with email 322 | In fact, this is what finally inspired me to do this blog 323 | How to balance full-time work with creative projects 324 | TL;DR: Find a better job, don&rsquo;t think you&rsquo;ll be able to do a lot anyway 325 | 326 | 327 | 328 | I should probably change the design and remove these titles 329 | https://fsis.site/blog/2019-03-10/ 330 | Sun, 10 Mar 2019 20:00:00 +0000 331 | 332 | https://fsis.site/blog/2019-03-10/ 333 | Programming Wave Function Collapse 334 | An algorihtm that generates bitmaps that are locally similar to the input bitmap. 335 | Can be used for e.g. texture or dungeon generation. 336 | NB: Has nothing quantum or wave-functiony in it. 337 | 3d City generator that uses it 338 | Articles Speaking of quantum theory, Quantum theory cannot consistently describe the use of itself 339 | Turns out, we&rsquo;d need some new theory to describe both microscopic and macroscopic events . 340 | 341 | 342 | 343 | Smallish one this time 344 | https://fsis.site/blog/2019-03-03/ 345 | Sun, 03 Mar 2019 17:10:24 +0000 346 | 347 | https://fsis.site/blog/2019-03-03/ 348 | Programming Why Don&rsquo;t People Use Formal Methods 349 | (because they are normally not worth the investment) 350 | Articles ETS isn&rsquo;t TLS and you should not use it 351 | We Need Chrome No More 352 | How Chickens Lost Their Penises 353 | Music Wooden Shjips - Back to Land (2013) 354 | Hard psychedelic rock, very psychedelic in fact. Highly recommended. 355 | Marxthrone - A Blaze in the Western Sky 356 | 357 | 358 | 359 | Links and updates 360 | https://fsis.site/blog/2019-02-24/ 361 | Sun, 24 Feb 2019 23:44:00 +0000 362 | 363 | https://fsis.site/blog/2019-02-24/ 364 | Programming C is Not a Low-level Language 365 | All programming languages are abstractions, in some way. Even simplistic ones, like C and x86 assembly, are now quite far from the bare bones of the hardware (at least, in the x86-world) &ndash; there&rsquo;s just too many layers of intermediate translations and optimizations that the CPU does by itself. 366 | RISC architectures are better in that regard, as even x86 CPUs are RISC-like internally nowadays (or so they say), so at least it&rsquo;s more transparent that way. 367 | 368 | 369 | 370 | Links and updates 371 | https://fsis.site/blog/2019-02-19/ 372 | Tue, 19 Feb 2019 18:14:18 +0000 373 | 374 | https://fsis.site/blog/2019-02-19/ 375 | In this post, I will try and gather some interesting stuff I found around the internets. 376 | Event sourcing is hard. I used to work on an large product which was based on event-sourcing, and can agree on many points made by the author. It was very fun, but also a lot of pain. 377 | I guess, eventually, when the right tooling will be sourced, it will be usable&hellip; (Apache Kafka/Zamza come to mind; then again, it were these exact things that made us consider event-sourcing in the first place) 378 | 379 | 380 | 381 | Hello 382 | https://fsis.site/blog/2019-01-26_first/ 383 | Sat, 26 Jan 2019 13:06:30 +0000 384 | 385 | https://fsis.site/blog/2019-01-26_first/ 386 | Hello and welcome to my personal space, where I will probably post random links for, like, two people to notice. 387 | I made it using gatsby.js, and the experience was OK. In retrospect I think I should&rsquo;ve used some better starter, and not try and bend the default one to my will. Oh well. 388 | This whole thing seems to be working, so yay. 389 | Here&rsquo;s a list of things that I still have to do at some point: 390 | 391 | 392 | 393 | 394 | -------------------------------------------------------------------------------- /__tests__/fixtures/mf-missing.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Good morning! Watch this video ... • Aaron Parecki 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 |
48 |
49 |
50 |
51 |
52 | 53 | 54 |
55 |
56 | 57 | 58 | 59 | 60 | 61 | 62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 | 70 |
71 |

Aaron Parecki

72 |
73 | 74 |
75 | 80 |
81 | 82 |
83 |
84 |
85 | 287 | 288 |
289 | Posted in 290 | /notes 291 | using 292 | quill.p3k.io 293 | 294 | 295 |
296 | 297 |
298 |
299 |
300 | 301 | 452 | 453 | 454 | 455 | 456 | 457 | 458 | -------------------------------------------------------------------------------- /__tests__/fixtures/rcp.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | pingback.ping 4 | 5 | https://remysharp.com/2017/08/12/but 6 | https://rems.life/but/ 7 | 8 | 9 | -------------------------------------------------------------------------------- /__tests__/fixtures/simon-links-atom.xml: -------------------------------------------------------------------------------- 1 | 2 | Simon Willison's Weblog: Blogmarkshttp://simonwillison.net/2019-05-30T04:35:42+00:00Simon WillisonLos Angeles Weedmaps analysis2019-05-30T04:35:42+00:002019-05-30T04:35:42+00:00http://simonwillison.net/2019/May/30/los-angeles-weedmaps-analysis/#atom-blogmarks<p><a href="https://nbviewer.jupyter.org/github/datadesk/la-weedmaps-analysis/blob/master/notebook.ipynb">Los Angeles Weedmaps analysis</a></p> 3 | <p>Ben Welsh at the LA Times published this Jupyter notebook showing the full working behind a story they published about LA&#39;s black market weed dispensaries. I picked up several useful tricks from it - including how to load points into a geopandas GeoDataFrame (in epsg:4326 aka WGS 84) and how to then join that against the LA Times neighborhoods GeoJSON boundaries file.</p> 4 | 5 | <p>Via <a href="https://twitter.com/palewire/status/1133723284116086784">Ben Welsh</a></p> 6 | 7 | Building a stateless API proxy2019-05-30T04:28:55+00:002019-05-30T04:28:55+00:00http://simonwillison.net/2019/May/30/building-a-stateless-api-proxy/#atom-blogmarks<p><a href="https://blog.thea.codes/building-a-stateless-api-proxy/">Building a stateless API proxy</a></p> 8 | <p>This is a really clever idea. The GitHub API is infuriatingly coarsely grained with its permissions: you often end up having to create a token with way more permissions than you actually need for your project. Thea Flowers proposes running your own proxy in front of their API that adds more finely grained permissions, based on custom encrypted proxy API tokens that use JWT to encode the original API key along with the permissions you want to grant to that particular token (as a list of regular expressions matching paths on the underlying API).</p> 9 | 10 | <p>Via <a href="https://twitter.com/theavalkyrie/status/1133864634178424832">@theavalkyrie</a></p> 11 | 12 | datasette-jq2019-05-30T01:52:57+00:002019-05-30T01:52:57+00:00http://simonwillison.net/2019/May/30/datasette-jq/#atom-blogmarks<p><a href="https://github.com/simonw/datasette-jq">datasette-jq</a></p> 13 | <p>I released another tiny Datasette plugin: datasette-jq registers a single custom SQL function, jq(), which lets you execute the jq expression language against a JSON column (or literal value) to filter and transform the JSON data. The README includes a link to a live demo - it&#39;s a neat way to play with the jq micro-language.</p> 14 | 15 | <p>Via <a href="https://twitter.com/simonw/status/1133912206859313152">@simonw</a></p> 16 | 17 | Falsehoods Programmers Believe About Search2019-05-29T20:09:23+00:002019-05-29T20:09:23+00:00http://simonwillison.net/2019/May/29/falsehoods-programmers-believe-about-search/#atom-blogmarks<p><a href="https://opensourceconnections.com/blog/2019/05/29/falsehoods-programmers-believe-about-search/">Falsehoods Programmers Believe About Search</a></p> 18 | <p>These are great. &quot;When you find the boolean operator ‘OR’, you always know it doesn’t mean Oregon&quot;.</p> 19 | 20 | <p>Via <a href="https://news.ycombinator.com/item?id=20039891">Hacker News</a></p> 21 | 22 | gls: Goroutine local storage2019-05-28T23:13:38+00:002019-05-28T23:13:38+00:00http://simonwillison.net/2019/May/28/gls-goroutine-local-storage/#atom-blogmarks<p><a href="https://github.com/jtolds/gls">gls: Goroutine local storage</a></p> 23 | <p>Go doesn&#39;t provide a mechanism for having &quot;goroutine local&quot; variables (like threadlocals in Python but for goroutines), and the structure of the language makes it really hard to get something working. JT Olio figured out a truly legendary hack: Go&#39;s introspection lets you see the current stack, so he figured out a way to encode a base-16 identifer tag into the call order of 16 special nested functions. I particularly like the &quot;What are people saying?&quot; section of the README: &quot;Wow, that&#39;s horrifying.&quot; - &quot;This is the most terrible thing I have seen in a very long time.&quot; - &quot;Where is it getting a context from? Is this serializing all the requests? What the heck is the client being bound to? What are these tags? Why does he need callers? Oh god no. No no no.&quot;</p> 24 | 25 | <p>Via <a href="https://twitter.com/aboodman/status/1133507328458649600">Aaron Boodman</a></p> 26 | 27 | Zdog2019-05-28T21:59:27+00:002019-05-28T21:59:27+00:00http://simonwillison.net/2019/May/28/zdog/#atom-blogmarks<p><a href="https://zzz.dog/">Zdog</a></p> 28 | <p>Well this is absolutely delightful: Zdog is a pseudo-3D engine for canvas and SVG that outputs 3D models rendered as super-stylish flat shapes. It&#39;s hard to describe with words - go play with the demos!</p> 29 | 30 | <p>Via <a href="https://twitter.com/desandro/status/1133373535542489088">Dave DeSandro</a></p> 31 | 32 | Using dependabot to bump Django on my blog from 2.2 to 2.2.12019-05-27T01:24:48+00:002019-05-27T01:24:48+00:00http://simonwillison.net/2019/May/27/dependabot/#atom-blogmarks<p><a href="https://github.com/simonw/simonwillisonblog/pull/25">Using dependabot to bump Django on my blog from 2.2 to 2.2.1</a></p> 33 | <p>GitHub recently acquired dependabot and made it free, and I decided to try it out on my blog. It&#39;s a really neat piece of automation: it scans your requirements.txt (plus a number of other packaging definitions across several different languages), checks for updates to your dependencies and opens pull requests against any that it finds. Combine it with a CI service such as Circle CI and your tests will run automatically against the pull request, letting you know if it&#39;s safe to merge. dependabot constantly rebases other changes against the pull request to try and ensure it will merge as cleanly as possible.</p> 34 | 35 | <p>Via <a href="https://nimbleindustries.io/2019/05/26/dependabot-is-now-free-and-its-amazing/">Dependabot is Now Free and It&#39;s Amazing</a></p> 36 | 37 | sqlite-utils 1.02019-05-25T01:20:37+00:002019-05-25T01:20:37+00:00http://simonwillison.net/2019/May/25/sqlite-utils-1/#atom-blogmarks<p><a href="https://sqlite-utils.readthedocs.io/en/latest/changelog.html#v1-0">sqlite-utils 1.0</a></p> 38 | <p>I just released sqlite-utils 1.0, with a couple of handy new features over 0.14: it can now automatically add columns to a database table if you attempt to insert data which doesn&#39;t quite fit (using alter=True in the Python API or the --alter option to the &quot;sqlite-utils insert&quot; command). It also has the ability to output nested JSON column values on the command-line using the new --json-cols option. This is the first project I&#39;ve marked as a 1.0 release in a very long time - I&#39;ll be sticking to semver for this project from now on, bumping the major version only in the case of a backwards incompatible change.</p> 39 | 40 | WebAssembly at eBay: A Real-World Use Case2019-05-22T20:30:58+00:002019-05-22T20:30:58+00:00http://simonwillison.net/2019/May/22/webassembly-ebay-real-world-use-case/#atom-blogmarks<p><a href="https://medium.com/ebaytech/webassembly-at-ebay-a-real-world-use-case-ef888f38b537">WebAssembly at eBay: A Real-World Use Case</a></p> 41 | <p>eBay used WebAssembly to run a C++ barcode reading library inside a web worker, passing images from the camera in order to provide a barcode scanning interface as part of their mobile web &quot;add listing&quot; page (a feature that had already proved successful in their native mobile apps). This is a great write-up, with lots of detail about how they compiled the library. They ended up running three barcode solutions in parallel web workers - two using WebAssembly, one in pure JavaScript - because their testing showed that racing between three implementations greatly increased the chance of a match due to how the different libraries handled poor quality or out-of-focus images.</p> 42 | 43 | <p>Via <a href="https://twitter.com/senthil_hi/status/1131252395520929792">@senthil_hi</a></p> 44 | 45 | Terrarium by Fastly Labs2019-05-21T20:51:37+00:002019-05-21T20:51:37+00:00http://simonwillison.net/2019/May/21/terrarium-fastly-labs/#atom-blogmarks<p><a href="https://wasm.fastlylabs.com/">Terrarium by Fastly Labs</a></p> 46 | <p>Fastly have been investing heavily in WebAssembly, which makes sense as it provides an excellent option for a sandboxed environment for executing server-side code at the edge of their CDN offering. Terrarium is their &quot;playground for experimenting with edge-side WebAssembly&quot; - it lets you write a program in Rust, C, TypeScript or Wat (WebAssembly text format), compile it to WebAssembly and deploy it to a URL with a single button-click. It&#39;s just a demo for the moment so deployments only persist for 15 minutes, but it&#39;s a fascinating sandbox to play around with.</p> 47 | 48 | Monaco Editor2019-05-21T20:47:12+00:002019-05-21T20:47:12+00:00http://simonwillison.net/2019/May/21/monaco-editor/#atom-blogmarks<p><a href="https://microsoft.github.io/monaco-editor/">Monaco Editor</a></p> 49 | <p>VS Code is MIT licensed and built on top of Electron. I thought &quot;huh, I wonder if I could run the editor component embedded in a web app&quot; - and it turns out Microsoft have already extracted out the code editor component into an open source JavaScript package called Monaco. Looks very slick, though sadly it&#39;s not supported in mobile browsers.</p> 50 | 51 | Public Data Release of Stack Overflow’s 2019 Developer Survey2019-05-21T18:51:43+00:002019-05-21T18:51:43+00:00http://simonwillison.net/2019/May/21/public-data-release-of-stack-overflows-2019-developer-survey/#atom-blogmarks<p><a href="https://stackoverflow.blog/2019/05/21/public-data-release-of-stack-overflows-2019-developer-survey/">Public Data Release of Stack Overflow’s 2019 Developer Survey</a></p> 52 | <p>Here&#39;s the Stack Overflow announcement of their developer survey public data release, which discusses the Glitch partnership and mentions Datasette.</p> 53 | 54 | Discover Insights in Developer Survey Results2019-05-21T18:50:22+00:002019-05-21T18:50:22+00:00http://simonwillison.net/2019/May/21/discover-insights-developer-survey-results/#atom-blogmarks<p><a href="https://glitch.com/culture/discover-insights-explore-developer-survey-results-2019/">Discover Insights in Developer Survey Results</a></p> 55 | <p>Stack Overflow partnered with Glitch and used Datasette to host the full data set from Stack Overflow&#39;s 2019 Developer Survey!</p> 56 | 57 | django-lifecycle2019-05-15T23:34:55+00:002019-05-15T23:34:55+00:00http://simonwillison.net/2019/May/15/django-lifecycle/#atom-blogmarks<p><a href="https://github.com/rsinger86/django-lifecycle">django-lifecycle</a></p> 58 | <p>Interesting alternative to Django signals by Robert Singer. It provides a model mixin class which over-rides the Django ORM&#39;s save() method, tracking which model attributes have been changed. Then it lets you add methods to your model with a @hook annotation allowing you to specify things like &quot;run this method before saving if the status changed&quot; or &quot;run this after an object has been deleted&quot;.</p> 59 | 60 | <p>Via <a href="https://twitter.com/webology/status/1128801534391836678">Jeff Triplett</a></p> 61 | 62 | Why I (Still) Love Tech: In Defense of a Difficult Industry2019-05-15T15:45:20+00:002019-05-15T15:45:20+00:00http://simonwillison.net/2019/May/15/in-defense-of-a-difficult-industry/#atom-blogmarks<p><a href="https://www.wired.com/story/why-we-love-tech-defense-difficult-industry/">Why I (Still) Love Tech: In Defense of a Difficult Industry</a></p> 63 | <p>If you only read one longform piece this week, make it this one. Utterly delightful prose and a bunch of different messages that resonated with me deeply.</p> 64 | 65 | <p>Via <a href="https://daringfireball.net/linked/2019/05/14/ford-loves-tech">Daring Fireball</a></p> 66 | 67 | 68 | -------------------------------------------------------------------------------- /__tests__/fixtures/summary.atom: -------------------------------------------------------------------------------- 1 | 2 | 3 | urn:uuid:f6d9e764-f597-5370-94e1-c01aa3928860 4 | Ctrl blog 5 | 6 | Daniel Aleksandersen 7 | https://www.daniel.priv.no/ 8 | 9 | Copyright © 2019 Daniel Aleksandersen. 10 | 11 | 12 | 2019-09-26T04:58:00Z 13 | daily 14 | 6 15 | 16 | urn:uuid:cc30d5e6-33d4-4f45-97ce-f0664622c6c2 17 | 18 | 2019-09-26T04:58:00Z 19 | 2019-09-26T04:58:00Z 20 | Semantic markup improves the quality of machine-translated technical texts 21 | Text-level semantic HTML can improve machine-translation of texts containing program names, programming instructions, file paths, URIs, etc. 22 | <p>The leading web browser, Google Chrome, and leading search engines — including Bing, Yandex, Google, and Baidu — can machine-translate any webpage in seconds. This enables anyone who understands a supported language to access documents written in any other supported language.</p> <p><a href="https://example.com/marker">Read more …</a></p> 23 | 24 | 25 | 26 | 27 | 28 | urn:uuid:03020027-792e-4468-9999-b84847d6aff6 29 | 30 | 2019-09-18T15:19:00Z 31 | 2019-09-18T15:19:00Z 32 | How Coil and Web Monetization works compared to Flattr 33 | I explore the new Coil micro-payment system and browser extension, based on the proposed Web Monetization web-standard, and compare it to Flattr. 34 | <p>Coil is a micro-payment service that resembles Flattr. Both services rely on a web browser extension to automatically pay participating websites and creators from customers’ 5 USD/monthly subscription funds.</p> <p><a href="https://www.ctrl.blog/entry/coil-web-monetization.html#src=feed">Read more …</a></p> 35 | 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /__tests__/h-entry.test.js: -------------------------------------------------------------------------------- 1 | const tap = require('tap'); 2 | const { findEntries } = require('../shared/lib/microformat/dom'); 3 | 4 | const fixture = { 5 | items: [ 6 | { 7 | type: ['h-entry'], 8 | properties: { 9 | name: [ 10 | 'Indie Map is a public IndieWeb social graph and dataset.\n\n\n\n2300 sites, 5.7M pages, 380GB HTML with microformats2.\nSocial graph API and interactive map: 631M links, 706K relationships.\nSQL queryable dataset, stats, raw crawl data.\n\n\nLearn more ➜', 11 | ], 12 | }, 13 | }, 14 | ], 15 | rels: { 16 | stylesheet: ['style.css'], 17 | }, 18 | 'rel-urls': { 19 | 'style.css': { 20 | type: 'text/css', 21 | rels: ['stylesheet'], 22 | }, 23 | }, 24 | }; 25 | 26 | tap.test('empty h-entry', (t) => { 27 | const res = findEntries(fixture.items); 28 | 29 | t.equal(res.length, 0); 30 | t.end(); 31 | }); 32 | -------------------------------------------------------------------------------- /__tests__/index.test.js: -------------------------------------------------------------------------------- 1 | const tap = require('tap'); 2 | const fs = require('fs'); 3 | const Webmention = require('../shared/lib/webmention'); 4 | const read = (f) => fs.readFileSync(__dirname + f, 'utf8'); 5 | 6 | tap.test('atom', (t) => { 7 | t.plan(1); 8 | const wm = new Webmention({ limit: 2 }); 9 | wm.on('end', () => { 10 | t.not(wm.mentions.length, 0); 11 | t.end(); 12 | }); 13 | wm.load(read('/fixtures/simon-links-atom.xml')); 14 | }); 15 | 16 | tap.test('xml', (t) => { 17 | t.plan(1); 18 | const wm = new Webmention({ limit: 2 }); 19 | wm.on('end', () => { 20 | t.not(wm.mentions.length, 0); 21 | t.end(); 22 | }); 23 | wm.load(read('/fixtures/fsis.xml')); 24 | }); 25 | 26 | // tap.test('local html', t => { 27 | // t.plan(2); 28 | // const wm = new Webmention(); 29 | // wm.on('end', () => { 30 | // const found = wm.endpoints.find(_ => _.source.includes('remysharp')); 31 | // if (!found) { 32 | // t.fail('endpoints missing'); 33 | // console.warn(wm.mentions); 34 | // } else { 35 | // t.equal(found.endpoint.type, 'pingback'); 36 | // t.equal(found.endpoint.url, 'https://rems.life/xmlrpc.php'); 37 | // } 38 | // t.end(); 39 | // }); 40 | // wm.load(read('/fixtures/but.html')); 41 | // }); 42 | 43 | tap.test('local rss', (t) => { 44 | t.plan(3); 45 | const wm = new Webmention(); 46 | wm.on('end', () => { 47 | t.equal(wm.mentions.length, 10); 48 | const found = wm.endpoints.find((_) => 49 | _.target.includes('paulrobertlloyd') 50 | ); 51 | t.equal(found.endpoint.type, 'webmention'); 52 | t.equal( 53 | found.endpoint.url, 54 | 'https://webmention.io/paulrobertlloyd.com/webmention' 55 | ); 56 | 57 | t.end(); 58 | }); 59 | wm.load(read('/fixtures/jeremy.xml')); 60 | }); 61 | 62 | // tap.test('local non-h-entry', t => { 63 | // t.plan(1); 64 | // const wm = new Webmention(); 65 | // wm.on('endpoints', e => { 66 | // t.equal(e.length, 2); 67 | // t.end(); 68 | // }); 69 | // wm.load(read('/fixtures/alt-but.html')); 70 | // }); 71 | 72 | tap.test('local h-feed nested', (t) => { 73 | t.plan(1); 74 | const wm = new Webmention(); 75 | wm.on('endpoints', (endpoints) => { 76 | t.equal(endpoints.length, 10); 77 | t.end(); 78 | }); 79 | wm.load(read('/fixtures/snarfed.html')); 80 | }); 81 | -------------------------------------------------------------------------------- /__tests__/mf.test.js: -------------------------------------------------------------------------------- 1 | const tap = require('tap'); 2 | const fs = require('fs'); 3 | const Webmention = require('../shared/lib/webmention'); 4 | const read = (f) => fs.readFileSync(__dirname + f, 'utf8'); 5 | 6 | tap.test('microformat', (t) => { 7 | t.plan(2); 8 | const wm = new Webmention(); 9 | wm.on('endpoints', (endpoints) => { 10 | t.ok(endpoints[0].source.includes('adactio.com')); 11 | t.ok(endpoints[0].target.includes('remysharp.com')); 12 | t.end(); 13 | }); 14 | wm.load(read('/fixtures/adactio-link.html')); 15 | }); 16 | 17 | tap.test('microformat missing', (t) => { 18 | t.plan(1); 19 | const wm = new Webmention(); 20 | wm.on('endpoints', (endpoints) => { 21 | t.equal(endpoints.length, 2); 22 | t.end(); 23 | }); 24 | wm.load(read('/fixtures/mf-missing.html')); 25 | }); 26 | -------------------------------------------------------------------------------- /__tests__/pingback.test.js: -------------------------------------------------------------------------------- 1 | const pingback = require('../shared/lib/send/pingback'); 2 | const tap = require('tap'); 3 | 4 | tap.test('pingback', (t) => { 5 | const endpoint = 'https://bavatuesdays.com/xmlrpc.php'; 6 | const source = 'https://remy.jsbin.me/icy-feather-c76/'; 7 | const target = 'https://bavatuesdays.com/hello-world/'; 8 | return pingback({ source, target, endpoint }) 9 | .then(() => { 10 | t.pass('worked'); 11 | }) 12 | .catch((e) => { 13 | t.failed(e); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /__tests__/request.test.js: -------------------------------------------------------------------------------- 1 | const tap = require('tap'); 2 | const request = require('../shared/lib/request'); 3 | 4 | tap.test('request a duff domain', (t) => { 5 | t.plan(1); 6 | const timeout = 2000; 7 | const now = Date.now(); 8 | request('http://www.this_url_should_not_resolve.com', timeout) 9 | .then(() => { 10 | t.fail('should not resolve'); 11 | }) 12 | .catch(() => { 13 | t.ok(Date.now() - now < timeout + 1000); 14 | }); 15 | }); 16 | -------------------------------------------------------------------------------- /app/api/auth.mjs: -------------------------------------------------------------------------------- 1 | export function get() { 2 | return { 3 | location: '/auth/', 4 | }; 5 | } 6 | -------------------------------------------------------------------------------- /app/api/auth/$$.mjs: -------------------------------------------------------------------------------- 1 | /* eslint-env node */ 2 | import '@remy/envy'; 3 | import passport from 'passport'; 4 | import { github } from '../../../shared/lib/passport.js'; 5 | 6 | passport.use('github', github); 7 | 8 | export async function get(req) { 9 | console.log('/auth/get'); 10 | return new Promise((resolve) => { 11 | let location = ''; 12 | passport.authenticate('github', (err, user) => { 13 | console.log('passport auth', { err: err ? err.__type : null, user }); 14 | if (err) { 15 | console.log(err); 16 | return resolve({ location: '/token-failure' }); 17 | } 18 | 19 | if (!user) { 20 | return resolve({ location: '/token-failure' }); 21 | } 22 | 23 | resolve({ 24 | session: { token: user }, 25 | location: '/token', 26 | }); 27 | })(req, { 28 | setHeader: (key, value) => { 29 | console.log('setHeader', { key, value }); 30 | if (key.toLowerCase() === 'location') { 31 | location = value; 32 | } 33 | }, 34 | end: (...args) => { 35 | console.log('end', args); 36 | if (location) { 37 | resolve({ 38 | location, 39 | }); 40 | } 41 | }, 42 | }); 43 | }); 44 | } 45 | -------------------------------------------------------------------------------- /app/api/check.mjs: -------------------------------------------------------------------------------- 1 | import '@remy/envy'; 2 | import ms from 'ms'; 3 | import { parse } from 'url'; 4 | import Webmention from '../../shared/lib/webmention.js'; 5 | import sendMention from '../../shared/lib/send/index.js'; 6 | import db from '../../shared/lib/db.js'; 7 | 8 | const rateWindow = 1000 * 60; // * 60 * 4; // 4 hours 9 | 10 | async function handleRequest(req) { 11 | let { url, token, limit = 10 } = req.query; 12 | const method = req.method.toLowerCase(); 13 | 14 | const now = new Date(); 15 | 16 | // Server-Timing: miss, db;dur=53, app;dur=47.2 (ms) 17 | const timings = { 18 | db: 0, 19 | webmention: 0, 20 | send: 0, 21 | }; 22 | 23 | if (!url) { 24 | return; // just render the get 25 | } 26 | 27 | if (!url.startsWith('http')) { 28 | url = `http://${url}`; 29 | } 30 | 31 | // ensure the url is properly encoded 32 | url = parse(url).href; 33 | 34 | const { origin = '', referer = '' } = req.headers; 35 | 36 | if (origin.includes('localhost') || referer.includes('webmention.app')) { 37 | // note that this token is rotated on a random basis, if you want to pinch 38 | // it, you can, but don't trust it'll continue to work. 39 | if (!token) token = '089edc08-9677-48fd-947c-06f9e2d90148-site'; 40 | } 41 | 42 | const validToken = token ? await db.updateTokenRequestCount(token) : null; 43 | 44 | if (!validToken) { 45 | // only allow one hit a day 46 | const data = await db.getRequestCount(url); 47 | 48 | if (data) { 49 | const delta = 50 | now.getTime() - rateWindow - new Date(data.lastRequested).getTime(); 51 | 52 | if (delta < 0) { 53 | return { 54 | statusCode: 429, 55 | json: { 56 | error: true, 57 | message: `Too many requests in time window. Try again in ${ms( 58 | delta * -1, 59 | { long: true } 60 | )}, or use a free token for no rate limits: https://webmention.app/token`, 61 | }, 62 | }; 63 | } 64 | } 65 | } 66 | 67 | return new Promise((resolve) => { 68 | const dbUpdate = db 69 | .updateRequestCount(url) 70 | .then(() => { 71 | timings.db = Date.now() - now.getTime(); 72 | }) 73 | .catch((e) => console.log(e)); 74 | 75 | const send = (data) => { 76 | data.url = url; 77 | return dbUpdate 78 | .then(() => { 79 | return { 80 | json: data, 81 | }; 82 | }) 83 | .catch((e) => { 84 | return { 85 | json: { error: true, message: e.message }, 86 | }; 87 | }) 88 | .then((reply) => resolve(reply)); 89 | }; 90 | 91 | console.log('>> %s %s', method === 'post' ? 'SEND' : 'QUERY', url); 92 | 93 | const wm = new Webmention({ limit }); 94 | wm.on('error', (e) => { 95 | send({ error: true, message: e.message }); 96 | }); 97 | 98 | // wm.on('log', (a) => console.log(a)); 99 | // wm.on('progress', (e) => { 100 | // const [[key, value]] = Object.entries(e); 101 | // console.log('progress', key, value); 102 | // }); 103 | 104 | wm.on('endpoints', (urls) => { 105 | timings.webmention = Date.now() - now.getTime(); 106 | 107 | if (method === 'post') { 108 | return Promise.all(urls.map(sendMention)).then((reply) => { 109 | if (reply.length) 110 | db.updateRequestCount( 111 | '__sent', 112 | reply.filter((_) => _.status < 400).length 113 | ).catch((E) => console.log('error updating __sent count', E)); 114 | timings.send = Date.now() - now.getTime(); 115 | return send({ urls: reply }); 116 | }); 117 | } 118 | 119 | if (urls.length === 0 && wm.mentions.length > 0) { 120 | return send({ 121 | error: true, 122 | message: `No webmention endpoints found in the links of the ${ 123 | wm.mentions.length 124 | } content ${wm.mentions.length === 1 ? 'entry' : 'entries'}`, 125 | }); 126 | } 127 | 128 | send({ urls }); 129 | }); 130 | 131 | wm.fetch(url); 132 | }); 133 | } 134 | 135 | export const get = handleRequest; 136 | export const post = handleRequest; 137 | -------------------------------------------------------------------------------- /app/api/docs/index.mjs: -------------------------------------------------------------------------------- 1 | export function get() { 2 | console.log('api/index/get'); 3 | return { 4 | json: { 5 | title: 'docs :: webmention.app', 6 | }, 7 | }; 8 | } 9 | -------------------------------------------------------------------------------- /app/api/index.mjs: -------------------------------------------------------------------------------- 1 | import db from '../../shared/lib/db.js'; 2 | 3 | export function get() { 4 | return db.getRequestCount('__sent').then((data) => { 5 | return { json: { total: data.hits } }; 6 | }); 7 | } 8 | -------------------------------------------------------------------------------- /app/api/stats.mjs: -------------------------------------------------------------------------------- 1 | import db from '../../shared/lib/db.js'; 2 | 3 | export function get() { 4 | // TODO work out how this fails… 5 | return db.getRecentURLs().then(({ Items: data }) => { 6 | return { 7 | json: { 8 | data: data 9 | .filter((_) => _.url !== '__sent') 10 | .sort((a, b) => (a.requested < b.requested ? 1 : -1)) 11 | .slice(0, 20), 12 | total: data.length - 1, 13 | sent: data.find((_) => _.url === '__sent').hits, 14 | }, 15 | }; 16 | }); 17 | } 18 | -------------------------------------------------------------------------------- /app/api/token.mjs: -------------------------------------------------------------------------------- 1 | export function get({ session }) { 2 | const token = session.token || null; 3 | return { 4 | json: { 5 | token, 6 | }, 7 | }; 8 | } 9 | -------------------------------------------------------------------------------- /app/browser/$.mjs: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | 3 | /** 4 | * Class representing array-like NodeCollection 5 | * 6 | * @class 7 | * @augments Array 8 | * @augments Element 9 | */ 10 | class NodeListArray extends Array { 11 | constructor() { 12 | super(); 13 | 14 | // allow setting any node property via proxy 15 | return new Proxy(this, { 16 | get(obj, prop) { 17 | const type = obj[0]; 18 | 19 | if (prop in obj) { 20 | return obj[prop]; 21 | } 22 | 23 | if (type && prop in type) { 24 | return type[prop]; 25 | } 26 | 27 | return undefined; 28 | }, 29 | 30 | set(obj, prop, value) { 31 | const type = obj[0]; 32 | 33 | if (type && prop in type) { 34 | return obj.filter((el) => { 35 | try { 36 | return (el[prop] = value); 37 | } catch (_) { 38 | // nop 39 | } 40 | }); 41 | } 42 | 43 | const res = (this[prop] = value); 44 | return res; 45 | }, 46 | }); 47 | } 48 | 49 | /** 50 | * Bind event to the collection for a given event type 51 | * 52 | * @param {string} event Event name 53 | * @param {Function} handler Event handler 54 | * @param {object} options Standard addEventListener options 55 | * @returns {Element[]} 56 | */ 57 | on(event, handler, options) { 58 | return this.filter((el) => el.addEventListener(event, handler, options)); 59 | } 60 | 61 | /** 62 | * Trigger the given event on all attached handlers 63 | * 64 | * @param {string} event 65 | * @param {*} data 66 | * @returns {Array} filtered result of dispatchEvent 67 | */ 68 | emit(event, data) { 69 | const e = new Event(event, { data }); 70 | return this.filter((el) => el.dispatchEvent(e)); 71 | } 72 | } 73 | 74 | /** 75 | * Query the DOM for an array like collection of nodes 76 | * 77 | * @param {string} selector CSS selector 78 | * @param {Element} [context=document] Query context 79 | * @returns {NodeListArray} New array-like node list 80 | */ 81 | export default function (selector, context = document) { 82 | const res = context.querySelectorAll(selector); 83 | 84 | if (res.length === 0) { 85 | console.warn(`${selector} zero results`); 86 | } 87 | 88 | return res.length ? NodeListArray.from(res) : {}; 89 | } 90 | -------------------------------------------------------------------------------- /app/browser/check-mention.mjs: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | import $ from './$.mjs'; 3 | import { mentionWrapper } from '../elements/check-mention.mjs'; 4 | 5 | window.$ = $; 6 | 7 | class CheckMention extends HTMLElement { 8 | static observedAttributes = ['loading']; 9 | 10 | get url() { 11 | return this.input.value; 12 | } 13 | 14 | set loading(value) { 15 | if (value) { 16 | this.setAttribute('loading', true); 17 | } else { 18 | this.removeAttribute('loading'); 19 | } 20 | } 21 | 22 | clear() { 23 | this.mentions.innerHTML = ''; 24 | } 25 | 26 | constructor() { 27 | super(); 28 | this.button = $('button', this); 29 | this.input = $('input', this); 30 | this.mentions = $('#mention-wrapper', this); 31 | this.sendForm = $('form[method="post"]', this); 32 | 33 | this.addEventListener('submit', (e) => { 34 | e.preventDefault(); 35 | this.check(this.url, e.target.method.toLowerCase() === 'post'); 36 | }); 37 | } 38 | 39 | /** 40 | * @param {string} url 41 | * @param {boolean} [send=false] 42 | */ 43 | async check(url, send = false) { 44 | if (!send) this.clear(); 45 | this.loading = true; 46 | const query = new URLSearchParams(); 47 | query.append('url', url); 48 | const res = await fetch(`/check?${query.toString()}`, { 49 | headers: { accept: 'application/json' }, 50 | method: send ? 'post' : 'get', 51 | }); 52 | const json = await res.json(); 53 | this.loading = false; 54 | 55 | this.mentions.innerHTML = mentionWrapper({ 56 | html(strings, ...values) { 57 | return String.raw({ raw: strings }, ...values); 58 | }, 59 | sent: send, 60 | urls: json.urls || [], 61 | url: json.url, 62 | }); 63 | 64 | this.sendForm.hidden = send; 65 | 66 | // if (!json.error) { 67 | // this.mentions = json.urls; 68 | // this.sent = true; 69 | // } else { 70 | // this.mentions = []; 71 | // this.error = json.message; 72 | // } 73 | } 74 | 75 | async send() { 76 | this.check(this.url, true); 77 | } 78 | 79 | attributeChangedCallback(name, oldValue, newValue) { 80 | if (name === 'loading') { 81 | this.button.disabled = !!newValue; 82 | } 83 | } 84 | } 85 | 86 | customElements.define('check-mention', CheckMention); 87 | -------------------------------------------------------------------------------- /app/browser/getting-started.mjs: -------------------------------------------------------------------------------- 1 | import enhance from '@enhance/element'; 2 | import GettingStarted from '../elements/getting-started.mjs'; 3 | 4 | enhance('getting-started', { 5 | attrs: [], 6 | /** 7 | * @param {HTMLElement} root 8 | */ 9 | init(root) { 10 | root.addEventListener('change', (event) => { 11 | if (event.target.nodeName === 'INPUT') { 12 | root 13 | .querySelector('#started-docs') 14 | .setAttribute('selected', event.target.value); 15 | } 16 | }); 17 | }, 18 | render: GettingStarted, 19 | connected() {}, 20 | }); 21 | -------------------------------------------------------------------------------- /app/browser/started-docs.mjs: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | import Component from '../elements/started-docs.mjs'; 3 | import enhance from '@enhance/element'; 4 | 5 | enhance('started-docs', { 6 | attrs: ['selected'], 7 | render: Component, 8 | }); 9 | -------------------------------------------------------------------------------- /app/browser/web-mention.mjs: -------------------------------------------------------------------------------- 1 | /* eslint-env browser */ 2 | import Component from '../elements/web-mention.mjs'; 3 | import enhance from '@enhance/element'; 4 | 5 | enhance('web-mention', { 6 | attrs: ['source', 'target', 'status', 'error'], 7 | render: Component, 8 | }); 9 | -------------------------------------------------------------------------------- /app/elements/app-footer.mjs: -------------------------------------------------------------------------------- 1 | export default function AppFooter({ html }) { 2 | return html` 3 | 23 | 24 | 29 | `; 30 | } 31 | -------------------------------------------------------------------------------- /app/elements/app-layout.mjs: -------------------------------------------------------------------------------- 1 | export default function AppLayout({ html }) { 2 | return html` 3 |
4 |
5 | 6 |
7 |
8 | 9 |
10 | 11 | 14 |
15 | `; 16 | } 17 | -------------------------------------------------------------------------------- /app/elements/app-logo.mjs: -------------------------------------------------------------------------------- 1 | export default function Logo({ html, state }) { 2 | const { attrs } = state; 3 | const { width } = attrs; 4 | 5 | let hover = false; 6 | 7 | return html` 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 41 | 49 | 50 | 55 | 56 | 57 | 58 | `; 80 | } 81 | 82 | { 83 | /* */ 93 | } 94 | -------------------------------------------------------------------------------- /app/elements/app-navigation.mjs: -------------------------------------------------------------------------------- 1 | export default function AppNavigation({ html }) { 2 | return html` `; 20 | } 21 | -------------------------------------------------------------------------------- /app/elements/check-mention.mjs: -------------------------------------------------------------------------------- 1 | export default function CheckMention({ html, state }) { 2 | const { store = {} } = state; 3 | const { urls = [], url = '', sent = false } = store; 4 | const hasResult = urls.length > 0; 5 | 6 | const error = false; 7 | 8 | return html` 12 | 13 |
14 |
15 | 16 | 17 | 18 |
19 |
20 |
21 | ${mentionWrapper({ html, sent, urls, error, url })} 22 |
23 | 24 |
25 | 26 | 27 |
`; 28 | } 29 | 30 | export function mentionWrapper({ html, sent, urls, error, url }) { 31 | const webMentions = urls 32 | .map( 33 | ({ source, target, status, error }) => 34 | html`` 40 | ) 41 | .join(''); 42 | 43 | const errorMessage = error ? `

${error}

` : ''; 44 | const sentNotifications = sent 45 | ? html`

46 | 47 | Sent ${urls.map((_) => _.status < 400).length} webmention 48 | notifications. 49 | 50 |

` 51 | : ''; 52 | 53 | const foundCount = 54 | url && !sent 55 | ? html`

56 | 57 | ${urls.length === 0 ? 'No' : urls.length} webmention supported links 58 | found. 59 | 60 |

` 61 | : ''; 62 | 63 | // Note: this form is part of the partial because _only_ this part of the 64 | // component gets updated, so the `hasResult` and `sent` are changed 65 | return html`${sentNotifications} ${foundCount} ${errorMessage} 66 | ${`
    ${webMentions}
`}`; 67 | } 68 | -------------------------------------------------------------------------------- /app/elements/getting-started.mjs: -------------------------------------------------------------------------------- 1 | export default function GettingStarted({ html }) { 2 | return html` 3 | 46 | 47 |
48 |

Which best describes your site:

49 | 90 |
91 | 92 | 93 | 94 | 95 | `; 96 | } 97 | -------------------------------------------------------------------------------- /app/elements/started-docs.mjs: -------------------------------------------------------------------------------- 1 | function feed(html) { 2 | return html`
3 |

4 | Your ideal approach is to use IFTTT to run a recipe when a new item is 5 | published to your feed and to trigger a call to webmention.app. 6 |

7 |

8 | 9 | Read the full walk through can be seen here. 10 | 11 |

12 |
`; 13 | } 14 | 15 | function url(html) { 16 | return html`
17 |

There's two options here:

18 |

19 | You can 20 | 21 | call this web site's webhook 22 | 23 | when your site is updated pointing to the URL of your homepage. 24 |

25 |

26 | Alternatively you can use 27 | IFTTT to run a repeating and scheduled recipe 30 | to check your homepage for new content. 31 |

32 |
`; 33 | } 34 | 35 | function cli(html) { 36 | return html`
37 |

38 | Your ideal approach is to use the 39 | webmention command line tool. It doesn't rely on this web 40 | site at all and can be executed on Windows, Mac and Unix-based platforms. 41 |

42 |

43 | Read how to install and use the command line tool. 46 |

47 |
`; 48 | } 49 | 50 | function complicated(html) { 51 | return html`
52 |

53 | You can visit this site whenever you need and test your new content given 54 | any URL and once the "dry-run" has completed, if any webmentions have been 55 | found, you'll be able to send those outgoing notifications. 56 |

57 |

58 | Use the manual test and send tool. 59 |

60 |
`; 61 | } 62 | 63 | function docs(html) { 64 | return html`
65 |

66 | Take a peruse of the documentation and see if there's anything that 67 | matches. If not, feel free to 68 | open an issue 71 | to see if there's something can be solved for you. 72 |

73 |

74 | Browse the documentation and recipes. 75 |

76 |
`; 77 | } 78 | 79 | const selection = { 80 | feed, 81 | docs, 82 | cli, 83 | complicated, 84 | url, 85 | }; 86 | 87 | export default function GettingStartedDocs({ html, state }) { 88 | const { attrs = {} } = state; 89 | const { selected = '' } = attrs; 90 | 91 | const body = selected ? selection[selected](html) : ''; 92 | return html`
${body}
93 | `; 94 | } 95 | -------------------------------------------------------------------------------- /app/elements/web-mention.mjs: -------------------------------------------------------------------------------- 1 | export default function WebMention({ html, state }) { 2 | const { attrs = {} } = state; 3 | const { source = '', status = null, error = false, target = '' } = attrs; 4 | 5 | let statusClassName = ''; 6 | 7 | if (status) { 8 | statusClassName = [ 9 | 'status', 10 | 'status-' + status.toString().slice(0, 1), 11 | ].join(' '); 12 | } 13 | 14 | return html`
  • 15 | source= 16 | ${source} 17 |
    18 | target= 19 | ${(status && 20 | html` 21 | ${status} 22 | `) || 23 | ''} 24 | ${target} 25 | ${(error && 26 | html` 27 |
    28 | ${error} 29 |     `) || 30 | ''} 31 |
    • `; 32 | } 33 | -------------------------------------------------------------------------------- /app/head.mjs: -------------------------------------------------------------------------------- 1 | export default function Head(state) { 2 | const { store = {} } = state; 3 | 4 | const title = store.title || 'Automate your outgoing webmentions'; 5 | 6 | return ` 7 | 8 | 9 | 10 | 11 | 12 | ${title} 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 25 | 26 | 30 | 31 | 32 | 36 | 37 | 38 | 39 | `; 40 | } 41 | -------------------------------------------------------------------------------- /app/pages/about.html: -------------------------------------------------------------------------------- 1 | 2 |

    About webmention.app

    3 |

    Remy Sharp wrote webmention.app because it seemed like webmention notification 4 | systems were thin on the ground, and wanted to use a notification system that was agnostic of the user software.

    5 |

    The service is currently free, and will remain so, as it is able to run on a single server with little to zero 6 | maintenance.

    7 |

    If you want to contact there's a few ways:

    8 | 14 |

    If you want to say thanks to support the project: You can buy 15 | (some) drinks ❤️

    16 | 17 |     18 | -------------------------------------------------------------------------------- /app/pages/check.mjs: -------------------------------------------------------------------------------- 1 | export default function Check({ html }) { 2 | return html` 3 |
    4 |

    Test and send webmentions

    5 |

    6 | Keep in mind that this test page only scans the 10 most recent 7 | h-entry elements on the target. 8 |

    9 | 10 |
    11 |     12 | 13 | `; 23 | } 24 | -------------------------------------------------------------------------------- /app/pages/docs/index.html: -------------------------------------------------------------------------------- 1 | 2 |
    3 |

    Documentation

    4 |

    Platform agnostic webmentions

    5 |

    webmention.app relies entirely on your markup and not your software, so no matter how your content is generated, 6 | you can send outgoing webmentions to other web sites.

    7 |

    8 | Supported formats include: plain HTML, 9 | h-entry (and 10 | hentry) Microformat markup, RSS and Atom. For backward compatibility both webmention and pingback 11 | protocols are supported. 12 |

    13 | 14 |
    15 | 16 |

    Send webmentions using the web service

    17 |     18 |
    19 |

    20 | You can send either URLs with HTML or RSS feeds. The service supports single entries and multiple entries. 21 | Multiple entries are found with well formed markup, specifically using 22 | 23 | .h-entry 24 | classes. 25 |

    26 |

    27 | You need to find a way to request a URL, in this example, we'll use the command line tool 28 | curl. Here's a real world example of sending webmentions: 29 |

    30 | 
    $ curl -X POST https://webmention.app/check?url=https://adactio.com/journal/15254
 31 | [
 32 |   {
 33 |     "endpoint": {
 34 |       "url": "https://webmention.io/indiewebcamp/webmention",
 35 |       "type": "webmention"
 36 |     },
 37 |     "source": "https://adactio.com/journal/15254",
 38 |     "target": "https://indieweb.org/Homebrew_Website_Club"
 39 |   },
 40 |   {
 41 |     "endpoint": {
 42 |       "url": "https://webmention.io/indiewebcamp/webmention",
 43 |       "type": "webmention"
 44 |     },
 45 |     "source": "https://adactio.com/journal/15254",
 46 |     "target": "https://indieweb.org/2019/Brighton"
 47 |   },
 48 |   {
 49 |     "endpoint": {
 50 |       "url": "https://brid.gy/publish/webmention",
 51 |       "type": "webmention"
 52 |     },
 53 |     "source": "https://adactio.com/journal/15254",
 54 |     "target": "https://benjamin.parry.is/"
 55 |   },
 56 |   {
 57 |     "endpoint": {
 58 |       "url": "https://webmention.io/remysharp.com/webmention",
 59 |       "type": "webmention"
 60 |     },
 61 |     "source": "https://adactio.com/journal/15254",
 62 |     "target": "https://remysharp.com/"
 63 |   }
 64 | ]
 65 |
    66 | 67 |

    68 | If your URL has any query string parameters (such as 69 | ?slug=my-great-post) make sure to properly 70 | encode the URL. 71 |

    72 | 73 |

    74 | In the example above, the 75 | -X POST curl argument is being use to ensure the request is a POST which notifies. If you want to 76 | perform a dry-run to see what would be sent, perform a GET request (and remove the 77 | -X POST). 78 |

    79 | 80 |

    81 | Remember to claim a token so that your requests are not rate limited. 82 |

    83 |
    84 |
    85 | 86 |
    87 | 88 |

    How to integrate with Netlify

    89 |     90 |
    91 |

    Netlify is a great platform for hosting static sites, and you can use webmention.app as part of your build 92 | process, or more simply if you also generate an RSS feed for you website you can provide a "deploy 93 | notification".

    94 |

    95 | Navigate to your Netlify project, and from the 96 | Build & Deploy menu, find 97 | Deploy Notifications. Add a new notification, selecting 98 | Outgoing webhook: 99 |

    100 |

    101 | 102 |

    103 |

    104 | Select the 105 | Deploy succeeded event, and the URL to send outgoing webmentions is: 106 |

    107 | 
    https://webmention.app/check?token=[your-token]&limit=1&url=[your-feed-url]
    108 |

    Now upon every new post you release, webmention.app will automatically handle your webmentions for you.

    109 |
    110 |
    111 | 112 |
    113 | 114 |

    Using IFTTT to trigger checks

    115 |     116 |
    117 |

    If you have an RSS feed on your website, then you can configure IFTTT the trigger a call to webmention.app 118 | when new posts are published.

    119 |
      120 |
    1. 121 | Start by 122 | creating a new applet on ifttt.com 123 |
      2. 124 |
    2. 125 | Click on 126 | +this and select 127 | RSS Feed 128 |
      3. 129 |
    3. 130 | Select 131 | New feed item and enter the URL to your feed 132 |
      4. 133 |
    4. 134 | Click on 135 | +that and find and select 136 | Webhooks 137 |
      5. 138 |
    5. 139 | For the URL, enter: 140 | 141 | https://webmention.app/check?url={{EntryUrl}}&token= 142 | [your-token] 143 | 144 |
      6. 145 |
    6. 146 | Change the method to 147 | POST 148 |
      7. 149 |
    7. 150 | Then click 151 | Create action then 152 | Finish 153 |
      8. 154 |
    155 | 156 |

    Now when you publish a post, IFTTT will tell webmention.app to check the new URL for webmentions and 157 | automatically send them out.

    158 |
    159 |
    160 | 161 |
    162 | 163 |

    Scheduling repeating checks

    164 |     165 |
    166 |

    If you publish your content to a URL that's constant, like your homepage, or mysite.com/articles, when you 167 | can use IFTTT to set up a regular check - either weekly, daily or hourly.

    168 |
      169 |
    1. 170 | Start by 171 | creating a new applet on ifttt.com 172 |
      2. 173 |
    2. 174 | Click on 175 | +this and select 176 | Date & Time 177 |
      3. 178 |
    3. Select the frequency that suits your website - unless you're prolific, daily or weekly might be best.
      4. 179 |
    4. Change the time from the default 12 AM - this eases everyone's requests coming at the same time
      5. 180 |
    5. 181 | Click on 182 | +that and find and select 183 | Webhooks 184 |
      6. 185 |
    6. 186 | For the URL, enter: 187 | 188 | https://webmention.app/check?url={YOUR_URL}&token= 189 | [your-token] 190 | (remember to swap 191 | {YOUR_URL} for your 192 | actual URL!) 193 |
      7. 194 |
    7. 195 | Change the method to 196 | POST 197 |
      8. 198 |
    8. 199 | Then click 200 | Create action then 201 | Finish 202 |
      9. 203 |
    204 | 205 |

    Now IFTTT will run a regular webmention notification request.

    206 |
    207 |
    208 | 209 |
    210 | 211 |

    Supported feed types

    212 |     213 |
    214 |

    You can use either the web service or the command line method to request a feed. You pass the URL of the feed 215 | to webmention.app just as you would any other URL.

    216 | 217 |

    218 | Note that both RSS and Atom feeds are supported. If you have another format in mind, please 219 | open an issue with details. 220 |

    221 | 222 |

    223 | By default, the service will 224 | only look at 225 | the first 10 items found in the feed. 226 |

    227 |
    228 |
    229 | 230 |
    231 | 232 |

    Using the command line

    233 |     234 |
    235 |

    The command line doesn't rely on webmention.app at all and doesn't require a token - so you can run it 236 | locally with the knowledge that if your site outlives this one, the tool will still work.

    237 |

    238 | The tool uses 239 | nodejs and once nodejs is installed, you can install the tool using: 240 |

    241 | 
    $ npm install @remy/webmention
    242 |

    243 | This provides an executable under the command 244 | webmention (also available as 245 | wm). Default usage allows you to pass a filename (like a newly generated RSS feed) or a specific 246 | URL. It will default to the 10 most recent entries found (using 247 | item for RSS and 248 | h-entry for HTML). 249 |

    250 |

    251 | By default, the command will perform a dry-run/discovery only. To complete the notification of webmentions use 252 | the 253 | --send flag. 254 |

    255 |

    The options available are:

    256 |
      257 |
    • 258 | --send (default: false) send the webmention to all valid endpoints 259 |
      • 260 |
    • 261 | --limit n (default: 10) limit to 262 | n entries found 263 |
      • 264 |
    • 265 | --debug (default: false) print internal debugging 266 |
      • 267 |
    268 |

    269 | Using 270 | npx you can invoke the tool to read the latest entry in your RSS feed: 271 |

    272 | 
    $ npx webmention https://yoursite.com/feed.xml --limit 1 --send
    273 |

    274 | Alternatively, you can make the tool part of your build workflow and have it execute during a 275 | postbuild phase: 276 |

    277 | 
    {
278 |   "scripts": {
279 |     "postbuild": "webmention dist/feed.xml --limit 1 --send"
280 |   }
281 | }
282 |
    283 |
    284 |
    285 | 286 |
    287 | 288 |

    289 | How can you scan 290 | every item in a feed or page? 291 |

    292 |     293 |
    294 |

    295 | Using 296 | --limit 0 will tell the software to ignore any limits. 297 |

    298 |

    299 | If you're using the web service, include a query parameter of 300 | &limit=0. 301 |

    302 |
    303 |
    304 | 305 |
    306 | 307 |

    308 | How can you 309 | receive webmentions? 310 |

    311 |     312 |
    313 |

    webmention.app is only used to notify of outgoing webmentions. However, I can recommend the following 314 | websites:

    315 |
      316 |
    • 317 | webmention.io - a service you can use to accept inbound webmention 318 | notifications (I use this on 319 | my own blog) 320 |
      • 321 |
    • 322 | bridgy - a service to gather and send notifications from sources such as 323 | Twitter 324 |
      • 325 |
    • 326 | Using Webmentions - Max Böck's 327 | excellent article on how to start showing webmentions on your own website 328 |
      • 329 |
    330 |
    331 |
    332 | 333 |
    334 | 335 |

    Further reading

    336 |     337 |
    338 | 349 |
    350 |
    351 |
    352 |     353 | 354 | 368 | 369 | 386 | -------------------------------------------------------------------------------- /app/pages/docs/todo.html: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 |

    TODO

    9 | 16 |

    As of v1.3.4 / 2019-06-18

    17 | 22 |

    As of v1.3.1 / 2019-06-13

    23 | 32 |

    As of 2019-06-04

    33 | 38 |

    As of v1.2.0 / 2019-05-30

    39 | 46 |
    47 |

    Please help and contribute with pull requests ❤️

    48 | 49 | 50 |     51 | -------------------------------------------------------------------------------- /app/pages/index.mjs: -------------------------------------------------------------------------------- 1 | export default function Index({ html, state }) { 2 | const { store = { total: '?' } } = state; 3 | const { total } = store; 4 | return html` 13 | 14 | 15 |
    16 |

    webmention.app

    17 |

    Automate your outgoing webmentions

    18 |

    19 | ${total.toLocaleString()} webmentions delivered so far. 22 |

    23 |
    24 |

    25 | This is a platform agnostic service that will check a given URL for 26 | links to other sites, discover if they support webmentions, then 27 | send a webmention to the target. 28 |

    29 | 30 |

    The notification API is reasonably simplistic:

    31 | 32 |
      33 |
    • 34 | POST https://webmention.app/check/?url=:url 35 |

      36 | Finds all links in your given URL, discovers those with valid 37 | Webmention endpoints, and sends the full webmention 38 | notifications. 39 |

      40 |

      41 | The URL should be escaped (though the protocol is not required), 42 | and an 43 | optional token can be used to avoid the 44 | rate limit. 45 |

      46 |
      • 47 |
    • 48 | GET https://webmention.app/check/?url=:url 49 |
      50 | Perform a dry run, reporting on all discovered Webmention 52 | endpoints 54 |
      • 55 |
    56 | 57 |

    Getting started

    58 | 59 | 60 |
    61 |
    62 |
    63 |

    Want to just try it out? Check a URL for webmentions

    64 | 65 |
    66 |
    67 |     `; 68 | } 69 | -------------------------------------------------------------------------------- /app/pages/stats.mjs: -------------------------------------------------------------------------------- 1 | import ms from 'ms'; 2 | 3 | export default function Stats({ html, state }) { 4 | const { store = {} } = state; 5 | const { data = [], total = 0, sent = 0 } = store; 6 | 7 | const list = data 8 | .map(({ url, requested, hits }) => { 9 | return html`
  • 10 | ${asHost(url)} 11 | @ ${toMs(requested)} ago (${hits.toLocaleString()} requests) 12 |
    • `; 13 | }) 14 | .join(''); 15 | 16 | return html` 17 |
    18 |

    Latest stats

    19 |

    20 | Webmentions sent: ${sent.toLocaleString()}
    Total 21 | unique scanned: ${total.toLocaleString()} 22 |

    23 |
      24 | ${list} 25 |
    26 |
    27 |     `; 28 | } 29 | 30 | function asHost(value) { 31 | const url = new URL(value); 32 | return url.hostname + (url.pathname === '/' ? '' : url.pathname); 33 | } 34 | 35 | function toMs(value) { 36 | return ms(Date.now() - new Date(value).getTime()); 37 | } 38 | -------------------------------------------------------------------------------- /app/pages/token-failure.html: -------------------------------------------------------------------------------- 1 | 2 |

    Failed to get auth token

    3 |

    Something went wrong trying to get the auth token.

    4 |

    You can try again or if it persists, open a new issue with any details of the problem.

    6 |     7 | -------------------------------------------------------------------------------- /app/pages/token.mjs: -------------------------------------------------------------------------------- 1 | export default function TokenPage({ html, state }) { 2 | const { store = {} } = state; 3 | const { token } = store; 4 | return html` 5 |
    6 |

    Your token

    7 |

    8 | Including a token in your requests allows you to avoid rate limits on 9 | requests. The anonymous requests are currently limited to once per 4 10 | hours per unique URL. 11 |

    12 | 13 |
    14 | ${token // eslint-disable-next-line indent 15 | ? html` 16 |

    17 | ${token} 18 | 21 |

    22 |

    23 | This token doesn't provide any write access, and is only used 24 | to identify you as a real person wanting to use this service. 25 |

    26 |

    Usage

    27 |

    28 | Include your token as a URL parameter in your calls to the 29 | check API: 30 |

    31 | 
    curl -X POST https://webmention.app/check?token=${token}&url=…
    32 | 57 | ` // eslint-disable-next-line indent 58 | : html`

    59 | Sign in using Github 60 |

    61 |

    62 | Important: the sign in process does not ask 63 | for any private data (nor email) and is only used to assign 64 | you a unique token that will allow you to make as many request 65 | as you need against this service. 66 |

    `} 67 |
    68 |
    69 |

    70 | Remember, if you use the command line tool you don't need a token as 71 | it runs entirely on your own machine. 72 |

    73 |
    74 |     75 | 76 | `; 113 | } 114 | -------------------------------------------------------------------------------- /bin/wm.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | /* eslint-disable no-process-exit */ 3 | const pkg = require('../package.json'); 4 | const opts = require('optimist') 5 | .usage( 6 | 'Parse, discover and send webmentions\n\n$ $0 [ url | file ]\n\nversion: ' + 7 | pkg.version 8 | ) 9 | .boolean('version') 10 | .describe('send', 'send webmention notifications') 11 | .default('limit', 10) 12 | .default('send', false) 13 | .describe('limit', 'int: entries to discover') 14 | .describe('version') 15 | .boolean('debug'); 16 | 17 | const argv = opts.argv; 18 | 19 | if (argv.version) { 20 | console.log(pkg.version); 21 | process.exit(0); 22 | } 23 | 24 | if (argv._.length == 0) { 25 | opts.showHelp(); 26 | process.exit(1); 27 | } 28 | 29 | const ui = require('clui'); 30 | const Progress = ui.Progress; 31 | const progressBar = new Progress(20); 32 | const existsSync = require('fs').existsSync; 33 | const readFileSync = require('fs').readFileSync; 34 | const target = argv._[0]; 35 | const Webmention = require('../shared/lib/webmention'); 36 | 37 | const { limit, debug, send } = argv; 38 | const wm = new Webmention({ limit, send }); 39 | 40 | const clearLine = () => { 41 | if (process.stdout.isTTY) { 42 | process.stdout.clearLine(); 43 | process.stdout.cursorTo(0); 44 | } 45 | }; 46 | 47 | let todo = 0; 48 | let done = 0; 49 | 50 | if (debug) { 51 | console.log('limit = ' + limit); 52 | console.log('send = ' + send); 53 | } 54 | 55 | wm.on('error', (e) => console.error(e)); 56 | wm.on('progress', (e) => { 57 | const [[key, value]] = Object.entries(e); 58 | 59 | if (key === 'endpoints') { 60 | todo = value; 61 | } 62 | 63 | if (key === 'endpoints-resolved') { 64 | done = value; 65 | } 66 | 67 | if (!debug && process.stdout.isTTY) { 68 | clearLine(); 69 | process.stdout.write(progressBar.update(done, todo)); 70 | } 71 | 72 | if (debug) { 73 | console.log( 74 | `${key} = ${value}${e.data ? ' ' + JSON.stringify(e.data) : ''}` 75 | ); 76 | } 77 | }); 78 | if (debug) wm.on('log', (e) => console.log(e)); 79 | wm.on('endpoints', clearLine); 80 | if (!send) { 81 | wm.on('endpoints', (res) => { 82 | if (res.length === 0) { 83 | if (wm.mentions.length) { 84 | console.log( 85 | 'No webmention endpoints found on %s entries found (try increasing with --limit N)', 86 | wm.mentions.length 87 | ); 88 | } else { 89 | console.log('No webmention endpoints found'); 90 | } 91 | } 92 | 93 | res.map((res) => { 94 | console.log('source = ' + res.source); 95 | console.log('target = ' + res.target); 96 | console.log(`endpoint = ${res.endpoint.url} (${res.endpoint.type})`); 97 | console.log(''); 98 | }); 99 | }); 100 | } 101 | 102 | wm.on('sent', (res) => { 103 | console.log('source = ' + res.source); 104 | console.log(`endpoint = ${res.endpoint.url} (${res.endpoint.type})`); 105 | console.log('target = ' + res.target); 106 | console.log(`status = ${res.status} ${res.status < 400 ? '✓' : '✗'}`); // ✖︎✓✔︎✗ 107 | if (res.error) console.log('error = ' + res.error); 108 | console.log(''); 109 | }); 110 | 111 | if (existsSync(target)) { 112 | wm.load(readFileSync(argv._[0], 'utf8')).catch((e) => { 113 | console.log('wm error', e.message); 114 | }); 115 | } else { 116 | wm.fetch(target); 117 | } 118 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | interface Endpoint { 2 | url: string; 3 | type: 'webmention' | 'pingback'; 4 | } 5 | 6 | interface WebMention { 7 | source: string; 8 | endpoint: Endpoint; 9 | } 10 | 11 | interface ProgressCallback { 12 | (event: string, { type: string, value: number, data: any }): void; 13 | } 14 | 15 | interface EndpointCallback { 16 | (error: Error | null, endpoint: Endpoint?); 17 | } 18 | 19 | // type ProgressCallback = { type: string; value: number; data: any }; 20 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "typeRoots": ["./index.d.ts", "./node_modules/@types"], 4 | "checkJs": true 5 | }, 6 | "exclude": ["node_modules"] 7 | } 8 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@remy/webmention", 3 | "version": "1.5.0", 4 | "description": "Web Mentions sender", 5 | "main": "./shared/lib/webmention.js", 6 | "repository": "github:remy/wm", 7 | "bin": { 8 | "wm": "./bin/wm.js", 9 | "webmention": "./bin/wm.js" 10 | }, 11 | "files": [ 12 | "shared/lib/**/*", 13 | "bin/wm.js" 14 | ], 15 | "types": "./index.d.ts", 16 | "scripts": { 17 | "start": "npx enhance dev --no-warnings", 18 | "lint": "eslint ./app/**/*.mjs --fix", 19 | "enhance": "enhance", 20 | "test:watch": "tap __tests__/*.test.js --no-coverage-report --watch", 21 | "test": "tap" 22 | }, 23 | "keywords": [ 24 | "webmention", 25 | "webmentions", 26 | "indieweb" 27 | ], 28 | "author": "Remy Sharp (https://remysharp.com)", 29 | "license": "MIT", 30 | "dependencies": { 31 | "cheerio": "^0.22.0", 32 | "clui": "^0.3.6", 33 | "decodeuricomponent": "^0.3.1", 34 | "follow-redirects": "^1.7.0", 35 | "li": "^1.3.0", 36 | "microformat-node": "^2.0.1", 37 | "ms": "^2.1.2", 38 | "node-fetch": "^2.6.1", 39 | "optimist": "^0.6.1", 40 | "rss-parser": "^3.7.0", 41 | "uuid": "^3.3.2" 42 | }, 43 | "devDependencies": { 44 | "@enhance/arc-plugin-enhance": "^6.2.3", 45 | "@enhance/cli": "latest", 46 | "@enhance/element": "^1.3.1", 47 | "@enhance/styles-cribsheet": "^0.0.9", 48 | "@enhance/types": "^0.6.1", 49 | "@remy/envy": "^4.0.2", 50 | "aws-sdk": "^2.466.0", 51 | "eslint": "^8.49.0", 52 | "markdown-it-named-headings": "^1.1.0", 53 | "markdown-it-task-lists": "^2.1.1", 54 | "passport": "^0.4.0", 55 | "passport-github2": "^0.1.11", 56 | "tap": "^16.3.9" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /prefs.arc: -------------------------------------------------------------------------------- 1 | @sandbox 2 | livereload true 3 | ports 4 | http 4444 5 | -------------------------------------------------------------------------------- /public/copy.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /public/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/remy/wm/a11bc5b11fd8bb7abaf9f04f7f8719c92c5f1407/public/favicon.png -------------------------------------------------------------------------------- /public/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /public/netlify.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/remy/wm/a11bc5b11fd8bb7abaf9f04f7f8719c92c5f1407/public/netlify.png -------------------------------------------------------------------------------- /public/style.css: -------------------------------------------------------------------------------- 1 | * { 2 | box-sizing: border-box; 3 | } 4 | 5 | body, 6 | input, 7 | button { 8 | font-size: 16px; 9 | line-height: 28px; 10 | font-family: 'Rubik', sans-serif; 11 | } 12 | 13 | [v-cloak] { 14 | display: none; 15 | } 16 | 17 | pre, 18 | code { 19 | font-family: 'Ubuntu mono', 'SFMono-Regular', Consolas, 'Liberation Mono', 20 | Menlo, Courier, monospace; 21 | } 22 | h1 { 23 | font-size: 2rem; 24 | line-height: 2.4rem; 25 | text-align: center; 26 | } 27 | 28 | h2 { 29 | font-size: 1.3rem; 30 | line-height: 1.6rem; 31 | color: rgb(148, 146, 146); 32 | } 33 | 34 | #index h2, 35 | #index h3 { 36 | text-align: center; 37 | } 38 | 39 | #docs h2 { 40 | text-align: left; 41 | } 42 | 43 | h3 { 44 | margin: 16px 0; 45 | } 46 | 47 | h2 * { 48 | font-size: 1.3rem; 49 | padding: 0; 50 | } 51 | 52 | h2 strong { 53 | border-bottom: 2px solid #9e9e9e; 54 | } 55 | 56 | strong { 57 | color: black; 58 | } 59 | 60 | pre { 61 | overflow: scroll; 62 | border-left: 4px solid #ccc; 63 | padding: 16px; 64 | padding-right: 32px; 65 | line-height: 20px; 66 | } 67 | 68 | pre code { 69 | width: 100%; 70 | } 71 | 72 | pre .prompt { 73 | font-family: inherit; 74 | font-size: inherit; 75 | user-select: none; 76 | opacity: 0.5; 77 | } 78 | 79 | hr { 80 | border: 0; 81 | border-bottom: 1px solid #ccc; 82 | margin: 0; 83 | padding: 0; 84 | margin-bottom: 1rem; 85 | padding-top: 1rem; 86 | } 87 | 88 | html { 89 | margin: 0; 90 | min-height: 100%; 91 | font-size: 1.2rem; 92 | } 93 | 94 | a, 95 | .link { 96 | color: #424242; 97 | text-decoration: none; 98 | border: 0; 99 | padding: 0; 100 | cursor: pointer; 101 | border-bottom: 1px solid #424242; 102 | margin-bottom: 1px; 103 | background: none; 104 | line-height: initial; 105 | } 106 | 107 | a:hover, 108 | .link:hover { 109 | border-width: 2px; 110 | margin-bottom: 0px; 111 | color: black; 112 | } 113 | 114 | p { 115 | color: #616161; 116 | margin: 24px 0; 117 | } 118 | 119 | p.summary { 120 | font-weight: bold; 121 | } 122 | 123 | footer, 124 | footer * { 125 | font-size: 12px; 126 | line-height: 22px; 127 | } 128 | 129 | html, 130 | body, 131 | #__nuxt, 132 | #__layout { 133 | min-height: 100%; 134 | height: 100%; 135 | margin: 0; 136 | } 137 | 138 | main { 139 | height: 100%; 140 | width: calc(100% - 40px); 141 | display: flex; 142 | flex-direction: column; 143 | margin: 0 auto; 144 | max-width: 600px; 145 | flex-grow: 1; 146 | padding-top: 40px; 147 | } 148 | 149 | main > div.content { 150 | flex-grow: 1; 151 | } 152 | 153 | main > footer { 154 | margin: 20px 0; 155 | } 156 | 157 | #navigation ul { 158 | list-style: none; 159 | margin: 0; 160 | padding: 0; 161 | display: flex; 162 | align-items: center; 163 | } 164 | 165 | #navigation li { 166 | margin: 10px; 167 | } 168 | 169 | #navigation li:first-child { 170 | margin-left: 0; 171 | } 172 | 173 | .results { 174 | margin: 0 auto; 175 | width: 600px; 176 | } 177 | 178 | .result { 179 | border-bottom: 1px solid #ccc; 180 | padding-bottom: 40px; 181 | margin-bottom: 40px; 182 | } 183 | 184 | .result:last-child { 185 | border: 0; 186 | } 187 | 188 | @media screen and (max-width: 600px) { 189 | .results { 190 | width: 100%; 191 | } 192 | } 193 | 194 | .block { 195 | width: 100%; 196 | /* background: blue; */ 197 | display: inline-block; 198 | height: 100%; 199 | position: relative; 200 | } 201 | 202 | .block span { 203 | position: absolute; 204 | /* height: 20px; */ 205 | background: #00bcd4; 206 | } 207 | 208 | tbody td:first-child { 209 | white-space: nowrap; 210 | width: 140px; 211 | padding-right: 20px; 212 | } 213 | 214 | hr { 215 | border: 0; 216 | border-bottom: 1px solid #ccc; 217 | } 218 | 219 | .cmd-results { 220 | color: gray; 221 | margin-top: 10px; 222 | display: inline-block; 223 | } 224 | 225 | /* @media screen and (max-width: 420px) { 226 | * { 227 | font-size: 0.6rem; 228 | line-height: 0.9rem; 229 | } 230 | } */ 231 | 232 | ul { 233 | padding-left: 1rem; 234 | /* list-style-type: square; */ 235 | } 236 | 237 | summary { 238 | cursor: pointer; 239 | } 240 | 241 | summary h2 { 242 | display: inline; 243 | } 244 | 245 | summary::-webkit-details-marker { 246 | color: rgb(158, 158, 158); 247 | } 248 | 249 | details, 250 | summary + div { 251 | margin-top: 20px; 252 | } 253 | 254 | /* details { 255 | margin: 20px 0; 256 | border: 1px solid rgb(238, 238, 238); 257 | border-radius: 4px; 258 | } 259 | 260 | summary::-webkit-details-marker { 261 | display: none; 262 | } 263 | 264 | summary:hover, 265 | [open] summary { 266 | background: #eee; 267 | } 268 | 269 | summary { 270 | display: flex; 271 | justify-content: space-between; 272 | align-items: center; 273 | cursor: pointer; 274 | flex-wrap: wrap; 275 | padding: 20px; 276 | } 277 | 278 | summary .subject { 279 | flex: 1; 280 | min-width: 100%; 281 | } */ 282 | 283 | /* details > div { 284 | margin: 20px; 285 | } */ 286 | 287 | .label { 288 | color: #717171; 289 | white-space: nowrap; 290 | } 291 | 292 | .logo { 293 | flex: 1; 294 | } 295 | 296 | .logo a { 297 | width: 64px; 298 | margin-bottom: 0; 299 | } 300 | 301 | .logo .logo-accent { 302 | fill: white; 303 | transform: fill 500ms ease-out; 304 | } 305 | 306 | li span { 307 | color: #616161; 308 | } 309 | 310 | details strong { 311 | color: black; 312 | } 313 | 314 | details ol { 315 | margin: 0; 316 | } 317 | 318 | details li a { 319 | white-space: nowrap; 320 | } 321 | 322 | details pre { 323 | max-height: 160px; 324 | padding-bottom: 20px; 325 | border-bottom: 1px solid #ccc; 326 | line-height: 24px; 327 | } 328 | 329 | details div.open pre { 330 | max-height: initial; 331 | } 332 | 333 | details div:last-child pre { 334 | border-bottom: 0; 335 | } 336 | 337 | .btn { 338 | display: inline-block; 339 | line-height: 28px; 340 | min-width: 100px; 341 | white-space: nowrap; 342 | border-radius: 4px; 343 | cursor: pointer; 344 | margin: 0; 345 | background-color: rgb(66, 66, 66); 346 | border: 1px solid rgb(66, 66, 66); 347 | color: white; 348 | text-transform: uppercase; 349 | font-size: 80%; 350 | padding: 4px 16px; 351 | transition: background-color 100ms ease-out, color 100ms ease-out; 352 | } 353 | 354 | .btn:disabled { 355 | cursor: wait; 356 | transition: opacity 0.3s ease; 357 | background-size: 30px 30px; 358 | background-image: linear-gradient( 359 | 45deg, 360 | rgba(0, 0, 0, 0.1) 25%, 361 | transparent 25%, 362 | transparent 50%, 363 | rgba(0, 0, 0, 0.1) 50%, 364 | rgba(0, 0, 0, 0.1) 75%, 365 | transparent 75%, 366 | transparent 367 | ); 368 | animation: loading 0.5s linear infinite; 369 | color: #aaa; 370 | background-color: white; 371 | } 372 | 373 | @keyframes loading { 374 | from { 375 | background-position: 0 0; 376 | } 377 | to { 378 | background-position: 60px 30px; 379 | } 380 | } 381 | 382 | .btn:hover, 383 | .btn:active { 384 | border-width: 1px; 385 | outline: none; 386 | background-color: white; 387 | color: rgb(66, 66, 66); 388 | } 389 | 390 | input[type='text'], 391 | input[type='url'] { 392 | width: 100%; 393 | border-radius: 4px; 394 | border: 1px solid #ccc; 395 | padding: 4px 8px; 396 | margin: 0 8px; 397 | } 398 | 399 | input:focus, 400 | input:hover { 401 | border-color: black; 402 | outline: none; 403 | } 404 | 405 | .sep { 406 | margin: 0 8px; 407 | } 408 | 409 | .flex-fields { 410 | display: flex; 411 | align-items: center; 412 | } 413 | 414 | #mentions { 415 | color: #9e9e9e; 416 | } 417 | 418 | #mentions:empty { 419 | margin: 0; 420 | } 421 | 422 | li { 423 | margin: 8px 0; 424 | } 425 | 426 | .flex-grow { 427 | flex-grow: 1; 428 | } 429 | 430 | .flex { 431 | display: flex; 432 | flex-direction: column; 433 | min-height: 100%; 434 | } 435 | 436 | footer ul { 437 | list-style: none; 438 | padding: 0; 439 | margin: 20px 0; 440 | display: flex; 441 | } 442 | 443 | footer li:after { 444 | content: ' // '; 445 | margin: 0 16px; 446 | color: #9e9e9e; 447 | display: inline-block; 448 | } 449 | 450 | footer li:last-child:after { 451 | content: none; 452 | } 453 | 454 | .cta { 455 | background: #03a9f4; 456 | margin: 0 auto; 457 | display: block; 458 | } 459 | 460 | #app-container > div { 461 | flex-grow: 1; 462 | } 463 | 464 | #navigation a { 465 | font-size: 14px; 466 | } 467 | 468 | #navigation li { 469 | line-height: 22px; 470 | } 471 | 472 | #navigation .logo a { 473 | border: 0; 474 | display: flex; 475 | flex-direction: column; 476 | align-items: center; 477 | color: black; 478 | justify-content: inherit; 479 | place-items: flex-start; 480 | } 481 | 482 | #navigation .logo a:hover img { 483 | background: red; 484 | } 485 | 486 | #navigation .logo a span { 487 | font-size: 12px; 488 | line-height: 14px; 489 | } 490 | 491 | .contains-task-list { 492 | list-style: none; 493 | padding: 0; 494 | } 495 | 496 | .status { 497 | border-radius: 4px; 498 | font-size: 80%; 499 | padding: 0px 6px; 500 | background: #2196f3; 501 | color: white; 502 | text-align: center; 503 | display: inline-block; 504 | line-height: 22px; 505 | } 506 | 507 | .status-2, 508 | .status-200, 509 | .status-201 { 510 | background: green; 511 | } 512 | 513 | .status-4, 514 | .status-5 { 515 | background: #b71c1c; 516 | } 517 | -------------------------------------------------------------------------------- /public/webmention-app-card.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/remy/wm/a11bc5b11fd8bb7abaf9f04f7f8719c92c5f1407/public/webmention-app-card.jpg -------------------------------------------------------------------------------- /shared/enhance-styles: -------------------------------------------------------------------------------- 1 | /Users/remy/dev/webmention.app/.enhance -------------------------------------------------------------------------------- /shared/lib/db.js: -------------------------------------------------------------------------------- 1 | var DynamoDB = require('aws-sdk/clients/dynamodb'); 2 | 3 | let client = new DynamoDB.DocumentClient({ 4 | region: 'eu-west-2', 5 | accessKeyId: process.env.DB_KEY_ID, 6 | secretAccessKey: process.env.DB_ACCESS_KEY, 7 | }); 8 | 9 | function _(TableName) { 10 | return { 11 | get(Key) { 12 | return client 13 | .get({ TableName, Key }) 14 | .promise() 15 | .then((res) => res.Item); 16 | }, 17 | put(Item) { 18 | return client.put({ TableName, Item }).promise(); 19 | }, 20 | }; 21 | } 22 | 23 | function getRecentURLs() { 24 | return client 25 | .scan({ 26 | TableName: 'wm-requests', 27 | ScanIndexForward: true, 28 | }) 29 | .promise(); 30 | } 31 | 32 | function getRequestCount(url) { 33 | return _('wm-requests').get({ url }); 34 | } 35 | 36 | async function updateRequestCount(url, increment = 1) { 37 | const requested = new Date().toJSON(); 38 | 39 | const update = { 40 | Key: { url }, 41 | TableName: 'wm-requests', 42 | UpdateExpression: 'set hits = hits + :val, requested = :requested', 43 | ExpressionAttributeValues: { 44 | ':val': increment, 45 | ':requested': requested, 46 | }, 47 | ReturnValues: 'UPDATED_NEW', 48 | }; 49 | 50 | return client 51 | .update(update) 52 | .promise() 53 | .catch((err) => { 54 | if (err && err.code === 'ValidationException') { 55 | // make it 56 | return _('wm-requests').put({ 57 | url, 58 | requested, 59 | hits: 1, 60 | }); 61 | } 62 | console.error( 63 | 'Unable to add item. Error JSON:', 64 | JSON.stringify(err, null, 2) 65 | ); 66 | throw err; 67 | }); 68 | } 69 | 70 | function getByUsername(username) { 71 | const ExpressionAttributeValues = { ':username': username }; 72 | const KeyConditionExpression = `username = :username`; 73 | 74 | const params = { 75 | TableName: 'wm-users', 76 | IndexName: 'username-index', 77 | KeyConditionExpression, 78 | ExpressionAttributeValues, 79 | }; 80 | 81 | return client 82 | .query(params) 83 | .promise() 84 | .then((res) => { 85 | return res.Items[0]; 86 | }) 87 | .catch(() => { 88 | // console.log('catch on getByUsername(%s): %s', username, e.message); 89 | return null; 90 | }); 91 | } 92 | 93 | function createUser({ username, token, service, id }) { 94 | return _('wm-users').put({ username, token, requests: 0, [service]: id }); 95 | } 96 | 97 | function updateTokenRequestCount(token) { 98 | const update = { 99 | Key: { token }, 100 | TableName: 'wm-users', 101 | UpdateExpression: 'set requests = requests + :val', 102 | ExpressionAttributeValues: { 103 | ':val': 1, 104 | }, 105 | ReturnValues: 'UPDATED_NEW', 106 | }; 107 | 108 | return client 109 | .update(update) 110 | .promise() 111 | .catch(() => null); 112 | } 113 | 114 | module.exports = { 115 | updateRequestCount, 116 | getByUsername, 117 | createUser, 118 | updateTokenRequestCount, 119 | getRequestCount, 120 | getRecentURLs, 121 | _, 122 | }; 123 | -------------------------------------------------------------------------------- /shared/lib/endpoint.js: -------------------------------------------------------------------------------- 1 | // @ts-check 2 | const { http, https } = require('follow-redirects'); 3 | const li = require('li'); 4 | const cheerio = require('cheerio'); 5 | const url = require('url'); 6 | 7 | const cache = new Map(); 8 | 9 | /** 10 | * @param {string} source 11 | * @returns {Promise} 12 | */ 13 | const main = (source) => 14 | new Promise((resolve, reject) => { 15 | if (cache.has(source)) { 16 | return resolve({ source, endpoint: cache.get(source) }); 17 | } 18 | getWebmentionUrl(source, (err, endpoint) => { 19 | if (err) { 20 | return reject(err); 21 | } 22 | 23 | cache.set(source, endpoint || null); 24 | resolve({ source, endpoint }); 25 | }); 26 | }); 27 | 28 | /** 29 | * 30 | * @param {string} html 31 | * @param {string} baseUrl 32 | * @returns {{url: string, type: string} | undefined} 33 | */ 34 | function findEndpoints(html, baseUrl) { 35 | const $ = cheerio.load(html); 36 | 37 | const res = $('link, a') 38 | .map(function (idx, el) { 39 | const rels = (el.attribs.rel || '').split(' ').filter(Boolean); 40 | 41 | if ( 42 | (rels.includes('webmention') || rels.includes('pingback')) && 43 | // We explicitly check for undefined because we want to catch empty strings, but those are falsy 44 | typeof el.attribs.href !== 'undefined' 45 | ) { 46 | return { 47 | url: url.resolve(baseUrl, el.attribs.href), 48 | type: rels.includes('webmention') ? 'webmention' : 'pingback', 49 | }; 50 | } 51 | 52 | return null; 53 | }) 54 | .get() 55 | .filter(Boolean); 56 | 57 | const webmention = res.find((_) => _.type === 'webmention'); 58 | return webmention || res[0]; 59 | } 60 | 61 | module.exports = main; 62 | 63 | module.exports.findEndpoints = findEndpoints; 64 | 65 | /** 66 | * 67 | * @param {object|string} opts 68 | * @param {EndpointCallback} callback 69 | */ 70 | function getWebmentionUrl(opts, realCallback) { 71 | var parsed; 72 | if (typeof opts === 'string') { 73 | parsed = url.parse(opts); 74 | } else if (opts.url) { 75 | parsed = url.parse(opts.url); 76 | } else { 77 | parsed = opts; 78 | } 79 | 80 | parsed.headers = { 81 | 'user-agent': 'webmention.app', 82 | }; 83 | 84 | parsed.timeout = 5 * 1000; 85 | 86 | const client = parsed.protocol === 'http:' ? http : https; 87 | 88 | let complete = false; 89 | const callback = (err, res) => { 90 | if (complete) return; 91 | complete = true; 92 | realCallback(err, res); 93 | }; 94 | 95 | const req = client.request(parsed, function (res) { 96 | if (res.statusCode < 200 || res.statusCode >= 400) { 97 | callback(); 98 | } 99 | 100 | if (res.headers['x-pingback']) { 101 | callback(undefined, { url: res.headers['x-pingback'], type: 'pingback' }); 102 | } 103 | 104 | if (res.headers.link) { 105 | const links = li.parse(res.headers.link); 106 | const endpoint = links.webmention || links['http://webmention.org/']; 107 | 108 | if (endpoint) { 109 | callback(undefined, { 110 | url: url.resolve(res.responseUrl, endpoint), 111 | type: 'webmention', 112 | }); 113 | } 114 | } 115 | 116 | // don't try to parse non-text (i.e. mp3s!) 117 | if ( 118 | !res.headers['content-type'] || 119 | !res.headers['content-type'].startsWith('text/') 120 | ) { 121 | callback(); 122 | } 123 | var buf = ''; 124 | 125 | res.on('data', (chunk) => (buf += chunk.toString())); 126 | res.on('end', () => { 127 | callback(null, findEndpoints(buf.toString(), res.responseUrl)); 128 | }); 129 | }); 130 | 131 | req.on('error', callback); 132 | 133 | req.on('timeout', () => { 134 | req.abort(); 135 | callback(new Error(`Timeout`)); 136 | }); 137 | 138 | req.end(); 139 | } 140 | -------------------------------------------------------------------------------- /shared/lib/get-wm-endpoints.js: -------------------------------------------------------------------------------- 1 | const parse = require('url').parse; 2 | const hosts = require('./ignored-endpoints'); 3 | const wm = require('./endpoint'); 4 | 5 | /** 6 | * 7 | * @param {string[]} urls 8 | * @param {ProgressCallback} progress 9 | * @param {number} [limit=10] 10 | * @returns {Promise} 11 | */ 12 | async function main(urls, progress = () => {}, limit = 10) { 13 | progress('log', 'URLs to check: ' + urls.length, limit); 14 | progress('progress-update', { type: 'endpoints', value: urls.length }); 15 | 16 | return Promise.all( 17 | urls 18 | .filter((source) => { 19 | const hostname = parse(source).hostname; 20 | return !hosts.find((_) => { 21 | if (_.indexOf('.') === 0) { 22 | const test = hostname.endsWith(_) || hostname === _.slice(1); 23 | if (test) { 24 | progress('log', 'skipping ' + source); 25 | return true; 26 | } 27 | } 28 | return hostname === _; 29 | }); 30 | }) 31 | .map((source) => { 32 | return wm(source) 33 | .catch((e) => { 34 | progress('error', `Get endpoint fail (${source}): ${e.message}`); 35 | return { source, endpoint: null }; 36 | }) 37 | .then((res) => { 38 | progress('progress-update', { 39 | type: 'endpoints-resolved', 40 | value: 1, 41 | data: { source, endpoint: res.endpoint }, 42 | }); 43 | return res; 44 | }); 45 | }) 46 | ) 47 | .then((res) => { 48 | return urls.map((url) => { 49 | return Object.assign( 50 | { url }, 51 | res.find(({ source }) => url.startsWith(source)) 52 | ); 53 | }); 54 | }) 55 | .then((res) => res.filter((_) => _.endpoint).slice(0, limit || undefined)); 56 | } 57 | 58 | module.exports = main; 59 | -------------------------------------------------------------------------------- /shared/lib/html/dom.js: -------------------------------------------------------------------------------- 1 | const cheerio = require('cheerio'); 2 | 3 | function main(html) { 4 | const $ = cheerio.load(html.toString()); 5 | 6 | let base = $('.h-entry, .hentry'); 7 | 8 | if (base.length === 0) { 9 | base = $('html'); 10 | } 11 | 12 | return { base, $ }; 13 | } 14 | 15 | module.exports = main; 16 | -------------------------------------------------------------------------------- /shared/lib/ignored-endpoints.js: -------------------------------------------------------------------------------- 1 | /* 2 | Add known websites that do not and are unlikely ever to have 3 | a webmention or pingback endpoint. 4 | 5 | If it's an exact hostname, then use it's name, if you want 6 | to match all cnames off the hostname, use a leading period, 7 | e.g. .github.com will match both github.com and gist.github.com 8 | */ 9 | 10 | module.exports = ` 11 | .amazon.com 12 | .amazonaws.com 13 | .cloudfront.net 14 | .facebook.com 15 | .flickr.com 16 | .github.com 17 | .google.com 18 | .instagram.com 19 | .linkedin.com 20 | .mapbox.com 21 | .medium.com 22 | .gravatar.com 23 | .npmjs.com 24 | .stackoverflow.com 25 | .swarmapp.com 26 | .twimg.com 27 | .twitter.com 28 | .wordpress.org 29 | .wp.com 30 | .youtube.com 31 | .w3.org 32 | bit.ly 33 | developer.mozilla.org 34 | gitlab.com 35 | html.spec.whatwg.org 36 | httparchive.org 37 | ifttt.com 38 | ind.ie 39 | indieauth.com 40 | instagram.com 41 | microformats.org 42 | www.brid.gy 43 | www.complexity-explorables.org 44 | www.frontendunited.org 45 | www.meltingasphalt.com 46 | www.newyorker.com 47 | www.wired.com 48 | youtu.be 49 | ${/* here begin known 404s */ ''} 50 | pipes.yahoo.com 51 | andreaarbogast.org 52 | ` 53 | .split('\n') 54 | .map((_) => _.trim()) 55 | .filter(Boolean); 56 | -------------------------------------------------------------------------------- /shared/lib/links.js: -------------------------------------------------------------------------------- 1 | const request = require('./request'); 2 | const dom = require('./html/dom'); 3 | const smellsLikeRSS = require('./rss/is'); 4 | const rss = require('./rss/dom'); 5 | const resolve = require('url').resolve; 6 | const parse = require('url').parse; 7 | 8 | function links({ $, base, url = '' }) { 9 | let baseHref = $('base, link[rel~="canonical"]').attr('href') || url; 10 | const hostname = parse(baseHref).hostname; 11 | 12 | return base 13 | .map((i, element) => { 14 | const $$ = $(element); 15 | 16 | let permalink = resolve( 17 | baseHref, 18 | $$.find('.u-url').attr('href') || element.link || '' 19 | ); 20 | 21 | if (!permalink.includes(hostname) && base.length === 1) { 22 | // it's probably bad, so let's reset it 23 | permalink = baseHref; 24 | } 25 | 26 | const anchors = $$.find('a[href^="http:"], a[href^="https:"]') 27 | .map((i, el) => $(el).attr('href')) 28 | .get(); 29 | 30 | try { 31 | // extraLinks can sometimes return structured microformats, so make 32 | // sure to flatten that into an array of strings 33 | let more = base.extraLinks().map((_) => { 34 | if (typeof _ === 'string') { 35 | return _; 36 | } 37 | 38 | if (_.value) { 39 | return _.value; 40 | } 41 | 42 | if (_.properties) { 43 | return _.properties.urls[0]; 44 | } 45 | }); 46 | 47 | anchors.push(...more); 48 | } catch (e) { 49 | // noop 50 | } 51 | 52 | // leaving for the time being 53 | const images = $$.find('img[src^="http"]') 54 | .map((i, el) => $(el).attr('src')) 55 | .get(); 56 | 57 | const media = []; 58 | $$.find('video, audio').each((i, el) => { 59 | const sources = $(el).find('source'); 60 | if (sources.length) { 61 | sources.each((i, el) => media.push($(el).attr('src'))); 62 | } else { 63 | media.push($(el).attr('src')); 64 | } 65 | }); 66 | 67 | // note: fragment identifiers on the URL are considered part of the target 68 | // that should be sent. 69 | return { 70 | permalink, 71 | links: [] 72 | .concat(anchors, images, media) 73 | .filter((url) => url.startsWith('http')) 74 | .filter((url) => url !== permalink) 75 | .filter((curr, i, self) => self.indexOf(curr) === i), // unique 76 | }; 77 | }) 78 | .get(); 79 | } 80 | 81 | function getLinksFromHTML({ html, url }) { 82 | return links({ ...dom(html), url }); 83 | } 84 | 85 | async function getLinksFromFeed({ xml, limit }) { 86 | const res = await rss(xml, limit); 87 | 88 | return links({ ...res, rss: true }); 89 | } 90 | 91 | function getFromContent(content, url, limit) { 92 | if (smellsLikeRSS(content)) { 93 | return getLinksFromFeed({ xml: content, limit }); 94 | } 95 | 96 | // else: html 97 | return getLinksFromHTML({ html: content, url }); 98 | } 99 | 100 | async function get(url, limit) { 101 | const content = await request(url); 102 | return getFromContent(content, url, limit); 103 | } 104 | 105 | module.exports = { 106 | links, 107 | getLinksFromHTML, 108 | getFromContent, 109 | get, 110 | }; 111 | -------------------------------------------------------------------------------- /shared/lib/microformat/dom.js: -------------------------------------------------------------------------------- 1 | const microformats = require('microformat-node'); 2 | const cheerio = require('cheerio'); 3 | 4 | function findEntries(mf) { 5 | if (Array.isArray(mf)) { 6 | return mf.reduce((acc, curr) => { 7 | const found = findEntries(curr); 8 | if (Array.isArray(found)) { 9 | acc.push(...found); 10 | } else if (found) { 11 | acc.push(found); 12 | } 13 | 14 | return acc; 15 | }, []); 16 | } 17 | 18 | if (mf.children) { 19 | return findEntries(mf.children); 20 | } 21 | 22 | if (mf.type.includes('h-entry')) { 23 | if (mf.value) { 24 | return mf; 25 | } 26 | if (mf.properties && mf.properties.content) { 27 | if (Array.isArray(mf.properties.content)) { 28 | return mf.properties.content.map((content) => { 29 | return { 30 | properties: { 31 | ...mf.properties, 32 | content: [content], 33 | }, 34 | type: mf.type, 35 | }; 36 | }); 37 | } else { 38 | return mf; 39 | } 40 | } 41 | 42 | return null; 43 | } 44 | } 45 | 46 | async function dom(html, { url, limit }) { 47 | const mf = await microformats.getAsync({ html, filter: ['h-entry'] }); 48 | let entries = findEntries(mf.items); 49 | 50 | if (limit) { 51 | entries = entries.slice(0, limit || undefined); 52 | } 53 | 54 | if (!url) { 55 | if (mf.rels.canonical) { 56 | url = mf.rels.canonical[0]; 57 | } else if (entries.length && entries[0].properties.url) { 58 | url = entries[0].properties.url[0]; 59 | } 60 | } 61 | 62 | const base = { 63 | length: entries.length, 64 | map: (callback) => { 65 | const res = entries.map((item, i) => { 66 | item.link = item.properties.url ? item.properties.url[0] : url; 67 | return callback(i, item); 68 | }); 69 | 70 | return { 71 | get: () => res, 72 | }; 73 | }, 74 | extraLinks: () => { 75 | const res = []; 76 | 77 | const types = [ 78 | 'in-reply-to', 79 | 'like-of', 80 | 'repost-of', 81 | 'bookmark-of', 82 | 'mention-of', 83 | 'rsvp', 84 | ]; 85 | 86 | types.forEach((type) => { 87 | if (!Array.isArray(entries[0].properties[type])) { 88 | return; 89 | } 90 | 91 | const urls = entries[0].properties[type].map((value) => { 92 | if (typeof value === 'string') { 93 | return value; 94 | } 95 | 96 | if (value.properties.url) { 97 | return value.properties.url[0]; 98 | } 99 | }); 100 | 101 | res.push(...urls); 102 | }); 103 | 104 | if (entries[0].properties.url) { 105 | res.push(...entries[0].properties.url); 106 | } 107 | return res; 108 | }, 109 | }; 110 | 111 | const $ = (element) => { 112 | if (typeof element === 'string') { 113 | return { 114 | attr: () => null, 115 | }; 116 | } 117 | 118 | if (element.properties) { 119 | // try encoded content first 120 | const content = element.properties.content 121 | ? element.properties.content[0].html 122 | : element.value; 123 | 124 | // wrapping in a div ensures there's a selectable dom 125 | return cheerio.load(`
    ${content}
    `)(':root'); 126 | } 127 | 128 | return cheerio.load(element)(':root'); 129 | }; 130 | 131 | return { base, $, url }; 132 | } 133 | 134 | module.exports = dom; 135 | module.exports.findEntries = findEntries; 136 | -------------------------------------------------------------------------------- /shared/lib/passport.js: -------------------------------------------------------------------------------- 1 | const Strategy = require('passport-github2').Strategy; 2 | const uuid = require('./uuid'); 3 | const db = require('./db'); 4 | 5 | const config = { 6 | clientID: process.env.GITHUB_CLIENT_ID, 7 | clientSecret: process.env.GITHUB_CLIENT_SECRET, 8 | }; 9 | 10 | exports.github = new Strategy( 11 | config, 12 | (accessToken, refreshToken, profile, done) => { 13 | const username = `github-${profile.id}`; 14 | 15 | db.getByUsername(username) 16 | .then((user) => { 17 | if (user) { 18 | return done(null, user.token); 19 | } 20 | 21 | const token = uuid(); 22 | 23 | db.createUser({ username, token, service: 'github', id: profile.id }) 24 | .then(() => done(null, token)) 25 | .catch((e) => done(e)); 26 | }) 27 | .catch(done); 28 | } 29 | ); 30 | -------------------------------------------------------------------------------- /shared/lib/query-string.js: -------------------------------------------------------------------------------- 1 | const url = require('url'); 2 | const qsParser = require('querystring').parse; 3 | const decodeUriComponent = require('decodeuricomponent'); 4 | 5 | module.exports = (req) => 6 | qsParser(url.parse(req.url).query, null, null, { 7 | decodeURIComponent: decodeUriComponent, 8 | }); 9 | -------------------------------------------------------------------------------- /shared/lib/request.js: -------------------------------------------------------------------------------- 1 | const { http, https } = require('follow-redirects'); 2 | const headers = { 'User-Agent': 'webmention.app' }; 3 | 4 | // by default timeout at 5 seconds 5 | function main(url, timeout = 500) { 6 | return new Promise((resolve, reject) => { 7 | const client = url.startsWith('http:') ? http : https; 8 | let timer = null; 9 | const req = client.request(url, { timeout, headers }, (res) => { 10 | clearTimeout(timer); 11 | if (res.statusCode < 200 || res.statusCode >= 400) { 12 | reject(new Error(`Bad response ${res.statusCode} on ${url}`)); 13 | return; 14 | } 15 | 16 | let reply = ''; 17 | 18 | res.on('data', (chunk) => (reply += chunk)); 19 | res.on('end', () => { 20 | resolve({ content: reply, responseUrl: res.responseUrl }); 21 | }); 22 | 23 | res.on('error', reject); 24 | }); 25 | req.on('timeout', () => { 26 | req.abort(); 27 | reject(new Error('Timeout')); 28 | }); 29 | req.on('error', (err) => reject(err)); 30 | timer = setTimeout(() => { 31 | req.abort(); 32 | reject(new Error('Timeout')); 33 | }, timeout); 34 | req.end(); 35 | }); 36 | } 37 | 38 | module.exports = main; 39 | -------------------------------------------------------------------------------- /shared/lib/rss/dom.js: -------------------------------------------------------------------------------- 1 | const Parser = require('rss-parser'); 2 | const cheerio = require('cheerio'); 3 | 4 | async function main(xml, limit = 10) { 5 | xml = xml.toString(); 6 | const rss = await new Parser({ 7 | customFields: { 8 | item: ['summary'], 9 | }, 10 | }).parseString(xml); 11 | const dollar = cheerio.load(xml); 12 | 13 | const url = rss.link; 14 | 15 | let items = rss.items; 16 | if (limit) { 17 | items = items.slice(0, limit || undefined); 18 | } 19 | 20 | const base = { 21 | map: (callback) => { 22 | const res = items.map((item, i) => { 23 | return callback(i, item); 24 | }); 25 | 26 | return { 27 | get: () => res, 28 | }; 29 | }, 30 | }; 31 | 32 | const $ = (element) => { 33 | if (typeof element === 'string') { 34 | return { 35 | attr: () => null, 36 | }; 37 | } 38 | 39 | if ( 40 | element.content && 41 | element.content.$ && 42 | element.content.$.type === 'html' 43 | ) { 44 | return dollar(element.content); 45 | } 46 | 47 | if (element.content) { 48 | // try encoded content first 49 | return dollar( 50 | element['content:encoded'] || `
    ${element.content}
    ` 51 | ); 52 | } 53 | 54 | if (element.summary) { 55 | if (element.summary.$ && element.summary.$.type === 'text') { 56 | return dollar(`

    ${element.summary._}

    `); 57 | } 58 | } 59 | 60 | return dollar(element); 61 | }; 62 | 63 | return { base, $, url, rss: { items } }; 64 | } 65 | 66 | module.exports = main; 67 | -------------------------------------------------------------------------------- /shared/lib/rss/is.js: -------------------------------------------------------------------------------- 1 | module.exports = (source) => { 2 | source = source.replace(/>\s+ ` 4 | 5 | pingback.ping 6 | 7 | ${source} 8 | ${target} 9 | 10 | `; 11 | 12 | async function main({ source, target, endpoint }) { 13 | const xmlBody = xml({ source, target }); 14 | 15 | const res = await fetch(endpoint, { 16 | method: 'POST', 17 | body: xmlBody, 18 | headers: { 19 | 'content-type': 'application/x-www-form-urlencoded', 20 | }, 21 | }); 22 | 23 | const body = await res.text(); 24 | 25 | if (body.includes('fault')) { 26 | const fault = new Error('Undefined XML RPC error'); 27 | const code = body.match( 28 | /([^<]+)<\/int>[\s\S]+?([^<]+)<\/string>/i 29 | ); 30 | 31 | if (code) { 32 | if (code[2]) fault.message = code[2]; 33 | fault.code = code[1]; 34 | } 35 | 36 | return { 37 | status: 400, 38 | target: res.url, 39 | source: target, 40 | error: fault.message, 41 | }; 42 | } 43 | 44 | return { 45 | source, 46 | status: res.status, 47 | target: res.url, 48 | }; 49 | } 50 | 51 | module.exports = main; 52 | -------------------------------------------------------------------------------- /shared/lib/send/webmention.js: -------------------------------------------------------------------------------- 1 | const fetch = require('node-fetch'); 2 | 3 | async function main({ source, target, endpoint }) { 4 | const res = await fetch(endpoint, { 5 | method: 'post', 6 | body: `source=${encodeURIComponent(source)}&target=${encodeURIComponent( 7 | target 8 | )}`, 9 | headers: { 10 | 'content-type': 'application/x-www-form-urlencoded', 11 | }, 12 | }); 13 | 14 | let error = null; 15 | 16 | if (res.status >= 400) { 17 | error = await res.text(); 18 | } 19 | 20 | const reply = { 21 | status: res.status, 22 | error, 23 | source: target, // this is confusing, but works in the output 24 | target: res.url, 25 | }; 26 | 27 | return reply; 28 | } 29 | 30 | module.exports = main; 31 | -------------------------------------------------------------------------------- /shared/lib/uuid.js: -------------------------------------------------------------------------------- 1 | module.exports = require('uuid/v4'); 2 | -------------------------------------------------------------------------------- /shared/lib/webmention.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const parse = require('url').parse; 3 | const links = require('./links').links; 4 | const getEndpoints = require('./get-wm-endpoints'); 5 | const request = require('./request'); 6 | const dom = require('./html/dom'); 7 | const mf = require('./microformat/dom'); 8 | const smellsLikeRSS = require('./rss/is'); 9 | const rss = require('./rss/dom'); 10 | const send = require('./send'); 11 | 12 | class Webmention extends EventEmitter { 13 | constructor({ limit = 10, send = false } = {}) { 14 | super(); 15 | 16 | this.url = null; 17 | this.limit = limit; 18 | this.send = send; 19 | this.__sending = false; 20 | this.mentions = []; 21 | this.endpoints = null; 22 | this.counts = {}; 23 | 24 | this.on('progress-update', ({ type, value, data }) => { 25 | const v = (this.counts[type] = (this.counts[type] || 0) + value); 26 | this.emit('progress', { [type]: v, data }); 27 | }); 28 | 29 | if (send) this.sendWebMentions(); 30 | } 31 | 32 | async getLinksFromHTML({ html }) { 33 | const url = this.url; 34 | const res = dom(html); 35 | this.emit('progress', { entries: res.base.length }); 36 | 37 | return links({ ...res, url }); 38 | } 39 | 40 | async getLinksFromMicroformats({ html }) { 41 | const res = await mf(html, this); 42 | const url = res.url; 43 | this.emit('progress', { entries: res.base.length }); 44 | return links({ ...res, url }); 45 | } 46 | 47 | async getLinksFromFeed({ xml }) { 48 | const res = await rss(xml, this.limit); 49 | this.url = res.url; 50 | this.emit('progress', { entries: res.rss.items.length }); 51 | 52 | return links({ ...res, rss: true }); 53 | } 54 | 55 | async process() { 56 | this.emit('progress', { 57 | mentions: this.mentions.reduce( 58 | (acc, curr) => (acc += curr.links.length), 59 | 0 60 | ), 61 | }); 62 | 63 | const ignoreOwn = (permalink) => (curr) => { 64 | if (!this.url) return true; 65 | const host = parse(this.url).hostname; 66 | if (curr.includes(host) || curr.includes(host + '/')) { 67 | return false; 68 | } 69 | 70 | if (curr === permalink) { 71 | return false; 72 | } 73 | 74 | return true; 75 | }; 76 | 77 | const urls = await Promise.all( 78 | this.mentions.map(async ({ permalink, links }) => { 79 | const endpoints = await getEndpoints( 80 | links.filter(ignoreOwn(permalink)), 81 | (type, data) => this.emit(type, data), 82 | this.limit 83 | ); 84 | 85 | if (endpoints.length === 0) return false; 86 | this.emit( 87 | 'log', 88 | `Webmention endpoint found: ${JSON.stringify(endpoints)}` 89 | ); 90 | 91 | return endpoints.map(({ url: target, endpoint }) => { 92 | return { 93 | endpoint, 94 | source: permalink, 95 | target, 96 | }; 97 | }); 98 | }) 99 | ); 100 | 101 | return [].concat(...urls.filter(Boolean)); 102 | } 103 | 104 | getFromContent(content) { 105 | if (smellsLikeRSS(content)) { 106 | this.emit('log', 'Content is RSS'); 107 | return this.getLinksFromFeed({ xml: content }); 108 | } 109 | 110 | if (content.includes('h-entry')) { 111 | // smells like microformats 112 | this.emit('log', 'Content has microformats'); 113 | return this.getLinksFromMicroformats({ html: content }); 114 | } 115 | 116 | // else: html 117 | this.emit('log', 'Content is HTML'); 118 | return this.getLinksFromHTML({ html: content }); 119 | } 120 | 121 | fetch(url) { 122 | if (!url.startsWith('http')) { 123 | url = `http://${url}`; 124 | } 125 | this.emit('log', `Fetching ${url}`); 126 | request(url, 2000) // url has 2 seconds to respond 127 | .then(({ content, responseUrl }) => { 128 | this.url = responseUrl; 129 | this.emit('request', this.url); 130 | return this.load(content); 131 | }) 132 | .catch((e) => { 133 | if (e.code === 'ECONNRESET') { 134 | // this.emit('log', 'timeout - blocking'); 135 | this.emit('error', { 136 | message: `${url} did not respond and timed out`, 137 | }); 138 | } else { 139 | this.emit('error', e); 140 | } 141 | }); 142 | } 143 | 144 | load(content) { 145 | this.content = content; 146 | return this.getFromContent(content) 147 | .then((res) => { 148 | this.mentions = res; 149 | return this.process().then((res) => { 150 | this.endpoints = res; 151 | this.emit('endpoints', res); 152 | if (!this.__sending) { 153 | this.emit('end'); 154 | } 155 | }); 156 | }) 157 | .catch((e) => this.emit('error', e)); 158 | } 159 | 160 | // FIXME work out why I can't call this `send` 161 | async sendWebMentions() { 162 | this.__sending = true; 163 | if (this.endpoints === null) { 164 | this.emit('log', 'queuing send'); 165 | this.on('endpoints', () => { 166 | this.sendWebMentions(); 167 | }); 168 | return; 169 | } 170 | 171 | this.emit('log', 'start send'); 172 | return Promise.all( 173 | this.endpoints.map((res) => { 174 | this.emit( 175 | 'log', 176 | `Sending ${res.source} to ${res.endpoint.url} (${res.endpoint.type})` 177 | ); 178 | return send(res) 179 | .then((_) => this.emit('sent', { ..._, ...res })) 180 | .catch((e) => { 181 | this.emit('log', e.message); 182 | }); 183 | }) 184 | ).then(() => { 185 | this.emit('end'); 186 | }); 187 | } 188 | } 189 | 190 | module.exports = Webmention; 191 | -------------------------------------------------------------------------------- /shared/static.json: -------------------------------------------------------------------------------- 1 | /Users/remy/dev/webmention.app/public/static.json --------------------------------------------------------------------------------