├── .gitignore
├── src
├── http
│ ├── get-webmention-000id
│ │ ├── view.js
│ │ └── index.js
│ ├── get-index
│ │ └── index.js
│ ├── get-webmention
│ │ └── index.js
│ └── post-webmention
│ │ └── index.js
├── shared
│ ├── valid-url.js
│ ├── layout.js
│ └── status.js
└── events
│ ├── send
│ └── index.js
│ ├── verify
│ └── index.js
│ └── parse
│ └── index.js
├── package.json
├── app.arc
├── LICENSE
└── README.md
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | node_modules
3 |
4 | .arc-env
5 | .env
6 | preferences.arc
7 | sam.json
8 | sam.yaml
9 | .sam.yaml.swp
10 |
--------------------------------------------------------------------------------
/src/http/get-webmention-000id/view.js:
--------------------------------------------------------------------------------
1 | function view (locals) {
2 | return `
3 | ${JSON.stringify(locals, null, 2)}
4 | `
5 | }
6 |
7 | module.exports = view
8 |
--------------------------------------------------------------------------------
/src/shared/valid-url.js:
--------------------------------------------------------------------------------
1 | function isValidURL (string) {
2 | try {
3 | new URL(string) // eslint-disable-line
4 | } catch (_) {
5 | return false
6 | }
7 | return true
8 | }
9 |
10 | module.exports = isValidURL
11 |
--------------------------------------------------------------------------------
/src/http/get-index/index.js:
--------------------------------------------------------------------------------
1 | exports.handler = async function http (req) {
2 | return {
3 | statusCode: 200,
4 | headers: {
5 | 'content-type': 'text/plain; charset=utf-8'
6 | },
7 | body: 'Webmention receiver'
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/src/shared/layout.js:
--------------------------------------------------------------------------------
1 | function layout (content) {
2 | return `
3 |
4 |
5 | Wembley
6 |
7 |
8 | ${content}
9 |
10 |
11 | `
12 | }
13 |
14 | module.exports = layout
15 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "wembley",
3 | "version": "0.1.0",
4 | "description": "Webmention receiver",
5 | "scripts": {
6 | "start": "PORT=3335 npx sandbox"
7 | },
8 | "devDependencies": {
9 | "@architect/architect": "^11.1.0"
10 | },
11 | "dependencies": {
12 | "@architect/functions": "^3.13.11",
13 | "node-fetch": "^2.6.1"
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/app.arc:
--------------------------------------------------------------------------------
1 | @app
2 | wembley
3 |
4 | @aws
5 | region eu-west-2
6 |
7 | @http
8 | get /
9 | get /webmention
10 | get /webmention/:id
11 | post /webmention
12 |
13 | @events
14 | parse
15 | send
16 | verify
17 |
18 | @tables
19 | blocks
20 | domain *String
21 | statuses
22 | id *String
23 | webmentions
24 | id *String
25 |
26 | @indexes
27 | statuses
28 | webmention_id *String
29 | created_at **String
30 | webmentions
31 | target *String
32 | created_at **String
33 |
--------------------------------------------------------------------------------
/src/shared/status.js:
--------------------------------------------------------------------------------
1 | const arc = require('@architect/functions')
2 |
3 | async function create (webmentionId, message, type) {
4 | const data = await arc.tables()
5 | const id = Math.random().toString(36).substring(2)
6 | await data.statuses.put({
7 | id,
8 | webmention_id: webmentionId,
9 | created_at: new Date().toISOString(),
10 | type,
11 | message
12 | })
13 | }
14 |
15 | async function log (webmentionId, message) {
16 | console.log(`${webmentionId} ${message}`)
17 | await create(webmentionId, message, 'log')
18 | }
19 |
20 | async function error (webmentionId, message) {
21 | console.error(`${webmentionId} ${message}`)
22 | await create(webmentionId, message, 'error')
23 | }
24 |
25 | module.exports = { log, error }
26 |
--------------------------------------------------------------------------------
/src/http/get-webmention/index.js:
--------------------------------------------------------------------------------
1 | const arc = require('@architect/functions')
2 |
3 | async function findWebmentionsByTarget (target) {
4 | const data = await arc.tables()
5 | const result = await data.webmentions.query({
6 | IndexName: 'target-created_at-index',
7 | ScanIndexForward: false,
8 | KeyConditionExpression: 'target = :target',
9 | ExpressionAttributeValues: {
10 | ':target': target
11 | }
12 | })
13 | return result.Items
14 | }
15 |
16 | async function http (request) {
17 | const { target } = request.query
18 | if (!target) {
19 | return {
20 | status: 400,
21 | json: { message: 'target query parameter is missing' }
22 | }
23 | }
24 |
25 | const webmentions = await findWebmentionsByTarget(target)
26 | if (!webmentions) {
27 | return {
28 | status: 404,
29 | json: { message: 'No webmentions found' }
30 | }
31 | }
32 |
33 | return {
34 | json: { webmentions }
35 | }
36 | }
37 |
38 | exports.handler = arc.http.async(http)
39 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 Barry Frost
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/src/events/send/index.js:
--------------------------------------------------------------------------------
1 | const fetch = require('node-fetch')
2 | const arc = require('@architect/functions')
3 | const status = require('@architect/shared/status')
4 | const isValidURL = require('@architect/shared/valid-url')
5 |
6 | async function handler (event) {
7 | const { id } = event
8 |
9 | const data = await arc.tables()
10 | const webmention = await data.webmentions.get({ id })
11 | if (!webmention) {
12 | status.error(id, 'Webmention could not be found')
13 | }
14 |
15 | const url = process.env.WEBHOOK_URL
16 | if (!(url)) {
17 | status.error(id, 'WEBHOOK_URL has not been defined in ENV')
18 | return
19 | }
20 | if (!isValidURL(url)) {
21 | status.error(id, 'WEBHOOK_URL is not a valid URL')
22 | return
23 | }
24 |
25 | const response = await fetch(url, {
26 | method: 'post',
27 | headers: { 'Content-Type': 'application/json' },
28 | body: JSON.stringify(webmention.post)
29 | })
30 |
31 | console.log('webhook', JSON.stringify(webmention.post, null, 2))
32 |
33 | if (response.ok) {
34 | status.log('Webhook post was successful')
35 | } else {
36 | status.error('Webhook post failed')
37 | }
38 |
39 | return response.ok
40 | }
41 |
42 | exports.handler = arc.events.subscribe(handler)
43 |
--------------------------------------------------------------------------------
/src/http/get-webmention-000id/index.js:
--------------------------------------------------------------------------------
1 | const arc = require('@architect/functions')
2 | const layout = require('@architect/shared/layout')
3 | const view = require('./view')
4 |
5 | async function findStatusesByWebmention (webmentionId) {
6 | const data = await arc.tables()
7 | const result = await data.statuses.query({
8 | IndexName: 'webmention_id-created_at-index',
9 | ScanIndexForward: false,
10 | KeyConditionExpression: 'webmention_id = :webmentionId',
11 | ExpressionAttributeValues: {
12 | ':webmentionId': webmentionId
13 | }
14 | })
15 | return result.Items
16 | }
17 |
18 | async function http (request) {
19 | const data = await arc.tables()
20 |
21 | const { id } = request.params
22 | if (!id) {
23 | return {
24 | status: 400,
25 | json: { message: 'id parameter is missing' }
26 | }
27 | }
28 |
29 | const webmention = await data.webmentions.get({ id })
30 | if (!webmention) {
31 | return {
32 | status: 404,
33 | json: { message: 'Webmention not found' }
34 | }
35 | }
36 |
37 | const statuses = await findStatusesByWebmention(id)
38 |
39 | if (request.headers.accept.startsWith('application/json')) {
40 | return {
41 | json: { ...webmention, statuses }
42 | }
43 | }
44 |
45 | const content = view({ webmention, statuses })
46 | return {
47 | html: layout(content)
48 | }
49 | }
50 |
51 | exports.handler = arc.http.async(http)
52 |
--------------------------------------------------------------------------------
/src/events/verify/index.js:
--------------------------------------------------------------------------------
1 | const fetch = require('node-fetch')
2 | const arc = require('@architect/functions')
3 | const status = require('@architect/shared/status')
4 |
5 | function validContentTypes (contentType) {
6 | const types = [
7 | 'text/html',
8 | 'application/json',
9 | 'text/plain'
10 | ]
11 | for (const type of types) {
12 | if (contentType.startsWith(type)) {
13 | return true
14 | }
15 | }
16 | }
17 |
18 | async function handler (event) {
19 | const { id } = event
20 |
21 | const data = await arc.tables()
22 | const webmention = await data.webmentions.get({ id })
23 | if (!webmention) {
24 | status.error(id, 'Webmention could not be found')
25 | return
26 | }
27 |
28 | // fetch default timeout is 5s and 20 redirects
29 | const response = await fetch(webmention.source, {
30 | method: 'get',
31 | size: 1024 * 1024 // 1mb
32 | })
33 | if (!response.ok) {
34 | status.error(id, 'Source could not be fetched')
35 | return
36 | }
37 | const contentType = response.headers.get('content-type')
38 | if (!validContentTypes(contentType)) {
39 | status.error(id, 'Source Content-Type was invalid')
40 | return
41 | }
42 | const text = await response.text()
43 | if (!text.match(webmention.target)) {
44 | status.error(id, 'Source does not include a link to target')
45 | return
46 | }
47 |
48 | await arc.events.publish({
49 | name: 'parse',
50 | payload: { id }
51 | })
52 |
53 | await status.log(id, 'Source links to target')
54 | }
55 |
56 | exports.handler = arc.events.subscribe(handler)
57 |
--------------------------------------------------------------------------------
/src/http/post-webmention/index.js:
--------------------------------------------------------------------------------
1 | const arc = require('@architect/functions')
2 | const status = require('@architect/shared/status')
3 | const isValidURL = require('@architect/shared/valid-url')
4 |
5 | async function findBlock (source) {
6 | const data = await arc.tables()
7 | const domain = new URL(source).host
8 | const block = data.blocks.get({ domain })
9 | return block
10 | }
11 |
12 | function validate (source, target) {
13 | if (!source) return 'source parameter is missing'
14 | if (!isValidURL(source)) return 'source is not a valid URL'
15 | if (!target) return 'target parameter is missing'
16 | if (!isValidURL(target)) return 'target is not a valid URL'
17 | if (source === target) return 'source and target are the same'
18 | if (findBlock(source)) return 'source is blocked'
19 | }
20 |
21 | async function create (source, target) {
22 | const data = await arc.tables()
23 | // random id for webmention record
24 | const id = Math.random().toString(36).substring(2)
25 | await data.webmentions.put({
26 | id,
27 | source,
28 | target,
29 | created_at: new Date().toISOString()
30 | })
31 | return id
32 | }
33 |
34 | async function http (request) {
35 | const { source, target } = request.body
36 |
37 | const message = validate(source, target)
38 | if (message) {
39 | return {
40 | status: 400,
41 | json: { message }
42 | }
43 | }
44 |
45 | const id = await create(source, target)
46 |
47 | await status.log(id, 'Received webmention')
48 |
49 | await arc.events.publish({
50 | name: 'verify',
51 | payload: { id }
52 | })
53 |
54 | return {
55 | status: 201,
56 | location: `${process.env.ROOT_URL}webmention/${id}`
57 | }
58 | }
59 |
60 | exports.handler = arc.http.async(http)
61 |
--------------------------------------------------------------------------------
/src/events/parse/index.js:
--------------------------------------------------------------------------------
1 | const fetch = require('node-fetch')
2 | const arc = require('@architect/functions')
3 | const status = require('@architect/shared/status')
4 |
5 | function getXRayUrl (url) {
6 | const xRayBaseUrl = 'https://xray.p3k.io/parse?url='
7 | const safeUrl = encodeURIComponent(url)
8 | return xRayBaseUrl + safeUrl
9 | }
10 |
11 | async function fetchSource (webmention) {
12 | const url = getXRayUrl(webmention.source)
13 | const response = await fetch(url)
14 | if (!response.ok) return
15 | const jf2 = await response.json()
16 | if (!('data' in jf2)) return
17 | return jf2
18 | }
19 |
20 | async function handler (event) {
21 | const { id } = event
22 |
23 | const data = await arc.tables()
24 | const webmention = await data.webmentions.get({ id })
25 | if (!webmention) {
26 | status.error(id, 'Webmention could not be found')
27 | return
28 | }
29 |
30 | const jf2 = await fetchSource(webmention)
31 | if (!jf2) {
32 | status.error(id, 'Source could not be parsed')
33 | return
34 | }
35 | status.log(id, 'Source was parsed')
36 |
37 | // TODO: upload photo to cloudinary
38 |
39 | const post = {
40 | url: jf2.url,
41 | type: jf2.data.type,
42 | published: jf2.data.published,
43 | author: jf2.data.author
44 | }
45 |
46 | switch (jf2.data['post-type']) {
47 | case 'like':
48 | post['like-of'] = jf2.data['like-of']
49 | post['wm-property'] = 'like-of'
50 | break
51 | case 'repost':
52 | post['repost-of'] = jf2.data['repost-of']
53 | post['wm-property'] = 'repost-of'
54 | break
55 | case 'reply':
56 | post['in-reply-to'] = jf2.data['in-reply-to'][0]
57 | post['wm-property'] = 'in-reply-to'
58 | post.content = jf2.data.content.text
59 | if (jf2.data.name) { post.name = jf2.data.name }
60 | break
61 | }
62 |
63 | /*
64 | "post": {
65 | "type": "entry",
66 | "author": {
67 | "name": "Amy Guy",
68 | "photo": "http://webmention.io/avatar/rhiaro.co.uk/829d3f6e7083d7ee8bd7b20363da84d88ce5b4ce094f78fd1b27d8d3dc42560e.png",
69 | "url": "http://rhiaro.co.uk/about#me"
70 | },
71 | "url": "http://rhiaro.co.uk/2015/11/1446953889",
72 | "published": "2015-11-08T03:38:09+00:00",
73 | "name": "repost of http://aaronparecki.com/notes/2015/11/07/4/indiewebcamp",
74 | "repost-of": "http://aaronparecki.com/notes/2015/11/07/4/indiewebcamp",
75 | "wm-property": "repost-of"
76 | }
77 | */
78 |
79 | data.webmentions.put({ post, ...webmention })
80 |
81 | await arc.events.publish({
82 | name: 'send',
83 | payload: { id }
84 | })
85 | }
86 |
87 | exports.handler = arc.events.subscribe(handler)
88 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Wembley
2 |
3 | This is a [Webmention](https://webmention.net) receiver to be used as an
4 | endpoint to receive webmentions for your website.
5 |
6 | It's still very much a work-in-progress and not ready for use, but I wanted to
7 | share progress as the spec evolves in the [IndieWeb](https://indieweb.org)
8 | community.
9 |
10 | * Node.js with [Architect framework](https://arc.codes)
11 | * Deploy to AWS using Lambda, API Gateway, DynamoDB, SNS
12 |
13 | ## Outline
14 |
15 | ### Receiving webmentions
16 |
17 | * **POST /webmention** with `source` and `target` form parameters
18 | * Return 400 if `source` and `target` are not valid URLs
19 | * Return 400 if `source` is the same as target
20 | * Return 400 if `target` is not a known domain (`DOMAINS` env var)
21 | * Return 400 if `source`'s domain is found in blocked domain table
22 | * Generate a unique id for the webmention
23 | * Store `id`, `source`, `target` and a timestamp in webmentions table
24 | * Log status "Received webmention"
25 | * Publish verify event with `id` payload
26 | * Return 201 with `Location` header `/webmention/:id`
27 | * Handle **verify** event with `id` payload
28 | * Get the webmention record from the database
29 | * Fetch the source, limiting to 1Mb, 20 redirects and a 5-second timeout
30 | * Log error if `Content-Type` is not HTML, JSON or text
31 | * Log error if `source` content does not include `target`
32 | * Publish parse event with `id`
33 | * Handle **parse** event with `id` payload
34 | * Get the webmention record from the database
35 | * Send source URL to [XRay](https://xray.p3k.io) to parse into JF2
36 | * Log status "Source was parsed" or error if unsuccessful
37 | * Upload author photo to Cloudinary
38 | * Create a JF2 object (format TBD - see
39 | https://github.com/indieweb/webmention-ecosystem/issues/2)
40 | * Update webmention record with JF2 post in table
41 | * Publish send event with `id` payload
42 | * Handle **send** event with `id` payload
43 | * Get the webmention record from the database
44 | * Send POST to config webhook URL with webmention record in JSON body
45 |
46 | ### Listing webmentions
47 |
48 | * **GET /webmention** with `target` query parameter
49 | * Query for webmentions matching `target`
50 | * Return 404 if no webmentions are found
51 | * Return 200 with list of webmentions as HTML (default) or JSON depending on
52 | `Accept` header
53 |
54 | ### Webmention statuses
55 |
56 | * **GET /webmention/:id**
57 | * Query for statuses matching `id`
58 | * Return 404 if no statuses are found
59 | * Return 200 with list of statuses as HTML (default) or JSON depending on
60 | `Accept` header
61 |
62 | ## env
63 |
64 | * `ROOT_URL` e.g. https://wembley.barryfrost.com/
65 | * `DOMAINS` e.g. barryfrost.com
66 | * `WEBHOOK_URL` e.g. https://api.barryfrost.com/webmention
67 |
--------------------------------------------------------------------------------