├── static ├── robots.txt └── favicon.png ├── .dockerignore ├── src ├── frontend │ ├── assets │ │ ├── img │ │ │ ├── 1.png │ │ │ ├── 2.png │ │ │ ├── 3.png │ │ │ ├── utlogo.png │ │ │ └── hnwlogo.svg │ │ └── font │ │ │ ├── social.eot │ │ │ ├── social.ttf │ │ │ ├── social.woff │ │ │ ├── social.woff2 │ │ │ ├── README.md │ │ │ └── social.svg │ ├── server-entry.js │ ├── browser-entry.js │ ├── .eslintrc.js │ ├── components │ │ ├── CountryFlag.vue │ │ ├── CopyButton.vue │ │ ├── VisitTableCell.vue │ │ ├── Timestamp.vue │ │ └── VisitTable.vue │ ├── lib │ │ ├── country-flag.js │ │ ├── copy-to-clipboard.js │ │ ├── timestamp.js │ │ └── fetch-updates.js │ ├── app.js │ ├── views │ │ ├── Trap.vue │ │ ├── Index.vue │ │ └── Monitor.vue │ └── App.vue └── backend │ ├── lib │ ├── gcloud.js │ ├── taskqueue.js │ ├── resolve.js │ ├── analytics.js │ └── store.js │ ├── worker.js │ └── app.js ├── index.yaml ├── Dockerfile ├── .gitignore ├── .eslintrc.js ├── circle.yml ├── LICENSE ├── package.json ├── webpack.config.js ├── app.yaml └── README.md /static/robots.txt: -------------------------------------------------------------------------------- 1 | User-Agent: * 2 | Disallow: /new$ 3 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | build/ 3 | Dockerfile 4 | .yarnclean 5 | -------------------------------------------------------------------------------- /static/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/static/favicon.png -------------------------------------------------------------------------------- /src/frontend/assets/img/1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/src/frontend/assets/img/1.png -------------------------------------------------------------------------------- /src/frontend/assets/img/2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/src/frontend/assets/img/2.png -------------------------------------------------------------------------------- /src/frontend/assets/img/3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/src/frontend/assets/img/3.png -------------------------------------------------------------------------------- /index.yaml: -------------------------------------------------------------------------------- 1 | indexes: 2 | - kind: Visit 3 | properties: 4 | - name: target 5 | - name: seqId 6 | direction: desc 7 | -------------------------------------------------------------------------------- /src/frontend/assets/font/social.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/src/frontend/assets/font/social.eot -------------------------------------------------------------------------------- /src/frontend/assets/font/social.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/src/frontend/assets/font/social.ttf -------------------------------------------------------------------------------- /src/frontend/assets/img/utlogo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/src/frontend/assets/img/utlogo.png -------------------------------------------------------------------------------- /src/frontend/assets/font/social.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/src/frontend/assets/font/social.woff -------------------------------------------------------------------------------- /src/frontend/assets/font/social.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HowNetWorks/uriteller/HEAD/src/frontend/assets/font/social.woff2 -------------------------------------------------------------------------------- /src/frontend/server-entry.js: -------------------------------------------------------------------------------- 1 | import app from "./app"; 2 | 3 | export default context => { 4 | const { view, data } = context.state; 5 | return app(view, data); 6 | }; 7 | -------------------------------------------------------------------------------- /src/frontend/browser-entry.js: -------------------------------------------------------------------------------- 1 | import app from "./app"; 2 | 3 | const element = document.getElementById("initial-state"); 4 | 5 | let state = {}; 6 | if (element) { 7 | state = JSON.parse(element.innerHTML); 8 | } 9 | 10 | const { view, data } = state; 11 | app(view, data).then(vm => vm.$mount("#app")); 12 | -------------------------------------------------------------------------------- /src/frontend/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "plugins": [ 3 | "vue" 4 | ], 5 | "env": { 6 | "browser": true, 7 | "node": false 8 | }, 9 | "parserOptions": { 10 | "sourceType": "module", 11 | "ecmaFeatures": { 12 | "experimentalObjectRestSpread": true 13 | } 14 | } 15 | }; 16 | -------------------------------------------------------------------------------- /src/frontend/assets/font/README.md: -------------------------------------------------------------------------------- 1 | `social.*` font files are a minimal selection of Font Awesome fonts, collected 2 | with [Fontello](http://fontello.com/). 3 | 4 | ## Font Awesome 5 | 6 | Copyright (C) 2016 by Dave Gandy 7 | 8 | Author: Dave Gandy 9 | License: SIL OFL 1.1 (http://scripts.sil.org/OFL) 10 | Homepage: http://fortawesome.github.com/Font-Awesome/ 11 | -------------------------------------------------------------------------------- /src/frontend/components/CountryFlag.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 20 | -------------------------------------------------------------------------------- /src/frontend/components/CopyButton.vue: -------------------------------------------------------------------------------- 1 | 6 | 7 | 20 | -------------------------------------------------------------------------------- /src/frontend/components/VisitTableCell.vue: -------------------------------------------------------------------------------- 1 | 9 | 10 | 22 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:9.9 AS builder 2 | COPY . /app 3 | RUN useradd -m app \ 4 | && chown -R app:app /app 5 | USER app 6 | WORKDIR /app 7 | RUN yarn --no-progress \ 8 | && yarn build \ 9 | && yarn --production --no-progress \ 10 | && yarn autoclean \ 11 | && tar cfz build.tar.gz build node_modules 12 | 13 | FROM node:9.9-slim 14 | COPY . /app 15 | COPY --from=builder /app/build.tar.gz /app 16 | RUN useradd -m app \ 17 | && chown -R app:app /app 18 | USER app 19 | WORKDIR /app 20 | ENV NODE_ENV production 21 | CMD tar xfz build.tar.gz && yarn start 22 | -------------------------------------------------------------------------------- /src/frontend/lib/country-flag.js: -------------------------------------------------------------------------------- 1 | export default function countryFlag(code) { 2 | // Turn two-letter ISO 3166-1 alpha-2 country codes to country flag emojis. 3 | // See https://en.wikipedia.org/wiki/Regional_Indicator_Symbol 4 | 5 | if (!code || code.length !== 2) { 6 | return ""; 7 | } 8 | 9 | const up = code.toUpperCase(); 10 | const a = up.charCodeAt(0) - 0x41; 11 | const b = up.charCodeAt(1) - 0x41; 12 | if (a < 0 || a >= 26 || b < 0 || b >= 26) { 13 | return ""; 14 | } 15 | 16 | const base = 0x1f1e6; 17 | return String.fromCodePoint(base + a, base + b); 18 | } 19 | -------------------------------------------------------------------------------- /src/backend/lib/gcloud.js: -------------------------------------------------------------------------------- 1 | let errorHandler = { 2 | report(err) { 3 | // eslint-disable-next-line no-console 4 | console.error(err); 5 | }, 6 | express(req, res, next) { 7 | next(); 8 | } 9 | }; 10 | 11 | if (process.env.NODE_ENV === "production") { 12 | require("@google-cloud/trace-agent").start(); 13 | require("@google-cloud/debug-agent").start(); 14 | errorHandler = require("@google-cloud/error-reporting")(); 15 | } 16 | 17 | module.exports = { 18 | pubsub() { 19 | return require("@google-cloud/pubsub")(); 20 | }, 21 | datastore() { 22 | return require("@google-cloud/datastore")(); 23 | }, 24 | errors: errorHandler 25 | }; 26 | -------------------------------------------------------------------------------- /src/frontend/app.js: -------------------------------------------------------------------------------- 1 | import Vue from "vue"; 2 | import App from "./App.vue"; 3 | import Index from "./views/Index.vue"; 4 | import Trap from "./views/Trap.vue"; 5 | import Monitor from "./views/Monitor.vue"; 6 | 7 | Vue.component("app", App); 8 | Vue.config.productionTip = false; 9 | 10 | const views = new Map(); 11 | views.set("index", Index); 12 | views.set("trap", Trap); 13 | views.set("monitor", Monitor); 14 | 15 | export default function(name, data) { 16 | const view = views.get(name); 17 | if (!view) { 18 | throw new Error("unknown view"); 19 | } 20 | 21 | const vm = new Vue(view); 22 | if (vm.setData) { 23 | return Promise.resolve(vm.setData(data)).then(() => vm); 24 | } 25 | return Promise.resolve(vm); 26 | } 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # node-waf configuration 24 | .lock-wscript 25 | 26 | # Compiled binary addons (http://nodejs.org/api/addons.html) 27 | build/Release 28 | 29 | # Dependency directories 30 | node_modules 31 | jspm_packages 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | 39 | # Build directory 40 | /build 41 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "es6": true, 4 | "node": true 5 | }, 6 | "extends": [ 7 | "eslint:recommended" 8 | ], 9 | "parserOptions": { 10 | "ecmaVersion": 2017 11 | }, 12 | "rules": { 13 | // Stylistic Issues 14 | "camelcase": "error", 15 | "indent": ["error", 2, { "MemberExpression": 1 }], 16 | "linebreak-style": ["error", "unix"], 17 | "quotes": ["error", "double"], 18 | "semi": ["error", "always"], 19 | "no-trailing-spaces": "error", 20 | "eol-last": "error", 21 | "no-multiple-empty-lines": ["error", { "max": 1 }], 22 | 23 | // Possible Errors 24 | "no-unsafe-negation": "error", 25 | 26 | // Best Practices 27 | "no-multi-spaces": "error", 28 | 29 | // ECMAScript 6 30 | "no-var": "error", 31 | "prefer-const": "error" 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: docker:17.06.0-ce 6 | working_directory: ~/workdir 7 | steps: 8 | - run: apk add --no-cache git openssh-client 9 | - checkout 10 | - setup_remote_docker: 11 | version: 17.06.0-ce 12 | - run: docker build -t ${DOCKER_HUB_IMAGE} . 13 | - deploy: 14 | name: Deploy the image to Docker Hub 15 | command: | 16 | if echo -n "${CIRCLE_TAG}" | grep -Eq ^v[0-9]+\.[0-9]+\.[0-9]+$; then 17 | docker tag ${DOCKER_HUB_IMAGE} ${DOCKER_HUB_IMAGE}:${CIRCLE_TAG} 18 | docker login -u ${DOCKER_HUB_USER} -p ${DOCKER_HUB_PASS} 19 | docker push ${DOCKER_HUB_IMAGE}:${CIRCLE_TAG} 20 | docker push ${DOCKER_HUB_IMAGE} 21 | fi 22 | 23 | deployment: 24 | workaround_for_circleci_2_0: 25 | tag: /.*/ 26 | -------------------------------------------------------------------------------- /src/frontend/components/Timestamp.vue: -------------------------------------------------------------------------------- 1 | 7 | 8 | 42 | -------------------------------------------------------------------------------- /src/frontend/lib/copy-to-clipboard.js: -------------------------------------------------------------------------------- 1 | export default function copyToClipboard(text) { 2 | const div = document.createElement("div"); 3 | div.appendChild(document.createTextNode(text)); 4 | 5 | const selection = window.getSelection(); 6 | const oldRanges = []; 7 | for (let i = 0, len = selection.rangeCount; i < len; i++) { 8 | oldRanges.push(selection.getRangeAt(i)); 9 | } 10 | 11 | document.body.appendChild(div); 12 | try { 13 | const range = document.createRange(); 14 | range.selectNode(div); 15 | 16 | const selection = window.getSelection(); 17 | selection.removeAllRanges(); 18 | selection.addRange(range); 19 | 20 | document.execCommand("copy"); 21 | } finally { 22 | document.body.removeChild(div); 23 | 24 | selection.removeAllRanges(); 25 | oldRanges.forEach(oldRange => { 26 | selection.addRange(oldRange); 27 | }); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/frontend/views/Trap.vue: -------------------------------------------------------------------------------- 1 | 27 | 28 | 43 | -------------------------------------------------------------------------------- /src/frontend/views/Index.vue: -------------------------------------------------------------------------------- 1 | 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 HowNetWorks 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/frontend/components/VisitTable.vue: -------------------------------------------------------------------------------- 1 | 31 | 32 | 47 | -------------------------------------------------------------------------------- /src/backend/lib/taskqueue.js: -------------------------------------------------------------------------------- 1 | const gcloud = require("./gcloud"); 2 | 3 | const pubsub = gcloud.pubsub(); 4 | 5 | function _topic(name) { 6 | return pubsub.createTopic(name) 7 | .then( 8 | data => data[0], 9 | err => { 10 | if (err.code === 409) { 11 | return pubsub.topic(name); 12 | } 13 | throw err; 14 | } 15 | ); 16 | } 17 | 18 | function _subscribe(topic, subName) { 19 | const config = { 20 | maxInProgress: 512, 21 | ackDeadlineSeconds: 120 22 | }; 23 | return topic.subscribe(subName, config).then(data => data[0]); 24 | } 25 | 26 | exports.subscribe = function(topicName, subName, handler) { 27 | return _topic(topicName) 28 | .then(topic => _subscribe(topic, subName)) 29 | .then( 30 | sub => { 31 | function onMsg(msg) { 32 | handler(null, msg); 33 | } 34 | function onErr(err) { 35 | sub.removeListener("message", onMsg); 36 | sub.removeListener("error", onErr); 37 | handler(err, null); 38 | } 39 | sub.on("message", onMsg); 40 | sub.on("error", onErr); 41 | }, 42 | err => { 43 | handler(err, null); 44 | } 45 | ); 46 | }; 47 | 48 | exports.publish = function(topicName, data) { 49 | return _topic(topicName).then(topic => topic.publish(data)); 50 | }; 51 | -------------------------------------------------------------------------------- /src/frontend/assets/img/hnwlogo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Group 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "uriteller", 3 | "version": "0.2.11", 4 | "private": true, 5 | "license": "MIT", 6 | "dependencies": { 7 | "@google-cloud/datastore": "^1.1.0", 8 | "@google-cloud/debug-agent": "^2.1.3", 9 | "@google-cloud/error-reporting": "^0.2.1", 10 | "@google-cloud/pubsub": "^0.13.1", 11 | "@google-cloud/trace-agent": "^2.1.3", 12 | "compression": "^1.7.2", 13 | "express": "^4.16.3", 14 | "express-static-gzip": "^0.3.2", 15 | "helmet": "^3.12.0", 16 | "ip-address": "^5.8.9", 17 | "request": "^2.85.0", 18 | "serialize-javascript": "^1.4.0", 19 | "vue-server-renderer": "^2.5.16" 20 | }, 21 | "scripts": { 22 | "start": "node src/backend/${SCRIPT:-app.js}", 23 | "lint": "eslint --ignore-path=.gitignore --ext=.js,.vue .", 24 | "dev": "webpack --mode development --watch", 25 | "build": "webpack --mode production" 26 | }, 27 | "devDependencies": { 28 | "autoprefixer": "^8.2.0", 29 | "babel-core": "^6.26.0", 30 | "babel-loader": "^7.1.4", 31 | "babel-plugin-transform-object-rest-spread": "^6.26.0", 32 | "babel-preset-env": "^1.6.1", 33 | "bootstrap": "4.0.0-alpha.6", 34 | "compression-webpack-plugin": "^1.1.11", 35 | "css-loader": "^0.28.11", 36 | "eslint": "^4.5.0", 37 | "eslint-plugin-vue": "^2.1.0", 38 | "file-loader": "^1.1.11", 39 | "mini-css-extract-plugin": "^0.2.0", 40 | "node-sass": "^4.7.2", 41 | "sass-loader": "^6.0.7", 42 | "vue": "^2.5.16", 43 | "vue-loader": "^14.2.1", 44 | "vue-template-compiler": "^2.5.16", 45 | "webpack": "^4.2.0", 46 | "webpack-cli": "^2.0.12", 47 | "webpack-merge": "^4.1.2" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/backend/worker.js: -------------------------------------------------------------------------------- 1 | const { errors } = require("./lib/gcloud"); 2 | const net = require("net"); 3 | const { URL } = require("url"); 4 | const request = require("request"); 5 | const store = require("./lib/store"); 6 | const taskQueue = require("./lib/taskqueue"); 7 | const resolve = require("./lib/resolve"); 8 | 9 | const WHEREABOUTS_URL = process.env.WHEREABOUTS_URL || "http://localhost:8080"; 10 | 11 | function whereabouts(ip) { 12 | return new Promise((resolve, reject) => { 13 | if (!net.isIP(ip)) { 14 | return resolve(undefined); 15 | } 16 | 17 | const url = new URL("api/whereabouts/" + ip, WHEREABOUTS_URL).toString(); 18 | return request({ url: String(url), json: true }, (err, res) => { 19 | if (err) { 20 | return reject(err); 21 | } 22 | 23 | const json = res.body; 24 | if (json && json.country && json.country.code) { 25 | return resolve(json.country.code); 26 | } 27 | resolve(undefined); 28 | }); 29 | }); 30 | } 31 | 32 | taskQueue.subscribe("trap-topic", "trap-subscription", (err, msg) => { 33 | if (err) { 34 | return errors.report(err); 35 | } 36 | 37 | const data = msg.data; 38 | if (!data.info || !data.info.ip) { 39 | return msg.ack(); 40 | } 41 | 42 | const ip = data.info.ip; 43 | Promise.all([resolve.ipToASNs(ip), resolve.reverse(ip), whereabouts(ip)]) 44 | .then(([asns, reverse, country]) => { 45 | data.info.reverse = reverse; 46 | data.info.asns = asns; 47 | data.info.country = country; 48 | return store.visit(data.target, data.timestamp, data.info); 49 | }) 50 | .then(() => msg.ack()) 51 | .catch(errors.report); 52 | }).catch(errors.report); 53 | -------------------------------------------------------------------------------- /src/frontend/assets/font/social.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Copyright (C) 2016 by original authors @ fontello.com 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/frontend/lib/timestamp.js: -------------------------------------------------------------------------------- 1 | const PAD_TWO = "00"; 2 | const PAD_FOUR = "0000"; 3 | 4 | function pad(number, padding) { 5 | const result = String(number); 6 | return padding.substr(result.length) + result; 7 | } 8 | 9 | export function formatAbsolute(timestamp) { 10 | const date = new Date(timestamp); 11 | const year = pad(date.getFullYear(), PAD_FOUR); 12 | const month = pad(date.getMonth() + 1, PAD_TWO); 13 | const day = pad(date.getDate(), PAD_TWO); 14 | const hour = pad(date.getHours(), PAD_TWO); 15 | const minute = pad(date.getMinutes(), PAD_TWO); 16 | const second = pad(date.getSeconds(), PAD_TWO); 17 | return year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second; 18 | } 19 | 20 | function formatSpan(delta) { 21 | const seconds = Math.round(delta / 1000); 22 | if (seconds < 45) { 23 | return "a few seconds"; 24 | } 25 | if (seconds < 90) { 26 | return "a minute"; 27 | } 28 | 29 | const minutes = Math.round(seconds / 60); 30 | if (minutes < 45) { 31 | return minutes + " minutes"; 32 | } 33 | if (minutes < 90) { 34 | return "an hour"; 35 | } 36 | 37 | const hours = Math.round(minutes / 60); 38 | if (hours < 22) { 39 | return hours + " hours"; 40 | } 41 | if (hours < 36) { 42 | return "a day"; 43 | } 44 | 45 | const days = Math.round(hours / 24); 46 | if (days < 26) { 47 | return days + " days"; 48 | } 49 | if (days < 45) { 50 | return "a month"; 51 | } 52 | if (days < 320) { 53 | return Math.round(days / 30) + " months"; 54 | } 55 | if (days < 548) { 56 | return "a year"; 57 | } 58 | return Math.round(days / 365) + " years"; 59 | } 60 | 61 | export function formatRelative(timestamp, origin=Date.now()) { 62 | // Return a relatively formatted time. The implementation aims to follow the 63 | // formatting described in the documentation of Moment.js 2.15.1. See 64 | // https://momentjs.com/docs/#/displaying/fromnow/ and 65 | // https://momentjs.com/docs/#/displaying/tonow/ 66 | 67 | if (timestamp > origin) { 68 | return "in " + formatSpan(timestamp - origin); 69 | } 70 | return formatSpan(origin - timestamp) + " ago"; 71 | } 72 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const merge = require("webpack-merge"); 3 | const MiniCssExtractPlugin = require("mini-css-extract-plugin"); 4 | const VueSsrClientPlugin = require("vue-server-renderer/client-plugin"); 5 | const VueSsrServerPlugin = require("vue-server-renderer/server-plugin"); 6 | const CompressionPlugin = require("compression-webpack-plugin"); 7 | const pkg = require("./package.json"); 8 | 9 | // A helper to create paths relative to this config file 10 | function p(...paths) { 11 | return path.join(__dirname, ...paths); 12 | } 13 | 14 | const babel = { 15 | loader: "babel-loader", 16 | options: { 17 | presets: [["env", { modules: false, targets: { uglify: true } }]], 18 | plugins: ["transform-object-rest-spread"] 19 | } 20 | }; 21 | 22 | const css = [MiniCssExtractPlugin.loader, "css-loader"]; 23 | 24 | const base = { 25 | output: { 26 | path: p("build"), 27 | filename: "assets/[chunkhash].js", 28 | publicPath: "/" 29 | }, 30 | module: { 31 | rules: [ 32 | { 33 | test: /\.js$/, 34 | include: [p("src")], 35 | use: [babel] 36 | }, 37 | { 38 | test: /\.vue$/, 39 | include: [p("src")], 40 | loader: "vue-loader", 41 | options: { 42 | loaders: { 43 | js: babel, 44 | css: css, 45 | scss: [...css, "sass-loader"] 46 | }, 47 | postcss: [require("autoprefixer")()] 48 | } 49 | }, 50 | { 51 | test: /\.(eot|woff2|woff|ttf|svg|png)$/, 52 | loader: "file-loader", 53 | options: { 54 | outputPath: "assets/" 55 | } 56 | } 57 | ] 58 | }, 59 | plugins: [ 60 | new MiniCssExtractPlugin({ 61 | filename: "assets/[chunkhash].css" 62 | }), 63 | new CompressionPlugin() 64 | ] 65 | }; 66 | 67 | module.exports = [ 68 | merge(base, { 69 | entry: p("src/frontend/browser-entry.js"), 70 | plugins: [new VueSsrClientPlugin()], 71 | devtool: "source-map" 72 | }), 73 | merge(base, { 74 | target: "node", 75 | entry: p("src/frontend/server-entry.js"), 76 | output: { 77 | libraryTarget: "commonjs2" 78 | }, 79 | externals: Object.keys(pkg.dependencies), 80 | plugins: [new VueSsrServerPlugin()] 81 | }) 82 | ]; 83 | -------------------------------------------------------------------------------- /src/frontend/lib/fetch-updates.js: -------------------------------------------------------------------------------- 1 | function timeout(delay) { 2 | return new Promise(resolve => { 3 | setTimeout(resolve, delay); 4 | }); 5 | } 6 | 7 | class ConnectionFailed extends Error {} 8 | class RequestFailed extends Error {} 9 | class ParsingFailed extends Error {} 10 | class Timeout extends Error {} 11 | 12 | function fetchJSON(url, timeout=15000) { 13 | return new Promise((resolve, reject) => { 14 | const xhr = new XMLHttpRequest(); 15 | xhr.open("GET", url); 16 | xhr.timeout = timeout; 17 | 18 | xhr.addEventListener("load", () => { 19 | if (xhr.status !== 200) { 20 | return reject(new RequestFailed(`status ${xhr.status} (${xhr.statusText})`)); 21 | } 22 | 23 | const response = xhr.response; 24 | let json; 25 | try { 26 | json = JSON.parse(response); 27 | } catch (err) { 28 | return reject(new ParsingFailed("parsing the request response failed")); 29 | } 30 | resolve(json); 31 | }, false); 32 | 33 | xhr.addEventListener("error", () => { 34 | reject(new ConnectionFailed("connection failed")); 35 | }, false); 36 | 37 | xhr.addEventListener("timeout", () => { 38 | reject(new Timeout("timeout")); 39 | }, false); 40 | 41 | xhr.send(); 42 | }); 43 | } 44 | 45 | function fetchLoop(baseUrl, cursor, interval, minInterval, maxInterval, currentErr, callback) { 46 | const a = document.createElement("a"); 47 | a.href = baseUrl; 48 | a.search = typeof cursor === "undefined" ? "" : `cursor=${cursor}`; 49 | 50 | const updateUrl = a.href; 51 | timeout(interval) 52 | .then(() => fetchJSON(updateUrl)) 53 | .then( 54 | json => { 55 | const visits = json.visits; 56 | if (currentErr || visits.length > 0) { 57 | callback(null, visits); 58 | } 59 | fetchLoop(baseUrl, json.cursor, minInterval, minInterval, maxInterval, null, callback); 60 | }, 61 | err => { 62 | if (!currentErr || String(currentErr) !== String(err)) { 63 | callback(err, null); 64 | } 65 | const newInterval = Math.min(interval * 2, maxInterval); 66 | fetchLoop(baseUrl, cursor, newInterval, minInterval, maxInterval, err, callback); 67 | } 68 | ); 69 | } 70 | 71 | export default function(baseUrl, cursor, minInterval, maxInterval, callback) { 72 | fetchLoop(baseUrl, cursor, minInterval, minInterval, maxInterval, null, callback); 73 | } 74 | -------------------------------------------------------------------------------- /app.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Ingress 3 | metadata: 4 | name: app-ingress 5 | annotations: 6 | kubernetes.io/ingress.global-static-ip-name: "uriteller-ingress" 7 | kubernetes.io/tls-acme: "true" 8 | kubernetes.io/ingress.class: "gce" 9 | spec: 10 | tls: 11 | - secretName: tls-secret 12 | hosts: 13 | - uriteller.io 14 | rules: 15 | - host: uriteller.io 16 | http: 17 | paths: 18 | - backend: 19 | serviceName: app 20 | servicePort: http 21 | 22 | --- 23 | 24 | apiVersion: v1 25 | kind: Service 26 | metadata: 27 | name: app 28 | spec: 29 | type: NodePort 30 | selector: 31 | name: app 32 | ports: 33 | - name: http 34 | port: 80 35 | targetPort: http 36 | 37 | --- 38 | 39 | apiVersion: extensions/v1beta1 40 | kind: Deployment 41 | metadata: 42 | name: app 43 | spec: 44 | replicas: 3 45 | template: 46 | metadata: 47 | labels: 48 | name: app 49 | spec: 50 | containers: 51 | - name: app 52 | image: hownetworks/uriteller:v0.2.11 53 | ports: 54 | - containerPort: 8080 55 | name: http 56 | envFrom: 57 | - configMapRef: 58 | name: uriteller-config 59 | readinessProbe: 60 | httpGet: 61 | path: /healthz 62 | port: http 63 | initialDelaySeconds: 5 64 | timeoutSeconds: 1 65 | 66 | --- 67 | 68 | apiVersion: extensions/v1beta1 69 | kind: Deployment 70 | metadata: 71 | name: worker 72 | spec: 73 | replicas: 3 74 | template: 75 | metadata: 76 | labels: 77 | name: worker 78 | spec: 79 | containers: 80 | - name: app 81 | image: hownetworks/uriteller:v0.2.11 82 | env: 83 | - name: SCRIPT 84 | value: worker.js 85 | - name: WHEREABOUTS_URL 86 | value: http://localhost:8080 87 | - name: whereabouts 88 | image: hownetworks/whereabouts:v0.3.3 89 | 90 | --- 91 | 92 | apiVersion: extensions/v1beta1 93 | kind: Deployment 94 | metadata: 95 | name: kube-lego 96 | spec: 97 | replicas: 1 98 | template: 99 | metadata: 100 | labels: 101 | name: kube-lego 102 | spec: 103 | containers: 104 | - name: kube-lego 105 | image: jetstack/kube-lego:0.1.5 106 | ports: 107 | - containerPort: 8080 108 | resources: 109 | requests: 110 | memory: 64Mi 111 | cpu: 25m 112 | env: 113 | - name: LEGO_EMAIL 114 | value: devel@hownetworks.io 115 | - name: LEGO_URL 116 | value: https://acme-v01.api.letsencrypt.org/directory 117 | - name: LEGO_POD_IP 118 | valueFrom: 119 | fieldRef: 120 | fieldPath: status.podIP 121 | readinessProbe: 122 | httpGet: 123 | path: /healthz 124 | port: 8080 125 | initialDelaySeconds: 5 126 | -------------------------------------------------------------------------------- /src/backend/lib/resolve.js: -------------------------------------------------------------------------------- 1 | const net = require("net"); 2 | const dns = require("dns"); 3 | const ipAddress = require("ip-address"); 4 | 5 | const OK_DNS_ERRORS = new Set([dns.NODATA, dns.NOTFOUND]); 6 | 7 | exports.reverse = function(ip) { 8 | if (!net.isIP(ip)) { 9 | return Promise.resolve([]); 10 | } 11 | 12 | return new Promise((resolve, reject) => { 13 | dns.reverse(ip, (err, hostnames) => { 14 | if (err) { 15 | return OK_DNS_ERRORS.has(err.code) ? resolve([]) : reject(err); 16 | } 17 | return resolve(hostnames); 18 | }); 19 | }); 20 | }; 21 | 22 | exports.ipToASNs = function(ip) { 23 | if (net.isIP(ip)) { 24 | const ipv4 = new ipAddress.Address4(ip); 25 | if (ipv4.isValid()) { 26 | return lookup(ipv4.toArray().reverse().join(".") + ".origin.asn.cymru.com"); 27 | } 28 | 29 | const ipv6 = new ipAddress.Address6(ip); 30 | if (ipv6.isValid()) { 31 | return lookup(ipv6.reverseForm({ omitSuffix: true }) + ".origin6.asn.cymru.com"); 32 | } 33 | } 34 | return Promise.resolve([]); 35 | }; 36 | 37 | function lookup(name) { 38 | return lookupASNs(name).then(asns => { 39 | const promises = asns.map(info => { 40 | return lookupASNames(info.asn).then(names => { 41 | return Object.assign({}, info, { names: names }); 42 | }); 43 | }); 44 | return Promise.all(promises); 45 | }); 46 | } 47 | 48 | function lookupASNs(name) { 49 | return new Promise((resolve, reject) => { 50 | dns.resolveTxt(name, (err, records) => { 51 | if (err) { 52 | return OK_DNS_ERRORS.has(err.code) ? resolve([]) : reject(err); 53 | } 54 | 55 | const asns = new Map(); 56 | records.forEach(record => { 57 | const fields = record.join("").split("|").map(x => x.trim()); 58 | 59 | const asn = fields[0]; 60 | if (!asn) { 61 | return; 62 | } 63 | const info = { asn: asn }; 64 | 65 | const country = fields[2]; 66 | if (country) { 67 | info.country = country; 68 | } 69 | asns.set(asn, info); 70 | }); 71 | return resolve(Array.from(asns.values())); 72 | }); 73 | }); 74 | } 75 | 76 | function cleanASName(name) { 77 | // Remove a trailing two-character country code, if it exists. Also trim 78 | // away surrounding whitespace. 79 | return name.replace(/,\s*[A-Z]{2}\s*$/, "").trim(); 80 | } 81 | 82 | function lookupASNames(asn) { 83 | return new Promise((resolve, reject) => { 84 | dns.resolveTxt("AS" + asn + ".asn.cymru.com", (err, records) => { 85 | if (err) { 86 | return OK_DNS_ERRORS.has(err.code) ? resolve([]) : reject(err); 87 | } 88 | 89 | const names = new Set(); 90 | records.forEach(record => { 91 | const pieces = record.join("").split("|"); 92 | if (pieces.length < 5) { 93 | return; 94 | } 95 | const name = cleanASName(pieces[4]); 96 | if (name) { 97 | names.add(name); 98 | } 99 | }); 100 | return resolve(Array.from(names)); 101 | }); 102 | }); 103 | } 104 | -------------------------------------------------------------------------------- /src/frontend/views/Monitor.vue: -------------------------------------------------------------------------------- 1 | 45 | 46 | 108 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | URI:teller 2 | 3 | [![CircleCI](https://circleci.com/gh/HowNetWorks/uriteller.svg?style=shield)](https://circleci.com/gh/HowNetWorks/uriteller) 4 | 5 | **NOTE: The hosted site, https:/​/uriteller.io, is no longer active. As a replacement check out [Canarytokens](https://canarytokens.org/) and [RequestBin](https://github.com/Runscope/requestbin).** 6 | 7 | URI:teller is a service for monitoring how chat apps, social network sites and such fetch their link previews. See the companion [blog post](https://medium.com/hownetworks/uri-teller-a-call-for-the-curious-20694617db1c) talking about the original motivation and further uses. 8 | 9 | The code in this repository is built for the Google Cloud Platform. See [Prerequisites](#prerequisites) for more info about that. 10 | 11 | ## Technical Name-dropping 12 | 13 | URI:teller uses Google's [Cloud Datastore](https://cloud.google.com/datastore/) as the database and [Cloud Pub/Sub](https://cloud.google.com/pubsub/) for passing work between services. Stackdriver [Debugger](https://cloud.google.com/debugger/), [Trace](https://cloud.google.com/trace/) and [Error Reporting](https://cloud.google.com/error-reporting/) work if their respective APIs are enabled. The frontend service sends analytics to Google Analytics whent the `GA_TRACKING_ID` environment variable is set. 14 | 15 | The code is written in ES2015 plus some extensions, such as modules and `.vue` component files. [Babel](https://babeljs.io/) then compiles the source to JavaScript that [Node.js](https://nodejs.org/en/) and browsers can handle. For styling: [SASS](http://sass-lang.com/). 16 | 17 | [Vue](https://vuejs.org/) allows reusing the same view code for both server-side and in-browser rendering. 18 | 19 | [Express](http://expressjs.com/) (with its [Helmet](https://helmetjs.github.io/) on) powers the server side code. 20 | 21 | On the browser [Bootstrap 4](https://v4-alpha.getbootstrap.com/) makes things look nice. [Webpack 2](https://webpack.github.io/) crumples the code, styles and other assets into an easily distributable bundle. 22 | 23 | [CircleCI](https://circleci.com/) runs the build process on every repository push. CircleCI also deploys the site whenever the `production` branch gets an update. 24 | 25 | ## Prerequisites 26 | 27 | ### Google Cloud Platform 28 | 29 | This project is made to be hosted in the Google Cloud Platform (namely in the 30 | [Google App Engine Node.js flexible 31 | environment](https://cloud.google.com/appengine/docs/flexible/nodejs/)), so you 32 | need an account: https://cloud.google.com/ 33 | 34 | Install and initialize Google Cloud SDK: https://cloud.google.com/sdk/docs/ 35 | 36 | Create a new project from the [Google Cloud Platform 37 | Console](https://console.cloud.google.com/) or use an already existing one. Set 38 | your project's id `PROJECT_ID` as default with: 39 | 40 | ```sh 41 | $ gcloud config set project 42 | ``` 43 | 44 | ### App Dependencies 45 | 46 | Install Node.js dependencies. The following command line examples use [`yarn`](https://yarnpkg.com/) but `npm` works just as well. 47 | 48 | ```sh 49 | $ yarn 50 | ``` 51 | 52 | 53 | ## Development 54 | 55 | ### Client 56 | 57 | Watch and rebuild client side assets on changes: 58 | 59 | ```sh 60 | $ yarn dev 61 | ``` 62 | 63 | ### Server 64 | 65 | If you want to run the server components you can use the [Application Default 66 | Credentials](https://cloud.google.com/docs/authentication#getting_credentials_for_server-centric_flow) - 67 | note that you need to get these only once for your environment: 68 | 69 | ```sh 70 | $ gcloud auth application-default login 71 | ``` 72 | 73 | Run `app.js` in port 8080: 74 | 75 | ```sh 76 | $ GCLOUD_PROJECT= APP_BASE_URL=http://localhost:8080/ yarn start 77 | ``` 78 | 79 | Run `worker.js` instead: 80 | 81 | ```sh 82 | $ GCLOUD_PROJECT= SCRIPT=worker.js yarn start 83 | ``` 84 | -------------------------------------------------------------------------------- /src/backend/lib/analytics.js: -------------------------------------------------------------------------------- 1 | const net = require("net"); 2 | const https = require("https"); 3 | const crypto = require("crypto"); 4 | const querystring = require("querystring"); 5 | const ipAddress = require("ip-address"); 6 | const request = require("request"); 7 | 8 | function clean(obj) { 9 | const result = {}; 10 | 11 | Object.keys(obj).forEach(key => { 12 | if (obj[key]) { 13 | result[key] = obj[key]; 14 | } 15 | }); 16 | 17 | return result; 18 | } 19 | 20 | function anonymizeIP(ip) { 21 | if (net.isIP(ip)) { 22 | const ipv4 = new ipAddress.Address4(ip + "/24"); 23 | if (ipv4.isValid()) { 24 | return ipv4.startAddress().correctForm(); 25 | } 26 | 27 | const ipv6 = new ipAddress.Address6(ip + "/48"); 28 | if (ipv6.isValid()) { 29 | return ipv6.startAddress().correctForm(); 30 | } 31 | } 32 | return void 0; 33 | } 34 | 35 | module.exports = class { 36 | constructor(trackingId) { 37 | this._trackingId = trackingId; 38 | this._timeout = 1000; 39 | 40 | // Maximum number of measurements to fit into one batch. Google's 41 | // Measurement protocol limits this to 20. 42 | this._maxBatchLength = 20; 43 | // Maximum number of batches to keep in the buffer. 44 | this._maxBatchBufferLength = 50; 45 | this._batchBuffer = []; 46 | this._currentBatch = null; 47 | 48 | this._agent = new https.Agent({ 49 | keepAlive: true, 50 | maxSockets: 1 51 | }); 52 | } 53 | 54 | _request() { 55 | if (this._currentBatch !== null) { 56 | return; 57 | } 58 | if (this._batchBuffer.length === 0) { 59 | return; 60 | } 61 | 62 | this._currentBatch = this._batchBuffer.shift(); 63 | 64 | const body = this._currentBatch.map(item => { 65 | return querystring.stringify(item.data); 66 | }).join("\n") + "\n"; 67 | 68 | request.post("https://www.google-analytics.com/batch", { 69 | body: body, 70 | agent: this._agent, 71 | timeout: this._timeout 72 | }, (err, res) => { 73 | const batch = this._currentBatch; 74 | this._currentBatch = null; 75 | 76 | if (err) { 77 | batch.forEach(item => { 78 | item.reject(err); 79 | }); 80 | } else if (res.statusCode !== 200) { 81 | batch.forEach(item => { 82 | item.reject(new Error(`Analytics request status ${res.statusCode}`)); 83 | }); 84 | } else { 85 | batch.forEach(item => { 86 | item.resolve(); 87 | }); 88 | } 89 | 90 | this._request(); 91 | }); 92 | } 93 | 94 | _push(item) { 95 | if (this._batchBuffer.length === 0) { 96 | this._batchBuffer.push([]); 97 | } 98 | 99 | const lastBatch = this._batchBuffer[this._batchBuffer.length - 1]; 100 | if (lastBatch.length < this._maxBatchLength) { 101 | lastBatch.push(item); 102 | } else { 103 | this._batchBuffer.push([item]); 104 | } 105 | 106 | if (this._batchBuffer.length > this._maxBatchBufferLength) { 107 | const dropped = this._batchBuffer.splice(0, this._batchBuffer.length - this._maxBatchBufferLength); 108 | dropped.forEach(item => { 109 | item.reject(new Error("Dropped")); 110 | }); 111 | } 112 | 113 | this._request(); 114 | } 115 | 116 | _send(req, overrides = {}) { 117 | if (!this._trackingId) { 118 | return Promise.resolve(); 119 | } 120 | 121 | const ip = anonymizeIP(req.ip); 122 | const ua = req.get("user-agent"); 123 | const cid = crypto.createHash("sha256").update(ip + ua).digest("hex"); 124 | 125 | const data = clean( 126 | Object.assign( 127 | { 128 | v: "1", 129 | tid: this._trackingId, 130 | cid: cid, 131 | 132 | dh: req.hostname, 133 | dp: req.path, 134 | dr: req.get("referrer"), // Express considers "referrer" and "referer" interchangeable 135 | uip: ip, 136 | aip: "1", 137 | ua: ua 138 | }, 139 | overrides 140 | ) 141 | ); 142 | 143 | return new Promise((resolve, reject) => { 144 | this._push({ 145 | data: data, 146 | resolve: resolve, 147 | reject: reject 148 | }); 149 | }); 150 | } 151 | 152 | pageView(req) { 153 | return this._send(req, { 154 | t: "pageview" 155 | }); 156 | } 157 | 158 | event(req, category, action, label, value) { 159 | return this._send(req, { 160 | t: "event", 161 | ec: category, 162 | ea: action, 163 | el: label, 164 | ev: value 165 | }); 166 | } 167 | }; 168 | -------------------------------------------------------------------------------- /src/backend/lib/store.js: -------------------------------------------------------------------------------- 1 | const crypto = require("crypto"); 2 | const gcloud = require("./gcloud"); 3 | 4 | const datastore = gcloud.datastore(); 5 | 6 | // Return a string id with at least `n` bits of randomness. The returned string 7 | // will contain only characters [a-zA-Z0-9_-]. 8 | function genPageId(n = 128) { 9 | const bytes = crypto.randomBytes(Math.ceil(n / 8.0)); 10 | return bytes.toString("base64").replace(/\//g, "_").replace(/\+/g, "-").replace(/=/g, ""); 11 | } 12 | 13 | const pageIdCache = new Map(); 14 | 15 | exports.get = function(id) { 16 | if (pageIdCache.has(id)) { 17 | return Promise.resolve(pageIdCache.get(id)); 18 | } 19 | 20 | return datastore.get(datastore.key(["Item", id])) 21 | .then(data => { 22 | const result = data[0] || null; 23 | if (result !== null) { 24 | pageIdCache.set(id, result); 25 | } 26 | return result; 27 | }); 28 | }; 29 | 30 | exports.create = function() { 31 | function tryCreate(resolve, reject) { 32 | const trapId = genPageId(); 33 | const viewId = genPageId(); 34 | 35 | const trapKey = datastore.key(["Item", trapId]); 36 | const viewKey = datastore.key(["Item", viewId]); 37 | 38 | const transaction = datastore.transaction(); 39 | transaction.run(err => { 40 | if (err) { 41 | return reject(err); 42 | } 43 | 44 | const trapData = { 45 | isView: false, 46 | other: viewId 47 | }; 48 | const viewData = { 49 | isView: true, 50 | other: trapId 51 | }; 52 | 53 | transaction.insert([ 54 | { 55 | key: trapKey, 56 | data: trapData 57 | }, 58 | { 59 | key: viewKey, 60 | data: viewData 61 | } 62 | ]); 63 | transaction.commit(err => { 64 | if (err && err.code === 409) { 65 | return tryCreate(resolve, reject); 66 | } 67 | if (err) { 68 | return reject(err); 69 | } 70 | pageIdCache.set(trapId, trapData); 71 | pageIdCache.set(viewId, viewData); 72 | return resolve(viewId); 73 | }); 74 | }); 75 | } 76 | 77 | return new Promise((resolve, reject) => { 78 | tryCreate(resolve, reject); 79 | }); 80 | }; 81 | 82 | const seqIdCache = new Map(); 83 | 84 | function getSeqId(target) { 85 | if (seqIdCache.has(target)) { 86 | return Promise.resolve(seqIdCache.get(target) + 1); 87 | } 88 | 89 | return queryMaxSeqId(target).then(seqId => { 90 | if (seqId === null) { 91 | return 0; 92 | } 93 | seenSeqId(seqId); 94 | return seqId + 1; 95 | }); 96 | } 97 | 98 | function seenSeqId(target, seqId) { 99 | const cached = seqIdCache.get(target); 100 | if (cached === undefined || cached < seqId) { 101 | seqIdCache.set(target, seqId); 102 | } 103 | } 104 | 105 | function queryMaxSeqId(target) { 106 | const query = datastore.createQuery("Visit") 107 | .filter("target", target) 108 | .filter("seqId", ">=", 0) 109 | .filter("seqId", "<=", Number.MAX_SAFE_INTEGER) 110 | .select("seqId") 111 | .order("seqId", { 112 | descending: true 113 | }) 114 | .limit(1); 115 | 116 | return query.run() 117 | .then(data => data[0]) 118 | .then(entities => { 119 | if (entities.length === 0) { 120 | return null; 121 | } 122 | return entities[0].seqId; 123 | }); 124 | } 125 | 126 | function seqIdKey(target, seqId) { 127 | return datastore.key(["Visit", seqId + "/" + target]); 128 | } 129 | 130 | const visits = new Map(); 131 | const VISIT_CHUNK_COUNT = 10; 132 | 133 | exports.visit = function(target, timestamp, info) { 134 | function insert(seqId) { 135 | const queue = visits.get(target); 136 | if (!queue) { 137 | return; 138 | } 139 | 140 | const chunk = queue.slice(0, VISIT_CHUNK_COUNT); 141 | datastore.insert({ 142 | key: seqIdKey(target, seqId), 143 | data: { 144 | target: target, 145 | seqId: seqId, 146 | visits: chunk.map(visit => visit.data) 147 | } 148 | }, err => { 149 | if (err && err.code === 409) { 150 | seenSeqId(target, seqId); 151 | return insert(seqId + 1); 152 | } 153 | 154 | if (err) { 155 | chunk.forEach(visit => visit.reject(err)); 156 | } else { 157 | seenSeqId(target, seqId); 158 | chunk.forEach(visit => visit.resolve()); 159 | } 160 | 161 | queue.splice(0, chunk.length); 162 | if (queue.length === 0) { 163 | visits.delete(target); 164 | } else { 165 | getSeqId(target).then(insert); 166 | } 167 | }); 168 | } 169 | 170 | return new Promise((resolve, reject) => { 171 | const queue = visits.get(target) || []; 172 | visits.set(target, queue); 173 | queue.push({ 174 | resolve: resolve, 175 | reject: reject, 176 | data: { 177 | timestamp: timestamp, 178 | info: info 179 | } 180 | }); 181 | 182 | if (queue.length === 1) { 183 | getSeqId(target).then(insert); 184 | } 185 | }); 186 | }; 187 | 188 | function getSeqIds(target, seqIds) { 189 | if (seqIds.length === 0) { 190 | return Promise.resolve([]); 191 | } 192 | const keys = seqIds.map(seqId => seqIdKey(target, seqId)); 193 | return datastore.get(keys).then(data => data[0]); 194 | } 195 | 196 | exports.list = function(target, cursor = 0) { 197 | const query = datastore.createQuery("Visit") 198 | .filter("target", target) 199 | .filter("seqId", ">=", cursor) 200 | .filter("seqId", "<=", Number.MAX_SAFE_INTEGER) 201 | .order("seqId", { descending: true }); 202 | 203 | return query.run() 204 | .then(data => data[0]) 205 | .then(entities => { 206 | const nextCursor = entities.reduce((previous, entity) => { 207 | const seqId = entity.seqId + 1; 208 | return seqId > previous ? seqId : previous; 209 | }, cursor); 210 | 211 | const available = new Set(entities.map(entity => entity.seqId)); 212 | const missing = []; 213 | for (let i = cursor; i < nextCursor; i++) { 214 | if (!available.has(i)) { 215 | missing.push(i); 216 | } 217 | } 218 | 219 | return getSeqIds(target, missing).then(newEntities => { 220 | return { 221 | cursor: nextCursor, 222 | entities: entities.concat(newEntities) 223 | }; 224 | }); 225 | }) 226 | .then(({ cursor, entities }) => { 227 | const visits = []; 228 | entities.forEach(entity => { 229 | entity.visits.forEach(visit => { 230 | visits.push(visit); 231 | }); 232 | }); 233 | 234 | return { 235 | cursor: cursor, 236 | visits: visits 237 | }; 238 | }); 239 | }; 240 | -------------------------------------------------------------------------------- /src/backend/app.js: -------------------------------------------------------------------------------- 1 | const { errors } = require("./lib/gcloud"); 2 | 3 | const url = require("url"); 4 | const path = require("path"); 5 | const helmet = require("helmet"); 6 | const express = require("express"); 7 | const compression = require("compression"); 8 | const expressStaticGzip = require("express-static-gzip"); 9 | const { createBundleRenderer } = require("vue-server-renderer"); 10 | const serialize = require("serialize-javascript"); 11 | 12 | const taskQueue = require("./lib/taskqueue"); 13 | const store = require("./lib/store"); 14 | const Analytics = require("./lib/analytics"); 15 | const serverBundle = require("../../build/vue-ssr-server-bundle.json"); 16 | const clientManifest = require("../../build/vue-ssr-client-manifest.json"); 17 | 18 | const renderer = createBundleRenderer(serverBundle, { 19 | runInNewContext: "once", 20 | clientManifest, 21 | inject: false, // Inject the assets manually, incl. the 54 | `; 55 | } 56 | }); 57 | stream.once("data", data => { 58 | res.set("Content-Type", "text/html"); 59 | res.write(data); 60 | 61 | stream.on("data", data => { 62 | res.write(data); 63 | }); 64 | }); 65 | stream.on("end", () => { 66 | res.end(); 67 | resolve(); 68 | }); 69 | stream.on("error", err => { 70 | reject(err); 71 | }); 72 | }); 73 | } 74 | 75 | function fullUrl(req, path) { 76 | const baseUrl = process.env.APP_BASE_URL; 77 | return url.resolve(baseUrl, url.resolve(req.baseUrl, path)); 78 | } 79 | 80 | const PAGE_ID_REGEX = /^\/([a-zA-Z0-9_-]{22})([./].*)?$/; 81 | 82 | const analytics = new Analytics(process.env.GA_TRACKING_ID); 83 | 84 | const app = express(); 85 | app.set("json spaces", 2); 86 | app.set("trust proxy", true); 87 | app.use( 88 | helmet({ 89 | contentSecurityPolicy: { 90 | directives: { 91 | defaultSrc: ["'self'"] 92 | } 93 | }, 94 | referrerPolicy: { 95 | policy: "no-referrer" 96 | } 97 | }) 98 | ); 99 | app.use(compression()); 100 | 101 | app.get("/healthz", (req, res) => { 102 | res.sendStatus(200); 103 | }); 104 | 105 | app.get(PAGE_ID_REGEX, (req, res, next) => { 106 | const id = req.params[0]; 107 | store.get(id) 108 | .then(item => { 109 | if (!item || item.isView) { 110 | return; 111 | } 112 | 113 | analytics.event(req, "trap", "view").catch(errors.report); 114 | 115 | const suffix = req.url.substring(1 + id.length) || undefined; 116 | return taskQueue.publish("trap-topic", { 117 | target: id, 118 | timestamp: Date.now(), 119 | info: { 120 | ip: req.ip, 121 | referrer: req.get("referrer"), // Express considers "referrer" and "referer" interchangeable 122 | userAgent: req.get("user-agent"), 123 | protocol: req.secure ? "https" : "http", 124 | suffix: suffix || undefined 125 | } 126 | }); 127 | }) 128 | .then( 129 | () => next(), 130 | err => next(err) 131 | ); 132 | }); 133 | 134 | app.use((req, res, next) => { 135 | if (req.secure) { 136 | next(); 137 | return; 138 | } 139 | 140 | const redirect = fullUrl(req, req.path); 141 | const protocol = url.parse(redirect).protocol; 142 | if (protocol === "https:") { 143 | res.redirect(redirect); 144 | } else if (process.env.NODE_ENV !== "production" && protocol === "http:") { 145 | next(); 146 | } else { 147 | next(new Error("can't decide how to redirect an insecure request")); 148 | } 149 | }); 150 | 151 | app.use("/", express.static(path.join(__dirname, "../../static"))); 152 | app.use("/assets", expressStaticGzip(path.join(__dirname, "../../build/assets"), { maxAge: "365d" })); 153 | 154 | app.get("/", (req, res, next) => { 155 | analytics.pageView(req).catch(errors.report); 156 | render(res, { view: "index" }).catch(next); 157 | }); 158 | 159 | app.get("/new", (req, res, next) => { 160 | analytics.pageView(req).catch(errors.report); 161 | 162 | store.create() 163 | .then(view => { 164 | res.redirect(fullUrl(req, view)); 165 | }) 166 | .catch(next); 167 | }); 168 | 169 | function getData(req, target, cursor) { 170 | return store.list(target, cursor).then(({ cursor, visits }) => { 171 | return { 172 | trapUrl: fullUrl(req, target), 173 | cursor: cursor, 174 | visits: visits.map(entity => { 175 | return Object.assign({}, entity.info, { 176 | protocol: entity.info.protocol || "https", 177 | timestamp: entity.timestamp 178 | }); 179 | }) 180 | }; 181 | }); 182 | } 183 | 184 | app.get(PAGE_ID_REGEX, (req, res, next) => { 185 | const id = req.params[0]; 186 | const rest = req.params[1]; 187 | 188 | store.get(id) 189 | .then(item => { 190 | if (item && !item.isView) { 191 | res.status(404); 192 | render(res, { 193 | view: "trap", 194 | data: { baseUrl: fullUrl(req, "/") } 195 | }); 196 | } 197 | 198 | if (item && item.isView && !rest) { 199 | analytics.pageView(req).catch(errors.report); 200 | return getData(req, item.other).then(data => { 201 | render(res, { 202 | view: "monitor", 203 | data: Object.assign( 204 | { 205 | updateUrl: fullUrl(req, id + ".json") 206 | }, 207 | data 208 | ) 209 | }); 210 | }); 211 | } 212 | 213 | if (item && item.isView && rest === ".json") { 214 | let cursor = req.query.cursor; 215 | if (Array.isArray(cursor)) { 216 | return res.sendStatus(400); 217 | } 218 | if (cursor !== undefined) { 219 | cursor = Number(cursor); 220 | } 221 | return getData(req, item.other, cursor).then(data => res.json(data)); 222 | } 223 | 224 | next(); 225 | }) 226 | .catch(next); 227 | }); 228 | 229 | app.use(errors.express); 230 | 231 | const server = app.listen(process.env.PORT || 8080, () => { 232 | const addr = server.address(); 233 | 234 | // eslint-disable-next-line no-console 235 | console.log("Listening on port %s...", addr.port); 236 | }); 237 | -------------------------------------------------------------------------------- /src/frontend/App.vue: -------------------------------------------------------------------------------- 1 | 31 | 32 | 37 | 38 | 421 | --------------------------------------------------------------------------------