├── .babelrc ├── .dockerignore ├── .editorconfig ├── .env.example ├── .eslintrc.json ├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── config ├── default.json └── production.json ├── docker-compose.override.yml ├── docker-compose.yml ├── index.js ├── package.json ├── public └── favicon.ico ├── scripts └── docker_push.sh ├── src ├── app.hooks.js ├── app.js ├── channels.js ├── hooks │ └── logger.js ├── index.js ├── middleware │ └── index.js ├── mongoose.js └── services │ ├── embeds │ ├── embeds.class.js │ ├── embeds.hooks.js │ ├── embeds.model.js │ ├── embeds.service.js │ └── providers │ │ ├── index.js │ │ ├── provider.default.js │ │ ├── provider.flickr.js │ │ └── provider.youtube.js │ ├── images │ └── images.service.js │ └── index.js ├── test ├── app.test.js └── services │ └── embeds.test.js └── yarn.lock /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "test": { 4 | "plugins": [ "istanbul" ] 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | npm-debug.log 3 | Dockerfile 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | EMBED_API_HOST=0.0.0.0 2 | EMBED_API_URL=http://0.0.0.0:3050 3 | EMBED_API_TOKEN=MYSUPERSECRETSTRING 4 | EMBED_API_MONGO_USER=MONGOUSER 5 | EMBED_API_MONGO_PASS=MONGOPASS -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "es6": true, 4 | "node": true, 5 | "mocha": true 6 | }, 7 | "parserOptions": { 8 | "ecmaVersion": 2017 9 | }, 10 | "extends": "eslint:recommended", 11 | "rules": { 12 | "indent": [ 13 | "error", 14 | 2 15 | ], 16 | "linebreak-style": [ 17 | "error", 18 | "unix" 19 | ], 20 | "quotes": [ 21 | "error", 22 | "single" 23 | ], 24 | "semi": [ 25 | "error", 26 | "always" 27 | ] 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | .nyc_output 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # Compiled binary addons (http://nodejs.org/api/addons.html) 21 | build/Release 22 | 23 | # Dependency directory 24 | # Commenting this out is preferred by some people, see 25 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 26 | node_modules 27 | 28 | # Users Environment Variables 29 | .lock-wscript 30 | 31 | # IDEs and editors (shamelessly copied from @angular/cli's .gitignore) 32 | /.idea 33 | .project 34 | .classpath 35 | .c9/ 36 | *.launch 37 | .settings/ 38 | *.sublime-workspace 39 | /.vs 40 | .vscode 41 | 42 | # IDE - VSCode 43 | .vscode/* 44 | !.vscode/settings.json 45 | !.vscode/tasks.json 46 | !.vscode/launch.json 47 | !.vscode/extensions.json 48 | 49 | ### Linux ### 50 | *~ 51 | 52 | # temporary files which can be created if a process still has a handle open of a deleted file 53 | .fuse_hidden* 54 | 55 | # KDE directory preferences 56 | .directory 57 | 58 | # Linux trash folder which might appear on any partition or disk 59 | .Trash-* 60 | 61 | # .nfs files are created when an open file is removed but is still being accessed 62 | .nfs* 63 | 64 | ### OSX ### 65 | *.DS_Store 66 | .AppleDouble 67 | .LSOverride 68 | 69 | # Icon must end with two \r 70 | Icon 71 | 72 | 73 | # Thumbnails 74 | ._* 75 | 76 | # Files that might appear in the root of a volume 77 | .DocumentRevisions-V100 78 | .fseventsd 79 | .Spotlight-V100 80 | .TemporaryItems 81 | .Trashes 82 | .VolumeIcon.icns 83 | .com.apple.timemachine.donotpresent 84 | 85 | # Directories potentially created on remote AFP share 86 | .AppleDB 87 | .AppleDesktop 88 | Network Trash Folder 89 | Temporary Items 90 | .apdisk 91 | 92 | ### Windows ### 93 | # Windows thumbnail cache files 94 | Thumbs.db 95 | ehthumbs.db 96 | ehthumbs_vista.db 97 | 98 | # Folder config file 99 | Desktop.ini 100 | 101 | # Recycle Bin used on file shares 102 | $RECYCLE.BIN/ 103 | 104 | # Windows Installer files 105 | *.cab 106 | *.msi 107 | *.msm 108 | *.msp 109 | 110 | # Windows shortcuts 111 | *.lnk 112 | 113 | # Others 114 | lib/ 115 | data/ 116 | 117 | # ignore internal .github files 118 | /.github 119 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "10" 4 | cache: 5 | yarn: true 6 | directories: 7 | - node_modules 8 | services: 9 | # we need docker for building the image and mongo for testing 10 | - docker 11 | 12 | env: 13 | - DOCKER_COMPOSE_VERSION=1.23.2 14 | 15 | before_install: 16 | - sudo rm /usr/local/bin/docker-compose 17 | - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose 18 | - chmod +x docker-compose 19 | - sudo mv docker-compose /usr/local/bin 20 | 21 | install: 22 | - docker build -t humanconnection/embed-api:latest . 23 | - docker-compose -f docker-compose.yml up -d # some of the tests need mongodb 24 | 25 | script: 26 | - docker-compose exec embed-api yarn run lint 27 | - docker-compose exec embed-api yarn run mocha 28 | 29 | after_success: 30 | - wget https://raw.githubusercontent.com/DiscordHooks/travis-ci-discord-webhook/master/send.sh 31 | - chmod +x send.sh 32 | - ./send.sh success $WEBHOOK_URL 33 | 34 | after_failure: 35 | - wget https://raw.githubusercontent.com/DiscordHooks/travis-ci-discord-webhook/master/send.sh 36 | - chmod +x send.sh 37 | - ./send.sh failure $WEBHOOK_URL 38 | 39 | deploy: 40 | - provider: script 41 | script: scripts/docker_push.sh 42 | on: 43 | branch: master 44 | tags: true 45 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:10-alpine 2 | LABEL Description="API Service for fetching URL Information like images, icons, descriptions etc. through OpenGraph, oEmbed and other standards. " Vendor="Human-Connection gGmbH" Version="1.0" Maintainer="Human-Connection gGmbH (developer@human-connection.org)" 3 | 4 | # expose the app port 5 | EXPOSE 3050 6 | 7 | # set environment variables 8 | # ENV NPM_CONFIG_PRODUCTION=false 9 | # ENV EMBED_API_HOST=0.0.0.0 10 | ENV NODE_ENV=production 11 | ENV EMBED_API_PORT=3050 12 | 13 | # create working directory 14 | RUN mkdir -p /var/www/ 15 | WORKDIR /var/www/ 16 | 17 | # install app dependencies 18 | COPY package.json /var/www/ 19 | COPY yarn.lock /var/www/ 20 | RUN yarn install --frozen-lockfile --non-interactive --production=false --ignore-engines 21 | 22 | # copy the code to the docker image 23 | COPY . /var/www/ 24 | 25 | # start the application in a autohealing cluster 26 | #CMD pm2 start server/index.js -n api -i 0 --attach 27 | # as we have issues with pm2 currently in conjunction with nuxt, we use the standard approach here 28 | CMD node src/ 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Grzegorz Leoniec 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Embed API 2 | 3 |

4 | Build Status 5 |

6 | 7 | > URL Embed API with Caching 8 | 9 | ## About 10 | 11 | This API uses [Metaphor](https://www.npmjs.com/package/metaphor) for fetching URL data from multiple sources. 12 | 13 | ## How to use this image 14 | 15 | Make sure you have installed [docker](https://www.docker.com/community-edition). 16 | 17 | ### Running locally 18 | 19 | To test this service on you local machine, simply run: 20 | 21 | ```bash 22 | docker run --rm --name embed-api humanconnection/embed-api:latest 23 | ``` 24 | 25 | You can use docker-compose, too: 26 | ```sh 27 | docker-compose up 28 | # wait until the services are up 29 | curl --header "authentication: embedapitoken" "http://localhost:3050/embeds?url=http://www.human-connection.org/" 30 | ``` 31 | 32 | ### Running in Production 33 | 34 | Make sure you provide an unique API Key as well as a mongoDB user and password on your production server: 35 | 36 | ```bash 37 | docker run --rm --name embed-api -e "EMBED_API_TOKEN=MYSUPERSECRETSTRING" -e "EMBED_API_MONGO_USER=MONGOUSER" -e "EMBED_API_MONGO_PASS=MONGOPASS" humanconnection/embed-api:latest 38 | ``` 39 | 40 | ## Development 41 | 42 | 1. Make sure you have [NodeJS](https://nodejs.org/), [yarn](https://yarnpkg.com) and [mongoDB](https://www.mongodb.com/download-center#community) installed. 43 | 2. Install your dependencies 44 | 45 | ``` 46 | cd path/to/embed-api; yarn install 47 | ``` 48 | 49 | 3. Start your app 50 | 51 | ``` 52 | yarn dev 53 | ``` 54 | 55 | 4. Get URL Information 56 | 57 | ``` 58 | http://localhost:3050/embeds?url=http://www.human-connection.org/ 59 | ``` 60 | 61 | 5. Do something creative with the following output 62 | (More information here: [Metaphor](https://www.npmjs.com/package/metaphor)) 63 | 64 | ``` 65 | { 66 | site_name: 'YouTube', 67 | url: 'https://www.youtube.com/watch?v=cWDdd5KKhts', 68 | title: 'Cheese Shop Sketch - Monty Python\'s Flying Circus', 69 | image: { url: 'https://i.ytimg.com/vi/cWDdd5KKhts/maxresdefault.jpg' }, 70 | description: 'Subscribe to the Official Monty Python Channel here - http://smarturl.it/SubscribeToPython Cleese plays an erudite customer attempting to purchase some chees...', 71 | type: 'video', 72 | video: [ 73 | { 74 | url: 'https://www.youtube.com/embed/cWDdd5KKhts', 75 | type: 'text/html', 76 | width: '480', 77 | height: '360' 78 | }, 79 | { 80 | url: 'https://www.youtube.com/v/cWDdd5KKhts?version=3&autohide=1', 81 | type: 'application/x-shockwave-flash', 82 | width: '480', 83 | height: '360', 84 | tag: ['Monty Python', 'Python (Monty) Pictures Limited', 'Comedy', 'flying circus', 'monty pythons flying circus', 'john cleese', 'micael palin', 'eric idle', 'terry jones', 'graham chapman', 'terry gilliam', 'funny', 'comedy', 'animation', '60s animation', 'humor', 'humour', 'sketch show', 'british comedy', 'cheese shop', 'monty python cheese', 'cheese shop sketch', 'cleese cheese', 'cheese'] 85 | } 86 | ], 87 | thumbnail: { 88 | url: 'https://i.ytimg.com/vi/cWDdd5KKhts/hqdefault.jpg', 89 | width: 480, 90 | height: 360 91 | }, 92 | embed: { 93 | type: 'video', 94 | height: 344, 95 | width: 459, 96 | html: '' 97 | }, 98 | app: { 99 | iphone: { 100 | name: 'YouTube', 101 | id: '544007664', 102 | url: 'vnd.youtube://www.youtube.com/watch?v=cWDdd5KKhts&feature=applinks' 103 | }, 104 | ipad: { 105 | name: 'YouTube', 106 | id: '544007664', 107 | url: 'vnd.youtube://www.youtube.com/watch?v=cWDdd5KKhts&feature=applinks' 108 | }, 109 | googleplay: { 110 | name: 'YouTube', 111 | id: 'com.google.android.youtube', 112 | url: 'https://www.youtube.com/watch?v=cWDdd5KKhts' 113 | } 114 | }, 115 | player: { 116 | url: 'https://www.youtube.com/embed/cWDdd5KKhts', 117 | width: '480', 118 | height: '360' 119 | }, 120 | twitter: { site_username: '@youtube' }, 121 | icon: { 122 | '32': 'https://s.ytimg.com/yts/img/favicon_32-vfl8NGn4k.png', 123 | '48': 'https://s.ytimg.com/yts/img/favicon_48-vfl1s0rGh.png', 124 | '96': 'https://s.ytimg.com/yts/img/favicon_96-vfldSA3ca.png', 125 | '144': 'https://s.ytimg.com/yts/img/favicon_144-vflWmzoXw.png', 126 | smallest: 'https://s.ytimg.com/yts/img/favicon_32-vfl8NGn4k.png' 127 | }, 128 | preview: 'Cheese Shop Sketch - Monty Python\'s Flying Circus
YouTube
Cheese Shop Sketch - Monty Python\'s Flying Circus
Subscribe to the Official Monty Python Channel here - http://smarturl.it/SubscribeToPython Cleese plays an erudite customer attempting to purchase some chees...
', 129 | sources: ['ogp', 'resource', 'oembed', 'twitter'] 130 | } 131 | ``` 132 | ## Testing 133 | 134 | Simply run `yarn test` and all your tests in the `test/` directory will be run. 135 | 136 | ### Testing with docker-compose 137 | 138 | You can run eslint and mocha with: 139 | ``` 140 | - docker-compose run --rm embed-api yarn run eslint 141 | - docker-compose run --rm embed-api yarn run mocha 142 | ``` 143 | 144 | ## License 145 | 146 | Copyright (c) 2018 147 | Grzegorz Leoniec 148 | 149 | Licensed under the [MIT license](LICENSE). 150 | -------------------------------------------------------------------------------- /config/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "localhost", 3 | "port": 3050, 4 | "baseUrl": "http://localhost:3050", 5 | "public": "../public/", 6 | "paginate": { 7 | "default": 10, 8 | "max": 50 9 | }, 10 | "token": "", 11 | "mongodb": "mongodb://localhost:27017/embed_api", 12 | "embedTypeWhitelist": ["video", "photo", "link"] 13 | } 14 | -------------------------------------------------------------------------------- /config/production.json: -------------------------------------------------------------------------------- 1 | { 2 | "host": "EMBED_API_HOST", 3 | "port": "EMBED_API_PORT", 4 | "baseUrl": "EMBED_API_URL", 5 | "token": "EMBED_API_TOKEN", 6 | "mongodb": "EMBED_API_MONGO_DB" 7 | } 8 | -------------------------------------------------------------------------------- /docker-compose.override.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | embed-api: 5 | volumes: 6 | - .:/var/www 7 | - /var/www/node_modules/ 8 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | 3 | services: 4 | mongodb: 5 | image: mongo:latest 6 | environment: 7 | - MONGO_INITDB_ROOT_USERNAME=${EMBED_API_MONGO_USER:-MONGOUSER} 8 | - MONGO_INITDB_ROOT_PASSWORD=${EMBED_API_MONGO_PASS:-MONGOPASS} 9 | - MONGO_DATA_DIR=/data/db 10 | - MONGO_LOG_DIR=/dev/null 11 | volumes: 12 | - mongo-data:/data/db 13 | networks: 14 | - hc-network 15 | command: mongod --smallfiles --logpath=/dev/null 16 | embed-api: 17 | build: . 18 | image: humanconnection/embed-api:latest 19 | volumes: 20 | - .:/var/www 21 | - /var/www/node_modules/ 22 | ports: 23 | - 3050:3050 24 | networks: 25 | - hc-network 26 | depends_on: 27 | - mongodb 28 | environment: 29 | - EMBED_API_HOST=${EMBED_API_HOST:-0.0.0.0} 30 | - EMBED_API_PORT=3050 31 | - EMBED_API_URL=${EMBED_API_URL:-http://0.0.0.0:3050} 32 | - EMBED_API_TOKEN=${EMBED_API_TOKEN:-embedapitoken} 33 | - EMBED_API_MONGO_DB=mongodb://${EMBED_API_MONGO_USER:-MONGOUSER}:${EMBED_API_MONGO_PASS:-MONGOPASS}@mongodb:27017/embed_api?authSource=admin 34 | 35 | networks: 36 | hc-network: 37 | 38 | volumes: 39 | mongo-data: 40 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Human-Connection/Embed-API/dd6c76c592e827039515c1929947982040b93b92/index.js -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "embed-api", 3 | "version": "0.0.1", 4 | "description": "Embed API with Cache", 5 | "license": "MIT", 6 | "homepage": "https://human-connection.org", 7 | "main": "src", 8 | "keywords": [ 9 | "feathers", 10 | "embeds", 11 | "url", 12 | "metainfo" 13 | ], 14 | "author": { 15 | "name": "Grzegorz Leoniec", 16 | "email": "greg@app-interactive.de" 17 | }, 18 | "contributors": [], 19 | "bugs": {}, 20 | "directories": { 21 | "lib": "src", 22 | "test": "test/" 23 | }, 24 | "engines": { 25 | "node": ">=8.0.0 <11.0.0", 26 | "npm": ">=5.0.0 <6.0.0" 27 | }, 28 | "scripts": { 29 | "test": "yarn run eslint && yarn run mocha", 30 | "lint": "eslint src/. test/. --config .eslintrc.json", 31 | "start": "node src/", 32 | "dev": "concurrently 'mongod --dbpath data --quiet &>/dev/null' 'wait-on tcp:27017 && DEBUG=feathers nodemon src/'", 33 | "dev:debug": "DEBUG=feathers nodemon --inspect src/", 34 | "mocha": "mocha test/ --recursive --exit" 35 | }, 36 | "dependencies": { 37 | "@feathersjs/configuration": "^1.0.2", 38 | "@feathersjs/errors": "^3.3.0", 39 | "@feathersjs/express": "^1.2.0", 40 | "@feathersjs/feathers": "^3.1.3", 41 | "compression": "^1.7.2", 42 | "cors": "^2.8.4", 43 | "cross-env": "^5.2.0", 44 | "got": "^8.3.0", 45 | "helmet": "^3.12.0", 46 | "lodash": "^4.17.13", 47 | "metaphor": "^3.8.3", 48 | "metascraper": "^3.12.1", 49 | "metascraper-author": "^3.9.2", 50 | "metascraper-date": "^3.3.0", 51 | "metascraper-description": "^3.9.2", 52 | "metascraper-title": "^3.9.2", 53 | "mongoose": "^5.0.11", 54 | "node-fetch": "^2.1.2", 55 | "nodemon": "^1.17.2", 56 | "serve-favicon": "^2.4.5", 57 | "winston": "^2.4.1" 58 | }, 59 | "devDependencies": { 60 | "babel-plugin-istanbul": "^5.1.0", 61 | "concurrently": "^4.1.0", 62 | "eslint": "^5.10.0", 63 | "istanbul": "^0.4.5", 64 | "mocha": "^5.0.5", 65 | "nyc": "^13.1.0", 66 | "request-promise": "^4.2.2", 67 | "wait-on": "^3.2.0" 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Human-Connection/Embed-API/dd6c76c592e827039515c1929947982040b93b92/public/favicon.ico -------------------------------------------------------------------------------- /scripts/docker_push.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 3 | docker push humanconnection/embed-api:latest 4 | -------------------------------------------------------------------------------- /src/app.hooks.js: -------------------------------------------------------------------------------- 1 | // Application hooks that run for every service 2 | const logger = require('./hooks/logger'); 3 | 4 | module.exports = { 5 | before: { 6 | all: [ logger() ], 7 | find: [], 8 | get: [], 9 | create: [], 10 | update: [], 11 | patch: [], 12 | remove: [] 13 | }, 14 | 15 | after: { 16 | all: [ logger() ], 17 | find: [], 18 | get: [], 19 | create: [], 20 | update: [], 21 | patch: [], 22 | remove: [] 23 | }, 24 | 25 | error: { 26 | all: [ logger() ], 27 | find: [], 28 | get: [], 29 | create: [], 30 | update: [], 31 | patch: [], 32 | remove: [] 33 | } 34 | }; 35 | -------------------------------------------------------------------------------- /src/app.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const favicon = require('serve-favicon'); 3 | const compress = require('compression'); 4 | const cors = require('cors'); 5 | const helmet = require('helmet'); 6 | const logger = require('winston'); 7 | 8 | const feathers = require('@feathersjs/feathers'); 9 | const configuration = require('@feathersjs/configuration'); 10 | const express = require('@feathersjs/express'); 11 | 12 | const middleware = require('./middleware'); 13 | const services = require('./services'); 14 | const appHooks = require('./app.hooks'); 15 | const channels = require('./channels'); 16 | 17 | const mongoose = require('./mongoose'); 18 | 19 | const app = express(feathers()); 20 | 21 | // Load app configuration 22 | app.configure(configuration()); 23 | // Enable CORS, security, compression, favicon and body parsing 24 | app.use(cors()); 25 | app.use(helmet()); 26 | app.use(compress()); 27 | app.use(express.json()); 28 | app.use(express.urlencoded({ extended: true })); 29 | app.use(favicon(path.join(app.get('public'), 'favicon.ico'))); 30 | // Host the public folder 31 | app.use('/', express.static(app.get('public'))); 32 | 33 | // Set up Plugins and providers 34 | app.configure(express.rest()); 35 | 36 | app.configure(mongoose); 37 | 38 | // Configure other middleware (see `middleware/index.js`) 39 | app.configure(middleware); 40 | // Set up our services (see `services/index.js`) 41 | app.configure(services); 42 | // Set up event channels (see channels.js) 43 | app.configure(channels); 44 | 45 | // Configure a middleware for 404s and the error handler 46 | app.use(express.notFound()); 47 | app.use(express.errorHandler({ logger })); 48 | 49 | app.hooks(appHooks); 50 | 51 | module.exports = app; 52 | -------------------------------------------------------------------------------- /src/channels.js: -------------------------------------------------------------------------------- 1 | module.exports = function(app) { 2 | if(typeof app.channel !== 'function') { 3 | // If no real-time functionality has been configured just return 4 | return; 5 | } 6 | 7 | app.on('connection', connection => { 8 | // On a new real-time connection, add it to the anonymous channel 9 | app.channel('anonymous').join(connection); 10 | }); 11 | 12 | app.on('login', (authResult, { connection }) => { 13 | // connection can be undefined if there is no 14 | // real-time connection, e.g. when logging in via REST 15 | if(connection) { 16 | // Obtain the logged in user from the connection 17 | // const user = connection.user; 18 | 19 | // The connection is no longer anonymous, remove it 20 | app.channel('anonymous').leave(connection); 21 | 22 | // Add it to the authenticated user channel 23 | app.channel('authenticated').join(connection); 24 | 25 | // Channels can be named anything and joined on any condition 26 | 27 | // E.g. to send real-time events only to admins use 28 | // if(user.isAdmin) { app.channel('admins').join(connection); } 29 | 30 | // If the user has joined e.g. chat rooms 31 | // if(Array.isArray(user.rooms)) user.rooms.forEach(room => app.channel(`rooms/${room.id}`).join(channel)); 32 | 33 | // Easily organize users by email and userid for things like messaging 34 | // app.channel(`emails/${user.email}`).join(channel); 35 | // app.channel(`userIds/$(user.id}`).join(channel); 36 | } 37 | }); 38 | 39 | // eslint-disable-next-line no-unused-vars 40 | app.publish((data, hook) => { 41 | // Here you can add event publishers to channels set up in `channels.js` 42 | // To publish only for a specific event use `app.publish(eventname, () => {})` 43 | 44 | console.log('Publishing all events to all authenticated users. See `channels.js` and https://docs.feathersjs.com/api/channels.html for more information.'); // eslint-disable-line 45 | 46 | // e.g. to publish all service events to all authenticated users use 47 | return app.channel('authenticated'); 48 | }); 49 | 50 | // Here you can also add service specific event publishers 51 | // e..g the publish the `users` service `created` event to the `admins` channel 52 | // app.service('users').publish('created', () => app.channel('admins')); 53 | 54 | // With the userid and email organization from above you can easily select involved users 55 | // app.service('messages').publish(() => { 56 | // return [ 57 | // app.channel(`userIds/${data.createdBy}`), 58 | // app.channel(`emails/${data.recipientEmail}`) 59 | // ]; 60 | // }); 61 | }; 62 | -------------------------------------------------------------------------------- /src/hooks/logger.js: -------------------------------------------------------------------------------- 1 | // A hook that logs service method before, after and error 2 | // See https://github.com/winstonjs/winston for documentation 3 | // about the logger. 4 | const logger = require('winston'); 5 | 6 | // To see more detailed messages, uncomment the following line 7 | // logger.level = 'debug'; 8 | 9 | module.exports = function () { 10 | return context => { 11 | // This debugs the service call and a stringified version of the hook context 12 | // You can customize the mssage (and logger) to your needs 13 | logger.debug(`${context.type} app.service('${context.path}').${context.method}()`); 14 | 15 | if(typeof context.toJSON === 'function') { 16 | logger.debug('Hook Context', JSON.stringify(context, null, ' ')); 17 | } 18 | 19 | if (context.error) { 20 | logger.error(context.error); 21 | } 22 | }; 23 | }; 24 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | const logger = require('winston'); 3 | const app = require('./app'); 4 | const port = app.get('port'); 5 | const server = app.listen(port); 6 | 7 | process.on('unhandledRejection', (reason, p) => 8 | logger.error('Unhandled Rejection at: Promise ', p, reason) 9 | ); 10 | 11 | server.on('listening', () => 12 | logger.info('Embed API started on http://%s:%d', app.get('host'), port) 13 | ); 14 | -------------------------------------------------------------------------------- /src/middleware/index.js: -------------------------------------------------------------------------------- 1 | // const cors = require('cors'); 2 | 3 | // eslint-disable-next-line no-unused-vars 4 | let allowedRoutes = [ 5 | '/images' 6 | ]; 7 | 8 | module.exports = function (app) { 9 | /* 10 | const corsOptions = { 11 | origin: 'http://localhost:8080', 12 | optionsSuccessStatus: 200 13 | }; 14 | app.use(cors(corsOptions)) 15 | */ 16 | app.use(function (req, res, next) { 17 | const authentication = req.header('authentication'); 18 | const token = app.get('token'); 19 | if (allowedRoutes.includes(req.path) || !token || authentication === token) { 20 | next(); 21 | } else { 22 | res.status(401).send('Request not authorized'); 23 | } 24 | }); 25 | }; 26 | -------------------------------------------------------------------------------- /src/mongoose.js: -------------------------------------------------------------------------------- 1 | const mongoose = require('mongoose'); 2 | 3 | const MONGODB_URI = process.env.EMBED_API_MONGO_DB || 'mongodb://localhost:27017/embeds'; 4 | 5 | module.exports = function (app) { 6 | mongoose.Promise = global.Promise; 7 | mongoose.connect(`${MONGODB_URI}`, { useNewUrlParser: true }).catch( (e) => { 8 | throw e; 9 | }); 10 | app.set('mongooseClient', mongoose); 11 | }; 12 | -------------------------------------------------------------------------------- /src/services/embeds/embeds.class.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-vars */ 2 | const errors = require('@feathersjs/errors'); 3 | const mongoose = require('mongoose'); 4 | const { URL } = require('url'); 5 | const metascraper = require('metascraper').load([ 6 | require('metascraper-date')(), 7 | require('metascraper-title')(), 8 | require('metascraper-description')(), 9 | require('metascraper-image')() 10 | ]); 11 | const got = require('got'); 12 | 13 | const Metaphor = require('metaphor'); 14 | const engine = new Metaphor.Engine({ 15 | preview: false, 16 | tweet: true 17 | }); 18 | 19 | const getMetadata = async (targetURL, Provider) => { 20 | const data = { 21 | metaphor: {}, 22 | metascraper: {} 23 | }; 24 | 25 | // only get data from requested services 26 | let promises = []; 27 | if (Provider.methods.metaphor) { 28 | promises.push(new Promise((resolve, reject) => { 29 | try { 30 | engine.describe(targetURL, (metadata) => { 31 | data.metaphor = metadata; 32 | resolve(metadata); 33 | }); 34 | } catch (err) { 35 | reject(err); 36 | } 37 | })); 38 | } 39 | if (Provider.methods.metascraper) { 40 | promises.push(new Promise(async (resolve, reject) => { 41 | try { 42 | let metadata = {}; 43 | const head = await got.head(targetURL); 44 | if (head.headers['content-type'].indexOf('text') < 0) { 45 | if (head.headers['content-type']) { 46 | metadata.overwrite = { 47 | type: head.headers['content-type'].split('/').shift(), 48 | contentType: head.headers['content-type'] 49 | }; 50 | } 51 | } else { 52 | const { body: html, url } = await got(targetURL); 53 | metadata = await metascraper({ html, url }); 54 | } 55 | 56 | data.metascraper = metadata; 57 | resolve(data); 58 | } catch (err) { 59 | resolve({err}); 60 | } 61 | })); 62 | } 63 | 64 | await Promise.all(promises); 65 | 66 | // if (!data.metaphor.icon || data.metaphor.icon.any) { 67 | // const { hostname } = new URL(data.metaphor.url); 68 | // data.metaphor.icon = Object.assign(data.metaphor.icon || {}, { 69 | // any: `https://logo.clearbit.com/${hostname}` 70 | // }); 71 | // } 72 | 73 | if (!data.metascraper) { 74 | return data.metaphor; 75 | } 76 | 77 | if (data.metascraper && data.metascraper.overwrite) { 78 | data.metaphor = Object.assign(data.metaphor, data.metascraper.overwrite); 79 | delete data.metascraper.overwrite; 80 | } 81 | 82 | if (!data.metaphor.image && data.metascraper.image) { 83 | data.metaphor.image = { 84 | url: data.metascraper.image 85 | }; 86 | } 87 | if (data.metascraper.title && data.metaphor.title !== data.metaphor.description) { 88 | data.metaphor.title = data.metascraper.title; 89 | } 90 | if (data.metaphor.title === data.metaphor.description) { 91 | data.metaphor.title = null; 92 | } 93 | 94 | if (data.metascraper.description) { 95 | data.metaphor.description = data.metascraper.description; 96 | } 97 | if (data.metascraper.date) { 98 | data.metaphor.date = data.metascraper.date; 99 | } 100 | 101 | return data.metaphor; 102 | }; 103 | 104 | class Service { 105 | constructor (options) { 106 | this.options = options || {}; 107 | if (!options.app) { 108 | throw new Error('embeds services missing option.app'); 109 | } 110 | this.app = options.app; 111 | this.embeds = mongoose.model('embeds'); 112 | this.Provider = require('./providers')(this.app); 113 | } 114 | 115 | async find (params) { 116 | let url = params.query.url; 117 | url = url.replace(/\/+$/, ''); 118 | if (url.indexOf('://') < 0) { 119 | url = `https://${url}`; 120 | } 121 | const Provider = new this.Provider(url); 122 | url = Provider.normalizeURL(url); 123 | // 1. check if there is already metadata 124 | let embed = await this.embeds.findOne({ 125 | $or: [ 126 | { url: url }, 127 | { url: `${url}/` }, 128 | { 'metadata.url': url }, 129 | { 'metadata.url': `${url}/` } 130 | ] 131 | }); 132 | if (embed) { 133 | return embed.metadata; 134 | } 135 | 136 | // 2. if not or not older then x minutes, get fresh data and save it to the database 137 | let metadata = await getMetadata(url, Provider); 138 | try { 139 | metadata = Provider.enrichMetadata(metadata); 140 | } catch (err) { 141 | return err; 142 | } 143 | 144 | if (!metadata.title && !metadata.site_name) { 145 | throw new errors.NotFound('no data found for url'); 146 | } 147 | 148 | try { 149 | await this.embeds.create({ 150 | url, 151 | metadata 152 | }); 153 | } catch (err) { 154 | return err; 155 | } 156 | 157 | // 3. return cached or fresh metadata 158 | return metadata; 159 | } 160 | 161 | async get (id, params) { 162 | throw errors.NotImplemented(); 163 | } 164 | 165 | async create (data, params) { 166 | throw errors.NotImplemented(); 167 | } 168 | 169 | async update (id, data, params) { 170 | throw errors.NotImplemented(); 171 | } 172 | 173 | async patch (id, data, params) { 174 | throw errors.NotImplemented(); 175 | } 176 | 177 | async remove (id, params) { 178 | throw errors.NotImplemented(); 179 | } 180 | } 181 | 182 | module.exports = function (options) { 183 | return new Service(options); 184 | }; 185 | 186 | module.exports.Service = Service; 187 | -------------------------------------------------------------------------------- /src/services/embeds/embeds.hooks.js: -------------------------------------------------------------------------------- 1 | 2 | 3 | module.exports = { 4 | before: { 5 | all: [], 6 | find: [], 7 | get: [], 8 | create: [], 9 | update: [], 10 | patch: [], 11 | remove: [] 12 | }, 13 | 14 | after: { 15 | all: [], 16 | find: [], 17 | get: [], 18 | create: [], 19 | update: [], 20 | patch: [], 21 | remove: [] 22 | }, 23 | 24 | error: { 25 | all: [], 26 | find: [], 27 | get: [], 28 | create: [], 29 | update: [], 30 | patch: [], 31 | remove: [] 32 | } 33 | }; 34 | -------------------------------------------------------------------------------- /src/services/embeds/embeds.model.js: -------------------------------------------------------------------------------- 1 | // embeds.model.js - A mongoose model 2 | // 3 | // See http://mongoosejs.com/docs/models.html 4 | // for more of what you can do here. 5 | module.exports = function (app) { 6 | const mongooseClient = app.get('mongooseClient'); 7 | const embeds = new mongooseClient.Schema({ 8 | url: { type: String, required: true, index: true, unique: true }, 9 | metadata: { type: Object, required: true }, 10 | createdAt: { type: Date, default: Date.now } 11 | }); 12 | 13 | embeds.index({ url: 1 }, { unique: true }); 14 | embeds.index({ 'metadata.url': 1 }); 15 | embeds.index({ createdAt: 1 }, { expireAfterSeconds: 60 * 60 * 24 }); 16 | 17 | return mongooseClient.model('embeds', embeds); 18 | }; 19 | -------------------------------------------------------------------------------- /src/services/embeds/embeds.service.js: -------------------------------------------------------------------------------- 1 | // Initializes the `url` service on path `/url` 2 | const createService = require('./embeds.class.js'); 3 | const createModel = require('./embeds.model.js'); 4 | const hooks = require('./embeds.hooks'); 5 | 6 | module.exports = function (app) { 7 | 8 | const paginate = app.get('paginate'); 9 | const Model = createModel(app); 10 | 11 | const options = { 12 | app, 13 | name: 'embeds', 14 | Model, 15 | paginate 16 | }; 17 | 18 | // Initialize our service with any options it requires 19 | app.use('/embeds', createService(options)); 20 | 21 | // Get our initialized service so that we can register hooks and filters 22 | const service = app.service('embeds'); 23 | 24 | service.hooks(hooks); 25 | }; 26 | -------------------------------------------------------------------------------- /src/services/embeds/providers/index.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = function (app) { 3 | const ProviderYouTube = require('./provider.youtube')(app); 4 | const ProviderFlickr = require('./provider.flickr')(app); 5 | const ProviderDefault = require('./provider.default')(app); 6 | 7 | return class Provider { 8 | constructor (url) { 9 | // list of possible providers plus the default fallback 10 | const providers = [ 11 | new ProviderYouTube(), 12 | new ProviderFlickr(), 13 | new ProviderDefault() 14 | ]; 15 | 16 | // find the matching provider by url 17 | let outputProvider; 18 | providers.forEach(provider => { 19 | if (!outputProvider && provider.checkURL(url)) { 20 | outputProvider = provider; 21 | return outputProvider; 22 | } 23 | }); 24 | return outputProvider; 25 | } 26 | }; 27 | }; 28 | -------------------------------------------------------------------------------- /src/services/embeds/providers/provider.default.js: -------------------------------------------------------------------------------- 1 | module.exports = function (app) { 2 | return class ProviderDefault { 3 | constructor (options = {}) { 4 | this.app = app; 5 | this.imageBaseUrl = `${app.get('baseUrl')}/images?url=`; 6 | 7 | options = Object.assign({ 8 | name: 'default', 9 | methods: { 10 | metaphor: true, // this is always needed at the moment 11 | metascraper: true // this is optional 12 | } 13 | }, options); 14 | this.methods = options.methods; 15 | this.name = options.name; 16 | } 17 | 18 | checkURL () { 19 | return true; 20 | } 21 | 22 | normalizeURL (url) { 23 | return url; 24 | } 25 | 26 | proxyImageUrl (url) { 27 | url = encodeURIComponent(url); 28 | return this.imageBaseUrl + url; 29 | } 30 | 31 | enrichMetadata (metadata) { 32 | if (metadata.icon && metadata.icon.any) { 33 | metadata.icon.any = this.proxyImageUrl(metadata.icon.any); 34 | } 35 | 36 | if (metadata.image && metadata.image.url) { 37 | metadata.image.url = this.proxyImageUrl(metadata.image.url); 38 | } 39 | 40 | if (metadata.embed && metadata.embed.type && metadata.embed.type === 'photo') { 41 | if (!metadata.image) { 42 | metadata.image = {}; 43 | } 44 | metadata.image.url = this.proxyImageUrl(metadata.url); 45 | } 46 | 47 | return metadata; 48 | } 49 | }; 50 | }; 51 | -------------------------------------------------------------------------------- /src/services/embeds/providers/provider.flickr.js: -------------------------------------------------------------------------------- 1 | module.exports = function (app) { 2 | const ProviderDefault = require('./provider.default')(app); 3 | return class ProviderFlickr extends ProviderDefault { 4 | constructor (options = {}) { 5 | options = Object.assign({ 6 | name: 'flickr', 7 | methods: { 8 | metaphor: true, 9 | metascraper: true 10 | } 11 | }, options); 12 | super(options); 13 | this.regex = /(?:(?:https?:\/\/)(?:www)?\.?(?:flickr)(?:\.com))/gi; 14 | } 15 | 16 | checkURL (url) { 17 | return this.regex.test(url); 18 | } 19 | 20 | normalizeURL (url) { 21 | return url; 22 | } 23 | 24 | enrichMetadata (metadata) { 25 | if (metadata.embed && metadata.embed.url) { 26 | metadata.image.url = metadata.embed.url; 27 | } 28 | return metadata; 29 | } 30 | }; 31 | }; 32 | -------------------------------------------------------------------------------- /src/services/embeds/providers/provider.youtube.js: -------------------------------------------------------------------------------- 1 | module.exports = function (app) { 2 | const ProviderDefault = require('./provider.default')(app); 3 | return class ProviderYouTube extends ProviderDefault { 4 | constructor (options = {}) { 5 | options = Object.assign({ 6 | name: 'youtube', 7 | methods: { 8 | metaphor: true, 9 | metascraper: false 10 | } 11 | }, options); 12 | super(options); 13 | this.regex = /(?:(?:https?:\/\/)(?:www)?\.?(?:youtu\.?be)(?:\.com)?\/(?:.*[=/])*)([^= &?/\r\n]{8,11})&(start=.*[0-9])/gi; 14 | } 15 | 16 | checkURL (url) { 17 | this.match = this.regex.exec(url); 18 | return this.match && this.match.length > 1; 19 | } 20 | 21 | normalizeURL (url) { 22 | if (this.match && this.match[1] && this.match[2]) { 23 | url = `https://www.youtube.com/watch?v=${this.match[1]}&${this.match[2]}`; 24 | } else { 25 | url = `https://www.youtube.com/watch?v=${this.match[1]}`; 26 | } 27 | 28 | return url; 29 | } 30 | 31 | enrichMetadata (metadata) { 32 | if (metadata.thumbnail && metadata.thumbnail.url && !metadata.image) { 33 | metadata.image = { 34 | url: metadata.thumbnail.url 35 | }; 36 | } 37 | return metadata; 38 | } 39 | }; 40 | }; 41 | -------------------------------------------------------------------------------- /src/services/images/images.service.js: -------------------------------------------------------------------------------- 1 | const fetch = require('node-fetch'); 2 | 3 | 4 | 5 | // ToDo: Add image cache / thumbnail creation 6 | module.exports = function (app) { 7 | 8 | 9 | app.use('/images', (req, res) => { 10 | if (!req.query.url) { 11 | res.status(404).end('no url found.'); 12 | } 13 | fetch(req.query.url) 14 | .then(result => { 15 | if (result.headers.get('content-type').indexOf('image') < 0) { 16 | res.status(404).end('no url found.'); 17 | } 18 | res.writeHead(200, { 19 | 'Content-Type': result.headers.get('content-type') 20 | }); 21 | result.buffer() 22 | .then(buffer => { 23 | res.end(buffer); 24 | }); 25 | }); 26 | }); 27 | }; 28 | -------------------------------------------------------------------------------- /src/services/index.js: -------------------------------------------------------------------------------- 1 | const embeds = require('./embeds/embeds.service.js'); 2 | const images = require('./images/images.service.js'); 3 | // eslint-disable-next-line no-unused-vars 4 | module.exports = function (app) { 5 | app.configure(embeds); 6 | app.configure(images); 7 | }; 8 | -------------------------------------------------------------------------------- /test/app.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const rp = require('request-promise'); 3 | const url = require('url'); 4 | const app = require('../src/app'); 5 | 6 | const port = 3051; 7 | const getUrl = pathname => url.format({ 8 | hostname: app.get('host') || 'localhost', 9 | protocol: 'http', 10 | port, 11 | pathname 12 | }); 13 | 14 | describe('Feathers application tests', () => { 15 | before(function(done) { 16 | this.server = app.listen(port); 17 | this.server.once('listening', () => done()); 18 | }); 19 | 20 | after(function(done) { 21 | this.server.close(done); 22 | }); 23 | 24 | describe('404', function() { 25 | it('shows a 404 HTML page', () => { 26 | return rp({ 27 | url: getUrl('path/to/nowhere'), 28 | headers: { 29 | 'Accept': 'text/html', 30 | 'authentication': 'embedapitoken' 31 | } 32 | }).catch(res => { 33 | assert.equal(res.statusCode, 404); 34 | assert.ok(res.error.indexOf('') !== -1); 35 | }); 36 | }); 37 | 38 | it('shows a 404 JSON error without stack trace', () => { 39 | return rp({ 40 | url: getUrl('path/to/nowhere'), 41 | headers: { 42 | 'authentication': 'embedapitoken' 43 | }, 44 | json: true 45 | }).catch(res => { 46 | assert.equal(res.statusCode, 404); 47 | assert.equal(res.error.code, 404); 48 | assert.equal(res.error.message, 'Page not found'); 49 | assert.equal(res.error.name, 'NotFound'); 50 | }); 51 | }); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /test/services/embeds.test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const app = require('../../src/app'); 3 | const service = app.service('embeds'); 4 | 5 | const port = 3051; 6 | 7 | describe('\'embeds\' service', () => { 8 | before(function(done) { 9 | this.server = app.listen(port); 10 | this.server.once('listening', () => done()); 11 | }); 12 | 13 | after(function(done) { 14 | this.server.close(done); 15 | }); 16 | 17 | it('registered the service', () => { 18 | assert.ok(service, 'Registered the service'); 19 | }); 20 | 21 | // Parameters for the find() method on the Service class. 22 | const params = { 23 | simpleUrl: { 24 | query: { 25 | url: 'https://www.youtube.com/watch?v=QWU9YsxXPqw' 26 | } 27 | }, 28 | startTimeUrl: { 29 | query: { 30 | url: 'https://www.youtube.com/watch?v=QWU9YsxXPqw&start=332' 31 | } 32 | } 33 | }; 34 | 35 | it('returned URL metadata', async () => { 36 | try { 37 | // Calls Embeds Service.find(params) method, which will trigger the URL validation 38 | let body = await service.find(params.simpleUrl); 39 | assert.equal(body.site_name.toLowerCase(), 'youtube'); 40 | assert.equal(body.url, params.simpleUrl.query.url); 41 | } catch (err) { 42 | return err; 43 | } 44 | }); 45 | 46 | it('returned URL metadata with start time', async () => { 47 | try { 48 | let body = await service.find(params.startTimeUrl); 49 | assert.equal(body.site_name.toLowerCase(), 'youtube'); 50 | // Start time is only appended to the "video" key and it always returns with a "?start" 51 | assert.equal(body.video[0].url, 'https://www.youtube.com/embed/QWU9YsxXPqw?start=332'); 52 | } catch (err) { 53 | return err; 54 | } 55 | }); 56 | }); 57 | --------------------------------------------------------------------------------