├── .gitignore
├── up.json
├── img
└── screen.png
├── package.json
├── LICENSE.txt
├── app.js
└── README.md
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | yarn.lock
3 |
--------------------------------------------------------------------------------
/up.json:
--------------------------------------------------------------------------------
1 | {
2 | "profile": "aws"
3 | }
4 |
--------------------------------------------------------------------------------
/img/screen.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/MustansirZia/serverless-link-preview/HEAD/img/screen.png
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "serverless-link-preview",
3 | "version": "0.0.1",
4 | "description": "Serverless service to get website description and preview deployed on AWS Lambda.",
5 | "main": "app.js",
6 | "author": "Mustansir Zia",
7 | "license": "MIT",
8 | "scripts": {
9 | "start": "node app.js"
10 | },
11 | "repository": {
12 | "type": "git",
13 | "url": "https://github.com/MustansirZia/serverless-link-preview"
14 | },
15 | "keywords": [
16 | "react",
17 | "react-native",
18 | "location",
19 | "gps",
20 | "fused",
21 | "android",
22 | "play-services",
23 | "npm",
24 | "native"
25 | ],
26 | "dependencies": {
27 | "@nunkisoftware/link-preview": "^0.2.0",
28 | "cors": "^2.8.4",
29 | "express": "^4.16.2",
30 | "memory-cache": "^0.2.0"
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2017 Mustansir Zia
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
23 |
--------------------------------------------------------------------------------
/app.js:
--------------------------------------------------------------------------------
1 | const express = require('express');
2 | const linkPreview = require('@nunkisoftware/link-preview');
3 | const mCache = require('memory-cache');
4 | const cors = require('cors');
5 |
6 | const app = express();
7 |
8 | // Apply cors to provide asynchronous access from browsers.
9 | app.use(cors());
10 |
11 | // Validation middleware to simply check the url query param.
12 | const validate = function (req, res, next) {
13 | const url = req.query.url;
14 | if (!url) {
15 | res.status(400).json({ message: 'url query param missing.' });
16 | return;
17 | }
18 | next();
19 | };
20 |
21 | // Function which returns an in memory cache middleware.
22 | const cache = function (duration) {
23 | return function (req, res, next) {
24 | const key = req.query.url;
25 |
26 | // Try to get cached response using url param as key.
27 | const cachedResponse = mCache.get(key);
28 |
29 | if (cachedResponse) {
30 |
31 | // Send cached response.
32 | res.json(cachedResponse);
33 | return;
34 |
35 | }
36 |
37 | // If cached response not present,
38 | // pass the request to the actual handler.
39 | res.originalJSON = res.json;
40 | res.json = function (result) {
41 |
42 | // Cache the newly generated response for later use
43 | // and send it to the client.
44 | mCache.put(key, result, duration * 1000);
45 | res.originalJSON(result);
46 |
47 | };
48 | next();
49 | };
50 | };
51 |
52 | // Actual get handler with cache set to 3 minutes.
53 | app.get('/', validate, cache(180), function (req, res) {
54 | const url = req.query.url;
55 |
56 | // Get the actual response from link-preview.
57 | // Wait for 5 secs before calling a timeout.
58 | linkPreview(url, 5000)
59 | .then(function (response) {
60 |
61 | if (!response.title) {
62 | // If the url given is incorrect.
63 | res.status(400).json({ message: 'Invalid URL given or timeout occured.' });
64 | return;
65 | }
66 |
67 | res.json(response);
68 | })
69 | .catch(function (err) {
70 | res.status(500).send('Internal Server Error.');
71 | });
72 | });
73 |
74 | // Listen on the port provided by Up.
75 | app.listen(process.env.PORT || 3000);
76 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # serverless-link-preview.
2 |
3 | [](https://opensource.org/licenses/mit-license.php)
4 |
5 | ### A serverless, scalable website preview service built using [Node.js](https://nodejs.org/en/), [Express.js](https://expressjs.com/), [memory-cache](https://github.com/ptarjan/node-cache) and deployed using [Up](https://github.com/apex/up).
6 |
7 |
8 | This repository is basically a follow up to an article that I wrote [here](https://mustansirzia.com/posts/link-preview/).
9 |
10 |
11 |
12 | It is a RESTful API service (a microservice) that will take in a website URL and reply with its title, description, a thumbnail preview of the first image found on the website along with the site name. Scrapping is done using [@nunkisoftware/link-preview](https://github.com/nunkisoftware/link-preview). It is serverless and runs on AWS Lambda as Function as a Service (FaaS). Since there is no server or other hardware considerations it can scale to mammoth proportions as Amazon will automatically deploy copies of our exported functions depending on the load.
13 |
14 |
15 |
16 |
17 | ## Installation.
18 | • Install Up globally.
19 | `$ npm i -g up`
20 |
21 |
22 |
23 | • Then, you've two choices. Either clone this repo, install local dependencies skip to the very last step.
24 | `$ git clone https://github.com/MustansirZia/serverless-link-preview`
25 |
26 | `$ npm i`
27 |
28 |
29 | OR follow along,
30 |
31 |
32 | • First, initialise the project yourself by creating these files.
33 | `$ touch package.json up.json app.js`
34 |
35 |
36 |
37 | • Then, add a few local packages.
38 | `$ npm i express memory-cache cors @nunkisoftware/link-preview --save`
39 |
40 |
41 |
42 | • Add a `scripts` section to your `package.json` so Up knows how to start your express server.
43 | ```json
44 | {
45 | "name": "serverless-link-preview",
46 | "version": "0.0.1",
47 | "description": "Serverless service to get website description and preview deployed on AWS Lambda.",
48 | "main": "app.js",
49 | "license": "MIT",
50 | "scripts": {
51 | "start": "node app.js"
52 | },
53 | "dependencies": {
54 | "@nunkisoftware/link-preview": "^0.2.0",
55 | "cors": "^2.8.4",
56 | "express": "^4.16.2",
57 | "memory-cache": "^0.2.0"
58 | }
59 | }
60 | ```
61 |
62 |
63 |
64 | • Write an express server inside `app.js` with a single GET endpoint at `/` which would take a query param `url`. This would be our website url whose preview we require.
65 |
66 | ```js
67 | const express = require('express');
68 | const linkPreview = require('@nunkisoftware/link-preview');
69 | const mCache = require('memory-cache');
70 | const cors = require('cors');
71 |
72 | const app = express();
73 |
74 | // Apply cors to provide asynchronous access from browsers.
75 | app.use(cors());
76 |
77 | // Validation middleware to simply check the url query param.
78 | const validate = function (req, res, next) {
79 | const url = req.query.url;
80 | if (!url) {
81 | res.status(400).json({ message: 'url query param missing.' });
82 | return;
83 | }
84 | next();
85 | };
86 |
87 | // Function which returns an in memory cache middleware.
88 | const cache = function (duration) {
89 | return function (req, res, next) {
90 | const key = req.query.url;
91 |
92 | // Try to get cached response using url param as key.
93 | const cachedResponse = mCache.get(key);
94 |
95 | if (cachedResponse) {
96 |
97 | // Send cached response.
98 | res.json(cachedResponse);
99 | return;
100 |
101 | }
102 |
103 | // If cached response not present,
104 | // pass the request to the actual handler.
105 | res.originalJSON = res.json;
106 | res.json = function (result) {
107 |
108 | // Cache the newly generated response for later use
109 | // and send it to the client.
110 | mCache.put(key, result, duration * 1000);
111 | res.originalJSON(result);
112 |
113 | };
114 | next();
115 | };
116 | };
117 |
118 | // Actual get handler with cache set to 3 minutes.
119 | app.get('/', validate, cache(180), function (req, res) {
120 | const url = req.query.url;
121 |
122 | // Get the actual response from link-preview.
123 | linkPreview(url)
124 | .then(function (response) {
125 |
126 | if (!response.title) {
127 | // If the url given is incorrect.
128 | res.status(400).json({ message: 'Invalid URL given.' });
129 | return;
130 | }
131 |
132 | res.json(response);
133 | })
134 | .catch(function (err) {
135 | res.status(500).send('Internal Server Error.');
136 | });
137 | });
138 |
139 | // Listen on the port provided by Up.
140 | app.listen(process.env.PORT || 3000);
141 | ```
142 |
143 | Please note that we also employ an in memory cache to store recent website previews so we don't query `link-preview` on every frequent homogenous request (as that's a time/resource expensive thing to do) and thus serve the cached result to our client.
144 |
145 | The following two steps can also be accomplished using environment variables but making a separate file is much cleaner and will make our deployment super easy by writing a single command, `Up`.
146 |
147 | • Add a single entry to our `up.json` so Up knows where and how to find our AWS credentials.
148 | ```json
149 | {
150 | "profile": "aws"
151 | }
152 | ```
153 |
154 |
155 | This is a one time step and won't be required for subsequent Up deployments.
156 |
157 | • Finally, create the aws credentials file at `~/.aws/` and fill in your IAM credentials.
158 |
159 |
160 | `$ mkdir -p ~/.aws && touch ~/.aws/credentials`
161 | `$ gedit ~/.aws/credentials` or `$ nano ~/.aws/credentials` and paste the following in.
162 |
163 | Replace `$YOUR_ACCESS_ID` and `$YOUR_ACCESS_KEY` with your own. Find them from [here.](https://help.bittitan.com/hc/en-us/articles/115008255268-How-do-I-find-my-AWS-Access-Key-and-Secret-Access-Key-) It could be beneficial to create a new IAM user just for this purpose.
164 |
165 | ```
166 | [aws]
167 | aws_access_key_id = $YOUR_ACCESS_ID
168 | aws_secret_access_key = $YOUR_ACCESS_KEY
169 | ```
170 |
171 | Save the file and that's it.
172 |
173 | To verify our installation, key in `npm start` from the directory that houses our `app.js`.
174 | From another terminal window, request our service like so.
175 | `$ curl localhost:3000?url=https://www.youtube.com/watch?v=NUWViXhvW3k`
176 | You should see a familiar JSON and this verifies our installation.
177 |
178 | ```json
179 | {
180 | "url": "https://www.youtube.com/watch?v=NUWViXhvW3k",
181 | "image": "https://i.ytimg.com/vi/NUWViXhvW3k/maxresdefault.jpg",
182 | "imageWidth": null,
183 | "imageHeight": null,
184 | "imageType": null,
185 | "title": "Building the CLEANEST Desk Setup!!!",
186 | "description": "My setup tour: https://goo.gl/nv0nja ADD ME ON SNAPCHAT TO STAY UP TO DATE WITH MY SETUP PROGRESS: Snapchat: Kenneth.YT or KDKHD SNAP CODE: http://kennethkre...",
187 | "siteName": "YouTube"
188 | }
189 | ```
190 |
191 | Inside the same directory, deploy the service with a single command.
192 |
193 | ### `$ up`
194 |
195 |
196 |
197 | After the deployment is complete, get the service's URL like so.
198 | `$ up url`
199 |
200 | The url with the query param could look similar to this.
201 | [`https://hfnuua77fd.execute-api.us-west-2.amazonaws.com/development?url=https://www.youtube.com/watch?v=NUWViXhvW3k`](https://hfnuua77fd.execute-api.us-west-2.amazonaws.com/development?url=https://www.youtube.com/watch?v=NUWViXhvW3k)
202 |
203 | And there you have it, your own serverless and scalable website preview service built and deployed on AWS Lambda.
204 | Query with your favourite http client inside any application.
205 |
206 | ## Further Reading.
207 | • Documentation for [Up.](https://up.docs.apex.sh/#introduction)
208 |
209 | ## License.
210 | • [MIT.](https://github.com/MustansirZia/serverless-link-preview/blob/master/LICENSE.txt)
211 |
--------------------------------------------------------------------------------