├── .gitignore ├── Dockerfile ├── docker-compose.yml ├── package.json ├── tests └── test.js ├── README.md ├── app ├── app.js └── spider.js └── LICENSE /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:lts-slim 2 | COPY package*.json . 3 | RUN npm install 4 | COPY /app /app 5 | EXPOSE 8000 6 | CMD ["npm", "start"] -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | redis: 3 | image: redis/redis-stack-server:latest 4 | ports: 5 | - 6379:6379 6 | tika: 7 | image: apache/tika:latest 8 | ports: 9 | - 9998:9998 10 | app: 11 | build: . 12 | environment: 13 | REDIS_URL: redis://redis:6379 14 | TIKA_URL: http://tika:9998 15 | APP_PORT: 8000 16 | ports: 17 | - 8000:8000 18 | depends_on: 19 | - redis 20 | - tika -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "doc-crawler", 3 | "version": "1.0.0", 4 | "description": "web crawling with redis indexing", 5 | "main": "app.js", 6 | "scripts": { 7 | "test": "node ./tests/test.js", 8 | "start": "node ./app/app.js" 9 | }, 10 | "keywords": [ 11 | "redis" 12 | ], 13 | "author": "joey whelan", 14 | "license": "See LICENSE file", 15 | "dependencies": { 16 | "axios": "^0.27.2", 17 | "cheerio": "^1.0.0-rc.12", 18 | "express": "^4.18.1", 19 | "redis": "^4.3.1", 20 | "uuid": "^9.0.0" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /tests/test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileoverview Client-side tests of the API 3 | */ 4 | 5 | 'use strict'; 6 | const axios = require('axios'); 7 | const appUrl = 'http://localhost:8000'; 8 | const sleep = ms => new Promise(r => setTimeout(r, ms)); 9 | 10 | (async () => { 11 | let res; 12 | 13 | console.log('*** app status ***'); 14 | res = await axios({ 15 | method: 'GET', 16 | url: appUrl, 17 | responseType: 'json' 18 | }); 19 | console.log(JSON.stringify(res.data, null, 4)); 20 | 21 | console.log('\n*** start a crawl task ***'); 22 | res = await axios({ 23 | method: 'POST', 24 | url: `${appUrl}/crawl`, 25 | data: {'fqdn': 'developer.redis.com'}, 26 | responseType: 'json' 27 | }); 28 | let taskID = res.data.taskID; 29 | console.log(JSON.stringify(res.data, null, 4)); 30 | 31 | console.log('\n*** check status on the crawl task ***'); 32 | let status; 33 | while (status != 'complete') { 34 | await sleep(10000); //sleep 10 sec 35 | try { 36 | res = await axios({ 37 | method: 'GET', 38 | url: `${appUrl}/status/tasks/${taskID}`, 39 | responseType: 'json' 40 | }); 41 | status = res.data.status; 42 | console.log(JSON.stringify(res.data, null, 4)); 43 | } 44 | catch (err) { 45 | console.log(err.message); 46 | } 47 | } 48 | 49 | console.log('\n*** document search ***'); 50 | res = await axios({ 51 | method: 'PUT', 52 | url: `${appUrl}/search`, 53 | data: {'term': 'Node.js'}, 54 | responseType: 'json' 55 | }); 56 | console.log(JSON.stringify(res.data, null, 4)); 57 | })(); -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Example of a Web Crawler with Redis Indexing 2 | 3 | ## Summary 4 | API server implementation of a web crawler. Apache Tika is leveraged to extract text from crawled docs (html, 5 | pdf, etc). The extracted text is then stored in Redis as JSON and indexed via RediSearch. 6 | 7 | ## Architecture 8 | ### High Level 9 | ![High-level Architecture](https://docs.google.com/drawings/d/e/2PACX-1vTSA_ZdSLYGTVXzALpSndVo1cjt4z9XumZsoI5skRkILGlYOIudkIGEwR6iQ9wjlIgQ3CJ0CoGYILr1/pub?w=663&h=380 "High Level Architecture") 10 | ### Detailed 11 | ![Detailed Architecture](https://docs.google.com/drawings/d/e/2PACX-1vQjHys8uZsTIntbtubgaPOkrqXVaGmGtl57B_NAzbkzBn3GvT02MKSYwlsYqiLVKQVgq2WFdxGzzRnA/pub?w=830&h=290 "Detailed Architecture") 12 | ## Application Flow 13 | ![Application Flow](https://docs.google.com/drawings/d/e/2PACX-1vQUqCKdebUOvKqUa7POsdNxJsOLLhog13krnQZmwIxvECC-GTzo24mwl4YiNVWR_4_RjMY-D989O2A1/pub?w=696&h=292 "Application Flow") 14 | 15 | ## Features 16 | - Implements a simple web crawler (cheerio-based) 17 | - Utilizes Apache Tika server for mime-type detection and text extraction 18 | - Utilizes RedisJSON for document storage and RediSearch for indexing. 19 | ## Prerequisites 20 | - Docker 21 | - Node.js 22 | - npm 23 | - Apache Tika 24 | - Redis w/RediSearch and RedisJSON modules 25 | ## Installation 26 | 1. Clone this repo. 27 | 28 | 2. Go to doc-crawler folder. 29 | ```bash 30 | cd doc-crawler 31 | ``` 32 | 3. Install Node.js requirements 33 | ```bash 34 | npm install 35 | ``` 36 | 4. Build and start docker containers 37 | ```bash 38 | docker compose up 39 | ``` 40 | ## Usage 41 | ### Test Client 42 | ```bash 43 | npm run test 44 | ``` 45 | ### CURL 46 | ```bash 47 | #app status 48 | curl -X GET http://localhost:8000 49 | 50 | {"status":"app running"} 51 | 52 | #start a crawl task 53 | curl -X POST http://localhost:8000/crawl \ 54 | -H 'Content-Type: application/json' \ 55 | -d '{"fqdn":"developer.redis.com"}' 56 | 57 | {"taskID":"ec135f4c-d3f5-4a9f-bafb-4eb90bfd8273"} 58 | 59 | #check status on a crawl task 60 | curl -X GET http://localhost:8000/status/tasks/ec135f4c-d3f5-4a9f-bafb-4eb90bfd8273 61 | 62 | {"status":"active"} 63 | 64 | curl -X GET http://localhost:8000/status/tasks/ec135f4c-d3f5-4a9f-bafb-4eb90bfd8273 65 | 66 | {"indexed":159,"errors":0,"time":56.81,"status":"complete"} 67 | 68 | #document search 69 | curl -X PUT http://localhost:8000/search \ 70 | -H 'Content-Type: application/json' \ 71 | -d '{"term":"Node.js"}' 72 | 73 | {"docs":["developer.redis.com/develop/node","developer.redis.com/develop/node/node-crash-course","developer.redis.com/develop/java/redis-and-spring-course/lesson_8","developer.redis.com/develop/node/nodecrashcourse/runningtheapplication","developer.redis.com/develop/node/nodecrashcourse/welcome","developer.redis.com/develop/node/nodecrashcourse/coursewrapup","developer.redis.com/develop/node/nodecrashcourse/redisbloom","developer.redis.com/develop/node/nodecrashcourse/sessionstorage","developer.redis.com/develop/node/nodecrashcourse/checkinswithstreams","developer.redis.com/develop/node/nodecrashcourse/redisearch"]} 74 | ``` -------------------------------------------------------------------------------- /app/app.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileoverview Express-based API server 3 | */ 4 | 5 | 'use strict'; 6 | const redis = require('redis'); 7 | const express = require('express'); 8 | const { v4: uuidv4 } = require('uuid'); 9 | const { Worker } = require('worker_threads'); 10 | const port = process.env.APP_PORT || 8000; 11 | const redisUrl = process.env.REDIS_URL || 'http://localhost:6379'; 12 | var client; 13 | 14 | /** 15 | * Helper function for creating redis client 16 | * @returns {redis} client 17 | */ 18 | async function clientFactory() { 19 | console.log(`app - clientFactory`); 20 | try { 21 | client.ping(); 22 | } 23 | catch (err) { 24 | console.error(`app - clientFactory - ${err.message}`); 25 | client = redis.createClient({url: redisUrl}); 26 | client.on('error', (err) => { 27 | console.error(`app - clientFactory - ${err.message}`); 28 | throw new Error(err.message); 29 | }); 30 | await client.connect(); 31 | } 32 | finally { 33 | return client; 34 | } 35 | } 36 | 37 | /** 38 | * Helper function for building the search index on the crawled docs 39 | */ 40 | async function buildIndex() { 41 | console.log(`app - buildIndex`); 42 | let rc = await clientFactory(); 43 | try { 44 | await rc.ft.create('docIdx', { 45 | '$.doc': { 46 | type: redis.SchemaFieldTypes.TEXT, 47 | AS: 'doc' 48 | }, 49 | '$.text': { 50 | type: redis.SchemaFieldTypes.TEXT, 51 | AS: 'text' 52 | } 53 | }, { 54 | ON: 'JSON', 55 | PREFIX: 'DOC' 56 | }); 57 | } 58 | catch(err) { 59 | console.error(`app - buildIndex - ${err.message}`); 60 | } 61 | } 62 | 63 | const app = express(); 64 | app.use(express.json()); 65 | 66 | /** 67 | * App status endpoint 68 | */ 69 | app.get('/', (req, res) => { 70 | console.log(`app - GET /`); 71 | res.status(200).json({'status': 'app running'}); 72 | }); 73 | 74 | /** 75 | * Crawl endpoint. Starts a Worker thread (spider.js) that crawls the given fqdn, extracts text via Tika, 76 | * and then store the resulting text in Redis as JSON documents 77 | * This returns immediately and provides a taskID for the Worker thread. 78 | */ 79 | app.post('/crawl', (req, res) => { 80 | console.log(`app - POST /crawl ${req.body.fqdn}`); 81 | const taskID = uuidv4(); 82 | try { 83 | new Worker('./app/spider.js', { workerData : { 'fqdn': req.body.fqdn, 'taskID': taskID }}); 84 | res.status(201).json({'taskID': taskID}); 85 | } 86 | catch (err) { 87 | console.error(`app - POST /crawl ${req.body.fqdn} - ${err.message}`) 88 | res.status(400).json({ 'error': err.message }); 89 | } 90 | }); 91 | 92 | /** 93 | * Task status endpoint. Worker threads are assigned a taskID and that ID + Worker status is stored 94 | * in Redis as a JSON doc. The client can poll this endpoint to determine when a crawl task is complete. 95 | */ 96 | app.get('/status/tasks/:taskID', async (req, res) => { 97 | console.log(`app - GET /status/tasks/${req.params.taskID}`) 98 | try { 99 | let rc = await clientFactory(); 100 | const status = await rc.json.get(`taskID:${req.params.taskID}`, '.'); 101 | if (status) { 102 | res.status(200).json(status); 103 | } 104 | else { 105 | throw new Error(`Task ${req.params.taskID} not found`); 106 | } 107 | } 108 | catch (err) { 109 | console.error(`app - GET /status/tasks/${req.params.taskID} - ${err.message}`) 110 | res.status(400).json({ 'error': err.message }); 111 | } 112 | }); 113 | 114 | /** 115 | * Doc search endpoint. This exposes a very simple interface for searching crawled documents. The client can 116 | * submit a term and an array of matching doc URLs are returned. 117 | */ 118 | app.put('/search', async (req, res) => { 119 | console.log(`app - PUT /search ${req.body.term}`) 120 | try { 121 | let rc = await clientFactory(); 122 | const sres = await rc.ft.search('docIdx', `@text:${req.body.term}`, { RETURN: ['doc'] }); 123 | if (sres && sres.documents) { 124 | let found = []; 125 | for (let item of sres.documents) { 126 | found.push(item.value.doc); 127 | } 128 | res.status(201).json({'docs': found}); 129 | } 130 | else { 131 | throw new Error(`Query error ${req.body.term}`); 132 | } 133 | } 134 | catch (err) { 135 | res.status(400).json({ 'error': err.message }); 136 | } 137 | }); 138 | 139 | app.listen(port, async () => { 140 | await buildIndex(); 141 | console.log(`app - listening on port ${port}`) 142 | }); -------------------------------------------------------------------------------- /app/spider.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @fileoverview Implements a web crawler that extracts text (html, pdf, etc) via Apache Tika and then 3 | * stores that text as JSON docs in Redis. This file is executed in a Node.js Worker thread. 4 | */ 5 | 6 | 'use strict'; 7 | const cheerio = require('cheerio'); 8 | const axios = require('axios'); 9 | const redis = require('redis'); 10 | const crypto = require('crypto'); 11 | const tikaUrl = process.env.TIKA_URL || 'http://localhost:9998'; 12 | const redisUrl = process.env.REDIS_URL || 'http://localhost:6379' 13 | 14 | const { workerData } = require('worker_threads'); 15 | const { Readable } = require('stream'); 16 | 17 | const PREFIX = 'DOC'; 18 | const CRAWL_BUDGET = 2500; // max number of iterations (recursions) for a given fqdn 19 | const ERR_BUDGET = 100; // max number of errors tolerated for a given fqdn 20 | 21 | /** 22 | * @class Implements a very simple/naive web crawler 23 | */ 24 | class Spider { 25 | /** 26 | * 27 | * @param {string} fqdn Domain to be crawled 28 | * @param {object} client Redis client 29 | */ 30 | constructor(fqdn, client) { 31 | this.fqdn = fqdn; 32 | this.start = Date.now(); 33 | this.docs = []; 34 | this.client = client; 35 | this.errors = 0; 36 | this.indexed = 0; 37 | this.iterations = 0; 38 | } 39 | 40 | /** 41 | * 42 | * @param {string} doc Current document being extracted and stored 43 | * @returns 44 | */ 45 | async crawl(doc=this.fqdn) { 46 | try { 47 | this.iterations++; 48 | if (this.errors < ERR_BUDGET && 49 | this.iterations < CRAWL_BUDGET && 50 | !this.docs.includes(doc)) 51 | { 52 | console.log(`spider - crawled doc: ${doc}`) 53 | const response = await axios({ //fetch the doc as an array buffer 54 | method: 'GET', 55 | url: `https://${doc}`, 56 | responseType: 'arrayBuffer' 57 | }); 58 | const data = response.data; 59 | this.docs.push(doc); // update visited pages 60 | 61 | // determine if document is already indexed 62 | const hash = crypto.createHash('sha256').update(data).digest('hex'); 63 | let result = await this.client.json.get(`${PREFIX}:${doc}`, '.'); //attempt to fetch doc from Redis 64 | if (!result || result.hash != hash) { //doc is not in Redis or a non-current version is in Redis 65 | await this.extract(doc, data, hash); //extract text via Tika and store in Redis 66 | this.indexed++; 67 | } 68 | 69 | // crawl links on document 70 | const $ = cheerio.load(data); 71 | const links = $('a'); 72 | for (let link of links) { 73 | link = $(link).attr('href'); 74 | if (link) { //recurse on any valid links on page 75 | link = link.replace(/[^a-z0-9]$/,''); //remove any trailing char 76 | if (link.startsWith(`https://${this.fqdn}`)) { //case 1 - absolute link 77 | await this.crawl(link.replace(/(^\w+:|^)\/\//, '')); //strip protocol 78 | } 79 | else if (link.startsWith('/')) { //case 2 - root link 80 | link = this.fqdn + link; 81 | await this.crawl(link); 82 | } 83 | else if (link.length > 0 && //case 3 - relative link 84 | !link.toLowerCase().startsWith('http') && 85 | !link.toLowerCase().startsWith('mailto')) { 86 | link = `${doc}/${link}`; 87 | await this.crawl(link); 88 | } 89 | else { //case 4 - not a valid link to crawl (link to foreign domain, etc) 90 | continue; 91 | } 92 | } 93 | } 94 | } 95 | } 96 | catch (err) { 97 | console.error('spider - ' + err.message); 98 | this.errors++; 99 | } 100 | finally { 101 | return { 102 | "indexed": this.indexed, 103 | "errors": this.errors, 104 | "time": parseFloat(((Date.now() - this.start)/1000).toFixed(2)) 105 | } 106 | } 107 | } 108 | 109 | /** 110 | * 111 | * @param {string} doc URL of document 112 | * @param {arrayBuffer} data Unformatted text of document 113 | * @param {string} hash SHA256 hash of the original document (prior to extraction to text) 114 | */ 115 | async extract(doc, data, hash) { 116 | const stream = Readable.from(data); //get a stream from the arrayBuffer obj 117 | const response = await axios({ //send that stream to Tika for automatic mime-type detection and text extraction 118 | method: 'PUT', 119 | url: `${tikaUrl}/tika`, 120 | data: stream, 121 | responseType: 'text', 122 | headers: { 123 | 'Content-Type': 'application/octet-stream', 124 | 'Accept': 'text/plain' 125 | } 126 | }); 127 | const json = { "doc": doc, "text": response.data, "hash": hash }; 128 | await this.client.json.set(`${PREFIX}:${doc}`, '.', json); 129 | } 130 | } 131 | 132 | (async () => { 133 | console.log(`spider - job started ${workerData.fqdn}`) 134 | const client = redis.createClient({url: redisUrl}); 135 | await client.connect(); 136 | await client.json.set(`taskID:${workerData.taskID}`, '.', {'status': 'active'}); 137 | const spider = new Spider(workerData.fqdn, client); 138 | let result = await spider.crawl(); 139 | result['status'] = 'complete'; 140 | await client.json.set(`taskID:${workerData.taskID}`, '.', result); 141 | await client.expire(`taskID:${workerData.taskID}`, 60*60*24); //24 hour expiration 142 | await client.quit(); 143 | console.log(`spider - ${workerData.fqdn} complete. docs indexed:${result.indexed},\ 144 | errors:${result.errors}, time:${result.time}`); 145 | })(); -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Server Side Public License 2 | VERSION 1, OCTOBER 16, 2018 3 | 4 | Copyright © 2022 Redis, Inc. 5 | 6 | Everyone is permitted to copy and distribute verbatim copies of this 7 | license document, but changing it is not allowed. 8 | 9 | TERMS AND CONDITIONS 10 | 11 | 0. Definitions. 12 | 13 | “This License” refers to Server Side Public License. 14 | 15 | “Copyright” also means copyright-like laws that apply to other kinds of 16 | works, such as semiconductor masks. 17 | 18 | “The Program” refers to any copyrightable work licensed under this 19 | License. Each licensee is addressed as “you”. “Licensees” and 20 | “recipients” may be individuals or organizations. 21 | 22 | To “modify” a work means to copy from or adapt all or part of the work in 23 | a fashion requiring copyright permission, other than the making of an 24 | exact copy. The resulting work is called a “modified version” of the 25 | earlier work or a work “based on” the earlier work. 26 | 27 | A “covered work” means either the unmodified Program or a work based on 28 | the Program. 29 | 30 | To “propagate” a work means to do anything with it that, without 31 | permission, would make you directly or secondarily liable for 32 | infringement under applicable copyright law, except executing it on a 33 | computer or modifying a private copy. Propagation includes copying, 34 | distribution (with or without modification), making available to the 35 | public, and in some countries other activities as well. 36 | 37 | To “convey” a work means any kind of propagation that enables other 38 | parties to make or receive copies. Mere interaction with a user through a 39 | computer network, with no transfer of a copy, is not conveying. 40 | 41 | An interactive user interface displays “Appropriate Legal Notices” to the 42 | extent that it includes a convenient and prominently visible feature that 43 | (1) displays an appropriate copyright notice, and (2) tells the user that 44 | there is no warranty for the work (except to the extent that warranties 45 | are provided), that licensees may convey the work under this License, and 46 | how to view a copy of this License. If the interface presents a list of 47 | user commands or options, such as a menu, a prominent item in the list 48 | meets this criterion. 49 | 50 | 1. Source Code. 51 | 52 | The “source code” for a work means the preferred form of the work for 53 | making modifications to it. “Object code” means any non-source form of a 54 | work. 55 | 56 | A “Standard Interface” means an interface that either is an official 57 | standard defined by a recognized standards body, or, in the case of 58 | interfaces specified for a particular programming language, one that is 59 | widely used among developers working in that language. The “System 60 | Libraries” of an executable work include anything, other than the work as 61 | a whole, that (a) is included in the normal form of packaging a Major 62 | Component, but which is not part of that Major Component, and (b) serves 63 | only to enable use of the work with that Major Component, or to implement 64 | a Standard Interface for which an implementation is available to the 65 | public in source code form. A “Major Component”, in this context, means a 66 | major essential component (kernel, window system, and so on) of the 67 | specific operating system (if any) on which the executable work runs, or 68 | a compiler used to produce the work, or an object code interpreter used 69 | to run it. 70 | 71 | The “Corresponding Source” for a work in object code form means all the 72 | source code needed to generate, install, and (for an executable work) run 73 | the object code and to modify the work, including scripts to control 74 | those activities. However, it does not include the work's System 75 | Libraries, or general-purpose tools or generally available free programs 76 | which are used unmodified in performing those activities but which are 77 | not part of the work. For example, Corresponding Source includes 78 | interface definition files associated with source files for the work, and 79 | the source code for shared libraries and dynamically linked subprograms 80 | that the work is specifically designed to require, such as by intimate 81 | data communication or control flow between those subprograms and other 82 | parts of the work. 83 | 84 | The Corresponding Source need not include anything that users can 85 | regenerate automatically from other parts of the Corresponding Source. 86 | 87 | The Corresponding Source for a work in source code form is that same work. 88 | 89 | 2. Basic Permissions. 90 | 91 | All rights granted under this License are granted for the term of 92 | copyright on the Program, and are irrevocable provided the stated 93 | conditions are met. This License explicitly affirms your unlimited 94 | permission to run the unmodified Program, subject to section 13. The 95 | output from running a covered work is covered by this License only if the 96 | output, given its content, constitutes a covered work. This License 97 | acknowledges your rights of fair use or other equivalent, as provided by 98 | copyright law. Subject to section 13, you may make, run and propagate 99 | covered works that you do not convey, without conditions so long as your 100 | license otherwise remains in force. You may convey covered works to 101 | others for the sole purpose of having them make modifications exclusively 102 | for you, or provide you with facilities for running those works, provided 103 | that you comply with the terms of this License in conveying all 104 | material for which you do not control copyright. Those thus making or 105 | running the covered works for you must do so exclusively on your 106 | behalf, under your direction and control, on terms that prohibit them 107 | from making any copies of your copyrighted material outside their 108 | relationship with you. 109 | 110 | Conveying under any other circumstances is permitted solely under the 111 | conditions stated below. Sublicensing is not allowed; section 10 makes it 112 | unnecessary. 113 | 114 | 3. Protecting Users' Legal Rights From Anti-Circumvention Law. 115 | 116 | No covered work shall be deemed part of an effective technological 117 | measure under any applicable law fulfilling obligations under article 11 118 | of the WIPO copyright treaty adopted on 20 December 1996, or similar laws 119 | prohibiting or restricting circumvention of such measures. 120 | 121 | When you convey a covered work, you waive any legal power to forbid 122 | circumvention of technological measures to the extent such circumvention is 123 | effected by exercising rights under this License with respect to the 124 | covered work, and you disclaim any intention to limit operation or 125 | modification of the work as a means of enforcing, against the work's users, 126 | your or third parties' legal rights to forbid circumvention of 127 | technological measures. 128 | 129 | 4. Conveying Verbatim Copies. 130 | 131 | You may convey verbatim copies of the Program's source code as you 132 | receive it, in any medium, provided that you conspicuously and 133 | appropriately publish on each copy an appropriate copyright notice; keep 134 | intact all notices stating that this License and any non-permissive terms 135 | added in accord with section 7 apply to the code; keep intact all notices 136 | of the absence of any warranty; and give all recipients a copy of this 137 | License along with the Program. You may charge any price or no price for 138 | each copy that you convey, and you may offer support or warranty 139 | protection for a fee. 140 | 141 | 5. Conveying Modified Source Versions. 142 | 143 | You may convey a work based on the Program, or the modifications to 144 | produce it from the Program, in the form of source code under the terms 145 | of section 4, provided that you also meet all of these conditions: 146 | 147 | a) The work must carry prominent notices stating that you modified it, 148 | and giving a relevant date. 149 | 150 | b) The work must carry prominent notices stating that it is released 151 | under this License and any conditions added under section 7. This 152 | requirement modifies the requirement in section 4 to “keep intact all 153 | notices”. 154 | 155 | c) You must license the entire work, as a whole, under this License to 156 | anyone who comes into possession of a copy. This License will therefore 157 | apply, along with any applicable section 7 additional terms, to the 158 | whole of the work, and all its parts, regardless of how they are 159 | packaged. This License gives no permission to license the work in any 160 | other way, but it does not invalidate such permission if you have 161 | separately received it. 162 | 163 | d) If the work has interactive user interfaces, each must display 164 | Appropriate Legal Notices; however, if the Program has interactive 165 | interfaces that do not display Appropriate Legal Notices, your work 166 | need not make them do so. 167 | 168 | A compilation of a covered work with other separate and independent 169 | works, which are not by their nature extensions of the covered work, and 170 | which are not combined with it such as to form a larger program, in or on 171 | a volume of a storage or distribution medium, is called an “aggregate” if 172 | the compilation and its resulting copyright are not used to limit the 173 | access or legal rights of the compilation's users beyond what the 174 | individual works permit. Inclusion of a covered work in an aggregate does 175 | not cause this License to apply to the other parts of the aggregate. 176 | 177 | 6. Conveying Non-Source Forms. 178 | 179 | You may convey a covered work in object code form under the terms of 180 | sections 4 and 5, provided that you also convey the machine-readable 181 | Corresponding Source under the terms of this License, in one of these 182 | ways: 183 | 184 | a) Convey the object code in, or embodied in, a physical product 185 | (including a physical distribution medium), accompanied by the 186 | Corresponding Source fixed on a durable physical medium customarily 187 | used for software interchange. 188 | 189 | b) Convey the object code in, or embodied in, a physical product 190 | (including a physical distribution medium), accompanied by a written 191 | offer, valid for at least three years and valid for as long as you 192 | offer spare parts or customer support for that product model, to give 193 | anyone who possesses the object code either (1) a copy of the 194 | Corresponding Source for all the software in the product that is 195 | covered by this License, on a durable physical medium customarily used 196 | for software interchange, for a price no more than your reasonable cost 197 | of physically performing this conveying of source, or (2) access to 198 | copy the Corresponding Source from a network server at no charge. 199 | 200 | c) Convey individual copies of the object code with a copy of the 201 | written offer to provide the Corresponding Source. This alternative is 202 | allowed only occasionally and noncommercially, and only if you received 203 | the object code with such an offer, in accord with subsection 6b. 204 | 205 | d) Convey the object code by offering access from a designated place 206 | (gratis or for a charge), and offer equivalent access to the 207 | Corresponding Source in the same way through the same place at no 208 | further charge. You need not require recipients to copy the 209 | Corresponding Source along with the object code. If the place to copy 210 | the object code is a network server, the Corresponding Source may be on 211 | a different server (operated by you or a third party) that supports 212 | equivalent copying facilities, provided you maintain clear directions 213 | next to the object code saying where to find the Corresponding Source. 214 | Regardless of what server hosts the Corresponding Source, you remain 215 | obligated to ensure that it is available for as long as needed to 216 | satisfy these requirements. 217 | 218 | e) Convey the object code using peer-to-peer transmission, provided you 219 | inform other peers where the object code and Corresponding Source of 220 | the work are being offered to the general public at no charge under 221 | subsection 6d. 222 | 223 | A separable portion of the object code, whose source code is excluded 224 | from the Corresponding Source as a System Library, need not be included 225 | in conveying the object code work. 226 | 227 | A “User Product” is either (1) a “consumer product”, which means any 228 | tangible personal property which is normally used for personal, family, 229 | or household purposes, or (2) anything designed or sold for incorporation 230 | into a dwelling. In determining whether a product is a consumer product, 231 | doubtful cases shall be resolved in favor of coverage. For a particular 232 | product received by a particular user, “normally used” refers to a 233 | typical or common use of that class of product, regardless of the status 234 | of the particular user or of the way in which the particular user 235 | actually uses, or expects or is expected to use, the product. A product 236 | is a consumer product regardless of whether the product has substantial 237 | commercial, industrial or non-consumer uses, unless such uses represent 238 | the only significant mode of use of the product. 239 | 240 | “Installation Information” for a User Product means any methods, 241 | procedures, authorization keys, or other information required to install 242 | and execute modified versions of a covered work in that User Product from 243 | a modified version of its Corresponding Source. The information must 244 | suffice to ensure that the continued functioning of the modified object 245 | code is in no case prevented or interfered with solely because 246 | modification has been made. 247 | 248 | If you convey an object code work under this section in, or with, or 249 | specifically for use in, a User Product, and the conveying occurs as part 250 | of a transaction in which the right of possession and use of the User 251 | Product is transferred to the recipient in perpetuity or for a fixed term 252 | (regardless of how the transaction is characterized), the Corresponding 253 | Source conveyed under this section must be accompanied by the 254 | Installation Information. But this requirement does not apply if neither 255 | you nor any third party retains the ability to install modified object 256 | code on the User Product (for example, the work has been installed in 257 | ROM). 258 | 259 | The requirement to provide Installation Information does not include a 260 | requirement to continue to provide support service, warranty, or updates 261 | for a work that has been modified or installed by the recipient, or for 262 | the User Product in which it has been modified or installed. Access 263 | to a network may be denied when the modification itself materially 264 | and adversely affects the operation of the network or violates the 265 | rules and protocols for communication across the network. 266 | 267 | Corresponding Source conveyed, and Installation Information provided, in 268 | accord with this section must be in a format that is publicly documented 269 | (and with an implementation available to the public in source code form), 270 | and must require no special password or key for unpacking, reading or 271 | copying. 272 | 273 | 7. Additional Terms. 274 | 275 | “Additional permissions” are terms that supplement the terms of this 276 | License by making exceptions from one or more of its conditions. 277 | Additional permissions that are applicable to the entire Program shall be 278 | treated as though they were included in this License, to the extent that 279 | they are valid under applicable law. If additional permissions apply only 280 | to part of the Program, that part may be used separately under those 281 | permissions, but the entire Program remains governed by this License 282 | without regard to the additional permissions. When you convey a copy of 283 | a covered work, you may at your option remove any additional permissions 284 | from that copy, or from any part of it. (Additional permissions may be 285 | written to require their own removal in certain cases when you modify the 286 | work.) You may place additional permissions on material, added by you to 287 | a covered work, for which you have or can give appropriate copyright 288 | permission. 289 | 290 | Notwithstanding any other provision of this License, for material you add 291 | to a covered work, you may (if authorized by the copyright holders of 292 | that material) supplement the terms of this License with terms: 293 | 294 | a) Disclaiming warranty or limiting liability differently from the 295 | terms of sections 15 and 16 of this License; or 296 | 297 | b) Requiring preservation of specified reasonable legal notices or 298 | author attributions in that material or in the Appropriate Legal 299 | Notices displayed by works containing it; or 300 | 301 | c) Prohibiting misrepresentation of the origin of that material, or 302 | requiring that modified versions of such material be marked in 303 | reasonable ways as different from the original version; or 304 | 305 | d) Limiting the use for publicity purposes of names of licensors or 306 | authors of the material; or 307 | 308 | e) Declining to grant rights under trademark law for use of some trade 309 | names, trademarks, or service marks; or 310 | 311 | f) Requiring indemnification of licensors and authors of that material 312 | by anyone who conveys the material (or modified versions of it) with 313 | contractual assumptions of liability to the recipient, for any 314 | liability that these contractual assumptions directly impose on those 315 | licensors and authors. 316 | 317 | All other non-permissive additional terms are considered “further 318 | restrictions” within the meaning of section 10. If the Program as you 319 | received it, or any part of it, contains a notice stating that it is 320 | governed by this License along with a term that is a further restriction, 321 | you may remove that term. If a license document contains a further 322 | restriction but permits relicensing or conveying under this License, you 323 | may add to a covered work material governed by the terms of that license 324 | document, provided that the further restriction does not survive such 325 | relicensing or conveying. 326 | 327 | If you add terms to a covered work in accord with this section, you must 328 | place, in the relevant source files, a statement of the additional terms 329 | that apply to those files, or a notice indicating where to find the 330 | applicable terms. Additional terms, permissive or non-permissive, may be 331 | stated in the form of a separately written license, or stated as 332 | exceptions; the above requirements apply either way. 333 | 334 | 8. Termination. 335 | 336 | You may not propagate or modify a covered work except as expressly 337 | provided under this License. Any attempt otherwise to propagate or modify 338 | it is void, and will automatically terminate your rights under this 339 | License (including any patent licenses granted under the third paragraph 340 | of section 11). 341 | 342 | However, if you cease all violation of this License, then your license 343 | from a particular copyright holder is reinstated (a) provisionally, 344 | unless and until the copyright holder explicitly and finally terminates 345 | your license, and (b) permanently, if the copyright holder fails to 346 | notify you of the violation by some reasonable means prior to 60 days 347 | after the cessation. 348 | 349 | Moreover, your license from a particular copyright holder is reinstated 350 | permanently if the copyright holder notifies you of the violation by some 351 | reasonable means, this is the first time you have received notice of 352 | violation of this License (for any work) from that copyright holder, and 353 | you cure the violation prior to 30 days after your receipt of the notice. 354 | 355 | Termination of your rights under this section does not terminate the 356 | licenses of parties who have received copies or rights from you under 357 | this License. If your rights have been terminated and not permanently 358 | reinstated, you do not qualify to receive new licenses for the same 359 | material under section 10. 360 | 361 | 9. Acceptance Not Required for Having Copies. 362 | 363 | You are not required to accept this License in order to receive or run a 364 | copy of the Program. Ancillary propagation of a covered work occurring 365 | solely as a consequence of using peer-to-peer transmission to receive a 366 | copy likewise does not require acceptance. However, nothing other than 367 | this License grants you permission to propagate or modify any covered 368 | work. These actions infringe copyright if you do not accept this License. 369 | Therefore, by modifying or propagating a covered work, you indicate your 370 | acceptance of this License to do so. 371 | 372 | 10. Automatic Licensing of Downstream Recipients. 373 | 374 | Each time you convey a covered work, the recipient automatically receives 375 | a license from the original licensors, to run, modify and propagate that 376 | work, subject to this License. You are not responsible for enforcing 377 | compliance by third parties with this License. 378 | 379 | An “entity transaction” is a transaction transferring control of an 380 | organization, or substantially all assets of one, or subdividing an 381 | organization, or merging organizations. If propagation of a covered work 382 | results from an entity transaction, each party to that transaction who 383 | receives a copy of the work also receives whatever licenses to the work 384 | the party's predecessor in interest had or could give under the previous 385 | paragraph, plus a right to possession of the Corresponding Source of the 386 | work from the predecessor in interest, if the predecessor has it or can 387 | get it with reasonable efforts. 388 | 389 | You may not impose any further restrictions on the exercise of the rights 390 | granted or affirmed under this License. For example, you may not impose a 391 | license fee, royalty, or other charge for exercise of rights granted 392 | under this License, and you may not initiate litigation (including a 393 | cross-claim or counterclaim in a lawsuit) alleging that any patent claim 394 | is infringed by making, using, selling, offering for sale, or importing 395 | the Program or any portion of it. 396 | 397 | 11. Patents. 398 | 399 | A “contributor” is a copyright holder who authorizes use under this 400 | License of the Program or a work on which the Program is based. The work 401 | thus licensed is called the contributor's “contributor version”. 402 | 403 | A contributor's “essential patent claims” are all patent claims owned or 404 | controlled by the contributor, whether already acquired or hereafter 405 | acquired, that would be infringed by some manner, permitted by this 406 | License, of making, using, or selling its contributor version, but do not 407 | include claims that would be infringed only as a consequence of further 408 | modification of the contributor version. For purposes of this definition, 409 | “control” includes the right to grant patent sublicenses in a manner 410 | consistent with the requirements of this License. 411 | 412 | Each contributor grants you a non-exclusive, worldwide, royalty-free 413 | patent license under the contributor's essential patent claims, to make, 414 | use, sell, offer for sale, import and otherwise run, modify and propagate 415 | the contents of its contributor version. 416 | 417 | In the following three paragraphs, a “patent license” is any express 418 | agreement or commitment, however denominated, not to enforce a patent 419 | (such as an express permission to practice a patent or covenant not to 420 | sue for patent infringement). To “grant” such a patent license to a party 421 | means to make such an agreement or commitment not to enforce a patent 422 | against the party. 423 | 424 | If you convey a covered work, knowingly relying on a patent license, and 425 | the Corresponding Source of the work is not available for anyone to copy, 426 | free of charge and under the terms of this License, through a publicly 427 | available network server or other readily accessible means, then you must 428 | either (1) cause the Corresponding Source to be so available, or (2) 429 | arrange to deprive yourself of the benefit of the patent license for this 430 | particular work, or (3) arrange, in a manner consistent with the 431 | requirements of this License, to extend the patent license to downstream 432 | recipients. “Knowingly relying” means you have actual knowledge that, but 433 | for the patent license, your conveying the covered work in a country, or 434 | your recipient's use of the covered work in a country, would infringe 435 | one or more identifiable patents in that country that you have reason 436 | to believe are valid. 437 | 438 | If, pursuant to or in connection with a single transaction or 439 | arrangement, you convey, or propagate by procuring conveyance of, a 440 | covered work, and grant a patent license to some of the parties receiving 441 | the covered work authorizing them to use, propagate, modify or convey a 442 | specific copy of the covered work, then the patent license you grant is 443 | automatically extended to all recipients of the covered work and works 444 | based on it. 445 | 446 | A patent license is “discriminatory” if it does not include within the 447 | scope of its coverage, prohibits the exercise of, or is conditioned on 448 | the non-exercise of one or more of the rights that are specifically 449 | granted under this License. You may not convey a covered work if you are 450 | a party to an arrangement with a third party that is in the business of 451 | distributing software, under which you make payment to the third party 452 | based on the extent of your activity of conveying the work, and under 453 | which the third party grants, to any of the parties who would receive the 454 | covered work from you, a discriminatory patent license (a) in connection 455 | with copies of the covered work conveyed by you (or copies made from 456 | those copies), or (b) primarily for and in connection with specific 457 | products or compilations that contain the covered work, unless you 458 | entered into that arrangement, or that patent license was granted, prior 459 | to 28 March 2007. 460 | 461 | Nothing in this License shall be construed as excluding or limiting any 462 | implied license or other defenses to infringement that may otherwise be 463 | available to you under applicable patent law. 464 | 465 | 12. No Surrender of Others' Freedom. 466 | 467 | If conditions are imposed on you (whether by court order, agreement or 468 | otherwise) that contradict the conditions of this License, they do not 469 | excuse you from the conditions of this License. If you cannot use, 470 | propagate or convey a covered work so as to satisfy simultaneously your 471 | obligations under this License and any other pertinent obligations, then 472 | as a consequence you may not use, propagate or convey it at all. For 473 | example, if you agree to terms that obligate you to collect a royalty for 474 | further conveying from those to whom you convey the Program, the only way 475 | you could satisfy both those terms and this License would be to refrain 476 | entirely from conveying the Program. 477 | 478 | 13. Offering the Program as a Service. 479 | 480 | If you make the functionality of the Program or a modified version 481 | available to third parties as a service, you must make the Service Source 482 | Code available via network download to everyone at no charge, under the 483 | terms of this License. Making the functionality of the Program or 484 | modified version available to third parties as a service includes, 485 | without limitation, enabling third parties to interact with the 486 | functionality of the Program or modified version remotely through a 487 | computer network, offering a service the value of which entirely or 488 | primarily derives from the value of the Program or modified version, or 489 | offering a service that accomplishes for users the primary purpose of the 490 | Program or modified version. 491 | 492 | “Service Source Code” means the Corresponding Source for the Program or 493 | the modified version, and the Corresponding Source for all programs that 494 | you use to make the Program or modified version available as a service, 495 | including, without limitation, management software, user interfaces, 496 | application program interfaces, automation software, monitoring software, 497 | backup software, storage software and hosting software, all such that a 498 | user could run an instance of the service using the Service Source Code 499 | you make available. 500 | 501 | 14. Revised Versions of this License. 502 | 503 | Redis, Inc. may publish revised and/or new versions of the Server Side 504 | Public License from time to time. Such new versions will be similar in 505 | spirit to the present version, but may differ in detail to address new 506 | problems or concerns. 507 | 508 | Each version is given a distinguishing version number. If the Program 509 | specifies that a certain numbered version of the Server Side Public 510 | License “or any later version” applies to it, you have the option of 511 | following the terms and conditions either of that numbered version or of 512 | any later version published by Redis, Inc. If the Program does not 513 | specify a version number of the Server Side Public License, you may 514 | choose any version ever published by Redis, Inc. 515 | 516 | If the Program specifies that a proxy can decide which future versions of 517 | the Server Side Public License can be used, that proxy's public statement 518 | of acceptance of a version permanently authorizes you to choose that 519 | version for the Program. 520 | 521 | Later license versions may give you additional or different permissions. 522 | However, no additional obligations are imposed on any author or copyright 523 | holder as a result of your choosing to follow a later version. 524 | 525 | 15. Disclaimer of Warranty. 526 | 527 | THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY 528 | APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT 529 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY 530 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, 531 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 532 | PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM 533 | IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF 534 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 535 | 536 | 16. Limitation of Liability. 537 | 538 | IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 539 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS 540 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING 541 | ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF 542 | THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO 543 | LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU 544 | OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 545 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 546 | POSSIBILITY OF SUCH DAMAGES. 547 | 548 | 17. Interpretation of Sections 15 and 16. 549 | 550 | If the disclaimer of warranty and limitation of liability provided above 551 | cannot be given local legal effect according to their terms, reviewing 552 | courts shall apply local law that most closely approximates an absolute 553 | waiver of all civil liability in connection with the Program, unless a 554 | warranty or assumption of liability accompanies a copy of the Program in 555 | return for a fee. 556 | 557 | END OF TERMS AND CONDITIONS --------------------------------------------------------------------------------