├── .tool-versions ├── .gitignore ├── jestSetup.js ├── jest.config.js ├── babel.config.js ├── rollup.config.js ├── .eslintrc ├── .circleci └── config.yml ├── createSearchIndex.js ├── LICENSE.txt ├── index.test.js ├── index.js ├── template.js ├── package.json └── README.md /.tool-versions: -------------------------------------------------------------------------------- 1 | nodejs 12.14.1 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist/ 2 | node_modules/ 3 | yarn-error.log 4 | -------------------------------------------------------------------------------- /jestSetup.js: -------------------------------------------------------------------------------- 1 | import "@testing-library/jest-dom"; 2 | 3 | global.fetch = require("jest-fetch-mock"); 4 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | setupFilesAfterEnv: ["/jestSetup.js"], 3 | transform: { 4 | "^.+\\.js?$": "babel-jest", 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | presets: ["@babel/preset-env"], 3 | env: { 4 | test: { 5 | presets: ["@babel/preset-env", "@babel/preset-react"], 6 | plugins: ["@babel/plugin-transform-runtime"], 7 | }, 8 | }, 9 | }; 10 | -------------------------------------------------------------------------------- /rollup.config.js: -------------------------------------------------------------------------------- 1 | import babel from "rollup-plugin-babel"; 2 | import pkg from "./package.json"; 3 | 4 | const config = { 5 | input: "index.js", 6 | output: [ 7 | { 8 | file: pkg.main, 9 | format: "cjs", 10 | }, 11 | { 12 | file: pkg.module, 13 | format: "esm", 14 | }, 15 | ], 16 | plugins: [babel()], 17 | external: ["react", "use-debounce"], 18 | }; 19 | 20 | export default config; 21 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true 4 | }, 5 | "extends": [ 6 | "standard", 7 | "standard-react", 8 | "prettier/standard", 9 | "plugin:prettier/recommended" 10 | ], 11 | "overrides": [ 12 | { 13 | "files": [ 14 | "**/*.test.js" 15 | ], 16 | "env": { 17 | "jest": true 18 | } 19 | } 20 | ], 21 | "parser": "babel-eslint", 22 | "plugins": ["react-hooks"], 23 | "rules": { 24 | "react-hooks/rules-of-hooks": "error", 25 | "react-hooks/exhaustive-deps": "error" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | test: 4 | working_directory: ~/netlify-flexsearch 5 | docker: 6 | - image: circleci/node:12-stretch 7 | steps: 8 | - checkout 9 | - restore_cache: 10 | key: dependency-cache-{{ checksum "yarn.lock" }} 11 | - run: 12 | name: Install dependencies 13 | command: yarn install 14 | - save_cache: 15 | key: dependency-cache-{{ checksum "yarn.lock" }} 16 | paths: 17 | - ~/.cache/yarn 18 | - ./node_modules 19 | - run: 20 | name: ESLint 21 | command: yarn lint 22 | - run: 23 | name: Jest 24 | command: yarn test 25 | 26 | workflows: 27 | version: 2 28 | test: 29 | jobs: 30 | - test 31 | -------------------------------------------------------------------------------- /createSearchIndex.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const path = require("path"); 3 | 4 | const capitalize = (str) => { 5 | return str.charAt(0).toUpperCase() + str.slice(1); 6 | }; 7 | 8 | module.exports = ({ index, data, functionsPath, flexSearchOptions }) => { 9 | const template = fs 10 | .readFileSync(path.join(__dirname, "template.js")) 11 | .toString(); 12 | 13 | let populated = template.replace( 14 | "const searchData = {};", 15 | `const searchData = ${JSON.stringify(data)};` 16 | ); 17 | 18 | if (flexSearchOptions) { 19 | populated = populated.replace( 20 | "const flexSearchOptions = {};", 21 | `const flexSearchOptions = ${JSON.stringify(flexSearchOptions)};` 22 | ); 23 | } 24 | 25 | fs.writeFileSync( 26 | path.join(process.cwd(), functionsPath, `search${capitalize(index)}.js`), 27 | populated 28 | ); 29 | }; 30 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2020 Aha! Labs Inc 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /index.test.js: -------------------------------------------------------------------------------- 1 | import React from "react"; 2 | import { render, fireEvent, screen, waitFor } from "@testing-library/react"; 3 | import { useSearch } from "./index"; 4 | 5 | describe("useSearch", () => { 6 | const Search = () => { 7 | const [searchTerm, setSearchTerm, results, loading, error] = useSearch( 8 | "blog" 9 | ); 10 | 11 | return ( 12 |
13 | setSearchTerm(e.target.value)} 16 | placeholder="Search" 17 | /> 18 | 19 | {loading && "Loading..."} 20 | {error && `Oh no! Something went wrong.`} 21 | {!(loading || error) && ( 22 | 29 | )} 30 |
31 | ); 32 | }; 33 | 34 | it("returns search results", async () => { 35 | global.fetch.once( 36 | JSON.stringify({ 37 | results: [{ id: "123", name: "Lorem ipsum" }], 38 | }) 39 | ); 40 | 41 | render(); 42 | fireEvent.change(screen.getByPlaceholderText("Search"), { 43 | target: { value: "Lorem" }, 44 | }); 45 | await waitFor(() => expect(screen.getByText("Lorem ipsum"))); 46 | }); 47 | }); 48 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | import { useEffect, useReducer, useState } from "react"; 2 | import { useDebounce } from "use-debounce"; 3 | 4 | const capitalize = (str) => { 5 | return str.charAt(0).toUpperCase() + str.slice(1); 6 | }; 7 | 8 | export const search = ({ index, term, excerpt, limit = 25 }) => { 9 | let searchUrl = `/.netlify/functions/search${capitalize(index)}?term=${term}`; 10 | if (limit) searchUrl = `${searchUrl}&limit=${limit}`; 11 | if (excerpt) searchUrl = `${searchUrl}&excerpt=${excerpt}`; 12 | return fetch(searchUrl).then((response) => response.json()); 13 | }; 14 | 15 | const reducer = (state, action) => { 16 | switch (action.type) { 17 | case "startSearch": 18 | return { loading: true, error: state.error, results: state.results }; 19 | case "endSearch": 20 | return { loading: false, error: action.error, results: action.results }; 21 | default: 22 | throw new Error(); 23 | } 24 | }; 25 | 26 | export const useSearch = ( 27 | index, 28 | { defaultSearchTerm = "", debounce = 250, limit = 25, excerpt = false } = {} 29 | ) => { 30 | const [searchTerm, setSearchTerm] = useState(defaultSearchTerm); 31 | const initialState = { 32 | loading: Boolean(defaultSearchTerm && defaultSearchTerm !== ""), 33 | error: null, 34 | results: [], 35 | }; 36 | const [{ loading, results, error }, dispatch] = useReducer( 37 | reducer, 38 | initialState 39 | ); 40 | const [term] = useDebounce(searchTerm, debounce); 41 | 42 | useEffect(() => { 43 | if (term && term !== "") { 44 | dispatch({ type: "startSearch" }); 45 | search({ index, term, limit, excerpt }) 46 | .then((response) => 47 | dispatch({ 48 | type: "endSearch", 49 | results: response.results, 50 | error: null, 51 | }) 52 | ) 53 | .catch((error) => dispatch({ type: "endSearch", results: [], error })); 54 | } 55 | }, [index, term, limit, excerpt]); 56 | 57 | return [searchTerm, setSearchTerm, results, loading, error]; 58 | }; 59 | -------------------------------------------------------------------------------- /template.js: -------------------------------------------------------------------------------- 1 | const FlexSearch = require("flexsearch"); 2 | 3 | // This will be populated with search data by the build script. 4 | const searchData = {}; 5 | const flexSearchOptions = {}; 6 | 7 | const index = FlexSearch.create(flexSearchOptions); 8 | 9 | Object.entries(searchData).forEach(([id, item]) => { 10 | index.add(id, item.text); 11 | }); 12 | 13 | exports.handler = async function ({ 14 | queryStringParameters: { limit, term, excerpt }, 15 | }) { 16 | const tokenizeStrategy = flexSearchOptions.tokenize || "strict"; 17 | 18 | const results = index.search(term, limit).map((id) => { 19 | const response = searchData[id].response; 20 | 21 | // If the excerpt param is set, find the match and the 22 | // n closest words in both directions. 23 | const excerptNum = parseInt(excerpt); 24 | if (!isNaN(excerptNum)) { 25 | const fullText = searchData[id].text.split(/\s/); 26 | const index = fullText.findIndex((word) => { 27 | const cleanWord = word.replace(/[^A-Za-z0-9\s]/g, "").toLowerCase(); 28 | const cleanTerm = term.toLowerCase(); 29 | 30 | switch (tokenizeStrategy) { 31 | case "forward": 32 | return cleanWord.startsWith(cleanTerm); 33 | case "reverse": 34 | return cleanWord.endsWith(cleanTerm); 35 | case "full": 36 | return cleanWord.includes(cleanTerm); 37 | case "strict": 38 | default: 39 | return cleanWord === cleanTerm; 40 | } 41 | }); 42 | if (index > -1) { 43 | const excerptStartIdx = Math.max(index - excerptNum, 0); 44 | const excerptEndIdx = Math.min( 45 | index + excerptNum + 1, 46 | fullText.length - 1 47 | ); 48 | 49 | response.excerpt = { 50 | text: fullText.slice(excerptStartIdx, excerptEndIdx).join(" "), 51 | matchIndex: index - excerptStartIdx, 52 | moreBefore: index - excerptNum > 0, 53 | moreAfter: index + excerptNum + 1 < fullText.length - 1, 54 | }; 55 | } 56 | } 57 | 58 | return response; 59 | }); 60 | 61 | return { 62 | statusCode: 200, 63 | body: JSON.stringify({ 64 | results, 65 | }), 66 | }; 67 | }; 68 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@aha-app/netlify-flexsearch", 3 | "version": "0.1.9", 4 | "description": "Implement search for static sites on Netlify, powered by FlexSearch and Netlify functions", 5 | "author": "Zach Schneider ", 6 | "license": "MIT", 7 | "main": "dist/index.js", 8 | "module": "dist/index.es.js", 9 | "keywords": [ 10 | "netlify", 11 | "search", 12 | "flexsearch" 13 | ], 14 | "repository": { 15 | "type": "git", 16 | "url": "git+https://github.com/aha-app/netlify-flexsearch.git" 17 | }, 18 | "bugs": { 19 | "url": "https://github.com/aha-app/netlify-flexsearch/issues" 20 | }, 21 | "homepage": "https://github.com/aha-app/netlify-flexsearch#readme", 22 | "files": [ 23 | "dist/", 24 | "createSearchIndex.js", 25 | "index.js", 26 | "template.js" 27 | ], 28 | "scripts": { 29 | "build": "rollup -c rollup.config.js", 30 | "format": "prettier --write '*.js'", 31 | "lint": "eslint '*.js'", 32 | "test": "jest" 33 | }, 34 | "dependencies": { 35 | "use-debounce": "^3.4.0" 36 | }, 37 | "peerDependencies": { 38 | "flexsearch": ">= 0.6.0", 39 | "react": ">= 16.8.0" 40 | }, 41 | "devDependencies": { 42 | "@babel/core": "^7.9.0", 43 | "@babel/plugin-transform-runtime": "^7.9.0", 44 | "@babel/preset-env": "^7.9.0", 45 | "@babel/preset-react": "^7.9.4", 46 | "@testing-library/jest-dom": "^5.3.0", 47 | "@testing-library/react": "^10.0.2", 48 | "babel-eslint": "^10.1.0", 49 | "babel-jest": "^25.2.6", 50 | "eslint": "^6.8.0", 51 | "eslint-config-prettier": "^6.10.1", 52 | "eslint-config-standard": "^14.1.1", 53 | "eslint-config-standard-react": "^9.2.0", 54 | "eslint-plugin-import": "^2.20.2", 55 | "eslint-plugin-node": "^11.1.0", 56 | "eslint-plugin-prettier": "^3.1.2", 57 | "eslint-plugin-promise": "^4.2.1", 58 | "eslint-plugin-react": "^7.19.0", 59 | "eslint-plugin-react-hooks": "^3.0.0", 60 | "eslint-plugin-standard": "^4.0.1", 61 | "flexsearch": "^0.6.32", 62 | "jest": "^25.2.6", 63 | "jest-fetch-mock": "^3.0.3", 64 | "prettier": "^2.0.2", 65 | "react": "^16.13.1", 66 | "react-dom": "^16.13.1", 67 | "rollup": "^2.3.2", 68 | "rollup-plugin-babel": "^4.4.0" 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CircleCI](https://circleci.com/gh/aha-app/netlify-flexsearch.svg?style=shield)](https://circleci.com/gh/aha-app/netlify-flexsearch) 2 | 3 | # netlify-flexsearch 4 | 5 | netlify-flexsearch provides a set of tools for implementing search for static sites hosted on Netlify, powered by [FlexSearch](https://github.com/nextapps-de/flexsearch) and [Netlify functions](https://www.netlify.com/products/functions). 6 | 7 | ## When do I need this? 8 | 9 | You might find this library useful if: 10 | 11 | 1. You are building a static site on Netlify and you would like to implement search. 12 | 2. You need to index a large enough quantity of data that building an in-browser search index will cause performance problems. 13 | 3. You don't want to add a third-party service such as Algolia. 14 | 15 | ## How does it work? 16 | 17 | [Netlify functions](https://www.netlify.com/products/functions) are serverless functions that Netlify automatically detects in your build and serves as APIs. This library helps you generate one or more Netlify functions during your build process which each contain a FlexSearch index. You can then make asynchronous requests to these functions to perform searches; the package also includes helper functions for making search requests. 18 | 19 | ## Installation 20 | 21 | ``` 22 | npm install flexsearch @aha-app/netlify-flexsearch 23 | ``` 24 | 25 | or 26 | 27 | ``` 28 | yarn add flexsearch @aha-app/netlify-flexsearch 29 | ``` 30 | 31 | ## Usage 32 | 33 | 1. Create a build script which invokes `createSearchIndex` one or more times. `createSearchIndex` accepts one object argument with the following options: 34 | 35 | | Name | Type | Description | 36 | | --- | --- | --- | 37 | | `index` (required) | `string` | Unique name for the search index (this determines its API path) | 38 | | `data` (required) | `object` | Search data to be indexed (see example below for the required format) | 39 | | `functionsPath` (required) | `string` | Directory where you have configured Netlify functions (relative to the root directory of your application) | 40 | | `flexSearchOptions` (optional) | `object` | Additional options to be passed to FlexSearch when creating the index (see the [FlexSearch documentation](https://github.com/nextapps-de/flexsearch#flexsearch.create) for available options) | 41 | 42 | ```javascript 43 | const createSearchIndex = require('@aha-app/netlify-flexsearch/createSearchIndex'); 44 | 45 | // Data must take on this exact format: an object where each key 46 | // is a unique ID and the value is an object containing `text` 47 | // (the text to be indexed for search) and `response` (the data that 48 | // should be returned when this record is found via search). 49 | const blogPosts = { 50 | "hello-world": { 51 | text: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque augue odio, accumsan eu turpis et, fermentum pellentesque justo.", 52 | response: { 53 | slug: "hello-world", 54 | title: "Hello World" 55 | } 56 | }, 57 | "second-post": { 58 | text: "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque augue odio, accumsan eu turpis et, fermentum pellentesque justo.", 59 | response: { 60 | slug: "second-post", 61 | title: "Second Post" 62 | } 63 | } 64 | } 65 | 66 | createSearchIndex({ 67 | index: 'blog', 68 | data: blogPosts, 69 | functionsPath: 'src/functions' 70 | }); 71 | ``` 72 | 73 | 2. You should add a `.gitignore` entry to prevent the output of this script from being checked into git. If your Netlify functions folder is `src/functions`, the `.gitignore` entry might be `src/functions/search*` (all output filenames will begin with `search`). 74 | 75 | 3. Modify your Netlify build command to invoke the script defined in step 1 in addition to your regular build. For example, if your build command is currently `gatsby build` and you named the script from step ` createSearch.js`, you might change the build command to `node createSearch.js && gatsby build`. 76 | 77 | 4. Import and use one of the helper functions for searching. The package provides both a React hook and a basic asynchronous function. 78 | 79 | ```javascript 80 | // Search.js (React-based usage) 81 | import { useSearch } from '@aha-app/netlify-flexsearch'; 82 | 83 | const Search = () => { 84 | // useSearch accepts two arguments: 85 | // 86 | // The first (required) is the string name of the index to 87 | // search -- this must match the index name provided when 88 | // building the inddex 89 | // 90 | // The second (optional) is an object of additional options: 91 | // - debounce (default 250) 92 | // - defaultSearchTerm (prefill the search with a term) 93 | // - limit (default 25; limit the number of results that are returned) 94 | // - excerpt (default none; will populate the response with the n words surrounding the first match, if possible) 95 | const [searchTerm, setSearchTerm, results, loading, error] = useSearch('blog', { debounce: 300 }); 96 | 97 | return ( 98 |
99 | setSearchTerm(e.target.value)} 103 | placeholder="Search..." 104 | /> 105 | 106 |

Search results

107 | 108 | {loading && "Loading..."} 109 | {error && `Oh no! Something went wrong.`} 110 | {!(loading || error) && ( 111 |
    112 | {results.map((result) => ( 113 | // Result data here matches what is provided in the 114 | // `response` object when building the index. 115 |
  • 116 | {result.name} 117 |

    118 | {result.excerpt.moreBefore && '...'} 119 | {result.excerpt.text} 120 | {result.excerpt.moreAfter && '...'} 121 |

    122 |
    • 123 | ))} 124 |
125 | )} 126 |
127 | ) 128 | } 129 | ``` 130 | 131 | ```javascript 132 | // Non-React helper 133 | import { search } from '@aha-app/netlify-flexsearch'; 134 | 135 | search({ index: 'blog', term: 'hello', limit: 10 }).then(response => { 136 | // Do something with the results 137 | console.log(response.results); 138 | }) 139 | ``` 140 | 141 | ## Caveats 142 | 143 | * By default, Netlify functions run with 1024MB of RAM, so if your search index is too large you will see a build error. 144 | * This library has also not been tested with search indexes larger than a few MB; you might see slow/unacceptable search performance if your search index is hundreds of MB, even if it's under the 1024MB limit. 145 | 146 | ## License 147 | 148 | MIT 149 | --------------------------------------------------------------------------------