├── .editorconfig ├── .eslintrc.json ├── .gitattributes ├── .gitignore ├── .travis.yml ├── .verb.md ├── LICENSE ├── README.md ├── contributing.md ├── example ├── config.example.json ├── config.js ├── docs │ ├── about.html │ └── index.html ├── index.js ├── indexer-algolia.js ├── indexer-lunr.js ├── lunr-search.json ├── search-algolia.js └── search-lunr.js ├── index.js ├── lib └── utils.js ├── package.json └── test └── test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | 10 | [*.md] 11 | trim_trailing_whitespace = false 12 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "ecmaFeatures": { 3 | "modules": true, 4 | "experimentalObjectRestSpread": true 5 | }, 6 | "env": { 7 | "browser": false, 8 | "es6": true, 9 | "node": true, 10 | "mocha": true 11 | }, 12 | "globals": { 13 | "document": false, 14 | "navigator": false, 15 | "window": false 16 | }, 17 | "rules": { 18 | "accessor-pairs": 2, 19 | "arrow-spacing": [ 20 | 2, 21 | { 22 | "before": true, 23 | "after": true 24 | } 25 | ], 26 | "block-spacing": [ 27 | 2, 28 | "always" 29 | ], 30 | "brace-style": [ 31 | 2, 32 | "1tbs", 33 | { 34 | "allowSingleLine": true 35 | } 36 | ], 37 | "comma-dangle": [ 38 | 2, 39 | "never" 40 | ], 41 | "comma-spacing": [ 42 | 2, 43 | { 44 | "before": false, 45 | "after": true 46 | } 47 | ], 48 | "comma-style": [ 49 | 2, 50 | "last" 51 | ], 52 | "constructor-super": 2, 53 | "curly": [ 54 | 2, 55 | "multi-line" 56 | ], 57 | "dot-location": [ 58 | 2, 59 | "property" 60 | ], 61 | "eol-last": 2, 62 | "eqeqeq": [ 63 | 2, 64 | "allow-null" 65 | ], 66 | "generator-star-spacing": [ 67 | 2, 68 | { 69 | "before": true, 70 | "after": true 71 | } 72 | ], 73 | "handle-callback-err": [ 74 | 2, 75 | "^(err|error)$" 76 | ], 77 | "indent": [ 78 | 2, 79 | 2, 80 | { 81 | "SwitchCase": 1 82 | } 83 | ], 84 | "key-spacing": [ 85 | 2, 86 | { 87 | "beforeColon": false, 88 | "afterColon": true 89 | } 90 | ], 91 | "keyword-spacing": [ 92 | 2, 93 | { 94 | "before": true, 95 | "after": true 96 | } 97 | ], 98 | "new-cap": [ 99 | 2, 100 | { 101 | "newIsCap": true, 102 | "capIsNew": false 103 | } 104 | ], 105 | "new-parens": 2, 106 | "no-array-constructor": 2, 107 | "no-caller": 2, 108 | "no-class-assign": 2, 109 | "no-cond-assign": 2, 110 | "no-const-assign": 2, 111 | "no-control-regex": 2, 112 | "no-debugger": 2, 113 | "no-delete-var": 2, 114 | "no-dupe-args": 2, 115 | "no-dupe-class-members": 2, 116 | "no-dupe-keys": 2, 117 | "no-duplicate-case": 2, 118 | "no-empty-character-class": 2, 119 | "no-eval": 2, 120 | "no-ex-assign": 2, 121 | "no-extend-native": 2, 122 | "no-extra-bind": 2, 123 | "no-extra-boolean-cast": 2, 124 | "no-extra-parens": [ 125 | 2, 126 | "functions" 127 | ], 128 | "no-fallthrough": 2, 129 | "no-floating-decimal": 2, 130 | "no-func-assign": 2, 131 | "no-implied-eval": 2, 132 | "no-inner-declarations": [ 133 | 2, 134 | "functions" 135 | ], 136 | "no-invalid-regexp": 2, 137 | "no-irregular-whitespace": 2, 138 | "no-iterator": 2, 139 | "no-label-var": 2, 140 | "no-labels": 2, 141 | "no-lone-blocks": 2, 142 | "no-mixed-spaces-and-tabs": 2, 143 | "no-multi-spaces": 2, 144 | "no-multi-str": 2, 145 | "no-multiple-empty-lines": [ 146 | 2, 147 | { 148 | "max": 1 149 | } 150 | ], 151 | "no-native-reassign": 0, 152 | "no-negated-in-lhs": 2, 153 | "no-new": 2, 154 | "no-new-func": 2, 155 | "no-new-object": 2, 156 | "no-new-require": 2, 157 | "no-new-wrappers": 2, 158 | "no-obj-calls": 2, 159 | "no-octal": 2, 160 | "no-octal-escape": 2, 161 | "no-proto": 0, 162 | "no-redeclare": 2, 163 | "no-regex-spaces": 2, 164 | "no-return-assign": 2, 165 | "no-self-compare": 2, 166 | "no-sequences": 2, 167 | "no-shadow-restricted-names": 2, 168 | "no-spaced-func": 2, 169 | "no-sparse-arrays": 2, 170 | "no-this-before-super": 2, 171 | "no-throw-literal": 2, 172 | "no-trailing-spaces": 0, 173 | "no-undef": 2, 174 | "no-undef-init": 2, 175 | "no-unexpected-multiline": 2, 176 | "no-unneeded-ternary": [ 177 | 2, 178 | { 179 | "defaultAssignment": false 180 | } 181 | ], 182 | "no-unreachable": 2, 183 | "no-unused-vars": [ 184 | 2, 185 | { 186 | "vars": "all", 187 | "args": "none" 188 | } 189 | ], 190 | "no-useless-call": 0, 191 | "no-with": 2, 192 | "one-var": [ 193 | 0, 194 | { 195 | "initialized": "never" 196 | } 197 | ], 198 | "operator-linebreak": [ 199 | 0, 200 | "after", 201 | { 202 | "overrides": { 203 | "?": "before", 204 | ":": "before" 205 | } 206 | } 207 | ], 208 | "padded-blocks": [ 209 | 0, 210 | "never" 211 | ], 212 | "quotes": [ 213 | 2, 214 | "single", 215 | "avoid-escape" 216 | ], 217 | "radix": 2, 218 | "semi": [ 219 | 2, 220 | "always" 221 | ], 222 | "semi-spacing": [ 223 | 2, 224 | { 225 | "before": false, 226 | "after": true 227 | } 228 | ], 229 | "space-before-blocks": [ 230 | 2, 231 | "always" 232 | ], 233 | "space-before-function-paren": [ 234 | 2, 235 | "never" 236 | ], 237 | "space-in-parens": [ 238 | 2, 239 | "never" 240 | ], 241 | "space-infix-ops": 2, 242 | "space-unary-ops": [ 243 | 2, 244 | { 245 | "words": true, 246 | "nonwords": false 247 | } 248 | ], 249 | "spaced-comment": [ 250 | 0, 251 | "always", 252 | { 253 | "markers": [ 254 | "global", 255 | "globals", 256 | "eslint", 257 | "eslint-disable", 258 | "*package", 259 | "!", 260 | "," 261 | ] 262 | } 263 | ], 264 | "use-isnan": 2, 265 | "valid-typeof": 2, 266 | "wrap-iife": [ 267 | 2, 268 | "any" 269 | ], 270 | "yoda": [ 271 | 2, 272 | "never" 273 | ] 274 | } 275 | } 276 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | tmp 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: node_js 3 | node_js: 4 | - 'node' 5 | - '6' 6 | - '5' 7 | - '4' 8 | -------------------------------------------------------------------------------- /.verb.md: -------------------------------------------------------------------------------- 1 | ## Usage 2 | 3 | ```js 4 | var search = require('{%= name %}'); 5 | ``` 6 | 7 | ## API 8 | {%= apidocs('index.js') %} 9 | 10 | ### Indexers 11 | 12 | Indexers are objects that have `collect` and `index` methods that will be executed when [collect](#collect) or [index](#index) are called on [app.search](#search). 13 | 14 | The indexer objects may be plain objects or instances created with those methods. See the [examples](examples) to see what indexers may look like. 15 | 16 | Simple object to be used in examples below. 17 | 18 | ```js 19 | var indexer = {}; 20 | ``` 21 | 22 | #### .collect 23 | 24 | The collect method on an indexer will be passed a `file` object and a `next` callback. The collect method 25 | should create an object to pass back to `next` that will be added to the `.files` collection on the `search` instance. 26 | 27 | If `file` is a view from [assemble][], we can collect information about the file that we want to index: 28 | 29 | ```js 30 | indexer.collect = function(file, next) { 31 | var obj = { 32 | key: file.key, 33 | title: file.data.title, 34 | category: file.data.category, 35 | url: file.data.url, 36 | body: file.content 37 | }; 38 | // return the object 39 | next(null, obj); 40 | }; 41 | ``` 42 | 43 | #### .index 44 | 45 | The index method on an indexer will be passed a `files` object containing all fo the collected files, an `options` object which is the same as the `options` passed into the [search.index](#index) method, and a callback function to call when indexing is complete. The callback function is the same as the one passed into the [search.index](#index) method so users may choose to return additional information if necessary. 46 | 47 | ```js 48 | indexer.index = function(files, options, cb) { 49 | for (var key in files) { 50 | if (files.hasOwnProperty(key)) { 51 | console.log(key); 52 | console.log(files[key]); 53 | console.log(); 54 | } 55 | } 56 | cb(); 57 | }; 58 | ``` 59 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Brian Woodward 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # base-search [![NPM version](https://img.shields.io/npm/v/base-search.svg?style=flat)](https://www.npmjs.com/package/base-search) [![NPM monthly downloads](https://img.shields.io/npm/dm/base-search.svg?style=flat)](https://npmjs.org/package/base-search) [![NPM total downloads](https://img.shields.io/npm/dt/base-search.svg?style=flat)](https://npmjs.org/package/base-search) [![Linux Build Status](https://img.shields.io/travis/node-base/base-search.svg?style=flat&label=Travis)](https://travis-ci.org/node-base/base-search) 2 | 3 | > Base plugin that adds methods for creating, updating and using search indices. 4 | 5 | ## Install 6 | 7 | Install with [npm](https://www.npmjs.com/): 8 | 9 | ```sh 10 | $ npm install --save base-search 11 | ``` 12 | 13 | ## Usage 14 | 15 | ```js 16 | var search = require('base-search'); 17 | ``` 18 | 19 | ## API 20 | 21 | ### [.search](index.js#L20) 22 | 23 | Plugin for [base](https://github.com/node-base/base) applications like [generate](https://github.com/generate/generate), [assemble](https://github.com/assemble/assemble), [verb](https://github.com/verbose/verb), and [update](https://github.com/update/update) to add an instance of [sarge](https://github.com/doowb/sarge) as `app.search` that has methods for creating search indices using [indexers](#indexers). 24 | 25 | **Params** 26 | 27 | * `config` **{Object}**: Configuration object used to specify default indexer to use. 28 | * `returns` **{Function}**: Plugin function passed to `app.use` methods. 29 | 30 | **Example** 31 | 32 | ```js 33 | var app = assemble(); 34 | app.use(search()); 35 | console.log(app.search); 36 | ``` 37 | 38 | ### Indexers 39 | 40 | Indexers are objects that have `collect` and `index` methods that will be executed when [collect](#collect) or [index](#index) are called on [app.search](#search). 41 | 42 | The indexer objects may be plain objects or instances created with those methods. See the [examples](examples) to see what indexers may look like. 43 | 44 | Simple object to be used in examples below. 45 | 46 | ```js 47 | var indexer = {}; 48 | ``` 49 | 50 | #### .collect 51 | 52 | The collect method on an indexer will be passed a `file` object and a `next` callback. The collect method 53 | should create an object to pass back to `next` that will be added to the `.files` collection on the `search` instance. 54 | 55 | If `file` is a view from [assemble](https://github.com/assemble/assemble), we can collect information about the file that we want to index: 56 | 57 | ```js 58 | indexer.collect = function(file, next) { 59 | var obj = { 60 | key: file.key, 61 | title: file.data.title, 62 | category: file.data.category, 63 | url: file.data.url, 64 | body: file.content 65 | }; 66 | // return the object 67 | next(null, obj); 68 | }; 69 | ``` 70 | 71 | #### .index 72 | 73 | The index method on an indexer will be passed a `files` object containing all fo the collected files, an `options` object which is the same as the `options` passed into the [search.index](#index) method, and a callback function to call when indexing is complete. The callback function is the same as the one passed into the [search.index](#index) method so users may choose to return additional information if necessary. 74 | 75 | ```js 76 | indexer.index = function(files, options, cb) { 77 | for (var key in files) { 78 | if (files.hasOwnProperty(key)) { 79 | console.log(key); 80 | console.log(files[key]); 81 | console.log(); 82 | } 83 | } 84 | cb(); 85 | }; 86 | ``` 87 | 88 | ## About 89 | 90 | ### Related projects 91 | 92 | * [assemble](https://www.npmjs.com/package/assemble): Get the rocks out of your socks! Assemble makes you fast at creating web projects… [more](https://github.com/assemble/assemble) | [homepage](https://github.com/assemble/assemble "Get the rocks out of your socks! Assemble makes you fast at creating web projects. Assemble is used by thousands of projects for rapid prototyping, creating themes, scaffolds, boilerplates, e-books, UI components, API documentation, blogs, building websit") 93 | * [base](https://www.npmjs.com/package/base): base is the foundation for creating modular, unit testable and highly pluggable node.js applications, starting… [more](https://github.com/node-base/base) | [homepage](https://github.com/node-base/base "base is the foundation for creating modular, unit testable and highly pluggable node.js applications, starting with a handful of common methods, like `set`, `get`, `del` and `use`.") 94 | * [generate](https://www.npmjs.com/package/generate): Command line tool and developer framework for scaffolding out new GitHub projects. Generate offers the… [more](https://github.com/generate/generate) | [homepage](https://github.com/generate/generate "Command line tool and developer framework for scaffolding out new GitHub projects. Generate offers the robustness and configurability of Yeoman, the expressiveness and simplicity of Slush, and more powerful flow control and composability than either.") 95 | * [sarge](https://www.npmjs.com/package/sarge): Register and use custom search indexers to create, update and use search indices. | [homepage](https://github.com/doowb/sarge "Register and use custom search indexers to create, update and use search indices.") 96 | * [search-indexer-algolia](https://www.npmjs.com/package/search-indexer-algolia): base-search indexer to enable collecting and adding records to an algolia search index | [homepage](https://github.com/doowb/search-indexer-algolia "base-search indexer to enable collecting and adding records to an algolia search index") 97 | * [update](https://www.npmjs.com/package/update): Be scalable! Update is a new, open source developer framework and CLI for automating updates… [more](https://github.com/update/update) | [homepage](https://github.com/update/update "Be scalable! Update is a new, open source developer framework and CLI for automating updates of any kind in code projects.") 98 | * [verb](https://www.npmjs.com/package/verb): Documentation generator for GitHub projects. Verb is extremely powerful, easy to use, and is used… [more](https://github.com/verbose/verb) | [homepage](https://github.com/verbose/verb "Documentation generator for GitHub projects. Verb is extremely powerful, easy to use, and is used on hundreds of projects of all sizes to generate everything from API docs to readmes.") 99 | 100 | ### Contributing 101 | 102 | Pull requests and stars are always welcome. For bugs and feature requests, [please create an issue](../../issues/new). 103 | 104 | Please read the [contributing guide](contributing.md) for advice on opening issues, pull requests, and coding standards. 105 | 106 | ### Building docs 107 | 108 | _(This project's readme.md is generated by [verb](https://github.com/verbose/verb-generate-readme), please don't edit the readme directly. Any changes to the readme must be made in the [.verb.md](.verb.md) readme template.)_ 109 | 110 | To generate the readme, run the following command: 111 | 112 | ```sh 113 | $ npm install -g verbose/verb#dev verb-generate-readme && verb 114 | ``` 115 | 116 | ### Running tests 117 | 118 | Install dev dependencies: 119 | 120 | ```sh 121 | $ npm install && npm test 122 | ``` 123 | 124 | ### Author 125 | 126 | **Brian Woodward** 127 | 128 | * [github/doowb](https://github.com/doowb) 129 | * [twitter/doowb](https://twitter.com/doowb) 130 | 131 | ### License 132 | 133 | Copyright © 2017, [Brian Woodward](https://github.com/doowb). 134 | MIT 135 | 136 | *** 137 | 138 | _This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.4.2, on February 01, 2017._ -------------------------------------------------------------------------------- /contributing.md: -------------------------------------------------------------------------------- 1 | # Contributing to base-search 2 | 3 | First and foremost, thank you! We appreciate that you want to contribute to base-search, your time is valuable, and your contributions mean a lot to us. 4 | 5 | **What does "contributing" mean?** 6 | 7 | Creating an issue is the simplest form of contributing to a project. But there are many ways to contribute, including the following: 8 | 9 | - Updating or correcting documentation 10 | - Feature requests 11 | - Bug reports 12 | 13 | If you'd like to learn more about contributing in general, the [Guide to Idiomatic Contributing](https://github.com/jonschlinkert/idiomatic-contributing) has a lot of useful information. 14 | 15 | **Showing support for base-search** 16 | 17 | Please keep in mind that open source software is built by people like you, who spend their free time creating things the rest the community can use. 18 | 19 | Don't have time to contribute? No worries, here are some other ways to show your support for base-search: 20 | 21 | - star the [project](https://github.com/node-base/base-search) 22 | - tweet your support for base-search 23 | 24 | ## Issues 25 | 26 | ### Before creating an issue 27 | 28 | Please try to determine if the issue is caused by an underlying library, and if so, create the issue there. Sometimes this is difficult to know. We only ask that you attempt to give a reasonable attempt to find out. Oftentimes the readme will have advice about where to go to create issues. 29 | 30 | Try to follow these guidelines 31 | 32 | - **Investigate the issue**: 33 | - **Check the readme** - oftentimes you will find notes about creating issues, and where to go depending on the type of issue. 34 | - Create the issue in the appropriate repository. 35 | 36 | ### Creating an issue 37 | 38 | Please be as descriptive as possible when creating an issue. Give us the information we need to successfully answer your question or address your issue by answering the following in your issue: 39 | 40 | - **version**: please note the version of base-search are you using 41 | - **extensions, plugins, helpers, etc** (if applicable): please list any extensions you're using 42 | - **error messages**: please paste any error messages into the issue, or a [gist](https://gist.github.com/) 43 | 44 | ## Above and beyond 45 | 46 | Here are some tips for creating idiomatic issues. Taking just a little bit extra time will make your issue easier to read, easier to resolve, more likely to be found by others who have the same or similar issue in the future. 47 | 48 | - read the [Guide to Idiomatic Contributing](https://github.com/jonschlinkert/idiomatic-contributing) 49 | - take some time to learn basic markdown. This [markdown cheatsheet](https://gist.github.com/jonschlinkert/5854601) is super helpful, as is the GitHub guide to [basic markdown](https://help.github.com/articles/markdown-basics/). 50 | - Learn about [GitHub Flavored Markdown](https://help.github.com/articles/github-flavored-markdown/). And if you want to really go above and beyond, read [mastering markdown](https://guides.github.com/features/mastering-markdown/). 51 | - use backticks to wrap code. This ensures that code will retain its format, making it much more readable to others 52 | - use syntax highlighting by adding the correct language name after the first "code fence" 53 | 54 | 55 | [node-glob]: https://github.com/isaacs/node-glob 56 | [micromatch]: https://github.com/jonschlinkert/micromatch 57 | [so]: http://stackoverflow.com/questions/tagged/base-search -------------------------------------------------------------------------------- /example/config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "ALGOLIA_APPLICATION_ID": "your-application-id-here", 3 | "ALGOLIA_SECRET_KEY": "your-secret-api-key-here" 4 | } 5 | -------------------------------------------------------------------------------- /example/config.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var extend = require('extend-shallow'); 4 | var exists = require('fs-exists-sync'); 5 | var path = require('path'); 6 | 7 | module.exports = function(options) { 8 | var config = {}; 9 | var config = {}; 10 | if (exists(path.join(__dirname, '../tmp/config.json'))) { 11 | config = require('../tmp/config.json'); 12 | } else if (exists(path.join(__dirname, 'config.json'))) { 13 | config = require('./config.json'); 14 | } 15 | config = extend({}, config, options); 16 | return config; 17 | }; 18 | -------------------------------------------------------------------------------- /example/docs/about.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | About 5 | 6 | 7 |
8 |

General information

9 |

Sub heading 1

10 | 17 |
18 | 19 | 20 | -------------------------------------------------------------------------------- /example/docs/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Home Page 5 | 6 | 7 |
8 |

Home Page

9 |

Sub heading 1

10 |

This is some content in the main content under sub heading 1

11 |

Sub heading 2

12 |

This is some content in the main content under sub heading 2

13 |

Sub heading 3

14 |

This is some content in the main content under sub heading 3

15 |
16 | 17 | 18 | -------------------------------------------------------------------------------- /example/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var assemble = require('assemble'); 4 | var search = require('../'); 5 | 6 | var args = require('yargs-parser')(process.argv.slice(2), { 7 | alias: {i: 'indexer'}, 8 | default: {i: 'default'} 9 | }); 10 | 11 | var config = require('./config')(args); 12 | var app = new assemble(); 13 | app.use(search({indexer: args.i})); 14 | app.search.indexer('lunr', require('./indexer-lunr')(config)) 15 | app.search.indexer('algolia', require('./indexer-algolia')(config)) 16 | 17 | app.src('docs/*.html', {cwd: __dirname}) 18 | .pipe(app.search.collect()) 19 | .on('data', function(){}) 20 | .on('end', function() { 21 | app.search.index(function(err) { 22 | if (err) { 23 | console.error(err); 24 | process.exit(1); 25 | } 26 | console.log(app.search.files); 27 | console.log('done'); 28 | process.exit(); 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /example/indexer-algolia.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var each = require('async-each'); 4 | var path = require('path'); 5 | var $ = require('cheerio'); 6 | var write = require('write'); 7 | var algolia = require('algoliasearch'); 8 | 9 | module.exports = function(config) { 10 | var client = algolia(config.ALGOLIA_APPLICATION_ID, config.ALGOLIA_SECRET_KEY); 11 | var idx = client.initIndex('base-search-example'); 12 | 13 | return { 14 | collect: function(file, next) { 15 | var title = $(file.content).find('title').text(); 16 | next(null, { 17 | objectID: file.key, 18 | key: file.key, 19 | title: file.data.title || title || file.key, 20 | tags: [], 21 | category: file.data.category || 'docs', 22 | description: file.data.description || file.data.title || file.key, 23 | body: $(file.content).find('.main-content').text() 24 | }); 25 | }, 26 | index: function(files, options, cb) { 27 | if (typeof options === 'function') { 28 | cb = options; 29 | options = {}; 30 | } 31 | 32 | each(Object.keys(files), function(key, next) { 33 | if (files.hasOwnProperty(key)) { 34 | idx.addObject(files[key], next); 35 | return; 36 | } 37 | next(); 38 | }, cb); 39 | } 40 | }; 41 | }; 42 | -------------------------------------------------------------------------------- /example/indexer-lunr.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var path = require('path'); 4 | var lunr = require('lunr'); 5 | var write = require('write'); 6 | var $ = require('cheerio'); 7 | 8 | module.exports = function(config) { 9 | var idx = lunr(function() { 10 | this.ref('key'); 11 | this.field('title', { boost: 10000 }); 12 | this.field('description', { boost: 100 }); 13 | this.field('tags', { boost: 10 }); 14 | this.field('category', { boost: 10 }); 15 | this.field('body'); 16 | }); 17 | 18 | return { 19 | collect: function(file, next) { 20 | var title = $(file.content).find('title').text(); 21 | next(null, { 22 | key: file.key, 23 | title: file.data.title || title || file.key, 24 | tags: [], 25 | category: file.data.category || 'docs', 26 | description: file.data.description || file.data.title || file.key, 27 | body: $(file.content).find('.main-content').text() 28 | }); 29 | }, 30 | index: function(files, options, cb) { 31 | if (typeof options === 'function') { 32 | cb = options; 33 | options = {}; 34 | } 35 | 36 | for (var key in files) { 37 | if (files.hasOwnProperty(key)) { 38 | idx.add(files[key]); 39 | } 40 | } 41 | var fp = options.base 42 | ? path.join(options.base, 'lunr-search.json') 43 | : path.join(__dirname, 'lunr-search.json'); 44 | var content = JSON.stringify({files: files, idx: idx}); 45 | write(fp, content, cb); 46 | } 47 | }; 48 | }; 49 | -------------------------------------------------------------------------------- /example/lunr-search.json: -------------------------------------------------------------------------------- 1 | {"files":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"key":"/Users/doowb/work/base/base-search/example/docs/about.html","title":"About","tags":[],"category":"docs","description":"/Users/doowb/work/base/base-search/example/docs/about.html","body":"\n General information\n Sub heading 1\n \n Info 1\n Info 2\n Info 3\n Info 4\n Info 5\n \n"},"/Users/doowb/work/base/base-search/example/docs/index.html":{"key":"/Users/doowb/work/base/base-search/example/docs/index.html","title":"Home Page","tags":[],"category":"docs","description":"/Users/doowb/work/base/base-search/example/docs/index.html","body":"\n Home Page\n Sub heading 1\n This is some content in the main content under sub heading 1\n Sub heading 2\n This is some content in the main content under sub heading 2\n Sub heading 3\n This is some content in the main content under sub heading 3\n"}},"idx":{"version":"0.7.2","fields":[{"name":"title","boost":10000},{"name":"description","boost":100},{"name":"tags","boost":10},{"name":"category","boost":10},{"name":"body","boost":1}],"ref":"key","tokenizer":"default","documentStore":{"store":{"/Users/doowb/work/base/base-search/example/docs/about.html":["1","2","3","4","5","doc","gener","head","info","inform","search/example/docs/about.html","sub","users/doowb/work/base/bas"],"/Users/doowb/work/base/base-search/example/docs/index.html":["1","2","3","content","doc","head","home","main","page","search/example/docs/index.html","sub","under","users/doowb/work/base/bas"]},"length":2},"tokenStore":{"root":{"1":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.13333333333333333},"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":0.0625}}},"2":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.06666666666666667},"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":0.0625}}},"3":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.06666666666666667},"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":0.0625}}},"4":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.06666666666666667}}},"5":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.06666666666666667}}},"docs":{},"d":{"docs":{},"o":{"docs":{},"c":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":10},"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":10}}}}},"g":{"docs":{},"e":{"docs":{},"n":{"docs":{},"e":{"docs":{},"r":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.06666666666666667}}}}}}},"h":{"docs":{},"e":{"docs":{},"a":{"docs":{},"d":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.06666666666666667},"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":0.1875}}}}},"o":{"docs":{},"m":{"docs":{},"e":{"docs":{"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":5000.03125}}}}}},"i":{"docs":{},"n":{"docs":{},"f":{"docs":{},"o":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.3333333333333333}},"r":{"docs":{},"m":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.06666666666666667}}}}}}}},"s":{"docs":{},"e":{"docs":{},"a":{"docs":{},"r":{"docs":{},"c":{"docs":{},"h":{"docs":{},"/":{"docs":{},"e":{"docs":{},"x":{"docs":{},"a":{"docs":{},"m":{"docs":{},"p":{"docs":{},"l":{"docs":{},"e":{"docs":{},"/":{"docs":{},"d":{"docs":{},"o":{"docs":{},"c":{"docs":{},"s":{"docs":{},"/":{"docs":{},"a":{"docs":{},"b":{"docs":{},"o":{"docs":{},"u":{"docs":{},"t":{"docs":{},".":{"docs":{},"h":{"docs":{},"t":{"docs":{},"m":{"docs":{},"l":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":50}}}}}}}}}}}},"i":{"docs":{},"n":{"docs":{},"d":{"docs":{},"e":{"docs":{},"x":{"docs":{},".":{"docs":{},"h":{"docs":{},"t":{"docs":{},"m":{"docs":{},"l":{"docs":{"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":50}}}}}}}}}}}}}}}}}}}}}}}}}}}}}}},"u":{"docs":{},"b":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":0.06666666666666667},"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":0.1875}}}}},"u":{"docs":{},"s":{"docs":{},"e":{"docs":{},"r":{"docs":{},"s":{"docs":{},"/":{"docs":{},"d":{"docs":{},"o":{"docs":{},"o":{"docs":{},"w":{"docs":{},"b":{"docs":{},"/":{"docs":{},"w":{"docs":{},"o":{"docs":{},"r":{"docs":{},"k":{"docs":{},"/":{"docs":{},"b":{"docs":{},"a":{"docs":{},"s":{"docs":{},"e":{"docs":{},"/":{"docs":{},"b":{"docs":{},"a":{"docs":{},"s":{"docs":{"/Users/doowb/work/base/base-search/example/docs/about.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/about.html","tf":50},"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":50}}}}}}}}}}}}}}}}}}}}}}}}}},"n":{"docs":{},"d":{"docs":{},"e":{"docs":{},"r":{"docs":{"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":0.09375}}}}}}},"c":{"docs":{},"o":{"docs":{},"n":{"docs":{},"t":{"docs":{},"e":{"docs":{},"n":{"docs":{},"t":{"docs":{"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":0.1875}}}}}}}}},"m":{"docs":{},"a":{"docs":{},"i":{"docs":{},"n":{"docs":{"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":0.09375}}}}}},"p":{"docs":{},"a":{"docs":{},"g":{"docs":{},"e":{"docs":{"/Users/doowb/work/base/base-search/example/docs/index.html":{"ref":"/Users/doowb/work/base/base-search/example/docs/index.html","tf":5000.03125}}}}}}},"length":26},"corpusTokens":["1","2","3","4","5","content","doc","gener","head","home","info","inform","main","page","search/example/docs/about.html","search/example/docs/index.html","sub","under","users/doowb/work/base/bas"],"pipeline":["trimmer","stopWordFilter","stemmer"]}} -------------------------------------------------------------------------------- /example/search-algolia.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var algolia = require('algoliasearch'); 4 | var args = require('yargs-parser')(process.argv.slice(2)); 5 | var config = require('./config')(args); 6 | 7 | var client = algolia(config.ALGOLIA_APPLICATION_ID, config.ALGOLIA_SECRET_KEY); 8 | var idx = client.initIndex('base-search-example'); 9 | var term = args._.join(' '); 10 | 11 | console.log(`Searching for "${term}"`); 12 | idx.search(term, function(err, results) { 13 | if (err) { 14 | console.log(err); 15 | process.exit(1); 16 | } 17 | console.log(`Found ${results.nbHits} result${results.nbHits === 1 ? '' : 's'}`); 18 | console.log(); 19 | results.hits.forEach(function(result) { 20 | var key = result.key; 21 | console.log(` - ${key}`); 22 | console.log(result); 23 | console.log(); 24 | }); 25 | process.exit(); 26 | }); 27 | -------------------------------------------------------------------------------- /example/search-lunr.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var path = require('path'); 4 | var lunr = require('lunr'); 5 | var exists = require('fs-exists-sync'); 6 | var args = require('yargs-parser')(process.argv.slice(2)); 7 | 8 | if (!exists(path.join(__dirname, 'lunr-search.json'))) { 9 | console.log('Run `$ node example -i lunr` first to generate the search index.'); 10 | process.exit(); 11 | } 12 | 13 | var data = require('./lunr-search.json'); 14 | var idx = lunr.Index.load(data.idx); 15 | var term = args._.join(' '); 16 | console.log(`Searching for "${term}"`); 17 | var results = idx.search(term); 18 | 19 | console.log(`Found ${results.length} result${results.length === 1 ? '' : 's'}`); 20 | console.log(); 21 | results.forEach(function(result) { 22 | var key = result.ref; 23 | var file = data.files[key]; 24 | console.log(` - ${key}`); 25 | console.log(file); 26 | console.log(); 27 | }); 28 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var utils = require('./lib/utils'); 4 | 5 | /** 6 | * Plugin for [base][] applications like [generate][], [assemble][], [verb][], and [update][] 7 | * to add an instance of [sarge][] as `app.search` that has methods for creating search indices using [indexers](#indexers). 8 | * 9 | * ```js 10 | * var app = assemble(); 11 | * app.use(search()); 12 | * console.log(app.search); 13 | * ``` 14 | * @name search 15 | * @param {Object} `config` Configuration object used to specify default indexer to use. 16 | * @return {Function} Plugin function passed to `app.use` methods. 17 | * @api public 18 | */ 19 | 20 | module.exports = function(config) { 21 | return function plugin(app) { 22 | if (!utils.isValid(app, 'base-search')) return; 23 | app.define('search', new utils.Sarge(config)); 24 | }; 25 | }; 26 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var utils = require('lazy-cache')(require); 4 | var fn = require; 5 | require = utils; 6 | 7 | /** 8 | * Lazily required module dependencies 9 | */ 10 | 11 | require('is-valid-app', 'isValid'); 12 | require('sarge', 'Sarge'); 13 | 14 | require = fn; 15 | 16 | /** 17 | * Expose `utils` modules 18 | */ 19 | 20 | module.exports = utils; 21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "base-search", 3 | "description": "Base plugin that adds methods for creating, updating and using search indices.", 4 | "version": "0.1.1", 5 | "homepage": "https://github.com/node-base/base-search", 6 | "author": "Brian Woodward (https://github.com/doowb)", 7 | "repository": "node-base/base-search", 8 | "bugs": { 9 | "url": "https://github.com/node-base/base-search/issues" 10 | }, 11 | "license": "MIT", 12 | "files": [ 13 | "index.js", 14 | "lib" 15 | ], 16 | "main": "index.js", 17 | "engines": { 18 | "node": ">=4" 19 | }, 20 | "scripts": { 21 | "test": "mocha" 22 | }, 23 | "dependencies": { 24 | "extend-shallow": "^2.0.1", 25 | "is-valid-app": "^0.2.1", 26 | "lazy-cache": "^2.0.2", 27 | "sarge": "^0.1.2" 28 | }, 29 | "devDependencies": { 30 | "algoliasearch": "^3.20.4", 31 | "assemble": "^0.18.1", 32 | "async-each": "^1.0.1", 33 | "base-app": "^0.2.6", 34 | "cheerio": "^0.22.0", 35 | "fs-exists-sync": "^0.1.0", 36 | "gulp-format-md": "^0.1.11", 37 | "lunr": "^0.7.2", 38 | "mocha": "^3.2.0", 39 | "write": "^0.3.2", 40 | "yargs-parser": "^4.2.1" 41 | }, 42 | "keywords": [ 43 | "algolia", 44 | "assemble", 45 | "base", 46 | "generate", 47 | "index", 48 | "indexer", 49 | "lunr", 50 | "node-base", 51 | "search", 52 | "update", 53 | "verb" 54 | ], 55 | "verb": { 56 | "toc": false, 57 | "layout": "default", 58 | "tasks": [ 59 | "readme" 60 | ], 61 | "plugins": [ 62 | "gulp-format-md" 63 | ], 64 | "lint": { 65 | "reflinks": true 66 | }, 67 | "related": { 68 | "list": [ 69 | "assemble", 70 | "base", 71 | "generate", 72 | "sarge", 73 | "search-indexer-algolia", 74 | "update", 75 | "verb" 76 | ] 77 | }, 78 | "reflinks": [ 79 | "assemble", 80 | "base", 81 | "generate", 82 | "update", 83 | "verb", 84 | "verb-generate-readme" 85 | ] 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | require('mocha'); 4 | var Base = require('base-app'); 5 | var assert = require('assert'); 6 | var each = require('async-each'); 7 | var search = require('../'); 8 | var app; 9 | 10 | describe('base-search', function() { 11 | beforeEach(function() { 12 | app = new Base(); 13 | app.use(search()); 14 | }); 15 | 16 | it('should export a function', function() { 17 | assert.equal(typeof search, 'function'); 18 | }); 19 | 20 | it('should add a `search` property to a base application', function() { 21 | assert(app.search, 'expected app to have a `search` property'); 22 | assert.equal(typeof app.search, 'object'); 23 | }); 24 | 25 | it('should register an indexer', function() { 26 | var foo = { 27 | collect: function() {}, 28 | index: function() {} 29 | }; 30 | 31 | app.search.indexer('foo', foo); 32 | assert.deepEqual(app.search.indexers.foo, foo); 33 | }); 34 | 35 | it('should get a registered indexer', function() { 36 | var foo = { 37 | collect: function() {}, 38 | index: function() {} 39 | }; 40 | 41 | app.search.indexer('foo', foo); 42 | assert.deepEqual(app.search.indexer('foo'), foo); 43 | }); 44 | 45 | it('should throw an error if getting an unregistered indexer', function(cb) { 46 | try { 47 | app.search.indexer('foo'); 48 | cb(new Error('expected an error')); 49 | } catch (err) { 50 | assert.equal(err.message, 'Unable to find indexer "foo"'); 51 | cb(); 52 | } 53 | }); 54 | 55 | describe('collect', function() { 56 | it('should return a stream', function() { 57 | var stream = app.search.collect(); 58 | assert(stream, 'expected a stream to be returned'); 59 | assert.equal(typeof stream.on, 'function'); 60 | assert.equal(typeof stream.pipe, 'function'); 61 | }); 62 | 63 | it('should collect file objects and add them to the files object', function(cb) { 64 | var stream = app.search.collect(); 65 | stream.once('error', cb); 66 | stream.on('data', function() {}); 67 | stream.on('end', function() { 68 | assert.deepEqual(app.search.files, { 69 | foo: {key: 'foo'}, 70 | bar: {key: 'bar'}, 71 | baz: {key: 'baz'} 72 | }); 73 | cb(); 74 | }); 75 | 76 | stream.write({key: 'foo'}); 77 | stream.write({key: 'bar'}); 78 | stream.write({key: 'baz'}); 79 | stream.end(); 80 | }); 81 | }); 82 | 83 | describe('index', function() { 84 | it('should index files with the index method on the indexer', function(cb) { 85 | var foo = { 86 | collect: function(file, next) { 87 | next(null, file); 88 | }, 89 | index: function(files, options, next) { 90 | assert.deepEqual(files, { 91 | foo: {key: 'foo'}, 92 | bar: {key: 'bar'}, 93 | baz: {key: 'baz'} 94 | }); 95 | next(); 96 | } 97 | }; 98 | app.search.indexer('foo', foo); 99 | var stream = app.search.collect({indexer: 'foo'}); 100 | stream.once('error', cb); 101 | stream.on('data', function() {}); 102 | stream.on('end', function() { 103 | app.search.index({indexer: 'foo'}, cb); 104 | }); 105 | 106 | stream.write({key: 'foo'}); 107 | stream.write({key: 'bar'}); 108 | stream.write({key: 'baz'}); 109 | stream.end(); 110 | }); 111 | 112 | it('should allow using multiple indexers', function(cb) { 113 | var count = 0; 114 | var foo = { 115 | index: function(files, options, next) { 116 | count++; 117 | assert.deepEqual(files, { 118 | foo: {key: 'foo'}, 119 | bar: {key: 'bar'}, 120 | baz: {key: 'baz'} 121 | }); 122 | next(); 123 | } 124 | }; 125 | 126 | var bar = { 127 | index: function(files, options, next) { 128 | count++; 129 | assert.deepEqual(files, { 130 | foo: {key: 'foo'}, 131 | bar: {key: 'bar'}, 132 | baz: {key: 'baz'} 133 | }); 134 | next(); 135 | } 136 | }; 137 | 138 | app.search.indexer('foo', foo); 139 | app.search.indexer('bar', bar); 140 | 141 | // use default indexer to collect files 142 | var stream = app.search.collect(); 143 | stream.once('error', cb); 144 | stream.on('data', function() {}); 145 | stream.on('end', function() { 146 | // use foo and bar indexers to index results 147 | each(['foo', 'bar'], function(name, next) { 148 | app.search.index({indexer: name}, next); 149 | }, function(err) { 150 | if (err) return cb(err); 151 | assert.equal(count, 2); 152 | cb(); 153 | }); 154 | }); 155 | 156 | stream.write({key: 'foo'}); 157 | stream.write({key: 'bar'}); 158 | stream.write({key: 'baz'}); 159 | stream.end(); 160 | }); 161 | }); 162 | }); 163 | --------------------------------------------------------------------------------