├── .gitignore ├── index.js ├── .editorconfig ├── .eslintrc.json ├── lib ├── indexers │ ├── index.js │ ├── algolia.js │ ├── elasticsearch-legacy.js │ ├── meilisearch.js │ └── elasticsearch.js ├── utils.js ├── index.js └── create-indexer.js ├── .prettierrc.json ├── jsconfig.json ├── package.json ├── types.ts ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | searchsync.config.js 3 | searchsync.config.json 4 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // Used only when place directly to extensions/hooks/directus-extension-searchsync 2 | module.exports = require('./lib/index.js'); 3 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root=true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | indent_style = tab 8 | trim_trailing_whitespace = true 9 | 10 | [{package.json,*.yml,*.yaml}] 11 | indent_style = space 12 | indent_size = 2 13 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "root": true, 3 | "env": { 4 | "browser": false, 5 | "node": true 6 | }, 7 | "plugins": ["prettier"], 8 | "extends": ["eslint:recommended", "prettier"], 9 | "rules": { 10 | "prettier/prettier": "error" 11 | }, 12 | "parserOptions": { 13 | "ecmaVersion": 2020 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lib/indexers/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @type {Record} 3 | */ 4 | module.exports = { 5 | algolia: require('./algolia'), 6 | meilisearch: require('./meilisearch'), 7 | elasticsearch: require('./elasticsearch'), 8 | elasticsearch_legacy: require('./elasticsearch-legacy.js'), 9 | }; 10 | -------------------------------------------------------------------------------- /.prettierrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "htmlWhitespaceSensitivity": "ignore", 3 | "printWidth": 120, 4 | "singleQuote": true, 5 | "useTabs": true, 6 | "proseWrap": "always", 7 | "editorconfig": true, 8 | "overrides": [ 9 | { 10 | "files": "package.json", 11 | "options": { 12 | "tabWidth": 2, 13 | "useTabs": false 14 | } 15 | } 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "checkJs": true, 5 | "strict": true, 6 | "skipDefaultLibCheck": false, 7 | "skipLibCheck": true, 8 | "lib": [ 9 | "es2019" 10 | ], 11 | "types": [ 12 | "node" 13 | ] 14 | }, 15 | "include": [ 16 | "types.ts", 17 | "lib/**/*.js", 18 | ], 19 | "exclude": [ 20 | "node_modules" 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "directus-extension-searchsync", 3 | "version": "1.0.3", 4 | "author": { 5 | "email": "dimitrov.adrian+gh@gmail.com", 6 | "name": "Adrian Dimitrov" 7 | }, 8 | "license": "MIT", 9 | "directus:extension": { 10 | "type": "hook", 11 | "path": "lib/index.js", 12 | "source": "lib/index.js", 13 | "host": "^v9.0.0", 14 | "hidden": false 15 | }, 16 | "homepage": "https://github.com/dimitrov-adrian/directus-extension-searchsync", 17 | "dependencies": { 18 | "striptags": "^3.2.0", 19 | "cosmiconfig": "^7.0.1", 20 | "axios": "^0.24.0" 21 | }, 22 | "devDependencies": { 23 | "eslint": "^8.4.0", 24 | "prettier": "^2.5.0", 25 | "@directus/shared": "^9.0.0" 26 | }, 27 | "files": [ 28 | "lib" 29 | ] 30 | } 31 | -------------------------------------------------------------------------------- /types.ts: -------------------------------------------------------------------------------- 1 | import { Filter } from '@directus/shared/types'; 2 | 3 | export type ExtensionConfig = { 4 | server: IndexerConfig; 5 | batchLimit?: number; 6 | collections: Record; 7 | }; 8 | 9 | export type CollectionConfig = { 10 | collection?: string; 11 | collectionName?: string; 12 | collectionField?: string; 13 | indexName?: string; 14 | fields?: string[]; 15 | filter?: Filter; 16 | transform?: (input: object, utils: Record, collectionName: string) => object; 17 | }; 18 | 19 | export type IndexerConfig = { 20 | type: string; 21 | appId?: string; 22 | key?: string; 23 | host?: string; 24 | headers?: Record; 25 | }; 26 | 27 | export type IndexerInterface = (config: IndexerConfig) => { 28 | createIndex: (collection: string) => Promise; 29 | deleteItems: (collection: string) => Promise; 30 | deleteItem: (collection: string, id: string) => Promise; 31 | updateItem: (collection: string, id: string, data: object, pk?: string) => Promise; 32 | }; 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Adrian Dimitrov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | flattenObject, 3 | objectMap, 4 | filteredObject, 5 | }; 6 | 7 | function flattenObject(ob, glue = '.') { 8 | const toReturn = {}; 9 | 10 | for (const i in ob) { 11 | if (!ob.hasOwnProperty(i)) continue; 12 | 13 | if (typeof ob[i] == 'object' && ob[i] !== null) { 14 | const flatObject = flattenObject(ob[i], glue); 15 | for (const x in flatObject) { 16 | if (!flatObject.hasOwnProperty(x)) continue; 17 | 18 | toReturn[i + glue + x] = flatObject[x]; 19 | } 20 | } else { 21 | toReturn[i] = ob[i]; 22 | } 23 | } 24 | 25 | return toReturn; 26 | } 27 | 28 | /** 29 | * Returns a new object with the values at each key mapped using mapFn(value) 30 | * @param {Record} object 31 | * @param {(value: any, key: string) => any} mapFn 32 | * @return {Record} 33 | */ 34 | function objectMap(object, mapFn) { 35 | return Object.keys(object).reduce(function (result, key) { 36 | const value = object[key]; 37 | if (value instanceof Object) { 38 | result[key] = value; 39 | } else { 40 | result[key] = mapFn(object[key], key); 41 | } 42 | 43 | return result; 44 | }, {}); 45 | } 46 | 47 | /** 48 | * @param {Record} object 49 | * @param {string[]} keys 50 | * @return {Record} 51 | */ 52 | function filteredObject(object, keys) { 53 | return Object.keys(object) 54 | .filter((key) => keys.includes(key)) 55 | .reduce((obj, key) => { 56 | return { 57 | ...obj, 58 | [key]: object[key], 59 | }; 60 | }, {}); 61 | } 62 | -------------------------------------------------------------------------------- /lib/indexers/algolia.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @type {import("axios").AxiosInstance} 3 | */ 4 | const axios = require('axios'); 5 | 6 | /** 7 | * @type {import("../../types.js").IndexerInterface} 8 | */ 9 | module.exports = function algolia(config) { 10 | const axiosConfig = { 11 | headers: { 12 | 'Content-Type': 'application/json; charset=UTF-8', 13 | ...(config.headers || {}), 14 | }, 15 | }; 16 | 17 | if (config.key) { 18 | axiosConfig.headers['X-Algolia-API-Key'] = config.key; 19 | } else { 20 | throw Error('No API Key set. The server.key is mandatory.'); 21 | } 22 | 23 | if (config.appId) { 24 | axiosConfig.headers['X-Algolia-Application-Id'] = config.appId; 25 | } else { 26 | throw Error('No Application ID set. The server.appId is mandatory.'); 27 | } 28 | 29 | const endpoint = `https://${config.appId}.algolia.net/1/indexes`; 30 | 31 | return { 32 | createIndex, 33 | deleteItems, 34 | deleteItem, 35 | updateItem, 36 | }; 37 | 38 | async function createIndex() {} 39 | 40 | async function deleteItems(collection) { 41 | try { 42 | await axios.post(`${endpoint}/${collection}/clear`, null, axiosConfig); 43 | } catch (error) { 44 | if (error.response && error.response.status === 404) return; 45 | 46 | throw error; 47 | } 48 | } 49 | 50 | async function deleteItem(collection, id) { 51 | try { 52 | await axios.delete(`${endpoint}/${collection}/${id}`, axiosConfig); 53 | } catch (error) { 54 | if (error.response && error.response.status === 404) return; 55 | 56 | throw error; 57 | } 58 | } 59 | 60 | async function updateItem(collection, id, data) { 61 | try { 62 | await axios.put(`${endpoint}/${collection}/${id}`, data, axiosConfig); 63 | } catch (error) { 64 | if (error.response) { 65 | throw { message: error.toString(), response: error.response }; 66 | } 67 | 68 | throw error; 69 | } 70 | } 71 | }; 72 | -------------------------------------------------------------------------------- /lib/indexers/elasticsearch-legacy.js: -------------------------------------------------------------------------------- 1 | const { URL } = require('url'); 2 | 3 | /** 4 | * @type {import("axios").AxiosInstance} 5 | */ 6 | const axios = require('axios'); 7 | 8 | /** 9 | * @type {import("../../types.js").IndexerInterface} 10 | */ 11 | module.exports = function elasticsearchLegacy(config) { 12 | const axiosConfig = { 13 | headers: { 14 | 'Content-Type': 'application/json', 15 | ...(config.headers || {}), 16 | }, 17 | }; 18 | 19 | if (!config.host) { 20 | throw Error('No HOST set. The server.host is mandatory.'); 21 | } 22 | 23 | const host = new URL(config.host); 24 | if (!host.hostname || !host.pathname || host.pathname === '/') { 25 | throw Error(`Invalid server.host, it must be like http://ee.example.com/indexname`); 26 | } 27 | 28 | return { 29 | createIndex, 30 | deleteItems, 31 | deleteItem, 32 | updateItem, 33 | }; 34 | 35 | async function createIndex(collection) {} 36 | 37 | async function deleteItems(collection) { 38 | try { 39 | await axios.post( 40 | `${config.host}/${collection}/_delete_by_query`, 41 | { 42 | query: { 43 | match_all: {}, 44 | }, 45 | }, 46 | axiosConfig 47 | ); 48 | } catch (error) { 49 | if (error.response && error.response.status === 404) return; 50 | 51 | throw error; 52 | } 53 | } 54 | 55 | async function deleteItem(collection, id) { 56 | try { 57 | await axios.delete(`${config.host}/${collection}/${id}`, axiosConfig); 58 | } catch (error) { 59 | if (error.response && error.response.status === 404) return; 60 | 61 | throw error; 62 | } 63 | } 64 | 65 | async function updateItem(collection, id, data) { 66 | try { 67 | await axios.post(`${config.host}/${collection}/${id}`, data, axiosConfig); 68 | } catch (error) { 69 | if (error.response) { 70 | throw { message: error.toString(), response: error.response }; 71 | } 72 | 73 | throw error; 74 | } 75 | } 76 | }; 77 | -------------------------------------------------------------------------------- /lib/indexers/meilisearch.js: -------------------------------------------------------------------------------- 1 | const { URL } = require('url'); 2 | 3 | /** 4 | * @type {import("axios").AxiosInstance} 5 | */ 6 | const axios = require('axios'); 7 | 8 | /** 9 | * @type {import("../../types.js").IndexerInterface} 10 | */ 11 | module.exports = function meilisearch(config) { 12 | const axiosConfig = { 13 | headers: config.headers || {}, 14 | }; 15 | 16 | if (config.key) { 17 | // Meilisearch changed their authorization in 0.25 from the 18 | // 'X-Meili-API-Key' header to Authorization bearer. 19 | 20 | // Include old headers for compatibility with pre-0.25 versions of Meilisearch 21 | axiosConfig.headers['X-Meili-API-Key'] = config.key; 22 | 23 | // New auth headers for 0.25+ 24 | axiosConfig.headers['Authorization'] = `Bearer ${config.key}`; 25 | } 26 | 27 | if (!config.host) { 28 | throw Error('No HOST set. The server.host is mandatory.'); 29 | } 30 | 31 | const host = new URL(config.host); 32 | if (!host.hostname || (host.pathname && host.pathname !== '/')) { 33 | throw Error(`Invalid server.host, it must be like http://meili.example.com/`); 34 | } 35 | 36 | return { 37 | createIndex, 38 | deleteItems, 39 | deleteItem, 40 | updateItem, 41 | }; 42 | 43 | async function createIndex(collection) {} 44 | 45 | async function deleteItems(collection) { 46 | try { 47 | await axios.delete(`${config.host}/indexes/${collection}`, axiosConfig); 48 | } catch (error) { 49 | if (error.response && error.response.status === 404) return; 50 | 51 | throw error; 52 | } 53 | } 54 | 55 | async function deleteItem(collection, id) { 56 | try { 57 | await axios.delete(`${config.host}/indexes/${collection}/documents/${id}`, axiosConfig); 58 | } catch (error) { 59 | if (error.response && error.response.status === 404) return; 60 | 61 | throw error; 62 | } 63 | } 64 | 65 | async function updateItem(collection, id, data, pk) { 66 | try { 67 | await axios.post( 68 | `${config.host}/indexes/${collection}/documents?primaryKey=${pk}`, 69 | [{ [pk]: id, ...data }], 70 | axiosConfig 71 | ); 72 | } catch (error) { 73 | if (error.response) { 74 | throw { message: error.toString(), response: error.response }; 75 | } 76 | 77 | throw error; 78 | } 79 | } 80 | }; 81 | -------------------------------------------------------------------------------- /lib/indexers/elasticsearch.js: -------------------------------------------------------------------------------- 1 | const { URL } = require('url'); 2 | const https = require('https'); 3 | 4 | /** 5 | * @type {import("axios").AxiosInstance} 6 | */ 7 | const axios = require('axios'); 8 | 9 | /** 10 | * @type {import("../../types.js").IndexerInterface} 11 | */ 12 | module.exports = function elasticsearch(config) { 13 | 14 | const axiosConfig = { 15 | headers: { 16 | 'Content-Type': 'application/json', 17 | ...(config.headers || {}), 18 | } 19 | }; 20 | 21 | if(config.username && config.password) { 22 | axiosConfig.auth = { 23 | username: config.username, 24 | password: config.password 25 | } 26 | } 27 | 28 | if(config.ignore_cert) { 29 | const agent = new https.Agent({ 30 | rejectUnauthorized: false 31 | }); 32 | 33 | axiosConfig.httpsAgent = agent; 34 | } 35 | 36 | if (!config.host) { 37 | throw Error('No HOST set. The server.host is mandatory.'); 38 | } 39 | 40 | const host = new URL(config.host); 41 | if (!host.hostname || !host.pathname || host.pathname !== '/') { 42 | throw Error('Invalid server.host, it must be like http://ee.example.com and without path for index'); 43 | } 44 | 45 | return { 46 | createIndex, 47 | deleteItems, 48 | deleteItem, 49 | updateItem, 50 | }; 51 | 52 | async function createIndex(collection) {} 53 | 54 | async function deleteItems(collection) { 55 | try { 56 | await axios.post( 57 | `${config.host}/${collection}/_delete_by_query`, 58 | { 59 | query: { 60 | match_all: {}, 61 | }, 62 | }, 63 | axiosConfig 64 | ); 65 | } catch (error) { 66 | if (error.response && error.response.status === 404) return; 67 | 68 | throw error; 69 | } 70 | } 71 | 72 | async function deleteItem(collection, id) { 73 | try { 74 | await axios.delete(`${config.host}/${collection}/_doc/${id}`, axiosConfig); 75 | } catch (error) { 76 | if (error.response && error.response.status === 404) return; 77 | 78 | throw error; 79 | } 80 | } 81 | 82 | async function updateItem(collection, id, data) { 83 | try { 84 | await axios.post(`${config.host}/${collection}/_doc/${id}`, data, axiosConfig); 85 | } catch (error) { 86 | if (error.response) { 87 | throw { message: error.toString(), response: error.response }; 88 | } 89 | 90 | throw error; 91 | } 92 | } 93 | }; 94 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | const { dirname } = require('path'); 2 | const { cosmiconfigSync } = require('cosmiconfig'); 3 | const createIndexer = require('./create-indexer.js'); 4 | 5 | module.exports = registerHook; 6 | 7 | /** 8 | * @type {import('@directus/shared/types').HookConfig} 9 | */ 10 | function registerHook({ action, init }, { services, env, database, logger, getSchema }) { 11 | const extensionConfig = loadConfig(); 12 | validateConfig(extensionConfig); 13 | 14 | const indexer = createIndexer(extensionConfig, { 15 | services, 16 | database, 17 | logger: logger.child({ extension: 'directus-extension-searchsync' }), 18 | getSchema, 19 | }); 20 | 21 | init('cli.before', ({ program }) => { 22 | const usersCommand = program.command('extension:searchsync'); 23 | 24 | usersCommand 25 | .command('index') 26 | .description( 27 | 'directus-extension-searchsync: Push all documents from all collections, that are setup in extension configuration' 28 | ) 29 | .action(initCollectionIndexesCommand); 30 | }); 31 | 32 | action('server.start', () => { 33 | if (!extensionConfig.reindexOnStart) return; 34 | indexer.initCollectionIndexes(); 35 | }); 36 | 37 | action('items.create', ({ collection, key }) => { 38 | if (!(extensionConfig.collections.hasOwnProperty(collection))) return; 39 | indexer.updateItemIndex(collection, [key]); 40 | }); 41 | 42 | action('items.update', ({ collection, keys }) => { 43 | if (!(extensionConfig.collections.hasOwnProperty(collection))) return; 44 | indexer.updateItemIndex(collection, keys); 45 | }); 46 | 47 | action('items.delete', ({ collection, payload }) => { 48 | if (!(extensionConfig.collections.hasOwnProperty(collection))) return; 49 | indexer.deleteItemIndex(collection, payload); 50 | }); 51 | 52 | async function initCollectionIndexesCommand() { 53 | try { 54 | await indexer.initCollectionIndexes(); 55 | process.exit(0); 56 | } catch (error) { 57 | logger.error(error); 58 | process.exit(1); 59 | } 60 | } 61 | 62 | function loadConfig() { 63 | const cosmiconfig = cosmiconfigSync('searchsync', { 64 | stopDir: dirname(env.CONFIG_PATH), 65 | }); 66 | 67 | if (env.EXTENSION_SEARCHSYNC_CONFIG_PATH) { 68 | const config = cosmiconfig.load(env.EXTENSION_SEARCHSYNC_CONFIG_PATH); 69 | if (!config || !config.config) { 70 | throw Error( 71 | `EXTENSION_SEARCHSYNC_CONFIG_PATH is set, but "${env.EXTENSION_SEARCHSYNC_CONFIG_PATH}" is missing or broken.` 72 | ); 73 | } 74 | 75 | return config.config; 76 | } 77 | 78 | const config = cosmiconfig.search(dirname(env.CONFIG_PATH)); 79 | if (!config || !config.config) { 80 | throw Error( 81 | 'Missing configuration, follow https://github.com/dimitrov-adrian/directus-extension-searchsync#configuration to set it up.' 82 | ); 83 | } 84 | 85 | return config.config; 86 | } 87 | 88 | /** 89 | * @param {any} config 90 | */ 91 | function validateConfig(config) { 92 | if (typeof config !== 'object') { 93 | throw Error('Broken config file. Configuration is not an object.'); 94 | } 95 | 96 | if (!config.collections) { 97 | throw Error('Broken config file. Missing "collections" section.'); 98 | } 99 | 100 | if (!config.server) { 101 | throw Error('Broken config file. Missing "server" section.'); 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simple search engine indexer 2 | 3 | ## Supported engines 4 | 5 | - MeiliSearch 6 | - ElasticSearch 7 | - Algolia 8 | 9 | ## How to install 10 | 11 | In your Directus installation root 12 | 13 | ``` 14 | npm install dimitrov-adrian/directus-extension-searchsync 15 | ``` 16 | 17 | or yarn 18 | 19 | ``` 20 | yarn add https://github.com/dimitrov-adrian/directus-extension-searchsync 21 | ``` 22 | 23 | Restart directus 24 | 25 | ## CLI Commands 26 | 27 | Usage: `npx directus extension:searchsync ` 28 | 29 | Subcommands: 30 | 31 | - `index` - Reindex all documents from configuration 32 | 33 | ## Configuration 34 | 35 | The extension uses [cosmiconfig](https://github.com/davidtheclark/cosmiconfig#cosmiconfig) for configuration loader with 36 | `searchsync` block or if `EXTENSION_SEARCHSYNC_CONFIG_PATH` is set will try to use the file. 37 | 38 | So, configuration should comes from one of next files: 39 | 40 | - package.json `"searchsync":{...}` 41 | - .searchsyncrc 42 | - .searchsyncrc.json 43 | - .searchsyncrc.yaml 44 | - .searchsyncrc.yml 45 | - .searchsyncrc.js 46 | - .searchsyncrc.cjs 47 | - searchsync.config.js 48 | - searchsync.config.cjs 49 | 50 | ### Environment variables 51 | 52 | ### References 53 | 54 | - `server: object` Holds configuration for the search engine 55 | - `batchLimit: number` Batch limit when performing index/reindex (defaults to 100) 56 | - `reindexOnStart: boolean` Performs full reindex of all documents upon Directus starts 57 | - `collections: object` Indexing data definition 58 | - `collections..filter: object` The filter query in format like Directus on which item must match to be 59 | indexed (check [Filter Rules ](https://docs.directus.io/reference/filter-rules/#filter-rules)) 60 | - `collections..fields: array` Fields that will be indexed in Directus format 61 | - `collections..transform: function` (Could be defined only if config file is .js) A callback to return 62 | transformed/formatted data for indexing. 63 | - `collections..indexName: string` Force collection name when storing in search index 64 | - `collections..collectionField: string` If set, such field with value of the collection name will be added 65 | to the indexed document. Useful with conjuction with the _indexName_ option 66 | 67 | ### Examples 68 | 69 | #### `.searchsyncrc.json` 70 | 71 | ```json 72 | { 73 | "server": { 74 | "type": "meilisearch", 75 | "host": "http://search:7700/myindex", 76 | "key": "the-private-key" 77 | }, 78 | "batchLimit": 100, 79 | "reindexOnStart": false, 80 | "collections": { 81 | "products": { 82 | "filter": { 83 | "status": "published", 84 | "stock": "inStock" 85 | }, 86 | "fields": ["title", "image.id", "category.title", "brand.title", "tags", "description", "price", "rating"] 87 | }, 88 | "posts": { 89 | "indexName": "blog_posts", 90 | "collectionField": "_collection", 91 | 92 | "filter": { 93 | "status": "published" 94 | }, 95 | "fields": ["title", "teaser", "body", "thumbnail.id"] 96 | } 97 | } 98 | } 99 | ``` 100 | 101 | #### `.searchsyncrc.js` 102 | 103 | ```javascript 104 | const config = { 105 | server: { 106 | type: 'meilisearch', 107 | host: 'http://search:7700', 108 | key: 'the-private-key', 109 | }, 110 | reindexOnStart: false, 111 | batchLimit: 100, 112 | collections: { 113 | pages: { 114 | filter: { 115 | status: 'published', 116 | }, 117 | fields: ['title', 'teaser', 'body', 'thumbnail.id'], 118 | transform: (item, { flattenObject, striptags }) => { 119 | return { 120 | ...flattenObject(item), 121 | body: striptags(item.body), 122 | someCustomValue: 'Hello World!', 123 | }; 124 | }, 125 | }, 126 | }, 127 | }; 128 | 129 | // Use as object. 130 | module.exports = config; 131 | ``` 132 | 133 | ##### Collection transformation callback description 134 | 135 | ```javascript 136 | /** 137 | * @param {Object} item 138 | * @param {{striptags, flattenObject, objectMap}} utils 139 | * @param {String} collectionName 140 | * @returns {Object} 141 | */ 142 | function (item, { striptags, flattenObject, objectMap }, collectionName) { 143 | return item 144 | } 145 | ``` 146 | 147 | #### Search engines config references 148 | 149 | ##### Meilisearch 150 | 151 | ```json 152 | { 153 | "type": "meilisearch", 154 | "host": "http://search:7700", 155 | "key": "the-private-key" 156 | } 157 | ``` 158 | 159 | ##### Algolia 160 | 161 | ```json 162 | { 163 | "type": "algolia", 164 | "appId": "Application-Id", 165 | "key": "secret-api-key" 166 | } 167 | ``` 168 | 169 | ##### ElasticSearch 170 | 171 | New typeless behaviour, use collection names as index name. 172 | 173 | ```json 174 | { 175 | "type": "elasticsearch", 176 | "host": "http://search:9200/" 177 | } 178 | ``` 179 | 180 | Use Authentification. 181 | 182 | ```json 183 | { 184 | "type": "elasticsearch", 185 | "host": "http://search:9200/", 186 | "username": "elastic", 187 | "password": "somepassword" 188 | } 189 | ``` 190 | 191 | Ignore ssl-certificate-error. 192 | 193 | ```json 194 | { 195 | "type": "elasticsearch", 196 | "host": "http://search:9200/", 197 | "ignore_cert": true, 198 | } 199 | ``` 200 | 201 | ##### ElasticSearch for 5.x and 6.x 202 | 203 | Old type behaviour, use collection names as types. 204 | 205 | ```json 206 | { 207 | "type": "elasticsearch_legacy", 208 | "host": "http://search:9200/projectindex" 209 | } 210 | ``` 211 | -------------------------------------------------------------------------------- /lib/create-indexer.js: -------------------------------------------------------------------------------- 1 | const striptags = require('striptags'); 2 | const { flattenObject, objectMap, filteredObject } = require('./utils.js'); 3 | const availableIndexers = require('./indexers/index.js'); 4 | 5 | module.exports = createIndexer; 6 | 7 | /** 8 | * @param {import('../types.js').ExtensionConfig} config 9 | * @param {any} context 10 | */ 11 | function createIndexer(config, { logger, database, services, getSchema }) { 12 | if (!config.server.type || !availableIndexers[config.server.type]) { 13 | throw Error(`Broken config file. Missing or invalid indexer type "${config.server.type || 'Unknown'}".`); 14 | } 15 | 16 | const indexer = availableIndexers[config.server.type](config.server); 17 | 18 | return { 19 | ensureCollectionIndex, 20 | initCollectionIndexes, 21 | 22 | initItemsIndex, 23 | updateItemIndex, 24 | deleteItemIndex, 25 | 26 | getCollectionIndexName, 27 | }; 28 | 29 | /** 30 | * @param {string} collection 31 | */ 32 | async function ensureCollectionIndex(collection) { 33 | const collectionIndex = getCollectionIndexName(collection); 34 | try { 35 | await indexer.createIndex(collectionIndex); 36 | } catch (error) { 37 | logger.warn(`Cannot create collection "${collectionIndex}". ${getErrorMessage(error)}`); 38 | logger.debug(error); 39 | } 40 | } 41 | 42 | async function initCollectionIndexes() { 43 | for (const collection of Object.keys(config.collections)) { 44 | await ensureCollectionIndex(collection); 45 | await initItemsIndex(collection); 46 | } 47 | } 48 | 49 | /** 50 | * @param {string} collection 51 | */ 52 | async function initItemsIndex(collection) { 53 | const schema = await getSchema(); 54 | 55 | if (!schema.collections[collection]) { 56 | logger.warn(`Collection "${collection}" does not exists.`); 57 | return; 58 | } 59 | 60 | const query = new services.ItemsService(collection, { database, schema }); 61 | 62 | try { 63 | await indexer.deleteItems(getCollectionIndexName(collection)); 64 | } catch (error) { 65 | logger.warn(`Cannot drop collection "${collection}". ${getErrorMessage(error)}`); 66 | logger.debug(error); 67 | } 68 | 69 | const pk = schema.collections[collection].primary; 70 | const limit = config.batchLimit || 100; 71 | 72 | for (let offset = 0; ; offset += limit) { 73 | const items = await query.readByQuery({ 74 | fields: [pk], 75 | filter: config.collections[collection].filter || [], 76 | limit, 77 | offset, 78 | }); 79 | 80 | if (!items || !items.length) break; 81 | 82 | await updateItemIndex( 83 | collection, 84 | items.map((/** @type {{ [x: string]: any; }} */ i) => i[pk]) 85 | ); 86 | } 87 | } 88 | 89 | /** 90 | * @param {string} collection 91 | * @param {string[]} ids 92 | */ 93 | async function deleteItemIndex(collection, ids) { 94 | const collectionIndex = getCollectionIndexName(collection); 95 | for (const id of ids) { 96 | try { 97 | await indexer.deleteItem(collectionIndex, id); 98 | } catch (error) { 99 | logger.warn(`Cannot delete "${collectionIndex}/${id}". ${getErrorMessage(error)}`); 100 | logger.debug(error); 101 | } 102 | } 103 | } 104 | 105 | /** 106 | * @param {string} collection 107 | * @param {string[]} ids 108 | */ 109 | async function updateItemIndex(collection, ids) { 110 | const schema = await getSchema(); 111 | 112 | const collectionIndex = getCollectionIndexName(collection); 113 | 114 | const query = new services.ItemsService(collection, { 115 | knex: database, 116 | schema: schema, 117 | }); 118 | 119 | const pk = schema.collections[collection].primary; 120 | 121 | const items = await query.readMany(ids, { 122 | fields: config.collections[collection].fields ? [pk, ...config.collections[collection].fields] : ['*'], 123 | filter: config.collections[collection].filter || [], 124 | }); 125 | 126 | /** 127 | * @type {string[]} 128 | */ 129 | const processedIds = []; 130 | 131 | for (const item of items) { 132 | const id = item[pk]; 133 | 134 | try { 135 | await indexer.updateItem(collectionIndex, id, prepareObject(item, collection), pk); 136 | 137 | processedIds.push(id); 138 | } catch (error) { 139 | logger.warn(`Cannot index "${collectionIndex}/${id}". ${getErrorMessage(error)}`); 140 | logger.debug(error); 141 | } 142 | } 143 | 144 | if (items.length < ids.length) { 145 | for (const id of ids.filter((x) => !processedIds.includes(x))) { 146 | try { 147 | await indexer.deleteItem(collectionIndex, id); 148 | } catch (error) { 149 | logger.warn(`Cannot index "${collectionIndex}/${id}". ${getErrorMessage(error)}`); 150 | logger.debug(error); 151 | } 152 | } 153 | } 154 | } 155 | 156 | /** 157 | * @param {object} body 158 | * @param {string} collection 159 | */ 160 | function prepareObject(body, collection) { 161 | const meta = {}; 162 | 163 | if (config.collections[collection].collectionField) { 164 | // @ts-ignore 165 | meta[config.collections[collection].collectionField] = collection; 166 | } 167 | 168 | if (config.collections[collection].transform) { 169 | return { 170 | // @ts-ignore 171 | ...config.collections[collection].transform( 172 | body, 173 | { 174 | striptags, 175 | flattenObject, 176 | objectMap, 177 | filteredObject, 178 | }, 179 | collection 180 | ), 181 | ...meta, 182 | }; 183 | } else if (config.collections[collection].fields) { 184 | return { 185 | ...filteredObject(flattenObject(body), config.collections[collection].fields), 186 | ...meta, 187 | }; 188 | } 189 | 190 | return { 191 | ...body, 192 | ...meta, 193 | }; 194 | } 195 | 196 | /** 197 | * @param {string} collection 198 | * @returns {string} 199 | */ 200 | function getCollectionIndexName(collection) { 201 | return config.collections[collection].indexName || collection; 202 | } 203 | 204 | /** 205 | * @param {any} error 206 | * @returns {string} 207 | */ 208 | function getErrorMessage(error) { 209 | if (error && error.message) return error.message; 210 | 211 | if (error && error.response && error.response.data && error.response.data.error) return error.response.data.error; 212 | 213 | return error.toString(); 214 | } 215 | } 216 | --------------------------------------------------------------------------------