├── .editorconfig
├── .gitattributes
├── .gitignore
├── LICENSE
├── README.md
├── admin
└── src
│ ├── Components
│ └── LeftMenuItem
│ │ ├── Wrapper.js
│ │ └── index.js
│ ├── assets
│ └── images
│ │ └── logo.svg
│ ├── containers
│ ├── App
│ │ └── index.js
│ ├── DataView
│ │ ├── Wrapper.js
│ │ └── index.js
│ ├── HomePage
│ │ ├── GlobalStyles.js
│ │ └── index.js
│ ├── Initializer
│ │ └── index.js
│ └── LeftMenu
│ │ ├── Wrapper.js
│ │ └── index.js
│ ├── index.js
│ ├── lifecycles.js
│ ├── pluginId.js
│ ├── translations
│ ├── ar.json
│ ├── cs.json
│ ├── de.json
│ ├── en.json
│ ├── es.json
│ ├── fr.json
│ ├── index.js
│ ├── it.json
│ ├── ko.json
│ ├── ms.json
│ ├── nl.json
│ ├── pl.json
│ ├── pt-BR.json
│ ├── pt.json
│ ├── ru.json
│ ├── sk.json
│ ├── th.json
│ ├── tr.json
│ ├── uk.json
│ ├── vi.json
│ ├── zh-Hans.json
│ └── zh.json
│ └── utils
│ └── getTrad.js
├── config
├── constants.js
├── functions
│ └── bootstrap.js
└── routes.json
├── controllers
└── elasticsearch.js
├── docs
└── CONFIG.md
├── middlewares
└── elastic
│ ├── defaults.json
│ └── index.js
├── package.json
├── services
├── functions.js
├── helper.js
├── index.js
├── logger.js
└── middleware.js
├── yarn-error.log
└── yarn.lock
/.editorconfig:
--------------------------------------------------------------------------------
1 | root = true
2 |
3 | [*]
4 | end_of_line = lf
5 | insert_final_newline = false
6 | indent_style = space
7 | indent_size = 2
8 |
--------------------------------------------------------------------------------
/.gitattributes:
--------------------------------------------------------------------------------
1 | # From https://github.com/Danimoth/gitattributes/blob/master/Web.gitattributes
2 |
3 | # Handle line endings automatically for files detected as text
4 | # and leave all files detected as binary untouched.
5 | * text=auto
6 |
7 | #
8 | # The above will handle all files NOT found below
9 | #
10 |
11 | #
12 | ## These files are text and should be normalized (Convert crlf => lf)
13 | #
14 |
15 | # source code
16 | *.php text
17 | *.css text
18 | *.sass text
19 | *.scss text
20 | *.less text
21 | *.styl text
22 | *.js text eol=lf
23 | *.coffee text
24 | *.json text
25 | *.htm text
26 | *.html text
27 | *.xml text
28 | *.svg text
29 | *.txt text
30 | *.ini text
31 | *.inc text
32 | *.pl text
33 | *.rb text
34 | *.py text
35 | *.scm text
36 | *.sql text
37 | *.sh text
38 | *.bat text
39 |
40 | # templates
41 | *.ejs text
42 | *.hbt text
43 | *.jade text
44 | *.haml text
45 | *.hbs text
46 | *.dot text
47 | *.tmpl text
48 | *.phtml text
49 |
50 | # git config
51 | .gitattributes text
52 | .gitignore text
53 | .gitconfig text
54 |
55 | # code analysis config
56 | .jshintrc text
57 | .jscsrc text
58 | .jshintignore text
59 | .csslintrc text
60 |
61 | # misc config
62 | *.yaml text
63 | *.yml text
64 | .editorconfig text
65 |
66 | # build config
67 | *.npmignore text
68 | *.bowerrc text
69 |
70 | # Heroku
71 | Procfile text
72 | .slugignore text
73 |
74 | # Documentation
75 | *.md text
76 | LICENSE text
77 | AUTHORS text
78 |
79 |
80 | #
81 | ## These files are binary and should be left untouched
82 | #
83 |
84 | # (binary is a macro for -text -diff)
85 | *.png binary
86 | *.jpg binary
87 | *.jpeg binary
88 | *.gif binary
89 | *.ico binary
90 | *.mov binary
91 | *.mp4 binary
92 | *.mp3 binary
93 | *.flv binary
94 | *.fla binary
95 | *.swf binary
96 | *.gz binary
97 | *.zip binary
98 | *.7z binary
99 | *.ttf binary
100 | *.eot binary
101 | *.woff binary
102 | *.pyc binary
103 | *.pdf binary
104 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Don't check auto-generated stuff into git
2 | coverage
3 | node_modules
4 | stats.json
5 | package-lock.json
6 |
7 | # Cruft
8 | .DS_Store
9 | npm-debug.log
10 | .idea
11 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2020 marefati110
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 | [](https://github.com/marefati110/strapi-plugin-elasticsearch/issues)
12 | [](https://github.com/marefati110/strapi-plugin-elasticsearch/network)
13 | [](https://github.com/marefati110/strapi-plugin-elasticsearch/stargazers)
14 | [](https://github.com/marefati110/strapi-plugin-elasticsearch)
15 |
16 |
17 |
18 |
19 | tested on strapi v3.x
20 |
21 | latest test: v3.4.0
22 |
23 |
24 |
25 |
26 | This plugin has not been tested on mongodb
27 |
28 |
29 |
30 |
31 | The purpose of developing this plugin is to use the elastic search engine in Strapi to help the application development process
32 |
33 |
34 | ## 📝 Table of Contents
35 |
36 | - [Prerequisites](#prerequisites)
37 | - [Getting Started](#getting_started)
38 | - [How plugin work](#how_work)
39 | - [Usage](#usage)
40 | - [scenario 1](#scenario-1)
41 | - [scenario 2](#scenario-2)
42 | - [scenario 3](#scenario-3)
43 | - [scenario 4](#scenario-4)
44 | - [Functions](#functions)
45 | - [Api](#api)
46 | - [Example](#example)
47 | - [Logging](#logging)
48 | - [Authors](#authors)
49 |
50 | ## Prerequisites
51 |
52 |
53 |
54 | Install Elasticsearch - https://www.elastic.co/downloads/elasticsearch
55 |
56 | Install plugin
57 |
58 | - Go to the project path
59 |
60 | - `cd PROJECT/plugins`
61 |
62 | - Clone the project
63 |
64 | - `git submodule add https://github.com/marefati110/strapi-plugin-elasticsearch.git ./elastic`
65 |
66 | - Install dependencies
67 |
68 | - `yarn install`
69 |
70 | # 🏁 Getting Started
71 |
72 | ## How plugin works?
73 |
74 | After the first run of the project, it creates a config file at `PROJECT/config/elasticsearch.js`
75 |
76 | **config file should look like the image**
77 |
78 |
79 |
80 |
81 |
82 | By default, syncing occurs in two ways
83 |
84 | The answer of any request that makes a change in the model is stored in Elasticsearch this is especially true for the Strap panel admin
85 |
86 | Or in response to any request, search for the pk of model the model in which the change was made, and after retrieving the information from the database, stores it in the elasticsearch
87 |
88 | In the following, solutions are prepared for more complex scenarios.
89 |
90 | After installing the plugin and running it, it creates an config file in the `PROJECT/config/elasticsearch.js`
91 |
92 | In the connections section, the settings related to the connection to the elasticsearch are listed, there is also a help link
93 |
94 | In the setting section, there are the initial settings related to the elastic plugin.
95 |
96 | In the models section for all models in the `Project/api/**` path there is a model being built and you can change the initial settings
97 |
98 |
99 |
100 | # 🎈 Usage
101 |
102 | ### Scenario 1
103 |
104 | For example, we want to make changes to the article model and then see the changes in the Elasticsearch.
105 |
106 | The first step is to activate in the settings related to this model
107 |
108 | After saving and restarting the plugin, it creates an index for this model in the elasticsearch.
109 |
110 | Note that the name selected for the index can be changed in the settings of the model.
111 |
112 | At the end of the settings should be as follows
113 |
114 | ```js
115 | {
116 | model: 'article',
117 | pk: 'id',
118 | plugin: null, // changed to true
119 | enabled: true,
120 | index: 'article',
121 | relations: [],
122 | conditions: {},
123 | supportAdminPanel: true,
124 | fillByResponse: true,
125 | migration: false,
126 | urls: [],
127 | },
128 | ```
129 |
130 | Now in the strapi admin panel, by making an creating , deleting or updating , you can see the changes in Elasticsearch.
131 |
132 | ### Scenario 2
133 |
134 | In this scenario, we want to make a change in the model using the rest api and see the result in Elasticsearch.
135 |
136 | After sending a post request to `/articles`, changes will be applied and we will receive a response to this
137 |
138 | ```json
139 | {
140 | "id": 1,
141 | "title": "title",
142 | "content": "content"
143 | }
144 | ```
145 |
146 | and model config should change to
147 |
148 | ```js
149 | {
150 | model: 'article',
151 | pk: 'id',
152 | plugin: null,
153 | enabled: true,
154 | index: 'article',
155 | relations: [],
156 | conditions: {},
157 | supportAdminPanel: true,
158 | fillByResponse: true, // default value
159 | migration: false,
160 | urls: ['/articles'], //changed
161 | },
162 | ```
163 |
164 | If the `fillByResponse` settings are enabled for the model, the same data will be stored in Elasticsearch, otherwise the data will be retrieved from the database using pk and stored in Elasticsearch.
165 |
166 | ### Scenario 3
167 |
168 | This scenario is quite similar to the previous scenario with these differences being the response
169 |
170 | ```json
171 | {
172 | "metaData": null,
173 | "data": {
174 | "articleID": 1,
175 | "title": "title",
176 | "content": "content"
177 | }
178 | }
179 | ```
180 |
181 | By default, the plugin looks for pk in the response or `ctx.body.id`
182 |
183 | We can rewrite these settings for a specific url
184 |
185 | config model should change to
186 |
187 | ```js
188 | {
189 | model: 'article',
190 | pk: 'id',
191 | plugin: null,
192 | enabled: true,
193 | index: 'article',
194 | relations: [],
195 | conditions: {},
196 | supportAdminPanel: true,
197 | fillByResponse: true,
198 | migration: false,
199 | urls: [
200 | {
201 | '/articles':{
202 | pk: 'data.articleID', // over write
203 | relations: [], // over write
204 | conditions: {}, // over write
205 | }
206 | }
207 | ],
208 | },
209 | ```
210 |
211 | ### Scenario 4
212 |
213 | In this scenario, no pk may be found in the request response
214 |
215 | ```json
216 | {
217 | "success": true
218 | }
219 | ```
220 |
221 | In this case, the synchronization operation can be performed on the controller
222 |
223 | there is some functions for help
224 |
225 | ```js
226 | const articleData = { title: 'title', content: 'content' };
227 | const article = await strapi.query('article').create(articleData);
228 |
229 | strapi.elastic.createOrUpdate('article', { data: article, id: article.id });
230 | // or
231 | strapi.elastic.migrateById('article', { id: article.id }); // execute new query
232 | ```
233 |
234 | and for delete data
235 |
236 | ```js
237 | const articleId = 1;
238 | const article = await strapi.query('article').delete(articleData);
239 |
240 | strapi.elastic.destroy('article', { id: articleID });
241 | ```
242 |
243 | # Functions
244 |
245 | | Command | Description | example |
246 | | :------------------------------ | :----------------------------- | :--------------------------: |
247 | | `strapi.elastic` | official elasticsearch package | [example](#elastic) |
248 | | `strapi.elastic.createOrUpdate` | Create to update data | [example](#create_or_update) |
249 | | `strapi.elastic.findOne` | Find specific data by id | [example](#findOne) |
250 | | `strapi.elastic.destroy` | delete data | [example](#destroy) |
251 | | `strapi.elastic.migrateById` | migrate data | [example](#migrateById) |
252 | | `strapi.elastic.migrateModel` | migrate specific data | [example](#migrateModel) |
253 | | `strapi.elastic.models` | migrate all enabled models | [example](#models) |
254 | | `strapi.log` | log data to elasticsearch | [example](#logging) |
255 |
256 | # Api
257 |
258 | | Url | Method | Description | body |
259 | | :-------------- | :----: | :------------------------- | ---------------------- |
260 | | /migrate-models | POST | Migrate all enabled Models | |
261 | | /migrate-Model | POST | Migrate specific model | `{model:'MODEL_NAME'}` |
262 |
263 | # Examples
264 |
265 | ### elastic
266 |
267 | For use official Elasticsearch package we can use `strapi.elastic`, and can access builtin function
268 | [elasticsearch reference api](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference.html)
269 |
270 | ```js
271 | const count = strapi.elastic.count({ index: 'article' }); // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference.html#_count
272 |
273 | const article = strapi.elastic.get({ index: 'article', id: 1 }); // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference.html#_get
274 | ```
275 |
276 | ### CreateOrUpdate
277 |
278 | ```js
279 | const result = strapi.elastic.createOrUpdate('article', {
280 | id: 1,
281 | data: { title: 'title', content: 'content' },
282 | });
283 | ```
284 |
285 | ### findOne
286 |
287 | ```js
288 | const result = strapi.elastic.findOne('article', { id: 1 });
289 | ```
290 |
291 | ### destroy
292 |
293 | ```js
294 | const result_one = strapi.elastic.destroy('article', { id: 1 });
295 | // or
296 | const result_two = strapi.elastic.destroy('article', { id_in: [1, 2, 3] });
297 | ```
298 |
299 | ### migrateById
300 |
301 | ```js
302 | const result_one = strapi.elastic.migrateById('article', { id: 1 });
303 |
304 | const result_two = strapi.elastic.migrateById('article', { id_in: [1, 2, 3] });
305 | ```
306 |
307 | ### migrateModel
308 |
309 | ```js
310 | const result = strapi.elastic.migrateModel('article', {
311 | conditions, // optional
312 | });
313 | ```
314 |
315 | ### migrateModels
316 |
317 | ```js
318 | const result = strapi.elastic.migrateModels({
319 | conditions, // optional (the conditions apply on all models)
320 | });
321 | ```
322 |
323 | # Logging
324 |
325 | strapi use Pino to logging but can store logs or send it to elasticsearch
326 |
327 | at now wen can send logs to elasticsearch by `strapi.elastic.log` there is no difference between `strapi.elastic.log` with `strapi.log` to call functions.
328 |
329 | ```js
330 | strapi.log.info('log message in console');
331 | strapi.elastic.log.info('log message console and store it to elasticsearch');
332 |
333 | strapi.log.debug('log message');
334 | strapi.elastic.log.debug('log message console and store it to elasticsearch');
335 |
336 | strapi.log.warn('log message');
337 | strapi.elastic.log.warn('log message console and store it to elasticsearch');
338 |
339 | strapi.log.error('log message');
340 | strapi.elastic.log.error('log message console and store it to elasticsearch');
341 |
342 | strapi.log.fatal('log message');
343 | strapi.elastic.log.fatal('log message console and store it to elasticsearch');
344 | ```
345 |
346 | Also there is some more options
347 |
348 | ```js
349 | // just send log to elastic and avoid to display in console
350 | strapi.elastic.log.info('some message', { setting: { show: false } });
351 |
352 | // just display relations, // optional ni console and avoid to save it to elastic search
353 | strapi.elastic.log.info('some message', { setting: { saveToElastic: false } });
354 |
355 | // send more data to elasticsearch
356 | const logData = { description: 'description' };
357 | strapi.elastic.log.info('some message', logData);
358 | ```
359 |
360 | **By default `strapi.log` send some metaData to elasticsearch such as `free memory`, `cpu load avg`, `current time`, `hostname` ,...**
361 |
362 | # Tricks
363 |
364 | to avoid config plugin for all model or write a lot of code we can create cron job for migration
365 |
366 | ```js
367 | const moment = require('moment');
368 | module.exports = {
369 | '*/10 * * * *': async () => {
370 | const updateTime = moment()
371 | .subtract(10, 'minutes')
372 | .format('YYYY-MM-DD HH:mm:ss');
373 |
374 | // currentTime
375 | await strapi.elastic.migrateModels({
376 | conditions: {
377 | updated_at_gt: updateTime,
378 | /* to utilise Draft/Publish feature & migrate only published entities
379 | you can add following in conditions
380 | */
381 | _publicationState: 'live'
382 | },
383 | });
384 | },
385 | };
386 | ```
387 |
388 | ### ✍️ Authors
389 |
390 | - [@marefati110](https://github.com/marefati110)
391 |
--------------------------------------------------------------------------------
/admin/src/Components/LeftMenuItem/Wrapper.js:
--------------------------------------------------------------------------------
1 | import styled from 'styled-components';
2 |
3 | const Wrapper = styled.div`
4 | white-space: nowrap;
5 | overflow: hidden;
6 | text-overflow: ellipsis;
7 | cursor: pointer;
8 | display: flex;
9 | align-items: center;
10 | width: 100%;
11 | height: 35px;
12 | padding-left: 15px;
13 | background-color: ${({ active }) => (active ? '#e9eaeb' : 'transparent')};
14 | opacity: ${({ enable }) => (enable ? '1' : '0.4')};
15 |
16 | svg {
17 | color: rgb(145, 155, 174);
18 | width: 5px;
19 | height: 5px;
20 | }
21 | `;
22 |
23 | export default Wrapper;
24 |
--------------------------------------------------------------------------------
/admin/src/Components/LeftMenuItem/index.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import PropTypes from 'prop-types';
3 | import Wrapper from './Wrapper';
4 |
5 | const LeftMenu = ({ label, onClick, active, enable }) => {
6 | return (
7 | {
9 | if (enable) onClick();
10 | }}
11 | active={active}
12 | enable={enable}
13 | >
14 |
29 |
30 | {label}
31 |
32 | );
33 | };
34 |
35 | LeftMenu.propTypes = {
36 | label: PropTypes.string.isRequired,
37 | onClick: PropTypes.func.isRequired,
38 | active: PropTypes.bool.isRequired,
39 | };
40 |
41 | export default LeftMenu;
42 |
--------------------------------------------------------------------------------
/admin/src/assets/images/logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
3 |
19 |
--------------------------------------------------------------------------------
/admin/src/containers/App/index.js:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | * This component is the skeleton around the actual pages, and should only
4 | * contain code that should be seen on all pages. (e.g. navigation bar)
5 | *
6 | */
7 |
8 | import React from 'react';
9 | import { Switch, Route } from 'react-router-dom';
10 | import { NotFound } from 'strapi-helper-plugin';
11 | // Utils
12 | import pluginId from '../../pluginId';
13 | // Containers
14 | import HomePage from '../HomePage';
15 |
16 | const App = () => {
17 | return (
18 |
19 |
20 |
21 |
22 |
23 |
24 | );
25 | };
26 |
27 | export default App;
28 |
--------------------------------------------------------------------------------
/admin/src/containers/DataView/Wrapper.js:
--------------------------------------------------------------------------------
1 | import styled from 'styled-components';
2 |
3 | const Wrapper = styled.div`
4 | margin-left: 25px;
5 | margin-top: 15px;
6 | width: 76%;
7 | h2 {
8 | margin-bottom: 20px;
9 | }
10 | `;
11 |
12 | export default Wrapper;
13 |
--------------------------------------------------------------------------------
/admin/src/containers/DataView/index.js:
--------------------------------------------------------------------------------
1 | import React, { memo, useMemo, useState } from 'react';
2 | import PropTypes from 'prop-types';
3 | import { useHistory } from 'react-router-dom';
4 | import { GlobalPagination, request } from 'strapi-helper-plugin';
5 | import { isObject } from 'lodash';
6 | import { Table, Button, Select } from '@buffetjs/core';
7 | import { LoadingBar } from '@buffetjs/styles';
8 | import Wrapper from './Wrapper';
9 |
10 | const LIMIT_OPTIONS = ['10', '20', '50', '100'];
11 |
12 | const DataView = ({
13 | data = [],
14 | activeModel = '',
15 | loading,
16 | page,
17 | limit,
18 | totalCount,
19 | onChangeParams,
20 | isMigrateActive,
21 | isDeleted,
22 | isCreated,
23 | refreshData,
24 | }) => {
25 | const history = useHistory();
26 | const tableHeaders = useMemo(
27 | () =>
28 | data && data.length
29 | ? Object.keys(data[0]).map((d) => ({ name: d, value: d }))
30 | : [],
31 | [data]
32 | );
33 |
34 | const tableData = useMemo(
35 | () =>
36 | data && data.length
37 | ? data.map((dataObject) => {
38 | let newObj = {};
39 | if (!dataObject) return newObj;
40 |
41 | for (let key in dataObject) {
42 | if (isObject(dataObject[key])) {
43 | newObj[key] = JSON.stringify(dataObject[key], null, 2);
44 | } else {
45 | newObj[key] = dataObject[key];
46 | }
47 | }
48 |
49 | return newObj;
50 | })
51 | : [],
52 | [data]
53 | );
54 |
55 | const [isMigrating, setIsMigrating] = useState(false);
56 | const [isCreating, setIsCreating] = useState(false);
57 | const [isDeleting, setIsDeleting] = useState(false);
58 |
59 | const migrate = (model) => {
60 | setIsMigrating(true);
61 | request(`/elastic/migrate-model`, {
62 | method: 'POST',
63 | body: { model },
64 | })
65 | .then((res) => {
66 | if (res.success) {
67 | strapi.notification.success(`${model} model migrated successfully`);
68 | refreshData();
69 | } else strapi.notification.error(`migration failed`);
70 | })
71 | .catch(() => strapi.notification.error(`migration failed`))
72 | .finally(() => setIsMigrating(false));
73 | };
74 |
75 | const deleteIndex = (model) => {
76 | setIsDeleting(true);
77 | request(`/elastic/delete-index`, {
78 | method: 'POST',
79 | body: { model },
80 | })
81 | .then((res) => {
82 | if (res.success) {
83 | refreshData();
84 | strapi.notification.success(`${model} index deleted`);
85 | } else {
86 | strapi.notification.error(`cannot delete ${model} index`);
87 | }
88 | })
89 | .catch(() => {
90 | strapi.notification.error(`cannot delete ${model} index`);
91 | })
92 | .finally(() => setIsDeleting(false));
93 | };
94 |
95 | const createIndex = (model) => {
96 | setIsCreating(true);
97 | request(`/elastic/create-index`, {
98 | method: 'POST',
99 | body: { model },
100 | })
101 | .then((res) => {
102 | refreshData();
103 | if (res.success) {
104 | strapi.notification.success(`${model} index created`);
105 | } else {
106 | strapi.notification.error(`cannot create ${model} index`);
107 | }
108 | })
109 | .catch(() => strapi.notification.error(`cannot create ${model} index`))
110 | .finally(() => setIsCreating(false));
111 | };
112 |
113 | return (
114 |
115 |
116 |
{activeModel?.index?.toUpperCase()}
117 |
128 |
139 |
150 |
151 |
152 | {loading ? (
153 | new Array(10).fill(0).map(() => (
154 | <>
155 |
156 |
157 | >
158 | ))
159 | ) : (
160 | <>
161 |
165 | history.push(
166 | `/plugins/content-manager/collectionType/${
167 | activeModel?.plugin
168 | ? `plugins::${activeModel?.plugin}.${activeModel?.model}`
169 | : `application::${activeModel?.model}.${activeModel?.model}`
170 | }/${data.id}`
171 | )
172 | }
173 | />
174 |
175 |
182 |
183 |
191 |
192 |
193 | >
194 | )}
195 |
196 | );
197 | };
198 |
199 | DataView.propTypes = {
200 | data: PropTypes.array.isRequired,
201 | refreshData: PropTypes.func.isRequired,
202 | activeModel: PropTypes.string.isRequired,
203 | loading: PropTypes.bool.isRequired,
204 | page: PropTypes.number.isRequired,
205 | totalCount: PropTypes.number.isRequired,
206 | limit: PropTypes.string.isRequired,
207 | onChangeParams: PropTypes.func.isRequired,
208 | isMigrateActive: PropTypes.bool.isRequired,
209 | isDeleted: PropTypes.bool.isRequired,
210 | isCreated: PropTypes.bool.isRequired,
211 | };
212 |
213 | export default memo(DataView);
214 |
--------------------------------------------------------------------------------
/admin/src/containers/HomePage/GlobalStyles.js:
--------------------------------------------------------------------------------
1 | import { createGlobalStyle } from 'styled-components';
2 |
3 | const GlobalStyle = createGlobalStyle`
4 | td {
5 | max-width: 300px;
6 | white-space: nowrap;
7 | overflow: hidden;
8 | text-overflow: ellipsis;
9 | }
10 | button {
11 | min-width: unset !important;
12 | }
13 | `;
14 | export default GlobalStyle;
15 |
--------------------------------------------------------------------------------
/admin/src/containers/HomePage/index.js:
--------------------------------------------------------------------------------
1 | /*
2 | *
3 | * HomePage
4 | *
5 | */
6 |
7 | import React, { useEffect, useState, memo, useMemo } from 'react';
8 | import { request } from 'strapi-helper-plugin';
9 | // import PropTypes from 'prop-types';
10 | // import pluginId from '../../pluginId';
11 | import DataView from '../DataView';
12 | import LeftMenu from '../LeftMenu';
13 | import GlobalStyle from './GlobalStyles';
14 |
15 | const INITIAL_PAGE = 1;
16 | const INITIAL_LIMIT = '10';
17 |
18 | const HomePage = () => {
19 | const [models, setModels] = useState([]);
20 | const [activeModel, setActiveModel] = useState({});
21 | const [modelData, setModelData] = useState([]);
22 | const [loading, setLoading] = useState(false);
23 | const [page, setPage] = useState(INITIAL_PAGE);
24 | const [limit, setLimit] = useState(INITIAL_LIMIT); // it should be string for select
25 | const [totalCount, setTotalCount] = useState(10); // it should be string for select
26 | const [isCreated, setIsCreated] = useState(true);
27 | const [isDeleted, setIsDeleted] = useState(true);
28 | const [hasMapping, setHasMapping] = useState(true);
29 |
30 | const onChangeParams = ({ target }) => {
31 | switch (target.name) {
32 | case 'params._page':
33 | setPage(target.value);
34 | break;
35 |
36 | case 'params._limit':
37 | setLimit(target.value);
38 | break;
39 |
40 | default:
41 | break;
42 | }
43 | };
44 |
45 | // fetch data for active model
46 | const fetchData = () => {
47 | if (activeModel && activeModel.index) {
48 | // fetch for the model data
49 | setLoading(true);
50 | request(
51 | `/elastic/model?index=${activeModel.index}&_start=${page}&_limit=${limit}`,
52 | {
53 | method: 'GET',
54 | }
55 | )
56 | .then((res) => {
57 | setIsCreated(res.status.created);
58 | setIsDeleted(res.status.deleted);
59 | setHasMapping(res.status.hasMapping);
60 | setModelData(res.data);
61 | setTotalCount(res.total || 10);
62 | })
63 | .finally(() => setLoading(false));
64 | }
65 | };
66 |
67 | useEffect(() => {
68 | // fetch all models
69 | request(`/elastic/models`, {
70 | method: 'GET',
71 | }).then((res) => {
72 | if (res && res.length && res.length > 0) {
73 | setModels(res);
74 | setActiveModel(res[0]);
75 | }
76 | });
77 | }, []);
78 |
79 | useEffect(() => {
80 | fetchData();
81 | }, [activeModel, page, limit]);
82 |
83 | return (
84 |
85 |
86 |
87 | {
91 | setPage(INITIAL_PAGE);
92 | setLimit(INITIAL_LIMIT);
93 | setActiveModel(model);
94 | }}
95 | />
96 |
110 |
111 |
112 | );
113 | };
114 |
115 | export default memo(HomePage);
116 |
--------------------------------------------------------------------------------
/admin/src/containers/Initializer/index.js:
--------------------------------------------------------------------------------
1 | import { useEffect, useRef } from 'react';
2 | import PropTypes from 'prop-types';
3 | import pluginId from '../../pluginId';
4 |
5 | const Initializer = ({ updatePlugin }) => {
6 | const ref = useRef();
7 | ref.current = updatePlugin;
8 |
9 | useEffect(() => {
10 | ref.current(pluginId, 'isReady', true);
11 | }, []);
12 |
13 | return null;
14 | };
15 |
16 | Initializer.propTypes = {
17 | updatePlugin: PropTypes.func.isRequired,
18 | };
19 |
20 | export default Initializer;
21 |
--------------------------------------------------------------------------------
/admin/src/containers/LeftMenu/Wrapper.js:
--------------------------------------------------------------------------------
1 | import styled from 'styled-components';
2 |
3 | import { sizes } from 'strapi-helper-plugin';
4 |
5 | const Wrapper = styled.div`
6 | width: 20%;
7 | height: calc(100vh - ${sizes.header.height});
8 | overflow-y: auto;
9 | overflow-x: hidden;
10 | background-color: #f2f3f4;
11 | margin-left: 15px;
12 | padding-top: 10px;
13 | `;
14 |
15 | export default Wrapper;
16 |
--------------------------------------------------------------------------------
/admin/src/containers/LeftMenu/index.js:
--------------------------------------------------------------------------------
1 | import React from 'react';
2 | import PropTypes from 'prop-types';
3 | import Wrapper from './Wrapper';
4 | import LeftMenuItem from '../../Components/LeftMenuItem';
5 |
6 | const LeftMenu = ({ models, activeModel, setActiveModel }) => {
7 | console.log(
8 | '🚀 ~ file: index.js ~ line 7 ~ LeftMenu ~ activeModel',
9 | activeModel
10 | );
11 | return (
12 |
13 | Models
14 |
15 | {models && models.length
16 | ? models.map((model) => (
17 | setActiveModel(model)}
20 | active={model.index === activeModel?.index}
21 | enable={model.enabled}
22 | />
23 | ))
24 | : null}
25 |
26 | );
27 | };
28 |
29 | LeftMenu.propTypes = {
30 | models: PropTypes.array.isRequired,
31 | activeModel: PropTypes.object.isRequired,
32 | setActiveModel: PropTypes.func.isRequired,
33 | };
34 |
35 | export default LeftMenu;
36 |
--------------------------------------------------------------------------------
/admin/src/index.js:
--------------------------------------------------------------------------------
1 | import pluginPkg from '../../package.json';
2 | import pluginId from './pluginId';
3 | import App from './containers/App';
4 | import Initializer from './containers/Initializer';
5 | import lifecycles from './lifecycles';
6 | import trads from './translations';
7 | import pluginLogo from './assets/images/logo.svg';
8 |
9 | export default (strapi) => {
10 | const pluginDescription =
11 | pluginPkg.strapi.description || pluginPkg.description;
12 | const icon = pluginPkg.strapi.icon;
13 | const name = pluginPkg.strapi.name;
14 |
15 | const plugin = {
16 | blockerComponent: null,
17 | blockerComponentProps: {},
18 | description: pluginDescription,
19 | icon,
20 | id: pluginId,
21 | initializer: Initializer,
22 | injectedComponents: [],
23 | isReady: false,
24 | isRequired: pluginPkg.strapi.required || false,
25 | layout: null,
26 | lifecycles,
27 | mainComponent: App,
28 | name,
29 | preventComponentRendering: false,
30 | trads,
31 | pluginLogo,
32 | menu: {
33 | pluginsSectionLinks: [
34 | {
35 | destination: `/plugins/${pluginId}`,
36 | icon,
37 | label: {
38 | id: `${pluginId}.plugin.name`,
39 | defaultMessage: name,
40 | },
41 | name,
42 | permissions: [
43 | // Uncomment to set the permissions of the plugin here
44 | // {
45 | // action: '', // the action name should be plugins::plugin-name.actionType
46 | // subject: null,
47 | // },
48 | ],
49 | },
50 | ],
51 | },
52 | };
53 |
54 | return strapi.registerPlugin(plugin);
55 | };
56 |
--------------------------------------------------------------------------------
/admin/src/lifecycles.js:
--------------------------------------------------------------------------------
1 | function lifecycles() {}
2 |
3 | export default lifecycles;
4 |
--------------------------------------------------------------------------------
/admin/src/pluginId.js:
--------------------------------------------------------------------------------
1 | const pluginPkg = require('../../package.json');
2 | const pluginId = pluginPkg.name.replace(
3 | /^strapi-plugin-/i,
4 | ''
5 | );
6 |
7 | module.exports = pluginId;
8 |
--------------------------------------------------------------------------------
/admin/src/translations/ar.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/cs.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/de.json:
--------------------------------------------------------------------------------
1 | {}
2 |
--------------------------------------------------------------------------------
/admin/src/translations/en.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/es.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/fr.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/index.js:
--------------------------------------------------------------------------------
1 | import ar from './ar.json';
2 | import cs from './cs.json';
3 | import de from './de.json';
4 | import en from './en.json';
5 | import es from './es.json';
6 | import fr from './fr.json';
7 | import it from './it.json';
8 | import ko from './ko.json';
9 | import ms from './ms.json';
10 | import nl from './nl.json';
11 | import pl from './pl.json';
12 | import ptBR from './pt-BR.json';
13 | import pt from './pt.json';
14 | import ru from './ru.json';
15 | import th from './th.json';
16 | import tr from './tr.json';
17 | import uk from './uk.json';
18 | import vi from './vi.json';
19 | import zhHans from './zh-Hans.json';
20 | import zh from './zh.json';
21 | import sk from './sk.json';
22 |
23 | const trads = {
24 | ar,
25 | cs,
26 | de,
27 | en,
28 | es,
29 | fr,
30 | it,
31 | ko,
32 | ms,
33 | nl,
34 | pl,
35 | 'pt-BR': ptBR,
36 | pt,
37 | ru,
38 | th,
39 | tr,
40 | uk,
41 | vi,
42 | 'zh-Hans': zhHans,
43 | zh,
44 | sk,
45 | };
46 |
47 | export default trads;
48 |
--------------------------------------------------------------------------------
/admin/src/translations/it.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/ko.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/ms.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/nl.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/pl.json:
--------------------------------------------------------------------------------
1 | {}
2 |
--------------------------------------------------------------------------------
/admin/src/translations/pt-BR.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/pt.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/ru.json:
--------------------------------------------------------------------------------
1 | {}
2 |
--------------------------------------------------------------------------------
/admin/src/translations/sk.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/th.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/tr.json:
--------------------------------------------------------------------------------
1 | {}
2 |
--------------------------------------------------------------------------------
/admin/src/translations/uk.json:
--------------------------------------------------------------------------------
1 | {}
2 |
--------------------------------------------------------------------------------
/admin/src/translations/vi.json:
--------------------------------------------------------------------------------
1 | {}
--------------------------------------------------------------------------------
/admin/src/translations/zh-Hans.json:
--------------------------------------------------------------------------------
1 | {}
2 |
--------------------------------------------------------------------------------
/admin/src/translations/zh.json:
--------------------------------------------------------------------------------
1 | {}
2 |
--------------------------------------------------------------------------------
/admin/src/utils/getTrad.js:
--------------------------------------------------------------------------------
1 | import pluginId from '../pluginId';
2 |
3 | const getTrad = id => `${pluginId}.${id}`;
4 |
5 | export default getTrad;
6 |
--------------------------------------------------------------------------------
/config/constants.js:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/marefati110/strapi-plugin-elastic/83007056f8cd309b771295f6d096194bebc98dbf/config/constants.js
--------------------------------------------------------------------------------
/config/functions/bootstrap.js:
--------------------------------------------------------------------------------
1 | const { Client } = require('@elastic/elasticsearch');
2 |
3 | const {
4 | helper: { generateMainConfig, initialStrapi },
5 | } = require('../../services');
6 | const {
7 | logger,
8 | migrateModel,
9 | migrateModels,
10 | find,
11 | findOne,
12 | createOrUpdate,
13 | destroy,
14 | migrateById,
15 | } = require('../../services');
16 |
17 | module.exports = async () => {
18 | /**
19 | * generate elasticsearch config file
20 | */
21 | generateMainConfig();
22 |
23 | /**
24 | * initialize strapi.elastic object
25 | */
26 | if (strapi.config.elasticsearch) {
27 | const { connection } = strapi.config.elasticsearch;
28 |
29 | const client = new Client(connection);
30 |
31 | strapi.elastic = client;
32 |
33 | initialStrapi();
34 |
35 | const functions = {
36 | findOne,
37 | find,
38 | destroy,
39 | createOrUpdate,
40 | migrateModel,
41 | transferModelData: migrateModel,
42 | migrateModels,
43 | transferModelsData: migrateModels,
44 | migrateById,
45 | transferModelDataById: migrateById,
46 | log: logger,
47 | };
48 |
49 | Object.assign(strapi.elastic, functions);
50 |
51 | strapi.log.info('The elastic plugin is running');
52 | }
53 | };
54 |
--------------------------------------------------------------------------------
/config/routes.json:
--------------------------------------------------------------------------------
1 | {
2 | "routes": [
3 | {
4 | "method": "POST",
5 | "path": "/migrate-Models",
6 | "handler": "elasticsearch.migrateModels",
7 | "config": {
8 | "policies": []
9 | }
10 | },
11 | {
12 | "method": "POST",
13 | "path": "/migrate-model",
14 | "handler": "elasticsearch.migrateModel",
15 | "config": {
16 | "policies": []
17 | }
18 | },
19 | {
20 | "method": "POST",
21 | "path": "/generate-mappings/:model",
22 | "handler": "elasticsearch.generateIndexConfig",
23 | "config": {
24 | "policies": []
25 | }
26 | },
27 | {
28 | "method": "POST",
29 | "path": "/create-index",
30 | "handler": "elasticsearch.createIndex",
31 | "config": {
32 | "policies": []
33 | }
34 | },
35 | {
36 | "method": "POST",
37 | "path": "/delete-index",
38 | "handler": "elasticsearch.deleteIndex",
39 | "config": {
40 | "policies": []
41 | }
42 | },
43 | {
44 | "method": "GET",
45 | "path": "/models",
46 | "handler": "elasticsearch.fetchModels",
47 | "config": {
48 | "policies": []
49 | }
50 | },
51 | {
52 | "method": "GET",
53 | "path": "/model",
54 | "handler": "elasticsearch.fetchModel",
55 | "config": {
56 | "policies": []
57 | }
58 | }
59 | ]
60 | }
61 |
--------------------------------------------------------------------------------
/controllers/elasticsearch.js:
--------------------------------------------------------------------------------
1 | const _ = require('lodash');
2 | const {
3 | helper: { generateMappings, findMappingConfig },
4 | } = require('../services');
5 |
6 | module.exports = {
7 | fetchModels: (ctx) => {
8 | const { models } = strapi.config.elasticsearch;
9 |
10 | const enabledModels = models.filter((model) => model.enabled);
11 |
12 | const sortedEnabledModels = _.orderBy(enabledModels, ['model'], ['asc']);
13 |
14 | const disabledModels = models.filter((model) => !model.enabled);
15 |
16 | const sortedDisabledModels = _.orderBy(disabledModels, ['model'], ['asc']);
17 |
18 | // there is a bug here
19 | // models are not sorted
20 | const allModels = [...sortedEnabledModels, ...sortedDisabledModels];
21 |
22 | const response = _.map(
23 | allModels,
24 | _.partialRight(_.pick, [
25 | 'model',
26 | 'plugin',
27 | 'index',
28 | 'migration',
29 | 'pk',
30 | 'enabled',
31 | ])
32 | );
33 |
34 | return ctx.send(response);
35 | },
36 | fetchModel: async (ctx) => {
37 | const { index, _start, _limit } = ctx.query;
38 | let data, count, map;
39 | let status = {};
40 |
41 | try {
42 | //
43 | count = await strapi.elastic.count({ index });
44 | //
45 | map = await strapi.elastic.indices.getMapping({ index });
46 | //
47 | status = {
48 | deleted: false,
49 | created: true,
50 | };
51 | //
52 | } catch (e) {
53 | status = {
54 | deleted: true,
55 | created: false,
56 | };
57 | }
58 | if (status.created && !_.isEmpty(map.body[index])) {
59 | //
60 | status.hasMapping = true;
61 | //
62 | } else {
63 | //
64 | status.hasMapping = false;
65 | //
66 | }
67 | try {
68 | data = await strapi.elastic.search({
69 | index,
70 | size: _limit || 10,
71 | from: _limit * (_start - 1),
72 | body: {
73 | sort: [
74 | {
75 | updated_at: {
76 | order: 'desc',
77 | },
78 | },
79 | ],
80 | query: {
81 | match_all: {},
82 | },
83 | },
84 | });
85 | } catch (e) {
86 | return ctx.send({ data: null, total: 0, status });
87 | }
88 |
89 | if (data.statusCode !== 200) return ctx.badRequest();
90 |
91 | const res = [];
92 | for (const item of data.body.hits.hits) {
93 | const source = item['_source'];
94 | if (!_.isEmpty(source)) {
95 | //
96 | const sourceKeys = Object.keys(source);
97 |
98 | for (const key of sourceKeys) {
99 | //
100 | if (_.isArray(source[key])) {
101 | //
102 | source[key] = '[Array]';
103 | //
104 | } else if (_.isObject(source[key])) {
105 | //
106 | source[key] = '[Object]';
107 | //
108 | }
109 | }
110 | res.push(source);
111 | }
112 | }
113 | return ctx.send({
114 | data: res,
115 | total: count && count.body && count.body.count,
116 | status,
117 | });
118 | },
119 | migrateModels: async (ctx) => {
120 | await ctx.send({
121 | message: 'on progress it can take a few minuets',
122 | });
123 |
124 | strapi.elastic.migrateModels();
125 | },
126 | migrateModel: async (ctx) => {
127 | const { model } = ctx.request.body;
128 |
129 | await strapi.elastic.migrateModel(model);
130 | return ctx.send({ success: true });
131 | },
132 | generateIndexConfig: async (ctx) => {
133 | const data = ctx.request.body;
134 | const { model } = ctx.params;
135 |
136 | if (!data || !model) return ctx.badRequest();
137 |
138 | await strapi.elastic.index({
139 | index: 'strapi_elastic_lab',
140 | body: data,
141 | });
142 |
143 | const map = await strapi.elastic.indices.getMapping({
144 | index: 'strapi_elastic_lab',
145 | });
146 |
147 | await strapi.elastic.indices.delete({
148 | index: 'strapi_elastic_lab',
149 | });
150 |
151 | const { models } = strapi.config.elasticsearch;
152 | const targetModel = models.find((item) => item.model === model);
153 |
154 | await generateMappings({
155 | data: map.body['strapi_elastic_lab'],
156 | targetModels: targetModel,
157 | });
158 |
159 | return ctx.send({ success: true });
160 | },
161 | createIndex: async (ctx) => {
162 | const { model } = ctx.request.body;
163 |
164 | const { models } = strapi.config.elasticsearch;
165 | const targetModel = models.find((item) => item.model === model);
166 |
167 | const mapping = await findMappingConfig({ targetModel });
168 |
169 | const indexConfig = strapi.elastic.indicesMapping[targetModel.model];
170 |
171 | const options = {
172 | index: targetModel.index,
173 | };
174 |
175 | if (mapping || indexConfig) {
176 | options.body = mapping[targetModel.index] || indexConfig;
177 | }
178 |
179 | await strapi.elastic.indices.create(options);
180 |
181 | return ctx.send({ success: true });
182 | },
183 | deleteIndex: async (ctx) => {
184 | const { model } = ctx.request.body;
185 |
186 | const { models } = strapi.config.elasticsearch;
187 | const targetModel = models.find((item) => item.model === model);
188 |
189 | try {
190 | await strapi.elastic.indices.delete({
191 | index: targetModel.index,
192 | });
193 | return ctx.send({ success: true });
194 | } catch (e) {
195 | return ctx.throw(500);
196 | }
197 | },
198 | };
199 |
--------------------------------------------------------------------------------
/docs/CONFIG.md:
--------------------------------------------------------------------------------
1 | ```javascript
2 | module.exports = ({ env }) => ({
3 | connection: {
4 | node: env('ELASTICSEARCH_HOST', 'localhost'),
5 | auth: {
6 | username: env('ELASTICSEARCH_USERNAME', 'USERNAME'),
7 | password: env('ELASTICSEARCH_PASSWORD', 'PASSWORD'),
8 | },
9 | },
10 | setting: {
11 | validStatus: [200, 201],
12 | validMethod: ['PUT', 'POST', 'DELETE'],
13 | fillByResponse: false,
14 | importLimit: 3000,
15 | removeExistIndexForMigration: false,
16 | migration: {
17 | allowEntities: ['all'],
18 | disallowEntities: [],
19 | },
20 | },
21 | urls: {},
22 | adminUrls: {}
23 | );
24 | ```
25 |
--------------------------------------------------------------------------------
/middlewares/elastic/defaults.json:
--------------------------------------------------------------------------------
1 | {
2 | "elastic": {
3 | "enabled": true
4 | }
5 | }
6 |
--------------------------------------------------------------------------------
/middlewares/elastic/index.js:
--------------------------------------------------------------------------------
1 | const { elasticsearchManager } = require('../../services');
2 |
3 | module.exports = (strapi) => ({
4 | initialize() {
5 | strapi.app.use(async (ctx, next) => {
6 | await next();
7 | elasticsearchManager(ctx);
8 | });
9 | },
10 | });
11 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "strapi-plugin-elastic",
3 | "version": "1.3.3",
4 | "description": "this package help you easily send your data to elasticsearch.",
5 | "strapi": {
6 | "name": "Elasticsearch",
7 | "icon": "plug",
8 | "description": "this package help you easily send your data to elasticsearch."
9 | },
10 | "dependencies": {
11 | "@elastic/elasticsearch": "^7.8.0",
12 | "axios": "^0.20.0",
13 | "lodash": "^4.17.21",
14 | "moment": "^2.27.0"
15 | },
16 | "author": {
17 | "name": "Ali Marefati",
18 | "email": "marefati110@gmail.com",
19 | "url": "https://github.com/marefati110"
20 | },
21 | "maintainers": [
22 | {
23 | "name": "Ali Marefati",
24 | "email": "marefati110@gmail.com",
25 | "url": "https://github.com/marefati110"
26 | }
27 | ],
28 | "engines": {
29 | "node": ">=12.0.0",
30 | "npm": ">=6.0.0"
31 | },
32 | "license": "MIT"
33 | }
34 |
--------------------------------------------------------------------------------
/services/functions.js:
--------------------------------------------------------------------------------
1 | const _ = require('lodash');
2 | const { compareDataWithMap } = require('./helper');
3 | module.exports = {
4 | /**
5 | *
6 | * @param {string} model
7 | * @param {Object} query
8 | */
9 | find: async (model, query) => {
10 | const { models } = strapi.config.elasticsearch;
11 | const targetModel = models.find((item) => item.model === model);
12 |
13 | if (!targetModel) {
14 | strapi.log.error('model notfound');
15 | return;
16 | }
17 |
18 | try {
19 | const res = await strapi.elastic.search({
20 | index: targetModel.index,
21 | ...query,
22 | });
23 | return res;
24 | } catch (e) {
25 | strapi.log.error(e.message);
26 | }
27 | },
28 | /**
29 | *
30 | * @param {string} model
31 | * @param {Object|number|string} param1
32 | */
33 | findOne: async (model, pk) => {
34 | const { models } = strapi.config.elasticsearch;
35 | const targetModel = models.find((item) => item.model === model);
36 |
37 | let id;
38 | if (_.isObject(pk)) {
39 | id = pk.id;
40 | } else {
41 | id = pk;
42 | }
43 |
44 | if (!id) {
45 | strapi.log.error('id parameter is not valid');
46 | return;
47 | }
48 |
49 | if (!targetModel) {
50 | strapi.log.error('model notfound');
51 | return;
52 | }
53 |
54 | const result = await strapi.elastic.get({
55 | index: targetModel.index,
56 | id,
57 | });
58 |
59 | return result;
60 | },
61 | /**
62 | *
63 | * @param {string} model
64 | * @param {Object|string|number} pk
65 | */
66 | destroy: async (model, pk) => {
67 | let id_in;
68 |
69 | if (pk.id_in && !_.isArray(pk.id_in)) {
70 | strapi.log.error('id_in must be array');
71 | return;
72 | }
73 |
74 | if (!_.isObject(pk)) {
75 | id_in = [pk];
76 | } else {
77 | id_in = pk.id_in || [pk.id];
78 | }
79 |
80 | const { models } = strapi.config.elasticsearch;
81 | const targetModel = models.find((item) => item.model === model);
82 |
83 | if (!id_in) {
84 | strapi.log.error('pk parameter is not valid');
85 | }
86 |
87 | if (!targetModel) {
88 | strapi.log.error('model notfound');
89 | return;
90 | }
91 |
92 | const a = [];
93 |
94 | const body = id_in.map((id) => {
95 | return {
96 | delete: {
97 | _index: targetModel.index,
98 | _type: '_doc',
99 | _id: id,
100 | },
101 | };
102 | });
103 |
104 | try {
105 | return strapi.elastic.bulk({ body });
106 | } catch (e) {
107 | strapi.log.error(e.message);
108 | }
109 | },
110 | /**
111 | *
112 | * @param {string} model
113 | * @param {Object} param1
114 | */
115 | createOrUpdate: async (model, { id, data }) => {
116 | const { models } = strapi.config.elasticsearch;
117 | const targetModel = await models.find((item) => item.model === model);
118 |
119 | if (!data) {
120 | strapi.log.error('data property is not valid');
121 | return;
122 | }
123 |
124 | if (!targetModel) {
125 | strapi.log.error('model notfound');
126 | return;
127 | }
128 |
129 | const indexConfig = strapi.elastic.indicesMapping[targetModel.model];
130 |
131 | if (
132 | indexConfig &&
133 | indexConfig.mappings &&
134 | indexConfig.mappings.properties
135 | ) {
136 | const res = await compareDataWithMap({
137 | docs: data,
138 | properties: indexConfig.mappings.properties,
139 | });
140 | data = res.result || data;
141 | }
142 |
143 | let result;
144 | if (!id && data) {
145 | result = await strapi.elastic.index({
146 | index: targetModel.index,
147 | body: data,
148 | });
149 | } else if (id && data) {
150 | result = await strapi.elastic.update({
151 | index: targetModel.index,
152 | id: data[targetModel.pk || 'id'],
153 | body: {
154 | doc: data,
155 | doc_as_upsert: true,
156 | },
157 | });
158 |
159 | return result;
160 | }
161 | },
162 | /**
163 | *
164 | * @param {string} model
165 | * @param {Object} param1
166 | */
167 | migrateById: async (model, { id, id_in, relations, conditions }) => {
168 | const { models } = strapi.config.elasticsearch;
169 |
170 | const targetModel = models.find((item) => item.model === model);
171 |
172 | if (!targetModel) return null;
173 |
174 | id_in = id_in || [id];
175 |
176 | relations = relations || targetModel.relations;
177 | conditions = conditions || targetModel.conditions;
178 |
179 | const data = await strapi
180 | .query(targetModel.model, targetModel.plugin)
181 | .find({ id_in: [...id_in], ...conditions }, [...relations]);
182 |
183 | if (!data) return null;
184 |
185 | const body = await data.flatMap((doc) => [
186 | {
187 | index: {
188 | _index: targetModel.index,
189 | _id: doc[targetModel.pk || 'id'],
190 | _type: '_doc',
191 | },
192 | },
193 | doc,
194 | ]);
195 |
196 | const result = await strapi.elastic.bulk({ refresh: true, body });
197 |
198 | return result;
199 | },
200 | /**
201 | *
202 | * @param {string} model
203 | * @param {Object} params
204 | * @returns
205 | */
206 | migrateModel: async (model, params = {}) => {
207 | // specific condition
208 | params.conditions = params.conditions || {};
209 |
210 | const { models, setting } = strapi.config.elasticsearch;
211 |
212 | // set default value
213 | setting.importLimit = setting.importLimit || 3000;
214 |
215 | const targetModel = models.find((item) => item.model === model);
216 |
217 | let indexConfig = strapi.elastic.indicesMapping[targetModel.model];
218 |
219 | const { indexExist } = await strapi.elastic.indices.exists({
220 | index: targetModel.index,
221 | });
222 |
223 | indexConfig = indexExist ? indexConfig : null;
224 |
225 | if (
226 | !targetModel ||
227 | targetModel.enabled === false ||
228 | targetModel.migration === false
229 | )
230 | return;
231 |
232 | let start = 0;
233 | strapi.elastic.log.debug(`Importing ${targetModel.model} to elasticsearch`);
234 |
235 | let index_length = await strapi.query(targetModel.model).count();
236 | index_length = parseInt(index_length / setting.importLimit);
237 |
238 | // eslint-disable-next-line no-constant-condition
239 | while (true) {
240 | const start_sql = Date.now();
241 |
242 | strapi.log.debug(`Getting ${targetModel.model} model data from database`);
243 | let result = await strapi
244 | .query(targetModel.model, targetModel.plugin)
245 | .find(
246 | {
247 | _limit: setting.importLimit,
248 | _start: setting.importLimit * start,
249 | ...targetModel.conditions,
250 | ...params.conditions,
251 | },
252 | [...targetModel.relations]
253 | );
254 | if (result.length === 0) break;
255 |
256 | if (
257 | indexConfig &&
258 | indexConfig.mappings &&
259 | indexConfig.mappings.properties
260 | ) {
261 | const res = compareDataWithMap({
262 | docs: result,
263 | properties: indexConfig.mappings.properties,
264 | });
265 |
266 | result = res.result || result;
267 | //
268 | }
269 |
270 | //
271 | const end_sql = Date.now();
272 | //
273 | const body = await result.flatMap((doc) => [
274 | {
275 | index: {
276 | _index: targetModel.index,
277 | _id: doc[targetModel.pk || 'id'],
278 | _type: '_doc',
279 | },
280 | },
281 | doc,
282 | ]);
283 | //
284 | const start_elastic = Date.now();
285 |
286 | strapi.log.debug(
287 | `Sending ${targetModel.model} model to elasticsearch...`
288 | );
289 | try {
290 | await strapi.elastic.bulk({ refresh: true, body });
291 | } catch (e) {
292 | strapi.log.error(e);
293 | return;
294 | }
295 |
296 | const end_elastic = Date.now();
297 |
298 | start++;
299 |
300 | // progress bar
301 | strapi.log.info(
302 | `(${start}/${index_length + 1}) Imported to ${
303 | targetModel.index
304 | } index | sql query took ${parseInt(
305 | (end_sql - start_sql) / 1000
306 | )}s and insert to elasticsearch took ${parseInt(
307 | (end_elastic - start_elastic) / 1000
308 | )}s`
309 | );
310 |
311 | //
312 | }
313 | },
314 | /**
315 | *
316 | * @param {Object} params
317 | */
318 | migrateModels: async (params = {}) => {
319 | const { setting, models } = strapi.config.elasticsearch;
320 |
321 | params.models = params.models || [];
322 | params.conditions = params.conditions || {};
323 |
324 | // remove elasticsearch index before migration
325 | if (setting.removeExistIndexForMigration) {
326 | models.forEach(async (model) => {
327 | if (model.enabled && model.migration) {
328 | await strapi.elastic.indices.delete({ index: model.index });
329 | }
330 | });
331 | }
332 |
333 | if (params.models.length !== 0) {
334 | const targetModels = models.filter((item) =>
335 | params.models.includes(item.model)
336 | );
337 |
338 | // call migrateModel function for each model
339 | for (const item of targetModels) {
340 | await this.migrateModel(item.model, params);
341 | }
342 | } else {
343 | // call migrateModel function for each model
344 | for (const item of models) {
345 | await this.migrateModel(item.model, params);
346 | }
347 | }
348 |
349 | strapi.log.info('All models imported...');
350 | },
351 | };
352 |
--------------------------------------------------------------------------------
/services/helper.js:
--------------------------------------------------------------------------------
1 | 'user strict';
2 |
3 | const axios = require('axios');
4 | const fs = require('fs');
5 | const path = require('path');
6 | const _ = require('lodash');
7 |
8 | const modelConfigTemplate = (model) => ({
9 | model,
10 | index: model,
11 | plugin: null,
12 | enabled: false,
13 | migration: false,
14 | pk: 'id',
15 | relations: [],
16 | conditions: {},
17 | fillByResponse: true,
18 | supportAdminPanel: true,
19 | urls: [],
20 | });
21 |
22 | const isModel = (config) => {
23 | return config.model !== '.gitkeep';
24 | };
25 |
26 | const elasticsearchConfigTemplate = (modelsConfig) => `
27 | module.exports = ({ env }) => ({
28 | connection: {
29 | // https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/auth-reference.html
30 | node: env('ELASTICSEARCH_HOST', 'http://127.0.0.1:9200'),
31 | },
32 | setting: {
33 | importLimit: 3000,
34 | removeExistIndexForMigration: false,
35 | },
36 | models: ${JSON.stringify(modelsConfig, null, 2)}
37 | });`;
38 |
39 | module.exports = {
40 | checkRequest: (ctx) => {
41 | const { setting } = strapi.config.elasticsearch;
42 |
43 | setting.validMethod = setting.validMethod || ['PUT', 'POST', 'DELETE'];
44 | setting.validStatus = setting.validStatus || [200, 201];
45 |
46 | return (
47 | setting.validMethod.includes(ctx.request.method) &&
48 | setting.validStatus.includes(ctx.response.status)
49 | );
50 | },
51 | findModel: async ({ reqUrl, models }) => {
52 | let res;
53 |
54 | await models.forEach((model) => {
55 | model.urls.forEach((items) => {
56 | const re = new RegExp(items);
57 | if (_.isString(items)) {
58 | const status = re.test(reqUrl);
59 | if (status && model.enabled) {
60 | const targetModel = model;
61 | res = targetModel;
62 | }
63 | } else if (_.isObject(items)) {
64 | const urls = Object.keys(items);
65 | for (const url of urls) {
66 | const re = new RegExp(url);
67 | const status = re.test(reqUrl);
68 |
69 | if (status && model.enabled) {
70 | const targetModel = model;
71 | targetModel.pk = items[url].pk;
72 | targetModel.relations = items[url].relations || [];
73 | targetModel.conditions = items[url].conditions || {};
74 | targetModel.fillByResponse = _.isBoolean(
75 | items[url].fillByResponse
76 | )
77 | ? items[url].fillByResponse
78 | : true;
79 | res = targetModel;
80 | }
81 | }
82 | }
83 | });
84 | });
85 | return res;
86 | },
87 | isContentManagerUrl: async ({ models, reqUrl }) => {
88 | //
89 | const contentManagerUrlPattern =
90 | /\/content-manager\/(?:collection-types|single-types)\/([a-zA-Z-_]+)::([a-zA-Z-_]+).([a-zA-Z0-9_-]+)(?:\/(\d+))?/;
91 |
92 | const result = reqUrl.match(contentManagerUrlPattern);
93 |
94 | if (!result) return;
95 |
96 | const [, , , model] = result;
97 |
98 | const targetModel = await models.find((item) => item.model === model);
99 |
100 | if (
101 | !targetModel ||
102 | targetModel.enabled !== true ||
103 | targetModel.supportAdminPanel !== true
104 | )
105 | return;
106 |
107 | return targetModel;
108 | },
109 | isDeleteAllUrl: async ({ models, reqUrl }) => {
110 | const contentManagerUrlPattern =
111 | /^\/content-manager\/(?:collection-types|single-types)\/(\w+)\/\w*::([a-zA-Z-]+).([a-zA-Z0-9_-]+)|\/(\d*)/;
112 |
113 | const result = reqUrl.match(contentManagerUrlPattern);
114 |
115 | if (!result) return;
116 |
117 | const [, , , model] = result;
118 |
119 | const targetModel = await models.find(
120 | (configModel) => configModel.model === model
121 | );
122 |
123 | if (
124 | !targetModel ||
125 | targetModel.enabled === false ||
126 | targetModel.supportAdminPanel === false
127 | )
128 | return;
129 |
130 | return targetModel;
131 | },
132 | getDeleteIds: async ({ body, reqUrl }) => {
133 | const contentManagerUrlPattern =
134 | /\/content-manager\/(?:collection-types|single-types)\/(\w+)::([a-zA-Z-_]+).([a-zA-Z0-9_-]+)\/actions\/bulkDelete/;
135 |
136 | const result = contentManagerUrlPattern.test(reqUrl);
137 |
138 | if (!result || !body) return;
139 |
140 | const ids = [];
141 |
142 | for (const data of body) {
143 | ids.push(data.id);
144 | }
145 |
146 | return ids;
147 | },
148 | generateMainConfig: () => {
149 | const rootPath = path.resolve(__dirname, '../../../');
150 | const configPath = rootPath + '/config/elasticsearch.js';
151 |
152 | const existConfigFile = fs.existsSync(configPath);
153 |
154 | if (existConfigFile) return;
155 |
156 | const models = fs.readdirSync(rootPath + '/api');
157 |
158 | const modelsConfig = [];
159 |
160 | models.map((model) => {
161 | const config = modelConfigTemplate(model);
162 |
163 | if (isModel(config)) {
164 | modelsConfig.push(config);
165 | }
166 | });
167 |
168 | const elasticsearchConfig = elasticsearchConfigTemplate(modelsConfig);
169 | fs.writeFileSync(configPath, elasticsearchConfig, (err) => {
170 | if (err) throw err;
171 | });
172 | },
173 | compareDataWithMap: ({ properties, docs }) => {
174 | // initial variable;
175 | const elasticSearchNumericTypes = [
176 | 'long',
177 | 'integer',
178 | 'short',
179 | 'byte',
180 | 'double',
181 | 'float',
182 | 'half_float',
183 | 'scaled_float',
184 | 'unsigned_long',
185 | ];
186 | let outputDataType = 'array';
187 | let newMappings = false;
188 |
189 | const result = [];
190 |
191 | // convert docs(object) to array
192 | if (!_.isArray(docs)) {
193 | docs = [docs];
194 |
195 | // outputDataType use for remind input data type to return with same type
196 | outputDataType = 'object';
197 | }
198 | const propertiesKeys = Object.keys(properties);
199 |
200 | for (const doc of docs) {
201 | //
202 | const res = {};
203 | const dockKeyUsed = [];
204 |
205 | const docKeys = Object.keys(doc);
206 |
207 | for (const docKey of docKeys) {
208 | // check type of data with mapping in config
209 |
210 | if (propertiesKeys.includes(docKey)) {
211 | //
212 |
213 | const DOC = doc[docKey];
214 | const DOC_PROPERTY = properties[docKey].type;
215 |
216 | // recursive function for nested object/array
217 | if (
218 | _.isObject(DOC) &&
219 | _.isObject(properties[docKey].properties) &&
220 | !_.isDate(DOC) &&
221 | !_.isEmpty(DOC) &&
222 | !_.isEmpty(properties[docKey].properties)
223 | ) {
224 | const filteredData = module.exports.compareDataWithMap({
225 | properties: properties[docKey].properties,
226 | docs: DOC,
227 | });
228 |
229 | if (!_.isEmpty(filteredData.result)) {
230 | // check all element
231 | const finalArray = [];
232 | if (_.isArray(filteredData.result)) {
233 | //
234 | filteredData.result.forEach((item) => {
235 | //
236 | if (!_.isEmpty(item)) {
237 | //
238 | finalArray.push(item);
239 | //
240 | }
241 | //
242 | });
243 | //
244 | filteredData.result = finalArray;
245 | //
246 | }
247 |
248 | res[docKey] = filteredData.result;
249 |
250 | dockKeyUsed.push(docKey);
251 | //
252 | } else {
253 | //
254 | // res[docKey] = null;
255 | dockKeyUsed.push(docKey);
256 | //
257 | }
258 | newMappings = filteredData.newMappings;
259 |
260 | // check numbers
261 | } else if (
262 | _.isNumber(DOC) &&
263 | elasticSearchNumericTypes.includes(DOC_PROPERTY)
264 | ) {
265 | //
266 | res[docKey] = DOC;
267 | dockKeyUsed.push(docKey);
268 |
269 | // check strings
270 | } else if (_.isString(DOC) && DOC_PROPERTY === 'text') {
271 | //
272 | res[docKey] = DOC;
273 | dockKeyUsed.push(docKey);
274 |
275 | // check boolean
276 | } else if (_.isBoolean(DOC) && DOC_PROPERTY === 'boolean') {
277 | //
278 | res[docKey] = DOC;
279 | dockKeyUsed.push(docKey);
280 |
281 | // check date
282 | } else if (_.isDate(DOC) && DOC_PROPERTY === 'date') {
283 | //
284 | res[docKey] = DOC;
285 | dockKeyUsed.push(docKey);
286 |
287 | // check date
288 | } else if (_.isString(DOC) && DOC_PROPERTY === 'date') {
289 | //
290 | res[docKey] = DOC;
291 | dockKeyUsed.push(docKey);
292 |
293 | // other types
294 | } else {
295 | //
296 | // res[docKey] = null;
297 | dockKeyUsed.push(docKey);
298 | //
299 | }
300 | } else {
301 | //
302 | //some logic
303 | //
304 | }
305 | }
306 | // push property that exist in mapping config but not in entered data
307 | const mainKeys = _.difference(propertiesKeys, dockKeyUsed);
308 | for (const key of mainKeys) {
309 | res[key] = null;
310 | }
311 | result.push(res);
312 | }
313 | // return data it depends on outputDataType
314 | if (outputDataType === 'array') {
315 | //
316 | return { result, newMappings };
317 | //
318 | } else if (outputDataType === 'object') {
319 | //
320 | return { result: result[0], newMappings };
321 | //
322 | }
323 | },
324 | generateMappings: async ({ targetModels, data }) => {
325 | //
326 | if (!_.isArray(targetModels)) targetModels = [targetModels];
327 |
328 | const rootPath = path.resolve(__dirname, '../../../');
329 | const exportPath = `${rootPath}/exports/elasticsearch`;
330 |
331 | for (const targetModel of targetModels) {
332 | let map = {};
333 | // get mapping;
334 | if (!data) {
335 | map = await strapi.elastic.indices.getMapping({
336 | index: targetModel.index,
337 | });
338 | }
339 |
340 | if ((map && map.body) || data) {
341 | fs.writeFile(
342 | `${exportPath}/${targetModel.model}.index.json`,
343 | JSON.stringify(map.body || data, null, 2),
344 | (err) => {
345 | if (err) throw err;
346 | }
347 | );
348 | }
349 |
350 | //
351 | }
352 | },
353 | checkEnableModels: async () => {
354 | const { models } = strapi.config.elasticsearch;
355 |
356 | const enableModels = models.filter((model) => model.enabled);
357 |
358 | await enableModels.forEach(async (model) => {
359 | const indicesMapping = {};
360 | // const indexName = model.index_postfix + model.index + model.index_postfix;
361 | try {
362 | const indexMap = await strapi.elastic.indices.getMapping({
363 | index: model.index,
364 | });
365 |
366 | if (indexMap.status === 200) {
367 | indicesMapping[model.index] = indexMap.body;
368 | }
369 | } catch (e) {}
370 |
371 | strapi.elastic.indicesMapping = indicesMapping;
372 | });
373 | },
374 | checkNewVersion: async () => {
375 | const { setting } = strapi.config.elasticsearch;
376 |
377 | const currentVersion = setting.version;
378 |
379 | const releases = await axios.default.get(
380 | 'https://api.github.com/repos/marefati110/strapi-plugin-elastic/releases'
381 | );
382 |
383 | const lastVersion = releases.data[0];
384 |
385 | if (
386 | currentVersion !== lastVersion.tag_name &&
387 | lastVersion.prerelease === false
388 | ) {
389 | strapi.log.warn(
390 | 'There is new version for strapi-plugin-elastic. please update plugin.'
391 | );
392 | }
393 | },
394 | findMappingConfig: async ({ targetModel }) => {
395 | //
396 | const rootPath = path.resolve(__dirname, '../../../');
397 |
398 | const mappingConfigFilePath = `${rootPath}/exports/elasticsearch/${targetModel.model}.index.json`;
399 |
400 | const indicesMapConfigFile = fs.existsSync(mappingConfigFilePath);
401 |
402 | if (!indicesMapConfigFile) return;
403 |
404 | const map = require(mappingConfigFilePath);
405 |
406 | return map;
407 | },
408 | initialStrapi: async () => {
409 | strapi.elastic.indicesMapping = {};
410 |
411 | const indexFilePattern = /([a-zA-z0-9-_]*)\.index\.json/;
412 |
413 | const { models } = strapi.config.elasticsearch;
414 |
415 | const rootPath = path.resolve(__dirname, '../../../');
416 |
417 | const exportPath = `${rootPath}/exports/elasticsearch`;
418 |
419 | fs.mkdirSync(rootPath + '/exports/elasticsearch', { recursive: true });
420 |
421 | const indicesMapConfigFile = fs.readdirSync(exportPath);
422 |
423 | const enableModels = models.filter((model) => model.enabled);
424 |
425 | for (const index of indicesMapConfigFile) {
426 | //
427 | if (indexFilePattern.test(index)) {
428 | //
429 | const map = require(`${exportPath}/${index}`);
430 |
431 | const [, model] = index.match(indexFilePattern);
432 |
433 | const targetModel = models.find((item) => item.model === model);
434 |
435 | if (targetModel && targetModel.enabled) {
436 | strapi.elastic.indicesMapping[targetModel.model] =
437 | map[targetModel.index];
438 | }
439 | }
440 | }
441 |
442 | for (const targetModel of enableModels) {
443 | //
444 | if (!strapi.elastic.indicesMapping[targetModel.model]) {
445 | //
446 | try {
447 | //
448 |
449 | const indexMap = await strapi.elastic.indices.getMapping({
450 | index: targetModel.index,
451 | });
452 |
453 | if (indexMap.statusCode === 200) {
454 | //
455 | strapi.elastic.indicesMapping[targetModel.model] =
456 | indexMap.body[targetModel.index];
457 |
458 | module.exports.generateMappings({
459 | targetModels: targetModel,
460 | data: indexMap.body,
461 | });
462 |
463 | //
464 | }
465 | } catch (e) {
466 | strapi.log.warn(
467 | `There is an error to get mapping of ${targetModel.index} index from Elasticsearch`
468 | );
469 | }
470 | }
471 | }
472 |
473 | //
474 | },
475 | };
476 |
--------------------------------------------------------------------------------
/services/index.js:
--------------------------------------------------------------------------------
1 | const {
2 | createOrUpdate,
3 | destroy,
4 | find,
5 | findOne,
6 | migrateById,
7 | migrateModel,
8 | migrateModels,
9 | } = require('./functions');
10 |
11 | const { elasticsearchManager } = require('./middleware');
12 |
13 | const logger = require('./logger');
14 |
15 | const helper = require('./helper');
16 |
17 | module.exports = {
18 | createOrUpdate,
19 | find,
20 | destroy,
21 | elasticsearchManager,
22 | findOne,
23 | migrateById,
24 | migrateModel,
25 | migrateModels,
26 | logger,
27 | helper,
28 | };
29 |
--------------------------------------------------------------------------------
/services/logger.js:
--------------------------------------------------------------------------------
1 | const os = require('os');
2 | const moment = require('moment');
3 |
4 | const sendToElasticsearch = (data) => {
5 | if (data && data.setting && data.setting.saveToElastic === false) return;
6 |
7 | const index =
8 | (data.setting && data.setting.index) || 'strapi_elasticsearch_log';
9 |
10 | data.metaData = {
11 | pid: process.pid,
12 | free_mem: os.freemem(),
13 | total_mem: os.totalmem(),
14 | hostname: os.hostname(),
15 | loadavg: os.loadavg(),
16 | time: moment().format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
17 | };
18 |
19 | delete data.setting;
20 |
21 | return strapi.elastic.index({
22 | index,
23 | body: data,
24 | });
25 | };
26 |
27 | const displayLog = (data) => {
28 | // deprecated property
29 | let show = data && data.setting && data.setting.show;
30 | show = typeof show === 'boolean' ? show : true;
31 |
32 | let display = data && data.setting && data.setting.display;
33 | show = typeof display === 'boolean' ? display : true;
34 |
35 | return show && display;
36 | };
37 |
38 | const log = ({ level, msg, data }) => {
39 | if (displayLog(data)) {
40 | strapi.log[level](msg);
41 | }
42 | };
43 |
44 | module.exports = {
45 | custom: (msg, data) => {
46 | log({ msg, data, level: false });
47 | sendToElasticsearch({ msg, ...data, level: 'custom' });
48 | },
49 | warn: (msg, data) => {
50 | log({ msg, level: 'warn', data });
51 | sendToElasticsearch({ msg, ...data, level: 'warn' });
52 | },
53 | fatal: (msg, data) => {
54 | log({ msg, level: 'fatal', data });
55 | sendToElasticsearch({ msg, ...data, level: 'fatal' });
56 | },
57 | info: (msg, data) => {
58 | log({ msg, level: 'info', data });
59 | sendToElasticsearch({ msg, ...data, level: 'info' });
60 | },
61 | debug: (msg, data) => {
62 | log({ msg, level: 'debug', data });
63 | sendToElasticsearch({ msg, ...data, level: 'debug' });
64 | },
65 | error: (msg, data) => {
66 | log({ msg, level: 'error', data });
67 | sendToElasticsearch({ msg, ...data, level: 'error' });
68 | },
69 | };
70 |
--------------------------------------------------------------------------------
/services/middleware.js:
--------------------------------------------------------------------------------
1 | const _ = require('lodash');
2 |
3 | const {
4 | findModel,
5 | isContentManagerUrl,
6 | isDeleteAllUrl,
7 | getDeleteIds,
8 | checkRequest,
9 | } = require('./helper');
10 |
11 | /**
12 | *
13 | * @param {string} models
14 | * @param {string} url
15 | * @returns {null| Object}
16 | */
17 | const findTargetModel = async (models, url) => {
18 | let targetModel;
19 |
20 | targetModel = await isContentManagerUrl({ models, reqUrl: url });
21 |
22 | if (!targetModel) {
23 | targetModel = await isDeleteAllUrl({ models, reqUrl: url });
24 | }
25 |
26 | if (!targetModel) {
27 | targetModel = await findModel({ models, reqUrl: url });
28 | }
29 |
30 | return targetModel;
31 | };
32 |
33 | /**
34 | *
35 | * @param {string} url request url
36 | * @param {object} body request body
37 | * @param {object} targetModel model config in elasticsearch config file
38 | * @param {number|string} id record primary key
39 | */
40 | const deleteData = async (url, body, targetModel, id) => {
41 | let deleteIds;
42 |
43 | deleteIds = await getDeleteIds({ reqUrl: url, body: body });
44 |
45 | const id_in = !_.isEmpty(deleteIds) ? deleteIds : [id];
46 |
47 | if (_.isEmpty(id_in)) return;
48 | await strapi.elastic.destroy(targetModel.model, { id_in });
49 | };
50 |
51 | /**
52 | *
53 | * @param {object} body request body
54 | * @param {object} targetModel model config in elasticsearch config file
55 | * @param {*} id record primary key
56 | */
57 | const createOrUpdateData = async (body, targetModel, id) => {
58 | let data;
59 | data = targetModel.fillByResponse ? body : null;
60 |
61 | if (!data) {
62 | data = await strapi
63 | .query(targetModel.model, targetModel.plugin)
64 | .findOne({ id, ...targetModel.conditions }, [
65 | ...targetModel.relations,
66 | 'created_by',
67 | 'updated_by',
68 | ]);
69 | }
70 |
71 | if (!data || !id) return;
72 |
73 | await strapi.elastic.createOrUpdate(targetModel.model, { id, data });
74 | };
75 |
76 | module.exports = {
77 | /**
78 | *
79 | * @param {Object} ctx request context
80 | */
81 | elasticsearchManager: async (ctx) => {
82 | const isValidReq = checkRequest(ctx);
83 | if (!isValidReq) return;
84 |
85 | const { url, method } = ctx.request;
86 | const { body } = ctx;
87 | const { models } = strapi.config.elasticsearch;
88 |
89 | const targetModel = await findTargetModel(models, url);
90 | if (!targetModel) return;
91 |
92 | // set default value
93 | targetModel.fillByResponse = _.isBoolean(targetModel.fillByResponse)
94 | ? targetModel.fillByResponse
95 | : true;
96 | const pk = targetModel.pk || 'id';
97 |
98 | const { id } =
99 | _.pick(body, pk) || _.pick(ctx.params, pk) || _.pick(ctx.query, pk);
100 |
101 | const isPostOrPutMethod = method === 'POST' || method === 'PUT';
102 | const isDeleteMethod = ctx.request.method === 'DELETE';
103 |
104 | if (isDeleteMethod) {
105 | await deleteData(url, body, targetModel, id);
106 | } else if (isPostOrPutMethod) {
107 | await createOrUpdateData(body, targetModel, id);
108 | }
109 | },
110 | };
111 |
--------------------------------------------------------------------------------
/yarn-error.log:
--------------------------------------------------------------------------------
1 | Arguments:
2 | /usr/bin/node /usr/bin/yarn audit --json
3 |
4 | PATH:
5 | /home/ali/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/usr/local/go/bin:/home/ali/google-cloud-sdk/bin:/home/ali/.cache/cloud-code/installer/google-cloud-sdk/bin
6 |
7 | Yarn version:
8 | 1.22.10
9 |
10 | Node version:
11 | 14.15.5
12 |
13 | Platform:
14 | linux x64
15 |
16 | Trace:
17 | Error: connect ENETUNREACH 104.16.22.35:443 - Local (0.0.0.0:0)
18 | at internalConnect (net.js:921:16)
19 | at defaultTriggerAsyncIdScope (internal/async_hooks.js:430:12)
20 | at emitLookup (net.js:1064:9)
21 | at /usr/lib/node_modules/yarn/lib/cli.js:107131:28
22 | at /usr/lib/node_modules/yarn/lib/cli.js:107040:13
23 | at RawTask.module.exports.RawTask.call (/usr/lib/node_modules/yarn/lib/cli.js:83614:19)
24 | at flush (/usr/lib/node_modules/yarn/lib/cli.js:83696:29)
25 | at processTicksAndRejections (internal/process/task_queues.js:75:11)
26 |
27 | npm manifest:
28 | {
29 | "name": "strapi-plugin-elastic",
30 | "version": "1.1.0",
31 | "description": "this package help you easily send your data to elasticsearch.",
32 | "strapi": {
33 | "name": "Elasticsearch",
34 | "icon": "plug",
35 | "description": "this package help you easily send your data to elasticsearch."
36 | },
37 | "dependencies": {
38 | "@elastic/elasticsearch": "^7.8.0",
39 | "moment": "^2.27.0",
40 | "axios": "^0.20.0",
41 | "flat": "^5.0.2"
42 | },
43 | "author": {
44 | "name": "Ali Marefati",
45 | "email": "marefati110@gmail.com",
46 | "url": "https://github.com/marefati110"
47 | },
48 | "maintainers": [
49 | {
50 | "name": "Ali Marefati",
51 | "email": "marefati110@gmail.com",
52 | "url": "https://github.com/marefati110"
53 | }
54 | ],
55 | "engines": {
56 | "node": ">=12.0.0",
57 | "npm": ">=6.0.0"
58 | },
59 | "license": "MIT"
60 | }
61 |
62 | yarn manifest:
63 | No manifest
64 |
65 | Lockfile:
66 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
67 | # yarn lockfile v1
68 |
69 |
70 | "@elastic/elasticsearch@^7.8.0":
71 | version "7.9.0"
72 | resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.9.0.tgz#098f8adbe45cba1864ea5edc59638ea983904fe5"
73 | integrity sha512-iSLQvQafspN03YayzccShkKgJeRsUbncbtIhIL2SeiH01xwdnOZcp0nCvSNaMsH28A3YQ4ogTs9K8eXe42UaUA==
74 | dependencies:
75 | debug "^4.1.1"
76 | decompress-response "^4.2.0"
77 | ms "^2.1.1"
78 | pump "^3.0.0"
79 | secure-json-parse "^2.1.0"
80 |
81 | debug@^4.1.1:
82 | version "4.2.0"
83 | resolved "https://registry.yarnpkg.com/debug/-/debug-4.2.0.tgz#7f150f93920e94c58f5574c2fd01a3110effe7f1"
84 | integrity sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==
85 | dependencies:
86 | ms "2.1.2"
87 |
88 | decompress-response@^4.2.0:
89 | version "4.2.1"
90 | resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986"
91 | integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==
92 | dependencies:
93 | mimic-response "^2.0.0"
94 |
95 | end-of-stream@^1.1.0:
96 | version "1.4.4"
97 | resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
98 | integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
99 | dependencies:
100 | once "^1.4.0"
101 |
102 | mimic-response@^2.0.0:
103 | version "2.1.0"
104 | resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43"
105 | integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==
106 |
107 | ms@2.1.2, ms@^2.1.1:
108 | version "2.1.2"
109 | resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
110 | integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
111 |
112 | once@^1.3.1, once@^1.4.0:
113 | version "1.4.0"
114 | resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
115 | integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
116 | dependencies:
117 | wrappy "1"
118 |
119 | os-utils@^0.0.14:
120 | version "0.0.14"
121 | resolved "https://registry.yarnpkg.com/os-utils/-/os-utils-0.0.14.tgz#29e511697b1982b8c627722175fe39797ef64156"
122 | integrity sha1-KeURaXsZgrjGJ3Ihdf45eX72QVY=
123 |
124 | pump@^3.0.0:
125 | version "3.0.0"
126 | resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
127 | integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==
128 | dependencies:
129 | end-of-stream "^1.1.0"
130 | once "^1.3.1"
131 |
132 | secure-json-parse@^2.1.0:
133 | version "2.1.0"
134 | resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.1.0.tgz#ae76f5624256b5c497af887090a5d9e156c9fb20"
135 | integrity sha512-GckO+MS/wT4UogDyoI/H/S1L0MCcKS1XX/vp48wfmU7Nw4woBmb8mIpu4zPBQjKlRT88/bt9xdoV4111jPpNJA==
136 |
137 | wrappy@1:
138 | version "1.0.2"
139 | resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
140 | integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
141 |
--------------------------------------------------------------------------------
/yarn.lock:
--------------------------------------------------------------------------------
1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
2 | # yarn lockfile v1
3 |
4 |
5 | "@elastic/elasticsearch@^7.8.0":
6 | version "7.9.0"
7 | resolved "https://registry.yarnpkg.com/@elastic/elasticsearch/-/elasticsearch-7.9.0.tgz#098f8adbe45cba1864ea5edc59638ea983904fe5"
8 | integrity sha512-iSLQvQafspN03YayzccShkKgJeRsUbncbtIhIL2SeiH01xwdnOZcp0nCvSNaMsH28A3YQ4ogTs9K8eXe42UaUA==
9 | dependencies:
10 | debug "^4.1.1"
11 | decompress-response "^4.2.0"
12 | ms "^2.1.1"
13 | pump "^3.0.0"
14 | secure-json-parse "^2.1.0"
15 |
16 | debug@^4.1.1:
17 | version "4.2.0"
18 | resolved "https://registry.yarnpkg.com/debug/-/debug-4.2.0.tgz#7f150f93920e94c58f5574c2fd01a3110effe7f1"
19 | integrity sha512-IX2ncY78vDTjZMFUdmsvIRFY2Cf4FnD0wRs+nQwJU8Lu99/tPFdb0VybiiMTPe3I6rQmwsqQqRBvxU+bZ/I8sg==
20 | dependencies:
21 | ms "2.1.2"
22 |
23 | decompress-response@^4.2.0:
24 | version "4.2.1"
25 | resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-4.2.1.tgz#414023cc7a302da25ce2ec82d0d5238ccafd8986"
26 | integrity sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==
27 | dependencies:
28 | mimic-response "^2.0.0"
29 |
30 | end-of-stream@^1.1.0:
31 | version "1.4.4"
32 | resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
33 | integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
34 | dependencies:
35 | once "^1.4.0"
36 |
37 | mimic-response@^2.0.0:
38 | version "2.1.0"
39 | resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-2.1.0.tgz#d13763d35f613d09ec37ebb30bac0469c0ee8f43"
40 | integrity sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==
41 |
42 | ms@2.1.2, ms@^2.1.1:
43 | version "2.1.2"
44 | resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
45 | integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
46 |
47 | once@^1.3.1, once@^1.4.0:
48 | version "1.4.0"
49 | resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
50 | integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E=
51 | dependencies:
52 | wrappy "1"
53 |
54 | os-utils@^0.0.14:
55 | version "0.0.14"
56 | resolved "https://registry.yarnpkg.com/os-utils/-/os-utils-0.0.14.tgz#29e511697b1982b8c627722175fe39797ef64156"
57 | integrity sha1-KeURaXsZgrjGJ3Ihdf45eX72QVY=
58 |
59 | pump@^3.0.0:
60 | version "3.0.0"
61 | resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64"
62 | integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==
63 | dependencies:
64 | end-of-stream "^1.1.0"
65 | once "^1.3.1"
66 |
67 | secure-json-parse@^2.1.0:
68 | version "2.1.0"
69 | resolved "https://registry.yarnpkg.com/secure-json-parse/-/secure-json-parse-2.1.0.tgz#ae76f5624256b5c497af887090a5d9e156c9fb20"
70 | integrity sha512-GckO+MS/wT4UogDyoI/H/S1L0MCcKS1XX/vp48wfmU7Nw4woBmb8mIpu4zPBQjKlRT88/bt9xdoV4111jPpNJA==
71 |
72 | wrappy@1:
73 | version "1.0.2"
74 | resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
75 | integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=
76 |
--------------------------------------------------------------------------------