├── .editorconfig ├── .gitattributes ├── .gitignore ├── README.md ├── admin └── src │ ├── components │ └── MappingTable │ │ ├── MappingOptions.js │ │ ├── TargetFieldSelect.js │ │ └── index.js │ ├── containers │ ├── App │ │ ├── actions.js │ │ ├── constants.js │ │ ├── index.js │ │ ├── reducer.js │ │ └── selectors.js │ ├── CreateImportPage │ │ ├── ExternalUrlForm │ │ │ ├── index.js │ │ │ └── styles.scss │ │ ├── InputFormatSettings │ │ │ ├── index.js │ │ │ └── styles.scss │ │ ├── RawInputForm │ │ │ ├── index.js │ │ │ └── styles.scss │ │ ├── UploadFileForm │ │ │ └── index.js │ │ ├── actions.js │ │ ├── constants.js │ │ ├── index.js │ │ ├── reducer.js │ │ ├── saga.js │ │ ├── selectors.js │ │ └── styles.scss │ ├── HomePage │ │ ├── actions.js │ │ ├── constants.js │ │ ├── index.js │ │ ├── reducer.js │ │ ├── saga.js │ │ ├── selectors.js │ │ └── styles.scss │ └── NotFoundPage │ │ └── index.js │ ├── pluginId.js │ └── translations │ ├── ar.json │ ├── de.json │ ├── en.json │ ├── es.json │ ├── fr.json │ ├── it.json │ ├── ko.json │ ├── nl.json │ ├── pl.json │ ├── pt-BR.json │ ├── pt.json │ ├── ru.json │ ├── tr.json │ ├── zh-Hans.json │ └── zh.json ├── azure-pipelines.yml ├── config ├── functions │ └── bootstrap.js ├── queries │ ├── bookshelf.js │ └── mongoose.js ├── routes.json └── settings.json ├── controllers └── ImportConfig.js ├── models ├── ImportConfig.js ├── ImportConfig.settings.json ├── Importeditem.js └── Importeditem.settings.json ├── package.json ├── services ├── ImportConfig.js ├── ImportItems.js ├── UndoItems.js └── utils │ ├── FileDataResolver.js │ ├── FileDataResolver.spec.js │ ├── analyzer.js │ ├── analyzer.spec.js │ ├── fieldUtils.js │ ├── fieldUtils.spec.js │ ├── fileFromBuffer.js │ ├── fileUtils.js │ ├── fileUtils.spec.js │ ├── getMediaUrlsFromFieldData.js │ ├── getMediaUrlsFromFieldData.spec.js │ ├── getUploadProvider.js │ ├── importFields.js │ ├── importFields.spec.js │ ├── importMediaFiles.js │ ├── stringIsEmail.js │ ├── stringIsEmail.spec.js │ ├── urlIsMedia.js │ ├── urlIsMedia.spec.js │ ├── validateUrl.js │ └── validateUrl.spec.js └── video_thumbnail.png /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = false 6 | indent_style = space 7 | indent_size = 2 8 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # From https://github.com/Danimoth/gitattributes/blob/master/Web.gitattributes 2 | 3 | # Handle line endings automatically for files detected as text 4 | # and leave all files detected as binary untouched. 5 | * text=auto 6 | 7 | # 8 | # The above will handle all files NOT found below 9 | # 10 | 11 | # 12 | ## These files are text and should be normalized (Convert crlf => lf) 13 | # 14 | 15 | # source code 16 | *.php text 17 | *.css text 18 | *.sass text 19 | *.scss text 20 | *.less text 21 | *.styl text 22 | *.js text eol=lf 23 | *.coffee text 24 | *.json text 25 | *.htm text 26 | *.html text 27 | *.xml text 28 | *.svg text 29 | *.txt text 30 | *.ini text 31 | *.inc text 32 | *.pl text 33 | *.rb text 34 | *.py text 35 | *.scm text 36 | *.sql text 37 | *.sh text 38 | *.bat text 39 | 40 | # templates 41 | *.ejs text 42 | *.hbt text 43 | *.jade text 44 | *.haml text 45 | *.hbs text 46 | *.dot text 47 | *.tmpl text 48 | *.phtml text 49 | 50 | # git config 51 | .gitattributes text 52 | .gitignore text 53 | .gitconfig text 54 | 55 | # code analysis config 56 | .jshintrc text 57 | .jscsrc text 58 | .jshintignore text 59 | .csslintrc text 60 | 61 | # misc config 62 | *.yaml text 63 | *.yml text 64 | .editorconfig text 65 | 66 | # build config 67 | *.npmignore text 68 | *.bowerrc text 69 | 70 | # Heroku 71 | Procfile text 72 | .slugignore text 73 | 74 | # Documentation 75 | *.md text 76 | LICENSE text 77 | AUTHORS text 78 | 79 | 80 | # 81 | ## These files are binary and should be left untouched 82 | # 83 | 84 | # (binary is a macro for -text -diff) 85 | *.png binary 86 | *.jpg binary 87 | *.jpeg binary 88 | *.gif binary 89 | *.ico binary 90 | *.mov binary 91 | *.mp4 binary 92 | *.mp3 binary 93 | *.flv binary 94 | *.fla binary 95 | *.swf binary 96 | *.gz binary 97 | *.zip binary 98 | *.7z binary 99 | *.ttf binary 100 | *.eot binary 101 | *.woff binary 102 | *.pyc binary 103 | *.pdf binary 104 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Don't check auto-generated stuff into git 2 | coverage 3 | node_modules 4 | stats.json 5 | package-lock.json 6 | admin/build 7 | 8 | # Cruft 9 | .DS_Store 10 | npm-debug.log 11 | .idea 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Strapi Import Content plugin 2 | 3 | Import RSS items to your content type with Strapi. 4 | 5 | [![Build Status](https://dev.azure.com/joebeuckman0156/Strapi%20Plugins/_apis/build/status/jbeuckm.strapi-plugin-import-content?branchName=master)](https://dev.azure.com/joebeuckman0156/Strapi%20Plugins/_build/latest?definitionId=1&branchName=master) 6 | 7 | ### Installation 8 | 9 | ``` 10 | cd my-strapi-project/plugins 11 | git clone https://github.com/jbeuckm/strapi-plugin-import-content.git import-content 12 | cd import-content && npm install 13 | cd ../.. 14 | npm run setup --plugins 15 | ``` 16 | 17 | _\* the last step takes a notoriously long time..._ 18 | 19 | ### Configuration 20 | 21 | When plugin has been installed, you need to allow access to the endpoints. 22 | 23 | 1. Navigate to Users & Permissions. 24 | 2. Pick the role you would like to give permission. 25 | 3. Scroll down and expand the section **Import Content**. 26 | 4. Check "Select All" for the endpoints under "Importconfig". 27 | 5. Scroll up and press "Save" 28 | 29 | ### Usage 30 | 31 | Click for video demo: 32 | [![Click for demo video](video_thumbnail.png)](https://youtu.be/NOFioYMKPJk) 33 | -------------------------------------------------------------------------------- /admin/src/components/MappingTable/MappingOptions.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import TargetFieldSelect from './TargetFieldSelect'; 3 | 4 | const MappingOptions = ({ stat, onChange, targetModel }) => { 5 | return ( 6 |
7 | {stat.format === 'xml' && ( 8 |
9 | 10 | onChange({ stripTags: e.target.checked })} 13 | /> 14 |
15 | )} 16 | {stat.hasMediaUrls && ( 17 |
18 | 19 | 22 | onChange({ importMediaToField: targetField }) 23 | } 24 | /> 25 |
26 | )} 27 |
28 | ); 29 | }; 30 | 31 | export default MappingOptions; 32 | -------------------------------------------------------------------------------- /admin/src/components/MappingTable/TargetFieldSelect.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | const TargetFieldSelect = ({ targetModel, onChange }) => ( 4 | 16 | ); 17 | 18 | export default TargetFieldSelect; 19 | -------------------------------------------------------------------------------- /admin/src/components/MappingTable/index.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import PropTypes from 'prop-types'; 3 | import MappingOptions from './MappingOptions'; 4 | import TargetFieldSelect from './TargetFieldSelect'; 5 | import _ from 'lodash'; 6 | 7 | class MappingTable extends Component { 8 | state = { mapping: {} }; 9 | 10 | changeMappingOptions = stat => options => { 11 | console.log(stat, options); 12 | 13 | let newState = _.cloneDeep(this.state); 14 | for (let key in options) { 15 | _.set(newState, `mapping[${stat.fieldName}][${key}]`, options[key]); 16 | } 17 | this.setState(newState, () => this.props.onChange(this.state.mapping)); 18 | }; 19 | 20 | setMapping = (source, targetField) => { 21 | const state = _.set( 22 | this.state, 23 | `mapping[${source}]['targetField']`, 24 | targetField 25 | ); 26 | this.setState(state, () => this.props.onChange(this.state.mapping)); 27 | }; 28 | 29 | render() { 30 | const { analysis, targetModel } = this.props; 31 | return ( 32 |
33 |
34 |
Found {analysis.itemCount} items...
35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | {analysis.fieldStats.map(stat => ( 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 61 | 71 | 72 | ))} 73 |
Field NameCountFormatMin LengthMax LengthAvg LengthOptionsDestination
{stat.fieldName}{stat.count}{stat.format}{stat.minLength}{stat.maxLength}{stat.meanLength} 55 | 60 | 62 | {targetModel && ( 63 | 66 | this.setMapping(stat.fieldName, targetField) 67 | } 68 | /> 69 | )} 70 |
74 |
75 |
76 | ); 77 | } 78 | } 79 | 80 | MappingTable.propTypes = { 81 | analysis: PropTypes.object.isRequired, 82 | targetModel: PropTypes.object, 83 | onChange: PropTypes.func 84 | }; 85 | 86 | export default MappingTable; 87 | -------------------------------------------------------------------------------- /admin/src/containers/App/actions.js: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * App actions 4 | * 5 | */ 6 | -------------------------------------------------------------------------------- /admin/src/containers/App/constants.js: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * App constants 4 | * 5 | */ 6 | -------------------------------------------------------------------------------- /admin/src/containers/App/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * This component is the skeleton around the actual pages, and should only 4 | * contain code that should be seen on all pages. (e.g. navigation bar) 5 | * 6 | */ 7 | 8 | import React from 'react'; 9 | import PropTypes from 'prop-types'; 10 | import { connect } from 'react-redux'; 11 | import { createStructuredSelector } from 'reselect'; 12 | import { Switch, Route } from 'react-router-dom'; 13 | import { compose } from 'redux'; 14 | 15 | import pluginId from 'pluginId'; 16 | 17 | import HomePage from 'containers/HomePage'; 18 | import NotFoundPage from 'containers/NotFoundPage'; 19 | import CreateImportPage from 'containers/CreateImportPage'; 20 | 21 | import reducer from './reducer'; 22 | 23 | class App extends React.Component { 24 | render() { 25 | return ( 26 |
27 | 28 | 29 | 34 | 35 | 36 |
37 | ); 38 | } 39 | } 40 | 41 | App.contextTypes = { 42 | plugins: PropTypes.object, 43 | updatePlugin: PropTypes.func 44 | }; 45 | 46 | App.propTypes = { 47 | history: PropTypes.object.isRequired 48 | }; 49 | 50 | const mapStateToProps = createStructuredSelector({}); 51 | 52 | const withConnect = connect(mapStateToProps); 53 | const withReducer = strapi.injectReducer({ key: 'global', reducer, pluginId }); 54 | 55 | export default compose( 56 | withReducer, 57 | withConnect 58 | )(App); 59 | -------------------------------------------------------------------------------- /admin/src/containers/App/reducer.js: -------------------------------------------------------------------------------- 1 | /* 2 | * 3 | * App reducer 4 | * 5 | */ 6 | 7 | import { fromJS } from 'immutable'; 8 | 9 | const initialState = fromJS({}); 10 | 11 | function appReducer(state = initialState, action) { 12 | switch (action.type) { 13 | default: 14 | return state; 15 | } 16 | } 17 | 18 | export default appReducer; 19 | -------------------------------------------------------------------------------- /admin/src/containers/App/selectors.js: -------------------------------------------------------------------------------- 1 | // import { createSelector } from 'reselect'; 2 | // import pluginId from 'pluginId'; 3 | 4 | /** 5 | * Direct selector to the list state domain 6 | */ 7 | 8 | // const selectGlobalDomain = () => state => state.get(`${pluginId}_global`); 9 | 10 | export {}; 11 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/ExternalUrlForm/index.js: -------------------------------------------------------------------------------- 1 | import React, { Component, Fragment } from 'react'; 2 | import PropTypes from 'prop-types'; 3 | import { injectIntl } from 'react-intl'; 4 | import Label from 'components/Label'; 5 | 6 | import styles from './styles.scss'; 7 | 8 | export class ExternalUrlForm extends Component { 9 | state = { 10 | url: null 11 | }; 12 | 13 | preAnalyzeImportFile = async event => { 14 | const url = event.target.value; 15 | 16 | this.props.onRequestAnalysis({ source: 'url', options: { url } }); 17 | }; 18 | 19 | render() { 20 | const { loadingAnalysis } = this.props; 21 | 22 | return ( 23 | 24 | 31 | ); 32 | } 33 | } 34 | 35 | ExternalUrlForm.propTypes = { 36 | onRequestAnalysis: PropTypes.func.isRequired, 37 | loadingAnalysis: PropTypes.bool.isRequired 38 | }; 39 | 40 | export default injectIntl(ExternalUrlForm); 41 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/ExternalUrlForm/styles.scss: -------------------------------------------------------------------------------- 1 | .urlInput { 2 | width: 400px; 3 | margin-left: 10px; 4 | background-color: #bbb; 5 | } 6 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/InputFormatSettings/index.js: -------------------------------------------------------------------------------- 1 | import React, { Component, Fragment } from 'react'; 2 | import PropTypes from 'prop-types'; 3 | import { injectIntl } from 'react-intl'; 4 | import _ from 'lodash'; 5 | 6 | import Label from 'components/Label'; 7 | import styles from './styles.scss'; 8 | 9 | export class InputFormatSettings extends Component { 10 | onChangeOption = option => event => { 11 | const settings = _.clone(this.props.settings); 12 | settings[option] = event.target.value; 13 | 14 | this.props.onChange(settings); 15 | }; 16 | 17 | render() { 18 | const { type, settings } = this.props; 19 | 20 | return ( 21 |
22 | {type === 'csv' && ( 23 | 24 | 39 | )} 40 |
41 | ); 42 | } 43 | } 44 | 45 | InputFormatSettings.propTypes = { 46 | onChange: PropTypes.func.isRequired, 47 | type: PropTypes.string.isRequired, 48 | settings: PropTypes.object.isRequired 49 | }; 50 | 51 | export default injectIntl(InputFormatSettings); 52 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/InputFormatSettings/styles.scss: -------------------------------------------------------------------------------- 1 | .settingsInputField { 2 | background-color: #fff; 3 | margin-left: 10px; 4 | margin-left: 15px; 5 | width: 50px; 6 | padding-left: 7px; 7 | padding-right: 7px; 8 | } 9 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/RawInputForm/index.js: -------------------------------------------------------------------------------- 1 | import React, { Component, Fragment } from 'react'; 2 | import PropTypes from 'prop-types'; 3 | import { injectIntl } from 'react-intl'; 4 | import Label from 'components/Label'; 5 | import InputTextArea from 'components/InputTextArea'; 6 | import InputSelect from 'components/InputSelect'; 7 | import Button from 'components/Button'; 8 | import InputSpacer from 'components/InputSpacer'; 9 | 10 | import styles from './styles.scss'; 11 | 12 | export class RawInputForm extends Component { 13 | state = { 14 | rawText: '', 15 | dataFormat: 'text/csv' 16 | }; 17 | 18 | dataFormats = [{ label: 'csv', value: 'text/csv' }]; 19 | 20 | changeDataFormat = event => { 21 | this.setState({ dataFormat: event.target.value }); 22 | }; 23 | 24 | textChanged = async event => { 25 | const rawText = event.target.value; 26 | this.setState({ rawText }); 27 | }; 28 | 29 | clickAnalyze = () => { 30 | const { dataFormat, rawText } = this.state; 31 | 32 | this.props.onRequestAnalysis({ 33 | source: 'raw', 34 | type: dataFormat, 35 | options: { rawText } 36 | }); 37 | }; 38 | 39 | render() { 40 | const { loadingAnalysis } = this.props; 41 | const { dataFormat, rawText } = this.state; 42 | 43 | return ( 44 | 45 | 46 | 47 | 50 | 58 | 66 | 67 |
48 | 51 | 57 | 59 |
68 | 69 | 78 | 79 | ); 80 | } 81 | } 82 | 83 | RawInputForm.propTypes = { 84 | onRequestAnalysis: PropTypes.func.isRequired, 85 | loadingAnalysis: PropTypes.bool.isRequired 86 | }; 87 | 88 | export default injectIntl(RawInputForm); 89 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/RawInputForm/styles.scss: -------------------------------------------------------------------------------- 1 | .rawTextInput { 2 | width: 100%; 3 | margin-left: 10px; 4 | background-color: #bbb; 5 | } 6 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/UploadFileForm/index.js: -------------------------------------------------------------------------------- 1 | import React, { Component, Fragment } from 'react'; 2 | import PropTypes from 'prop-types'; 3 | import { injectIntl } from 'react-intl'; 4 | 5 | import Button from 'components/Button'; 6 | import Label from 'components/Label'; 7 | import InputSpacer from 'components/InputSpacer'; 8 | 9 | function readFileContent(file) { 10 | const reader = new FileReader(); 11 | return new Promise((resolve, reject) => { 12 | reader.onload = event => resolve(event.target.result); 13 | reader.onerror = reject; 14 | reader.readAsText(file); 15 | }); 16 | } 17 | 18 | export class UploadFileForm extends Component { 19 | state = { 20 | source: 'url', 21 | file: null, 22 | type: null, 23 | options: { 24 | filename: null 25 | } 26 | }; 27 | 28 | onChangeImportFile = event => { 29 | const file = event.target.files[0]; 30 | 31 | this.setState({ 32 | file, 33 | type: file.type, 34 | options: { 35 | ...this.state.options, 36 | filename: file.name 37 | } 38 | }); 39 | }; 40 | 41 | clickAnalyzeUploadFile = async () => { 42 | const { file, options } = this.state; 43 | 44 | const data = await readFileContent(file); 45 | 46 | this.props.onRequestAnalysis({ 47 | source: 'upload', 48 | type: file.type, 49 | options, 50 | data 51 | }); 52 | }; 53 | 54 | onChangeOption = option => event => { 55 | this.setState({ 56 | options: { ...this.state.options, [option]: event.target.value } 57 | }); 58 | }; 59 | 60 | render() { 61 | const { loadingAnalysis } = this.props; 62 | 63 | return ( 64 | 65 | 66 | 73 | 81 | 82 |
67 | 72 | 74 |
83 | ); 84 | } 85 | } 86 | 87 | UploadFileForm.propTypes = { 88 | onRequestAnalysis: PropTypes.func.isRequired, 89 | loadingAnalysis: PropTypes.bool.isRequired 90 | }; 91 | 92 | export default injectIntl(UploadFileForm); 93 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/actions.js: -------------------------------------------------------------------------------- 1 | import { 2 | LOAD_MODELS, 3 | LOAD_MODELS_SUCCESS, 4 | LOAD_MODELS_ERROR, 5 | PRE_ANALYZE, 6 | PRE_ANALYZE_SUCCESS, 7 | PRE_ANALYZE_ERROR, 8 | SAVE_IMPORT_CONFIG, 9 | SAVE_IMPORT_CONFIG_ERROR, 10 | SAVE_IMPORT_CONFIG_SUCCESS 11 | } from "./constants"; 12 | 13 | export const loadModels = () => ({ 14 | type: LOAD_MODELS 15 | }); 16 | export const loadModelsSuccess = models => ({ 17 | type: LOAD_MODELS_SUCCESS, 18 | payload: { models } 19 | }); 20 | export const loadModelsError = error => ({ 21 | type: LOAD_MODELS_ERROR, 22 | error: true, 23 | payload: error 24 | }); 25 | 26 | export const preAnalyze = importConfig => ({ 27 | type: PRE_ANALYZE, 28 | payload: { importConfig } 29 | }); 30 | export const preAnalyzeSuccess = analysis => ({ 31 | type: PRE_ANALYZE_SUCCESS, 32 | payload: { analysis } 33 | }); 34 | export const preAnalyzeError = error => ({ 35 | type: PRE_ANALYZE_ERROR, 36 | error: true, 37 | payload: error 38 | }); 39 | 40 | export const saveImportConfig = importConfig => ({ 41 | type: SAVE_IMPORT_CONFIG, 42 | payload: { importConfig } 43 | }); 44 | export const saveImportConfigSuccess = saved => ({ 45 | type: SAVE_IMPORT_CONFIG_SUCCESS, 46 | payload: { saved } 47 | }); 48 | export const saveImportConfigError = error => ({ 49 | type: SAVE_IMPORT_CONFIG_ERROR, 50 | error: true, 51 | payload: error 52 | }); 53 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/constants.js: -------------------------------------------------------------------------------- 1 | import pluginId from "pluginId"; 2 | 3 | const buildString = suffix => `${pluginId}/CreateImportPage/${suffix}`; 4 | 5 | export const LOAD_MODELS = buildString("LOAD_MODELS"); 6 | export const LOAD_MODELS_SUCCESS = buildString("LOAD_MODELS_SUCCESS"); 7 | export const LOAD_MODELS_ERROR = buildString("LOAD_MODELS_ERROR"); 8 | 9 | export const PRE_ANALYZE = buildString("PRE_ANALYZE"); 10 | export const PRE_ANALYZE_SUCCESS = buildString("PRE_ANALYZE_SUCCESS"); 11 | export const PRE_ANALYZE_ERROR = buildString("PRE_ANALYZE_ERROR"); 12 | 13 | export const SAVE_IMPORT_CONFIG = buildString("SAVE_IMPORT_CONFIG"); 14 | export const SAVE_IMPORT_CONFIG_SUCCESS = buildString( 15 | "SAVE_IMPORT_CONFIG_SUCCESS" 16 | ); 17 | export const SAVE_IMPORT_CONFIG_ERROR = buildString("SAVE_IMPORT_CONFIG_ERROR"); 18 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/index.js: -------------------------------------------------------------------------------- 1 | import React, { Component, Fragment } from 'react'; 2 | import PropTypes from 'prop-types'; 3 | import { connect } from 'react-redux'; 4 | import { createStructuredSelector } from 'reselect'; 5 | import { injectIntl } from 'react-intl'; 6 | import { compose } from 'redux'; 7 | import pluginId from 'pluginId'; 8 | 9 | import Button from 'components/Button'; 10 | import PluginHeader from 'components/PluginHeader'; 11 | import InputSelect from 'components/InputSelect'; 12 | import InputSpacer from 'components/InputSpacer'; 13 | import Label from 'components/Label'; 14 | 15 | import ExternalUrlForm from './ExternalUrlForm'; 16 | import UploadFileForm from './UploadFileForm'; 17 | import RawInputForm from './RawInputForm'; 18 | import InputFormatSettings from './InputFormatSettings'; 19 | import MappingTable from '../../components/MappingTable'; 20 | 21 | import styles from './styles.scss'; 22 | import { loadModels, preAnalyze, saveImportConfig } from './actions'; 23 | import { 24 | makeSelectLoading, 25 | makeSelectModels, 26 | makeSelectAnalyzing, 27 | makeSelectAnalysis, 28 | makeSelectCreated, 29 | makeSelectSaving 30 | } from './selectors'; 31 | import reducer from './reducer'; 32 | import saga from './saga'; 33 | 34 | export class CreateImportPage extends Component { 35 | importSources = [ 36 | { label: 'External URL ', value: 'url' }, 37 | { label: 'Upload file', value: 'upload' }, 38 | { label: 'Raw text', value: 'raw' } 39 | ]; 40 | 41 | state = { 42 | importSource: 'url', 43 | analysisConfig: null, 44 | selectedContentType: null, 45 | fieldMapping: {}, 46 | inputFormatSettings: { delimiter: ',', skipRows: 0 } 47 | }; 48 | 49 | componentDidMount() { 50 | this.props.loadModels(); 51 | } 52 | 53 | componentWillReceiveProps(nextProps) { 54 | if (nextProps.models && !this.state.selectedContentType) { 55 | this.setState({ selectedContentType: nextProps.models[0].name }); 56 | } 57 | if (!this.props.created && nextProps.created) { 58 | this.props.history.push(`/plugins/${pluginId}`); 59 | } 60 | } 61 | 62 | getAnalysisConfigWithSettings = analysisConfig => { 63 | const { inputFormatSettings } = this.state; 64 | 65 | return { 66 | ...analysisConfig, 67 | options: { 68 | ...analysisConfig.options, 69 | ...inputFormatSettings 70 | } 71 | }; 72 | }; 73 | 74 | onRequestAnalysis = async analysisConfig => { 75 | this.analysisConfig = analysisConfig; 76 | 77 | const analysisConfigWithSettings = this.getAnalysisConfigWithSettings( 78 | analysisConfig 79 | ); 80 | 81 | this.props.preAnalyze(analysisConfigWithSettings); 82 | }; 83 | 84 | selectContentType = event => { 85 | const selectedContentType = event.target.value; 86 | 87 | this.setState({ selectedContentType }); 88 | }; 89 | 90 | getTargetModel = () => { 91 | const { models } = this.props; 92 | if (!models) return null; 93 | 94 | return models.find(model => model.name === this.state.selectedContentType); 95 | }; 96 | 97 | setFieldMapping = fieldMapping => { 98 | this.setState({ fieldMapping }); 99 | }; 100 | 101 | onSaveImport = () => { 102 | const { selectedContentType, fieldMapping } = this.state; 103 | const { analysisConfig } = this; 104 | 105 | const analysisConfigWithSettings = this.getAnalysisConfigWithSettings( 106 | analysisConfig 107 | ); 108 | 109 | const importConfig = { 110 | ...analysisConfigWithSettings, 111 | contentType: selectedContentType, 112 | fieldMapping 113 | }; 114 | 115 | this.props.saveImportConfig(importConfig); 116 | }; 117 | 118 | selectImportSource = event => { 119 | this.setState({ importSource: event.target.value }); 120 | }; 121 | 122 | updateInputFormatSettings = newSettings => { 123 | this.setState({ inputFormatSettings: newSettings }); 124 | }; 125 | 126 | render() { 127 | const { models, loading, loadingAnalysis, saving, analysis } = this.props; 128 | 129 | const { importSource, inputFormatSettings, fieldMapping } = this.state; 130 | 131 | const saveDisabled = loading || saving || fieldMapping === {}; 132 | 133 | const modelOptions = 134 | models && 135 | models.map(({ name }) => ({ 136 | label: name, 137 | value: name 138 | })); 139 | 140 | return ( 141 |
142 | 143 | 144 |
145 |
146 | 147 | 148 | 156 | 168 | 169 |
149 | 157 | {loading &&

Loading content types...

} 158 | {modelOptions && ( 159 | 160 | 166 | )} 167 |
170 | 171 | 172 | 173 | {importSource === 'upload' && ( 174 | 178 | )} 179 | 180 | {importSource === 'url' && ( 181 | 185 | )} 186 | 187 | {importSource === 'raw' && ( 188 | 192 | )} 193 | 194 | 195 | 200 | 201 | {loadingAnalysis &&

Analyzing...

} 202 | 203 |
204 |
205 | 206 |
207 |
208 | {analysis && ( 209 | 214 | )} 215 | 216 | 217 | 218 |
225 |
226 |
227 | ); 228 | } 229 | } 230 | 231 | CreateImportPage.contextTypes = { 232 | router: PropTypes.object 233 | }; 234 | 235 | CreateImportPage.propTypes = { 236 | models: PropTypes.object.isRequired, 237 | loadModels: PropTypes.func.isRequired, 238 | preAnalyze: PropTypes.func.isRequired, 239 | loading: PropTypes.bool.isRequired, 240 | saving: PropTypes.bool.isRequired, 241 | created: PropTypes.object.isRequired 242 | }; 243 | 244 | const mapDispatchToProps = { 245 | loadModels, 246 | preAnalyze, 247 | saveImportConfig 248 | }; 249 | 250 | const mapStateToProps = createStructuredSelector({ 251 | loading: makeSelectLoading(), 252 | models: makeSelectModels(), 253 | loadingAnalysis: makeSelectAnalyzing(), 254 | analysis: makeSelectAnalysis(), 255 | created: makeSelectCreated(), 256 | saving: makeSelectSaving() 257 | }); 258 | 259 | const withConnect = connect( 260 | mapStateToProps, 261 | mapDispatchToProps 262 | ); 263 | 264 | const withReducer = strapi.injectReducer({ 265 | key: 'createImportPage', 266 | reducer, 267 | pluginId 268 | }); 269 | 270 | const withSaga = strapi.injectSaga({ key: 'createImportPage', saga, pluginId }); 271 | 272 | export default compose( 273 | withReducer, 274 | withSaga, 275 | withConnect 276 | )(injectIntl(CreateImportPage)); 277 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/reducer.js: -------------------------------------------------------------------------------- 1 | import { fromJS } from "immutable"; 2 | 3 | import { 4 | LOAD_MODELS, 5 | LOAD_MODELS_SUCCESS, 6 | LOAD_MODELS_ERROR, 7 | PRE_ANALYZE, 8 | PRE_ANALYZE_SUCCESS, 9 | PRE_ANALYZE_ERROR, 10 | SAVE_IMPORT_CONFIG, 11 | SAVE_IMPORT_CONFIG_SUCCESS, 12 | SAVE_IMPORT_CONFIG_ERROR 13 | } from "./constants"; 14 | 15 | const initialState = fromJS({ 16 | loading: false, 17 | models: null, 18 | error: null, 19 | 20 | analyzing: false, 21 | analysis: null, 22 | preAnalyzeError: null, 23 | 24 | saving: false, 25 | created: null, 26 | saveError: null 27 | }); 28 | 29 | function createImportPageReducer(state = initialState, action) { 30 | const { type, payload } = action; 31 | 32 | switch (type) { 33 | case LOAD_MODELS: 34 | return state.set("loading", true); 35 | case LOAD_MODELS_SUCCESS: { 36 | const filtered = payload.models.filter( 37 | model => !["importconfig", "importeditem"].includes(model.name) 38 | ); 39 | return state.set("loading", false).set("models", filtered); 40 | } 41 | case LOAD_MODELS_ERROR: 42 | return state.set("loading", false).set("error", payload); 43 | 44 | case PRE_ANALYZE: 45 | return state.set("analyzing", true).set("analysis", null); 46 | case PRE_ANALYZE_SUCCESS: { 47 | return state.set("analyzing", false).set("analysis", payload.analysis); 48 | } 49 | case PRE_ANALYZE_ERROR: 50 | return state.set("analyzing", false).set("preAnalyzeError", payload); 51 | 52 | case SAVE_IMPORT_CONFIG: 53 | return state.set("saving", true).set("created", null); 54 | case SAVE_IMPORT_CONFIG_SUCCESS: { 55 | return state.set("saving", false).set("created", payload.saved); 56 | } 57 | case SAVE_IMPORT_CONFIG_ERROR: 58 | return state.set("loading", false).set("saveError", payload); 59 | 60 | default: 61 | return state; 62 | } 63 | } 64 | 65 | export default createImportPageReducer; 66 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/saga.js: -------------------------------------------------------------------------------- 1 | import { fork, takeLatest, call, put } from "redux-saga/effects"; 2 | import request from "utils/request"; 3 | 4 | import { 5 | loadModelsSuccess, 6 | loadModelsError, 7 | preAnalyzeSuccess, 8 | preAnalyzeError, 9 | saveImportConfigError, 10 | saveImportConfigSuccess 11 | } from "./actions"; 12 | import { LOAD_MODELS, PRE_ANALYZE, SAVE_IMPORT_CONFIG } from "./constants"; 13 | 14 | export function* loadModels() { 15 | try { 16 | const { allModels } = yield call(request, "/content-type-builder/models", { 17 | method: "GET" 18 | }); 19 | 20 | yield put(loadModelsSuccess(allModels)); 21 | } catch (err) { 22 | strapi.notification.error("notification.error"); 23 | yield put(loadModelsError(err)); 24 | } 25 | } 26 | 27 | export function* preAnalyze(event) { 28 | try { 29 | const { importConfig } = event.payload; 30 | 31 | const analysis = yield call( 32 | request, 33 | "/import-content/preAnalyzeImportFile", 34 | { 35 | method: "POST", 36 | body: importConfig 37 | } 38 | ); 39 | 40 | yield put(preAnalyzeSuccess(analysis)); 41 | } catch (error) { 42 | strapi.notification.error("notification.error"); 43 | yield put(preAnalyzeError(error)); 44 | } 45 | } 46 | 47 | export function* saveImportConfig(event) { 48 | try { 49 | const { importConfig } = event.payload; 50 | 51 | const saved = yield call(request, "/import-content", { 52 | method: "POST", 53 | body: importConfig 54 | }); 55 | 56 | yield put(saveImportConfigSuccess(saved)); 57 | } catch (error) { 58 | strapi.notification.error("notification.error"); 59 | yield put(saveImportConfigError(error)); 60 | } 61 | } 62 | 63 | export function* defaultSaga() { 64 | yield fork(takeLatest, LOAD_MODELS, loadModels); 65 | yield fork(takeLatest, PRE_ANALYZE, preAnalyze); 66 | yield fork(takeLatest, SAVE_IMPORT_CONFIG, saveImportConfig); 67 | } 68 | 69 | export default defaultSaga; 70 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/selectors.js: -------------------------------------------------------------------------------- 1 | import { createSelector } from "reselect"; 2 | import pluginId from "pluginId"; 3 | 4 | /** 5 | * Direct selector to the examplePage state domain 6 | */ 7 | const selectCreateImportPageDomain = () => state => 8 | state.get(`${pluginId}_createImportPage`); 9 | 10 | /** 11 | * Default selector used by HomePage 12 | */ 13 | 14 | const makeSelectLoading = () => 15 | createSelector( 16 | selectCreateImportPageDomain(), 17 | substate => substate.get("loading") 18 | ); 19 | const makeSelectModels = () => 20 | createSelector( 21 | selectCreateImportPageDomain(), 22 | substate => substate.get("models") 23 | ); 24 | 25 | const makeSelectAnalyzing = () => 26 | createSelector( 27 | selectCreateImportPageDomain(), 28 | substate => substate.get("analyzing") 29 | ); 30 | const makeSelectAnalysis = () => 31 | createSelector( 32 | selectCreateImportPageDomain(), 33 | substate => substate.get("analysis") 34 | ); 35 | 36 | const makeSelectCreated = () => 37 | createSelector( 38 | selectCreateImportPageDomain(), 39 | substate => substate.get("created") 40 | ); 41 | const makeSelectSaving = () => 42 | createSelector( 43 | selectCreateImportPageDomain(), 44 | substate => substate.get("saving") 45 | ); 46 | 47 | export { 48 | makeSelectLoading, 49 | makeSelectModels, 50 | makeSelectAnalyzing, 51 | makeSelectAnalysis, 52 | makeSelectCreated, 53 | makeSelectSaving 54 | }; 55 | -------------------------------------------------------------------------------- /admin/src/containers/CreateImportPage/styles.scss: -------------------------------------------------------------------------------- 1 | .createImportPage { 2 | padding: 1.7rem 3rem; 3 | background: rgba(14, 22, 34, 0.02); 4 | min-height: calc(100vh - 6rem); 5 | } 6 | -------------------------------------------------------------------------------- /admin/src/containers/HomePage/actions.js: -------------------------------------------------------------------------------- 1 | import { 2 | LOAD_IMPORT_CONFIGS, 3 | LOAD_IMPORT_CONFIGS_ERROR, 4 | LOAD_IMPORT_CONFIGS_SUCCESS, 5 | UNDO_IMPORT, 6 | UNDO_IMPORT_SUCCESS, 7 | UNDO_IMPORT_ERROR, 8 | DELETE_IMPORT, 9 | DELETE_IMPORT_SUCCESS, 10 | DELETE_IMPORT_ERROR 11 | } from "./constants"; 12 | 13 | export const loadImportConfigs = () => ({ 14 | type: LOAD_IMPORT_CONFIGS 15 | }); 16 | export const loadImportConfigsSuccess = importConfigs => ({ 17 | type: LOAD_IMPORT_CONFIGS_SUCCESS, 18 | payload: { importConfigs } 19 | }); 20 | export const loadImportConfigsError = error => ({ 21 | type: LOAD_IMPORT_CONFIGS_ERROR, 22 | payload: error, 23 | error: true 24 | }); 25 | 26 | export const undoImport = id => ({ 27 | type: UNDO_IMPORT, 28 | payload: { id } 29 | }); 30 | export const undoImportSuccess = () => ({ 31 | type: UNDO_IMPORT_SUCCESS 32 | }); 33 | export const undoImportError = error => ({ 34 | type: UNDO_IMPORT_ERROR, 35 | payload: error, 36 | error: true 37 | }); 38 | 39 | export const deleteImport = id => ({ 40 | type: DELETE_IMPORT, 41 | payload: { id } 42 | }); 43 | export const deleteImportSuccess = () => ({ 44 | type: DELETE_IMPORT_SUCCESS 45 | }); 46 | export const deleteImportError = error => ({ 47 | type: DELETE_IMPORT_ERROR, 48 | payload: error, 49 | error: true 50 | }); 51 | -------------------------------------------------------------------------------- /admin/src/containers/HomePage/constants.js: -------------------------------------------------------------------------------- 1 | import pluginId from "pluginId"; 2 | 3 | const buildString = suffix => `${pluginId}/HomePage/${suffix}`; 4 | 5 | export const LOAD_IMPORT_CONFIGS = buildString("LOAD_IMPORT_CONFIGS"); 6 | export const LOAD_IMPORT_CONFIGS_ERROR = buildString( 7 | "LOAD_IMPORT_CONFIGS_ERROR" 8 | ); 9 | export const LOAD_IMPORT_CONFIGS_SUCCESS = buildString( 10 | "LOAD_IMPORT_CONFIGS_SUCCESS" 11 | ); 12 | 13 | export const UNDO_IMPORT = buildString("UNDO_IMPORT"); 14 | export const UNDO_IMPORT_SUCCESS = buildString("UNDO_IMPORT_SUCCESS"); 15 | export const UNDO_IMPORT_ERROR = buildString("UNDO_IMPORT_ERROR"); 16 | 17 | export const DELETE_IMPORT = buildString("DELETE_IMPORT"); 18 | export const DELETE_IMPORT_SUCCESS = buildString("DELETE_IMPORT_SUCCESS"); 19 | export const DELETE_IMPORT_ERROR = buildString("DELETE_IMPORT_ERROR"); 20 | -------------------------------------------------------------------------------- /admin/src/containers/HomePage/index.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react'; 2 | import Button from 'components/Button'; 3 | import IcoContainer from 'components/IcoContainer'; 4 | import PropTypes from 'prop-types'; 5 | import { connect } from 'react-redux'; 6 | import { createStructuredSelector } from 'reselect'; 7 | import { injectIntl } from 'react-intl'; 8 | import { compose } from 'redux'; 9 | import pluginId from 'pluginId'; 10 | import moment from 'moment'; 11 | 12 | import { 13 | selectImportConfigs, 14 | selectImportConfigsError, 15 | selectImportConfigsLoading 16 | } from './selectors'; 17 | 18 | import styles from './styles.scss'; 19 | 20 | import { loadImportConfigs, undoImport, deleteImport } from './actions'; 21 | import reducer from './reducer'; 22 | import saga from './saga'; 23 | 24 | export class HomePage extends Component { 25 | componentDidMount() { 26 | this.props.loadImportConfigs(); 27 | } 28 | 29 | componentWillReceiveProps(nextProps) { 30 | if (this.props.loading && !nextProps.loading) { 31 | if ( 32 | nextProps.importConfigs && 33 | nextProps.importConfigs.some(config => config.ongoing) 34 | ) { 35 | setTimeout(() => this.props.loadImportConfigs(), 2000); 36 | } 37 | } 38 | } 39 | 40 | navigateToCreateImport = () => { 41 | this.props.history.push(`/plugins/${pluginId}/create`); 42 | }; 43 | 44 | deleteImport = id => () => { 45 | this.props.deleteImport(id); 46 | }; 47 | 48 | undoImport = id => () => { 49 | this.props.undoImport(id); 50 | }; 51 | 52 | getSourceText = item => { 53 | switch (item.source) { 54 | case 'upload': 55 | return item.options.filename; 56 | case 'url': 57 | return item.options.url; 58 | } 59 | }; 60 | 61 | render() { 62 | const { importConfigs } = this.props; 63 | 64 | return ( 65 |
66 |
116 | ); 117 | } 118 | } 119 | 120 | HomePage.contextTypes = { 121 | router: PropTypes.object 122 | }; 123 | 124 | HomePage.propTypes = { 125 | history: PropTypes.object.isRequired, 126 | loadImports: PropTypes.func.isRequired, 127 | importConfigs: PropTypes.array, 128 | undoImport: PropTypes.func.isRequired, 129 | deleteImport: PropTypes.func.isRequired 130 | }; 131 | 132 | const mapDispatchToProps = { 133 | loadImportConfigs, 134 | undoImport, 135 | deleteImport 136 | }; 137 | 138 | const mapStateToProps = createStructuredSelector({ 139 | importConfigs: selectImportConfigs(), 140 | loading: selectImportConfigsLoading(), 141 | error: selectImportConfigsError() 142 | }); 143 | 144 | const withConnect = connect( 145 | mapStateToProps, 146 | mapDispatchToProps 147 | ); 148 | 149 | const withReducer = strapi.injectReducer({ 150 | key: 'homePage', 151 | reducer, 152 | pluginId 153 | }); 154 | const withSaga = strapi.injectSaga({ key: 'homePage', saga, pluginId }); 155 | 156 | export default compose( 157 | withReducer, 158 | withSaga, 159 | withConnect 160 | )(injectIntl(HomePage)); 161 | -------------------------------------------------------------------------------- /admin/src/containers/HomePage/reducer.js: -------------------------------------------------------------------------------- 1 | import { fromJS } from 'immutable'; 2 | 3 | import { 4 | LOAD_IMPORT_CONFIGS, 5 | LOAD_IMPORT_CONFIGS_SUCCESS, 6 | LOAD_IMPORT_CONFIGS_ERROR 7 | } from './constants'; 8 | 9 | const initialState = fromJS({ 10 | importConfigs: null, 11 | loading: false, 12 | error: null 13 | }); 14 | 15 | function homePageReducer(state = initialState, action) { 16 | const { type, payload } = action; 17 | 18 | switch (type) { 19 | case LOAD_IMPORT_CONFIGS: 20 | return state.set('loading', true); 21 | 22 | case LOAD_IMPORT_CONFIGS_SUCCESS: 23 | return state 24 | .set('loading', false) 25 | .set('importConfigs', payload.importConfigs); 26 | 27 | case LOAD_IMPORT_CONFIGS_ERROR: 28 | return state.set('loading', false).set('error', payload); 29 | 30 | default: 31 | return state; 32 | } 33 | } 34 | 35 | export default homePageReducer; 36 | -------------------------------------------------------------------------------- /admin/src/containers/HomePage/saga.js: -------------------------------------------------------------------------------- 1 | import { fork, takeLatest, call, put } from "redux-saga/effects"; 2 | import request from "utils/request"; 3 | 4 | import { 5 | loadImportConfigs, 6 | loadImportConfigsSuccess, 7 | loadImportConfigsError, 8 | undoImportError, 9 | undoImportSuccess, 10 | deleteImportError, 11 | deleteImportSuccess 12 | } from "./actions"; 13 | import { LOAD_IMPORT_CONFIGS, DELETE_IMPORT, UNDO_IMPORT } from "./constants"; 14 | 15 | export function* loadImportConfigsSaga() { 16 | try { 17 | const importConfigs = yield call(request, "/import-content", { 18 | method: "GET" 19 | }); 20 | 21 | yield put(loadImportConfigsSuccess(importConfigs)); 22 | } catch (error) { 23 | strapi.notification.error("notification.error"); 24 | yield put(loadImportConfigsError(error)); 25 | } 26 | } 27 | 28 | export function* deleteImportSaga(event) { 29 | const { id } = event.payload; 30 | 31 | try { 32 | const importConfigs = yield call(request, `/import-content/${id}`, { 33 | method: "DELETE" 34 | }); 35 | 36 | yield put(deleteImportSuccess(importConfigs)); 37 | yield put(loadImportConfigs()); 38 | } catch (error) { 39 | strapi.notification.error("notification.error"); 40 | yield put(deleteImportError(error)); 41 | } 42 | } 43 | 44 | export function* undoImportSaga(event) { 45 | const { id } = event.payload; 46 | 47 | try { 48 | const importConfigs = yield call(request, `/import-content/${id}/undo`, { 49 | method: "POST" 50 | }); 51 | 52 | yield put(undoImportSuccess(importConfigs)); 53 | yield put(loadImportConfigs()); 54 | } catch (error) { 55 | strapi.notification.error("notification.error"); 56 | yield put(undoImportError(error)); 57 | } 58 | } 59 | 60 | export function* defaultSaga() { 61 | yield fork(takeLatest, LOAD_IMPORT_CONFIGS, loadImportConfigsSaga); 62 | yield fork(takeLatest, UNDO_IMPORT, undoImportSaga); 63 | yield fork(takeLatest, DELETE_IMPORT, deleteImportSaga); 64 | } 65 | 66 | export default defaultSaga; 67 | -------------------------------------------------------------------------------- /admin/src/containers/HomePage/selectors.js: -------------------------------------------------------------------------------- 1 | import { createSelector } from 'reselect'; 2 | import pluginId from 'pluginId'; 3 | /** 4 | * Direct selector to the homePage state domain 5 | */ 6 | const selectHomePageDomain = () => state => state.get(`${pluginId}_homePage`); 7 | 8 | /** 9 | * Default selector used by HomePage 10 | */ 11 | 12 | export const selectImportConfigs = () => 13 | createSelector(selectHomePageDomain(), substate => 14 | substate.get('importConfigs') 15 | ); 16 | 17 | export const selectImportConfigsError = () => 18 | createSelector(selectHomePageDomain(), substate => substate.get('error')); 19 | 20 | export const selectImportConfigsLoading = () => 21 | createSelector(selectHomePageDomain(), substate => substate.get('loading')); 22 | -------------------------------------------------------------------------------- /admin/src/containers/HomePage/styles.scss: -------------------------------------------------------------------------------- 1 | .homePage { 2 | padding: 1.7rem 3rem; 3 | background: rgba(14, 22, 34, 0.02); 4 | min-height: calc(100vh - 6rem); 5 | } 6 | 7 | .inProgress { 8 | background-color: rgba(180, 255, 180, 0.5); 9 | } 10 | 11 | th, 12 | td { 13 | padding-right: 15px; 14 | } 15 | -------------------------------------------------------------------------------- /admin/src/containers/NotFoundPage/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * NotFoundPage 3 | * 4 | * This is the page we show when the user visits a url that doesn't have a route 5 | * 6 | * NOTE: while this component should technically be a stateless functional 7 | * component (SFC), hot reloading does not currently support SFCs. If hot 8 | * reloading is not a neccessity for you then you can refactor it and remove 9 | * the linting exception. 10 | */ 11 | 12 | import React from 'react'; 13 | 14 | import NotFound from 'components/NotFound'; 15 | 16 | export default class NotFoundPage extends React.Component { 17 | render() { 18 | return ; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /admin/src/pluginId.js: -------------------------------------------------------------------------------- 1 | const pluginPkg = require('../../package.json'); 2 | const pluginId = pluginPkg.name.replace( 3 | /^strapi-plugin-/i, 4 | '' 5 | ); 6 | 7 | module.exports = pluginId; 8 | -------------------------------------------------------------------------------- /admin/src/translations/ar.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/de.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /admin/src/translations/en.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/es.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/fr.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/it.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/ko.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/nl.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/pl.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /admin/src/translations/pt-BR.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/pt.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /admin/src/translations/ru.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbeuckm/strapi-plugin-import-content/6c6131826bff162b107294005b035ab0c43a72a2/admin/src/translations/ru.json -------------------------------------------------------------------------------- /admin/src/translations/tr.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /admin/src/translations/zh-Hans.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbeuckm/strapi-plugin-import-content/6c6131826bff162b107294005b035ab0c43a72a2/admin/src/translations/zh-Hans.json -------------------------------------------------------------------------------- /admin/src/translations/zh.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /azure-pipelines.yml: -------------------------------------------------------------------------------- 1 | # Node.js with React 2 | # Build a Node.js project that uses React. 3 | # Add steps that analyze code, save build artifacts, deploy, and more: 4 | # https://docs.microsoft.com/azure/devops/pipelines/languages/javascript 5 | 6 | trigger: 7 | - master 8 | 9 | pool: 10 | vmImage: 'Ubuntu-16.04' 11 | 12 | steps: 13 | - task: NodeTool@0 14 | inputs: 15 | versionSpec: '10.x' 16 | displayName: 'Install Node.js' 17 | 18 | - script: | 19 | npm install 20 | npm run test 21 | displayName: 'npm install and test' 22 | -------------------------------------------------------------------------------- /config/functions/bootstrap.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const resetOngoingImports = async () => { 4 | const entries = await strapi 5 | .query('importconfig', 'import-content') 6 | .find({ ongoing: true }); 7 | 8 | const resetImportsPromises = entries.map(importConfig => 9 | strapi 10 | .query('importconfig', 'import-content') 11 | .update({ id: importConfig.id }, { ongoing: false }) 12 | ); 13 | 14 | return await Promise.all(resetImportsPromises); 15 | }; 16 | 17 | const findAuthenticatedRole = async () => { 18 | const result = await strapi 19 | .query('role', 'users-permissions') 20 | .findOne({ type: 'authenticated' }); 21 | 22 | return result; 23 | }; 24 | 25 | const setDefaultPermissions = async () => { 26 | const role = await findAuthenticatedRole(); 27 | 28 | const permissions = await strapi 29 | .query('permission', 'users-permissions') 30 | .find({ type: 'import-content', role: role.id }); 31 | 32 | await Promise.all( 33 | permissions.map(p => 34 | strapi 35 | .query('permission', 'users-permissions') 36 | .update({ id: p.id }, { enabled: true }) 37 | ) 38 | ); 39 | }; 40 | 41 | const isFirstRun = async () => { 42 | const pluginStore = strapi.store({ 43 | environment: strapi.config.environment, 44 | type: 'plugin', 45 | name: 'import-content' 46 | }); 47 | 48 | const initHasRun = await pluginStore.get({ key: 'initHasRun' }); 49 | 50 | await pluginStore.set({ key: 'initHasRun', value: true }); 51 | 52 | return !initHasRun; 53 | }; 54 | 55 | module.exports = async callback => { 56 | const shouldSetDefaultPermissions = await isFirstRun(); 57 | 58 | if (shouldSetDefaultPermissions) { 59 | await setDefaultPermissions(); 60 | } 61 | 62 | await resetOngoingImports(); 63 | 64 | callback(); 65 | }; 66 | -------------------------------------------------------------------------------- /config/queries/bookshelf.js: -------------------------------------------------------------------------------- 1 | const _ = require("lodash"); 2 | const { convertRestQueryParams, buildQuery } = require("strapi-utils"); 3 | 4 | module.exports = { 5 | find: async function(params, populate) { 6 | const model = this; 7 | 8 | const filters = convertRestQueryParams(params); 9 | 10 | return this.query(buildQuery({ model, filters })) 11 | .fetchAll({ 12 | withRelated: populate || this.associations.map(x => x.alias) 13 | }) 14 | .then(data => data.toJSON()); 15 | }, 16 | 17 | count: async function(params = {}) { 18 | const model = this; 19 | 20 | const { where } = convertRestQueryParams(params); 21 | 22 | return this.query(buildQuery({ model, filters: { where } })).count(); 23 | }, 24 | 25 | findOne: async function(params, populate) { 26 | const primaryKey = params[this.primaryKey] || params.id; 27 | 28 | if (primaryKey) { 29 | params = { 30 | [this.primaryKey]: primaryKey 31 | }; 32 | } 33 | 34 | const record = await this.forge(params).fetch({ 35 | withRelated: populate || this.associations.map(x => x.alias) 36 | }); 37 | 38 | return record ? record.toJSON() : record; 39 | }, 40 | 41 | create: async function(params) { 42 | return this.forge() 43 | .save( 44 | Object.keys(params).reduce((acc, current) => { 45 | if ( 46 | _.get(this._attributes, [current, "type"]) || 47 | _.get(this._attributes, [current, "model"]) 48 | ) { 49 | acc[current] = params[current]; 50 | } 51 | 52 | return acc; 53 | }, {}) 54 | ) 55 | .catch(err => { 56 | if (err.detail) { 57 | const field = _.last(_.words(err.detail.split("=")[0])); 58 | err = { message: `This ${field} is already taken`, field }; 59 | } 60 | 61 | throw err; 62 | }); 63 | }, 64 | 65 | update: async function(search, params = {}) { 66 | if (_.isEmpty(params)) { 67 | params = search; 68 | } 69 | 70 | const primaryKey = search[this.primaryKey] || search.id; 71 | 72 | if (primaryKey) { 73 | search = { 74 | [this.primaryKey]: primaryKey 75 | }; 76 | } else { 77 | const entry = await module.exports.findOne.call(this, search); 78 | 79 | search = { 80 | [this.primaryKey]: entry[this.primaryKey] || entry.id 81 | }; 82 | } 83 | 84 | return this.forge(search) 85 | .save(params, { 86 | patch: true 87 | }) 88 | .catch(err => { 89 | console.log("SQLite does not parse this error correctly", { err }); 90 | const field = _.last(_.words(err.detail.split("=")[0])); 91 | const error = { message: `This ${field} is already taken`, field }; 92 | 93 | throw error; 94 | }); 95 | }, 96 | 97 | delete: async function(params) { 98 | return await this.forge({ 99 | [this.primaryKey]: params[this.primaryKey] || params.id 100 | }).destroy(); 101 | }, 102 | 103 | search: async function(params) { 104 | return this.query(function(qb) { 105 | qb.whereRaw(`LOWER(hash) LIKE ?`, [`%${params.id}%`]).orWhereRaw( 106 | `LOWER(name) LIKE ?`, 107 | [`%${params.id}%`] 108 | ); 109 | }).fetchAll(); 110 | }, 111 | 112 | addPermission: async function(params) { 113 | return this.forge(params).save(); 114 | }, 115 | 116 | removePermission: async function(params) { 117 | return this.forge({ 118 | [this.primaryKey]: params[this.primaryKey] || params.id 119 | }).destroy(); 120 | } 121 | }; 122 | -------------------------------------------------------------------------------- /config/queries/mongoose.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const { convertRestQueryParams, buildQuery } = require('strapi-utils'); 3 | 4 | module.exports = { 5 | find: async function(params, populate) { 6 | const model = this; 7 | const filters = convertRestQueryParams(params); 8 | 9 | return buildQuery({ 10 | model, 11 | filters, 12 | populate: populate || model.associations.map(x => x.alias), 13 | }).lean(); 14 | }, 15 | 16 | count: async function(params) { 17 | const model = this; 18 | 19 | const filters = convertRestQueryParams(params); 20 | 21 | return buildQuery({ 22 | model, 23 | filters: { where: filters.where }, 24 | }).count(); 25 | }, 26 | 27 | findOne: async function(params, populate) { 28 | const primaryKey = params[this.primaryKey] || params.id; 29 | 30 | if (primaryKey) { 31 | params = { 32 | [this.primaryKey]: primaryKey, 33 | }; 34 | } 35 | 36 | return this.findOne(params) 37 | .populate(populate || this.associations.map(x => x.alias).join(' ')) 38 | .lean(); 39 | }, 40 | 41 | create: async function(params) { 42 | // Exclude relationships. 43 | const values = Object.keys(params).reduce((acc, current) => { 44 | if ( 45 | _.get(this._attributes, [current, 'type']) || 46 | _.get(this._attributes, [current, 'model']) 47 | ) { 48 | acc[current] = params[current]; 49 | } 50 | 51 | return acc; 52 | }, {}); 53 | 54 | return this.create(values).catch(err => { 55 | if (err.message.indexOf('index:') !== -1) { 56 | const message = err.message.split('index:'); 57 | const field = _.words(_.last(message).split('_')[0]); 58 | const error = { message: `This ${field} is already taken`, field }; 59 | 60 | throw error; 61 | } 62 | 63 | throw err; 64 | }); 65 | }, 66 | 67 | update: async function(search, params = {}) { 68 | if (_.isEmpty(params)) { 69 | params = search; 70 | } 71 | 72 | const primaryKey = search[this.primaryKey] || search.id; 73 | 74 | if (primaryKey) { 75 | search = { 76 | [this.primaryKey]: primaryKey, 77 | }; 78 | } 79 | 80 | return this.updateOne(search, params, { 81 | strict: false, 82 | }).catch(error => { 83 | const field = _.last(_.words(error.message.split('_')[0])); 84 | const err = { message: `This ${field} is already taken`, field }; 85 | 86 | throw err; 87 | }); 88 | }, 89 | 90 | delete: async function(params) { 91 | // Delete entry. 92 | return this.remove({ 93 | [this.primaryKey]: params[this.primaryKey] || params.id, 94 | }); 95 | }, 96 | 97 | search: async function(params) { 98 | const re = new RegExp(params.id, 'i'); 99 | 100 | return this.find({ 101 | $or: [{ hash: re }, { name: re }], 102 | }); 103 | }, 104 | 105 | addPermission: async function(params) { 106 | return this.create(params); 107 | }, 108 | 109 | removePermission: async function(params) { 110 | return this.remove(params); 111 | }, 112 | }; 113 | -------------------------------------------------------------------------------- /config/routes.json: -------------------------------------------------------------------------------- 1 | { 2 | "routes": [ 3 | { 4 | "method": "POST", 5 | "path": "/preAnalyzeImportFile", 6 | "handler": "ImportConfig.preAnalyzeImportFile", 7 | "config": { 8 | "policies": [] 9 | } 10 | }, 11 | { 12 | "method": "POST", 13 | "path": "/", 14 | "handler": "ImportConfig.create", 15 | "config": { 16 | "policies": [] 17 | } 18 | }, 19 | { 20 | "method": "GET", 21 | "path": "/", 22 | "handler": "ImportConfig.index", 23 | "config": { 24 | "policies": [] 25 | } 26 | }, 27 | { 28 | "method": "POST", 29 | "path": "/:importId/undo", 30 | "handler": "ImportConfig.undo", 31 | "config": { 32 | "policies": [] 33 | } 34 | }, 35 | { 36 | "method": "DELETE", 37 | "path": "/:importId", 38 | "handler": "ImportConfig.delete", 39 | "config": { 40 | "policies": [] 41 | } 42 | } 43 | ] 44 | } 45 | -------------------------------------------------------------------------------- /config/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "IMPORT_THROTTLE": 100, 3 | "UNDO_THROTTLE": 100 4 | } 5 | -------------------------------------------------------------------------------- /controllers/ImportConfig.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const { 3 | resolveFileDataFromRequest 4 | } = require('../services/utils/FileDataResolver'); 5 | 6 | module.exports = { 7 | index: async ctx => { 8 | const entries = await strapi.query('importconfig', 'import-content').find(); 9 | 10 | const withCounts = entries.map(entry => ({ 11 | ...entry, 12 | importedCount: entry.importeditems.length, 13 | importeditems: undefined 14 | })); 15 | 16 | ctx.send(withCounts); 17 | }, 18 | 19 | create: async ctx => { 20 | const services = strapi.plugins['import-content'].services; 21 | const importConfig = ctx.request.body; 22 | console.log('create', importConfig); 23 | importConfig.ongoing = true; 24 | 25 | const record = await strapi 26 | .query('importconfig', 'import-content') 27 | .create(importConfig); 28 | 29 | ctx.send(record); 30 | 31 | const { contentType, body } = await resolveFileDataFromRequest(ctx); 32 | 33 | services['importitems'].importItems(record, { contentType, body }); 34 | }, 35 | 36 | undo: async ctx => { 37 | const importId = ctx.params.importId; 38 | 39 | const importConfig = await strapi 40 | .query('importconfig', 'import-content') 41 | .findOne({ id: importId }); 42 | 43 | console.log('undo', importId); 44 | 45 | ctx.send(importConfig); 46 | 47 | strapi.plugins['import-content'].services['undoitems'].undoItems( 48 | importConfig 49 | ); 50 | }, 51 | 52 | delete: async ctx => { 53 | const importId = ctx.params.importId; 54 | 55 | await strapi.query('importconfig', 'import-content').delete({ 56 | id: importId 57 | }); 58 | 59 | ctx.send({ message: 'ok' }); 60 | }, 61 | 62 | preAnalyzeImportFile: async ctx => { 63 | const services = strapi.plugins['import-content'].services; 64 | 65 | const { contentType, body, options } = await resolveFileDataFromRequest( 66 | ctx 67 | ); 68 | 69 | try { 70 | const data = await services['importconfig'].preAnalyzeImportFile({ 71 | contentType, 72 | body, 73 | options 74 | }); 75 | 76 | ctx.send(data); 77 | } catch (error) { 78 | ctx.response.notAcceptable('could not parse', error); 79 | } 80 | } 81 | }; 82 | -------------------------------------------------------------------------------- /models/ImportConfig.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | // Before saving a value. 5 | // Fired before an `insert` or `update` query. 6 | // beforeSave: async (model) => {}, 7 | // After saving a value. 8 | // Fired after an `insert` or `update` query. 9 | // afterSave: async (model, result) => {}, 10 | // Before fetching all values. 11 | // Fired before a `fetchAll` operation. 12 | // beforeFetchAll: async (model) => {}, 13 | // After fetching all values. 14 | // Fired after a `fetchAll` operation. 15 | // afterFetchAll: async (model, results) => {}, 16 | // Fired before a `fetch` operation. 17 | // beforeFetch: async (model) => {}, 18 | // After fetching a value. 19 | // Fired after a `fetch` operation. 20 | // afterFetch: async (model, result) => {}, 21 | // Before creating a value. 22 | // Fired before `insert` query. 23 | // beforeCreate: async (model) => {}, 24 | // After creating a value. 25 | // Fired after `insert` query. 26 | // afterCreate: async (model, result) => {}, 27 | // Before updating a value. 28 | // Fired before an `update` query. 29 | // beforeUpdate: async (model) => {}, 30 | // After updating a value. 31 | // Fired after an `update` query. 32 | // afterUpdate: async (model, result) => {}, 33 | // Before destroying a value. 34 | // Fired before a `delete` query. 35 | // beforeDestroy: async (model) => {}, 36 | // After destroying a value. 37 | // Fired after a `delete` query. 38 | // afterDestroy: async (model, result) => {} 39 | }; 40 | -------------------------------------------------------------------------------- /models/ImportConfig.settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "connection": "default", 3 | "info": { 4 | "name": "importconfig", 5 | "description": 6 | "Record an executed import and relate its products for possible later deletion" 7 | }, 8 | "options": { 9 | "timestamps": true 10 | }, 11 | "attributes": { 12 | "date": { 13 | "type": "date" 14 | }, 15 | "source": { 16 | "type": "string" 17 | }, 18 | "options": { 19 | "type": "json" 20 | }, 21 | "contentType": { 22 | "type": "string" 23 | }, 24 | "fieldMapping": { 25 | "type": "json" 26 | }, 27 | "ongoing": { 28 | "type": "boolean" 29 | }, 30 | "importeditems": { 31 | "collection": "importeditem", 32 | "via": "importconfig", 33 | "plugin": "import-content" 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /models/Importeditem.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /** 4 | * Lifecycle callbacks for the `Importeditem` model. 5 | */ 6 | 7 | module.exports = { 8 | // Before saving a value. 9 | // Fired before an `insert` or `update` query. 10 | // beforeSave: async (model, attrs, options) => {}, 11 | 12 | // After saving a value. 13 | // Fired after an `insert` or `update` query. 14 | // afterSave: async (model, response, options) => {}, 15 | 16 | // Before fetching a value. 17 | // Fired before a `fetch` operation. 18 | // beforeFetch: async (model, columns, options) => {}, 19 | 20 | // After fetching a value. 21 | // Fired after a `fetch` operation. 22 | // afterFetch: async (model, response, options) => {}, 23 | 24 | // Before fetching all values. 25 | // Fired before a `fetchAll` operation. 26 | // beforeFetchAll: async (model, columns, options) => {}, 27 | 28 | // After fetching all values. 29 | // Fired after a `fetchAll` operation. 30 | // afterFetchAll: async (model, response, options) => {}, 31 | 32 | // Before creating a value. 33 | // Fired before an `insert` query. 34 | // beforeCreate: async (model, attrs, options) => {}, 35 | 36 | // After creating a value. 37 | // Fired after an `insert` query. 38 | // afterCreate: async (model, attrs, options) => {}, 39 | 40 | // Before updating a value. 41 | // Fired before an `update` query. 42 | // beforeUpdate: async (model, attrs, options) => {}, 43 | 44 | // After updating a value. 45 | // Fired after an `update` query. 46 | // afterUpdate: async (model, attrs, options) => {}, 47 | 48 | // Before destroying a value. 49 | // Fired before a `delete` query. 50 | // beforeDestroy: async (model, attrs, options) => {}, 51 | 52 | // After destroying a value. 53 | // Fired after a `delete` query. 54 | // afterDestroy: async (model, attrs, options) => {} 55 | }; 56 | -------------------------------------------------------------------------------- /models/Importeditem.settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "connection": "default", 3 | "info": { 4 | "name": "importeditem", 5 | "description": 6 | "A record of an item that has been imported with a given ImportConfig" 7 | }, 8 | "options": { 9 | "increments": true, 10 | "timestamps": true, 11 | "comment": "" 12 | }, 13 | "attributes": { 14 | "ContentType": { 15 | "type": "string" 16 | }, 17 | "ContentId": { 18 | "type": "integer" 19 | }, 20 | "importconfig": { 21 | "model": "importconfig", 22 | "via": "importeditems", 23 | "plugin": "import-content" 24 | }, 25 | "importedFiles": { 26 | "type": "json" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "strapi-plugin-import-content", 3 | "version": "0.3.4", 4 | "description": "Import your content into Strapi.", 5 | "repository": { 6 | "type": "git", 7 | "url": "git@github.com:jbeuckm/strapi-plugin-import-content.git" 8 | }, 9 | "strapi": { 10 | "name": "Import Content", 11 | "icon": "plug", 12 | "description": "Import RSS feed or uploaded CSV data into Strapi." 13 | }, 14 | "scripts": { 15 | "analyze:clean": "node ./node_modules/strapi-helper-plugin/node_modules/.bin/rimraf stats.json", 16 | "preanalyze": "npm run analyze:clean", 17 | "analyze": "node ./node_modules/strapi-helper-plugin/lib/internals/scripts/analyze.js", 18 | "prebuild": "npm run build:clean", 19 | "build:dev": "node ./node_modules/strapi-helper-plugin/node_modules/.bin/cross-env NODE_ENV=development node ./node_modules/strapi-helper-plugin/node_modules/.bin/webpack --config node_modules/strapi-helper-plugin/lib/internals/webpack/webpack.prod.babel.js --color -p --progress", 20 | "build": "node ./node_modules/strapi-helper-plugin/node_modules/.bin/cross-env NODE_ENV=production node node_modules/strapi-helper-plugin/node_modules/.bin/webpack --config node_modules/strapi-helper-plugin/lib/internals/webpack/webpack.prod.babel.js --color -p --progress", 21 | "build:clean": "node ./node_modules/strapi-helper-plugin/node_modules/.bin/rimraf admin/build", 22 | "start": "node ./node_modules/strapi-helper-plugin/node_modules/.bin/cross-env NODE_ENV=development node ./node_modules/strapi-helper-plugin/lib/server", 23 | "generate": "node ./node_modules/strapi-helper-plugin/node_modules/plop --plopfile ./node_modules/strapi-helper-plugin/lib/internals/generators/index.js", 24 | "lint": "node ./node_modules/strapi-helper-plugin/node_modules/.bin/eslint --ignore-path .gitignore --ignore-pattern '/admin/build/' --config ./node_modules/strapi-helper-plugin/lib/internals/eslint/.eslintrc.json admin", 25 | "prettier": "node ./node_modules/strapi-helper-plugin/node_modules/.bin/prettier --single-quote --trailing-comma es5 --write \"{admin,__{tests,mocks}__}/**/*.js\"", 26 | "test": "jest", 27 | "test:watch": "jest --watch", 28 | "prepublishOnly": "npm run build" 29 | }, 30 | "dependencies": { 31 | "content-type-parser": "^1.0.2", 32 | "csv-parse": "^4.4.1", 33 | "get-urls": "^9.1.0", 34 | "jest": "^24.7.1", 35 | "joi": "^14.3.1", 36 | "lodash": "^4.17.11", 37 | "moment": "^2.24.0", 38 | "nock": "^10.0.6", 39 | "request": "^2.88.0", 40 | "rss-parser": "^3.7.0", 41 | "simple-statistics": "^7.0.2", 42 | "striptags": "^3.1.1" 43 | }, 44 | "devDependencies": { 45 | "strapi-helper-plugin": "3.0.0-alpha.26" 46 | }, 47 | "author": { 48 | "name": "Joseph Beuckman", 49 | "email": "joe@beigerecords.com", 50 | "url": "https://github.com/jbeuckm" 51 | }, 52 | "maintainers": [ 53 | { 54 | "name": "Joseph Beuckman", 55 | "email": "joe@beigerecords.com", 56 | "url": "https://github.com/jbeuckm" 57 | } 58 | ], 59 | "engines": { 60 | "node": ">= 10.0.0", 61 | "npm": ">= 6.0.0" 62 | }, 63 | "license": "MIT" 64 | } 65 | -------------------------------------------------------------------------------- /services/ImportConfig.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const fileUtils = require('./utils/fileUtils'); 3 | const analyzer = require('./utils/analyzer'); 4 | module.exports = { 5 | preAnalyzeImportFile: async ({ contentType, body, options }) => { 6 | const { sourceType, items } = await fileUtils.getItemsForFileData({ 7 | contentType, 8 | body, 9 | options 10 | }); 11 | 12 | const analysis = analyzer.analyze(sourceType, items); 13 | 14 | return { sourceType, ...analysis }; 15 | } 16 | }; 17 | -------------------------------------------------------------------------------- /services/ImportItems.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const _ = require('lodash'); 3 | const fileUtils = require('./utils/fileUtils'); 4 | const importFields = require('./utils/importFields'); 5 | const importMediaFiles = require('./utils/importMediaFiles'); 6 | 7 | const queues = {}; 8 | 9 | const importNextItem = async importConfig => { 10 | const sourceItem = queues[importConfig.id].shift(); 11 | if (!sourceItem) { 12 | console.log('import complete'); 13 | 14 | await strapi 15 | .query('importconfig', 'import-content') 16 | .update({ id: importConfig.id }, { ongoing: false }); 17 | 18 | return; 19 | } 20 | 21 | const importedItem = await importFields( 22 | sourceItem, 23 | importConfig.fieldMapping 24 | ); 25 | 26 | const savedContent = await strapi.models[importConfig.contentType] 27 | .forge(importedItem) 28 | .save(); 29 | 30 | const uploadedFiles = await importMediaFiles( 31 | savedContent, 32 | sourceItem, 33 | importConfig 34 | ); 35 | const fileIds = _.map(_.flatten(uploadedFiles), 'id'); 36 | 37 | await strapi.query('importeditem', 'import-content').create({ 38 | importconfig: importConfig.id, 39 | ContentId: savedContent.id, 40 | ContentType: importConfig.contentType, 41 | importedFiles: { fileIds } 42 | }); 43 | 44 | const { IMPORT_THROTTLE } = strapi.plugins['import-content'].config; 45 | setTimeout(() => importNextItem(importConfig), IMPORT_THROTTLE); 46 | }; 47 | 48 | module.exports = { 49 | importItems: (importConfig, { contentType, body }) => 50 | new Promise(async (resolve, reject) => { 51 | const importConfigRecord = importConfig.attributes; 52 | console.log('importitems', importConfigRecord); 53 | 54 | try { 55 | const { items } = await fileUtils.getItemsForFileData({ 56 | contentType, 57 | body, 58 | options: importConfigRecord.options 59 | }); 60 | 61 | queues[importConfigRecord.id] = items; 62 | } catch (error) { 63 | reject(error); 64 | } 65 | 66 | resolve({ 67 | status: 'import started', 68 | importConfigId: importConfigRecord.id 69 | }); 70 | 71 | importNextItem(importConfigRecord); 72 | }) 73 | }; 74 | -------------------------------------------------------------------------------- /services/UndoItems.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const _ = require('lodash'); 3 | 4 | const queues = {}; 5 | 6 | const removeImportedFiles = async (fileIds, uploadConfig) => { 7 | const removePromises = fileIds.map(id => 8 | strapi.plugins['upload'].services.upload.remove({ id }, uploadConfig) 9 | ); 10 | 11 | return await Promise.all(removePromises); 12 | }; 13 | 14 | const undoNextItem = async (importConfig, uploadConfig) => { 15 | const item = queues[importConfig.id].shift(); 16 | 17 | if (!item) { 18 | console.log('undo complete'); 19 | 20 | await strapi 21 | .query('importconfig', 'import-content') 22 | .update({ id: importConfig.id }, { ongoing: false }); 23 | 24 | return; 25 | } 26 | 27 | await strapi.models[importConfig.contentType] 28 | .forge({ id: item.ContentId }) 29 | .destroy(); 30 | 31 | const importedFileIds = _.compact(item.importedFiles.fileIds); 32 | 33 | await removeImportedFiles(importedFileIds, uploadConfig); 34 | 35 | await strapi.query('importeditem', 'import-content').delete({ 36 | id: item.id 37 | }); 38 | 39 | const { UNDO_THROTTLE } = strapi.plugins['import-content'].config; 40 | setTimeout(() => undoNextItem(importConfig, uploadConfig), UNDO_THROTTLE); 41 | }; 42 | 43 | module.exports = { 44 | undoItems: importConfig => 45 | new Promise(async (resolve, reject) => { 46 | try { 47 | queues[importConfig.id] = importConfig.importeditems; 48 | } catch (error) { 49 | reject(error); 50 | } 51 | 52 | await strapi 53 | .query('importconfig', 'import-content') 54 | .update({ id: importConfig.id }, { ongoing: true }); 55 | 56 | resolve({ 57 | status: 'undo started', 58 | importConfigId: importConfig.id 59 | }); 60 | 61 | const uploadConfig = await strapi 62 | .store({ 63 | environment: strapi.config.environment, 64 | type: 'plugin', 65 | name: 'upload' 66 | }) 67 | .get({ key: 'provider' }); 68 | 69 | undoNextItem(importConfig, uploadConfig); 70 | }) 71 | }; 72 | -------------------------------------------------------------------------------- /services/utils/FileDataResolver.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const request = require('request'); 3 | const validateUrl = require('./validateUrl'); 4 | 5 | const getDataFromUrl = url => { 6 | return new Promise((resolve, reject) => { 7 | if (!validateUrl(url)) return reject('invalid URL'); 8 | 9 | request(url, null, async (err, res, body) => { 10 | if (err) { 11 | reject(err); 12 | } 13 | 14 | resolve({ contentType: res.headers['content-type'], body }); 15 | }); 16 | }); 17 | }; 18 | 19 | const resolveFileDataFromRequest = async ctx => { 20 | const { source, type, options, data } = ctx.request.body; 21 | 22 | switch (source) { 23 | case 'upload': 24 | return { contentType: type, body: data, options }; 25 | 26 | case 'url': 27 | const { contentType, body } = await getDataFromUrl(options.url); 28 | return { contentType, body, options }; 29 | 30 | case 'raw': 31 | return { 32 | contentType: type, 33 | body: options.rawText, 34 | options 35 | }; 36 | } 37 | }; 38 | 39 | module.exports = { 40 | resolveFileDataFromRequest, 41 | getDataFromUrl 42 | }; 43 | -------------------------------------------------------------------------------- /services/utils/FileDataResolver.spec.js: -------------------------------------------------------------------------------- 1 | const nock = require('nock'); 2 | const { 3 | resolveFileDataFromRequest, 4 | getDataFromUrl 5 | } = require('./FileDataResolver'); 6 | 7 | const TEST_BASE_URL = 'http://test'; 8 | const TEST_PATH = '/'; 9 | 10 | describe('resolveFileDataFromRequest', () => { 11 | it('requests data for "url" source', async () => { 12 | nock(TEST_BASE_URL) 13 | .get(TEST_PATH) 14 | .reply(200, 'data', { 'Content-Type': 'text/csv' }); 15 | 16 | const ctx = { 17 | request: { 18 | body: { source: 'url', options: { url: TEST_BASE_URL + TEST_PATH } } 19 | } 20 | }; 21 | 22 | const response = await resolveFileDataFromRequest(ctx); 23 | 24 | console.log(response); 25 | expect(response).toHaveProperty('contentType'); 26 | expect(response).toHaveProperty('body'); 27 | expect(response).toHaveProperty('options'); 28 | }); 29 | 30 | it('parses data for "raw" source', async () => { 31 | const ctx = { 32 | request: { 33 | body: { 34 | source: 'raw', 35 | type: 'text/csv', 36 | options: {}, 37 | data: 'key1,key2\nvalue1,value2' 38 | } 39 | } 40 | }; 41 | 42 | const response = await resolveFileDataFromRequest(ctx); 43 | 44 | console.log(response); 45 | expect(response).toHaveProperty('contentType'); 46 | expect(response).toHaveProperty('body'); 47 | expect(response).toHaveProperty('options'); 48 | }); 49 | 50 | it('parses data for "upload" source', async () => { 51 | const ctx = { 52 | request: { body: { source: 'upload', data: 'abc123' } } 53 | }; 54 | 55 | const response = await resolveFileDataFromRequest(ctx); 56 | console.log(response); 57 | expect(response).toHaveProperty('contentType'); 58 | expect(response).toHaveProperty('body'); 59 | expect(response).toHaveProperty('options'); 60 | }); 61 | }); 62 | 63 | describe('getDataFromUrl', () => { 64 | it('fetches type and data from a url', async () => { 65 | nock(TEST_BASE_URL) 66 | .get(TEST_PATH) 67 | .reply(200, 'data', { 'Content-Type': 'text/csv' }); 68 | 69 | const response = await getDataFromUrl(TEST_BASE_URL + TEST_PATH); 70 | 71 | expect(response).toHaveProperty('contentType'); 72 | expect(response).toHaveProperty('body'); 73 | }); 74 | }); 75 | -------------------------------------------------------------------------------- /services/utils/analyzer.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const _ = require('lodash'); 3 | var ss = require('simple-statistics'); 4 | const { compileStatsForFieldData } = require('./fieldUtils'); 5 | 6 | const getFieldNameSet = items => { 7 | const fieldNames = new Set(); 8 | items.forEach(item => { 9 | try { 10 | Object.keys(item).forEach(fieldName => fieldNames.add(fieldName)); 11 | } catch (e) { 12 | console.log(e); 13 | } 14 | }); 15 | 16 | return fieldNames; 17 | }; 18 | 19 | const analyze = (sourceType, items) => { 20 | const fieldNames = getFieldNameSet(items); 21 | 22 | const fieldAnalyses = {}; 23 | fieldNames.forEach(fieldName => (fieldAnalyses[fieldName] = [])); 24 | 25 | items.forEach(item => { 26 | fieldNames.forEach(fieldName => { 27 | const fieldData = item[fieldName]; 28 | const fieldStats = compileStatsForFieldData(fieldData); 29 | fieldAnalyses[fieldName].push(fieldStats); 30 | }); 31 | }); 32 | 33 | const fieldStats = Object.keys(fieldAnalyses).map(fieldName => { 34 | const fieldAnalysis = fieldAnalyses[fieldName]; 35 | 36 | const fieldStat = { fieldName, count: fieldAnalysis.length }; 37 | 38 | try { 39 | fieldStat.format = _ 40 | .chain(fieldAnalysis) 41 | .countBy('format') 42 | .map((value, key) => ({ count: value, type: key })) 43 | .sortBy('count') 44 | .reverse() 45 | .head() 46 | .get('type') 47 | .value(); 48 | } catch (e) { 49 | console.log(e); 50 | } 51 | 52 | fieldStat.hasMediaUrls = fieldAnalysis.some(fa => Boolean(fa.hasMediaUrls)); 53 | 54 | const lengths = _.map(fieldAnalysis, 'length'); 55 | 56 | fieldStat.minLength = ss.min(lengths); 57 | fieldStat.maxLength = ss.max(lengths); 58 | fieldStat.meanLength = ss.mean(lengths).toFixed(2); 59 | 60 | return fieldStat; 61 | }); 62 | 63 | return { itemCount: items.length, fieldStats }; 64 | }; 65 | 66 | module.exports = { getFieldNameSet, analyze }; 67 | -------------------------------------------------------------------------------- /services/utils/analyzer.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { getFieldNameSet, analyze } = require('./analyzer'); 4 | 5 | describe('getFieldNameSet', () => { 6 | it('finds set of all field names', () => { 7 | const fieldNames = getFieldNameSet([{ a: 1 }, { a: 1, b: 1 }, { c: 1 }]); 8 | 9 | expect(fieldNames).toEqual(new Set(['a', 'b', 'c'])); 10 | }); 11 | }); 12 | 13 | describe('analyze', () => { 14 | it('generates an analysis from an item array', () => { 15 | const analysis = analyze({}, []); 16 | 17 | expect(analysis.itemCount).toEqual(0); 18 | expect(analysis.fieldStats).toEqual([]); 19 | }); 20 | }); 21 | -------------------------------------------------------------------------------- /services/utils/fieldUtils.js: -------------------------------------------------------------------------------- 1 | const getUrls = require('get-urls'); 2 | const urlIsMedia = require('./urlIsMedia'); 3 | const striptags = require('striptags'); 4 | const stringIsEmail = require('./stringIsEmail'); 5 | 6 | const detectStringFieldFormat = data => { 7 | if (new Date(data).toString() !== 'Invalid Date') return 'date'; 8 | 9 | if (stringIsEmail(data)) return 'email'; 10 | 11 | if (data.length !== striptags(data).length) { 12 | return 'xml'; 13 | } 14 | 15 | return 'string'; 16 | }; 17 | 18 | const detectFieldFormat = data => { 19 | switch (typeof data) { 20 | case 'number': 21 | return 'number'; 22 | 23 | case 'boolean': 24 | return 'boolean'; 25 | 26 | case 'object': 27 | return 'object'; 28 | 29 | case 'string': 30 | return detectStringFieldFormat(data); 31 | } 32 | }; 33 | 34 | const compileStatsForFieldData = fieldData => { 35 | const stats = {}; 36 | 37 | switch (typeof fieldData) { 38 | case 'string': 39 | try { 40 | const urls = Array.from(getUrls(fieldData)); 41 | 42 | const l = urls.length; 43 | for (let i = 0; i < l; ++i) { 44 | if (urlIsMedia(urls[i])) { 45 | stats.hasMediaUrls = true; 46 | break; 47 | } 48 | } 49 | } catch (e) { 50 | console.log(e); 51 | } 52 | stats.length = fieldData.length; 53 | break; 54 | 55 | case 'object': 56 | if (urlIsMedia(fieldData.url)) { 57 | stats.hasMediaUrls = true; 58 | } 59 | stats.length = JSON.stringify(fieldData).length; 60 | break; 61 | 62 | default: 63 | console.log(typeof fieldData, fieldData); 64 | } 65 | 66 | stats.format = detectFieldFormat(fieldData); 67 | 68 | return stats; 69 | }; 70 | 71 | module.exports = { 72 | detectStringFieldFormat, 73 | detectFieldFormat, 74 | compileStatsForFieldData 75 | }; 76 | -------------------------------------------------------------------------------- /services/utils/fieldUtils.spec.js: -------------------------------------------------------------------------------- 1 | const fieldUtils = require('./fieldUtils'); 2 | 3 | describe('detectStringFieldFormat', () => { 4 | it('detects a date string', () => { 5 | const dateString = new Date().toString(); 6 | const result = fieldUtils.detectStringFieldFormat(dateString); 7 | expect(result).toEqual('date'); 8 | }); 9 | 10 | it('detects en email', () => { 11 | const email = 'joe@beigerecords.com'; 12 | const result = fieldUtils.detectStringFieldFormat(email); 13 | expect(result).toEqual('email'); 14 | }); 15 | }); 16 | 17 | describe('detectFieldFormat', () => { 18 | it('detects a number', () => { 19 | const result = fieldUtils.detectFieldFormat(1); 20 | expect(result).toEqual('number'); 21 | }); 22 | 23 | it('detects a boolean', () => { 24 | const result = fieldUtils.detectFieldFormat(false); 25 | expect(result).toEqual('boolean'); 26 | }); 27 | 28 | it('detects a string', () => { 29 | const result = fieldUtils.detectFieldFormat('hello'); 30 | expect(result).toEqual('string'); 31 | }); 32 | }); 33 | 34 | describe('compileStatsForFieldData', () => { 35 | it('returns an object', () => { 36 | const stringData = 'hello http://website'; 37 | const stats = fieldUtils.compileStatsForFieldData(stringData); 38 | expect(stats.length).toEqual(stringData.length); 39 | expect(stats.format).toEqual('string'); 40 | }); 41 | }); 42 | -------------------------------------------------------------------------------- /services/utils/fileFromBuffer.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | const uuid = require('uuid/v4'); 3 | 4 | function niceHash(buffer) { 5 | return crypto 6 | .createHash('sha256') 7 | .update(buffer) 8 | .digest('base64') 9 | .replace(/=/g, '') 10 | .replace(/\//g, '-') 11 | .replace(/\+/, '_'); 12 | } 13 | 14 | const fileFromBuffer = (mimeType, extension, buffer) => { 15 | const fid = uuid(); 16 | 17 | return { 18 | buffer, 19 | sha256: niceHash(buffer), 20 | hash: fid.replace(/-/g, ''), 21 | 22 | name: `${fid}.${extension}`, 23 | ext: `.${extension}`, 24 | mime: mimeType, 25 | size: (buffer.length / 1000).toFixed(2) 26 | }; 27 | }; 28 | 29 | module.exports = fileFromBuffer; 30 | -------------------------------------------------------------------------------- /services/utils/fileUtils.js: -------------------------------------------------------------------------------- 1 | const contentTypeParser = require('content-type-parser'); 2 | const RssParser = require('rss-parser'); 3 | const parse = require('csv-parse/lib/sync'); 4 | 5 | const getItemsForFileData = ({ contentType, body, options }) => 6 | new Promise(async (resolve, reject) => { 7 | const parsedContentType = contentTypeParser(contentType); 8 | 9 | if (parsedContentType.isXML()) { 10 | const parser = new RssParser(); 11 | const feed = await parser.parseString(body); 12 | 13 | return resolve({ sourceType: 'rss', items: feed.items }); 14 | } 15 | 16 | if (contentType === 'text/csv') { 17 | const items = parse(body, { 18 | ...options, 19 | columns: true 20 | }); 21 | return resolve({ sourceType: 'csv', items }); 22 | } 23 | 24 | reject({ 25 | contentType: parsedContentType.toString() 26 | }); 27 | }); 28 | 29 | module.exports = { getItemsForFileData }; 30 | -------------------------------------------------------------------------------- /services/utils/fileUtils.spec.js: -------------------------------------------------------------------------------- 1 | const { getItemsForFileData } = require("./fileUtils"); 2 | 3 | jest.mock("content-type-parser", () => () => ({ 4 | isXML: () => true 5 | })); 6 | 7 | jest.mock( 8 | "rss-parser", 9 | () => 10 | class RssParser { 11 | parseString() { 12 | return { items: [] }; 13 | } 14 | } 15 | ); 16 | describe("getItemsForFileData", () => { 17 | it("returns items from an rss file", async () => { 18 | const response = await getItemsForFileData("application/xml", ""); 19 | 20 | expect(response.items).toEqual([]); 21 | expect(response.sourceType).toEqual("rss"); 22 | }); 23 | }); 24 | -------------------------------------------------------------------------------- /services/utils/getMediaUrlsFromFieldData.js: -------------------------------------------------------------------------------- 1 | const getUrls = require('get-urls'); 2 | const urlIsMedia = require('./urlIsMedia'); 3 | 4 | const getMediaUrlsFromFieldData = fieldData => { 5 | switch (typeof fieldData) { 6 | case 'string': 7 | return Array.from(getUrls(fieldData)).filter(urlIsMedia); 8 | 9 | case 'object': 10 | return urlIsMedia(fieldData.url) ? [fieldData.url] : []; 11 | } 12 | }; 13 | 14 | module.exports = getMediaUrlsFromFieldData; 15 | -------------------------------------------------------------------------------- /services/utils/getMediaUrlsFromFieldData.spec.js: -------------------------------------------------------------------------------- 1 | const getMediaUrlsFromFieldData = require('./getMediaUrlsFromFieldData'); 2 | 3 | const TEST_URL = 'http://site.com/image.png'; 4 | 5 | describe('getMediaUrlsFromFieldData', () => { 6 | it('finds urls in string', () => { 7 | expect(getMediaUrlsFromFieldData(TEST_URL)).toEqual([TEST_URL]); 8 | expect(getMediaUrlsFromFieldData('💩')).toEqual([]); 9 | }); 10 | 11 | it('finds urls in object', () => { 12 | const TEST_OBJECT = { url: TEST_URL }; 13 | expect(getMediaUrlsFromFieldData(TEST_OBJECT)).toEqual([TEST_URL]); 14 | expect(getMediaUrlsFromFieldData({})).toEqual([]); 15 | }); 16 | }); 17 | -------------------------------------------------------------------------------- /services/utils/getUploadProvider.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const path = require('path'); 3 | 4 | module.exports = async () => { 5 | const uploadProviderConfig = await strapi 6 | .store({ 7 | environment: strapi.config.environment, 8 | type: 'plugin', 9 | name: 'upload' 10 | }) 11 | .get({ key: 'provider' }); 12 | 13 | // Get upload provider settings to configure the provider to use. 14 | const provider = _.find(strapi.plugins.upload.config.providers, { 15 | provider: uploadProviderConfig.provider 16 | }); 17 | 18 | if (!provider) { 19 | throw new Error( 20 | `The provider package isn't installed. Please run \`npm install strapi-provider-upload-${ 21 | uploadProviderConfig.provider 22 | }\`` 23 | ); 24 | } 25 | 26 | const getPath = file => 27 | uploadProviderConfig.provider === 'local' 28 | ? path.join(strapi.config.appPath, strapi.config.public.path, file.url) 29 | : file.url; 30 | 31 | return { 32 | config: uploadProviderConfig, 33 | provider, 34 | actions: await provider.init(uploadProviderConfig), 35 | getPath 36 | }; 37 | }; 38 | -------------------------------------------------------------------------------- /services/utils/importFields.js: -------------------------------------------------------------------------------- 1 | const striptags = require('striptags'); 2 | 3 | const importFields = async (sourceItem, fieldMapping) => { 4 | const importedItem = {}; 5 | 6 | Object.keys(fieldMapping).forEach(async sourceField => { 7 | const { targetField, stripTags } = fieldMapping[sourceField]; 8 | if (!targetField || targetField === 'none') { 9 | return; 10 | } 11 | 12 | const originalValue = sourceItem[sourceField]; 13 | 14 | importedItem[targetField] = stripTags 15 | ? striptags(originalValue) 16 | : originalValue; 17 | }); 18 | return importedItem; 19 | }; 20 | 21 | module.exports = importFields; 22 | -------------------------------------------------------------------------------- /services/utils/importFields.spec.js: -------------------------------------------------------------------------------- 1 | const importFields = require('./importFields'); 2 | 3 | describe('importField', () => { 4 | it('copies field data into result', async () => { 5 | const result = await importFields( 6 | { a: 'hello' }, 7 | { a: { targetField: 'b' } } 8 | ); 9 | expect(result['b']).toEqual('hello'); 10 | }); 11 | }); 12 | -------------------------------------------------------------------------------- /services/utils/importMediaFiles.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const request = require('request'); 3 | const getUploadProvider = require('./getUploadProvider'); 4 | const fileFromBuffer = require('./fileFromBuffer'); 5 | const getMediaUrlsFromFieldData = require('./getMediaUrlsFromFieldData'); 6 | 7 | const fetchAndStoreFiles = url => 8 | new Promise((resolve, reject) => { 9 | request({ url, method: 'GET', encoding: null }, async (err, res, body) => { 10 | if (err) { 11 | reject(err); 12 | } 13 | 14 | const mimeType = res.headers['content-type'].split(';').shift(); 15 | 16 | const parsed = new URL(url); 17 | const extension = parsed.pathname 18 | .split('.') 19 | .pop() 20 | .toLowerCase(); 21 | 22 | const { provider, actions } = await getUploadProvider(); 23 | 24 | const fileDescriptor = fileFromBuffer(mimeType, extension, body); 25 | 26 | await actions.upload(fileDescriptor); 27 | 28 | delete fileDescriptor.buffer; 29 | 30 | fileDescriptor.provider = provider.provider; 31 | 32 | resolve(fileDescriptor); 33 | }); 34 | }); 35 | 36 | const relateFileToContent = async ({ 37 | contentType, 38 | contentId, 39 | targetField, 40 | fileDescriptor 41 | }) => { 42 | fileDescriptor.related = [ 43 | { 44 | refId: contentId, 45 | ref: contentType, 46 | source: 'content-manager', 47 | field: targetField 48 | } 49 | ]; 50 | 51 | return await strapi.plugins['upload'].services.upload.add(fileDescriptor); 52 | }; 53 | 54 | const importMediaFiles = async (savedContent, sourceItem, importConfig) => { 55 | const { fieldMapping, contentType } = importConfig; 56 | 57 | const uploadedFileDescriptors = _.mapValues( 58 | fieldMapping, 59 | async (mapping, sourceField) => { 60 | if (mapping.importMediaToField) { 61 | const urls = getMediaUrlsFromFieldData(sourceItem[sourceField]); 62 | 63 | const uploadPromises = _.uniq(urls).map(fetchAndStoreFiles); 64 | 65 | const fileDescriptors = await Promise.all(uploadPromises); 66 | 67 | const relateContentPromises = fileDescriptors.map(fileDescriptor => 68 | relateFileToContent({ 69 | contentType, 70 | contentId: savedContent.id, 71 | targetField: mapping.importMediaToField, 72 | fileDescriptor 73 | }) 74 | ); 75 | 76 | return await Promise.all(relateContentPromises); 77 | } 78 | } 79 | ); 80 | 81 | return await Promise.all(_.values(uploadedFileDescriptors)); 82 | }; 83 | 84 | module.exports = importMediaFiles; 85 | -------------------------------------------------------------------------------- /services/utils/stringIsEmail.js: -------------------------------------------------------------------------------- 1 | const EMAIL_REGEXP = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/; 2 | 3 | const stringIsEmail = data => { 4 | EMAIL_REGEXP.lastIndex = 0; 5 | return EMAIL_REGEXP.test(data); 6 | }; 7 | 8 | module.exports = stringIsEmail; 9 | -------------------------------------------------------------------------------- /services/utils/stringIsEmail.spec.js: -------------------------------------------------------------------------------- 1 | const stringIsEmail = require('./stringIsEmail'); 2 | 3 | const GOOD_EMAILS = ['jim@jimspage.com', 'j45987098712@gmail.org']; 4 | const NON_EMAILS = ['💩', 'http://businesstime.com']; 5 | 6 | describe('stringIsEmail', () => { 7 | it('recognizes good emails', () => { 8 | GOOD_EMAILS.forEach(str => expect(stringIsEmail(str)).toBeTruthy()); 9 | }); 10 | it('recognizes non emails', () => { 11 | NON_EMAILS.forEach(str => expect(stringIsEmail(str)).toBeFalsy()); 12 | }); 13 | }); 14 | -------------------------------------------------------------------------------- /services/utils/urlIsMedia.js: -------------------------------------------------------------------------------- 1 | const urlIsMedia = url => { 2 | try { 3 | const parsed = new URL(url); 4 | 5 | const extension = parsed.pathname 6 | .split('.') 7 | .pop() 8 | .toLowerCase(); 9 | 10 | switch (extension) { 11 | case 'png': 12 | case 'gif': 13 | case 'jpg': 14 | case 'jpeg': 15 | case 'svg': 16 | case 'bmp': 17 | case 'tif': 18 | case 'tiff': 19 | return true; 20 | 21 | case 'mp3': 22 | case 'wav': 23 | case 'ogg': 24 | return true; 25 | 26 | case 'mp4': 27 | case 'avi': 28 | return true; 29 | 30 | default: 31 | return false; 32 | } 33 | } catch (error) { 34 | // Was likely a bad URL 35 | return false; 36 | } 37 | }; 38 | 39 | module.exports = urlIsMedia; 40 | -------------------------------------------------------------------------------- /services/utils/urlIsMedia.spec.js: -------------------------------------------------------------------------------- 1 | const urlIsMedia = require('./urlIsMedia'); 2 | 3 | describe('urlIsMedia', () => { 4 | it('returns true for image urls', () => { 5 | const URLS = [ 6 | 'http://site.com/image.gif', 7 | 'http://other.site.com/image.gif?hello=1', 8 | 'http://127.0.0.1/folder/image.PNG', 9 | 'http://www.website.com/two/folders/gif.jpeg.JPEG', 10 | 'https://site.com/thing.jpg' 11 | ]; 12 | 13 | URLS.forEach(url => { 14 | expect(urlIsMedia(url)).toBeTruthy(); 15 | }); 16 | }); 17 | 18 | it('returns false for non-image urls', () => { 19 | const URLS = [ 20 | 'http://site.com/gif.text', 21 | 'https://homepage/location.gis', 22 | 'https://localhost:333/page.html' 23 | ]; 24 | 25 | URLS.forEach(url => { 26 | expect(urlIsMedia(url)).toBeFalsy(); 27 | }); 28 | }); 29 | }); 30 | -------------------------------------------------------------------------------- /services/utils/validateUrl.js: -------------------------------------------------------------------------------- 1 | var urlRegEx = /((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-]*)?\??(?:[\-\+=&;%@\.\w]*)#?(?:[\.\!\/\\\w]*))?)/g; 2 | 3 | const URL_REGEXP = new RegExp(urlRegEx); 4 | 5 | const validateUrl = url => { 6 | URL_REGEXP.lastIndex = 0; 7 | return URL_REGEXP.test(url); 8 | }; 9 | 10 | module.exports = validateUrl; 11 | -------------------------------------------------------------------------------- /services/utils/validateUrl.spec.js: -------------------------------------------------------------------------------- 1 | const validateUrl = require('./validateUrl'); 2 | 3 | const GOOD_URLS = [ 4 | 'http://cernalerts.web.cern.ch/cernalerts/?feed=cern%20hot%20news', 5 | 'https://www.nasa.gov/rss/dyn/lg_image_of_the_day.rss' 6 | ]; 7 | 8 | const BAD_URLS = ['hello my name is', '1234']; 9 | 10 | describe('validateUrl', () => { 11 | it('returns true for valid urls', () => { 12 | GOOD_URLS.forEach(url => expect(validateUrl(url)).toBeTruthy()); 13 | }); 14 | it('returns false for not urls', () => { 15 | BAD_URLS.forEach(url => expect(validateUrl(url)).toBeFalsy()); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /video_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbeuckm/strapi-plugin-import-content/6c6131826bff162b107294005b035ab0c43a72a2/video_thumbnail.png --------------------------------------------------------------------------------