├── test ├── fixtures │ ├── gzip │ │ ├── topic-draft │ │ │ ├── attachments │ │ │ │ └── .gitkeep │ │ │ ├── aposAttachments.json │ │ │ └── aposDocs.json │ │ ├── fr-topic-draft │ │ │ ├── attachments │ │ │ │ └── .gitkeep │ │ │ ├── aposAttachments.json │ │ │ └── aposDocs.json │ │ ├── topic-draft-published │ │ │ ├── attachments │ │ │ │ └── .gitkeep │ │ │ ├── aposAttachments.json │ │ │ └── aposDocs.json │ │ ├── topic-draft-lastPublishedAt │ │ │ ├── attachments │ │ │ │ └── .gitkeep │ │ │ ├── aposAttachments.json │ │ │ └── aposDocs.json │ │ ├── topic-draft-published-aposDocId │ │ │ ├── attachments │ │ │ │ └── .gitkeep │ │ │ ├── aposAttachments.json │ │ │ └── aposDocs.json │ │ └── topic-draft-published-lastPublishedAt │ │ │ ├── attachments │ │ │ └── .gitkeep │ │ │ ├── aposAttachments.json │ │ │ └── aposDocs.json │ ├── topic-title.csv │ ├── topic-type-title-lastPublishedAt.csv │ ├── default-page-type-title-main.csv │ ├── default-page-type-title-lastPublishedAt.csv │ ├── topic-title-description-main.csv │ ├── topic-type-title-description-main.csv │ ├── topic-type-titleKey-title-lastPublishedAt.csv │ ├── default-page-type-titleKey-title-lastPublishedAt.csv │ ├── default-page-type-titleKey-title-main.csv │ ├── topic-type-titleKey-title-slug-lastPublishedAt.csv │ ├── default-page-type-titleKey-title-slug-lastPublishedAt.csv │ └── topic-type-titleKey-title-description-main.csv ├── public │ └── test-image.jpg ├── modules │ └── @apostrophecms │ │ ├── log │ │ └── index.js │ │ └── home-page │ │ └── views │ │ └── page.html ├── package.json ├── util │ └── index.js ├── overrideLocales.js └── overrideDuplicates.js ├── .stylelintrc.json ├── images ├── bulk-export.png ├── page-export.png ├── page-import.png ├── template-export.png ├── import-file-modal.png ├── page-export-modal.png ├── piece-batch-export.png ├── different-locale-modal.png └── page-import-with-translation.png ├── eslint.config.js ├── lib ├── formats │ ├── index.js │ ├── csv.js │ └── gzip.js ├── apiRoutes.js ├── handlers.js └── methods │ ├── import-page.js │ ├── index.js │ └── export.js ├── modules └── @apostrophecms │ ├── import-export-user │ └── index.js │ ├── import-export-asset │ └── index.js │ ├── import-export-page │ └── index.js │ └── import-export-piece-type │ └── index.js ├── .editorconfig ├── .gitignore ├── LICENSE.md ├── index.js ├── package.json ├── .github └── workflows │ └── main.yml ├── i18n ├── sk.json ├── pt-BR.json ├── it.json ├── fr.json ├── de.json ├── es.json └── en.json ├── ui └── apos │ ├── apps │ └── index.js │ └── components │ ├── AposImportModal.vue │ ├── AposDuplicateImportModal.vue │ └── AposExportModal.vue ├── CHANGELOG.md └── README.md /test/fixtures/gzip/topic-draft/attachments/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/topic-title.csv: -------------------------------------------------------------------------------- 1 | title 2 | "topic1" 3 | -------------------------------------------------------------------------------- /test/fixtures/gzip/fr-topic-draft/attachments/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft/aposAttachments.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /test/fixtures/gzip/fr-topic-draft/aposAttachments.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published/attachments/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-lastPublishedAt/attachments/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published/aposAttachments.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published-aposDocId/attachments/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.stylelintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "stylelint-config-apostrophe" 3 | } 4 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-lastPublishedAt/aposAttachments.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published-aposDocId/aposAttachments.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published-lastPublishedAt/attachments/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published-lastPublishedAt/aposAttachments.json: -------------------------------------------------------------------------------- 1 | [] 2 | -------------------------------------------------------------------------------- /images/bulk-export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/bulk-export.png -------------------------------------------------------------------------------- /images/page-export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/page-export.png -------------------------------------------------------------------------------- /images/page-import.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/page-import.png -------------------------------------------------------------------------------- /images/template-export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/template-export.png -------------------------------------------------------------------------------- /test/public/test-image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/test/public/test-image.jpg -------------------------------------------------------------------------------- /images/import-file-modal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/import-file-modal.png -------------------------------------------------------------------------------- /images/page-export-modal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/page-export-modal.png -------------------------------------------------------------------------------- /images/piece-batch-export.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/piece-batch-export.png -------------------------------------------------------------------------------- /images/different-locale-modal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/different-locale-modal.png -------------------------------------------------------------------------------- /test/fixtures/topic-type-title-lastPublishedAt.csv: -------------------------------------------------------------------------------- 1 | type,title,lastPublishedAt 2 | "topic","topic1","2021-01-01T00:00:00.000Z" 3 | -------------------------------------------------------------------------------- /test/fixtures/default-page-type-title-main.csv: -------------------------------------------------------------------------------- 1 | type,title,main 2 | "default-page","page1","

rich text

" 3 | -------------------------------------------------------------------------------- /images/page-import-with-translation.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/apostrophecms/import-export/main/images/page-import-with-translation.png -------------------------------------------------------------------------------- /test/fixtures/default-page-type-title-lastPublishedAt.csv: -------------------------------------------------------------------------------- 1 | type,title,lastPublishedAt 2 | "default-page","page1","2021-01-01T00:00:00.000Z" 3 | -------------------------------------------------------------------------------- /test/fixtures/topic-title-description-main.csv: -------------------------------------------------------------------------------- 1 | title,description,main 2 | "topic1","description1","

rich text

" 3 | -------------------------------------------------------------------------------- /test/fixtures/topic-type-title-description-main.csv: -------------------------------------------------------------------------------- 1 | type,title,description,main 2 | "topic","topic1","description1","

rich text

" 3 | -------------------------------------------------------------------------------- /test/fixtures/topic-type-titleKey-title-lastPublishedAt.csv: -------------------------------------------------------------------------------- 1 | type,title:key,title,lastPublishedAt 2 | "topic","topic1","topic1 - edited","2021-01-01T00:00:00.000Z" 3 | -------------------------------------------------------------------------------- /test/fixtures/default-page-type-titleKey-title-lastPublishedAt.csv: -------------------------------------------------------------------------------- 1 | type,title:key,title,lastPublishedAt 2 | "default-page","page1","page1 - edited","2021-01-01T00:00:00.000Z" 3 | -------------------------------------------------------------------------------- /test/fixtures/default-page-type-titleKey-title-main.csv: -------------------------------------------------------------------------------- 1 | type,title:key,title,main 2 | "default-page","page1","page1 - edited","

rich text - edited

" 3 | -------------------------------------------------------------------------------- /test/fixtures/topic-type-titleKey-title-slug-lastPublishedAt.csv: -------------------------------------------------------------------------------- 1 | type,title:key,title,slug,lastPublishedAt 2 | "topic","topic1 aaa","topic1 bbb","topic1-bbb","2021-01-01T00:00:00.000Z" 3 | -------------------------------------------------------------------------------- /test/fixtures/default-page-type-titleKey-title-slug-lastPublishedAt.csv: -------------------------------------------------------------------------------- 1 | type,title:key,title,slug,lastPublishedAt 2 | "default-page","page1 aaa","page1 bbb","/page1-bbb","2021-01-01T00:00:00.000Z" 3 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | const apostrophe = require('eslint-config-apostrophe').default; 2 | const { defineConfig } = require('eslint/config'); 3 | 4 | module.exports = defineConfig([ 5 | apostrophe 6 | ]); 7 | -------------------------------------------------------------------------------- /lib/formats/index.js: -------------------------------------------------------------------------------- 1 | const gzip = require('./gzip'); 2 | const csv = require('./csv'); 3 | 4 | // Keep gzip at the top of the list so it's the default format 5 | module.exports = { 6 | gzip, 7 | csv 8 | }; 9 | -------------------------------------------------------------------------------- /test/fixtures/topic-type-titleKey-title-description-main.csv: -------------------------------------------------------------------------------- 1 | type,title:key,title,description,main 2 | "topic","topic1","topic1 - edited","description1 - edited","

rich text - edited

" 3 | -------------------------------------------------------------------------------- /modules/@apostrophecms/import-export-user/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | improve: '@apostrophecms/user', 3 | options: { 4 | importExport: { 5 | import: false, 6 | export: false 7 | } 8 | } 9 | }; 10 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft/aposDocs.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "4:en:draft", 4 | "aposDocId": "4", 5 | "aposLocale": "en:draft", 6 | "aposMode": "draft", 7 | "slug": "topic1", 8 | "title": "topic1", 9 | "type": "topic" 10 | } 11 | ] 12 | -------------------------------------------------------------------------------- /test/fixtures/gzip/fr-topic-draft/aposDocs.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "4:fr:draft", 4 | "aposDocId": "4", 5 | "aposLocale": "fr:draft", 6 | "aposMode": "draft", 7 | "slug": "topic1-fr", 8 | "title": "topic1 FR", 9 | "type": "topic" 10 | } 11 | ] 12 | -------------------------------------------------------------------------------- /modules/@apostrophecms/import-export-asset/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | improve: '@apostrophecms/asset', 3 | 4 | init(self) { 5 | self.iconMap['apos-import-export-download-icon'] = 'Download'; 6 | self.iconMap['apos-import-export-upload-icon'] = 'Upload'; 7 | } 8 | }; 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | charset = utf-8 7 | end_of_line = lf 8 | indent_size = 2 9 | indent_style = space 10 | insert_final_newline = true 11 | trim_trailing_whitespace = true 12 | 13 | [*.md] 14 | trim_trailing_whitespace = false 15 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-lastPublishedAt/aposDocs.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "4:en:draft", 4 | "aposDocId": "4", 5 | "aposLocale": "en:draft", 6 | "aposMode": "draft", 7 | "lastPublishedAt": "2021-01-01T00:00:00.000Z", 8 | "slug": "topic1-draft", 9 | "title": "topic1 DRAFT", 10 | "type": "topic" 11 | } 12 | ] 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore MacOS X metadata forks (fusefs) 2 | ._* 3 | package-lock.json 4 | *.DS_Store 5 | node_modules 6 | 7 | # Never commit a CSS map file, anywhere 8 | *.css.map 9 | 10 | # vim swp files 11 | .*.sw* 12 | 13 | # Dont commit test generated css 14 | test/public/css/*.css 15 | test/public/css/master-*.less 16 | 17 | # Dont commit test uploads 18 | /test/data 19 | /test/fixtures/*.tar.gz 20 | /test/public/exports 21 | /test/public/uploads 22 | -------------------------------------------------------------------------------- /test/modules/@apostrophecms/log/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | options: { 3 | filter: { 4 | // By module name, or *. We can specify any mix of severity levels and specific 5 | // event types, and entries are kept if *either* criterion is met 6 | '*': { 7 | severity: [ 'warn', 'error' ] 8 | }, 9 | '@apostrophecms/login': { 10 | events: [ 'incorrect-user', 'incorrect-password' ] 11 | } 12 | } 13 | } 14 | }; 15 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published/aposDocs.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "4:en:draft", 4 | "aposDocId": "4", 5 | "aposLocale": "en:draft", 6 | "aposMode": "draft", 7 | "slug": "topic1-draft", 8 | "title": "topic1 DRAFT", 9 | "type": "topic" 10 | }, 11 | { 12 | "_id": "4:en:published", 13 | "aposDocId": "4", 14 | "aposLocale": "en:published", 15 | "aposMode": "published", 16 | "slug": "topic1-published", 17 | "title": "topic1 PUBLISHED", 18 | "type": "topic" 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published-aposDocId/aposDocs.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "4:en:draft", 4 | "aposDocId": "4", 5 | "aposLocale": "en:draft", 6 | "aposMode": "draft", 7 | "slug": "topic1-draft", 8 | "title": "topic1 DRAFT", 9 | "type": "topic" 10 | }, 11 | { 12 | "_id": "4:en:published", 13 | "aposDocId": "4", 14 | "aposLocale": "en:published", 15 | "aposMode": "published", 16 | "slug": "topic1-published", 17 | "title": "topic1 PUBLISHED", 18 | "type": "topic" 19 | } 20 | ] 21 | -------------------------------------------------------------------------------- /test/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "/**": "This package.json file is not actually installed.", 3 | " * ": "Apostrophe requires that all npm modules to be loaded by moog", 4 | " */": "exist in package.json at project level, which for a test is here", 5 | "dependencies": { 6 | "@apostrophecms-pro/automatic-translation": "git+https://github.com/apostrophecms/automatic-translation", 7 | "@apostrophecms/import-export": "git+https://github.com/apostrophecms/import-export.git", 8 | "apostrophe": "git+https://github.com/apostrophecms/apostrophe.git" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /test/modules/@apostrophecms/home-page/views/page.html: -------------------------------------------------------------------------------- 1 | {{ data.page.title }} 2 |

Home Page Template

3 | {# This is necessary to the login.js tests. -Tom #} 4 | {% if data.user %} 5 | logged in as {{ data.user.title }} 6 | {% else %} 7 | logged out 8 | {% endif %} 9 | {# Necessary to the @apostrophecms/global tests. #} 10 | counts: {{ data.global.counts }} 11 |

Translations

12 | 17 | -------------------------------------------------------------------------------- /test/fixtures/gzip/topic-draft-published-lastPublishedAt/aposDocs.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "4:en:draft", 4 | "aposDocId": "4", 5 | "aposLocale": "en:draft", 6 | "aposMode": "draft", 7 | "lastPublishedAt": "2021-01-01T00:00:00.000Z", 8 | "slug": "topic1-draft", 9 | "title": "topic1 DRAFT", 10 | "type": "topic" 11 | }, 12 | { 13 | "_id": "4:en:published", 14 | "aposDocId": "4", 15 | "aposLocale": "en:published", 16 | "aposMode": "published", 17 | "lastPublishedAt": "2021-01-01T00:00:00.000Z", 18 | "slug": "topic1-published", 19 | "title": "topic1 PUBLISHED", 20 | "type": "topic" 21 | } 22 | ] 23 | -------------------------------------------------------------------------------- /lib/apiRoutes.js: -------------------------------------------------------------------------------- 1 | module.exports = self => { 2 | if (self.options.importExport?.export === false) { 3 | return {}; 4 | } 5 | 6 | return { 7 | get: { 8 | async related(req) { 9 | if (!req.user) { 10 | throw self.apos.error('forbidden'); 11 | } 12 | 13 | const types = self.apos.launder.strings(req.query.types); 14 | 15 | const related = types.reduce((acc, type) => { 16 | const manager = self.apos.doc.getManager(type); 17 | if (!manager) { 18 | throw self.apos.error('invalid'); 19 | } 20 | 21 | return self.getRelatedTypes(req, manager.schema, acc); 22 | }, []); 23 | 24 | return related; 25 | } 26 | }, 27 | post: { 28 | async overrideDuplicates(req) { 29 | return self.overrideDuplicates(req); 30 | }, 31 | 32 | async cleanExport(req) { 33 | return self.cleanExport(req); 34 | } 35 | } 36 | }; 37 | }; 38 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2023 Apostrophe Technologies 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const fs = require('node:fs'); 2 | const path = require('node:path'); 3 | const handlers = require('./lib/handlers.js'); 4 | const methods = require('./lib/methods/index.js'); 5 | const apiRoutes = require('./lib/apiRoutes.js'); 6 | const formats = require('./lib/formats/index.js'); 7 | 8 | module.exports = { 9 | bundle: { 10 | directory: 'modules', 11 | modules: getBundleModuleNames() 12 | }, 13 | icons: { 14 | 'database-import-icon': 'DatabaseImport' 15 | }, 16 | options: { 17 | name: '@apostrophecms/import-export', 18 | i18n: { 19 | ns: 'aposImportExport', 20 | browser: true 21 | }, 22 | preventUpdateAssets: false, 23 | importDraftsOnlyDefault: false 24 | }, 25 | init(self) { 26 | self.formats = { 27 | ...formats, 28 | ...self.options.formats || {} 29 | }; 30 | 31 | self.enableBrowserData(); 32 | self.timeoutIds = {}; 33 | }, 34 | handlers, 35 | methods, 36 | apiRoutes 37 | }; 38 | 39 | function getBundleModuleNames() { 40 | const source = path.join(__dirname, './modules/@apostrophecms'); 41 | return fs 42 | .readdirSync(source, { withFileTypes: true }) 43 | .filter(dirent => dirent.isDirectory()) 44 | .map(dirent => `@apostrophecms/${dirent.name}`); 45 | } 46 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@apostrophecms/import-export", 3 | "version": "3.5.0", 4 | "description": "Import Export Documents for ApostropheCMS", 5 | "main": "index.js", 6 | "scripts": { 7 | "eslint": "eslint --ext .js,.vue .", 8 | "stylelint": "stylelint ui/**/*.{scss,vue}", 9 | "lint": "npm run eslint && npm run stylelint", 10 | "mocha": "mocha", 11 | "test": "npm run lint && mocha --ignore=test/import-page.js && mocha test/import-page.js" 12 | }, 13 | "repository": { 14 | "type": "git", 15 | "url": "git+https://github.com/apostrophecms/import-export.git" 16 | }, 17 | "homepage": "https://github.com/apostrophecms/import-export#readme", 18 | "author": "Apostrophe Technologies", 19 | "license": "UNLICENSED", 20 | "devDependencies": { 21 | "@apostrophecms-pro/automatic-translation": "github:apostrophecms/automatic-translation", 22 | "apostrophe": "github:apostrophecms/apostrophe", 23 | "eslint-config-apostrophe": "^6.0.2", 24 | "form-data": "^4.0.0", 25 | "mocha": "^10.2.0", 26 | "stylelint": "^16.0.0", 27 | "stylelint-config-apostrophe": "^4.1.0" 28 | }, 29 | "dependencies": { 30 | "bluebird": "^3.7.2", 31 | "bson": "^6.0.0", 32 | "csv-parse": "^5.5.5", 33 | "csv-stringify": "^6.4.6", 34 | "dayjs": "^1.9.8", 35 | "lodash": "^4.17.21", 36 | "tar-stream": "^3.1.6" 37 | } 38 | } -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: tests 4 | 5 | # Controls when the action will run. 6 | on: 7 | push: 8 | branches: ["main"] 9 | pull_request: 10 | branches: ["*"] 11 | 12 | # Allows you to run this workflow manually from the Actions tab 13 | workflow_dispatch: 14 | 15 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 16 | jobs: 17 | # This workflow contains a single job called "build" 18 | build: 19 | # The type of runner that the job will run on 20 | runs-on: ubuntu-latest 21 | strategy: 22 | matrix: 23 | node-version: [20, 22, 24] 24 | mongodb-version: ["6.0", "7.0", "8.0"] 25 | 26 | # Steps represent a sequence of tasks that will be executed as part of the job 27 | steps: 28 | - name: Git checkout 29 | uses: actions/checkout@v4 30 | 31 | - name: Use Node.js ${{ matrix.node-version }} 32 | uses: actions/setup-node@v4 33 | with: 34 | node-version: ${{ matrix.node-version }} 35 | 36 | - name: Grant private repositories access 37 | uses: webfactory/ssh-agent@v0.9.0 38 | with: 39 | ssh-private-key: | 40 | ${{ secrets.AUTOMATIC_TRANSLATION_DEPLOYMENT_KEY }} 41 | 42 | - name: Start MongoDB 43 | uses: supercharge/mongodb-github-action@1.11.0 44 | with: 45 | mongodb-version: ${{ matrix.mongodb-version }} 46 | 47 | - run: npm install 48 | 49 | - run: npm test 50 | env: 51 | CI: true 52 | -------------------------------------------------------------------------------- /lib/formats/csv.js: -------------------------------------------------------------------------------- 1 | const fs = require('node:fs'); 2 | const { stringify } = require('csv-stringify'); 3 | const { parse } = require('csv-parse'); 4 | 5 | module.exports = { 6 | label: 'CSV', 7 | extension: '.csv', 8 | allowedExtension: '.csv', 9 | allowedTypes: [ 'text/csv' ], 10 | async input(filepath) { 11 | const reader = fs.createReadStream(filepath); 12 | const parser = reader 13 | .pipe( 14 | parse({ 15 | columns: true, 16 | bom: true, 17 | cast(value, context) { 18 | if (context.header) { 19 | return value; 20 | } 21 | 22 | try { 23 | return JSON.parse(value); 24 | } catch { 25 | return value; 26 | } 27 | } 28 | }) 29 | ); 30 | 31 | const docs = []; 32 | 33 | parser.on('readable', function() { 34 | let doc; 35 | while ((doc = parser.read()) !== null) { 36 | docs.push(doc); 37 | } 38 | }); 39 | 40 | return new Promise((resolve, reject) => { 41 | reader.on('error', reject); 42 | parser.on('error', reject); 43 | parser.on('end', () => { 44 | console.info(`[csv] docs read from ${filepath}`); 45 | resolve({ docs }); 46 | }); 47 | }); 48 | }, 49 | async output(filepath, { docs }) { 50 | const writer = fs.createWriteStream(filepath); 51 | const stringifier = stringify({ 52 | header: true, 53 | columns: getColumnsNames(docs), 54 | cast: { 55 | date(value) { 56 | return value.toISOString(); 57 | }, 58 | boolean(value) { 59 | return value ? 'true' : 'false'; 60 | } 61 | } 62 | }); 63 | 64 | stringifier.pipe(writer); 65 | 66 | // plunge each doc into the stream 67 | docs.forEach(record => { 68 | stringifier.write(record); 69 | }); 70 | 71 | stringifier.end(); 72 | 73 | return new Promise((resolve, reject) => { 74 | stringifier.on('error', reject); 75 | writer.on('error', reject); 76 | writer.on('finish', () => { 77 | console.info(`[csv] export file written to ${filepath}`); 78 | resolve(); 79 | }); 80 | }); 81 | } 82 | }; 83 | 84 | function getColumnsNames(docs) { 85 | const columns = new Set(); 86 | docs.forEach(doc => { 87 | Object.keys(doc).forEach(key => columns.add(key)); 88 | }); 89 | return Array.from(columns); 90 | } 91 | -------------------------------------------------------------------------------- /lib/handlers.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = self => { 3 | return { 4 | 'apostrophe:modulesRegistered': { 5 | setContextOperations() { 6 | const excludedExportTypes = []; 7 | const excludedImportTypes = []; 8 | for (const mod of Object.values(self.apos.modules)) { 9 | if (!self.apos.instanceOf(mod, '@apostrophecms/doc-type')) { 10 | continue; 11 | } 12 | 13 | if (mod.options.importExport?.export === false) { 14 | excludedExportTypes.push({ 15 | type: { 16 | $ne: mod.__meta.name 17 | } 18 | }); 19 | } 20 | 21 | if (!mod.options.singleton || mod.options.importExport?.import === false) { 22 | excludedImportTypes.push({ 23 | type: { 24 | $ne: mod.__meta.name 25 | } 26 | }); 27 | } 28 | } 29 | 30 | const exportOperation = { 31 | action: 'import-export-export', 32 | context: 'update', 33 | label: 'aposImportExport:export', 34 | modal: 'AposExportModal', 35 | props: { 36 | action: 'import-export-export' 37 | }, 38 | if: { $and: excludedExportTypes } 39 | }; 40 | const importOperation = { 41 | action: 'import-export-import', 42 | context: 'update', 43 | replaces: true, 44 | label: 'aposImportExport:import', 45 | modal: 'AposImportModal', 46 | props: { 47 | action: 'import-export-import' 48 | }, 49 | if: { $and: excludedImportTypes }, 50 | conditions: [ 'canEdit' ] 51 | }; 52 | 53 | self.apos.doc.addContextOperation(exportOperation); 54 | self.apos.doc.addContextOperation(importOperation); 55 | } 56 | }, 57 | 'apostrophe:destroy': { 58 | async clearTimers() { 59 | const ids = Object.keys(self.timeoutIds); 60 | if (!ids.length) { 61 | return; 62 | } 63 | self.apos.util.debug(`Clearing ${ids.length} timer(s)`); 64 | for (const key of ids) { 65 | const entry = self.timeoutIds[key]; 66 | delete self.timeoutIds[key]; 67 | if (!entry) { 68 | continue; 69 | } 70 | clearTimeout(entry.timeoutId); 71 | if (entry.handler) { 72 | await entry.handler(); 73 | } 74 | } 75 | self.apos.util.debug('Timer(s) cleared'); 76 | } 77 | } 78 | }; 79 | }; 80 | -------------------------------------------------------------------------------- /i18n/sk.json: -------------------------------------------------------------------------------- 1 | { 2 | "dayjsTitleDateFormat": "MMM D, YYYY, hh:mm:ss A", 3 | "export": "Exportovať {{ type }}", 4 | "exportAttachmentError": "Niektoré prílohy sa nedali pridať do súboru {{ format }}", 5 | "exportFailed": "Export zlyhal", 6 | "exportFileGenerationError": "Generovanie súboru {{ format }} zlyhalo", 7 | "exportModalDescription": "Vybrali ste {{ count }} {{ type }} na export", 8 | "exportModalDocumentFormat": "Formát dokumentu", 9 | "exportModalIncludeChildren": "Zahrnúť deti tejto stránky", 10 | "exportModalIncludeRelated": "Zahrnúť súvisiace dokumenty", 11 | "exportModalIncludeRelatedSettings": "Nastavenia súvisiacich dokumentov", 12 | "exportModalNoRelatedTypes": "Žiadne súvisiace typy", 13 | "exportModalRelatedDocumentDescription": "Zahrnúť nasledujúce typy dokumentov tam, kde je to relevantné", 14 | "exportModalSettingsLabel": "Nastavenia exportu", 15 | "exported": "Exportované {{ count }} {{ type }}", 16 | "exportedFailed": "Exportovanie {{ type }} zlyhalo", 17 | "exportedWithFailures": "Exportované {{ count }} {{ type }} ({{ bad }} z {{ total }} zlyhalo)", 18 | "exporting": "Exportovanie {{ type }}...", 19 | "import": "Importovať {{ type }}", 20 | "importCleanFailed": "Údržba importovaného súboru na serveri zlyhala.", 21 | "importDuplicateContinue": "Pokračovať v importe", 22 | "importDuplicateDetected": "Zistené duplicity.", 23 | "importDuplicateMessage": "Skontrolujte položky, ktoré chcete pri importe prepísať.", 24 | "importFailed": "Import zlyhal", 25 | "importFailedForSome": "{{ count }} dokumentov malo pri importe chybu", 26 | "importFileError": "Import súboru {{ format }} zlyhal", 27 | "importModalDescription": "Import obsahu vyžaduje súbor {{ formats }}. Pozrite si našu oficiálnu dokumentáciu.", 28 | "importSucceed": "Import úspešný!", 29 | "importWarning": "Importovanie exportného súboru Apostrophe na nekompatibilnej webovej stránke nemusí fungovať, ako sa očakáva.", 30 | "importWithCurrentLocaleDescription": "Tento súbor bol exportovaný z jazyka \"{{ docsLocale }}\". Ste si istí, že ho chcete importovať do jazyka \"{{ currentLocale }}\"?", 31 | "importWithCurrentLocaleHeading": "Zistený iný jazyk", 32 | "imported": "Importované", 33 | "importing": "Importovanie {{ type }}...", 34 | "lastEdited": "Nap posledne editované", 35 | "or": "alebo", 36 | "title": "Názov", 37 | "type": "Typ", 38 | "typeUnknown": "Typ \"{{ type }}\" nie je platný.", 39 | "unsupportedFileType": "Typ súboru {{ type }} nie je podporovaný." 40 | } 41 | -------------------------------------------------------------------------------- /i18n/pt-BR.json: -------------------------------------------------------------------------------- 1 | { 2 | "dayjsTitleDateFormat": "MMM D, YYYY, hh:mm:ss A", 3 | "export": "Exportar {{ type }}", 4 | "exportAttachmentError": "Alguns anexos não puderam ser adicionados ao arquivo {{ format }}", 5 | "exportFailed": "Falha na exportação", 6 | "exportFileGenerationError": "A geração do arquivo {{ format }} falhou", 7 | "exportModalDescription": "Você selecionou {{ count }} {{ type }} para exportação", 8 | "exportModalDocumentFormat": "Formato do Documento", 9 | "exportModalIncludeChildren": "Incluir filhos desta página", 10 | "exportModalIncludeRelated": "Incluir documentos relacionados", 11 | "exportModalIncludeRelatedSettings": "Configurações de Documentos Relacionados", 12 | "exportModalNoRelatedTypes": "Nenhum Tipo Relacionado", 13 | "exportModalRelatedDocumentDescription": "Incluir os seguintes tipos de documentos onde aplicável", 14 | "exportModalSettingsLabel": "Configurações de Exportação", 15 | "exported": "Exportados {{ count }} {{ type }}", 16 | "exportedFailed": "Falha ao exportar {{ type }}", 17 | "exportedWithFailures": "Exportados {{ count }} {{ type }} ({{ bad }} de {{ total }} falharam)", 18 | "exporting": "Exportando {{ type }}...", 19 | "import": "Importar {{ type }}", 20 | "importCleanFailed": "A limpeza do arquivo importado no servidor falhou.", 21 | "importDuplicateContinue": "Continuar Importação", 22 | "importDuplicateDetected": "Duplicatas Detectadas.", 23 | "importDuplicateMessage": "Marque os itens que você deseja sobrescrever durante a importação.", 24 | "importFailed": "Falha na importação", 25 | "importFailedForSome": "{{ count }} documentos encontraram erro durante a importação", 26 | "importFileError": "A importação do arquivo {{ format }} falhou", 27 | "importModalDescription": "Importar conteúdo requer um arquivo {{ formats }}. Veja nossa documentação oficial.", 28 | "importSucceed": "Importação bem-sucedida!", 29 | "importWarning": "Importar um arquivo de exportação do Apostrophe em um site incompatível pode não funcionar como esperado.", 30 | "importWithCurrentLocaleDescription": "Este arquivo foi exportado da localidade \"{{ docsLocale }}\". Você tem certeza de que deseja importá-lo para a localidade \"{{ currentLocale }}\"?", 31 | "importWithCurrentLocaleHeading": "Localidade diferente detectada", 32 | "imported": "Importado", 33 | "importing": "Importando {{ type }}...", 34 | "lastEdited": "Última edição", 35 | "or": "ou", 36 | "title": "Título", 37 | "type": "Tipo", 38 | "typeUnknown": "Tipo \"{{ type }}\" não é válido.", 39 | "unsupportedFileType": "Não há suporte para o tipo de arquivo {{ type }}." 40 | } 41 | -------------------------------------------------------------------------------- /i18n/it.json: -------------------------------------------------------------------------------- 1 | { 2 | "dayjsTitleDateFormat": "MMM D, YYYY, hh:mm:ss A", 3 | "export": "Esporta {{ type }}", 4 | "exportAttachmentError": "Alcuni allegati non possono essere aggiunti al file {{ format }}", 5 | "exportFailed": "Esportazione non riuscita", 6 | "exportFileGenerationError": "La generazione del file {{ format }} è fallita", 7 | "exportModalDescription": "Hai selezionato {{ count }} {{ type }} da esportare", 8 | "exportModalDocumentFormat": "Formato documento", 9 | "exportModalIncludeChildren": "Includi i figli di questa pagina", 10 | "exportModalIncludeRelated": "Includi documenti correlati", 11 | "exportModalIncludeRelatedSettings": "Impostazioni documenti correlati", 12 | "exportModalNoRelatedTypes": "Nessun tipo correlato", 13 | "exportModalRelatedDocumentDescription": "Includi i seguenti tipi di documento dove applicabile", 14 | "exportModalSettingsLabel": "Impostazioni di esportazione", 15 | "exported": "Esportati {{ count }} {{ type }}", 16 | "exportedFailed": "Esportazione di {{ type }} non riuscita", 17 | "exportedWithFailures": "Esportati {{ count }} {{ type }} ({{ bad }} di {{ total }} non riusciti)", 18 | "exporting": "Esportazione {{ type }}...", 19 | "import": "Importa {{ type }}", 20 | "importCleanFailed": "La pulizia del file importato sul server è fallita.", 21 | "importDuplicateContinue": "Continua importazione", 22 | "importDuplicateDetected": "Duplicati rilevati.", 23 | "importDuplicateMessage": "Controlla gli elementi che desideri sovrascrivere durante l'importazione.", 24 | "importFailed": "Importazione non riuscita", 25 | "importFailedForSome": "{{ count }} documenti hanno riscontrato errori durante l'importazione", 26 | "importFileError": "L'importazione del file {{ format }} è fallita", 27 | "importModalDescription": "L'importazione di contenuti richiede un file {{ formats }}. Consulta la nostra documentazione ufficiale.", 28 | "importSucceed": "Importazione riuscita!", 29 | "importWarning": "Importare un file di esportazione di Apostrophe su un sito web incompatibile potrebbe non comportarsi come previsto.", 30 | "importWithCurrentLocaleDescription": "Questo file è stato esportato dalla locale \"{{ docsLocale }}\". Sei sicuro di volerlo importare nella locale \"{{ currentLocale }}\"?", 31 | "importWithCurrentLocaleHeading": "Locale diverso rilevato", 32 | "imported": "Importato", 33 | "importing": "Importazione {{ type }}...", 34 | "lastEdited": "Ultima modifica", 35 | "or": "o", 36 | "title": "Titolo", 37 | "type": "Tipo", 38 | "typeUnknown": "Tipo \"{{ type }}\" non valido.", 39 | "unsupportedFileType": "Il tipo di file {{tipo }} non è supportato." 40 | } 41 | -------------------------------------------------------------------------------- /i18n/fr.json: -------------------------------------------------------------------------------- 1 | { 2 | "dayjsTitleDateFormat": "MMM D, YYYY, hh:mm:ss A", 3 | "export": "Exporter {{ type }}", 4 | "exportAttachmentError": "Certaines pièces jointes n'ont pas pu être ajoutées au fichier {{ format }}", 5 | "exportFailed": "Échec de l'exportation", 6 | "exportFileGenerationError": "La génération du fichier {{ format }} a échoué", 7 | "exportModalDescription": "Vous avez sélectionné {{ count }} {{ type }} pour l'exportation", 8 | "exportModalDocumentFormat": "Format du document", 9 | "exportModalIncludeChildren": "Inclure les enfants de cette page", 10 | "exportModalIncludeRelated": "Inclure les documents associés", 11 | "exportModalIncludeRelatedSettings": "Paramètres des documents associés", 12 | "exportModalNoRelatedTypes": "Aucun type associé", 13 | "exportModalRelatedDocumentDescription": "Inclure les types de documents suivants le cas échéant", 14 | "exportModalSettingsLabel": "Paramètres d'exportation", 15 | "exported": "Exporté {{ count }} {{ type }}", 16 | "exportedFailed": "L'exportation de {{ type }} a échoué", 17 | "exportedWithFailures": "Exporté {{ count }} {{ type }} ({{ bad }} sur {{ total }} ont échoué)", 18 | "exporting": "Exportation de {{ type }}...", 19 | "import": "Importer {{ type }}", 20 | "importCleanFailed": "Le nettoyage du fichier importé sur le serveur a échoué.", 21 | "importDuplicateContinue": "Continuer l'importation", 22 | "importDuplicateDetected": "Doublons détectés.", 23 | "importDuplicateMessage": "Cochez les éléments que vous souhaitez écraser lors de l'importation.", 24 | "importFailed": "L'importation a échoué", 25 | "importFailedForSome": "{{ count }} documents ont rencontré une erreur lors de l'importation", 26 | "importFileError": "L'importation du fichier {{ format }} a échoué", 27 | "importModalDescription": "L'importation de contenu nécessite un fichier {{ formats }}. Consultez notre documentation officielle.", 28 | "importSucceed": "Importation réussie !", 29 | "importWarning": "L'importation d'un fichier d'exportation Apostrophe sur un site Web incompatible peut ne pas se dérouler comme prévu.", 30 | "importWithCurrentLocaleDescription": "Ce fichier a été exporté à partir de la locale \"{{ docsLocale }}\". Êtes-vous sûr de vouloir l'importer dans la locale \"{{ currentLocale }}\"?", 31 | "importWithCurrentLocaleHeading": "Locale différente détectée", 32 | "imported": "Importé", 33 | "importing": "Importation de {{ type }}...", 34 | "lastEdited": "Dernière modification", 35 | "or": "ou", 36 | "title": "Titre", 37 | "type": "Type", 38 | "typeUnknown": "Type \"{{ type }}\" n'est pas valide.", 39 | "unsupportedFileType": "Le type de fichier {{ type }} n'est pas supporté." 40 | } 41 | -------------------------------------------------------------------------------- /i18n/de.json: -------------------------------------------------------------------------------- 1 | { 2 | "dayjsTitleDateFormat": "MMM D, YYYY, hh:mm:ss A", 3 | "export": "Exportiere {{ type }}", 4 | "exportAttachmentError": "Einige Anhänge konnten nicht zur {{ format }}-Datei hinzugefügt werden", 5 | "exportFailed": "Export fehlgeschlagen", 6 | "exportFileGenerationError": "Die Erstellung der {{ format }}-Datei ist fehlgeschlagen", 7 | "exportModalDescription": "Sie haben {{ count }} {{ type }} zum Export ausgewählt", 8 | "exportModalDocumentFormat": "Dokumentformat", 9 | "exportModalIncludeChildren": "Kinderelemente dieser Seite einschließen", 10 | "exportModalIncludeRelated": "Verwandte Dokumente einbeziehen", 11 | "exportModalIncludeRelatedSettings": "Einstellungen für verwandte Dokumente", 12 | "exportModalNoRelatedTypes": "Keine verwandten Typen", 13 | "exportModalRelatedDocumentDescription": "Die folgenden Dokumenttypen bei Bedarf einbeziehen", 14 | "exportModalSettingsLabel": "Exporteinstellungen", 15 | "exported": "Exportiert {{ count }} {{ type }}", 16 | "exportedFailed": "Export von {{ type }} fehlgeschlagen", 17 | "exportedWithFailures": "Exportiert {{ count }} {{ type }} ({{ bad }} von {{ total }} fehlgeschlagen)", 18 | "exporting": "Exportiere {{ type }}...", 19 | "import": "Importiere {{ type }}", 20 | "importCleanFailed": "Die Bereinigung der importierten Datei auf dem Server ist fehlgeschlagen.", 21 | "importDuplicateContinue": "Import fortsetzen", 22 | "importDuplicateDetected": "Duplikate erkannt.", 23 | "importDuplicateMessage": "Überprüfen Sie die Elemente, die Sie während des Imports überschreiben möchten.", 24 | "importFailed": "Import fehlgeschlagen", 25 | "importFailedForSome": "{{ count }} Dokumente hatten während des Imports einen Fehler", 26 | "importFileError": "Der Import der {{ format }}-Datei ist fehlgeschlagen", 27 | "importModalDescription": "Das Importieren von Inhalten erfordert eine {{ formats }}-Datei. Siehe unsere offizielle Dokumentation.", 28 | "importSucceed": "Import erfolgreich!", 29 | "importWarning": "Das Importieren einer Apostrophe-Exportdatei auf einer inkompatiblen Website kann unvorhersehbar sein.", 30 | "importWithCurrentLocaleDescription": "Diese Datei wurde aus der \"{{ docsLocale }}\"-Sprache exportiert. Sind Sie sicher, dass Sie sie in die \"{{ currentLocale }}\"-Sprache importieren möchten?", 31 | "importWithCurrentLocaleHeading": "Andere Sprache erkannt", 32 | "imported": "Importiert", 33 | "importing": "Importiere {{ type }}...", 34 | "lastEdited": "Zuletzt bearbeitet", 35 | "or": "oder", 36 | "title": "Titel", 37 | "type": "Typ", 38 | "typeUnknown": "Typ \"{{ type }}\" ist ungültig.", 39 | "unsupportedFileType": "Der Dateityp {{ type }} wird nicht unterstützt." 40 | } 41 | -------------------------------------------------------------------------------- /i18n/es.json: -------------------------------------------------------------------------------- 1 | { 2 | "dayjsTitleDateFormat": "MMM D, YYYY, hh:mm:ss A", 3 | "export": "Exportar {{ type }}", 4 | "exportAttachmentError": "Algunos archivos adjuntos no se pudieron agregar al archivo {{ format }}", 5 | "exportFailed": "La exportación falló", 6 | "exportFileGenerationError": "La generación del archivo {{ format }} falló", 7 | "exportModalDescription": "Has seleccionado {{ count }} {{ type }} para exportar", 8 | "exportModalDocumentFormat": "Formato del Documento", 9 | "exportModalIncludeChildren": "Incluir hijos de esta página", 10 | "exportModalIncludeRelated": "Incluir documentos relacionados", 11 | "exportModalIncludeRelatedSettings": "Configuraciones de Documentos Relacionados", 12 | "exportModalNoRelatedTypes": "No hay Tipos Relacionados", 13 | "exportModalRelatedDocumentDescription": "Incluir los siguientes tipos de documentos donde sea aplicable", 14 | "exportModalSettingsLabel": "Configuraciones de Exportación", 15 | "exported": "Exportado {{ count }} {{ type }}", 16 | "exportedFailed": "Error al exportar {{ type }}", 17 | "exportedWithFailures": "Exportado {{ count }} {{ type }} ({{ bad }} de {{ total }} fallaron)", 18 | "exporting": "Exportando {{ type }}...", 19 | "import": "Importar {{ type }}", 20 | "importCleanFailed": "La limpieza del archivo importado en el servidor falló.", 21 | "importDuplicateContinue": "Continuar Importación", 22 | "importDuplicateDetected": "Duplicados Detectados.", 23 | "importDuplicateMessage": "Marca los elementos que te gustaría sobrescribir durante la importación.", 24 | "importFailed": "La importación falló", 25 | "importFailedForSome": "{{ count }} documentos encontraron errores durante la importación", 26 | "importFileError": "La importación del archivo {{ format }} falló", 27 | "importModalDescription": "Importar contenido requiere un archivo {{ formats }}. Consulta nuestra documentación oficial.", 28 | "importSucceed": "Importación Exitosa!", 29 | "importWarning": "Importar un archivo de exportación de Apostrophe en un sitio web incompatible puede no comportarse como se esperaba.", 30 | "importWithCurrentLocaleDescription": "Este archivo fue exportado desde la configuración regional \"{{ docsLocale }}\". ¿Estás seguro de que deseas importarlo a la configuración regional \"{{ currentLocale }}\"?", 31 | "importWithCurrentLocaleHeading": "Se detectó una configuración regional diferente", 32 | "imported": "Importado", 33 | "importing": "Importando {{ type }}...", 34 | "lastEdited": "Última edición", 35 | "or": "o", 36 | "title": "Título", 37 | "type": "Tipo", 38 | "typeUnknown": "Tipo \"{{ type }}\" no es válido.", 39 | "unsupportedFileType": "El tipo de archivo {{ type }} no es compatible." 40 | } 41 | -------------------------------------------------------------------------------- /modules/@apostrophecms/import-export-page/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | improve: '@apostrophecms/page', 3 | 4 | utilityOperations (self) { 5 | if (self.options.importExport?.import === false) { 6 | return {}; 7 | } 8 | 9 | return { 10 | add: { 11 | 'import-export-import': { 12 | label: 'aposImportExport:import', 13 | modalOptions: { 14 | modal: 'AposImportModal' 15 | }, 16 | canCreate: true, 17 | canEdit: true 18 | } 19 | } 20 | }; 21 | }, 22 | batchOperations(self) { 23 | if (self.options.importExport?.export === false) { 24 | return {}; 25 | } 26 | 27 | return { 28 | add: { 29 | 'import-export-export-batch': { 30 | label: 'aposImportExport:export', 31 | messages: { 32 | progress: 'aposImportExport:exporting', 33 | completed: 'aposImportExport:exported', 34 | completedWithFailures: 'aposImportExport:exportedWithFailures', 35 | failed: 'aposImportExport:exportedFailed', 36 | icon: 'database-export-icon', 37 | resultsEventName: 'import-export-export-download' 38 | }, 39 | modal: 'AposExportModal' 40 | } 41 | }, 42 | group: { 43 | more: { 44 | icon: 'dots-vertical-icon', 45 | operations: [ 'import-export-export-batch' ] 46 | } 47 | } 48 | }; 49 | }, 50 | apiRoutes(self) { 51 | return { 52 | post: { 53 | ...self.options.importExport?.import !== false && { 54 | importExportImport: [ 55 | self.apos.http.bigUploadMiddleware(), 56 | async (req) => { 57 | return self.apos.modules['@apostrophecms/import-export'] 58 | .import(req, self.__meta.name); 59 | } 60 | ] 61 | }, 62 | ...self.options.importExport?.export !== false && { 63 | importExportExport(req) { 64 | // Add the page label to req.body for notifications. 65 | req.body.type = req.t('apostrophe:page'); 66 | 67 | return self.apos.modules['@apostrophecms/import-export'] 68 | .export(req, self); 69 | } 70 | }, 71 | importExportExportBatch(req) { 72 | // Add the pages label to req.body for notifications. 73 | // Should be done before calling the job's `run` method. 74 | req.body.type = req.t('apostrophe:pages'); 75 | 76 | return self.apos.modules['@apostrophecms/job'].run( 77 | req, 78 | (req, reporting) => self.apos.modules['@apostrophecms/import-export'] 79 | .export(req, self, reporting), 80 | { 81 | action: 'export', 82 | ids: req.body._ids, 83 | docTypes: [] 84 | } 85 | ); 86 | } 87 | } 88 | }; 89 | } 90 | }; 91 | -------------------------------------------------------------------------------- /i18n/en.json: -------------------------------------------------------------------------------- 1 | { 2 | "dayjsTitleDateFormat": "MMM D, YYYY, hh:mm:ss A", 3 | "errorCantImportType": "Can't import type \"{{ type }}\"", 4 | "errorInsertingDocument": "Inserting/updating failed with message: {{ message }}", 5 | "export": "Export {{ type }}", 6 | "exportAttachmentError": "Some attachments could not be added to the {{ format }} file", 7 | "exportFailed": "Export failed", 8 | "exportFileGenerationError": "The {{ format }} file generation failed", 9 | "exportModalDescription": "You've selected {{ count }} {{ type }} for export", 10 | "exportModalDocumentFormat": "Document Format", 11 | "exportModalIncludeChildren": "Include children of this page", 12 | "exportModalIncludeRelated": "Include related documents", 13 | "exportModalIncludeRelatedSettings": "Related Documents Settings", 14 | "exportModalNoRelatedTypes": "No Related Types", 15 | "exportModalRelatedDocumentDescription": "Include the following document types where applicable", 16 | "exportModalSettingsLabel": "Export Settings", 17 | "exportModalToggleAllRelated": "Toggle all", 18 | "exported": "Exported {{ count }} {{ type }}", 19 | "exportedFailed": "Exporting {{ type }} failed", 20 | "exportedWithFailures": "Exported {{ count }} {{ type }} ({{ bad }} of {{ total }} failed)", 21 | "exporting": "Exporting {{ type }}...", 22 | "import": "Import {{ type }}", 23 | "importCleanFailed": "The cleaning of the imported file on the server failed.", 24 | "importDraftsOnly": "Import all documents as drafts", 25 | "importDraftsOnlyTooltip": "Content types that do not have separate drafts will be imported normally.", 26 | "importDuplicateContinue": "Continue Import", 27 | "importDuplicateDetected": "Duplicates Detected.", 28 | "importDuplicateMessage": "Check the items you'd like to overwrite during import.", 29 | "importFailed": "Import failed", 30 | "importFailedForSome": "{{ count }} documents encountered error during import", 31 | "importFileError": "The {{ type }} file import failed", 32 | "importModalDescription": "Importing content requires a {{ formats }} file. See our official documentation.", 33 | "importSucceed": "Import Succeeded!", 34 | "importTranslate": "Translate text content with AI", 35 | "importTranslateTooltip": "Translate the text content with AI while importing. Only available with .gz import files.", 36 | "importWarning": "Importing an Apostrophe export file on an incompatible website may not behave as expected.", 37 | "importWithCurrentLocaleDescription": "This file was exported from the \"{{ docsLocale }}\" locale. Are you sure you want to import it into the \"{{ currentLocale }}\" locale?", 38 | "importWithCurrentLocaleHeading": "Different locale detected", 39 | "imported": "Imported", 40 | "importing": "Importing {{ type }}...", 41 | "lastEdited": "Last edited", 42 | "mode": "Mode", 43 | "or": "or", 44 | "title": "Title", 45 | "type": "Type", 46 | "typeUnknown": "Type \"{{ type }}\" is not valid.", 47 | "unsupportedFileType": "File type {{ type }} is not supported." 48 | } 49 | -------------------------------------------------------------------------------- /lib/methods/import-page.js: -------------------------------------------------------------------------------- 1 | const getTargetId = async ({ 2 | manager, 3 | doc, 4 | req, 5 | duplicatedDocs = [] 6 | }) => { 7 | if (doc.archived) { 8 | return '_archive'; 9 | } 10 | 11 | const duplicatedDocsMapping = Object.fromEntries( 12 | duplicatedDocs.map(duplicate => [ duplicate.aposDocId, duplicate.replaceId ]) 13 | ); 14 | 15 | const { path = '' } = doc; 16 | const ancestorIds = path.split('/').reverse().slice(1); 17 | for (const ancestorId of ancestorIds) { 18 | try { 19 | const { aposDocId } = await manager.getTarget( 20 | req, 21 | duplicatedDocsMapping[ancestorId] || ancestorId 22 | ); 23 | return aposDocId; 24 | } catch (error) { 25 | // continue search 26 | } 27 | } 28 | 29 | // If no target is found, try by slug. Remove the last segment of the path 30 | // and search for the slug in the database. 31 | const parentSlug = doc.slug?.split('/').slice(0, -1).join('/'); 32 | const aposDocId = await getTargetIdBySlug( 33 | req, 34 | { 35 | manager, 36 | slug: parentSlug, 37 | mode: doc.aposMode 38 | } 39 | ); 40 | if (aposDocId) { 41 | return aposDocId; 42 | } 43 | 44 | return '_home'; 45 | }; 46 | 47 | async function getTargetIdBySlug(req, { 48 | manager, 49 | slug, 50 | mode 51 | }) { 52 | if (!slug || slug === '/') { 53 | return null; 54 | } 55 | 56 | const criteria = { 57 | slug, 58 | aposLocale: `${req.locale}:${mode}` 59 | }; 60 | const target = await manager.find(req, criteria) 61 | .project({ 62 | _id: 1, 63 | aposDocId: 1 64 | }) 65 | .permission(false) 66 | .archived(null) 67 | .areas(false) 68 | .toObject(); 69 | 70 | if (!target) { 71 | return null; 72 | } 73 | 74 | try { 75 | const { aposDocId } = await manager.getTarget( 76 | req, 77 | target.aposDocId 78 | ); 79 | return aposDocId; 80 | } catch (error) { 81 | // ignore 82 | } 83 | 84 | return null; 85 | } 86 | 87 | const insert = async ({ 88 | manager, 89 | doc, 90 | req, 91 | duplicatedDocs, 92 | modified 93 | }) => { 94 | const targetId = await getTargetId({ 95 | manager, 96 | doc, 97 | req, 98 | duplicatedDocs 99 | }); 100 | const position = 'lastChild'; 101 | 102 | return manager.insert( 103 | req, 104 | targetId, 105 | position, 106 | doc, 107 | { setModified: modified } 108 | ); 109 | }; 110 | 111 | const update = async ({ 112 | manager, 113 | doc, 114 | req, 115 | duplicatedDocs 116 | }) => { 117 | 118 | const { 119 | _id, 120 | aposDocId, 121 | path, 122 | rank, 123 | level, 124 | ...patch 125 | } = doc; 126 | 127 | const move = doc.parkedId 128 | ? {} 129 | : { 130 | _targetId: await getTargetId({ 131 | manager, 132 | doc, 133 | req, 134 | duplicatedDocs 135 | }), 136 | _position: 'lastChild' 137 | }; 138 | 139 | return manager.patch( 140 | req.clone({ 141 | body: { 142 | ...patch, 143 | ...move 144 | } 145 | }), 146 | _id, 147 | { 148 | fetchRelationships: false 149 | } 150 | ); 151 | }; 152 | 153 | module.exports = { 154 | insert, 155 | update 156 | }; 157 | -------------------------------------------------------------------------------- /modules/@apostrophecms/import-export-piece-type/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | improve: '@apostrophecms/piece-type', 3 | cascades: [ 'batchOperations' ], 4 | utilityOperations (self) { 5 | if (self.options.importExport?.import === false) { 6 | return {}; 7 | } 8 | 9 | return { 10 | add: { 11 | 'import-export-import': { 12 | label: 'aposImportExport:import', 13 | modalOptions: { 14 | modal: 'AposImportModal' 15 | }, 16 | canCreate: true, 17 | canEdit: true 18 | } 19 | } 20 | }; 21 | }, 22 | batchOperations(self) { 23 | if (self.options.importExport?.export === false) { 24 | return; 25 | } 26 | 27 | return { 28 | add: { 29 | 'import-export-export-batch': { 30 | label: 'aposImportExport:export', 31 | messages: { 32 | progress: 'aposImportExport:exporting', 33 | completed: 'aposImportExport:exported', 34 | completedWithFailures: 'aposImportExport:exportedWithFailures', 35 | failed: 'aposImportExport:exportedFailed', 36 | icon: 'database-export-icon', 37 | resultsEventName: 'import-export-export-download' 38 | }, 39 | modal: 'AposExportModal' 40 | } 41 | }, 42 | group: { 43 | more: { 44 | icon: 'dots-vertical-icon', 45 | operations: [ 'import-export-export-batch' ] 46 | } 47 | } 48 | }; 49 | }, 50 | apiRoutes(self) { 51 | return { 52 | post: { 53 | ...self.options.importExport?.import !== false && { 54 | importExportImport: [ 55 | self.apos.http.bigUploadMiddleware(), 56 | async (req) => { 57 | return self.apos.modules['@apostrophecms/import-export'] 58 | .import(req, self.__meta.name); 59 | } 60 | ] 61 | }, 62 | 63 | ...self.options.importExport?.export !== false && { 64 | importExportExport(req) { 65 | // Add the piece type label to req.body for notifications. 66 | req.body.type = req.t(self.options.label); 67 | 68 | return self.apos.modules['@apostrophecms/import-export'].export(req, self); 69 | }, 70 | // NOTE: this route is used in batch operations, and its method should be POST 71 | // in order to make the job work with the progress notification. 72 | // The other 'export' routes that are used by context operations on each doc 73 | // are also POST for consistency. 74 | importExportExportBatch(req) { 75 | // Add the piece type label to req.body for notifications. 76 | // Should be done before calling the job's `run` method. 77 | req.body.type = req.body._ids.length === 1 78 | ? req.t(self.options.label) 79 | : req.t(self.options.pluralLabel); 80 | 81 | return self.apos.modules['@apostrophecms/job'].run( 82 | req, 83 | (req, reporting) => self.apos.modules['@apostrophecms/import-export'] 84 | .export(req, self, reporting), 85 | { 86 | action: 'export', 87 | ids: req.body._ids, 88 | docTypes: [] 89 | } 90 | ); 91 | } 92 | } 93 | } 94 | }; 95 | } 96 | }; 97 | -------------------------------------------------------------------------------- /lib/methods/index.js: -------------------------------------------------------------------------------- 1 | const fsp = require('node:fs/promises'); 2 | const importMethods = require('./import.js'); 3 | const exportMethods = require('./export.js'); 4 | 5 | module.exports = self => { 6 | return { 7 | registerFormats(formats = {}) { 8 | verifyFormats(formats); 9 | 10 | self.formats = { 11 | ...self.formats, 12 | ...formats 13 | }; 14 | }, 15 | // No need to override, the parent method returns `{}`. 16 | getBrowserData() { 17 | return { 18 | formats: Object 19 | .entries(self.formats) 20 | .map(([ key, value ]) => ({ 21 | name: key, 22 | label: value.label, 23 | allowedExtension: value.allowedExtension 24 | })), 25 | importDraftsOnlyDefault: self.options.importDraftsOnlyDefault 26 | }; 27 | }, 28 | // Filter our docs that have their module with the import or export option set to 29 | // false and docs that have "admin only" permissions when the user is not an admin. 30 | // If a user does not have at lease the permission to view the draft, he won't 31 | // be able to import or export it. 32 | canImportOrExport(req, docType, action) { 33 | const docModule = self.apos.modules[docType]; 34 | if (!docModule) { 35 | return false; 36 | } 37 | if (docModule.options.importExport?.[action] === false) { 38 | return false; 39 | } 40 | // TODO: Do we need to keep that since done by managers? 41 | if (!self.apos.permission.can(req, 'view', docType)) { 42 | return false; 43 | } 44 | 45 | return true; 46 | }, 47 | 48 | async remove(filepath) { 49 | try { 50 | const stat = await fsp.lstat(filepath); 51 | if (stat.isDirectory()) { 52 | await fsp.rm(filepath, { 53 | recursive: true, 54 | force: true 55 | }); 56 | } else { 57 | await fsp.unlink(filepath); 58 | } 59 | console.info(`removed: ${filepath}`); 60 | } catch (err) { 61 | console.trace(err); 62 | self.apos.util.error( 63 | `Error while trying to remove the file or folder: ${filepath}. You might want to remove it yourself.` 64 | ); 65 | } 66 | }, 67 | 68 | ...importMethods(self), 69 | ...exportMethods(self) 70 | }; 71 | }; 72 | 73 | function verifyFormats(formats) { 74 | if (typeof formats !== 'object') { 75 | throw new Error('formats must be an object'); 76 | } 77 | 78 | Object 79 | .entries(formats) 80 | .forEach(([ formatName, format ]) => { 81 | const requiredKeys = [ 'label', 'extension', 'allowedExtension', 'allowedTypes', 'input', 'output' ]; 82 | const allowedKeys = [ ...requiredKeys, 'includeAttachments' ]; 83 | 84 | const keys = Object.keys(format); 85 | 86 | if (requiredKeys.some(requiredKey => !keys.includes(requiredKey))) { 87 | throw new Error(`${formatName}.label, ${formatName}.extension, ${formatName}.allowedExtension, ${formatName}.allowedTypes, ${formatName}.input and ${formatName}.output are required keys`); 88 | } 89 | keys.forEach(key => { 90 | if (!allowedKeys.includes(key)) { 91 | throw new Error(`${formatName}.${key} is not a valid key`); 92 | } 93 | }); 94 | keys.forEach(key => { 95 | if (key === 'label') { 96 | if (typeof format[key] !== 'string') { 97 | throw new Error(`${formatName}.${key} must be a string`); 98 | } 99 | } else if (key === 'extension') { 100 | if (typeof format[key] !== 'string') { 101 | throw new Error(`${formatName}.${key} must be a string`); 102 | } 103 | } else if (key === 'allowedExtension') { 104 | if (typeof format[key] !== 'string') { 105 | throw new Error(`${formatName}.${key} must be a string`); 106 | } 107 | } else if (key === 'allowedTypes') { 108 | if (!Array.isArray(format[key])) { 109 | throw new Error(`${formatName}.${key} must be an array`); 110 | } 111 | format[key].forEach(allowedType => { 112 | if (typeof allowedType !== 'string') { 113 | throw new Error(`${formatName}.${key} must be an array of strings`); 114 | } 115 | }); 116 | } else if (key === 'includeAttachments') { 117 | if (typeof format[key] !== 'boolean') { 118 | throw new Error(`${formatName}.${key} must be a boolean`); 119 | } 120 | } else if (key === 'input') { 121 | if (typeof format[key] !== 'function') { 122 | throw new Error(`${formatName}.${key} must be a function`); 123 | } 124 | } else if (key === 'output') { 125 | if (typeof format[key] !== 'function') { 126 | throw new Error(`${formatName}.${key} must be a function`); 127 | } 128 | } 129 | }); 130 | }); 131 | }; 132 | -------------------------------------------------------------------------------- /ui/apos/apps/index.js: -------------------------------------------------------------------------------- 1 | export default () => { 2 | let ready = false; 3 | 4 | apos.util.onReady(() => { 5 | if (!ready) { 6 | ready = true; 7 | apos.bus.$on('import-export-export-download', openUrl); 8 | apos.bus.$on('import-export-import-started', addBeforeUnloadListener); 9 | apos.bus.$on('import-export-import-ended', removeBeforeUnloadListenerAndReport); 10 | apos.bus.$on('import-export-import-locale-differs', handleDifferentLocale); 11 | apos.bus.$on('import-export-import-duplicates', handleDuplicates); 12 | } 13 | }); 14 | 15 | function openUrl(event) { 16 | if (event.url) { 17 | window.open(event.url, '_blank'); 18 | } 19 | } 20 | 21 | function addBeforeUnloadListener() { 22 | window.addEventListener('beforeunload', warningImport); 23 | } 24 | 25 | function removeBeforeUnloadListenerAndReport(event) { 26 | window.removeEventListener('beforeunload', warningImport); 27 | showReportModal(event); 28 | } 29 | 30 | async function showReportModal(event) { 31 | if (!event?.failedLog?.length) { 32 | return; 33 | } 34 | 35 | const locales = Object.entries(window.apos.i18n.locales).map( 36 | ([ locale, options ]) => { 37 | return { 38 | name: locale, 39 | label: options.label || locale 40 | }; 41 | } 42 | ); 43 | 44 | const items = event.failedLog.map((log) => { 45 | const locale = locales 46 | .find( 47 | (locale) => locale.name === log.aposLocale?.split(':')[0] 48 | )?.label || (log.aposLocale ? 'n/a' : '-'); 49 | const mode = log._id.split(':')[2]; 50 | return { 51 | ...log, 52 | localeLabel: locale, 53 | title: log.title || '-', 54 | typeLabel: typeLabel(log.type), 55 | aposModeLabel: mode ?? '-' 56 | }; 57 | }); 58 | 59 | await apos.report( 60 | { 61 | heading: 'aposImportExport:importFailedForSome', 62 | footerMessageDanger: 'aposImportExport:importFailedForSome', 63 | items, 64 | headers: [ 65 | { 66 | name: '_id', 67 | label: '_id', 68 | visibility: 'export' 69 | }, 70 | { 71 | name: 'aposDocId', 72 | label: '_id', 73 | format: 'last:5', 74 | visibility: 'table' 75 | }, 76 | { 77 | name: 'aposModeLabel', 78 | label: 'aposImportExport:mode', 79 | visibility: 'table' 80 | }, 81 | { 82 | name: 'typeLabel', 83 | label: 'apostrophe:type', 84 | sortable: true, 85 | translate: true 86 | }, 87 | { 88 | name: 'type', 89 | label: 'type', 90 | visibility: 'export' 91 | }, 92 | { 93 | name: 'localeLabel', 94 | label: 'apostrophe:locale', 95 | sortable: true 96 | }, 97 | { 98 | name: 'title', 99 | label: 'apostrophe:title', 100 | width: '20%', 101 | sortable: true 102 | }, 103 | { 104 | name: 'detail', 105 | label: 'apostrophe:details', 106 | width: '20%' 107 | } 108 | ] 109 | }, 110 | { 111 | // We don't need the `log.type` property for now (it's our doc.type). 112 | mode: 'all' 113 | } 114 | ); 115 | } 116 | 117 | async function handleDifferentLocale(event) { 118 | // Do not ask for confirmation if the editor already 119 | // opted-in for translation. 120 | const continueImport = event.translate 121 | ? true 122 | : await apos.modal.execute('AposModalConfirm', event); 123 | 124 | if (continueImport) { 125 | try { 126 | const moduleAction = apos.modules[event.moduleName].action; 127 | 128 | await apos.http.post(`${moduleAction}/import-export-import`, { 129 | body: { 130 | importDraftsOnly: event.importDraftsOnly, 131 | translate: event.translate, 132 | overrideLocale: true, 133 | exportId: event.exportId, 134 | formatLabel: event.formatLabel 135 | } 136 | }); 137 | } catch (error) { 138 | apos.notify('aposImportExport:importFailed', { 139 | type: 'danger', 140 | dismiss: true 141 | }); 142 | } 143 | 144 | return; 145 | } 146 | 147 | // If not, we still need to clean the uploaded archive 148 | try { 149 | await apos.http.post('/api/v1/@apostrophecms/import-export/clean-export', { 150 | body: { 151 | exportId: event.exportId 152 | } 153 | }); 154 | } catch (error) { 155 | apos.notify('aposImportExport:importCleanFailed', { 156 | type: 'warning' 157 | }); 158 | } 159 | } 160 | 161 | async function handleDuplicates(event) { 162 | if (event.duplicatedDocs.length) { 163 | await apos.modal.execute('AposDuplicateImportModal', event); 164 | } 165 | } 166 | 167 | function warningImport(event) { 168 | event.preventDefault(); 169 | event.returnValue = ''; 170 | } 171 | 172 | // Convert doc.type into a human readable label. 173 | function typeLabel(name) { 174 | const module = apos.modules[name] || {}; 175 | if (module.action === '@apostrophecms/page') { 176 | return 'apostrophe:page'; 177 | } 178 | return module.label || name; 179 | } 180 | }; 181 | -------------------------------------------------------------------------------- /lib/formats/gzip.js: -------------------------------------------------------------------------------- 1 | const path = require('node:path'); 2 | const fs = require('node:fs'); 3 | const fsp = require('node:fs/promises'); 4 | const stream = require('node:stream/promises'); 5 | const zlib = require('node:zlib'); 6 | const tar = require('tar-stream'); 7 | const { EJSON } = require('bson'); 8 | 9 | module.exports = { 10 | label: 'gzip', 11 | extension: '.tar.gz', 12 | allowedExtension: '.gz', 13 | allowedTypes: [ 14 | 'application/gzip', 15 | 'application/x-gzip' 16 | ], 17 | includeAttachments: true, 18 | async input(filepath) { 19 | let exportPath = filepath; 20 | 21 | // If the given path is actually the archive, we first need to extract it. 22 | // Then we no longer need the archive file, so we remove it. 23 | if (filepath.endsWith(this.allowedExtension)) { 24 | exportPath = filepath.replace(this.allowedExtension, ''); 25 | 26 | console.info(`[gzip] extracting ${filepath} into ${exportPath}`); 27 | await extract(filepath, exportPath); 28 | 29 | console.info(`[gzip] removing ${filepath}`); 30 | await remove(filepath); 31 | } 32 | 33 | const docsPath = path.join(exportPath, 'aposDocs.json'); 34 | const attachmentsPath = path.join(exportPath, 'aposAttachments.json'); 35 | const attachmentFilesPath = path.join(exportPath, 'attachments'); 36 | 37 | console.info(`[gzip] reading docs from ${docsPath}`); 38 | console.info(`[gzip] reading attachments from ${attachmentsPath}`); 39 | console.info(`[gzip] reading attachment files from ${attachmentFilesPath}`); 40 | 41 | const docs = await fsp.readFile(docsPath); 42 | const attachments = await fsp.readFile(attachmentsPath); 43 | 44 | const parsedDocs = EJSON.parse(docs); 45 | const parsedAttachments = EJSON.parse(attachments); 46 | 47 | const attachmentsInfo = parsedAttachments.map(attachment => ({ 48 | attachment, 49 | file: { 50 | name: `${attachment.name}.${attachment.extension}`, 51 | path: path.join(attachmentFilesPath, `${attachment._id}-${attachment.name}.${attachment.extension}`) 52 | } 53 | })); 54 | 55 | return { 56 | docs: parsedDocs, 57 | attachmentsInfo, 58 | exportPath 59 | }; 60 | }, 61 | async output( 62 | filepath, 63 | { 64 | docs, 65 | attachments = [], 66 | attachmentUrls = {} 67 | }, 68 | processAttachments 69 | ) { 70 | const data = { 71 | json: { 72 | 'aposDocs.json': EJSON.stringify(docs, undefined, 2), 73 | 'aposAttachments.json': EJSON.stringify(attachments, undefined, 2) 74 | }, 75 | attachments: attachmentUrls 76 | }; 77 | 78 | const writeStream = fs.createWriteStream(filepath); 79 | const pack = tar.pack(); 80 | const gzip = zlib.createGzip(); 81 | 82 | pack 83 | .pipe(gzip) 84 | .pipe(writeStream); 85 | 86 | let result; 87 | 88 | return new Promise((resolve, reject) => { 89 | writeStream.on('error', reject); 90 | gzip.on('error', reject); 91 | pack.on('error', reject); 92 | 93 | writeStream.on('finish', () => { 94 | console.info(`[gzip] export file written to ${filepath}`); 95 | resolve(result); 96 | }); 97 | 98 | for (const [ filename, content ] of Object.entries(data.json || {})) { 99 | console.info(`[gzip] adding ${filename} to the tarball`); 100 | addTarEntry(pack, { name: filename }, content).catch(reject); 101 | } 102 | 103 | addTarEntry(pack, { 104 | name: 'attachments/', 105 | type: 'directory' 106 | }) 107 | .then(() => { 108 | processAttachments(data.attachments, async (attachmentPath, name, size) => { 109 | console.info(`[gzip] adding attachments/${name} to the tarball`); 110 | 111 | const readStream = fs.createReadStream(attachmentPath); 112 | const entryStream = pack.entry({ 113 | name: `attachments/${name}`, 114 | size 115 | }); 116 | 117 | await stream.pipeline([ readStream, entryStream ]); 118 | }) 119 | .then((res) => { 120 | result = res; 121 | pack.finalize(); 122 | }); 123 | }) 124 | .catch(reject); 125 | }); 126 | } 127 | }; 128 | 129 | async function extract(filepath, exportPath) { 130 | if (fs.existsSync(exportPath)) { 131 | return; 132 | } 133 | 134 | await fsp.mkdir(exportPath); 135 | 136 | const readStream = fs.createReadStream(filepath); 137 | const gunzip = zlib.createGunzip(); 138 | const extract = tar.extract(); 139 | 140 | readStream 141 | .pipe(gunzip) 142 | .pipe(extract); 143 | 144 | return new Promise((resolve, reject) => { 145 | readStream.on('error', reject); 146 | gunzip.on('error', reject); 147 | extract.on('error', reject); 148 | 149 | extract.on('entry', (header, stream, next) => { 150 | if (header.type === 'directory') { 151 | fsp 152 | .mkdir(path.join(exportPath, header.name)) 153 | .then(next) 154 | .catch(reject); 155 | } else { 156 | stream.pipe(fs.WriteStream(path.join(exportPath, header.name))); 157 | stream.on('end', next); 158 | } 159 | }); 160 | extract.on('finish', resolve); 161 | }); 162 | } 163 | 164 | // This independent function is designed for file removal. 165 | // Avoid invoking `self.remove` within this script, 166 | // as it should remain separate from the apos context. 167 | async function remove(filepath) { 168 | try { 169 | await fsp.unlink(filepath); 170 | } catch (error) { 171 | console.error(error); 172 | } 173 | } 174 | 175 | function addTarEntry(pack, options, data = null) { 176 | return new Promise((resolve, reject) => { 177 | pack.entry(options, data, error => { 178 | if (error) { 179 | reject(error); 180 | return; 181 | } 182 | 183 | resolve(); 184 | }); 185 | }); 186 | } 187 | -------------------------------------------------------------------------------- /test/util/index.js: -------------------------------------------------------------------------------- 1 | const assert = require('node:assert'); 2 | const path = require('node:path'); 3 | const fs = require('node:fs/promises'); 4 | const { createReadStream } = require('node:fs'); 5 | const FormData = require('form-data'); 6 | const { output: gzip } = require('../../lib/formats/gzip.js'); 7 | 8 | module.exports = { 9 | getAppConfig, 10 | extractFileNames, 11 | getExtractedFiles, 12 | buildFixtures, 13 | copyFixtures, 14 | cleanFixtures, 15 | cleanData, 16 | deletePiecesAndPages, 17 | deleteAttachments, 18 | insertPiecesAndPages, 19 | login, 20 | insertAdminUser 21 | }; 22 | 23 | function getAppConfig(modules = {}, options = {}) { 24 | return { 25 | '@apostrophecms/express': { 26 | options: { 27 | session: { secret: 'supersecret' } 28 | } 29 | }, 30 | '@apostrophecms/import-export': {}, 31 | '@apostrophecms/uploadfs': { 32 | options: { 33 | storage: 'local' 34 | } 35 | }, 36 | 'home-page': { 37 | extend: '@apostrophecms/page-type' 38 | }, 39 | 'default-page': { 40 | extend: '@apostrophecms/page-type', 41 | fields: { 42 | add: { 43 | main: { 44 | label: 'Main', 45 | type: 'area', 46 | options: { 47 | widgets: { 48 | '@apostrophecms/rich-text': {}, 49 | '@apostrophecms/image': {}, 50 | '@apostrophecms/video': {} 51 | }, 52 | importAsRichText: true 53 | } 54 | }, 55 | _articles: { 56 | label: 'Articles', 57 | type: 'relationship', 58 | withType: 'article' 59 | } 60 | } 61 | } 62 | }, 63 | nonLocalized: { 64 | extend: '@apostrophecms/piece-type', 65 | options: { 66 | alias: 'nonLocalized', 67 | localized: false 68 | } 69 | }, 70 | article: { 71 | extend: '@apostrophecms/piece-type', 72 | options: { 73 | alias: 'article', 74 | autopublish: options.autopublish ?? true 75 | }, 76 | fields: { 77 | add: { 78 | image: { 79 | type: 'attachment', 80 | group: 'images' 81 | }, 82 | _topics: { 83 | label: 'Topics', 84 | type: 'relationship', 85 | withType: 'topic', 86 | builders: { 87 | project: { 88 | title: 1, 89 | image: 1, 90 | main: 1, 91 | aposMode: 1 92 | } 93 | } 94 | }, 95 | main: { 96 | label: '', 97 | type: 'area', 98 | options: { 99 | widgets: { 100 | '@apostrophecms/image': {} 101 | } 102 | } 103 | } 104 | } 105 | } 106 | }, 107 | 108 | topic: { 109 | extend: '@apostrophecms/piece-type', 110 | options: { 111 | alias: 'topic' 112 | }, 113 | fields: { 114 | add: { 115 | description: { 116 | label: 'Description', 117 | type: 'string' 118 | }, 119 | main: { 120 | label: 'Main', 121 | type: 'area', 122 | options: { 123 | widgets: { 124 | '@apostrophecms/rich-text': {} 125 | }, 126 | importAsRichText: true 127 | } 128 | }, 129 | _topics: { 130 | label: 'Related Topics', 131 | type: 'relationship', 132 | withType: 'topic', 133 | max: 2 134 | } 135 | } 136 | } 137 | }, 138 | 139 | ...modules 140 | }; 141 | } 142 | 143 | function extractFileNames (files) { 144 | return files.map((fullname) => { 145 | const regex = /-([\w\d-]+)\./; 146 | const [ , name ] = regex.exec(fullname); 147 | 148 | return name; 149 | }); 150 | } 151 | 152 | async function getExtractedFiles(extractPath) { 153 | const docsData = await fs.readFile( 154 | path.join(extractPath, 'aposDocs.json'), 155 | { encoding: 'utf8' } 156 | ); 157 | 158 | const attachmentsData = await fs.readFile( 159 | path.join(extractPath, 'aposAttachments.json'), 160 | { encoding: 'utf8' } 161 | ); 162 | 163 | const attachmentFiles = await fs.readdir(path.join(extractPath, 'attachments')); 164 | 165 | return { 166 | docs: JSON.parse(docsData), 167 | attachments: JSON.parse(attachmentsData), 168 | attachmentFiles 169 | }; 170 | } 171 | 172 | async function buildFixtures(apos) { 173 | const target = path.join(apos.rootDir, 'data/tmp/uploads'); 174 | const directories = (await fs.readdir(path.join(target, 'gzip'), { withFileTypes: true })) 175 | .filter(entry => entry.isDirectory()); 176 | for (const directory of directories) { 177 | const { default: docs } = await import( 178 | path.join(directory.parentPath, directory.name, 'aposDocs.json'), 179 | { with: { type: 'json' } } 180 | ); 181 | const { default: attachments } = await import( 182 | path.join(directory.parentPath, directory.name, 'aposAttachments.json'), 183 | { with: { type: 'json' } } 184 | ); 185 | await gzip( 186 | path.join(target, `${directory.name}.tar.gz`), 187 | { 188 | docs, 189 | attachments 190 | }, 191 | async () => {} 192 | ); 193 | } 194 | } 195 | 196 | async function copyFixtures(apos) { 197 | const source = path.join(apos.rootDir, 'fixtures'); 198 | const target = path.join(apos.rootDir, 'data/tmp/uploads'); 199 | 200 | await fs.cp(source, target, { recursive: true }); 201 | } 202 | 203 | async function cleanFixtures(apos) { 204 | const target = path.join(apos.rootDir, 'data/tmp/uploads'); 205 | 206 | await cleanData([ target ]); 207 | } 208 | 209 | async function cleanData(paths) { 210 | for (const filePath of paths) { 211 | try { 212 | const files = await fs.readdir(filePath); 213 | for (const name of files) { 214 | await fs.rm(path.join(filePath, name), { recursive: true }); 215 | } 216 | } catch (error) { 217 | console.error(error); // eslint-disable-line no-console 218 | } 219 | } 220 | } 221 | 222 | async function deletePiecesAndPages(apos) { 223 | await apos.doc.db.deleteMany({ type: /default-page|article|topic|@apostrophecms\/image/ }); 224 | } 225 | 226 | async function deleteAttachments(apos, attachmentPath) { 227 | await apos.attachment.db.deleteMany({}); 228 | await cleanData([ attachmentPath ]); 229 | } 230 | 231 | async function insertPiecesAndPages(apos) { 232 | const req = apos.task.getReq(); 233 | 234 | const formData = new FormData(); 235 | formData.append( 236 | 'file', 237 | createReadStream(path.join(apos.rootDir, '/public/test-image.jpg')) 238 | ); 239 | 240 | const jar = await login(apos.http); 241 | 242 | const attachment = await apos.http.post('/api/v1/@apostrophecms/attachment/upload', { 243 | body: formData, 244 | jar 245 | }); 246 | 247 | await apos.nonLocalized.insert(req, { 248 | ...apos.nonLocalized.newInstance(), 249 | title: 'nonLocalized1' 250 | }); 251 | 252 | const image1 = await apos.image.insert(req, { 253 | ...apos.image.newInstance(), 254 | title: 'image1', 255 | attachment 256 | }); 257 | 258 | const topic3 = await apos.topic.insert(req, { 259 | ...apos.topic.newInstance(), 260 | title: 'topic3' 261 | }); 262 | 263 | const topic2 = await apos.topic.insert(req, { 264 | ...apos.topic.newInstance(), 265 | title: 'topic2' 266 | }); 267 | 268 | const topic1 = await apos.topic.insert(req, { 269 | ...apos.topic.newInstance(), 270 | title: 'topic1', 271 | _topics: [ topic3 ] 272 | }); 273 | 274 | await apos.article.insert(req, { 275 | ...apos.article.newInstance(), 276 | title: 'article1', 277 | _topics: [ topic2 ], 278 | image: attachment 279 | }); 280 | 281 | const article2 = await apos.article.insert(req, { 282 | ...apos.article.newInstance(), 283 | title: 'article2', 284 | _topics: [ topic1 ] 285 | }); 286 | 287 | const pageInstance = await apos.http.post('/api/v1/@apostrophecms/page', { 288 | body: { 289 | _newInstance: true 290 | }, 291 | jar 292 | }); 293 | 294 | await apos.page.insert(req, '_home', 'lastChild', { 295 | ...pageInstance, 296 | title: 'page1', 297 | type: 'default-page', 298 | _articles: [ article2 ], 299 | main: { 300 | _id: 'areaId', 301 | items: [ 302 | { 303 | _id: 'cllp5ubqk001320613gaz2vmv', 304 | metaType: 'widget', 305 | type: '@apostrophecms/image', 306 | aposPlaceholder: false, 307 | imageIds: [ image1.aposDocId ], 308 | imageFields: { 309 | [image1.aposDocId]: { 310 | top: null, 311 | left: null, 312 | width: null, 313 | height: null, 314 | x: null, 315 | y: null 316 | } 317 | } 318 | } 319 | ], 320 | metaType: 'area' 321 | } 322 | }); 323 | } 324 | 325 | async function login(http, username = 'admin') { 326 | const jar = http.jar(); 327 | 328 | await http.post('/api/v1/@apostrophecms/login/login', { 329 | body: { 330 | username, 331 | password: username, 332 | session: true 333 | }, 334 | jar 335 | }); 336 | 337 | const loggedPage = await http.get('/', { 338 | jar 339 | }); 340 | 341 | assert(loggedPage.match(/logged in/)); 342 | 343 | return jar; 344 | } 345 | 346 | async function insertAdminUser(apos) { 347 | const user = { 348 | ...apos.user.newInstance(), 349 | title: 'admin', 350 | username: 'admin', 351 | password: 'admin', 352 | role: 'admin' 353 | }; 354 | 355 | await apos.user.insert(apos.task.getReq(), user); 356 | } 357 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 3.5.0 (2025-11-25) 4 | 5 | ### Adds 6 | 7 | * Adds failed export batch notifications. 8 | 9 | ## 3.4.0 (2025-10-30) 10 | 11 | ### Changes 12 | 13 | * Save import files in uploadfs. Import-export now works on projects using multiple node servers. 14 | 15 | ## 3.3.1 (2025-09-03) 16 | 17 | ### Fixes 18 | 19 | * CSV imports should not require a `type` column when importing pieces. 20 | * Removed unused `connect-multiparty` middleware, eliminating superfluous warnings. 21 | 22 | ## 3.3.0 (2025-08-06) 23 | 24 | ### Adds 25 | 26 | * Adds a checkbox to select and deselect all related document types in the export modal. 27 | 28 | ## 3.2.1 (2025-05-14) 29 | 30 | ### Fixes 31 | 32 | * Fixes the missing failed imports report modal when cancelling the import during override duplicates phase. 33 | * Compatible with `apostrophe` version 4.17.0 and up. Do not use 4.16.0, which was briefly incompatible with this module. 34 | This version correctly handles inserts of documents before their related images in the context of the latest `apostrophe` release. 35 | 36 | ## 3.2.0 (2025-04-16) 37 | 38 | ### Adds 39 | 40 | * Adds support for translating content while importing via the Automatic Translation module (when available). 41 | * Allow page position to be adapted when importing based on slug hierarchy. 42 | 43 | ### Fixes 44 | 45 | * Fix "include related documents" modal height. 46 | * Fixes `moduleLabel` computed. 47 | * Fixes missing call to `recomputeAllDocReferences` when duplicated have been overridden. 48 | 49 | ### Changes 50 | 51 | * Bumbs `eslint-config-apostrophe` to `5`, fixes errors, removes unused dependencies. 52 | * Exports attachments by default, without treating them like related documents. 53 | 54 | ## 3.1.0 (2025-03-19) 55 | 56 | ### Adds 57 | 58 | * Add error reporting UI (modal) for failed imports. 59 | 60 | ### Fixes 61 | 62 | * Better english wording when import succeeds. 63 | * Adds missing information to export and import batch operation jobs. 64 | 65 | ## 3.0.0 (2025-02-19) 66 | 67 | ### Changes 68 | 69 | * **Major version change:** 3.0.0 because this release is compatible only with ApostropheCMS 4.x. If you have not upgraded to ApostropheCMS 4.x, [you should do so.](https://docs.apostrophecms.org/guide/migration/upgrading-3-to-4.html#how-to-upgrade-your-project). For your ApostropheCMS 4.x project, make sure you change the dependency in `package.json` to `^3.0.0` to get this version. 70 | * Hide the `importDraftsOnly` checkbox for autopublished documents. 71 | 72 | ### Adds 73 | 74 | * Adds a method to sort all exported documents, makes sure that parent pages come always first. 75 | * Uses the new big-upload feature of apostrophe to allow very large export files to be imported. 76 | * Handles progress notification when uploading document (before running the actual import). 77 | 78 | ## 2.6.0 (2025-01-27) 79 | 80 | ### Adds 81 | 82 | * Pages linked a page via the "Internal Page" option in the rich text editor are now candidates to be exported as related documents. 83 | * Images embedded inline in rich text widgets via the `insert: [ 'image' ]` option are now candidates to be exported as related documents. 84 | * Uses `moduleLabels` prop when available. 85 | 86 | ### Fixes 87 | 88 | * Fixes Export modal related types slide animation. Uses `checkedTypes` when passed from parent component. 89 | 90 | ## 2.5.0 (2024-11-08) 91 | 92 | ### Adds 93 | 94 | * Adds a checkbox to the import modal that, when selected, imports published versions of documents from an export file **as drafts**. If a document doesn’t have a published version, it will still be imported. Setting the `importDraftsOnlyDefault: true` option selects this checkbox by default. 95 | 96 | ## 2.4.2 (2024-10-31) 97 | 98 | ### Adds 99 | 100 | * Adds AI-generated missing translations. 101 | 102 | ## 2.4.1 (2024-10-03) 103 | 104 | ### Fixes 105 | 106 | * Fixes SASS warnings. 107 | * Fixes a bug where related documents were not exported if they were only in one (draft or published) mode. 108 | * Fixes a bug where additional crops of an image were not imported properly when the image was already present on the receiving site. 109 | * Clean up timers on instance destroy. 110 | 111 | ## Changes 112 | 113 | * Update devDependencies. 114 | 115 | ## 2.4.0 (2024-09-05) 116 | 117 | ### Adds 118 | 119 | * Singletons can now be imported through `contextOperations` since they have no manager modal and thus no `utilityOperation` available. 120 | * Pages can be exported via an Export batch operation. 121 | 122 | ### Fixes 123 | 124 | * Exported related documents now contain the entire document and not only the projected fields. 125 | * The `related` route also returns the related types of the exported documents related documents. 126 | * Greatly improved performance when imports involve attachments that already exist on the target site. 127 | * Cropped images are imported properly. 128 | * Page tree relationships are maintained. 129 | * In general: many fixes to bring this module up to speed for use cases that involve selecting multiple documents. 130 | 131 | ## 2.3.0 (2024-08-08) 132 | 133 | ### Adds 134 | 135 | * Add a scrollbar to the duplicate import modal to handle too many duplicates, and fixed the "Type" column to display the correct document type. Thanks to (Borel Kuomo)() for this contribution. 136 | 137 | ### Fixes 138 | 139 | * Adds a method `checkDuplicates` to pre check for duplicates before importing, allows to insert attachments before documents to avoid them being stripped. 140 | * Uses the new method `simulateRelationshipsFromStorage` from core to simulate relationships on data from DB to be able to pass the convert, 141 | also uses the new option `fetchRelationships: false` on the convert to avoid fetching relationships from the DB. 142 | It prevents issues when a relationship has not been inserted yet. 143 | * Requires a duplicate confirmation for existing singleton documents (including parked pages), keeping their original ID's while importing (if the user chooses to do so). 144 | 145 | ## 2.2.0 (2024-07-12) 146 | 147 | ### Adds 148 | 149 | * Adds a `preventUpdateAssets` to the module that will not try to update already existing assets on import. 150 | 151 | ## 2.1.1 (2024-06-21) 152 | 153 | ### Fixes 154 | 155 | * Fix export relationship when an area has grouped widgets. Thanks to Michelin for contributing this fix. 156 | 157 | ## 2.1.0 (2024-06-12) 158 | 159 | ### Adds 160 | 161 | * Add the possibility to set a **key column** in your import CSV file in order to update existing pieces and pages. 162 | Thanks to this, this module reaches parity with the deprecated [`@apostrophecms/piece-type-importer`](https://github.com/apostrophecms/piece-type-importer) module. 163 | 164 | ### Fixes 165 | 166 | * We can now import pieces or pages with an import file that contains just the required fields. 167 | 168 | ## 2.0.0 (2024-05-15) 169 | 170 | ### Changes 171 | 172 | * Corrects documentation of required permissions. 173 | 174 | ### Adds 175 | 176 | * Add CSV format. 177 | 178 | ### Breaking changes 179 | 180 | * This is a new major version, 2.0.0. To update to this version you must edit your `package.json` file and change the dependency to `^2.0.0` or similar. 181 | * The signature of the `output` function from the gzip format has changed. It no longer takes the `apos` instance and now requires a `processAttachments` callback. 182 | * `import` and `overrideDuplicates` functions now require `formatLabel` to be passed in `req`. 183 | 184 | ## 1.4.1 (2024-03-20) 185 | 186 | ### Changes 187 | 188 | * Documentation updates. 189 | 190 | ### Fixes 191 | 192 | * Fixes imported data with the wrong mode because `req.mode` was always `published`, even for draft documents. 193 | 194 | ## 1.4.0 (2024-03-12) 195 | 196 | ### Changes 197 | 198 | * Compatible with both Apostrophe 3.x and Apostrophe 4.x (both Vue 2 and Vue 3). 199 | 200 | ### Fixes 201 | 202 | * Bug fix. When a piece or a page is created, published, then unpublished, and subsequently exported and re-imported, the manager modal incorrectly showed no published version. This occurs because the `lastPublishedAt` property of the draft document was set to null upon import, misleading the representation of the document's published state. Now it retains the original document's `lastPublishedAt` value. 203 | 204 | ## 1.3.0 (2024-02-21) 205 | 206 | ### Changes 207 | 208 | * Requires the create and edit permissions to use the import utility operation 209 | 210 | ## 1.2.1 (2024-01-24) 211 | 212 | ### Security 213 | 214 | * Fixed a security issue that allowed a correctly crafted 215 | HTTP request to delete arbitrary files and folders, subject to the permissions with which the Node.js 216 | process was run. No user account was required to exploit this issue. All users of this module should immediately run `npm update @apostrophecms/import-export` and deploy the latest version of this module. The module has been carefully audited for similar issues and best practices have been put in place to prevent any similar issue in future. 217 | 218 | ### Changes 219 | 220 | * Prefix routes and events to avoid conflicts with the old [`@apostrophecms/piece-type-importer`](https://github.com/apostrophecms/piece-type-importer) and [`@apostrophecms/piece-type-exporter`](https://github.com/apostrophecms/piece-type-exporter) modules. 221 | 222 | ## 1.2.0 (2023-11-29) 223 | 224 | ### Adds 225 | 226 | * Import now detects if the locale found in the exported docs is different from the current site one. 227 | If the site has only one locale configured, then the docs are automatically re-written with the site locale. 228 | If the site has multiple locales configured, the user is given the possibility to abort the import or re-write the docs with the site locale. 229 | Please note that this change is dependent on core changes found in 3.60.0. 230 | 231 | ### Changes 232 | 233 | * Hide "Duplicates Detected." notification. 234 | 235 | ## 1.1.0 (2023-11-03) 236 | 237 | ### Adds 238 | 239 | * Display more information about duplicated documents. 240 | * Export file name more meaningful, containing project name, module name and current date and time. 241 | 242 | ### Fixes 243 | 244 | * Fix progress bar going over `100%` when importing docs that are archived after being exported. 245 | * Adds missing dependency. 246 | 247 | ## 1.0.2 (2023-10-13) 248 | 249 | ### Fixes 250 | 251 | * Use `uploadfs.copyOut` to ensure success in more attachment export situations, such as debugging a multisite Assembly project 252 | on a local machine where Chrome considers subdomains of `localhost` to be your machine but Node.js does not. 253 | * Minor refactoring for maintainability and performance. 254 | * Error and warning notifications stay in place until dismissed. 255 | 256 | ## 1.0.1 (2023-10-12) 257 | 258 | ### Fixes 259 | 260 | Move documentation images to our own hosting. 261 | 262 | ## 1.0.0 (2023-10-12) 263 | 264 | ### Adds 265 | 266 | Initial release. 267 | -------------------------------------------------------------------------------- /ui/apos/components/AposImportModal.vue: -------------------------------------------------------------------------------- 1 | 105 | 106 | 273 | 274 | 379 | -------------------------------------------------------------------------------- /test/overrideLocales.js: -------------------------------------------------------------------------------- 1 | const assert = require('node:assert/strict'); 2 | const path = require('node:path'); 3 | const t = require('apostrophe/test-lib/util.js'); 4 | const { 5 | getAppConfig, 6 | insertAdminUser, 7 | deletePiecesAndPages, 8 | deleteAttachments, 9 | buildFixtures, 10 | copyFixtures, 11 | cleanFixtures 12 | } = require('./util/index.js'); 13 | 14 | describe('#import - overriding locales integration tests', function() { 15 | this.timeout(t.timeout); 16 | 17 | let apos; 18 | let importExportManager; 19 | let attachmentPath; 20 | 21 | describe('when the site has only one locale', function() { 22 | before(async function() { 23 | apos = await t.create({ 24 | root: module, 25 | testModule: true, 26 | modules: getAppConfig() 27 | }); 28 | 29 | attachmentPath = path.join(apos.rootDir, 'public/uploads/attachments'); 30 | importExportManager = apos.modules['@apostrophecms/import-export']; 31 | 32 | await insertAdminUser(apos); 33 | }); 34 | 35 | after(async function() { 36 | await t.destroy(apos); 37 | }); 38 | 39 | this.beforeEach(async function() { 40 | await deletePiecesAndPages(apos); 41 | await deleteAttachments(apos, attachmentPath); 42 | await cleanFixtures(apos); 43 | await copyFixtures(apos); 44 | await buildFixtures(apos); 45 | }); 46 | 47 | it('should not rewrite the docs locale nor ask about it when the locale is the same', async function() { 48 | const req = apos.task.getReq({ 49 | locale: 'en', 50 | body: {} 51 | }); 52 | 53 | const { 54 | duplicatedDocs, 55 | notificationId 56 | } = await importExportManager.import( 57 | req.clone({ 58 | files: { 59 | file: { 60 | path: path.join(apos.rootDir, 'data/tmp/uploads/topic-draft.tar.gz'), 61 | type: importExportManager.formats.gzip.allowedTypes[0] 62 | } 63 | } 64 | }) 65 | ); 66 | 67 | const notification = await apos.notification.db 68 | .findOne({ _id: notificationId }); 69 | const topics = await apos.doc.db 70 | .find({ type: 'topic' }) 71 | .toArray(); 72 | 73 | const actual = { 74 | duplicatedDocs, 75 | notification: { 76 | name: notification.event?.name 77 | }, 78 | topics 79 | }; 80 | const expected = { 81 | duplicatedDocs: [], 82 | notification: { 83 | name: undefined 84 | }, 85 | topics: [ 86 | { 87 | ...topics.at(0), 88 | _id: topics.at(0).aposDocId.concat(':en:draft'), 89 | aposMode: 'draft', 90 | aposLocale: 'en:draft', 91 | slug: 'topic1', 92 | title: 'topic1', 93 | type: 'topic' 94 | } 95 | ] 96 | }; 97 | 98 | assert.deepEqual(actual, expected); 99 | }); 100 | 101 | // FIX 102 | it('should rewrite the docs locale without asking about it when the locale is different', async function() { 103 | const req = apos.task.getReq({ 104 | locale: 'en', 105 | body: {} 106 | }); 107 | 108 | const { 109 | duplicatedDocs, 110 | notificationId 111 | } = await importExportManager.import( 112 | req.clone({ 113 | files: { 114 | file: { 115 | path: path.join(apos.rootDir, 'data/tmp/uploads/fr-topic-draft.tar.gz'), 116 | type: importExportManager.formats.gzip.allowedTypes[0] 117 | } 118 | } 119 | }) 120 | ); 121 | 122 | const notification = await apos.notification.db 123 | .findOne({ _id: notificationId }); 124 | const topics = await apos.doc.db 125 | .find({ type: 'topic' }) 126 | .toArray(); 127 | 128 | const actual = { 129 | duplicatedDocs, 130 | notification: { 131 | name: notification.event?.name 132 | }, 133 | topics 134 | }; 135 | const expected = { 136 | duplicatedDocs: [], 137 | notification: { 138 | name: undefined 139 | }, 140 | topics: [ 141 | { 142 | ...topics.at(0), 143 | _id: topics.at(0).aposDocId.concat(':en:draft'), 144 | aposMode: 'draft', 145 | aposLocale: 'en:draft', 146 | slug: 'topic1-fr', 147 | title: 'topic1 FR', 148 | type: 'topic' 149 | } 150 | ] 151 | }; 152 | 153 | assert.deepEqual(actual, expected); 154 | }); 155 | }); 156 | 157 | describe('when the site has multiple locales', function() { 158 | before(async function() { 159 | apos = await t.create({ 160 | root: module, 161 | testModule: true, 162 | modules: getAppConfig({ 163 | '@apostrophecms/express': { 164 | options: { 165 | session: { secret: 'supersecret' } 166 | } 167 | }, 168 | '@apostrophecms/i18n': { 169 | options: { 170 | defaultLocale: 'en', 171 | locales: { 172 | en: { label: 'English' }, 173 | fr: { 174 | label: 'French', 175 | prefix: '/fr' 176 | } 177 | } 178 | } 179 | } 180 | }) 181 | }); 182 | 183 | attachmentPath = path.join(apos.rootDir, 'public/uploads/attachments'); 184 | importExportManager = apos.modules['@apostrophecms/import-export']; 185 | 186 | await insertAdminUser(apos); 187 | }); 188 | 189 | after(async function() { 190 | await t.destroy(apos); 191 | }); 192 | 193 | this.beforeEach(async function() { 194 | await deletePiecesAndPages(apos); 195 | await deleteAttachments(apos, attachmentPath); 196 | await cleanFixtures(apos); 197 | await copyFixtures(apos); 198 | await buildFixtures(apos); 199 | }); 200 | 201 | it('should not rewrite the docs locale nor ask about it when the locale is the same', async function() { 202 | const req = apos.task.getReq({ 203 | locale: 'fr', 204 | body: {} 205 | }); 206 | 207 | await apos.topic.insert( 208 | apos.task.getReq({ 209 | locale: 'fr', 210 | mode: 'draft' 211 | }), 212 | { 213 | ...apos.topic.newInstance(), 214 | _id: '4:fr:draft', 215 | slug: 'topic1-fr-existing-draft', 216 | title: 'topic1 FR EXISTING DRAFT' 217 | } 218 | ); 219 | 220 | await apos.topic.insert( 221 | apos.task.getReq({ 222 | locale: 'fr', 223 | mode: 'published' 224 | }), 225 | { 226 | ...apos.topic.newInstance(), 227 | _id: '4:fr:published', 228 | slug: 'topic1-fr-existing-published', 229 | title: 'topic1 FR EXISTING PUBLISHED' 230 | } 231 | ); 232 | 233 | const { 234 | duplicatedDocs, 235 | importedAttachments, 236 | exportId, 237 | jobId, 238 | notificationId, 239 | formatLabel 240 | } = await importExportManager.import( 241 | req.clone({ 242 | files: { 243 | file: { 244 | path: path.join(apos.rootDir, 'data/tmp/uploads/fr-topic-draft.tar.gz'), 245 | type: importExportManager.formats.gzip.allowedTypes[0] 246 | } 247 | } 248 | }) 249 | ); 250 | 251 | const _req = req.clone({ 252 | body: { 253 | ...req.body, 254 | docIds: duplicatedDocs.map(({ aposDocId }) => aposDocId), 255 | duplicatedDocs, 256 | importedAttachments, 257 | exportId, 258 | jobId, 259 | notificationId, 260 | formatLabel 261 | } 262 | }); 263 | 264 | await importExportManager.overrideDuplicates(_req); 265 | 266 | const notification = await apos.notification.db 267 | .findOne({ _id: notificationId }); 268 | const topics = await apos.doc.db 269 | .find({ type: 'topic' }) 270 | .toArray(); 271 | 272 | const actual = { 273 | notification: { 274 | name: notification.event?.name 275 | }, 276 | topics 277 | }; 278 | const expected = { 279 | notification: { 280 | name: undefined 281 | }, 282 | topics: [ 283 | { 284 | ...topics.at(0), 285 | _id: topics.at(0).aposDocId.concat(':fr:draft'), 286 | aposLocale: 'fr:draft', 287 | aposMode: 'draft', 288 | modified: true, 289 | slug: 'topic1-fr', 290 | title: 'topic1 FR' 291 | }, 292 | { 293 | ...topics.at(1), 294 | _id: topics.at(1).aposDocId.concat(':fr:published'), 295 | aposLocale: 'fr:published', 296 | aposMode: 'published', 297 | slug: 'topic1-fr-existing-published', 298 | title: 'topic1 FR EXISTING PUBLISHED' 299 | } 300 | 301 | ] 302 | }; 303 | 304 | assert.deepEqual(actual, expected); 305 | }); 306 | 307 | it('should not rewrite the docs locales nor insert them but ask about it when the locale is different', async function() { 308 | const req = apos.task.getReq({ 309 | locale: 'fr', 310 | body: {} 311 | }); 312 | 313 | const { 314 | notificationId 315 | } = await importExportManager.import( 316 | req.clone({ 317 | files: { 318 | file: { 319 | path: path.join(apos.rootDir, 'data/tmp/uploads/topic-draft.tar.gz'), 320 | type: importExportManager.formats.gzip.allowedTypes[0] 321 | } 322 | } 323 | }) 324 | ); 325 | 326 | const notification = await apos.notification.db 327 | .findOne({ _id: notificationId }); 328 | const topics = await apos.doc.db 329 | .find({ type: 'topic' }) 330 | .toArray(); 331 | 332 | const actual = { 333 | notification: { 334 | name: notification.event.name 335 | }, 336 | topics 337 | }; 338 | const expected = { 339 | notification: { 340 | name: 'import-export-import-locale-differs' 341 | }, 342 | topics: [] 343 | }; 344 | 345 | assert.deepEqual(actual, expected); 346 | }); 347 | 348 | it('should rewrite the docs locale when the locale is different and the `overrideLocale` param is provided', async function() { 349 | const req = apos.task.getReq({ 350 | locale: 'en', 351 | body: {} 352 | }); 353 | 354 | const { 355 | exportId, 356 | formatLabel, 357 | importDraftsOnly, 358 | translate, 359 | notificationId: notificationId1 360 | } = await importExportManager.import( 361 | req.clone({ 362 | files: { 363 | file: { 364 | path: path.join(apos.rootDir, 'data/tmp/uploads/fr-topic-draft.tar.gz'), 365 | type: importExportManager.formats.gzip.allowedTypes[0] 366 | } 367 | } 368 | }) 369 | ); 370 | 371 | const { 372 | duplicatedDocs, 373 | notificationId: notificationId2 374 | } = await importExportManager.import( 375 | req.clone({ 376 | body: { 377 | ...req.body, 378 | importDraftsOnly, 379 | translate, 380 | overrideLocale: true, 381 | exportId, 382 | formatLabel 383 | } 384 | }) 385 | ); 386 | 387 | const notification1 = await apos.notification.db 388 | .findOne({ _id: notificationId1 }); 389 | const notification2 = await apos.notification.db 390 | .findOne({ _id: notificationId2 }); 391 | const topics = await apos.doc.db 392 | .find({ type: 'topic' }) 393 | .toArray(); 394 | 395 | const actual = { 396 | duplicatedDocs, 397 | notification1: { 398 | name: notification1.event?.name 399 | }, 400 | notification2: { 401 | name: notification2.event?.name 402 | }, 403 | topics 404 | }; 405 | const expected = { 406 | duplicatedDocs: [], 407 | notification1: { 408 | name: 'import-export-import-locale-differs' 409 | }, 410 | notification2: { 411 | name: undefined 412 | }, 413 | topics: [ 414 | { 415 | ...topics.at(0), 416 | _id: topics.at(0).aposDocId.concat(':en:draft'), 417 | aposMode: 'draft', 418 | aposLocale: 'en:draft', 419 | slug: 'topic1-fr', 420 | title: 'topic1 FR', 421 | type: 'topic' 422 | } 423 | ] 424 | }; 425 | 426 | assert.deepEqual(actual, expected); 427 | }); 428 | }); 429 | }); 430 | -------------------------------------------------------------------------------- /ui/apos/components/AposDuplicateImportModal.vue: -------------------------------------------------------------------------------- 1 | 106 | 107 | 271 | 272 | 449 | -------------------------------------------------------------------------------- /ui/apos/components/AposExportModal.vue: -------------------------------------------------------------------------------- 1 | 150 | 151 | 329 | 330 | 499 | -------------------------------------------------------------------------------- /test/overrideDuplicates.js: -------------------------------------------------------------------------------- 1 | const assert = require('node:assert/strict'); 2 | const path = require('node:path'); 3 | const t = require('apostrophe/test-lib/util.js'); 4 | const { 5 | getAppConfig, 6 | insertAdminUser, 7 | deletePiecesAndPages, 8 | deleteAttachments, 9 | buildFixtures, 10 | copyFixtures, 11 | cleanFixtures, 12 | insertPiecesAndPages 13 | } = require('./util/index.js'); 14 | 15 | describe('#overrideDuplicates - overriding locales integration tests', function() { 16 | this.timeout(t.timeout); 17 | 18 | let apos; 19 | let importExportManager; 20 | let attachmentPath; 21 | 22 | describe('when the site has only one locale', function() { 23 | before(async function() { 24 | apos = await t.create({ 25 | root: module, 26 | testModule: true, 27 | modules: getAppConfig() 28 | }); 29 | 30 | attachmentPath = path.join(apos.rootDir, 'public/uploads/attachments'); 31 | importExportManager = apos.modules['@apostrophecms/import-export']; 32 | 33 | await insertAdminUser(apos); 34 | }); 35 | 36 | after(async function() { 37 | await t.destroy(apos); 38 | }); 39 | 40 | this.beforeEach(async function () { 41 | await deletePiecesAndPages(apos); 42 | await deleteAttachments(apos, attachmentPath); 43 | await cleanFixtures(apos); 44 | await copyFixtures(apos); 45 | await buildFixtures(apos); 46 | }); 47 | 48 | it('should not rewrite the docs locale when the locale is the same', async function() { 49 | const req = apos.task.getReq({ 50 | locale: 'en', 51 | body: { 52 | formatLabel: 'gzip' 53 | } 54 | }); 55 | 56 | await apos.topic.insert(apos.task.getReq({ mode: 'draft' }), { 57 | ...apos.topic.newInstance(), 58 | _id: '4:en:draft', 59 | slug: 'topic1-existing-draft', 60 | title: 'topic1 EXISTING DRAFT' 61 | }); 62 | 63 | await apos.topic.insert(apos.task.getReq({ mode: 'published' }), { 64 | ...apos.topic.newInstance(), 65 | _id: '4:en:published', 66 | slug: 'topic1-existing-published', 67 | title: 'topic1 EXISTING PUBLISHED' 68 | }); 69 | 70 | const { 71 | duplicatedDocs, 72 | importedAttachments, 73 | exportId, 74 | jobId, 75 | notificationId, 76 | formatLabel 77 | } = await importExportManager.import( 78 | req.clone({ 79 | files: { 80 | file: { 81 | path: path.join(apos.rootDir, 'data/tmp/uploads/topic-draft.tar.gz'), 82 | type: importExportManager.formats.gzip.allowedTypes[0] 83 | } 84 | } 85 | }) 86 | ); 87 | 88 | const _req = req.clone({ 89 | body: { 90 | ...req.body, 91 | docIds: duplicatedDocs.map(({ aposDocId }) => aposDocId), 92 | duplicatedDocs, 93 | importedAttachments, 94 | exportId, 95 | jobId, 96 | notificationId, 97 | formatLabel 98 | } 99 | }); 100 | 101 | await importExportManager.overrideDuplicates(_req); 102 | 103 | const topics = await apos.doc.db 104 | .find({ type: 'topic' }) 105 | .toArray(); 106 | 107 | const actual = topics; 108 | const expected = [ 109 | { 110 | ...topics.at(0), 111 | _id: topics.at(0).aposDocId.concat(':en:draft'), 112 | aposLocale: 'en:draft', 113 | aposMode: 'draft', 114 | modified: true, 115 | slug: 'topic1', 116 | title: 'topic1' 117 | }, 118 | { 119 | ...topics.at(1), 120 | _id: topics.at(1).aposDocId.concat(':en:published'), 121 | aposLocale: 'en:published', 122 | aposMode: 'published', 123 | slug: 'topic1-existing-published', 124 | title: 'topic1 EXISTING PUBLISHED' 125 | } 126 | ]; 127 | 128 | assert.deepEqual(actual, expected); 129 | }); 130 | 131 | it('should rewrite the docs locale when the locale is different', async function() { 132 | const req = apos.task.getReq({ 133 | locale: 'en', 134 | body: { 135 | formatLabel: 'gzip' 136 | } 137 | }); 138 | 139 | await apos.topic.insert(apos.task.getReq({ mode: 'draft' }), { 140 | ...apos.topic.newInstance(), 141 | _id: '4:en:draft', 142 | slug: 'topic1-existing-draft', 143 | title: 'topic1 EXISTING DRAFT' 144 | }); 145 | 146 | await apos.topic.insert(apos.task.getReq({ mode: 'published' }), { 147 | ...apos.topic.newInstance(), 148 | _id: '4:en:published', 149 | slug: 'topic1-existing-published', 150 | title: 'topic1 EXISTING PUBLISHED' 151 | }); 152 | 153 | const { 154 | duplicatedDocs, 155 | importedAttachments, 156 | exportId, 157 | jobId, 158 | notificationId, 159 | formatLabel 160 | } = await importExportManager.import( 161 | req.clone({ 162 | files: { 163 | file: { 164 | path: path.join(apos.rootDir, 'data/tmp/uploads/fr-topic-draft.tar.gz'), 165 | type: importExportManager.formats.gzip.allowedTypes[0] 166 | } 167 | } 168 | }) 169 | ); 170 | 171 | const _req = req.clone({ 172 | body: { 173 | ...req.body, 174 | docIds: duplicatedDocs.map(({ aposDocId }) => aposDocId), 175 | duplicatedDocs, 176 | importedAttachments, 177 | exportId, 178 | jobId, 179 | notificationId, 180 | formatLabel 181 | } 182 | }); 183 | 184 | await importExportManager.overrideDuplicates(_req); 185 | 186 | const topics = await apos.doc.db 187 | .find({ type: 'topic' }) 188 | .toArray(); 189 | 190 | const actual = topics; 191 | const expected = [ 192 | { 193 | ...topics.at(0), 194 | _id: topics.at(0).aposDocId.concat(':en:draft'), 195 | aposMode: 'draft', 196 | aposLocale: 'en:draft', 197 | slug: 'topic1-fr', 198 | title: 'topic1 FR', 199 | type: 'topic' 200 | }, 201 | { 202 | ...topics.at(1), 203 | _id: topics.at(1).aposDocId.concat(':en:published'), 204 | aposMode: 'published', 205 | aposLocale: 'en:published', 206 | slug: 'topic1-existing-published', 207 | title: 'topic1 EXISTING PUBLISHED', 208 | type: 'topic' 209 | } 210 | ]; 211 | 212 | assert.deepEqual(actual, expected); 213 | }); 214 | }); 215 | 216 | describe('when the site has multiple locales', function() { 217 | before(async function() { 218 | apos = await t.create({ 219 | root: module, 220 | testModule: true, 221 | modules: getAppConfig({ 222 | '@apostrophecms/express': { 223 | options: { 224 | session: { secret: 'supersecret' } 225 | } 226 | }, 227 | '@apostrophecms/i18n': { 228 | options: { 229 | defaultLocale: 'en', 230 | locales: { 231 | en: { label: 'English' }, 232 | fr: { 233 | label: 'French', 234 | prefix: '/fr' 235 | } 236 | } 237 | } 238 | }, 239 | '@apostrophecms/page': { 240 | options: { 241 | park: [ 242 | { 243 | parkedId: 'search-parked', 244 | slug: '/search', 245 | title: 'Search', 246 | type: '@apostrophecms/search' 247 | } 248 | ] 249 | } 250 | } 251 | }) 252 | }); 253 | 254 | attachmentPath = path.join(apos.rootDir, 'public/uploads/attachments'); 255 | importExportManager = apos.modules['@apostrophecms/import-export']; 256 | 257 | await insertAdminUser(apos); 258 | }); 259 | 260 | after(async function() { 261 | await t.destroy(apos); 262 | }); 263 | 264 | this.beforeEach(async function () { 265 | await deletePiecesAndPages(apos); 266 | await deleteAttachments(apos, attachmentPath); 267 | await cleanFixtures(apos); 268 | await copyFixtures(apos); 269 | await buildFixtures(apos); 270 | }); 271 | 272 | it('should check if documents to import have duplicates in the current locale', async function() { 273 | await insertPiecesAndPages(apos); 274 | 275 | const req = apos.task.getReq({ mode: 'draft' }); 276 | const frReq = apos.task.getReq({ 277 | locale: 'fr', 278 | mode: 'draft' 279 | }); 280 | const [ nonLocalized ] = await apos.doc.db.find({ title: 'nonLocalized1' }).toArray(); 281 | const enArticles = await apos.article.find(req).toArray(); 282 | const parkedPages = await apos.page 283 | .find(req, { parkedId: { $exists: true } }) 284 | .toArray(); 285 | const singleton = await apos.global.findGlobal(req); 286 | 287 | const failedIds = []; 288 | const reporting = { failure: () => {} }; 289 | const enDocs = enArticles.concat([ nonLocalized, singleton ]).concat(parkedPages); 290 | const enDuplicates = await importExportManager.checkDuplicates(req, { 291 | reporting, 292 | docs: enDocs, 293 | failedIds, 294 | failedLog: {} 295 | }); 296 | 297 | const frDocs = enDocs.map((doc) => { 298 | return { 299 | ...doc, 300 | _id: doc._id.replace('en', 'fr'), 301 | aposLocale: 'fr:draft' 302 | }; 303 | }); 304 | const frDuplicates = await importExportManager.checkDuplicates(frReq, { 305 | reporting, 306 | docs: frDocs, 307 | failedIds, 308 | failedLog: {} 309 | }); 310 | 311 | const actual = { 312 | enDuplicates: [ 313 | enDocs.every((doc) => enDuplicates.duplicatedIds.has(doc.aposDocId)), 314 | enDuplicates.duplicatedDocs.length, 315 | enDuplicates.duplicatedDocs.filter(doc => !!doc.replaceId).length 316 | ], 317 | frDuplicates: [ 318 | frDocs.every((doc) => frDuplicates.duplicatedIds.has(doc.aposDocId)), 319 | frDuplicates.duplicatedIds.has(nonLocalized.aposDocId), 320 | frDuplicates.duplicatedDocs.length, 321 | enDuplicates.duplicatedDocs.filter(doc => !!doc.replaceId).length 322 | ] 323 | }; 324 | const expected = { 325 | enDuplicates: [ true, 6, 3 ], 326 | frDuplicates: [ false, true, 4, 3 ] 327 | }; 328 | 329 | assert.deepEqual(actual, expected); 330 | }); 331 | 332 | it('should not rewrite the docs locale when the locale is the same', async function() { 333 | const req = apos.task.getReq({ 334 | locale: 'en', 335 | body: { 336 | formatLabel: 'gzip' 337 | } 338 | }); 339 | 340 | await apos.topic.insert(apos.task.getReq({ mode: 'draft' }), { 341 | ...apos.topic.newInstance(), 342 | _id: '4:en:draft', 343 | slug: 'topic1-existing-draft', 344 | title: 'topic1 EXISTING DRAFT' 345 | }); 346 | 347 | await apos.topic.insert(apos.task.getReq({ mode: 'published' }), { 348 | ...apos.topic.newInstance(), 349 | _id: '4:en:published', 350 | slug: 'topic1-existing-published', 351 | title: 'topic1 EXISTING PUBLISHED' 352 | }); 353 | 354 | const { 355 | duplicatedDocs, 356 | importedAttachments, 357 | exportId, 358 | jobId, 359 | notificationId, 360 | formatLabel 361 | } = await importExportManager.import( 362 | req.clone({ 363 | files: { 364 | file: { 365 | path: path.join(apos.rootDir, 'data/tmp/uploads/topic-draft.tar.gz'), 366 | type: importExportManager.formats.gzip.allowedTypes[0] 367 | } 368 | } 369 | }) 370 | ); 371 | 372 | const _req = req.clone({ 373 | body: { 374 | ...req.body, 375 | docIds: duplicatedDocs.map(({ aposDocId }) => aposDocId), 376 | duplicatedDocs, 377 | importedAttachments, 378 | exportId, 379 | jobId, 380 | notificationId, 381 | formatLabel 382 | } 383 | }); 384 | 385 | await importExportManager.overrideDuplicates(_req); 386 | 387 | const topics = await apos.doc.db 388 | .find({ type: 'topic' }) 389 | .toArray(); 390 | 391 | const actual = topics; 392 | const expected = [ 393 | { 394 | ...topics.at(0), 395 | _id: topics.at(0).aposDocId.concat(':en:draft'), 396 | aposLocale: 'en:draft', 397 | aposMode: 'draft', 398 | modified: true, 399 | slug: 'topic1', 400 | title: 'topic1' 401 | }, 402 | { 403 | ...topics.at(1), 404 | _id: topics.at(1).aposDocId.concat(':en:published'), 405 | aposLocale: 'en:published', 406 | aposMode: 'published', 407 | slug: 'topic1-existing-published', 408 | title: 'topic1 EXISTING PUBLISHED' 409 | } 410 | ]; 411 | 412 | assert.deepEqual(actual, expected); 413 | }); 414 | 415 | it('should rewrite the docs locale when the locale is different and the `overrideLocale` param is provided', async function() { 416 | const req = apos.task.getReq({ 417 | locale: 'en', 418 | body: { 419 | formatLabel: 'gzip' 420 | } 421 | }); 422 | 423 | await apos.topic.insert(apos.task.getReq({ mode: 'draft' }), { 424 | ...apos.topic.newInstance(), 425 | _id: '4:en:draft', 426 | slug: 'topic1-existing-draft', 427 | title: 'topic1 EXISTING DRAFT' 428 | }); 429 | 430 | await apos.topic.insert(apos.task.getReq({ mode: 'published' }), { 431 | ...apos.topic.newInstance(), 432 | _id: '4:en:published', 433 | slug: 'topic1-existing-published', 434 | title: 'topic1 EXISTING PUBLISHED' 435 | }); 436 | 437 | const { 438 | exportId, 439 | formatLabel, 440 | importDraftsOnly, 441 | translate 442 | } = await importExportManager.import( 443 | req.clone({ 444 | files: { 445 | file: { 446 | path: path.join(apos.rootDir, 'data/tmp/uploads/fr-topic-draft.tar.gz'), 447 | type: importExportManager.formats.gzip.allowedTypes[0] 448 | } 449 | } 450 | }) 451 | ); 452 | 453 | // // TODO: check notification name 454 | // // apos.notify = async (req, message, options) => { 455 | // // assert.equal(options.event.name, 'import-export-import-locale-differs'); 456 | // // }; 457 | 458 | const { 459 | duplicatedDocs, 460 | importedAttachments, 461 | jobId, 462 | notificationId 463 | } = await importExportManager.import( 464 | req.clone({ 465 | body: { 466 | ...req.body, 467 | importDraftsOnly, 468 | translate, 469 | overrideLocale: true, 470 | exportId, 471 | formatLabel 472 | } 473 | }) 474 | ); 475 | 476 | const _req = req.clone({ 477 | body: { 478 | ...req.body, 479 | docIds: duplicatedDocs.map(({ aposDocId }) => aposDocId), 480 | duplicatedDocs, 481 | importedAttachments, 482 | overrideLocale: true, 483 | exportId, 484 | jobId, 485 | notificationId, 486 | formatLabel 487 | } 488 | }); 489 | 490 | await importExportManager.overrideDuplicates(_req); 491 | 492 | const topics = await apos.doc.db 493 | .find({ type: 'topic' }) 494 | .toArray(); 495 | 496 | const actual = topics; 497 | const expected = [ 498 | { 499 | ...topics.at(0), 500 | _id: topics.at(0).aposDocId.concat(':en:draft'), 501 | aposMode: 'draft', 502 | aposLocale: 'en:draft', 503 | modified: true, 504 | slug: 'topic1-fr', 505 | title: 'topic1 FR', 506 | type: 'topic' 507 | }, 508 | { 509 | ...topics.at(1), 510 | _id: topics.at(1).aposDocId.concat(':en:published'), 511 | aposMode: 'published', 512 | aposLocale: 'en:published', 513 | slug: 'topic1-existing-published', 514 | title: 'topic1 EXISTING PUBLISHED', 515 | type: 'topic' 516 | } 517 | ]; 518 | 519 | assert.deepEqual(actual, expected); 520 | }); 521 | }); 522 | }); 523 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | ApostropheCMS logo 3 | 4 |

Apostrophe Import Export Module

5 |

6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 |

16 |
17 | 18 | This module enables import and export of pages and pieces, with or without related 19 | documents such as files, images and other related types. 20 | 21 | ## Installation 22 | 23 | To install the module, use the command line to run this command in an Apostrophe project's root directory: 24 | 25 | ``` 26 | npm install @apostrophecms/import-export 27 | ``` 28 | 29 | ## Usage 30 | 31 | Configure the module in the `app.js` file: 32 | 33 | ```javascript 34 | require('apostrophe')({ 35 | shortName: 'my-project', 36 | modules: { 37 | '@apostrophecms/import-export': {} 38 | } 39 | }); 40 | ``` 41 | ## Exporting Files 42 | 43 | ### Pages 44 | ![Screenshot highlighting the export menu item in the context menu of the bulk operations in the manager.](https://static.apostrophecms.com/apostrophecms/import-export/images/bulk-export.png) 45 | 46 | Pages can be exported by either selecting "Export" from the context menu for an individual page, or by selecting multiple pages and then selecting "Export" from the bulk operations context menu. When exporting a page it will retain its publication status upon import. A page that is published when exported will be published on import unless you select the option to only import as a draft document. Draft pages will remain in draft status. 47 | 48 | Exporting the parent page in a set of nested documents will **not** export the child pages. Each child page must be exported separately. To maintain the page order, import the file for the parent document first, and then the files for the child documents. 49 | 50 | If a page exists in multiple locales, only the page for the current locale will be exported, not for any other locale. The page must be exported separately for each locale. 51 | 52 |
53 | 54 | ![Screenshot of the page export modal](https://static.apostrophecms.com/apostrophecms/import-export/images/page-export-modal.png) 55 | 56 | Clicking export will bring up a dialog box with two input controls. 57 | 58 | The first is for selecting the file format to use for the exported file. Currently, the module only exports in the `.tar.gz` format, but this will possibly be expanded in the future. 59 | 60 | The second input toggles whether documents associated with the page, like images, files, pieces, or pieces related via relationship fields in the document or its widgets, should be included in the download. Any or all can be unselected. Note that this doesn't mean that the particular page includes all of those document types, just that those document types both exist in your project and aren't disabled for export. 61 | 62 | Clicking on the "Export Page" button will trigger a download of the export file to your local computer. 63 | 64 |
65 | 66 | ### Pieces 67 | ![Screenshot of exporting pieces using the batch method](https://static.apostrophecms.com/apostrophecms/import-export/images/piece-batch-export.png) 68 | 69 | Unlike pages, pieces can be exported either individually from the context menu to the right of the piece, or as a batch using the context menu above the pieces list in the manager. For example, in the image above two articles have been selected for export. As with the page-type, all exporting is per-locale. 70 | 71 | Clicking export will bring up the same dialog box that appears when exporting pages, allowing for the selection of related documents that should also be exported. 72 | 73 |
74 | 75 | ### Templates 76 | ![Screenshot of exporting templates](https://static.apostrophecms.com/apostrophecms/import-export/images/template-export.png) 77 | 78 | If you have the [Template Library Pro](https://apostrophecms.com/extensions/template-library) module installed, templates are exported by selecting one or more within the template manager and then using the context menu located to the right of the batch operations. Like with the piece-type exporter you can elect to batch export just one or multiple documents depending on how many are checked. As with the page-type, all exporting is per-locale. 79 | 80 | Clicking export will bring up the same dialog box that appears when exporting pages, allowing for the selection of related documents that also should be exported. 81 | 82 |
83 | 84 | ## Importing files 85 | 86 | ![Screenshot of the utilities context menu for importing in the page manager](https://static.apostrophecms.com/apostrophecms/import-export/images/page-import.png) 87 | Any export file, no matter the content, can be imported using the utility context menu located at the top of the content managers, typically located to the left of the button to add new content of that type. This includes the page manager, any piece manager, or the document template manager if installed. 88 | 89 | ![Screenshot of the file upload modal for importing files](https://static.apostrophecms.com/apostrophecms/import-export/images/page-import-with-translation.png) 90 | 91 | Clicking on the "Import" menu item will bring up a dialog box to select the export file you wish to import. You can only select one file at a time and the selection of an additional file will replace the first. At this stage, you can select to import any published pages as [drafts](#importing-as-drafts-only). If you have the optional [`@apostrophecms-pro/automatic-translation` extension](https://apostrophecms.com/extensions/automatic-translation) installed and configured you can elect to translate the content into the language of the locale where you are performing the import. After you select the exported file and click on the import button, a progress bar will be shown and a success or failure notification when the file has been fully imported. 92 | 93 | If the file you select has documents that already exist in your project, you'll get a notification and list of the documents that would be over-written. From that list you can choose documents you don't want imported. Note, if you have any documents that were previously published and then archived, they will trigger a duplicate overwrite warning. 94 | 95 | When importing a page, piece(s), or template(s) that was exported from one locale while currently in another locale, and you do not have the automatic translation module or have not elected to translate content, you will get a notification asking if you want to proceed. 96 | 97 | When importing several files that comprise a set of nested pages, importing the file with the parental file before importing the child files will preserve the page order in the tree. If a file for a child document is imported before or independently of the parent document, then it will be added as a child of the homepage. 98 | 99 | Warning - While you can rename the exported file, you must not change the file extension (`.tar.gz`) or content, or the import might not go as planned. You will also get an error when trying to import an incompatible file made with another file exporter, a file with an incorrect extension, or a file made with a version of the `Import/Export` module that has breaking changes from the currently installed version. 100 | 101 | >One caveat of sharing documents between sites is that the modules for page-types and piece-types must be the same in each. For example, if site A is using a page-type of `contact-page`, a page of that type can only be imported into a site that also has a `contact-page` module. If the codebase of the two projects are significantly different, it can either cause the import to fail, or some data to be lost if schema fields can't be reconciled. 102 | 103 | ### Permissions 104 | 105 | Users who have both 'create' and 'edit' permissions for a document type can export and import those documents. However, users with other permissions for a document type, but lacking both 'create' or 'edit' permissions, cannot import documents. They can still export documents of that type. 106 | 107 | ### Options 108 | 109 | You can disable the export and/or the import for any page- or piece-type using the `importExport` option. This option takes an object with `import` and `export` keys that take both can take boolean values. 110 | 111 | ```javascript 112 | export default { 113 | extend: '@apostrophecms/piece-type', 114 | options: { 115 | importExport: { 116 | import: true, 117 | export: false 118 | }, 119 | label: 'Article', 120 | pluralLabel: 'Articles' 121 | ``` 122 | 123 | In this example, the 'Article' piece-type can be imported from a file, but users won't be allowed to export any of this type. This will also cause the export option to disappear from both the batch and individual 'Article' piece context menus in the 'Article' piece-type manager. Note that this will not impact other piece-types. The removal of the export menus from the respective managers will also occur if page-type or document templates are disabled for exporting. Finally, any disabled document type will not appear on the list of related documents. 124 | 125 | Disabling a document type from being imported will remove the import item from the utility context menu of the respective document manager. It will also block insertion of any documents of that type when importing from the context menu of another manager. Finally, any export file that contains a related document type that is disabled for import will still successfully import for those document types that are allowed. 126 | 127 | The `export` key can also take an object with an `expiration` property. The value of this property sets how long the export file will persist before being deleted. The default is 600000ms (10 min.), but can be extended if it is anticipated that the download will be delayed for some reason. 128 | 129 | ```javascript 130 | export default { 131 | extend: '@apostrophecms/piece-type', 132 | options: { 133 | importExport: { 134 | import: true, 135 | export: { 136 | expiration: 600000 137 | } 138 | }, 139 | label: 'Article', 140 | pluralLabel: 'Articles' 141 | ``` 142 | 143 | ## Importing documents from another locale 144 | 145 | Regardless of the original locale, imported documents are imported into the current locale. You will be prompted to confirm if the locale is different and you do not have, or are not opting to use, the automatic translation module. 146 | 147 | If multiple locales are set up, the user will be prompted to choose between canceling the import or proceeding with it, unless the user has elected to translate the content. 148 | 149 | ![Screenshot highlighting the confirm modal letting the user choose between aborting on continuing the import when the docs locale is different from the site one.](https://static.apostrophecms.com/apostrophecms/import-export/images/different-locale-modal.png) 150 | 151 | ### Automatic translation 152 | 153 | If the `@apostrophecms-pro/automatic-translation` module is installed, the import modal will offer a "Translate" checkbox. When selected, the module will automatically translate the imported documents to the current site's locale using the configured translation service. 154 | 155 | When the "Translate" checkbox is selected, the confirm modal letting the user choose between aborting on continuing the import when the docs locale is different from the site, will be suppressed and the import will proceed without any further user interaction. 156 | 157 | ## Importing as drafts only 158 | 159 | The import modal includes a checkbox labeled "_Import all documents as drafts_" that, when selected, imports the published version of documents in draft status only. This feature allows users to review and make modifications to imported documents before they are published. 160 | 161 | By default, this checkbox can be set to selected by enabling the `importDraftsOnlyDefault: true` option in this module. 162 | 163 | If a document in the import file has no published version, it will still be imported. Content types that do not have separate drafts will be imported normally. 164 | 165 | ## Updating existing pieces using CSV format 166 | 167 | You can also update existing pieces via this module. 168 | 169 | To do that, you will need one (and only one) **key column** in your file. This column's name **must be exactly the name of the existing field** that uniquely identifies each row as an update of a specific existing piece, **followed by `:key`**. 170 | 171 | For instance, if you need to change the usernames of users in bulk, you might prepare a CSV file like this: 172 | 173 | ``` 174 | username:key,username 175 | bobsmith,bob.smith 176 | janedoe,jane.doe 177 | ``` 178 | 179 | The key column is the *old value*. You may optionally also present a *new value* for that same column in a separate column without `:key`. You may also include other columns, as you see fit. The important thing is that you must have one and only one `:key` column in order to carry out updates. 180 | 181 | Please note that both the draft and the published versions will be updated, even if one of them does not match the key column value. 182 | 183 | ## Mixing inserts and updates 184 | 185 | If a row has no value for your `:key` column, it is treated as an insert, rather than an update. 186 | 187 | ## Importing rich text (HTML) rather than plaintext 188 | 189 | By default, if you create a column in your CSV file for a field of type `area`, it will be imported as plaintext. Any special characters like `<` and `>` will be escaped so the user can see them. HTML is not supported. 190 | 191 | To import areas as rich text HTML markup, add the `importAsRichText: true` option to the `area` field in your schema. 192 | 193 | ## How to add a new format? 194 | 195 | ### Create a file for your format: 196 | 197 | Add your format under `lib/formats/.js` and export it in l`ib/formats/index.js`. 198 | 199 | **Simple example** (for a single file without attachment files): 200 | 201 | ```js 202 | // lib/formats/ods.js 203 | module.exports = { 204 | label: 'ODS', 205 | extension: '.ods', 206 | allowedExtension: '.ods', 207 | allowedTypes: [ 'application/vnd.oasis.opendocument.spreadsheet' ], 208 | async input(filepath) { 209 | // Read `filepath` using `fs.createReadStream` 210 | // or any reader provided by a third-party library 211 | 212 | // Return parsed docs as an array 213 | return { docs }; 214 | }, 215 | async output(filepath, { docs }) { 216 | // Write `docs` into `filepath` using `fs.createWriteStream` 217 | // or any writer provided by a third-party library 218 | } 219 | }; 220 | ``` 221 | 222 | **Note**: The `input` and `output` functions should remain agnostic of any apostrophe logic. 223 | 224 | ```js 225 | // lib/formats/index.js 226 | const ods = require('./ods'); 227 | 228 | module.exports = { 229 | // ... 230 | ods 231 | }; 232 | ``` 233 | 234 | ### For formats with attachment files: 235 | 236 | If you want to add a format that includes attachment files such as an archive, you can enable the `includeAttachments` option and utilize extra arguments provided in the `input` and `output` functions. 237 | 238 | **Advanced example**: 239 | 240 | ```js 241 | // lib/formats/zip.js 242 | module.exports = { 243 | label: 'ZIP', 244 | extension: '.zip', 245 | allowedExtension: '.zip', 246 | allowedTypes: [ 247 | 'application/zip', 248 | 'application/x-zip', 249 | 'application/x-zip-compressed' 250 | ], 251 | includeAttachments: true, 252 | async input(filepath) { 253 | let exportPath = filepath; 254 | 255 | // If the given path is the archive, we first need to extract it 256 | // and define `exportPath` to the extracted folder, not the archive 257 | if (filepath.endsWith(this.allowedExtension)) { 258 | exportPath = filepath.replace(this.allowedExtension, ''); 259 | 260 | // Use format-specif extraction 261 | await extract(filepath, exportPath); 262 | await fsp.unlink(filepath); 263 | } 264 | 265 | // Read docs and attachments from `exportPath` 266 | // given that they are stored in aposDocs.json and aposAttachments.json files: 267 | const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); 268 | const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); 269 | const parsedDocs = EJSON.parse(docs); 270 | const parsedAttachments = EJSON.parse(attachments); 271 | 272 | // Add the attachment names and their path where they are going to be written to 273 | const attachmentsInfo = parsedAttachments.map(attachment => ({ 274 | attachment, 275 | file: { 276 | name: `${attachment.name}.${attachment.extension}`, 277 | path: path.join(exportPath, 'attachments', `${attachment._id}-${attachment.name}.${attachment.extension}`) 278 | } 279 | })); 280 | 281 | // Return parsed docs as an array, attachments with their extra files info 282 | // and `exportPath` since it we need to inform the caller where the extracted data is: 283 | return { 284 | docs: parsedDocs, 285 | attachmentsInfo, 286 | exportPath 287 | }; 288 | }, 289 | async output( 290 | filepath, 291 | { 292 | docs, 293 | attachments = [], 294 | attachmentUrls = {} 295 | }, 296 | processAttachments 297 | ) { 298 | // Store the docs and attachments into `aposDocs.json` and `aposAttachments.json` files 299 | // and add them to the archive 300 | 301 | // Create a `attachments/` directory in the archive and store the attachment files inside it: 302 | const addAttachment = async (attachmentPath, name, size) => { 303 | // Read attachment from `attachmentPath` 304 | // and store it into `attachments/` inside the archive 305 | } 306 | const { attachmentError } = await processAttachments(attachmentUrls, addAttachment); 307 | 308 | // Write the archive that contains `aposDocs.json`, `aposAttachments.json` and `attachments/` 309 | // into `filepath` using `fs.createWriteStream` or any writer provided by a third-party library 310 | 311 | // Return potential attachment processing error so that the caller is aware of it: 312 | return { attachmentError }; 313 | } 314 | }; 315 | ``` 316 | 317 | ### Add formats via a separate module 318 | 319 | You might want to scope one or multiple formats in another module for several reasons: 320 | 321 | - The formats rely on a dependency that is not hosted on NPM (which is the case with [@apostrophecms/import-export-xlsx](https://github.com/apostrophecms/import-export-xlsx)) 322 | - You want to fully scope the format in a separate module and repository for an easier maintenance 323 | - ... 324 | 325 | To do so, simply create an apostrophe module that improves `@apostrophecms/import-export` and register the formats in the `init` method. 326 | 327 | 328 | Example with an `import-export-excel` module: 329 | 330 | ```js 331 | const formats: { 332 | xls: { 333 | label: 'XLS', 334 | extension: '.xls', 335 | allowedExtension: '.xls', 336 | allowedTypes: [ 'application/vnd.ms-excel' ], 337 | async input(filepath) {}, 338 | async output(filepath, { docs }) {} 339 | }, 340 | xlsx: { 341 | label: 'XLSX', 342 | extension: '.xlsx', 343 | allowedExtension: '.xlsx', 344 | allowedTypes: [ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], 345 | async input(filepath) {}, 346 | async output(filepath, { docs }) {} 347 | } 348 | }; 349 | 350 | module.exports = { 351 | improve: '@apostrophecms/import-export', 352 | init(self) { 353 | self.registerFormats(formats); 354 | } 355 | }; 356 | ``` 357 | 358 | Then add the module to the project **package.json** and **app.js**. 359 | -------------------------------------------------------------------------------- /lib/methods/export.js: -------------------------------------------------------------------------------- 1 | const path = require('node:path'); 2 | const util = require('node:util'); 3 | const fsp = require('node:fs/promises'); 4 | const Promise = require('bluebird'); 5 | const dayjs = require('dayjs'); 6 | const { uniqueId } = require('lodash'); 7 | 8 | const MAX_RECURSION = 10; 9 | 10 | module.exports = self => { 11 | return { 12 | async export(req, manager, reporting = null) { 13 | if (!req.user) { 14 | throw self.apos.error('forbidden'); 15 | } 16 | 17 | if (reporting) { 18 | reporting.setTotal(req.body._ids.length); 19 | } 20 | 21 | const ids = self.apos.launder.ids(req.body._ids); 22 | const relatedTypes = self.apos.launder.strings(req.body.relatedTypes); 23 | const expiration = self.options.importExport?.export?.expiration && 24 | self.apos.launder.integer(self.options.importExport.export.expiration); 25 | 26 | const [ defaultFormatName ] = Object.keys(self.formats); 27 | const formatName = self.apos.launder.string(req.body.formatName, defaultFormatName); 28 | 29 | const format = self.formats[formatName]; 30 | if (!format) { 31 | throw self.apos.error('invalid'); 32 | } 33 | 34 | const hasRelatedTypes = Boolean(relatedTypes.length); 35 | 36 | const docs = (await self.getDocs(req, ids, manager, reporting)) 37 | .map((doc) => self.apos.util.clonePermanent(doc)); 38 | 39 | if (!hasRelatedTypes) { 40 | self.sortDocs(docs); 41 | const { attachments, attachmentUrls } = await self.getDocsAttachments(docs); 42 | return self.exportFile( 43 | req, 44 | reporting, 45 | format, 46 | { 47 | docs, 48 | attachments, 49 | attachmentUrls 50 | }, 51 | expiration 52 | ); 53 | } 54 | 55 | const allDocs = [ ...docs ]; 56 | for (const doc of docs) { 57 | await self.getRelatedDocsFromSchema(req, { 58 | doc, 59 | schema: self.apos.modules[doc.type].schema, 60 | relatedTypes, 61 | storedData: allDocs 62 | }); 63 | } 64 | 65 | self.sortDocs(allDocs); 66 | 67 | if (!format.includeAttachments) { 68 | return self.exportFile( 69 | req, 70 | reporting, 71 | format, 72 | { docs: allDocs }, 73 | expiration 74 | ); 75 | } 76 | 77 | const { 78 | attachments, 79 | attachmentUrls 80 | } = await self.getDocsAttachments(allDocs); 81 | 82 | return self.exportFile( 83 | req, 84 | reporting, 85 | format, 86 | { 87 | docs: allDocs, 88 | attachments, 89 | attachmentUrls 90 | }, 91 | expiration 92 | ); 93 | }, 94 | 95 | sortDocs(docs) { 96 | docs.sort((a, b) => { 97 | if (a.aposMode === 'draft' && b.aposMode === 'published') { 98 | return -1; 99 | } 100 | if (a.aposMode === 'published' && b.aposMode === 'draft') { 101 | return 1; 102 | } 103 | if (!self.apos.page.isPage(a) && !self.apos.page.isPage(b)) { 104 | return 0; 105 | } 106 | if (self.apos.page.isPage(a) && !self.apos.page.isPage(b)) { 107 | return -1; 108 | } 109 | if (!self.apos.page.isPage(a) && self.apos.page.isPage(b)) { 110 | return 1; 111 | } 112 | if (a.level > b.level) { 113 | return 1; 114 | } 115 | if (a.level < b.level) { 116 | return -1; 117 | } 118 | if (a.rank < b.rank) { 119 | return -1; 120 | } 121 | if (a.rank > b.rank) { 122 | return 1; 123 | } 124 | return 0; 125 | }); 126 | }, 127 | 128 | // Get docs via their manager in order to populate them 129 | // so that we can retrieve their relationships IDs later, 130 | // and to let the manager handle permissions. 131 | async getDocs(req, docsIds, manager, reporting) { 132 | // For BC, used to accept hasRelatedTypes as third param 133 | if (arguments.length === 5) { 134 | manager = arguments[3]; 135 | reporting = arguments[4]; 136 | } 137 | if (!docsIds.length) { 138 | return []; 139 | } 140 | 141 | const { draftIds, publishedIds } = self.getAllModesIds(docsIds); 142 | const isReqDraft = req.mode === 'draft'; 143 | 144 | const docs = []; 145 | 146 | const draftReq = isReqDraft ? req : req.clone({ mode: 'draft' }); 147 | const draftDocs = await manager 148 | .findForEditing(draftReq, { 149 | _id: { 150 | $in: draftIds 151 | } 152 | }) 153 | .relationships(false) 154 | .toArray(); 155 | 156 | docs.push(...draftDocs); 157 | 158 | const publishedReq = isReqDraft ? req.clone({ mode: 'published' }) : req; 159 | const publishedDocs = await manager 160 | .findForEditing(publishedReq, { 161 | _id: { 162 | $in: publishedIds 163 | } 164 | }) 165 | .relationships(false) 166 | .toArray(); 167 | 168 | docs.push(...publishedDocs); 169 | 170 | if (reporting) { 171 | const docsId = docs.map(doc => doc._id); 172 | 173 | // Verify that each id sent in the body has its corresponding doc fetched 174 | docsIds.forEach(id => { 175 | const fn = docsId.includes(id) 176 | ? 'success' 177 | : 'failure'; 178 | 179 | reporting[fn](); 180 | }); 181 | } 182 | 183 | return docs.filter(doc => self.canExport(req, doc.type)); 184 | }, 185 | 186 | // Add the published version ID next to each draft ID, 187 | // so we always get both the draft and the published ID. 188 | // If somehow published IDs are sent from the frontend, 189 | // that will not be an issue thanks to this method. 190 | getAllModesIds(ids) { 191 | return ids.reduce(({ draftIds, publishedIds }, id) => { 192 | return { 193 | draftIds: [ ...draftIds, id.replace('published', 'draft') ], 194 | publishedIds: [ ...publishedIds, id.replace('draft', 'published') ] 195 | }; 196 | }, { 197 | draftIds: [], 198 | publishedIds: [] 199 | }); 200 | }, 201 | 202 | getAttachments(ids) { 203 | if (!ids.length) { 204 | return []; 205 | } 206 | 207 | return self.apos.attachment.db 208 | .find({ 209 | _id: { 210 | $in: ids 211 | } 212 | }) 213 | .toArray(); 214 | }, 215 | 216 | canExport(req, docType) { 217 | return self.canImportOrExport(req, docType, 'export'); 218 | }, 219 | 220 | async getRelatedDocsFromSchema(req, { 221 | doc, 222 | schema, 223 | relatedTypes, 224 | storedData, 225 | recursion = 0, 226 | mode = doc.aposMode || req.mode 227 | }) { 228 | recursion++; 229 | if (doc.type === '@apostrophecms/rich-text') { 230 | await self.getRelatedDocsFromRichTextWidget(req, { 231 | doc, 232 | relatedTypes, 233 | storedData, 234 | recursion, 235 | mode 236 | }); 237 | } 238 | for (const field of schema) { 239 | const fieldValue = doc[field.name]; 240 | const shouldRecurse = recursion <= MAX_RECURSION; 241 | 242 | if (!field.withType && !fieldValue) { 243 | continue; 244 | } 245 | if ( 246 | field.withType && 247 | relatedTypes && 248 | !self.relatedTypesIncludes(field.withType, relatedTypes) 249 | ) { 250 | continue; 251 | } 252 | if (field.withType && !self.canExport(req, field.withType)) { 253 | continue; 254 | } 255 | 256 | if (shouldRecurse && field.type === 'array') { 257 | for (const subField of fieldValue) { 258 | await self.getRelatedDocsFromSchema(req, { 259 | doc: subField, 260 | schema: field.schema, 261 | relatedTypes, 262 | recursion, 263 | storedData, 264 | mode 265 | }); 266 | } 267 | continue; 268 | } 269 | 270 | if (shouldRecurse && field.type === 'object') { 271 | await self.getRelatedDocsFromSchema(req, { 272 | doc: fieldValue, 273 | schema: field.schema, 274 | relatedTypes, 275 | recursion, 276 | storedData, 277 | mode 278 | }); 279 | continue; 280 | } 281 | 282 | if (shouldRecurse && field.type === 'area') { 283 | for (const widget of (fieldValue.items || [])) { 284 | const schema = self.apos.modules[`${widget?.type}-widget`]?.schema || []; 285 | await self.getRelatedDocsFromSchema(req, { 286 | doc: widget, 287 | schema, 288 | relatedTypes, 289 | recursion, 290 | storedData, 291 | mode 292 | }); 293 | } 294 | continue; 295 | } 296 | 297 | if (field.type === 'relationship') { 298 | await self.handleRelatedField(req, { 299 | doc, 300 | field, 301 | fieldValue, 302 | relatedTypes, 303 | storedData, 304 | shouldRecurse, 305 | recursion, 306 | mode 307 | }); 308 | } 309 | } 310 | }, 311 | 312 | relatedTypesIncludes(name, relatedTypes) { 313 | if ([ '@apostrophecms/any-page-type', '@apostrophecms/page' ].includes(name)) { 314 | return relatedTypes.some(type => { 315 | const module = self.apos.modules[type]; 316 | return self.apos.instanceOf(module, '@apostrophecms/page-type'); 317 | }); 318 | } 319 | return relatedTypes.includes(name); 320 | }, 321 | 322 | getDocsAttachmentsIds(docOrDocs) { 323 | const getIds = (list, doc) => { 324 | const schema = self.apos.modules[doc.type].schema; 325 | for (const field of schema) { 326 | if ( 327 | field.type === 'attachment' && 328 | doc[field.name]?._id && 329 | !list.includes(doc[field.name]._id) 330 | ) { 331 | list.push(doc[field.name]._id); 332 | } 333 | } 334 | return list; 335 | }; 336 | 337 | if (Array.isArray(docOrDocs)) { 338 | return docOrDocs.reduce(getIds, []); 339 | } 340 | 341 | return getIds([], docOrDocs); 342 | }, 343 | 344 | async getDocsAttachments(docs) { 345 | const attachmentsIds = self.getDocsAttachmentsIds(docs); 346 | const attachments = await self.getAttachments(attachmentsIds); 347 | const attachmentUrls = Object.fromEntries( 348 | attachments.map(attachment => { 349 | const name = `${attachment._id}-${attachment.name}.${attachment.extension}`; 350 | return [ 351 | name, self.apos.attachment.url(attachment, { 352 | size: 'original', 353 | uploadfsPath: true 354 | }) 355 | ]; 356 | }) 357 | ); 358 | 359 | return { 360 | attachments, 361 | attachmentUrls 362 | }; 363 | }, 364 | 365 | async getRelatedDocsFromRichTextWidget(req, { 366 | doc, 367 | relatedTypes, 368 | storedData, 369 | recursion, 370 | mode 371 | }) { 372 | let linkedDocs = await self.apos.doc.db.find({ 373 | aposDocId: { 374 | $in: doc.permalinkIds 375 | } 376 | }).project({ 377 | type: 1, 378 | aposDocId: 1, 379 | slug: 1 380 | }).toArray(); 381 | // We're likely going to fetch them all with an @apostrophecms/any-page-type query, 382 | // so we need to do our real related types check early or we'll allow all page types 383 | // whenever we allow even one 384 | linkedDocs = linkedDocs.filter(doc => relatedTypes.includes(doc.type)); 385 | const linkedIdsByType = new Map(); 386 | for (const linkedDoc of linkedDocs) { 387 | // Normalization is a little different here because these 388 | // are individual pages or pieces 389 | const docType = linkedDoc.slug.startsWith('/') 390 | ? '@apostrophecms/any-page-type' 391 | : linkedDoc.type; 392 | const isTypeStored = linkedIdsByType.has(docType); 393 | const linkedIds = isTypeStored ? linkedIdsByType.get(docType) : new Set(); 394 | linkedIds.add(linkedDoc.aposDocId); 395 | if (!isTypeStored) { 396 | linkedIdsByType.set(docType, linkedIds); 397 | } 398 | } 399 | if (doc.imageIds?.length > 0) { 400 | linkedIdsByType.set('@apostrophecms/image', new Set(doc.imageIds)); 401 | } 402 | const virtualDoc = { 403 | type: '@apostrophecms/rich-text_related' 404 | }; 405 | const virtualSchema = []; 406 | for (const [ linkedType, linkedIds ] of linkedIdsByType.entries()) { 407 | const baseName = self.apos.util.slugify(linkedType); 408 | const fieldName = `_${baseName}`; 409 | const idsStorage = `${baseName}Ids`; 410 | virtualSchema.push({ 411 | name: fieldName, 412 | type: 'relationship', 413 | withType: linkedType, 414 | idsStorage 415 | }); 416 | const ids = [ ...linkedIds.values() ]; 417 | virtualDoc[idsStorage] = ids; 418 | virtualDoc[fieldName] = ids.map(id => ({ aposDocId: id })); 419 | } 420 | await self.getRelatedDocsFromSchema(req, { 421 | doc: virtualDoc, 422 | schema: virtualSchema, 423 | relatedTypes, 424 | storedData, 425 | recursion, 426 | mode 427 | }); 428 | }, 429 | 430 | async handleRelatedField(req, { 431 | doc, 432 | field, 433 | fieldValue, 434 | relatedTypes, 435 | type, 436 | storedData, 437 | shouldRecurse, 438 | recursion 439 | }) { 440 | const manager = self.apos.doc.getManager(field.withType); 441 | const relatedIds = doc[field.idsStorage]; 442 | if (!relatedIds?.length) { 443 | return; 444 | } 445 | const criteria = { 446 | aposDocId: { $in: relatedIds }, 447 | $or: [ 448 | { aposLocale: `${req.locale}:draft` }, 449 | { aposLocale: `${req.locale}:published` }, 450 | { aposLocale: null } 451 | ] 452 | }; 453 | 454 | const foundRelated = await manager 455 | .findForEditing(req, criteria) 456 | .permission('view') 457 | .relationships(false) 458 | .areas(false) 459 | .locale(null) 460 | .toArray(); 461 | 462 | for (const related of foundRelated) { 463 | const alreadyAdded = storedData.some(({ _id }) => _id === related._id); 464 | if (alreadyAdded) { 465 | continue; 466 | } 467 | 468 | storedData.push(self.apos.util.clonePermanent(related)); 469 | if (!shouldRecurse) { 470 | continue; 471 | } 472 | 473 | const relatedManager = self.apos.doc.getManager(related.type); 474 | await self.getRelatedDocsFromSchema(req, { 475 | doc: related, 476 | schema: relatedManager.schema, 477 | relatedTypes, 478 | recursion, 479 | storedData 480 | }); 481 | } 482 | }, 483 | 484 | async exportFile(req, reporting, format, data, expiration) { 485 | const date = dayjs().format('YYYYMMDDHHmmss'); 486 | const filename = `${self.apos.shortName}-${req.body.type.toLowerCase()}-export-${date}${format.extension}`; 487 | const filepath = path.join(self.apos.attachment.uploadfs.getTempPath(), filename); 488 | 489 | try { 490 | const result = await format.output(filepath, data, self.processAttachments); 491 | 492 | if (result?.attachmentError) { 493 | await self.apos.notify(req, 'aposImportExport:exportAttachmentError', { 494 | interpolate: { format: format.label }, 495 | icon: 'alert-circle-icon', 496 | type: 'warning' 497 | }); 498 | } 499 | } catch (error) { 500 | await self.apos.notify(req, 'aposImportExport:exportFileGenerationError', { 501 | interpolate: { format: format.label }, 502 | icon: 'alert-circle-icon', 503 | type: 'danger' 504 | }); 505 | throw self.apos.error(error.message); 506 | } 507 | 508 | // Must copy it to uploadfs, the server that created it 509 | // and the server that delivers it might be different 510 | const downloadPath = path.join('/exports', filename); 511 | const downloadUrl = `${self.apos.attachment.uploadfs.getUrl()}${downloadPath}`; 512 | const copyIn = util.promisify(self.apos.attachment.uploadfs.copyIn); 513 | console.info(`[export] copying ${filepath} to ${self.apos.rootDir}/public/uploads${downloadPath}`); 514 | try { 515 | await copyIn(filepath, downloadPath); 516 | } catch (error) { 517 | await self.remove(filepath); 518 | throw error; 519 | } 520 | 521 | if (reporting) { 522 | // Setting the download url which will automatically be 523 | // opened on the browser thanks to an event triggered 524 | // by a notification handled by the reporting. 525 | reporting.setResults({ 526 | url: downloadUrl 527 | }); 528 | } else { 529 | // No reporting means no batch operation: 530 | // We need to handle the notification manually 531 | // when exporting a single document: 532 | await self.apos.notification.trigger(req, 'aposImportExport:exported', { 533 | interpolate: { 534 | count: req.body._ids.length, 535 | type: req.body.type 536 | }, 537 | dismiss: true, 538 | icon: 'database-export-icon', 539 | type: 'success', 540 | event: { 541 | name: 'import-export-export-download', 542 | data: { 543 | url: downloadUrl 544 | } 545 | } 546 | }); 547 | } 548 | 549 | await self.remove(filepath); 550 | self.removeFromUploadFs(downloadPath, expiration); 551 | 552 | return { url: downloadUrl }; 553 | }, 554 | 555 | async processAttachments(attachments, addAttachment) { 556 | let attachmentError = false; 557 | 558 | const copyOut = Promise.promisify(self.apos.attachment.uploadfs.copyOut); 559 | 560 | await Promise.map(Object.entries(attachments), processAttachment, { 561 | concurrency: 5 562 | }); 563 | 564 | return { attachmentError }; 565 | 566 | async function processAttachment([ name, url ]) { 567 | const temp = self.apos.attachment.uploadfs.getTempPath() + '/' + self.apos.util.generateId(); 568 | console.info(`[export] processing attachment ${name} temporarily stored in ${temp}`); 569 | try { 570 | await copyOut(url, temp); 571 | const { size } = await fsp.stat(temp); 572 | // Looking at the source code, the tar-stream module 573 | // (when using the `gzip` format) 574 | // probably doesn't protect against two input streams 575 | // pushing mishmashed bytes into the tarball at the 576 | // same time, so stream in just one at a time. We still 577 | // get good concurrency on copyOut which is much slower 578 | // than this operation 579 | await self.apos.lock.withLock('import-export-copy-out', async () => { 580 | // Add attachment into the specific format 581 | await addAttachment(temp, name, size); 582 | }); 583 | } catch (e) { 584 | attachmentError = true; 585 | self.apos.util.error(e); 586 | } finally { 587 | await self.remove(temp); 588 | } 589 | } 590 | }, 591 | 592 | // Report is available for 10 minutes by default 593 | removeFromUploadFs(downloadPath, expiration) { 594 | const ms = expiration || 1000 * 60 * 10; 595 | const id = uniqueId(); 596 | console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs in ${ms / 1000 / 60} minutes`); 597 | const handler = () => { 598 | console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs`); 599 | delete self.timeoutIds[id]; 600 | return new Promise((resolve, _reject) => { 601 | self.apos.attachment.uploadfs.remove(downloadPath, error => { 602 | if (error) { 603 | self.apos.util.error(error); 604 | } 605 | resolve(); 606 | }); 607 | }); 608 | }; 609 | const timeoutId = setTimeout(handler, ms); 610 | self.timeoutIds[id] = { 611 | handler, 612 | timeoutId 613 | }; 614 | }, 615 | // The entry point. Modifies `related`, also returns `related` because code elsewhere 616 | // expects that 617 | getRelatedTypes(req, schema = [], related = []) { 618 | self.findSchemaRelatedTypes(req, schema, related, 0); 619 | return related; 620 | }, 621 | // Called recursively for you. Modifies `related`, has no useful return value 622 | findSchemaRelatedTypes(req, schema, related, recursions) { 623 | recursions++; 624 | if (recursions >= MAX_RECURSION) { 625 | return; 626 | } 627 | for (const field of schema) { 628 | if ( 629 | field.type === 'relationship' && 630 | self.canExport(req, field.withType) && 631 | !related.includes(field.withType) 632 | ) { 633 | self.pushRelatedType(req, related, field.withType, recursions); 634 | } else if ([ 'array', 'object' ].includes(field.type)) { 635 | self.findSchemaRelatedTypes(req, field.schema, related, recursions); 636 | } else if (field.type === 'area') { 637 | const widgets = self.apos.area.getWidgets(field.options); 638 | for (const [ widget, options ] of Object.entries(widgets)) { 639 | const schema = self.apos.area.getWidgetManager(widget).schema || []; 640 | if (widget === '@apostrophecms/rich-text') { 641 | self.getRelatedTypesFromRichTextWidget(req, options, related, recursions); 642 | } 643 | self.findSchemaRelatedTypes(req, schema, related, recursions); 644 | } 645 | } 646 | } 647 | }, 648 | pushRelatedType(req, related, type, recursions) { 649 | if ((type === '@apostrophecms/page') || (type === '@apostrophecms/any-page-type')) { 650 | const pageTypes = Object.entries(self.apos.doc.managers).filter( 651 | ([ name, module ]) => self.apos.instanceOf(module, '@apostrophecms/page-type')) 652 | .map(([ name, module ]) => name); 653 | for (const type of pageTypes) { 654 | if ([ '@apostrophecms/archive-page', '@apostrophecms/search' ].includes(type)) { 655 | // It is never appropriate to export the root page of the trash, and 656 | // while you *could* link to the search page it is extremely unlikely it 657 | // would have interesting content to export, just confusing to have it here 658 | continue; 659 | } 660 | self.pushRelatedType(req, related, type, recursions); 661 | } 662 | return; 663 | } 664 | if (!related.includes(type)) { 665 | related.push(type); 666 | const relatedManager = self.apos.doc.getManager(type); 667 | self.findSchemaRelatedTypes(req, relatedManager.schema, related, recursions); 668 | } 669 | }, 670 | // Does not currently utilize req, but it could be relevant in overrides and is 671 | // always the first argument by convention, so it is included in the signature 672 | getRelatedTypesFromRichTextWidget(req, options, related, recursions) { 673 | const manager = self.apos.modules['@apostrophecms/rich-text-widget']; 674 | const rteOptions = { 675 | ...manager.options.defaultOptions, 676 | ...options 677 | }; 678 | if ( 679 | (rteOptions.toolbar?.includes('image') || rteOptions.insert?.includes('image')) && 680 | !related.includes('@apostrophecms/image') 681 | ) { 682 | self.pushRelatedType(req, related, '@apostrophecms/image', recursions); 683 | } 684 | if (rteOptions.toolbar?.includes('link')) { 685 | const choices = manager.linkFields.linkTo.choices.map(choice => choice.value); 686 | for (const name of choices) { 687 | if (self.apos.doc.getManager(name) && !related.includes(name)) { 688 | self.pushRelatedType(req, related, name, recursions); 689 | } 690 | } 691 | } 692 | } 693 | }; 694 | }; 695 | --------------------------------------------------------------------------------