├── .gitignore ├── .eslintrc ├── ui └── apos │ └── apps │ └── index.js ├── test ├── package.json └── test.js ├── .stylelintrc ├── CHANGELOG.md ├── LICENSE.md ├── package.json ├── .circleci └── config.yml ├── index.js ├── lib ├── excel.js └── export.js └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore MacOS X metadata forks (fusefs) 2 | ._* 3 | package-lock.json 4 | *.DS_Store 5 | node_modules 6 | # Never commit a CSS map file, anywhere 7 | *.css.map 8 | 9 | .vscode 10 | # vim swp files 11 | .*.sw* 12 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ "apostrophe" ], 3 | "rules": { 4 | "no-var": "error" 5 | }, 6 | "overrides": [ 7 | { 8 | "files": [ 9 | "**/ui/**/*.js" 10 | ], 11 | "globals": { 12 | "apos": true 13 | } 14 | } 15 | ] 16 | } -------------------------------------------------------------------------------- /ui/apos/apps/index.js: -------------------------------------------------------------------------------- 1 | export default () => { 2 | // Allow Apostrophe to create the bus first 3 | setTimeout(function() { 4 | apos.bus && apos.bus.$on('export-download', (event) => { 5 | if (event.url) { 6 | window.open(event.url, '_blank'); 7 | } 8 | }); 9 | }, 0); 10 | }; 11 | -------------------------------------------------------------------------------- /test/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "-//": "This package.json file is not actually installed.", 3 | "/-/": "Apostrophe requires that all npm modules to be loaded by moog", 4 | "//-": "exist in package.json at project level, which for a test is here", 5 | "dependencies": { 6 | "apostrophe": "^3.4.0", 7 | "@apostrophecms/piece-type-exporter": "git://github.com/apostrophecms/piece-type-exporter.git" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /.stylelintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "stylelint-config-apostrophe", 3 | "rules": { 4 | "scale-unlimited/declaration-strict-value": null, 5 | "scss/at-import-partial-extension": null, 6 | "scss/at-mixin-named-arguments": null, 7 | "scss/dollar-variable-first-in-block": null, 8 | "scss/dollar-variable-pattern": null, 9 | "scss/selector-nest-combinators": null, 10 | "scss/no-duplicate-mixins": null, 11 | "property-no-vendor-prefix": [ 12 | true, 13 | { 14 | "ignoreProperties": ["appearance"] 15 | } 16 | ] 17 | } 18 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 1.1.0 (2024-07-10) 4 | 5 | * Deprecate module. 6 | 7 | ## 1.0.1 (2024-01-25) 8 | 9 | * Change label to "Export as CSV" to avoid confusion with the new [`@apostrophecms/import-export`](https://github.com/apostrophecms/import-export) module. 10 | 11 | ## 1.0.0 - 2023-01-16 12 | 13 | * Declared stable after many quiet moons of no code changes. 14 | 15 | ## 1.0.0-beta.1 - 2021-12-08 16 | 17 | * Renames methods to decrease potential for conflicts with other modules. 18 | 19 | ## 1.0.0-beta 20 | 21 | * Initial release. Adds exporter features to pieces in Apostrophe 3 projects when configured. 22 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 Apostrophe Technologies 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@apostrophecms/piece-type-exporter", 3 | "version": "1.1.0", 4 | "description": "An Apostrophe 3 module to support exporting piece data", 5 | "main": "index.js", 6 | "scripts": { 7 | "lint": "npm run eslint", 8 | "eslint": "eslint .", 9 | "test": "npm audit --production && npm run lint && mocha" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/apostrophecms/piece-type-exporter.git" 14 | }, 15 | "homepage": "https://github.com/apostrophecms/piece-type-exporter#readme", 16 | "author": "Apostrophe Technologies", 17 | "license": "MIT", 18 | "devDependencies": { 19 | "apostrophe": "^3.8.0", 20 | "eslint": "^7.9.0", 21 | "eslint-config-apostrophe": "^3.4.0", 22 | "eslint-config-standard": "^14.1.1", 23 | "eslint-plugin-import": "^2.22.0", 24 | "eslint-plugin-node": "^11.1.0", 25 | "eslint-plugin-promise": "^4.2.1", 26 | "eslint-plugin-standard": "^4.0.1", 27 | "mocha": "^7.1.2", 28 | "stylelint": "^13.7.1", 29 | "stylelint-config-apostrophe": "^1.0.0" 30 | }, 31 | "dependencies": { 32 | "csv-stringify": "^5.6.5", 33 | "xlsx": "^0.17.3" 34 | }, 35 | "publishConfig": { 36 | "access": "public" 37 | } 38 | } -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build-node14-mongo5: 4 | docker: 5 | - image: circleci/node:14-browsers 6 | - image: mongo:5.0 7 | steps: 8 | - checkout 9 | - run: 10 | name: update-npm 11 | command: 'sudo npm install -g npm@7' 12 | - restore_cache: 13 | key: dependency-cache-{{ checksum "package.json" }} 14 | - run: 15 | name: install-npm-wee 16 | command: npm install 17 | - save_cache: 18 | key: dependency-cache-{{ checksum "package.json" }} 19 | paths: 20 | - ./node_modules 21 | - run: 22 | name: test 23 | command: npm test 24 | build-node14-mongo4: 25 | docker: 26 | - image: circleci/node:14-browsers 27 | - image: mongo:4.4 28 | steps: 29 | - checkout 30 | - run: 31 | name: update-npm 32 | command: 'sudo npm install -g npm@6' 33 | - restore_cache: 34 | key: dependency-cache-{{ checksum "package.json" }} 35 | - run: 36 | name: install-npm-wee 37 | command: npm install 38 | - save_cache: 39 | key: dependency-cache-{{ checksum "package.json" }} 40 | paths: 41 | - ./node_modules 42 | - run: 43 | name: test 44 | command: npm test 45 | build-node12: 46 | docker: 47 | - image: circleci/node:12-browsers 48 | - image: mongo:3.6.11 49 | steps: 50 | - checkout 51 | - run: 52 | name: update-npm 53 | command: "sudo npm install -g npm" 54 | - restore_cache: 55 | key: dependency-cache-{{ .Branch }}-{{ checksum "package-lock.json" }} 56 | - run: 57 | name: install-npm-wee 58 | command: npm install 59 | - save_cache: 60 | key: dependency-cache-{{ .Branch }}-{{ checksum "package-lock.json" }} 61 | paths: 62 | - ./node_modules 63 | - run: 64 | name: test 65 | command: npm test 66 | workflows: 67 | version: 2 68 | build: 69 | jobs: 70 | - build-node14-mongo5 71 | - build-node14-mongo4 72 | - build-node12 -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const stringify = require('csv-stringify'); 2 | const fs = require('fs'); 3 | 4 | module.exports = { 5 | improve: '@apostrophecms/piece-type', 6 | batchOperations (self) { 7 | if (!self.options.export) { 8 | return {}; 9 | } 10 | 11 | return { 12 | add: { 13 | export: { 14 | label: 'Export as CSV', 15 | route: '/export', 16 | messages: { 17 | progress: 'Exporting {{ type }}...' 18 | }, 19 | requestOptions: { 20 | extension: 'csv' 21 | }, 22 | modalOptions: { 23 | title: 'Export {{ type }}', 24 | description: 'Are you sure you want to export {{ count }} {{ type }}', 25 | confirmationButton: 'Yes, export content' 26 | } 27 | } 28 | }, 29 | group: { 30 | more: { 31 | icon: 'dots-vertical-icon', 32 | operations: [ 'export' ] 33 | } 34 | } 35 | }; 36 | }, 37 | init (self) { 38 | if (!self.options.export) { 39 | return; 40 | } 41 | 42 | self.exportFormats = { 43 | csv: { 44 | label: 'CSV (comma-separated values)', 45 | output: function (filename) { 46 | const out = stringify({ header: true }); 47 | out.pipe(fs.createWriteStream(filename)); 48 | return out; 49 | } 50 | }, 51 | tsv: { 52 | label: 'TSV (tab-separated values)', 53 | output: function (filename) { 54 | const out = stringify({ 55 | header: true, 56 | delimiter: '\t' 57 | }); 58 | out.pipe(fs.createWriteStream(filename)); 59 | return out; 60 | } 61 | }, 62 | xlsx: require('./lib/excel.js')(self), 63 | ...(self.options.exportFormats || {}) 64 | }; 65 | }, 66 | methods (self) { 67 | if (!self.options.export) { 68 | return {}; 69 | } 70 | 71 | return { 72 | ...require('./lib/export')(self) 73 | }; 74 | }, 75 | apiRoutes (self) { 76 | if (!self.options.export) { 77 | return {}; 78 | } 79 | 80 | return { 81 | post: { 82 | export (req) { 83 | if (!Array.isArray(req.body._ids)) { 84 | throw self.apos.error('invalid'); 85 | } 86 | // Reassigning this since it is referenced off of req elsewhere. 87 | req.body._ids = self.apos.launder.ids(req.body._ids); 88 | 89 | const extension = self.apos.launder.string(req.body.extension); 90 | const batchSize = typeof self.options.export === 'object' && 91 | self.apos.launder.integer(self.options.export.batchSize); 92 | const expiration = typeof self.options.export === 'object' && 93 | self.apos.launder.integer(self.options.export.expiration); 94 | 95 | if (!self.exportFormats[extension]) { 96 | throw self.apos.error('invalid'); 97 | } 98 | 99 | // Add the piece type label to req.body for notifications. 100 | req.body.type = req.body._ids.length === 1 ? self.options.label : self.options.pluralLabel; 101 | 102 | const format = self.exportFormats[extension]; 103 | 104 | return self.apos.modules['@apostrophecms/job'].run( 105 | req, 106 | function (req, reporting) { 107 | return self.exportRun(req, reporting, { 108 | extension, 109 | format, 110 | batchSize, 111 | expiration 112 | }); 113 | }, 114 | {} 115 | ); 116 | } 117 | } 118 | }; 119 | } 120 | }; 121 | -------------------------------------------------------------------------------- /lib/excel.js: -------------------------------------------------------------------------------- 1 | // http://talesofthefluxfox.com/2016/10/07/writing-to-xlsx-spreadsheets-in-node-js/ 2 | 3 | const ALPHA = [ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 4 | 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z' 5 | ]; 6 | 7 | const XLSX = require('xlsx'); 8 | 9 | function objectsToWorkbook (objects, selectedFields, spreadsheetName) { 10 | const rowsOfData = objects; 11 | let lineNum = 1; 12 | const worksheetColumns = []; 13 | 14 | selectedFields.forEach(function () { 15 | worksheetColumns.push({ 16 | wch: 25 17 | }); 18 | }); 19 | 20 | const workbook = { 21 | SheetNames: [ spreadsheetName ], 22 | Sheets: { 23 | [spreadsheetName]: 24 | { 25 | '!ref': 'A1:', 26 | '!cols': worksheetColumns 27 | } 28 | } 29 | }; 30 | 31 | for (let i = 0; i < selectedFields.length; i++) { 32 | worksheetColumns.push( 33 | { 34 | wch: 25 35 | }); 36 | const currentCell = _calculateCurrentCellReference(i, lineNum); 37 | workbook.Sheets[spreadsheetName][currentCell] = { 38 | t: 's', 39 | v: selectedFields[i], 40 | s: { 41 | font: 42 | { 43 | bold: true 44 | } 45 | } 46 | }; 47 | } 48 | lineNum++; 49 | rowsOfData.forEach(function (offer) { 50 | for (let i = 0; i < selectedFields.length; i++) { 51 | const displayValue = offer[selectedFields[i]]; 52 | const currentCell = _calculateCurrentCellReference(i, lineNum); 53 | workbook.Sheets[spreadsheetName][currentCell] = { 54 | t: 's', 55 | v: displayValue, 56 | s: { 57 | font: { 58 | sz: '11', 59 | bold: false 60 | }, 61 | alignment: { 62 | wrapText: true, 63 | vertical: 'top' 64 | }, 65 | fill: { 66 | fgColor: { 67 | rgb: 'ffffff' 68 | } 69 | }, 70 | border: { 71 | left: { 72 | style: 'thin', 73 | color: { 74 | auto: 1 75 | } 76 | }, 77 | right: 78 | { 79 | style: 'thin', 80 | color: { 81 | auto: 1 82 | } 83 | }, 84 | top: 85 | { 86 | style: 'thin', 87 | color: { 88 | auto: 1 89 | } 90 | }, 91 | bottom: 92 | { 93 | style: 'thin', 94 | color: { 95 | auto: 1 96 | } 97 | } 98 | } 99 | } 100 | }; 101 | } 102 | lineNum++; 103 | }); 104 | const lastColumnInSheet = selectedFields.length - 1; 105 | const endOfRange = _calculateCurrentCellReference(lastColumnInSheet, lineNum); 106 | workbook.Sheets[spreadsheetName]['!ref'] += endOfRange; 107 | return workbook; 108 | } 109 | 110 | function _calculateCurrentCellReference (index, lineNumber) { 111 | return (index > 25) ? ALPHA[Math.floor((index / 26) - 1)] + ALPHA[index % 26] + lineNumber : ALPHA[index] + lineNumber; 112 | } 113 | 114 | module.exports = function (self) { 115 | return { 116 | label: 'Excel (.xlsx)', 117 | output: function (filename, objects, callback) { 118 | try { 119 | const fields = self.schema.map(self.schema, field => field.name); 120 | const label = self.pluralLabel || self.label || self.__meta.name; 121 | const workbook = objectsToWorkbook(objects, fields, label); 122 | 123 | XLSX.writeFile(workbook, filename); 124 | } catch (e) { 125 | return callback(e); 126 | } 127 | return setImmediate(callback); 128 | } 129 | }; 130 | }; 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![CircleCI](https://circleci.com/gh/apostrophecms/piece-type-exporter/tree/main.svg?style=svg)](https://circleci.com/gh/apostrophecms/piece-type-exporter/tree/main) 2 | [![Chat on Discord](https://img.shields.io/discord/517772094482677790.svg)](https://chat.apostrophecms.org) 3 | 4 | # Apostrophe Pieces Exporter 5 | 6 | > ⚠️ **NOTE:** This module is deprecated, its functionality has been incorporated into [@apostrophecms/import-export](https://github.com/apostrophecms/import-export). 7 | 8 | This module adds an optional export feature to all piece type modules in an [Apostrophe](https://apostrophecms.com) project. This feature enables exporting *published* pieces of piece types where it is configured. Requires Apostrophe 3. 9 | 10 | ## Installation 11 | 12 | ```bash 13 | npm install @apostrophecms/piece-type-exporter 14 | ``` 15 | 16 | ## Use 17 | 18 | ### Initialization 19 | 20 | Configure `@apostrophecms/piece-type-exporter` and the form widgets in `app.js`. 21 | 22 | ```javascript 23 | require('apostrophe')({ 24 | shortName: 'my-project', 25 | modules: { 26 | // The exporter module 27 | '@apostrophecms/piece-type-exporter': {}, 28 | // A piece type that you want to make exportable 29 | 'article': { 30 | options: { 31 | export: true 32 | } 33 | } 34 | } 35 | }); 36 | ``` 37 | 38 | The Pieces Exporter module improves all piece types in the site to add export functionality to them. To enable that functionality, **you must add the `export: true` option on the appropriate piece type(s)**. The example above demonstrates doing this in the `app.js` file. More often it will be preferable to set this option in the module's `index.js` file. 39 | 40 | ```javascript 41 | // modules/article/index.js 42 | module.exports = { 43 | extend: '@apostrophecms/piece-type', 44 | options: { 45 | label: 'Article', 46 | pluralLabel: 'Articles', 47 | export: true // 👈 Adding the export option. 48 | }, 49 | // Other properties... 50 | } 51 | ``` 52 | 53 | ### Additional options 54 | 55 | #### `omitFields` 56 | 57 | You can specify properties to omit from exported documents with this option. The `export` option on the exportable piece type becomes an object with an `omitFields` property. `omitFields` takes an array of field names to omit. 58 | 59 | For example, if you wanted to exclude the `archive` field from the export, you would configure your piece like this: 60 | 61 | ```javascript 62 | // modules/article/index.js 63 | module.exports = { 64 | extend: '@apostrophecms/piece-type', 65 | options: { 66 | label: 'Article', 67 | pluralLabel: 'Articles', 68 | export: { 69 | omitFields: [ 'archive' ] 70 | } 71 | }, 72 | // Other properties... 73 | } 74 | ``` 75 | 76 | #### `expiration` 77 | 78 | By default, exported files are automatically deleted after one hour. You can change this time span by setting an `expiration` property on the `export` option. It should be set to an integer representing the number of milliseconds until expiration. 79 | 80 | ```javascript 81 | // modules/article/index.js 82 | module.exports = { 83 | extend: '@apostrophecms/piece-type', 84 | options: { 85 | label: 'Article', 86 | pluralLabel: 'Articles', 87 | export: { 88 | // 👇 Set to expire after two hours. Tip: Writing as an expression can 89 | // help make it clearer to other people. 90 | expiration: 1000 * 60 * 120 91 | } 92 | }, 93 | // Other properties... 94 | } 95 | ``` 96 | 97 | #### Export areas as plain text with `exportPlainText` 98 | 99 | By default, this module exports areas as rich text. You will receive simple HTML markup corresponding to any rich text widgets present in those areas. 100 | 101 | If you prefer, you can set the `exportPlainText: true` option *on an `area` schema field* to export it as plain text. In this case, tags are stripped and entities are un-escaped. 102 | 103 | ```javascript 104 | // modules/article/index.js 105 | module.exports = { 106 | extend: '@apostrophecms/piece-type', 107 | options: { 108 | label: 'Article', 109 | pluralLabel: 'Articles', 110 | export: true 111 | }, 112 | fields: { 113 | add: { 114 | textArea: { 115 | type: 'area', 116 | widgets: { 117 | '@apostrophecms/rich-text': {} 118 | }, 119 | options: { 120 | // 👇 The option set to export this area as plain text. 121 | exportPlainText: true 122 | } 123 | } 124 | } 125 | } 126 | } 127 | ``` 128 | -------------------------------------------------------------------------------- /lib/export.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const path = require('path'); 3 | 4 | module.exports = (self) => { 5 | return { 6 | exportCleanup(file) { 7 | try { 8 | fs.unlinkSync(file); 9 | } catch (e) { 10 | self.apos.util.error(e); 11 | } 12 | }, 13 | async exportWriteBatch (req, out, _ids, lastId = '', reporting, options) { 14 | let batch; 15 | 16 | try { 17 | batch = await self.find(req, { 18 | $and: [ 19 | { 20 | _id: { $gt: lastId } 21 | }, 22 | { 23 | _id: { $in: _ids } 24 | } 25 | ] 26 | }) 27 | .sort({ _id: 1 }) 28 | .limit(options.batchSize || 100).toArray(); 29 | } catch (error) { 30 | self.apos.util.error(error); 31 | throw self.apos.error('notfound'); 32 | } 33 | 34 | if (!batch.length) { 35 | return null; 36 | } 37 | 38 | lastId = batch[batch.length - 1]._id; 39 | 40 | for (const piece of batch) { 41 | try { 42 | const record = await self.exportRecord(req, piece); 43 | 44 | reporting.success(); 45 | out.write(record); 46 | } catch (error) { 47 | self.apos.util.error('exportRecord error', piece._id, error); 48 | reporting.failure(); 49 | } 50 | } 51 | 52 | return lastId; 53 | }, 54 | async exportRun (req, reporting, options) { 55 | if (typeof reporting.setTotal === 'function') { 56 | reporting.setTotal(req.body._ids.length); 57 | } 58 | 59 | const extension = options.extension; 60 | const format = options.format; 61 | 62 | const filename = `${self.apos.util.generateId()}-export.${extension}`; 63 | const filepath = path.join(self.apos.attachment.uploadfs.getTempPath(), filename); 64 | 65 | let out; 66 | let data; 67 | let reported = false; 68 | 69 | if (format.output.length === 1) { 70 | // Now kick off the stream processing 71 | out = format.output(filepath); 72 | } else { 73 | // Create a simple writable stream that just buffers up the objects. Allows the simpler type of output function to drive the same methods that otherwise write to an output stream. 74 | data = []; 75 | out = { 76 | write: function (o) { 77 | data.push(o); 78 | }, 79 | end: function () { 80 | return format.output(filepath, data, function (err) { 81 | if (err) { 82 | out.emit('error', err); 83 | } else { 84 | out.emit('finish'); 85 | } 86 | }); 87 | }, 88 | on: function (name, fn) { 89 | out.listeners[name] = out.listeners[name] || []; 90 | out.listeners[name].push(fn); 91 | }, 92 | emit: function (name, value) { 93 | (out.listeners[name] || []).forEach(function (fn) { 94 | fn(value); 95 | }); 96 | }, 97 | listeners: {} 98 | }; 99 | } 100 | 101 | const result = new Promise((resolve, reject) => { 102 | out.on('error', function (err) { 103 | if (!reported) { 104 | reported = true; 105 | self.exportCleanup(filepath); 106 | self.apos.util.error(err); 107 | 108 | return reject(self.apos.error('error')); 109 | } 110 | }); 111 | 112 | out.on('finish', async function () { 113 | if (!reported) { 114 | reported = true; 115 | // Must copy it to uploadfs, the server that created it 116 | // and the server that delivers it might be different 117 | const filename = `${self.apos.util.generateId()}.${extension}`; 118 | const downloadPath = path.join('/exports', filename); 119 | 120 | const copyIn = require('util').promisify(self.apos.attachment.uploadfs.copyIn); 121 | 122 | try { 123 | await copyIn(filepath, downloadPath); 124 | } catch (error) { 125 | self.exportCleanup(filepath); 126 | return reject(error); 127 | } 128 | 129 | const downloadUrl = self.apos.attachment.uploadfs.getUrl() + downloadPath; 130 | 131 | reporting.setResults({ 132 | url: downloadUrl 133 | }); 134 | 135 | await self.apos.notification.trigger(req, 'Exported {{ count }} {{ type }}.', { 136 | interpolate: { 137 | count: req.body._ids.length, 138 | type: req.body.type || req.t('apostrophe:document') 139 | }, 140 | dismiss: true, 141 | icon: 'database-export-icon', 142 | type: 'success', 143 | event: { 144 | name: 'export-download', 145 | data: { 146 | url: downloadUrl 147 | } 148 | } 149 | }); 150 | 151 | self.exportCleanup(filepath); 152 | 153 | const expiration = self.options.export && self.options.export.expiration; 154 | 155 | // Report is available for one hour 156 | setTimeout(function () { 157 | self.apos.attachment.uploadfs.remove(downloadPath, function (err) { 158 | if (err) { 159 | self.apos.util.error(err); 160 | } 161 | }); 162 | }, expiration || 1000 * 60 * 60); 163 | 164 | return resolve(null); 165 | } 166 | }); 167 | 168 | }); 169 | 170 | let lastId = ''; 171 | const _ids = req.body._ids.map(id => id.replace(':draft', ':published')); 172 | const pubReq = req.clone({ mode: 'published' }); 173 | 174 | do { 175 | lastId = await self.exportWriteBatch(pubReq, out, _ids, lastId, reporting, { 176 | ...options 177 | }); 178 | } while (lastId); 179 | 180 | self.closeExportStream(out); 181 | 182 | return result; 183 | }, 184 | async exportRecord (req, piece) { 185 | const schema = self.schema; 186 | const record = {}; 187 | // Schemas don't have built-in exporters, for strings or otherwise. 188 | // Follow a format that reverses well if fed back to our importer 189 | // (although the importer can't accept an attachment via URL yet, 190 | // that is a plausible upgrade). Export schema fields only, 191 | // plus _id. 192 | record._id = piece._id; 193 | 194 | schema.forEach(function (field) { 195 | if (self.options.export.omitFields && self.options.export.omitFields.includes(field.name)) { 196 | return; 197 | } 198 | let value = piece[field.name]; 199 | if ((typeof value) === 'object') { 200 | if (field.type === 'relationship') { 201 | value = (value || []).map(function (item) { 202 | return item.title; 203 | }).join(','); 204 | } else if (field.type === 'attachment') { 205 | value = self.apos.attachment.url(value); 206 | } else if ((field.type === 'area')) { 207 | if (field.options && field.options.exportPlainText) { 208 | value = self.apos.area.plaintext(value); 209 | } else { 210 | value = self.apos.area.richText(value); 211 | } 212 | } else { 213 | value = ''; 214 | } 215 | } else { 216 | if (value) { 217 | value = value.toString(); 218 | } 219 | } 220 | record[field.name] = value; 221 | }); 222 | 223 | await self.beforeExport(req, piece, record); 224 | 225 | return record; 226 | }, 227 | closeExportStream(stream) { 228 | stream.end(); 229 | }, 230 | beforeExport (req, piece, record) { 231 | return record; 232 | } 233 | }; 234 | }; 235 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const testUtil = require('apostrophe/test-lib/test'); 3 | 4 | describe('Pieces Exporter', function () { 5 | let apos; 6 | 7 | this.timeout(10000); 8 | 9 | after(async function () { 10 | testUtil.destroy(apos); 11 | }); 12 | 13 | it('should improve piece types on the apos object', async function () { 14 | apos = await testUtil.create({ 15 | shortname: 'test-exporter', 16 | testModule: true, 17 | modules: { 18 | '@apostrophecms/express': { 19 | options: { 20 | port: 4242, 21 | trustProxy: true, 22 | apiKeys: { 23 | testKey: { 24 | role: 'admin' 25 | } 26 | }, 27 | csrf: { 28 | exceptions: [ 29 | '/api/v1/@apostrophecms/article/export' 30 | ] 31 | }, 32 | session: { secret: 'test-the-exporter' } 33 | } 34 | }, 35 | '@apostrophecms/piece-type-exporter': { 36 | options: { 37 | // A meaningless option to confirm the piece types are "improved." 38 | exporterActive: true 39 | } 40 | }, 41 | article: { 42 | extend: '@apostrophecms/piece-type', 43 | options: { 44 | export: { 45 | batchSize: 10, 46 | expiration: 5000 47 | } 48 | }, 49 | fields: { 50 | add: { 51 | richText: { 52 | type: 'area', 53 | widgets: { 54 | '@apostrophecms/rich-text': {} 55 | } 56 | } 57 | } 58 | } 59 | }, 60 | product: { 61 | extend: '@apostrophecms/piece-type', 62 | options: { 63 | export: { 64 | omitFields: [ 'secret' ], 65 | batchSize: 10, 66 | expiration: 5000 67 | } 68 | }, 69 | fields: { 70 | add: { 71 | secret: { 72 | type: 'string' 73 | }, 74 | plainText: { 75 | type: 'area', 76 | widgets: { 77 | '@apostrophecms/rich-text': {} 78 | }, 79 | options: { 80 | exportPlainText: true 81 | } 82 | } 83 | } 84 | } 85 | } 86 | } 87 | }); 88 | 89 | const articleModule = apos.modules.article; 90 | const productModule = apos.modules.product; 91 | 92 | assert(articleModule.__meta.name === 'article'); 93 | assert(typeof articleModule.options.export === 'object'); 94 | assert(productModule.__meta.name === 'product'); 95 | assert(typeof productModule.options.export === 'object'); 96 | // Pieces exporter is working and improving piece types. 97 | assert(productModule.options.exporterActive === true); 98 | }); 99 | 100 | const richText = '

This is rich text.

'; 101 | let _ids1 = []; 102 | 103 | it('can insert many test articles', async function () { 104 | const req = apos.task.getReq(); 105 | 106 | const data = { 107 | richText: { 108 | metaType: 'area', 109 | items: [ 110 | { 111 | type: '@apostrophecms/rich-text', 112 | metaType: 'widget', 113 | content: richText 114 | } 115 | ] 116 | } 117 | }; 118 | 119 | const promises = []; 120 | 121 | for (let i = 1; i <= 50; i++) { 122 | promises.push(insert(req, apos.modules.article, 'article', data, i)); 123 | } 124 | 125 | const inserted = await Promise.all(promises); 126 | _ids1 = inserted.map(doc => doc._id); 127 | 128 | assert(inserted.length === 50); 129 | assert(!!inserted[0]._id); 130 | }); 131 | 132 | it('can export the articles as a CSV', async function () { 133 | const req = apos.task.getReq(); 134 | req.body = req.body || {}; 135 | req.body._ids = _ids1; 136 | 137 | let good = 0; 138 | let bad = 0; 139 | let results; 140 | 141 | const reporting = { 142 | success: function () { 143 | good++; 144 | }, 145 | failure: function () { 146 | bad++; 147 | }, 148 | setResults: function (_results) { 149 | results = _results; 150 | } 151 | }; 152 | 153 | try { 154 | await apos.modules.article.exportRun(req, reporting, { 155 | extension: 'csv', 156 | format: apos.modules.article.exportFormats.csv 157 | }); 158 | } catch (error) { 159 | assert(!error); 160 | } 161 | 162 | assert(results.url); 163 | assert(good === 50); 164 | assert(!bad); 165 | 166 | const exportedArticles = await apos.http.get(results.url); 167 | assert(exportedArticles.match(/,article #00001,/)); 168 | assert(exportedArticles.indexOf(`,${richText}`) !== -1); 169 | }); 170 | 171 | const plainText = 'This is plain text.'; 172 | const plainTextWrapped = `

${plainText}

`; 173 | const secret = 'hide-me'; 174 | let _ids2 = []; 175 | it('can insert many test products', async function () { 176 | const req = apos.task.getReq(); 177 | 178 | const data = { 179 | plainText: { 180 | metaType: 'area', 181 | items: [ 182 | { 183 | type: '@apostrophecms/rich-text', 184 | metaType: 'widget', 185 | content: plainTextWrapped 186 | } 187 | ] 188 | }, 189 | secret 190 | }; 191 | 192 | const promises = []; 193 | 194 | for (let i = 1; i <= 30; i++) { 195 | promises.push(insert(req, apos.modules.product, 'product', data, i)); 196 | } 197 | 198 | const inserted = await Promise.all(promises); 199 | _ids2 = inserted.map(doc => doc._id); 200 | 201 | assert(inserted.length === 30); 202 | assert(!!inserted[0]._id); 203 | }); 204 | 205 | let exportedProducts; 206 | 207 | it('can export the products as a CSV', async function () { 208 | const req = apos.task.getReq(); 209 | req.body = req.body || {}; 210 | req.body._ids = _ids2; 211 | 212 | let good = 0; 213 | let bad = 0; 214 | let results; 215 | 216 | const reporting = { 217 | success: function () { 218 | good++; 219 | }, 220 | failure: function () { 221 | bad++; 222 | }, 223 | setResults: function (_results) { 224 | results = _results; 225 | } 226 | }; 227 | 228 | try { 229 | await apos.modules.product.exportRun(req, reporting, { 230 | extension: 'csv', 231 | format: apos.modules.product.exportFormats.csv 232 | }); 233 | } catch (error) { 234 | assert(!error); 235 | } 236 | 237 | assert(results.url); 238 | assert(good === 30); 239 | assert(!bad); 240 | 241 | exportedProducts = await apos.http.get(results.url); 242 | assert(exportedProducts.match(/,product #00001,/)); 243 | }); 244 | 245 | it('can convert product rich text to plain text', async function () { 246 | assert(exportedProducts.indexOf(`,${plainTextWrapped}`) === -1); 247 | assert(exportedProducts.indexOf(`,${plainText}`) !== -1); 248 | }); 249 | 250 | it('can omit the secret product field', async function () { 251 | assert(exportedProducts.indexOf(`,${secret}`) === -1); 252 | }); 253 | 254 | // API Route test 255 | let jobInfo; 256 | 257 | it('can get a job ID from the export route', async function () { 258 | jobInfo = await apos.http.post('/api/v1/article/export?apikey=testKey', { 259 | body: { 260 | extension: 'csv', 261 | _ids: _ids1 262 | } 263 | }); 264 | 265 | assert(jobInfo.jobId); 266 | }); 267 | 268 | it('can eventually get the exported CSV url back from the job', async function () { 269 | const complete = await checkJob(jobInfo.jobId); 270 | 271 | assert(complete && complete.results.url); 272 | 273 | async function checkJob (id) { 274 | let job; 275 | 276 | try { 277 | job = await apos.http.get(`/api/v1/@apostrophecms/job/${id}?apikey=testKey`, {}); 278 | } catch (error) { 279 | assert(!error); 280 | return null; 281 | } 282 | 283 | if (!job.ended) { 284 | await new Promise(resolve => { 285 | setTimeout(resolve, 2000); 286 | }); 287 | 288 | return checkJob(id); 289 | } 290 | 291 | return job; 292 | } 293 | }); 294 | }); 295 | 296 | function padInteger (i, places) { 297 | let s = i + ''; 298 | while (s.length < places) { 299 | s = '0' + s; 300 | } 301 | return s; 302 | } 303 | 304 | async function insert (req, pieceModule, title, data, i) { 305 | const docData = Object.assign(pieceModule.newInstance(), { 306 | title: `${title} #${padInteger(i, 5)}`, 307 | slug: `${title}-${padInteger(i, 5)}`, 308 | ...data 309 | }); 310 | 311 | return pieceModule.insert(req, docData); 312 | }; 313 | --------------------------------------------------------------------------------