├── .eslintrc
├── .gitignore
├── CHANGELOG.md
├── .stylelintrc
├── package.json
├── LICENSE.md
├── lib
└── findJoins.js
├── README.md
└── index.js
/.eslintrc:
--------------------------------------------------------------------------------
1 | {
2 | "extends": [ "apostrophe" ]
3 | }
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # Ignore MacOS X metadata forks (fusefs)
2 | ._*
3 | package-lock.json
4 | *.DS_Store
5 | node_modules
6 |
7 | # Never commit a CSS map file, anywhere
8 | *.css.map
9 |
10 | # vim swp files
11 | .*.sw*
12 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Changelog
2 |
3 | ## 1.0.0-alpha.3
4 |
5 | * Removes `apostrophe` as a peer dependency.
6 |
7 | ## 1.0.0-alpha.2
8 |
9 | * Bug fix for the case where a single locale is synced with `--workflow-locale` to ensure committing is possible afterwards, and a check for a common user error at startup.
10 |
11 | ## 1.0.0-alpha
12 |
13 | * Initial release.
14 |
--------------------------------------------------------------------------------
/.stylelintrc:
--------------------------------------------------------------------------------
1 | {
2 | "extends": "stylelint-config-apostrophe",
3 | "rules": {
4 | "scale-unlimited/declaration-strict-value": null,
5 | "scss/at-import-partial-extension": null,
6 | "scss/at-mixin-named-arguments": null,
7 | "scss/dollar-variable-first-in-block": null,
8 | "scss/dollar-variable-pattern": null,
9 | "scss/selector-nest-combinators": null,
10 | "scss/no-duplicate-mixins": null,
11 | "property-no-vendor-prefix": [
12 | true,
13 | {
14 | "ignoreProperties": ["appearance"]
15 | }
16 | ]
17 | }
18 | }
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "@apostrophecms/sync-content",
3 | "version": "1.0.0-alpha.3",
4 | "description": "Content sync utility for ApostropheCMS sites",
5 | "main": "index.js",
6 | "scripts": {
7 | "test": "echo \"Error: no test specified\" && exit 1"
8 | },
9 | "repository": {
10 | "type": "git",
11 | "url": "git+https://github.com/apostrophecms/sync-content.git"
12 | },
13 | "keywords": [
14 | "backup",
15 | "apostrophecms"
16 | ],
17 | "author": "Apostrophe Technologies",
18 | "license": "MIT",
19 | "bugs": {
20 | "url": "https://github.com/apostrophecms/sync-content/issues"
21 | },
22 | "homepage": "https://github.com/apostrophecms/sync-content#readme",
23 | "dependencies": {
24 | "bson": "^4.6.1",
25 | "compression": "^1.7.4",
26 | "lodash": "^4.17.21",
27 | "node-fetch": "^2.6.7",
28 | "qs": "^6.10.3",
29 | "stream-json": "^1.7.3"
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | Copyright (c) 2021 Apostrophe Technologies
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
4 |
5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
6 |
7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
8 |
--------------------------------------------------------------------------------
/lib/findJoins.js:
--------------------------------------------------------------------------------
1 | const _ = require('lodash');
2 |
3 | // Code borrowed verbatim from the workflow module
4 |
5 | module.exports = (self, options) => {
6 | // Given a doc, find all joins related to that doc: those in its own schema,
7 | // or in the schemas of its own widgets. These are returned as an array of
8 | // objects with `doc` and `field` properties, where `doc` may be the doc
9 | // itself or a widget within it, and `field` is the schema field definition
10 | // of the join. Only forward joins are returned.
11 |
12 | self.findJoinsInDoc = function(doc) {
13 | return self.findJoinsInDocSchema(doc).concat(self.findJoinsInAreas(doc));
14 | };
15 |
16 | // Given a doc, invoke `findJoinsInSchema` with that doc and its schema according to
17 | // its doc type manager, and return the result.
18 |
19 | self.findJoinsInDocSchema = function(doc) {
20 | if (!doc.type) {
21 | // Cannot determine schema, so we cannot fetch joins;
22 | // don't crash, so we behave reasonably if very light
23 | // projections are present
24 | return [];
25 | }
26 | var schema = self.apos.docs.getManager(doc.type).schema;
27 | return self.findJoinsInSchema(doc, schema);
28 | };
29 |
30 | // Given a doc, find joins in the schemas of widgets contained in the
31 | // areas of that doc and return an array in which each element is an object with
32 | // `doc` and `field` properties. `doc` is a reference to the individual widget
33 | // in question, and `field` is the join field definition for that widget.
34 | // Only forward joins are returned.
35 |
36 | self.findJoinsInAreas = function(doc) {
37 | var widgets = [];
38 | self.apos.areas.walk(doc, function(area, dotPath) {
39 | widgets = widgets.concat(area.items);
40 | });
41 | var joins = [];
42 | _.each(widgets, function(widget) {
43 | if (!widget.type) {
44 | // Don't crash on bad data or strange projections etc.
45 | return;
46 | }
47 | var manager = self.apos.areas.getWidgetManager(widget.type);
48 | if (!manager) {
49 | // We already warn about obsolete widgets elsewhere, don't crash
50 | return;
51 | }
52 | var schema = manager.schema;
53 | joins = joins.concat(self.findJoinsInSchema(widget, schema));
54 | });
55 | return joins;
56 | };
57 |
58 | // Given a doc (or widget) and a schema, find joins described by that schema and
59 | // return an array in which each element is an object with
60 | // `doc`, `field` and `value` properties. `doc` is a reference to the doc
61 | // passed to this method, `field` is a field definition, and `value` is the
62 | // value of the join if available (the doc was loaded with joins).
63 | //
64 | // Only forward joins are returned.
65 |
66 | self.findJoinsInSchema = function(doc, schema) {
67 | var fromArrays = [];
68 | return _.map(
69 | _.filter(
70 | schema, function(field) {
71 | if ((field.type === 'joinByOne') || (field.type === 'joinByArray')) {
72 | return true;
73 | }
74 | if (field.type === 'array') {
75 | _.each(doc[field.name] || [], function(doc) {
76 | fromArrays = fromArrays.concat(self.findJoinsInSchema(doc, field.schema));
77 | });
78 | }
79 | if (field.type === 'object' && typeof doc[field.name] === 'object') {
80 | fromArrays = fromArrays.concat(self.findJoinsInSchema(doc[field.name], field.schema));
81 | }
82 | }
83 | ), function(field) {
84 | return { doc: doc, field: field, value: doc[field.name] };
85 | }
86 | ).concat(fromArrays);
87 | };
88 | };
89 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | 
5 |
6 |
7 |
Sync Content for ApostropheCMS
8 |
9 |
10 | The Sync Content module allows syncing ApostropheCMS site content between different server environments, without the need for direct access to remote databases, directories, S3 buckets, etc.
11 |
12 | **Status:** ⚠️ In use, but still an alpha release. Not all planned features are implemented, but those discussed here are available.
13 |
14 | ## Purpose
15 |
16 | This module is useful when migrating content between development, staging and production environments without direct access to the underlying database and media storage. It is also useful when migrating between projects, however bear in mind that the new project must have the same doc and widget types available with the same fields in order to function properly.
17 |
18 | ## Installation
19 |
20 | To install the module, use the command line to run this command in an Apostrophe project's root directory:
21 |
22 | ```
23 | npm install @apostrophecms/sync-content
24 | ```
25 |
26 | ## ⚠️ Warnings
27 |
28 | This tool makes big changes to your database. There are no confirmation prompts in the current command line interface. Syncing "from" staging or production to your local development environment is generally safe, but take care to think about what you are doing.
29 |
30 | ## Usage
31 |
32 | Configure the `@apostrophecms/sync-content` module in the `app.js` file:
33 |
34 | ```javascript
35 | require('apostrophe')({
36 | shortName: 'my-project',
37 | modules: {
38 | '@apostrophecms/sync-content': {
39 | // Our API key, for incoming sync requests
40 | apiKey: 'choose-a-very-secure-random-key',
41 | environments: {
42 | staging: {
43 | label: 'Staging',
44 | url: 'https://mysite.staging.mycompany.com',
45 | // Their API key, for outgoing sync requests
46 | apiKey: 'choose-a-very-secure-random-key'
47 | }
48 | }
49 | }
50 | }
51 | });
52 | ```
53 |
54 | ### Syncing via command line tasks
55 |
56 | #### Syncing from another site
57 |
58 | ```bash
59 | # sync all content from another environment
60 | node app @apostrophecms/sync-content:sync --from=staging
61 |
62 | # Pass a site's base URL and api key directly, bypassing `environments`
63 | node app @apostrophecms/sync-content:sync --from=https://site.com --api-key=xyz
64 |
65 | # sync content of one piece type only, plus any related
66 | # documents. If other content already exists locally, purge it
67 | node app @apostrophecms/sync-content:sync --from=staging --type=article
68 |
69 | # Same, but keep existing content of this type too
70 | node app @apostrophecms/sync-content:sync --from=staging --type=article --keep
71 |
72 | # sync content of one piece type only, without related documents
73 | node app @apostrophecms/sync-content:sync --from=staging --type=article --related=false
74 |
75 | # sync content of one piece type only, matching a query
76 | node app @apostrophecms/sync-content:sync --from=staging --type=article --query=tags[]=blue
77 | ```
78 |
79 | * You must specify `--from` to specify the environment to sync with, as seen in your configuration above, where `staging` is an example. You can also specify the base URL of the other environment directly for `--from`, in which case you must pass `--api-key` as well. At a later date support for `--to` may also be added.
80 | * You may specify `--type=typename` to specify one content type only. This must be a piece type, and must match the `name` option of the type (**not** the module name, unless they are the same).
81 | * When using `--type`, you may also specify `--keep` to keep preexisting pieces whose `_id` does not appear in the synced content. **For data integrity reasons, this is not available when syncing an entire site.**
82 | * By default, syncing a piece type will also sync directly related documents, such as images and other pieces referenced by joins in the document's own schema or those of its own array fields, object fields, and widgets. If you do not want this, specify `--related=false`.
83 | * The `--query` option is best used by observing the query string while on a pieces page with various filters applied. Any valid Apostrophe cursor filter may be used.
84 |
85 | Note that the `--keep`, `--related` and `--query` options are only valid with `--type`. They may be combined with each other.
86 |
87 | ### Syncing via the UI
88 |
89 | Currently no UI is available, however at least some UI functionality is planned.
90 |
91 | ### Security restrictions
92 |
93 | For security reasons, and to avoid chicken and egg problems when using the UI, users and groups are **not** synced. You will have the same users and groups as before the sync operation.
94 |
95 | ### Additional notes
96 |
97 | Syncing a site takes time, especially if the site has media. Get a cup of coffee.
98 |
99 | It is not uncommon to see quite a few warnings about missing attachments at the end, particularly if another image size was added to the project without running the `apostrophe-attachments:rescale` task.
100 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | const fs = require('fs');
2 | const { EJSON } = require('bson');
3 | const util = require('util');
4 | const unlink = util.promisify(fs.unlink);
5 | const fetch = require('node-fetch');
6 | const compression = require('compression');
7 | const pipeline = require('util').promisify(require('stream').pipeline);
8 | const Stream = require('stream');
9 | const { parser } = require('stream-json');
10 | const { streamValues } = require('stream-json/streamers/StreamValues');
11 | const qs = require('qs');
12 |
13 | module.exports = {
14 | construct(self, options) {
15 | require('./lib/findJoins')(self, options);
16 | self.neverCollections = [ 'aposUsersSafe', 'sessions', 'aposCache', 'aposLocks', 'aposNotifications', 'aposBlessings', 'aposDocVersions' ];
17 | self.neverTypes = [ 'apostrophe-user', 'apostrophe-group' ];
18 | self.apos.on('csrfExceptions', function(list) {
19 | // For syncTo and the POST routes
20 | list.push(`${self.action}/content`);
21 | list.push(`${self.action}/uploadfs`);
22 | });
23 | self.addTask('sync', 'Syncs content to or from another server environment', async (apos, argv) => {
24 | const peer = argv.from || argv.to;
25 | if (!peer) {
26 | throw 'You must specify either --from or --to.';
27 | }
28 | if (argv.from && argv.to) {
29 | throw 'You must not specify both --from and --to (one end is always local).';
30 | }
31 | if (argv.type) {
32 | argv.related = self.apos.launder.boolean(argv.related, true);
33 | argv.keep = self.apos.launder.boolean(argv.keep);
34 | argv.query = qs.parse(self.apos.launder.string(argv.query));
35 | } else {
36 | if (argv.related) {
37 | throw '--related not available without --type';
38 | }
39 | if (argv.keep) {
40 | throw '--keep not available without --type';
41 | }
42 | if (argv.query) {
43 | throw '--query not available without --type';
44 | }
45 | }
46 | if (argv.from) {
47 | const env = self.getEnv(argv.from, argv);
48 | await self.syncFrom(env, argv);
49 | } else if (argv.to) {
50 | throw '--to is not yet implemented';
51 | } else {
52 | throw '--from is required';
53 | }
54 | });
55 | self.route('get', 'content', compression({
56 | filter: (req) => true
57 | }), async (req, res) => {
58 | try {
59 | if (!self.options.apiKey) {
60 | throw 'API key not configured';
61 | }
62 | const apiKey = getAuthorizationApiKey(req);
63 | if (apiKey !== self.options.apiKey) {
64 | throw 'Invalid API key';
65 | }
66 | const type = self.apos.launder.string(req.query.type);
67 | const related = self.apos.launder.boolean(req.query.related);
68 | // Naming it req.query.workflowLocale causes it to be stolen by
69 | // the workflow module, so we don't do that
70 | const workflowLocale = self.apos.launder.string(req.query.locale);
71 | const query = (typeof req.query.query === 'object') ? req.query.query : null;
72 | if (!type) {
73 | if (related) {
74 | throw 'related is only permitted with type';
75 | }
76 | if (query) {
77 | throw 'query is only permitted with type';
78 | }
79 | }
80 | // Ask nginx not to buffer this large response, better that it
81 | // flow continuously to the other end
82 | res.setHeader('X-Accel-Buffering', 'no');
83 | res.write(JSON.stringify({
84 | '@apostrophecms/sync-content': true,
85 | version: 1
86 | }));
87 | const never = self.neverCollections;
88 | const docs = self.getCollection('aposDocs');
89 | const collections = type ? [ docs ] : (await self.apos.db.collections()).filter(collection => {
90 | if (collection.collectionName.match(/^system\./)) {
91 | return false;
92 | }
93 | if (never.includes(collection.collectionName)) {
94 | return false;
95 | }
96 | return true;
97 | });
98 | let attachmentIds = [];
99 | for (const collection of collections) {
100 | await handleCollection(collection);
101 | }
102 | if (attachmentIds.length) {
103 | await handleCollection(self.getCollection('aposAttachments'), attachmentIds);
104 | }
105 | async function handleCollection(collection, ids) {
106 | const seen = new Set();
107 | if (query && (collection.collectionName === 'aposDocs') && !ids) {
108 | const manager = self.apos.docs.getManager(type);
109 | const reqParams = {};
110 | if (workflowLocale) {
111 | reqParams.locale = workflowLocale;
112 | }
113 | const criteria = {};
114 | ids = (await manager.find(self.apos.tasks.getReq(reqParams), {}, { _id: 1 })
115 | .queryToFilters(query, 'manage')
116 | .toArray())
117 | .map(doc => doc._id);
118 | }
119 | const criteria = ids ? {
120 | _id: {
121 | $in: ids
122 | }
123 | } : ((collection.collectionName === 'aposDocs') && type) ? {
124 | type
125 | } : (collection.collectionName === 'aposDocs') ? {
126 | type: {
127 | $nin: self.neverTypes
128 | }
129 | } : {};
130 | if ((collection.collectionName === 'aposDocs') && workflowLocale) {
131 | criteria.workflowLocale = {
132 | $in: [ null, workflowLocale ]
133 | };
134 | }
135 | await self.apos.migrations.each(collection, criteria, async (doc) => {
136 | // "sent" keeps track of whether we have started to buffer
137 | // output in RAM, if we have then after this batch we'll
138 | // wait for the output to drain
139 | await write(doc);
140 | if (type && (collection.collectionName === 'aposDocs')) {
141 | attachmentIds = attachmentIds.concat(self.apos.attachments.all(doc).map(attachment => attachment._id));
142 | }
143 | if (related && (collection.collectionName === 'aposDocs')) {
144 | const joins = self.findJoinsInDoc(doc);
145 | let ids = [];
146 | for (const join of joins) {
147 | const id = join.field.idField && join.doc[join.field.idField];
148 | if (id) {
149 | ids.push(id);
150 | }
151 | const joinIds = join.field.idsField && join.doc[join.field.idsField];
152 | if (joinIds) {
153 | ids = ids.concat(joinIds);
154 | }
155 | }
156 | const relatedDocs = await collection.find({
157 | _id: {
158 | $in: ids
159 | },
160 | type: {
161 | $nin: self.neverTypes
162 | },
163 | // Pages are never considered related for this purpose
164 | // because it leads to data integrity issues
165 | slug: /^[^\/]/
166 | }).toArray();
167 | for (const relatedDoc of relatedDocs) {
168 | if (type) {
169 | attachmentIds = attachmentIds.concat(self.apos.attachments.all(relatedDoc).map(attachment => attachment._id));
170 | }
171 | await write(relatedDoc);
172 | }
173 | }
174 | });
175 | async function write(doc) {
176 | if (seen.has(doc._id)) {
177 | // Don't send a related doc twice
178 | return;
179 | }
180 | seen.add(doc._id);
181 | return new Promise((resolve, reject) => {
182 | try {
183 | const result = res.write(EJSON.stringify({
184 | collection: collection.collectionName,
185 | doc
186 | }));
187 | if (!result) {
188 | // Node streams backpressure is fussy, you don't get a
189 | // drain event for the second of two consecutive false returns,
190 | // we must wait every time we do get one
191 | res.once('drain', () => {
192 | return resolve();
193 | });
194 | } else {
195 | return resolve();
196 | }
197 | } catch (e) {
198 | return reject(e);
199 | }
200 | });
201 | }
202 | }
203 | res.write(JSON.stringify({
204 | 'end': true
205 | }));
206 | res.end();
207 | } catch (e) {
208 | self.apos.utils.error(e);
209 | return res.status(400).send('invalid');
210 | }
211 | });
212 |
213 | self.route('get', 'uploadfs', async (req, res) => {
214 | const disable = util.promisify(self.apos.attachments.uploadfs.disable);
215 | const enable = util.promisify(self.apos.attachments.uploadfs.enable);
216 | const copyOut = util.promisify(self.apos.attachments.uploadfs.copyOut);
217 | try {
218 | self.checkAuthorizationApiKey(req);
219 | const path = self.apos.launder.string(req.query.path);
220 | const disabled = self.apos.launder.boolean(req.query.disabled);
221 | if (!path.length) {
222 | throw self.apos.utils.error('invalid');
223 | }
224 | if (!disabled) {
225 | let url = self.apos.attachments.uploadfs.getUrl() + path;
226 | if (url.startsWith('/') && !url.startsWith('//') && req.baseUrl) {
227 | url = req.baseUrl + url;
228 | }
229 | return res.redirect(url);
230 | } else {
231 | const tempPath = self.getTempPath(path);
232 | // This workaround is not ideal, in future uploadfs will provide
233 | // better guarantees that copyOut works when direct URL access does not
234 | await enable(path);
235 | await copyOut(path, tempPath);
236 | await disable(path);
237 | res.on('finish', async () => {
238 | await unlink(tempPath);
239 | });
240 | return res.download(tempPath);
241 | }
242 | } catch (e) {
243 | self.apos.utils.error(e);
244 | return res.status(400).send('invalid');
245 | }
246 | });
247 |
248 | self.syncFrom = async (env, options) => {
249 | if (env.url.endsWith('/')) {
250 | throw 'URL for environment must not end with /, should be a base URL for the site';
251 | }
252 | const updatePermissions = util.promisify(self.apos.attachments.updatePermissions);
253 | let ended = false;
254 | if (!options.type) {
255 | if (options.keep) {
256 | throw 'keep option not available without type option';
257 | }
258 | if (options.related) {
259 | throw 'related option not available without type option';
260 | }
261 | }
262 | const query = qs.stringify({
263 | type: options.type,
264 | keep: !!options.keep,
265 | related: !!options.related,
266 | // We have to rename this one in the query string to work around
267 | // the fact that the workflow module steals it otherwise
268 | locale: options.workflowLocale,
269 | query: options.query
270 | });
271 | const response = await fetch(`${env.url}/modules/@apostrophecms/sync-content/content?${query}`, {
272 | headers: {
273 | 'Authorization': `ApiKey ${env.apiKey}`
274 | }
275 | });
276 | if (response.status >= 400) {
277 | throw await response.text();
278 | }
279 | let version = null;
280 | const collections = {};
281 | let docIds = [];
282 |
283 | // This solution with a custom writable stream appears to handle backpressure properly,
284 | // several others appeared prettier but did not do that, so they would exhaust RAM
285 | // on a large site
286 |
287 | const sink = new Stream.Writable({
288 | objectMode: true
289 | });
290 | let attachmentIds = [];
291 | sink._write = async (value, encoding, callback) => {
292 | try {
293 | await handleObject(value);
294 | return callback(null);
295 | } catch (e) {
296 | return callback(e);
297 | }
298 | };
299 |
300 | await pipeline(
301 | response.body,
302 | parser({
303 | jsonStreaming: true
304 | }),
305 | streamValues(),
306 | sink
307 | );
308 |
309 | if (!ended) {
310 | throw 'Incomplete stream';
311 | }
312 |
313 | await self.syncUploadfsFrom(env, { attachmentIds });
314 | // Fix attachment permissions once all the facts are in
315 | await updatePermissions();
316 |
317 | async function handleObject(value) {
318 | value = value.value;
319 | if (!version) {
320 | if (!(value && (value['@apostrophecms/sync-content'] === true))) {
321 | throw 'This response does not contain an @apostrophecms/sync-content stream';
322 | }
323 | if (value.version !== 1) {
324 | throw `This site does not support stream version ${value.version}`;
325 | }
326 | version = value.version;
327 | const never = self.neverCollections;
328 | // Purge
329 | if (!options.type) {
330 | const collections = (await self.apos.db.collections()).filter(collection => !collection.collectionName.match(/^system\./) && !never.includes(collection.collectionName));
331 | for (const collection of collections) {
332 | const criteria = (collection.collectionName === 'aposDocs') ? {
333 | type: {
334 | $nin: self.neverTypes
335 | }
336 | } : {};
337 | await collection.removeMany(criteria);
338 | }
339 | } else if (!options.keep) {
340 | // TODO attachments need to update their references when this happens
341 | await self.apos.docs.db.removeMany({
342 | type: options.type
343 | });
344 | }
345 | } else if (value.collection) {
346 | if (!collections[value.collection]) {
347 | collections[value.collection] = self.getCollection(value.collection);
348 | }
349 | const collection = collections[value.collection];
350 | try {
351 | value.doc = EJSON.parse(JSON.stringify(value.doc));
352 | if (value.collection === 'aposDocs') {
353 | docIds.push(value.doc._id);
354 | }
355 | if ((value.collection === 'aposAttachments') && options.type) {
356 | await self.mergeAttachment(value.doc, docIds);
357 | attachmentIds.push(value.doc._id);
358 | } else {
359 | for (let attempt = 0; (attempt < 10); attempt++) {
360 | try {
361 | if (options.keep) {
362 | await collection.replaceOne(
363 | {
364 | _id: value.doc._id
365 | },
366 | value.doc,
367 | {
368 | upsert: true
369 | }
370 | );
371 | } else {
372 | await collection.insertOne(value.doc);
373 | }
374 | // This is A2, so when only one locale is synced
375 | // we must always match draft with live or vice versa
376 | if (query.locale && value.doc.workflowGuid) {
377 | const isDraft = query.locale.includes('-draft');
378 | const peerLocale = isDraft ? query.locale.replace('-draft', '') : (query.locale + '-draft');
379 | const existing = await collections.findOne({
380 | workflowGuid: value.doc.workflowGuid,
381 | workflowLocale: peerLocale
382 | });
383 | if (existing) {
384 | await collections.replaceOne({
385 | workflowGuid: value.doc.workflowGuid,
386 | workflowLocale: peerLocale
387 | }, {
388 | ...value.doc,
389 | _id: existing._id,
390 | workflowLocale: value.doc.workflowLocale
391 | });
392 | } else {
393 | await collections.insertOne({
394 | ...value.doc,
395 | _id: self.apos.utils.generateId(),
396 | workflowLocale: value.doc.workflowLocale
397 | });
398 | }
399 | }
400 | } catch (e) {
401 | if ((collection.collectionName === 'aposDocs') && self.apos.docs.isUniqueError(e)) {
402 | value.doc.slug += Math.floor(Math.random() * 10);
403 | continue;
404 | } else {
405 | throw e;
406 | }
407 | }
408 | break;
409 | }
410 | }
411 | } catch (e) {
412 | console.error(JSON.stringify(value, null, ' '));
413 | throw e;
414 | }
415 | } else if (value.end) {
416 | ended = true;
417 | } else {
418 | console.error(value);
419 | throw 'Unexpected object in JSON stream';
420 | }
421 | return true;
422 | }
423 | };
424 |
425 | self.mergeAttachment = async (attachment, docIds) => {
426 | // Merge what the sending and receiving sites know about docIds and trashDocIds to
427 | // ensure updatePermissions does the right thing for this attachment
428 | const existing = await self.apos.attachments.db.findOne({
429 | _id: attachment._id
430 | });
431 | const newDocIds = attachment.docIds.filter(id => docIds.includes(id));
432 | const newTrashDocIds = attachment.trashDocIds.filter(id => docIds.includes(id));
433 | if (!existing) {
434 | attachment.docIds = newDocIds;
435 | attachment.trashDocIds = newTrashDocIds;
436 | return self.apos.attachments.db.insertOne(attachment);
437 | }
438 | const oldDocIds = existing.docIds.filter(id => !docIds.includes(id));
439 | const oldTrashDocIds = attachment.trashDocIds.filter(id => !docIds.includes(id));
440 | attachment.docIds = newDocIds.concat(oldDocIds);
441 | attachment.trashDocIds = newTrashDocIds.concat(oldTrashDocIds);
442 | await self.apos.attachments.db.replaceOne({
443 | _id: attachment._id
444 | }, attachment);
445 | };
446 |
447 | self.syncUploadfsFrom = async (env, { attachmentIds }) => {
448 |
449 | const disable = util.promisify(self.apos.attachments.uploadfs.disable);
450 | const remove = util.promisify(self.apos.attachments.uploadfs.remove);
451 | const copyIn = util.promisify(self.apos.attachments.uploadfs.copyIn);
452 | const criteria = (options.type && attachmentIds) ? {
453 | _id: {
454 | $in: attachmentIds
455 | }
456 | } : {};
457 | await self.apos.migrations.each(self.apos.attachments.db, criteria, 5, async (attachment) => {
458 | const files = [];
459 | push(attachment, 'original', null);
460 | for (const size of self.apos.attachments.imageSizes) {
461 | push(attachment, size.name, null);
462 | }
463 | for (const crop of (attachment.crops || [])) {
464 | push(attachment, 'original', crop);
465 | for (const size of self.apos.attachments.imageSizes) {
466 | push(attachment, size.name, crop);
467 | }
468 | }
469 | for (const file of files) {
470 | const tempPath = self.getTempPath(file.path);
471 | await attempt(false);
472 | async function attempt(retryingDisabled) {
473 | try {
474 | const params = qs.stringify({
475 | ...file,
476 | disabled: retryingDisabled ? !file.disabled : file.disabled
477 | });
478 | const response = await fetch(`${env.url}/modules/@apostrophecms/sync-content/uploadfs?${params}`, {
479 | headers: {
480 | 'Authorization': `ApiKey ${env.apiKey}`
481 | }
482 | });
483 | if (response.status !== 200) {
484 | throw response.status;
485 | }
486 | await pipeline(
487 | response.body,
488 | fs.createWriteStream(tempPath)
489 | );
490 | try {
491 | await remove(file.path);
492 | } catch (e) {
493 | // Nonfatal, we are just making sure we don't get into conflict
494 | // with previous permissions settings if the receiving site
495 | // did have this file
496 | }
497 | await copyIn(tempPath, file.path);
498 | if (file.disabled) {
499 | await disable(file.path);
500 | }
501 | } catch (e) {
502 | if (!retryingDisabled) {
503 | // Work around the fact that the disabled state of the file
504 | // sometimes does not match what is expected on the sending end,
505 | // possibly due to an A2 bug. This way the receiving end gets
506 | // to the right outcome either way
507 | return await attempt(true);
508 | }
509 | // Missing attachments are not unusual and should not flunk the entire process
510 | self.apos.utils.error(`Error fetching uploadfs path ${file.path}, continuing:`);
511 | self.apos.utils.error(e);
512 | } finally {
513 | if (fs.existsSync(tempPath)) {
514 | await unlink(tempPath);
515 | }
516 | }
517 | }
518 | }
519 | function push(attachment, size, crop) {
520 | files.push({
521 | path: self.apos.attachments.url(attachment, {
522 | size,
523 | uploadfsPath: true,
524 | crop
525 | }),
526 | disabled: attachment.trash && (size !== self.apos.attachments.sizeAvailableInTrash)
527 | });
528 | }
529 | });
530 | };
531 |
532 | self.getTempPath = (file) => {
533 | return self.apos.attachments.uploadfs.getTempPath() + '/' + self.apos.utils.generateId() + require('path').extname(file);
534 | };
535 |
536 | self.getEnv = (envName, argv) => {
537 | if (envName.match(/^https?:/)) {
538 | if (!argv['api-key']) {
539 | throw '--api-key is required if --from specifies a URL';
540 | }
541 | return {
542 | label: envName,
543 | url: envName,
544 | apiKey: argv['api-key']
545 | };
546 | }
547 |
548 | const env = self.options.environments && self.options.environments[envName];
549 | if (!env) {
550 | throw new Error(`${envName} does not appear as a subproperty of the environments option`);
551 | }
552 | return env;
553 | };
554 | self.checkAuthorizationApiKey = (req) => {
555 | if (!self.options.apiKey) {
556 | throw 'API key not configured';
557 | }
558 | const apiKey = getAuthorizationApiKey(req);
559 | if (apiKey !== self.options.apiKey) {
560 | throw 'Invalid API key';
561 | }
562 | };
563 | self.getCollection = (name) => {
564 | const collection = self.apos.db.collection(name);
565 | // Should not be necessary according to the mongodb docs, but when
566 | // this method is used to obtain a collection object we don't get
567 | // a collectionName property
568 | collection.collectionName = name;
569 | return collection;
570 | };
571 | }
572 | };
573 |
574 | function getAuthorizationApiKey(req) {
575 | const header = req.headers.authorization;
576 | if (!header) {
577 | return null;
578 | }
579 | const matches = header.match(/^ApiKey\s+(\S.*)$/i);
580 | if (!matches) {
581 | return null;
582 | }
583 | return matches[1];
584 | }
585 |
--------------------------------------------------------------------------------