├── .editorconfig ├── .babelrc ├── .gitignore ├── example-config.json ├── test ├── runner └── run-space-sync-test.js ├── index.js ├── .travis.yml ├── LICENSE ├── bin └── space-sync ├── lib ├── get-transformed-destination-response.js ├── dump-error-buffer.js ├── run-space-sync.js └── usageParams.js ├── package.json └── README.md /.editorconfig: -------------------------------------------------------------------------------- 1 | [*.js] 2 | indent_style = space 3 | indent_size = 2 4 | insert_final_newline = true 5 | -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["es2015"], 3 | "env": { 4 | "test": { 5 | "sourceMaps": "inline", 6 | "plugins": ["babel-plugin-rewire"] 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | *.log 3 | coverage 4 | .lock-wscript 5 | node_modules 6 | 7 | dist 8 | 9 | # space-sync specific 10 | contentful-space-sync-* 11 | test-*.json 12 | *.swp 13 | -------------------------------------------------------------------------------- /example-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "sourceSpace": "source space id", 3 | "destinationSpace": "destination space id", 4 | "sourceDeliveryToken": "source space delivery token", 5 | "managementToken": "destination space management token" 6 | } 7 | -------------------------------------------------------------------------------- /test/runner: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var log = require('npmlog') 3 | log.level = 'silent' 4 | 5 | require('require-all')({ 6 | dirname: process.cwd() + '/test', 7 | filter: process.argv[2] || /\-test\.js/, 8 | recursive: true 9 | }) 10 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | try { 2 | module.exports = require('./dist/run-space-sync').default 3 | } catch (err) { 4 | if (err.code === 'MODULE_NOT_FOUND') { 5 | require('babel-register') 6 | module.exports = require('./lib/run-space-sync').default 7 | } else { 8 | console.log(err) 9 | process.exit(1) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | before_install: 3 | - npm install -g npm@latest 4 | language: node_js 5 | 6 | script: 7 | - npm run build 8 | - npm run test 9 | node_js: 10 | - "0.10" 11 | - "3.3" 12 | - "4.2" 13 | - "5.0" 14 | after_success: 15 | - curl -Lo travis_after_all.py https://raw.githubusercontent.com/contentful/travis_after_all/master/travis_after_all.py 16 | - python travis_after_all.py 17 | - cat ./coverage/lcov.info | ./node_modules/.bin/coveralls 18 | - export $(cat .to_export_back) &> /dev/null 19 | - npm run semantic-release 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Contentful GmbH - https://www.contentful.com 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /bin/space-sync: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var log = require('npmlog') 3 | var fs = require('fs') 4 | var runSpaceSync = require('../') 5 | var usageParams 6 | 7 | try { 8 | usageParams = require('../dist/usageParams') 9 | } catch (err) { 10 | if (err.code === 'MODULE_NOT_FOUND') { 11 | usageParams = require('../lib/usageParams') 12 | } else { 13 | console.log(err) 14 | process.exit(1) 15 | } 16 | } 17 | 18 | // welcome the user and let them know what's gonna happen 19 | log.info('Contentful Space Sync:\n' + 20 | 'Let\'s sync some content across spaces!') 21 | 22 | var hasTokenFile 23 | try { 24 | hasTokenFile = !!fs.statSync(usageParams.syncTokenFile) 25 | } catch (e) { 26 | hasTokenFile = false 27 | } 28 | 29 | if (!hasTokenFile) { 30 | log.info('No previous sync token found.\n' + 31 | 'Synchronizing fresh content from ' + usageParams.opts.sourceSpace + ' to ' + usageParams.opts.destinationSpace) 32 | } else { 33 | log.info('Synchronizing content from ' + usageParams.opts.sourceSpace + ' to ' + 34 | usageParams.opts.destinationSpace + 'with existing token from ' + usageParams.syncTokenFile) 35 | } 36 | 37 | // Allow the user some time to cancel their action after the previous warning 38 | setTimeout(function () { 39 | runSpaceSync(usageParams) 40 | .catch(function (err) { 41 | log.error('Failed with\n', err) 42 | process.exit(1) 43 | }) 44 | }, 3000) 45 | -------------------------------------------------------------------------------- /lib/get-transformed-destination-response.js: -------------------------------------------------------------------------------- 1 | import log from 'npmlog' 2 | import { map } from 'lodash/collection' 3 | import getOutdatedDestinationContent from 'contentful-batch-libs/get/get-outdated-destination-content' 4 | 5 | /** 6 | * Gets the response from the destination space with the content that needs 7 | * to be updated. If it's the initial sync, and content exists, we abort 8 | * and tell the user why. 9 | */ 10 | export default function getTransformedDestinationResponse ({managementClient, spaceId, sourceResponse, skipContentModel}) { 11 | return getOutdatedDestinationContent({ 12 | managementClient: managementClient, 13 | spaceId: spaceId, 14 | webhooks: map(sourceResponse.webhooks, 'sys.id'), 15 | entryIds: map(sourceResponse.entries, 'sys.id'), 16 | assetIds: map(sourceResponse.assets, 'sys.id') 17 | }) 18 | .then((destinationResponse) => { 19 | if (skipContentModel) { 20 | destinationResponse.contentTypes = [] 21 | destinationResponse.locales = [] 22 | } 23 | 24 | if (sourceResponse.isInitialSync && (destinationResponse.contentTypes.length > 0 || destinationResponse.assets.length > 0)) { 25 | log.error(` 26 | Your destination space already has some content. 27 | 28 | If you have a token file, please place it on the same directory which you are currently in. 29 | 30 | Otherwise, please run this tool on an empty space. 31 | 32 | If you'd like more information, please consult the README at: 33 | https://github.com/contentful/contentful-space-sync 34 | `) 35 | process.exit(1) 36 | } 37 | return destinationResponse 38 | }) 39 | } 40 | -------------------------------------------------------------------------------- /lib/dump-error-buffer.js: -------------------------------------------------------------------------------- 1 | import {get} from 'lodash/object' 2 | import {partialRight} from 'lodash/function' 3 | import {find} from 'lodash/collection' 4 | import log from 'npmlog' 5 | import fs from 'fs' 6 | import errorBuffer from 'contentful-batch-libs/utils/error-buffer' 7 | 8 | export default function dumpErrorBuffer (params, message = 'Additional errors were found') { 9 | const {destinationSpace, sourceSpace, errorLogFile} = params 10 | const loggedErrors = errorBuffer.drain() 11 | if (loggedErrors.length > 0) { 12 | const errorOutput = { 13 | additionalInfo: {} 14 | } 15 | errorOutput.errors = loggedErrors.map(partialRight(logErrorsWithAppLinks, sourceSpace, destinationSpace)) 16 | const unresolvedLinks = additionalInfoForUnresolvedLinks(errorOutput.errors) 17 | if (unresolvedLinks) { 18 | errorOutput.additionalInfo.unresolvedLinks = unresolvedLinks 19 | } 20 | fs.writeFileSync(errorLogFile, JSON.stringify(errorOutput, null, ' ')) 21 | log.warn(message) 22 | log.warn(`Check ${errorLogFile} for details.`) 23 | } 24 | } 25 | 26 | function logErrorsWithAppLinks (err, idx, loggedErrors, sourceSpace, destinationSpace) { 27 | const parsedError = JSON.parse(err.message) 28 | const requestUri = get(parsedError, 'request.url') 29 | if (requestUri) { 30 | parsedError.webAppUrl = parseEntityUrl(sourceSpace, destinationSpace, requestUri) 31 | } 32 | return parsedError 33 | } 34 | 35 | function parseEntityUrl (sourceSpace, destinationSpace, url) { 36 | return url.replace(/api.contentful/, 'app.contentful') 37 | .replace(/:443/, '') 38 | .replace(destinationSpace, sourceSpace) 39 | .split('/').splice(0, 7).join('/') 40 | } 41 | 42 | function additionalInfoForUnresolvedLinks (errors) { 43 | if (find(errors, {name: 'UnresolvedLinks'})) { 44 | return 'Unresolved links were found in your entries. See the errors list for more details. ' + 45 | 'Look at https://github.com/contentful/contentful-link-cleaner if you\'d like ' + 46 | 'a quicker way to fix all unresolved links in your space.' 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "contentful-space-sync", 3 | "description": "Syncs Contentful spaces", 4 | "bin": "bin/space-sync", 5 | "main": "index.js", 6 | "scripts": { 7 | "build": "rimraf dist && mkdirp dist && babel lib --out-dir dist", 8 | "clean": "rimraf dist && rimraf coverage", 9 | "prepublish": "in-publish && npm run build || not-in-publish", 10 | "postpublish": "npm run clean", 11 | "pretest": "standard", 12 | "test": "npm run test:cover", 13 | "test:cover": "BABEL_ENV=test babel-node ./node_modules/istanbul/lib/cli.js cover test/runner", 14 | "test:only": "BABEL_ENV=test babel-node ./test/runner", 15 | "test:debug": "BABEL_ENV=test babel-node debug ./test/runner", 16 | "browser-coverage": "npm run test:cover && opener coverage/lcov-report/index.html", 17 | "semantic-release": "semantic-release pre && npm publish && semantic-release post", 18 | "devmanage:build": "pushd ../contentful-management.js && npm run build && popd", 19 | "devmanage:clean": "pushd ../contentful-management.js && npm run clean && popd", 20 | "devmanage:install": "npm run devmanage:build && rm -rf node_modules/contentful-management.js && npm install ../contentful-management.js && npm run devmanage:clean", 21 | "devmanage:uninstall": "npm run devmanage:clean && rimraf node_modules/contentful-management.js", 22 | "devdep:build": "pushd ../contentful-batch-libs && npm run build && popd", 23 | "devdep:clean": "pushd ../contentful-batch-libs && npm run clean && popd", 24 | "devdep:install": "npm run devdep:build && rm -rf node_modules/contentful-batch-libs && npm install ../contentful-batch-libs && npm run devdep:clean", 25 | "devdep:uninstall": "npm run devdep:clean && rimraf node_modules/contentful-batch-libs" 26 | }, 27 | "dependencies": { 28 | "bluebird": "^3.3.3", 29 | "contentful-batch-libs": "4.6.1", 30 | "lodash": "^4.0.0", 31 | "npmlog": "^2.0.0", 32 | "yargs": "^4.2.0" 33 | }, 34 | "devDependencies": { 35 | "babel-cli": "^6.3.17", 36 | "babel-eslint": "^6.0.2", 37 | "babel-plugin-rewire": "^1.0.0-beta-3", 38 | "babel-preset-es2015": "^6.3.13", 39 | "babel-register": "^6.3.13", 40 | "coveralls": "^2.11.6", 41 | "cz-conventional-changelog": "^1.1.4", 42 | "ghooks": "^1.3.2", 43 | "in-publish": "^2.0.0", 44 | "istanbul": "^1.0.0-alpha.2", 45 | "mkdirp": "^0.5.1", 46 | "opener": "^1.4.1", 47 | "require-all": "^2.0.0", 48 | "rimraf": "^2.5.0", 49 | "semantic-release": "^4.3.5", 50 | "sinon": "^1.17.2", 51 | "standard": "^6.0.7", 52 | "tape": "^4.5.1" 53 | }, 54 | "engines": { 55 | "node": "<6.0.0" 56 | }, 57 | "files": [ 58 | "bin", 59 | "dist", 60 | "example-config.json", 61 | "index.js" 62 | ], 63 | "repository": { 64 | "type": "git", 65 | "url": "https://github.com/contentful/contentful-space-sync.git" 66 | }, 67 | "keywords": [ 68 | "contentful", 69 | "sync", 70 | "space" 71 | ], 72 | "author": "Tiago Rodrigues ", 73 | "license": "MIT", 74 | "bugs": { 75 | "url": "https://github.com/contentful/contentful-space-sync/issues" 76 | }, 77 | "homepage": "https://github.com/contentful/contentful-space-sync#readme", 78 | "standard": { 79 | "parser": "babel-eslint" 80 | }, 81 | "config": { 82 | "commitizen": { 83 | "path": "./node_modules/cz-conventional-changelog" 84 | }, 85 | "ghooks": { 86 | "pre-commit": "npm run test:only" 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /test/run-space-sync-test.js: -------------------------------------------------------------------------------- 1 | import test from 'tape' 2 | import sinon from 'sinon' 3 | import Promise from 'bluebird' 4 | 5 | import runSpaceSync from '../lib/run-space-sync' 6 | import errorBuffer from 'contentful-batch-libs/utils/error-buffer' 7 | 8 | const sourceResponse = { 9 | nextSyncToken: 'nextsynctoken', 10 | contentTypes: [ 11 | {original: {sys: {id: 'exists'}}} 12 | ], 13 | locales: [{original: {code: 'en-US'}}] 14 | } 15 | const destinationResponse = { 16 | contentTypes: [ 17 | {sys: {id: 'exists'}}, 18 | {sys: {id: 'doesntexist'}} 19 | ], 20 | locales: [{code: 'en-US'}, {code: 'en-GB'}] 21 | } 22 | 23 | const createClientsStub = sinon.stub().returns({ source: {delivery: {}}, destination: {management: {}} }) 24 | runSpaceSync.__Rewire__('createClients', createClientsStub) 25 | 26 | const getSourceSpaceViaSyncStub = sinon.stub().returns(Promise.resolve(sourceResponse)) 27 | runSpaceSync.__Rewire__('getSourceSpaceViaSync', getSourceSpaceViaSyncStub) 28 | 29 | const getTransformedDestinationResponseStub = sinon.stub().returns(Promise.resolve(destinationResponse)) 30 | runSpaceSync.__Rewire__('getTransformedDestinationResponse', getTransformedDestinationResponseStub) 31 | 32 | const transformSpaceStub = sinon.stub().returns(Promise.resolve(sourceResponse)) 33 | runSpaceSync.__Rewire__('transformSpace', transformSpaceStub) 34 | 35 | const pushToSpaceStub = sinon.stub().returns(Promise.resolve({})) 36 | runSpaceSync.__Rewire__('pushToSpace', pushToSpaceStub) 37 | 38 | const dumpErrorBufferStub = sinon.stub() 39 | runSpaceSync.__Rewire__('dumpErrorBuffer', dumpErrorBufferStub) 40 | 41 | const fsMock = { 42 | writeFileSync: sinon.stub() 43 | } 44 | runSpaceSync.__Rewire__('fs', fsMock) 45 | 46 | test('Runs space sync', (t) => { 47 | const preparedResponses = { 48 | sourceContent: { 49 | deletedContentTypes: [{sys: {id: 'doesntexist'}}], 50 | deletedLocales: [{code: 'en-GB'}], 51 | contentTypes: [{original: {sys: {id: 'exists'}}}], 52 | locales: [{original: {code: 'en-US'}}], 53 | nextSyncToken: 'nextsynctoken' 54 | }, 55 | destinationContent: Object.assign({}, destinationResponse) 56 | } 57 | 58 | errorBuffer.push({ 59 | request: { 60 | uri: 'erroruri' 61 | } 62 | }) 63 | 64 | runSpaceSync({ 65 | opts: {}, 66 | syncTokenFile: 'synctokenfile', 67 | errorLogFile: 'errorlogfile' 68 | }) 69 | .then(() => { 70 | t.ok(createClientsStub.called, 'creates clients') 71 | t.ok(getSourceSpaceViaSyncStub.called, 'gets source space') 72 | t.ok(getTransformedDestinationResponseStub.called, 'gets destination space') 73 | t.ok(transformSpaceStub.called, 'transforms space') 74 | t.deepLooseEqual(pushToSpaceStub.args[0][0].sourceContent, preparedResponses.sourceContent, 'sends source content to destination space') 75 | t.deepLooseEqual(pushToSpaceStub.args[0][0].destinationContent, preparedResponses.destinationContent, 'sends destination content to destination space') 76 | t.ok(fsMock.writeFileSync.calledWith('synctokenfile', 'nextsynctoken'), 'token file created') 77 | t.ok(dumpErrorBufferStub.called, 'error objects are logged') 78 | 79 | runSpaceSync.__ResetDependency__('createClients') 80 | runSpaceSync.__ResetDependency__('getSourceSpaceViaSync') 81 | runSpaceSync.__ResetDependency__('getTransformedDestinationResponse') 82 | runSpaceSync.__ResetDependency__('transformSpace') 83 | runSpaceSync.__ResetDependency__('pushToSpace') 84 | runSpaceSync.__ResetDependency__('dumpErrorBuffer') 85 | runSpaceSync.__ResetDependency__('fs') 86 | t.end() 87 | }) 88 | }) 89 | -------------------------------------------------------------------------------- /lib/run-space-sync.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird' 2 | import log from 'npmlog' 3 | import fs from 'fs' 4 | Promise.promisifyAll(fs) 5 | 6 | import { find, filter } from 'lodash/collection' 7 | import createClients from 'contentful-batch-libs/utils/create-clients' 8 | import getSourceSpaceViaSync from 'contentful-batch-libs/get/get-source-space-via-sync' 9 | import transformSpace from 'contentful-batch-libs/transform/transform-space' 10 | import pushToSpace from 'contentful-batch-libs/push/push-to-space' 11 | 12 | import dumpErrorBuffer from './dump-error-buffer' 13 | import getTransformedDestinationResponse from './get-transformed-destination-response' 14 | 15 | export default function runSpaceSync (usageParams) { 16 | let {opts, errorLogFile, syncTokenFile} = usageParams 17 | opts.sourceManagementToken = opts.sourceManagementToken || opts.managementToken 18 | opts.destinationManagementToken = opts.destinationManagementToken || opts.managementToken 19 | const tokenDir = opts.syncTokenDir || process.cwd() 20 | errorLogFile = errorLogFile || tokenDir + '/contentful-space-sync-' + Date.now() + '.log' 21 | syncTokenFile = syncTokenFile || tokenDir + '/contentful-space-sync-token-' + opts.sourceSpace + '-to-' + opts.destinationSpace 22 | 23 | if (opts.proxyHost && opts.proxyPort) { 24 | opts.proxy = {host: opts.proxyHost, port: opts.proxyPort} 25 | } 26 | const clients = createClients(opts) 27 | return getSourceSpaceViaSync({ 28 | deliveryClient: clients.source.delivery, 29 | managementClient: clients.source.management, 30 | sourceSpaceId: clients.source.spaceId, 31 | nextSyncTokenFile: syncTokenFile 32 | }) 33 | 34 | // Prepare object with both source and destination existing content 35 | .then((sourceResponse) => { 36 | return Promise.props({ 37 | source: sourceResponse, 38 | destination: getTransformedDestinationResponse({ 39 | managementClient: clients.destination.management, 40 | spaceId: clients.destination.spaceId, 41 | sourceResponse: sourceResponse, 42 | skipLocales: opts.skipLocales, 43 | skipContentModel: opts.skipContentModel 44 | }) 45 | }) 46 | }) 47 | .then((responses) => { 48 | return Promise.props({ 49 | source: transformSpace(responses.source, responses.destination), 50 | destination: responses.destination 51 | }) 52 | }) 53 | 54 | // Get deleted content types 55 | .then((responses) => { 56 | responses.source.deletedContentTypes = filter(responses.destination.contentTypes, (contentType) => { 57 | return !find(responses.source.contentTypes, {original: {sys: {id: contentType.sys.id}}}) 58 | }) 59 | responses.source.deletedLocales = filter(responses.destination.locales, (locale) => { 60 | return !find(responses.source.locales, {original: {code: locale.code}}) 61 | }) 62 | return responses 63 | }) 64 | 65 | // push source space content to destination space 66 | .then((responses) => { 67 | return pushToSpace({ 68 | sourceContent: responses.source, 69 | destinationContent: responses.destination, 70 | managementClient: clients.destination.management, 71 | spaceId: clients.destination.spaceId, 72 | prePublishDelay: opts.prePublishDelay, 73 | contentModelOnly: opts.contentModelOnly, 74 | skipLocales: opts.skipLocales, 75 | skipContentModel: opts.skipContentModel 76 | }) 77 | .then(() => { 78 | const nextSyncToken = responses.source.nextSyncToken 79 | if (!opts.contentModelOnly && nextSyncToken) { 80 | fs.writeFileSync(syncTokenFile, nextSyncToken) 81 | log.info('Successfully sychronized the content and saved the sync token to:\n ', opts.syncTokenFile) 82 | } else { 83 | log.info('Successfully sychronized the content model') 84 | } 85 | dumpErrorBuffer({ 86 | destinationSpace: opts.destinationSpace, 87 | sourceSpace: opts.sourceSpace, 88 | errorLogFile: errorLogFile 89 | }, 'However, additional errors were found') 90 | 91 | return { 92 | nextSyncToken: nextSyncToken 93 | } 94 | }) 95 | }) 96 | 97 | // Output any errors caught along the way 98 | .catch((err) => { 99 | dumpErrorBuffer({ 100 | destinationSpace: opts.destinationSpace, 101 | sourceSpace: opts.sourceSpace, 102 | errorLogFile: errorLogFile 103 | }) 104 | throw err 105 | }) 106 | } 107 | -------------------------------------------------------------------------------- /lib/usageParams.js: -------------------------------------------------------------------------------- 1 | var yargs = require('yargs') 2 | var log = require('npmlog') 3 | var packageFile = require('../package') 4 | 5 | var opts = yargs 6 | .version(packageFile.version || 'Version only available on installed package') 7 | .usage('Usage: $0 [options]') 8 | .option('source-space', { 9 | describe: 'ID of Space with source data', 10 | type: 'string', 11 | demand: true 12 | }) 13 | .option('destination-space', { 14 | describe: 'ID of Space data will be copied to', 15 | type: 'string', 16 | demand: true 17 | }) 18 | .option('rate-limit', { 19 | describe: 'How many requests to perform per period of time, default is 6', 20 | type: 'number', 21 | default: 6 22 | }) 23 | .option('rate-limit-period', { 24 | describe: 'How much time to wait before retry in ms, default 1000', 25 | type: 'number', 26 | default: 1000 27 | }) 28 | .option('source-delivery-token', { 29 | describe: 'Delivery API token for source space', 30 | type: 'string', 31 | demand: true 32 | }) 33 | .option('management-token', { 34 | describe: 'Management API token for both spaces.', 35 | type: 'string' 36 | }) 37 | .option('source-management-token', { 38 | describe: 'Management API token for source space, if different from --management-token.', 39 | type: 'string' 40 | }) 41 | .option('destination-management-token', { 42 | describe: 'Management API token for destination space if different from --management-token.', 43 | type: 'string' 44 | }) 45 | .option('pre-publish-delay', { 46 | describe: 'Delay in milliseconds to account for delay after creating entities, due to internal database indexing', 47 | type: 'number', 48 | default: 5000 49 | }) 50 | .option('sync-token-dir', { 51 | describe: 'Defines the path for storing sync token files (default path is the current directory)', 52 | type: 'string' 53 | }) 54 | .option('content-model-only', { 55 | describe: 'Copies only content types and locales', 56 | type: 'boolean' 57 | }) 58 | .option('skip-content-model', { 59 | describe: 'Skips content types and locales. Copies only entries and assets', 60 | type: 'boolean' 61 | }) 62 | .option('skip-locales', { 63 | describe: 'Skips locales. Must be used with content-model-only. Copies only content-types', 64 | type: 'boolean' 65 | }) 66 | .option('delivery-host', { 67 | describe: 'Host for the Delivery API.', 68 | type: 'string' 69 | }) 70 | .option('delivery-port', { 71 | describe: 'Port for the Delivery API.', 72 | type: 'string' 73 | }) 74 | .option('delivery-insecure', { 75 | describe: 'If the Delivery API should use http instead of the default https.', 76 | type: 'boolean' 77 | }) 78 | .option('management-host', { 79 | describe: 'Host for the Management API.', 80 | type: 'string' 81 | }) 82 | .option('management-port', { 83 | describe: 'Port for the Management API.', 84 | type: 'string' 85 | }) 86 | .option('management-insecure', { 87 | describe: 'If the Management API should use http instead of the default https.', 88 | type: 'boolean' 89 | }) 90 | .option('proxy-host', { 91 | describe: 'hostname of the proxy server', 92 | type: 'string' 93 | }) 94 | .option('proxy-port', { 95 | describe: 'port of the proxy server', 96 | type: 'string' 97 | }) 98 | .config('config', 'Configuration file with required values') 99 | .check(function (argv) { 100 | if (!argv.proxyHost && !argv.proxyPort) { 101 | return true 102 | } 103 | if (argv.proxyPort && argv.proxyHost) { 104 | return true 105 | } 106 | log.error('--proxy-host and --proxy-port must be both defined') 107 | process.exit(1) 108 | }) 109 | .check(function (argv) { 110 | if (!argv.sourceManagementToken && 111 | !argv.destinationManagementToken && 112 | argv.managementToken) { 113 | return true 114 | } 115 | if (argv.sourceManagementToken && 116 | argv.destinationManagementToken && 117 | !argv.managementToken) { 118 | return true 119 | } 120 | if ((!argv.sourceManagementToken || 121 | !argv.destinationManagementToken) && 122 | argv.managementToken) { 123 | return true 124 | } 125 | log.error( 126 | 'Please provide a --management-token to be used for both source and delivery\n' + 127 | 'spaces, or separate --source-management-token and --delivery-management-token.\n' + 128 | '\n' + 129 | 'You also provide --management-token and use just one of the other options to\n' + 130 | 'override it.\n' + 131 | '\n' + 132 | 'See https://www.npmjs.com/package/contentful-space-sync for more information.\n' 133 | ) 134 | process.exit(1) 135 | }) 136 | .check(function (argv) { 137 | if (argv.skipContentModel && argv.contentModelOnly) { 138 | log.error('--skipContentModel and --contentModelOnly cannot be used together') 139 | process.exit(1) 140 | } 141 | return true 142 | }) 143 | .check(function (argv) { 144 | if (argv.skipLocales && !argv.contentModelOnly) { 145 | log.error('--skip-locales can only be used with --content-model-only') 146 | process.exit(1) 147 | } 148 | return true 149 | }) 150 | .argv 151 | 152 | module.exports = { 153 | opts: opts 154 | } 155 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Contentful Space Sync [![deprecated](http://badges.github.io/stability-badges/dist/deprecated.svg)](http://github.com/badges/stability-badges) 2 | 3 | 4 | [![npm](https://img.shields.io/npm/v/contentful-space-sync.svg)](https://www.npmjs.com/package/contentful-space-sync) 5 | [![Build Status](https://travis-ci.org/contentful/contentful-space-sync.svg?branch=master)](https://travis-ci.org/contentful/contentful-space-sync) 6 | [![Coverage Status](https://coveralls.io/repos/github/contentful/contentful-space-sync/badge.svg?branch=master)](https://coveralls.io/github/contentful/contentful-space-sync?branch=master) 7 | [![Dependency Status](https://david-dm.org/contentful/contentful-space-sync.svg)](https://david-dm.org/contentful/contentful-space-sync) 8 | [![devDependency Status](https://david-dm.org/contentful/contentful-space-sync/dev-status.svg)](https://david-dm.org/contentful/contentful-space-sync#info=devDependencies) 9 | 10 | [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) 11 | [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/) 12 | 13 | ## Deprecation Notice 14 | 15 | We have replaced this tool with the [contentful-export](https://github.com/contentful/contentful-export/) and [contentful-import](https://github.com/contentful/contentful-import/) tools and it is now considered deprecated. We won't offer support regarding the usage of this tool. 16 | 17 | The new export tool allows you to export all content, including content types, assets and webhooks from a space. The new import tool enables you to import external content into a new space. 18 | 19 | Read more about the steps involved in [our guide for managing synchronization between multiple spaces](https://www.contentful.com/developers/docs/concepts/multiple-environments/). 20 | 21 | ## What this tool is for 22 | 23 | This tool allows you to perform a **one way** synchronization of **published** content from one Contentful space to another. 24 | 25 | The tool makes use of Contentful's [Synchronization API](https://www.contentful.com/developers/docs/concepts/sync/) which means that if you run the tool in the future with the provided token, you will only synchronize new and updated Entries and Assets, as well as remove any that have been deleted. 26 | 27 | ### Development environments 28 | 29 | - You have a Production space where your content editors create and publish content 30 | - You want your developers to work on new features without touching the production content 31 | - Use the tool to create copies of your Production space where your Developers can try things out at will 32 | 33 | ### Field deletion for published Content Types / Entries 34 | 35 | - See the [Deleting Fields](#deleting-fields) section 36 | 37 | ### Creating new spaces with a similar content model 38 | 39 | - If you want to start out with a content model similar to what you have on another space, you can use the `--content-model-only` option 40 | 41 | ## What this tool can be used for (but isn't advised) 42 | 43 | ### Published content backups 44 | 45 | While this is possible, we do not advise that you use this tool for backups for the following reasons: 46 | 47 | - This tool only synchronizes **published** content. Anything in Draft mode or any unpublished changes to a published Entry will not be synchronized. 48 | - Your content might have broken links (see [contentful-link-cleaner](https://github.com/contentful/contentful-link-cleaner)) 49 | - The tool attempts to create every Content Type, Entry and Asset separately, so if failures such as network failures occur your copy might not be complete 50 | - Contentful already [backups your content](https://www.contentful.com/faq/backup-security-and-hosting/) and provides extra offsite backup capabilities 51 | 52 | ## What this tool shouldn't be used for 53 | 54 | ### Workflow management 55 | 56 | - Initially, this tool was born as a replacement for [contentful-publication](https://github.com/jsebfranck/contentful-publication), a tool built to manage publication workflows, in which editors would work in a Source space, and content approved and meant for publishing would be synchronized to a Destination space 57 | - However, Contentful now has an improved [Roles and Permissions](https://www.contentful.com/r/knowledgebase/roles-and-permissions/) system, which allows for an easier content approval process 58 | 59 | # How does it work? 60 | 61 | Each time you run the tool it stores a [synchronization token](https://www.contentful.com/developers/docs/concepts/sync/) so only new Entries and Assets get copied, and so that deleted items can also be deleted on the destination space. See the [Synchronization](https://www.contentful.com/developers/docs/concepts/sync/) documentation for more details. 62 | 63 | Content Types will always be updated as they are not retrieved by the synchronization API, and Content Types which don't exist anymore in the source space will be deleted in the destination space as well. 64 | 65 | If you make any manual changes in the destination space, be aware that **this tool will overwrite any changes** you've made to entities with the same ids as those existent on the source space. 66 | 67 | Also, avoid creating new Content Types and Entries in the destination space. This tool is intended to be used with a workflow where you create content on one space and then regularly copy it somewhere else in an automated way. 68 | 69 | # Changelog 70 | 71 | Check out the [releases](https://github.com/contentful/contentful-space-sync/releases) page. 72 | 73 | # Install 74 | 75 | `npm install -g contentful-space-sync` 76 | 77 | # Usage 78 | 79 | Usage: contentful-space-sync [options] 80 | 81 | Options: 82 | --version Show version number 83 | 84 | --source-space ID of Space with source data 85 | [string] [required] 86 | 87 | --destination-space ID of Space data will be copied to 88 | [string] [required] 89 | 90 | --source-delivery-token Delivery API token for source space 91 | [string] [required] 92 | 93 | --management-token Management API token for both spaces. 94 | [string] 95 | 96 | --source-management-token Management API token for source space, if 97 | different from --management-token. 98 | [string] 99 | 100 | --destination-management-token Management API token for destination space if 101 | different from --management-token. 102 | [string] 103 | 104 | --pre-publish-delay Delay in milliseconds to account for delay 105 | after creating entities, due to internal 106 | database indexing 107 | [default: 5000] 108 | 109 | --sync-token-dir Defines the path for storing sync token files 110 | (default path is the current directory) 111 | [string] 112 | 113 | --content-model-only Copies only content types and locales 114 | [boolean] 115 | 116 | --skip-content-model Skips content types and locales. Copies only entries and assets 117 | [boolean] 118 | 119 | --skip-locales Skips locales. Must be used with --content-model-only. 120 | Copies only content types. 121 | [boolean] 122 | 123 | --delivery-host Host for the Delivery API. 124 | [string] 125 | 126 | --delivery-port Port for the Delivery API. 127 | [string] 128 | 129 | --delivery-insecure If the Delivery API should use http instead of the default https. 130 | [boolean] 131 | 132 | --management-host Host for the Management API. 133 | [string] 134 | 135 | --management-port Port for the Management API. 136 | [string] 137 | 138 | --management-insecure If the Management API should use http instead of the default https. 139 | [boolean] 140 | 141 | --proxy-host hostname of the proxy server. [string] 142 | 143 | --proxy-port port of the proxy server. [string] 144 | 145 | --rate-limit How many request per period of time, default 6 [number] 146 | 147 | --rate-limit-period How much time to wait before retry in ms, default 1000 [number] 148 | 149 | --config Configuration file with required values 150 | 151 | The `--management-token` parameter allows you to specify a token which will be used for both spaces. If you get a token from and your user account has access to both spaces, this should be enough. 152 | 153 | In case you actually need a different management token for any of the spaces, you can use the `--source-management-token` and `--destination-management-token` options to override it. 154 | 155 | Check the `example-config.json` file for an example of what a configuration file would look like. If you use the config file, you don't need to specify the other options for tokens and space ids. 156 | 157 | # Example usage 158 | 159 | contentful-space-sync \ 160 | --source-space sourceSpaceId \ 161 | --source-delivery-token sourceSpaceDeliveryToken \ 162 | --destination-space destinationSpaceId \ 163 | --destination-management-token destinationSpaceManagementToken 164 | 165 | or 166 | 167 | contentful-space-sync --config example-config.json 168 | 169 | You can create your own config file based on the [`example-config.json`](example-config.json) file. 170 | 171 | # Usage as a library 172 | 173 | While this tool is mostly intended to be used as a command line tool, it can also be used as a Node library: 174 | 175 | ```js 176 | var spaceSync = require('contentful-space-sync') 177 | 178 | spaceSync(options) 179 | .then((output) => { 180 | console.log('sync token', output.nextSyncToken) 181 | }) 182 | .catch((err) => { 183 | console.log('oh no! errors occurred!', err) 184 | }) 185 | ``` 186 | 187 | The options object can contain any of the CLI options but written with a camelCase pattern instead, and no dashes. So `--source-space` would become `sourceSpace`. 188 | 189 | Apart from those options, there are two additional ones that can be passed to it: 190 | 191 | - `errorLogFile` - File to where any errors will be written. 192 | - `syncTokenFile` - File to where the sync token will be written. 193 | 194 | The method returns a promise, where, if successful, you'll have an object which contains the `nextSyncToken` (only thing there at the moment). If not successful, it will contain an object with errors. 195 | 196 | You can look at [`bin/space-sync`](bin/space-sync) to check how the CLI tool uses the library. 197 | 198 | # Synchronizing a space over time 199 | 200 | This tool uses the Contentful [Synchronization](https://www.contentful.com/developers/docs/concepts/sync/) endpoint to keep content synchronized over repeated runs of the script. 201 | 202 | Behind the scenes, when you use the sync endpoint, apart from the content it also returns a sync token in its response. The sync token encodes information about the last synchronized content, so that when you request a new synchronization, you can supply it this content and you'll only get new and updated content, as well a list of what content has been deleted. 203 | 204 | When you run this tool, it will create a file in the current directory named `contentful-space-sync-token-sourceSpaceId-destinationSpaceId`. If you run the tool again in the directory where this file resides, with the same source and destination space IDs, it will read the token from the file. If you have a token from somewhere else you can just create the file manually. 205 | 206 | # The error log 207 | 208 | If any errors occur during synchronization, the tool will also create a time stamped log file (`contentful-space-sync-timestamp.log`) with a list of any errors which occurred, and links to the entities in the source space which might have problems that need to be fixed. 209 | 210 | The most common problem will probably be an `UnresolvedLinks` error, which means a published entry A links to another entry B or asset C which has been deleted since publishing of the entry A. 211 | 212 | If you come across this problem, you can use [contentful-link-cleaner](https://github.com/contentful/contentful-link-cleaner) to clean all of those unresolved references. 213 | 214 | # Copying only the content model 215 | 216 | By using the `--content-model-only` option, you can copy only Content Types and Locales. This means you'll get a space with the same content structure, but with no content at all. 217 | 218 | This might be useful if you have been trying things out but want to start fresh with your content, or if you have a need to [delete fields](#deleting-fields) from your existing Content Types. 219 | 220 | # Copying only content 221 | 222 | By using the `--skip-content-model` option, you can copy only Entries and Assets. This assumes you have used this script before with the `--content-model-only` option or created the exact same content structure by hand. 223 | 224 | Every time you run the script without any of these options, it will attempt to update the content model as well, so on subsequent syncs it might be desirable to use this option to make things a bit faster. 225 | 226 | # Deleting fields 227 | 228 | While we used to recommend this tool as a way to do field deletion through a specific workflow, this feature is now available on the Contentful UI. 229 | 230 | # What happened to --force-overwrite and --fresh ? 231 | 232 | These options were very problematic and caused more problems than they solved, so they were removed on version 4. You can see more details [here](https://github.com/contentful/contentful-space-sync/commit/066c629fec0e4c41b2094cbf6f5b01697c0b525f) and [here](https://github.com/contentful/contentful-space-sync/commit/44f1ac81ec12850c4342d91d53e0386dff68de32). 233 | 234 | If you think you need these options, we advise you create a new, empty space, and restart the synchronization process from scratch. 235 | 236 | If you really really really REALLY need those options and you are aware of how much trouble they can cause, you can always use an older version of the tool. 237 | --------------------------------------------------------------------------------