├── .circleci └── config.yml ├── .contentful └── vault-secrets.yaml ├── .eslintrc ├── .github ├── CODEOWNERS ├── dependabot.yml └── workflows │ └── dependabot-approve-and-request-merge.yml ├── .gitignore ├── .nvmrc ├── CONTRIBUTION.md ├── LICENSE ├── README.md ├── babel.config.json ├── bin └── contentful-export ├── catalog-info.yaml ├── example-config.json ├── example-config.test.json ├── lib ├── index.js ├── parseOptions.js ├── tasks │ ├── download-assets.js │ ├── get-space-data.js │ └── init-client.js ├── usageParams.js └── utils │ ├── embargoedAssets.js │ └── headers.js ├── package-lock.json ├── package.json ├── test ├── integration │ └── export-lib.test.js └── unit │ ├── index.test.js │ ├── mocks │ ├── download-assets.js │ └── get-space-data.js │ ├── parseOptions.test.js │ ├── tasks │ ├── download-assets.test.js │ ├── get-space-data.test.js │ └── init-client.test.js │ └── utils │ ├── embargoedAssets.test.js │ └── headers.test.js ├── tsconfig.json └── types.d.ts /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | vault: contentful/vault@1 5 | 6 | jobs: 7 | unit: 8 | docker: 9 | - image: cimg/node:18.18.0 10 | steps: 11 | - checkout 12 | - restore_cache: 13 | key: dependency-cache-{{ checksum "package.json" }} 14 | - run: npm install 15 | - run: npm run build 16 | - run: npm run test:unit 17 | integration: 18 | docker: 19 | - image: cimg/node:18.18.0 20 | steps: 21 | - checkout 22 | - restore_cache: 23 | key: dependency-cache-{{ checksum "package.json" }} 24 | - run: npm install 25 | - run: npm run build 26 | - run: npm run test:integration 27 | release: 28 | docker: 29 | - image: cimg/node:18.18.0 30 | steps: 31 | - checkout 32 | - vault/get-secrets: # Loads vault secrets 33 | template-preset: "semantic-release-ecosystem" 34 | - run: npm install 35 | - run: npm run build 36 | - run: npm run semantic-release 37 | workflows: 38 | build_and_test: 39 | jobs: 40 | - unit 41 | - integration 42 | - release: 43 | context: 44 | - vault 45 | requires: 46 | - unit 47 | - integration 48 | filters: 49 | branches: 50 | only: 51 | - main 52 | - beta 53 | -------------------------------------------------------------------------------- /.contentful/vault-secrets.yaml: -------------------------------------------------------------------------------- 1 | version: 1 2 | services: 3 | github-action: 4 | policies: 5 | - dependabot 6 | circleci: 7 | policies: 8 | - semantic-release-ecosystem 9 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "standard", 4 | "eslint:recommended", 5 | "plugin:@typescript-eslint/eslint-recommended", 6 | "plugin:@typescript-eslint/recommended" 7 | ], 8 | "plugins": [ 9 | "standard", 10 | "promise", 11 | "jest" 12 | ], 13 | "env": { 14 | "jest/globals": true 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @contentful/team-developer-experience 2 | 3 | package.json 4 | package-lock.json 5 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: npm 5 | directory: "/" 6 | schedule: 7 | interval: daily 8 | time: "00:00" 9 | timezone: UTC 10 | open-pull-requests-limit: 10 11 | ignore: 12 | - dependency-name: husky 13 | versions: 14 | - ">=5.0.0" 15 | - dependency-name: figures # Pure ESM module. Remove when supporting ESM 16 | versions: 17 | - ">=4.0.0" 18 | - dependency-name: bfj 19 | versions: 20 | - ">=9.0.0" 21 | - dependency-name: semantic-release 22 | versions: 23 | - ">=23.0.0" 24 | commit-message: 25 | prefix: build 26 | include: scope 27 | groups: 28 | production-dependencies: 29 | applies-to: version-updates 30 | dependency-type: production 31 | update-types: 32 | - minor 33 | - patch 34 | patterns: 35 | - '*' 36 | dev-dependencies: 37 | applies-to: version-updates 38 | dependency-type: development 39 | update-types: 40 | - minor 41 | - patch 42 | patterns: 43 | - '*' 44 | -------------------------------------------------------------------------------- /.github/workflows/dependabot-approve-and-request-merge.yml: -------------------------------------------------------------------------------- 1 | name: "dependabot approve-and-request-merge" 2 | 3 | on: pull_request_target 4 | 5 | jobs: 6 | worker: 7 | permissions: 8 | contents: write 9 | id-token: write 10 | runs-on: ubuntu-latest 11 | if: github.actor == 'dependabot[bot]' 12 | steps: 13 | - uses: contentful/github-auto-merge@v1 14 | with: 15 | VAULT_URL: ${{ secrets.VAULT_URL }} 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | contentful-export-* 2 | 3 | dist 4 | gh-pages 5 | 6 | # Docker 7 | *.dockerfile 8 | *.dockerignore 9 | 10 | # Node package managers 11 | .npmrc 12 | yarn.lock 13 | 14 | # Created by https://www.gitignore.io/api/vim,code,linux,macos,windows,sublimetext,node 15 | 16 | ### Code ### 17 | # Visual Studio Code - https://code.visualstudio.com/ 18 | .settings/ 19 | .vscode/ 20 | jsconfig.json 21 | 22 | # Export config 23 | config.json 24 | 25 | ### Linux ### 26 | *~ 27 | 28 | # temporary files which can be created if a process still has a handle open of a deleted file 29 | .fuse_hidden* 30 | 31 | # KDE directory preferences 32 | .directory 33 | 34 | # Linux trash folder which might appear on any partition or disk 35 | .Trash-* 36 | 37 | # .nfs files are created when an open file is removed but is still being accessed 38 | .nfs* 39 | 40 | ### macOS ### 41 | *.DS_Store 42 | .AppleDouble 43 | .LSOverride 44 | 45 | # Icon must end with two \r 46 | Icon 47 | 48 | # Thumbnails 49 | ._* 50 | 51 | # Files that might appear in the root of a volume 52 | .DocumentRevisions-V100 53 | .fseventsd 54 | .Spotlight-V100 55 | .TemporaryItems 56 | .Trashes 57 | .VolumeIcon.icns 58 | .com.apple.timemachine.donotpresent 59 | 60 | # Directories potentially created on remote AFP share 61 | .AppleDB 62 | .AppleDesktop 63 | Network Trash Folder 64 | Temporary Items 65 | .apdisk 66 | 67 | ### Node ### 68 | # Logs 69 | logs 70 | *.log 71 | npm-debug.log* 72 | yarn-debug.log* 73 | yarn-error.log* 74 | 75 | # Runtime data 76 | pids 77 | *.pid 78 | *.seed 79 | *.pid.lock 80 | 81 | # Directory for instrumented libs generated by jscoverage/JSCover 82 | lib-cov 83 | 84 | # Coverage directory used by tools like istanbul 85 | coverage 86 | 87 | # nyc test coverage 88 | .nyc_output 89 | 90 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 91 | .grunt 92 | 93 | # Bower dependency directory (https://bower.io/) 94 | bower_components 95 | 96 | # node-waf configuration 97 | .lock-wscript 98 | 99 | # Compiled binary addons (http://nodejs.org/api/addons.html) 100 | build/Release 101 | 102 | # Dependency directories 103 | node_modules/ 104 | jspm_packages/ 105 | 106 | # Typescript v1 declaration files 107 | typings/ 108 | 109 | # Optional npm cache directory 110 | .npm 111 | 112 | # Optional eslint cache 113 | .eslintcache 114 | 115 | # Optional REPL history 116 | .node_repl_history 117 | 118 | # Output of 'npm pack' 119 | *.tgz 120 | 121 | # Yarn Integrity file 122 | .yarn-integrity 123 | 124 | # dotenv environment variables file 125 | .env 126 | .envrc 127 | 128 | ### SublimeText ### 129 | # cache files for sublime text 130 | *.tmlanguage.cache 131 | *.tmPreferences.cache 132 | *.stTheme.cache 133 | 134 | # workspace files are user-specific 135 | *.sublime-workspace 136 | 137 | # project files should be checked into the repository, unless a significant 138 | # proportion of contributors will probably not be using SublimeText 139 | # *.sublime-project 140 | 141 | # sftp configuration file 142 | sftp-config.json 143 | 144 | # Package control specific files 145 | Package Control.last-run 146 | Package Control.ca-list 147 | Package Control.ca-bundle 148 | Package Control.system-ca-bundle 149 | Package Control.cache/ 150 | Package Control.ca-certs/ 151 | Package Control.merged-ca-bundle 152 | Package Control.user-ca-bundle 153 | oscrypto-ca-bundle.crt 154 | bh_unicode_properties.cache 155 | 156 | # Sublime-github package stores a github token in this file 157 | # https://packagecontrol.io/packages/sublime-github 158 | GitHub.sublime-settings 159 | 160 | ### Vim ### 161 | # swap 162 | .sw[a-p] 163 | .*.sw[a-p] 164 | # session 165 | Session.vim 166 | # temporary 167 | .netrwhist 168 | # auto-generated tag files 169 | tags 170 | 171 | ### Windows ### 172 | # Windows thumbnail cache files 173 | Thumbs.db 174 | ehthumbs.db 175 | ehthumbs_vista.db 176 | 177 | # Folder config file 178 | Desktop.ini 179 | 180 | # Recycle Bin used on file shares 181 | $RECYCLE.BIN/ 182 | 183 | # Windows Installer files 184 | *.cab 185 | *.msi 186 | *.msm 187 | *.msp 188 | 189 | # Windows shortcuts 190 | *.lnk 191 | 192 | .idea 193 | .tool-versions 194 | 195 | # End of https://www.gitignore.io/api/vim,code,linux,macos,windows,sublimetext,node 196 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | v18 2 | -------------------------------------------------------------------------------- /CONTRIBUTION.md: -------------------------------------------------------------------------------- 1 | We appreciate any community contributions to this project, whether in the form of issues or Pull Requests. 2 | 3 | This document outlines the we'd like you to follow in terms of commit messages and code style. 4 | 5 | It also explains what to do in case you want to setup the project locally and run tests. 6 | 7 | # Setup 8 | 9 | This project is written in ES2015 and transpiled to ES5 using Babel, to the `dist` directory. This should generally only happen at publishing time, or for testing purposes only. 10 | 11 | Run `npm install` to install all necessary dependencies. When running `npm install` locally, `dist` is not compiled. 12 | 13 | # Code style 14 | 15 | This project uses [standard](https://github.com/feross/standard). Install a relevant editor plugin if you'd like. 16 | 17 | Everywhere where it isn't applicable, follow a style similar to the existing code. 18 | 19 | # Commit messages and issues 20 | 21 | This project uses the [Angular JS Commit Message Conventions](https://docs.google.com/document/d/1QrDFcIiPjSLDn3EL15IJygNPiHORgU1_OOAqWjiDU5Y/edit), via semantic-release. See the semantic-release [Default Commit Message Format](https://github.com/semantic-release/semantic-release#default-commit-message-format) section for more details. 22 | 23 | # Running tests 24 | 25 | This project has unit and integration tests. Both of these run on both Node.js and Browser environments. 26 | 27 | Both of these test environments are setup to deal with Babel and code transpiling, so there's no need to worry about that 28 | 29 | - `npm test` runs all three kinds of tests and generates a coverage report 30 | - `npm run test:only` runs Node.js unit tests without coverage. `npm run test:cover` to run Node.js unit tests with coverage. `npm run test:debug` runs babel-node in debug mode (same as running `node debug`). 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Contentful 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Contentful export tool 2 | 3 | [![npm](https://img.shields.io/npm/v/contentful-export.svg)](https://www.npmjs.com/package/contentful-export) 4 | [![Build Status](https://travis-ci.org/contentful/contentful-export.svg?branch=master)](https://travis-ci.org/contentful/contentful-export) 5 | [![Dependency Status](https://img.shields.io/david/contentful/contentful-export.svg)](https://david-dm.org/contentful/contentful-export) 6 | [![devDependency Status](https://img.shields.io/david/dev/contentful/contentful-import.svg)](https://david-dm.org/contentful/contentful-export#info=devDependencies) 7 | 8 | [![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/) 9 | 10 | [Contentful](https://www.contentful.com) provides a content infrastructure for digital teams to power content in websites, apps, and devices. Unlike a CMS, Contentful was built to integrate with the modern software stack. It offers a central hub for structured content, powerful management and delivery APIs, and a customizable web app that enable developers and content creators to ship digital products faster. 11 | 12 | This is a library that helps you backup your Content Model, Content and Assets or move them to a new Contentful space. _It will support Roles & Permissions in a future version._ 13 | 14 | To import your exported data, please refer to the [contentful-import](https://github.com/contentful/contentful-import) repository. 15 | 16 | ## :exclamation: Usage as CLI 17 | > We moved the CLI version of this tool into our [Contentful CLI](https://github.com/contentful/contentful-cli). This allows our users to use and install only one single CLI tool to get the full Contentful experience. 18 | > 19 | > Please have a look at the [Contentful CLI export command documentation](https://github.com/contentful/contentful-cli/tree/master/docs/space/export) to learn more about how to use this as command line tool. 20 | 21 | 22 | ## :cloud: Pre-requisites && Installation 23 | 24 | ### Pre-requisites 25 | 26 | - Node LTS 27 | 28 | ### :cloud: Installation 29 | 30 | ```bash 31 | npm install contentful-export 32 | ``` 33 | 34 | ## :hand: Usage 35 | 36 | ```javascript 37 | const contentfulExport = require('contentful-export') 38 | 39 | const options = { 40 | spaceId: '', 41 | managementToken: '', 42 | ... 43 | } 44 | 45 | contentfulExport(options) 46 | .then((result) => { 47 | console.log('Your space data:', result) 48 | }) 49 | .catch((err) => { 50 | console.log('Oh no! Some errors occurred!', err) 51 | }) 52 | ``` 53 | 54 | ### Querying 55 | 56 | To scope your export, you are able to pass query parameters. All search parameters of our API are supported as documented in our [API documentation](https://www.contentful.com/developers/docs/references/content-delivery-api/#/reference/search-parameters). 57 | 58 | ```javascript 59 | const contentfulExport = require('contentful-export') 60 | 61 | const options = { 62 | spaceId: '', 63 | managementToken: '', 64 | queryEntries: ['content_type='] 65 | } 66 | 67 | contentfulExport(options) 68 | ... 69 | ``` 70 | 71 | The Export tool also support multiple inline queries. 72 | 73 | ```javascript 74 | const contentfulExport = require('contentful-export') 75 | 76 | const options = { 77 | spaceId: '', 78 | managementToken: '', 79 | queryEntries: [ 80 | 'content_type=', 81 | 'sys.id=' 82 | ] 83 | } 84 | 85 | contentfulExport(options) 86 | ... 87 | ``` 88 | 89 | `queryAssets` uses the same syntax as `queryEntries` 90 | 91 | ### Export an environment 92 | 93 | ```javascript 94 | const contentfulExport = require('contentful-export') 95 | 96 | const options = { 97 | spaceId: '', 98 | managementToken: '', 99 | environmentId: '' 100 | } 101 | 102 | contentfulExport(options) 103 | ... 104 | ``` 105 | 106 | ## :gear: Configuration options 107 | 108 | ### Basics 109 | 110 | #### `spaceId` [string] [required] 111 | ID of the space with source data 112 | 113 | #### `environmentId` [string] [default: 'master'] 114 | ID of the environment in the source space 115 | 116 | #### `managementToken` [string] [required] 117 | Contentful management API token for the space to be exported 118 | 119 | #### `deliveryToken` [string] 120 | Contentful Content Delivery API (CDA) token for the space to be exported. 121 | 122 | Providing `deliveryToken` will export both entries and assets from the 123 | Contentful Delivery API, instead of the Contentful Management API. 124 | This may be useful if you want to export the latest _published_ versions, 125 | as the management API always only exports the entirety of items, with the latest 126 | unpublished content. So if you want to make sure only to see the latest 127 | published changes, provide the `deliveryToken`. 128 | 129 | Just to clarify: When Contentful Management API always returns the latest version (e.g. 50 in this case): 130 | 131 | ``` 132 | "createdAt": "2020-01-06T12:00:00.000Z", 133 | "updatedAt": "2020-04-07T11:00:00.000Z", 134 | "publishedVersion": 23, 135 | "publishedAt": "2020-04-05T14:00:00.000Z", 136 | "publishedCounter": 1, 137 | "version": 50, 138 | ``` 139 | 140 | the Content Delivery API would return the `publishedVersion` (23). CDA responses don't include 141 | version number. 142 | 143 | Note: Tags are only available on the Contentful Management API, so they will not be exported if you provide a Contenful Delivery Token. Tags is a new feature that not all users have access to. 144 | 145 | ### Output 146 | 147 | #### `exportDir` [string] [default: current process working directory] 148 | Defines the path for storing the export JSON file 149 | 150 | #### `saveFile` [boolean] [default: true] 151 | Save the export as a JSON file 152 | 153 | #### `contentFile` [string] 154 | The filename for the exported data 155 | 156 | ### Filtering 157 | 158 | #### `includeDrafts` [boolean] [default: false] 159 | Include drafts in the exported entries. 160 | 161 | The `deliveryToken` option is ignored 162 | when `includeDrafts` has been set as `true`. 163 | If you want to include drafts, there's no point of getting them through the 164 | Content Delivery API. 165 | 166 | #### `includeArchived` [boolean] [default: false] 167 | Include archived entries in the exported entries 168 | 169 | #### `skipContentModel` [boolean] [default: false] 170 | Skip exporting content models 171 | 172 | #### `skipEditorInterfaces` [boolean] [default: false] 173 | Skip exporting editor interfaces 174 | 175 | #### `skipContent` [boolean] [default: false] 176 | Skip exporting assets and entries. 177 | 178 | #### `skipRoles` [boolean] [default: false] 179 | Skip exporting roles and permissions 180 | 181 | #### `skipTags` [boolean] [default: false] 182 | Skip exporting tags 183 | 184 | #### `skipWebhooks` [boolean] [default: false] 185 | Skip exporting webhooks 186 | 187 | #### `stripTags` [boolean] [default: false] 188 | Untag assets and entries 189 | 190 | #### `contentOnly` [boolean] [default: false] 191 | Only export entries and assets 192 | 193 | #### `queryEntries` [array] 194 | Only export entries that match these queries 195 | 196 | #### `queryAssets` [array] 197 | Only export assets that match these queries 198 | 199 | #### `downloadAssets` [boolean] 200 | Download actual asset files 201 | 202 | ### Connection 203 | 204 | #### `host` [string] [default: 'api.contentful.com'] 205 | The Management API host 206 | 207 | #### `hostDelivery` [string] [default: 'cdn.contentful.com'] 208 | The Delivery API host 209 | 210 | #### `proxy` [string] 211 | Proxy configuration in HTTP auth format: `host:port` or `user:password@host:port` 212 | 213 | #### `rawProxy` [boolean] 214 | Pass proxy config to Axios instead of creating a custom httpsAgent 215 | 216 | #### `maxAllowedLimit` [number] [default: 1000] 217 | The number of items per page per request 218 | 219 | #### `headers` [object] 220 | Additional headers to attach to the requests. 221 | 222 | ### Other 223 | 224 | #### `errorLogFile` [string] 225 | Full path to the error log file 226 | 227 | #### `useVerboseRenderer` [boolean] [default: false] 228 | Display progress in new lines instead of displaying a busy spinner and the status in the same line. Useful for CI. 229 | 230 | ## :rescue_worker_helmet: Troubleshooting 231 | 232 | ### Proxy 233 | 234 | Unable to connect to Contentful through your proxy? Try to set the `rawProxy` option to `true`. 235 | 236 | ```javascript 237 | contentfulExport({ 238 | proxy: 'https://cat:dog@example.com:1234', 239 | rawProxy: true, 240 | ... 241 | }) 242 | ``` 243 | 244 | ### Error: 400 - Bad Request - Response size too big. 245 | 246 | Contentful response sizes are limited (find more info in our [technical limit docs](https://www.contentful.com/developers/docs/technical-limits/)). In order to resolve this issue, limit the amount of entities received within a single request by setting the [`maxAllowedLimit`](#maxallowedlimit-number-default-1000) option: 247 | 248 | ```javascript 249 | contentfulExport({ 250 | proxy: 'https://cat:dog@example.com:1234', 251 | rawProxy: true, 252 | maxAllowedLimit: 50 253 | ... 254 | }) 255 | ``` 256 | 257 | ### Embargoed Assets 258 | 259 | If a space is configured to use the [embargoed assets feature](https://www.contentful.com/help/media/embargoed-assets/), certain options will need to be set to use the export/import tooling. When exporting content, the `downloadAssets` option must be set to `true`. This will download the asset files to your local machine. Then, when importing content ([using `contentful-import`](https://github.com/contentful/contentful-import)), the `uploadAssets` option must be set to `true` and the `assetsDirectory` must be set to the directory that contains all of the exported asset folders. 260 | 261 | ```javascript 262 | const contentfulExport = require('contentful-export') 263 | 264 | const options = { 265 | spaceId: '', 266 | managementToken: '', 267 | downloadAssets: true 268 | } 269 | 270 | contentfulExport(options) 271 | ``` 272 | 273 | ## :card_file_box: Exported data structure 274 | 275 | This is an overview of the exported data: 276 | 277 | ```json 278 | { 279 | "contentTypes": [], 280 | "entries": [], 281 | "assets": [], 282 | "locales": [], 283 | "tags": [], 284 | "webhooks": [], 285 | "roles": [], 286 | "editorInterfaces": [] 287 | } 288 | ``` 289 | 290 | *Note:* Tags feature is not available for all users. If you do not have access to this feature, the tags array will always be empty. 291 | 292 | ## :warning: Limitations 293 | 294 | - This tool currently does **not** support the export of space memberships. 295 | - Exported webhooks with credentials will be exported as normal webhooks. Credentials should be added manually afterwards. 296 | - If you have custom UI extensions, you need to reinstall them manually in the new space. 297 | 298 | ## :memo: Changelog 299 | 300 | Read the [releases](https://github.com/contentful/contentful-export/releases) page for more information. 301 | 302 | ## :scroll: License 303 | 304 | This project is licensed under MIT license 305 | 306 | [1]: https://www.contentful.com 307 | -------------------------------------------------------------------------------- /babel.config.json: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": { 7 | "node": "12" 8 | } 9 | } 10 | ] 11 | ], 12 | "plugins": [ 13 | "@babel/plugin-proposal-object-rest-spread", 14 | "add-module-exports" 15 | ] 16 | } 17 | -------------------------------------------------------------------------------- /bin/contentful-export: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | // eslint-disable-next-line 4 | const runContentfulExport = require('../dist/index') 5 | // eslint-disable-next-line 6 | const usageParams = require('../dist/usageParams') 7 | 8 | console.log('We moved the CLI version of this tool into our Contentful CLI.\nThis allows our users to use and install only one single CLI tool to get the full Contentful experience.\nFor more info please visit https://github.com/contentful/contentful-cli/tree/master/docs/space/export') 9 | 10 | runContentfulExport(usageParams) 11 | .then(() => { 12 | process.exit(0) 13 | }) 14 | .catch((err) => { 15 | if (err.name !== 'ContentfulMultiError') { 16 | console.error(err) 17 | } 18 | process.exit(1) 19 | }) 20 | -------------------------------------------------------------------------------- /catalog-info.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: backstage.io/v1alpha1 2 | kind: Component 3 | metadata: 4 | name: contentful-export 5 | description: | 6 | This tool allows you to export a Contentful space to a JSON dump. 7 | annotations: 8 | circleci.com/project-slug: github/contentful/contentful-export 9 | github.com/project-slug: contentful/contentful-export 10 | contentful.com/ci-alert-slack: prd-ecosystem-dx-bots 11 | contentful.com/service-tier: "4" 12 | tags: 13 | - tier-4 14 | spec: 15 | type: cli 16 | lifecycle: production 17 | owner: group:team-developer-experience 18 | -------------------------------------------------------------------------------- /example-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "spaceId": "source space id", 3 | "environmentId": "master", 4 | "managementToken": "destination space management token", 5 | "deliveryToken": "token to export both entries and assets from the Contentful Delivery API", 6 | "exportDir": "/path/to/export/directory", 7 | "saveFile": true, 8 | "contentFile": "export.json", 9 | "includeDrafts": false, 10 | "includeArchived": false, 11 | "skipContentModel": false, 12 | "skipEditorInterfaces": false, 13 | "skipContent": false, 14 | "skipRoles": false, 15 | "skipWebhooks": false, 16 | "contentOnly": false, 17 | "queryEntries": [ 18 | "content_type=", 19 | "sys.id=", 20 | "limit=1000" 21 | ], 22 | "queryAssets": [ 23 | "fields.title=Example" 24 | ], 25 | "downloadAssets": false, 26 | "host": "api.contentful.com", 27 | "proxy": "https://user:password@host:port", 28 | "rawProxy": false, 29 | "maxAllowedLimit": 1000, 30 | "errorLogFile": "/path/to/error.log", 31 | "useVerboseRenderer": false 32 | } -------------------------------------------------------------------------------- /example-config.test.json: -------------------------------------------------------------------------------- 1 | { 2 | "spaceId": "source space id", 3 | "managementToken": "destination space management token" 4 | } 5 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | import { access } from 'fs' 2 | 3 | import bfj from 'bfj' 4 | import Promise from 'bluebird' 5 | import Table from 'cli-table3' 6 | import Listr from 'listr' 7 | import UpdateRenderer from 'listr-update-renderer' 8 | import VerboseRenderer from 'listr-verbose-renderer' 9 | import startCase from 'lodash.startcase' 10 | import mkdirp from 'mkdirp' 11 | import { differenceInSeconds } from 'date-fns/differenceInSeconds' 12 | import { formatDistance } from 'date-fns/formatDistance' 13 | 14 | import { 15 | setupLogging, 16 | displayErrorLog, 17 | wrapTask, 18 | writeErrorLogFile 19 | } from 'contentful-batch-libs' 20 | 21 | import downloadAssets from './tasks/download-assets' 22 | import getSpaceData from './tasks/get-space-data' 23 | import initClient from './tasks/init-client' 24 | 25 | import parseOptions from './parseOptions' 26 | 27 | const accessP = Promise.promisify(access) 28 | 29 | const tableOptions = { 30 | // remove ANSI color codes for better CI/CD compatibility 31 | style: { head: [], border: [] } 32 | } 33 | 34 | function createListrOptions (options) { 35 | if (options.useVerboseRenderer) { 36 | return { 37 | renderer: VerboseRenderer 38 | } 39 | } 40 | return { 41 | renderer: UpdateRenderer, 42 | collapse: false 43 | } 44 | } 45 | 46 | export default function runContentfulExport (params) { 47 | const log = [] 48 | const options = parseOptions(params) 49 | 50 | const listrOptions = createListrOptions(options) 51 | 52 | // Setup custom error listener to store errors for later 53 | setupLogging(log) 54 | 55 | const tasks = new Listr( 56 | [ 57 | { 58 | title: 'Initialize client', 59 | task: wrapTask((ctx) => { 60 | try { 61 | // CMA client 62 | ctx.client = initClient(options) 63 | if (options.deliveryToken && !options.includeDrafts) { 64 | // CDA client for fetching only public entries 65 | ctx.cdaClient = initClient(options, true) 66 | } 67 | return Promise.resolve() 68 | } catch (err) { 69 | return Promise.reject(err) 70 | } 71 | }) 72 | }, 73 | { 74 | title: 'Fetching data from space', 75 | task: (ctx) => { 76 | return getSpaceData({ 77 | client: ctx.client, 78 | cdaClient: ctx.cdaClient, 79 | spaceId: options.spaceId, 80 | environmentId: options.environmentId, 81 | maxAllowedLimit: options.maxAllowedLimit, 82 | includeDrafts: options.includeDrafts, 83 | includeArchived: options.includeArchived, 84 | skipContentModel: options.skipContentModel, 85 | skipEditorInterfaces: options.skipEditorInterfaces, 86 | skipContent: options.skipContent, 87 | skipWebhooks: options.skipWebhooks, 88 | skipRoles: options.skipRoles, 89 | skipTags: options.skipTags, 90 | stripTags: options.stripTags, 91 | listrOptions, 92 | queryEntries: options.queryEntries, 93 | queryAssets: options.queryAssets 94 | }) 95 | } 96 | }, 97 | { 98 | title: 'Download assets', 99 | task: wrapTask(downloadAssets(options)), 100 | skip: (ctx) => 101 | !options.downloadAssets || 102 | !Object.prototype.hasOwnProperty.call(ctx.data, 'assets') 103 | }, 104 | { 105 | title: 'Write export log file', 106 | task: () => { 107 | return new Listr([ 108 | { 109 | title: 'Lookup directory to store the logs', 110 | task: (ctx) => { 111 | return accessP(options.exportDir) 112 | .then(() => { 113 | ctx.logDirectoryExists = true 114 | }) 115 | .catch(() => { 116 | ctx.logDirectoryExists = false 117 | }) 118 | } 119 | }, 120 | { 121 | title: 'Create log directory', 122 | task: () => { 123 | return mkdirp(options.exportDir) 124 | }, 125 | skip: (ctx) => !ctx.logDirectoryExists 126 | }, 127 | { 128 | title: 'Writing data to file', 129 | task: (ctx) => { 130 | return bfj.write(options.logFilePath, ctx.data, { 131 | circular: 'ignore', 132 | space: 2 133 | }) 134 | } 135 | } 136 | ]) 137 | }, 138 | skip: () => !options.saveFile 139 | } 140 | ], 141 | listrOptions 142 | ) 143 | 144 | return tasks 145 | .run({ 146 | data: {} 147 | }) 148 | .then((ctx) => { 149 | const resultTypes = Object.keys(ctx.data) 150 | if (resultTypes.length) { 151 | const resultTable = new Table(tableOptions) 152 | 153 | resultTable.push([{ colSpan: 2, content: 'Exported entities' }]) 154 | 155 | resultTypes.forEach((type) => { 156 | resultTable.push([startCase(type), ctx.data[type].length]) 157 | }) 158 | 159 | console.log(resultTable.toString()) 160 | } else { 161 | console.log('No data was exported') 162 | } 163 | 164 | if ('assetDownloads' in ctx) { 165 | const downloadsTable = new Table(tableOptions) 166 | downloadsTable.push([ 167 | { colSpan: 2, content: 'Asset file download results' } 168 | ]) 169 | downloadsTable.push(['Successful', ctx.assetDownloads.successCount]) 170 | downloadsTable.push(['Warnings ', ctx.assetDownloads.warningCount]) 171 | downloadsTable.push(['Errors ', ctx.assetDownloads.errorCount]) 172 | console.log(downloadsTable.toString()) 173 | } 174 | 175 | const endTime = new Date() 176 | const durationHuman = formatDistance(endTime, options.startTime) 177 | const durationSeconds = differenceInSeconds(endTime, options.startTime) 178 | 179 | console.log(`The export took ${durationHuman} (${durationSeconds}s)`) 180 | if (options.saveFile) { 181 | console.log( 182 | `\nStored space data to json file at: ${options.logFilePath}` 183 | ) 184 | } 185 | return ctx.data 186 | }) 187 | .catch((err) => { 188 | log.push({ 189 | ts: new Date().toJSON(), 190 | level: 'error', 191 | error: err 192 | }) 193 | }) 194 | .then((data) => { 195 | // @todo this should live in batch libs 196 | const errorLog = log.filter( 197 | (logMessage) => 198 | logMessage.level !== 'info' && logMessage.level !== 'warning' 199 | ) 200 | const displayLog = log.filter( 201 | (logMessage) => logMessage.level !== 'info' 202 | ) 203 | displayErrorLog(displayLog) 204 | 205 | if (errorLog.length) { 206 | return writeErrorLogFile(options.errorLogFile, errorLog).then(() => { 207 | const multiError = new Error('Errors occured') 208 | multiError.name = 'ContentfulMultiError' 209 | Object.assign(multiError, { errors: errorLog }) 210 | throw multiError 211 | }) 212 | } 213 | 214 | console.log('The export was successful.') 215 | 216 | return data 217 | }) 218 | } 219 | -------------------------------------------------------------------------------- /lib/parseOptions.js: -------------------------------------------------------------------------------- 1 | import { addSequenceHeader, agentFromProxy, proxyStringToObject } from 'contentful-batch-libs' 2 | import { format } from 'date-fns/format' 3 | import { resolve } from 'path' 4 | import qs from 'querystring' 5 | 6 | import { version } from '../package.json' 7 | import { getHeadersConfig } from './utils/headers' 8 | 9 | export default function parseOptions (params) { 10 | const defaultOptions = { 11 | environmentId: 'master', 12 | exportDir: process.cwd(), 13 | includeDrafts: false, 14 | includeArchived: false, 15 | skipRoles: false, 16 | skipContentModel: false, 17 | skipEditorInterfaces: false, 18 | skipContent: false, 19 | skipWebhooks: false, 20 | skipTags: false, 21 | stripTags: false, 22 | maxAllowedLimit: 1000, 23 | saveFile: true, 24 | useVerboseRenderer: false, 25 | rawProxy: false 26 | } 27 | 28 | const configFile = params.config 29 | // eslint-disable-next-line @typescript-eslint/no-require-imports 30 | ? require(resolve(process.cwd(), params.config)) 31 | : {} 32 | 33 | const options = { 34 | ...defaultOptions, 35 | ...configFile, 36 | ...params, 37 | headers: addSequenceHeader(params.headers || getHeadersConfig(params.header)) 38 | } 39 | 40 | // Validation 41 | if (!options.spaceId) { 42 | throw new Error('The `spaceId` option is required.') 43 | } 44 | 45 | if (!options.managementToken) { 46 | throw new Error('The `managementToken` option is required.') 47 | } 48 | 49 | options.startTime = new Date() 50 | options.contentFile = options.contentFile || `contentful-export-${options.spaceId}-${options.environmentId}-${format(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json` 51 | 52 | options.logFilePath = resolve(options.exportDir, options.contentFile) 53 | 54 | if (!options.errorLogFile) { 55 | options.errorLogFile = resolve(options.exportDir, `contentful-export-error-log-${options.spaceId}-${options.environmentId}-${format(options.startTime, "yyyy-MM-dd'T'HH-mm-ss")}.json`) 56 | } else { 57 | options.errorLogFile = resolve(process.cwd(), options.errorLogFile) 58 | } 59 | 60 | // Further processing 61 | options.accessToken = options.managementToken 62 | 63 | if (options.proxy) { 64 | if (typeof options.proxy === 'string') { 65 | const proxySimpleExp = /.+:\d+/ 66 | const proxyAuthExp = /.+:.+@.+:\d+/ 67 | if (!(proxySimpleExp.test(options.proxy) || proxyAuthExp.test(options.proxy))) { 68 | throw new Error('Please provide the proxy config in the following format:\nhost:port or user:password@host:port') 69 | } 70 | options.proxy = proxyStringToObject(options.proxy) 71 | } 72 | 73 | if (!options.rawProxy) { 74 | options.httpsAgent = agentFromProxy(options.proxy) 75 | delete options.proxy 76 | } 77 | } 78 | 79 | if (options.queryEntries && options.queryEntries.length > 0) { 80 | const querystr = options.queryEntries.join('&') 81 | options.queryEntries = qs.parse(querystr) 82 | } 83 | 84 | if (options.queryAssets && options.queryAssets.length > 0) { 85 | const querystr = options.queryAssets.join('&') 86 | options.queryAssets = qs.parse(querystr) 87 | } 88 | 89 | if (options.contentOnly) { 90 | options.skipRoles = true 91 | options.skipContentModel = true 92 | options.skipWebhooks = true 93 | } 94 | 95 | options.application = options.managementApplication || `contentful.export/${version}` 96 | options.feature = options.managementFeature || 'library-export' 97 | return options 98 | } 99 | -------------------------------------------------------------------------------- /lib/tasks/download-assets.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird' 2 | import { getEntityName } from 'contentful-batch-libs' 3 | import figures from 'figures' 4 | import { createWriteStream, promises as fs } from 'fs' 5 | import path from 'path' 6 | import { pipeline } from 'stream' 7 | import { promisify } from 'util' 8 | import { calculateExpiryTimestamp, isEmbargoedAsset, signUrl } from '../utils/embargoedAssets' 9 | import axios from 'axios' 10 | 11 | const streamPipeline = promisify(pipeline) 12 | 13 | /** 14 | * @param {Object} options - The options for downloading the asset. 15 | * @param {string} options.url - The URL of the asset to download. 16 | * @param {string} options.directory - The directory where the asset should be saved. 17 | * @param {import('axios').AxiosInstance} options.httpClient - The HTTP client to use for downloading the asset. 18 | */ 19 | async function downloadAsset ({ url, directory, httpClient }) { 20 | // handle urls without protocol 21 | if (url.startsWith('//')) { 22 | url = 'https:' + url 23 | } 24 | 25 | // build local file path from the url for the download 26 | const parsedUrl = new URL(url) 27 | const localFile = path.join(directory, parsedUrl.host, parsedUrl.pathname) 28 | 29 | // ensure directory exists and create file stream 30 | await fs.mkdir(path.dirname(localFile), { recursive: true }) 31 | const file = createWriteStream(localFile) 32 | 33 | try { 34 | // download asset 35 | const assetRequest = await httpClient.get(url, { 36 | responseType: "stream", 37 | transformResponse: [(data) => data], 38 | }) 39 | 40 | // Wait for stream to be consumed before returning local file 41 | await streamPipeline(assetRequest.data, file) 42 | return localFile 43 | } catch (e) { 44 | /** 45 | * @type {import('axios').AxiosError} 46 | */ 47 | const axiosError = e 48 | throw new Error(`error response status: ${axiosError.response.status}`) 49 | } 50 | } 51 | 52 | export default function downloadAssets (options) { 53 | return (ctx, task) => { 54 | let successCount = 0 55 | let warningCount = 0 56 | let errorCount = 0 57 | 58 | const httpClient = axios.create({ 59 | headers: options.headers, 60 | timeout: options.timeout, 61 | httpAgent: options.httpAgent, 62 | httpsAgent: options.httpsAgent, 63 | proxy: options.proxy 64 | }) 65 | 66 | return Promise.map(ctx.data.assets, (asset) => { 67 | const entityName = getEntityName(asset) 68 | if (!asset.fields.file) { 69 | task.output = `${figures.warning} asset ${entityName} has no file(s)` 70 | warningCount++ 71 | return 72 | } 73 | const locales = Object.keys(asset.fields.file) 74 | return Promise.mapSeries(locales, (locale) => { 75 | const url = asset.fields.file[locale].url 76 | if (!url) { 77 | task.output = `${figures.cross} asset '${entityName}' doesn't contain an url in path asset.fields.file[${locale}].url` 78 | errorCount++ 79 | 80 | return Promise.resolve() 81 | } 82 | 83 | let startingPromise = Promise.resolve({ url, directory: options.exportDir, httpClient }) 84 | 85 | if (isEmbargoedAsset(url)) { 86 | const { host, accessToken, spaceId, environmentId } = options 87 | const expiresAtMs = calculateExpiryTimestamp() 88 | 89 | startingPromise = signUrl(host, accessToken, spaceId, environmentId, url, expiresAtMs, httpClient) 90 | .then((signedUrl) => ({ url: signedUrl, directory: options.exportDir, httpClient })) 91 | } 92 | 93 | return startingPromise 94 | .then(downloadAsset) 95 | .then((_downLoadedFile) => { 96 | task.output = `${figures.tick} downloaded ${entityName} (${url})` 97 | successCount++ 98 | }) 99 | .catch((error) => { 100 | task.output = `${figures.cross} error downloading ${url}: ${error.message}` 101 | errorCount++ 102 | }) 103 | }) 104 | }, { 105 | concurrency: 6 106 | }) 107 | .then(() => { 108 | ctx.assetDownloads = { 109 | successCount, 110 | warningCount, 111 | errorCount 112 | } 113 | }) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /lib/tasks/get-space-data.js: -------------------------------------------------------------------------------- 1 | import Promise from 'bluebird' 2 | import { logEmitter, wrapTask } from 'contentful-batch-libs' 3 | import Listr from 'listr' 4 | import verboseRenderer from 'listr-verbose-renderer' 5 | 6 | const MAX_ALLOWED_LIMIT = 1000 7 | let pageLimit = MAX_ALLOWED_LIMIT 8 | 9 | /** 10 | * Gets all the content from a space via the management API. This includes 11 | * content in draft state. 12 | */ 13 | export default function getFullSourceSpace ({ 14 | client, 15 | cdaClient, 16 | spaceId, 17 | environmentId = 'master', 18 | skipContentModel, 19 | skipContent, 20 | skipWebhooks, 21 | skipRoles, 22 | skipEditorInterfaces, 23 | skipTags, 24 | stripTags, 25 | includeDrafts, 26 | includeArchived, 27 | maxAllowedLimit, 28 | listrOptions, 29 | queryEntries, 30 | queryAssets 31 | }) { 32 | pageLimit = maxAllowedLimit || MAX_ALLOWED_LIMIT 33 | listrOptions = listrOptions || { 34 | renderer: verboseRenderer 35 | } 36 | 37 | return new Listr([ 38 | { 39 | title: 'Connecting to space', 40 | task: wrapTask((ctx) => { 41 | return client.getSpace(spaceId) 42 | .then((space) => { 43 | ctx.space = space 44 | return space.getEnvironment(environmentId) 45 | }) 46 | .then((environment) => { 47 | ctx.environment = environment 48 | }) 49 | }) 50 | }, 51 | { 52 | title: 'Fetching content types data', 53 | task: wrapTask((ctx) => { 54 | return pagedGet({ source: ctx.environment, method: 'getContentTypes' }) 55 | .then(extractItems) 56 | .then((items) => { 57 | ctx.data.contentTypes = items 58 | }) 59 | }), 60 | skip: () => skipContentModel 61 | }, 62 | { 63 | title: 'Fetching tags data', 64 | task: wrapTask((ctx) => { 65 | return pagedGet({ source: ctx.environment, method: 'getTags' }) 66 | .then(extractItems) 67 | .then((items) => { 68 | ctx.data.tags = items 69 | }) 70 | .catch(() => { 71 | ctx.data.tags = [] 72 | }) 73 | }), 74 | skip: () => skipTags 75 | }, 76 | { 77 | title: 'Fetching editor interfaces data', 78 | task: wrapTask((ctx) => { 79 | return getEditorInterfaces(ctx.data.contentTypes) 80 | .then((editorInterfaces) => { 81 | ctx.data.editorInterfaces = editorInterfaces.filter((editorInterface) => { 82 | return editorInterface !== null 83 | }) 84 | }) 85 | }), 86 | skip: (ctx) => skipContentModel || skipEditorInterfaces || (ctx.data.contentTypes.length === 0 && 'Skipped since no content types downloaded') 87 | }, 88 | { 89 | title: 'Fetching content entries data', 90 | task: wrapTask((ctx) => { 91 | const source = cdaClient?.withAllLocales || ctx.environment 92 | if (cdaClient) { 93 | // let's not fetch children when using Content Delivery API 94 | queryEntries = queryEntries || {} 95 | queryEntries.include = 0 96 | } 97 | return pagedGet({ source, method: 'getEntries', query: queryEntries }) 98 | .then(extractItems) 99 | .then((items) => filterDrafts(items, includeDrafts, cdaClient)) 100 | .then((items) => filterArchived(items, includeArchived)) 101 | .then((items) => removeTags(items, stripTags)) 102 | .then((items) => { 103 | ctx.data.entries = items 104 | }) 105 | }), 106 | skip: () => skipContent 107 | }, 108 | { 109 | title: 'Fetching assets data', 110 | task: wrapTask((ctx) => { 111 | const source = cdaClient?.withAllLocales || ctx.environment 112 | queryAssets = queryAssets || {} 113 | return pagedGet({ source, method: 'getAssets', query: queryAssets }) 114 | .then(extractItems) 115 | .then((items) => filterDrafts(items, includeDrafts, cdaClient)) 116 | .then((items) => filterArchived(items, includeArchived)) 117 | .then((items) => removeTags(items, stripTags)) 118 | .then((items) => { 119 | ctx.data.assets = items 120 | }) 121 | }), 122 | skip: () => skipContent 123 | }, 124 | { 125 | title: 'Fetching locales data', 126 | task: wrapTask((ctx) => { 127 | return pagedGet({ source: ctx.environment, method: 'getLocales' }) 128 | .then(extractItems) 129 | .then((items) => { 130 | ctx.data.locales = items 131 | }) 132 | }), 133 | skip: () => skipContentModel 134 | }, 135 | { 136 | title: 'Fetching webhooks data', 137 | task: wrapTask((ctx) => { 138 | return pagedGet({ source: ctx.space, method: 'getWebhooks' }) 139 | .then(extractItems) 140 | .then((items) => { 141 | ctx.data.webhooks = items 142 | }) 143 | }), 144 | skip: () => skipWebhooks || (environmentId !== 'master' && 'Webhooks can only be exported from master environment') 145 | }, 146 | { 147 | title: 'Fetching roles data', 148 | task: wrapTask((ctx) => { 149 | return pagedGet({ source: ctx.space, method: 'getRoles' }) 150 | .then(extractItems) 151 | .then((items) => { 152 | ctx.data.roles = items 153 | }) 154 | }), 155 | skip: () => skipRoles || (environmentId !== 'master' && 'Roles can only be exported from master environment') 156 | } 157 | ], listrOptions) 158 | } 159 | 160 | function getEditorInterfaces (contentTypes) { 161 | return Promise.map(contentTypes, (contentType) => { 162 | return contentType.getEditorInterface() 163 | .then((editorInterface) => { 164 | logEmitter.emit('info', `Fetched editor interface for ${contentType.name}`) 165 | return editorInterface 166 | }) 167 | .catch(() => { 168 | // old contentTypes may not have an editor interface but we'll handle in a later stage 169 | // but it should not stop getting the data process 170 | logEmitter.emit('warning', `No editor interface found for ${contentType}`) 171 | return Promise.resolve(null) 172 | }) 173 | }, { 174 | concurrency: 6 175 | }) 176 | } 177 | 178 | /** 179 | * Gets all the existing entities based on pagination parameters. 180 | * The first call will have no aggregated response. Subsequent calls will 181 | * concatenate the new responses to the original one. 182 | */ 183 | function pagedGet ({ source, method, skip = 0, aggregatedResponse = null, query = null }) { 184 | const userQueryLimit = query && query.limit 185 | const fetchedTotal = aggregatedResponse && aggregatedResponse.items.length 186 | const limit = userQueryLimit ? Math.min(pageLimit, userQueryLimit - fetchedTotal) : pageLimit 187 | 188 | const requestQuery = Object.assign({}, 189 | { 190 | skip, 191 | order: 'sys.createdAt,sys.id' 192 | }, 193 | query, 194 | { 195 | limit 196 | } 197 | ) 198 | 199 | return source[method](requestQuery) 200 | .then((response) => { 201 | if (!aggregatedResponse) { 202 | aggregatedResponse = response 203 | } else { 204 | aggregatedResponse.items = aggregatedResponse.items.concat(response.items) 205 | } 206 | 207 | const totalItemsLength = aggregatedResponse.items.length 208 | const total = response.total 209 | 210 | logPagingStatus(response, requestQuery, userQueryLimit) 211 | 212 | const gotAllQueryLimitedItems = userQueryLimit && totalItemsLength >= userQueryLimit 213 | const gotAllItems = totalItemsLength >= total 214 | const gotNoItems = totalItemsLength <= 0 215 | if (gotAllQueryLimitedItems || gotAllItems || gotNoItems) { 216 | return aggregatedResponse 217 | } 218 | return pagedGet({ source, method, skip: skip + response.items.length, aggregatedResponse, query }) 219 | }) 220 | } 221 | 222 | function logPagingStatus (response, requestQuery, userLimit) { 223 | const { total, limit, items } = response 224 | const pagedItemsLength = items.length 225 | 226 | // sometimes our pageLimit or queryLimit of 1000 is overridden by the API (like in locales) 227 | const imposedLimit = limit || requestQuery.limit 228 | const limitedTotal = userLimit ? Math.min(userLimit, total) : total 229 | const page = Math.ceil(requestQuery.skip / imposedLimit) + 1 230 | const pages = Math.ceil(limitedTotal / imposedLimit) 231 | logEmitter.emit('info', `Fetched ${pagedItemsLength} of ${total} items (Page ${page}/${pages})`) 232 | } 233 | 234 | function extractItems (response) { 235 | return response.items 236 | } 237 | 238 | function filterDrafts (items, includeDrafts, cdaClient) { 239 | // CDA filters drafts based on host, no need to do filtering here 240 | return (includeDrafts || cdaClient) ? items : items.filter((item) => !!item.sys.publishedVersion || !!item.sys.archivedVersion) 241 | } 242 | 243 | function filterArchived (items, includeArchived) { 244 | return includeArchived ? items : items.filter((item) => !item.sys.archivedVersion) 245 | } 246 | 247 | function removeTags (items, stripTags) { 248 | if (stripTags) { 249 | items.forEach(item => { 250 | if (item.metadata?.tags) { 251 | item.metadata.tags = [] 252 | } 253 | }) 254 | } 255 | return items 256 | } 257 | -------------------------------------------------------------------------------- /lib/tasks/init-client.js: -------------------------------------------------------------------------------- 1 | import { createClient as createCdaClient } from 'contentful' 2 | import { logEmitter } from 'contentful-batch-libs' 3 | import { createClient as createCmaClient } from 'contentful-management' 4 | 5 | function logHandler (level, data) { 6 | logEmitter.emit(level, data) 7 | } 8 | 9 | export default function initClient (opts, useCda = false) { 10 | const defaultOpts = { 11 | timeout: 10000, 12 | logHandler 13 | } 14 | const config = { 15 | ...defaultOpts, 16 | ...opts 17 | } 18 | if (useCda) { 19 | const cdaConfig = { 20 | ...config, 21 | space: config.spaceId, 22 | accessToken: config.deliveryToken, 23 | environment: config.environmentId, 24 | host: config.hostDelivery 25 | } 26 | return createCdaClient(cdaConfig).withoutLinkResolution 27 | } 28 | return createCmaClient(config) 29 | } 30 | -------------------------------------------------------------------------------- /lib/usageParams.js: -------------------------------------------------------------------------------- 1 | import yargs from 'yargs' 2 | import packageFile from '../package.json' 3 | 4 | export default yargs 5 | .version(packageFile.version || 'Version only available on installed package') 6 | .usage('Usage: $0 [options]') 7 | .option('space-id', { 8 | describe: 'ID of Space with source data', 9 | type: 'string', 10 | demand: true 11 | }) 12 | .option('environment-id', { 13 | describe: 'ID of Environment with source data', 14 | type: 'string', 15 | default: 'master' 16 | }) 17 | .option('management-token', { 18 | describe: 'Contentful management API token for the space to be exported', 19 | type: 'string', 20 | demand: true 21 | }) 22 | .option('delivery-token', { 23 | describe: 'Contentful Content Delivery API token for the space to be exported', 24 | type: 'string' 25 | }) 26 | .option('export-dir', { 27 | describe: 'Defines the path for storing the export json file (default path is the current directory)', 28 | type: 'string' 29 | }) 30 | .option('include-drafts', { 31 | describe: 'Include drafts in the exported entries', 32 | type: 'boolean', 33 | default: false 34 | }) 35 | .option('include-archived', { 36 | describe: 'Include archived entries in the exported entries', 37 | type: 'boolean', 38 | default: false 39 | }) 40 | .option('skip-content-model', { 41 | describe: 'Skip exporting content models', 42 | type: 'boolean', 43 | default: false 44 | }) 45 | .option('skip-content', { 46 | describe: 'Skip exporting assets and entries', 47 | type: 'boolean', 48 | default: false 49 | }) 50 | .option('skip-roles', { 51 | describe: 'Skip exporting roles and permissions', 52 | type: 'boolean', 53 | default: false 54 | }) 55 | .options('skip-tags', { 56 | describe: 'Skip exporting tags', 57 | type: 'boolean', 58 | default: false 59 | }) 60 | .option('skip-webhooks', { 61 | describe: 'Skip exporting webhooks', 62 | type: 'boolean', 63 | default: false 64 | }) 65 | .options('strip-tags', { 66 | describe: 'Untag assets and entries', 67 | type: 'boolean', 68 | default: false 69 | }) 70 | .option('content-only', { 71 | describe: 'only export entries and assets', 72 | type: 'boolean', 73 | default: false 74 | }) 75 | .option('download-assets', { 76 | describe: 'With this flags asset files will also be downloaded', 77 | type: 'boolean' 78 | }) 79 | .option('max-allowed-limit', { 80 | describe: 'How many items per page per request', 81 | type: 'number', 82 | default: 1000 83 | }) 84 | .option('host', { 85 | describe: 'Management API host', 86 | type: 'string', 87 | default: 'api.contentful.com' 88 | }) 89 | .option('host-delivery', { 90 | describe: 'Delivery API host', 91 | type: 'string', 92 | default: 'cdn.contentful.com' 93 | }) 94 | .option('proxy', { 95 | describe: 'Proxy configuration in HTTP auth format: [http|https]://host:port or [http|https]://user:password@host:port', 96 | type: 'string' 97 | }) 98 | .option('raw-proxy', { 99 | describe: 'Pass proxy config to Axios instead of creating a custom httpsAgent', 100 | type: 'boolean', 101 | default: false 102 | }) 103 | .option('error-log-file', { 104 | describe: 'Full path to the error log file', 105 | type: 'string' 106 | }) 107 | .option('query-entries', { 108 | describe: 'Exports only entries that matches these queries', 109 | type: 'array' 110 | }) 111 | .option('query-assets', { 112 | describe: 'Exports only assets that matches these queries', 113 | type: 'array' 114 | }) 115 | .option('content-file', { 116 | describe: 'The filename for the exported data', 117 | type: 'string' 118 | }) 119 | .option('save-file', { 120 | describe: 'Save the export as a json file', 121 | type: 'boolean', 122 | default: true 123 | }) 124 | .option('use-verbose-renderer', { 125 | describe: 'Display progress in new lines instead of displaying a busy spinner and the status in the same line. Useful for CI.', 126 | type: 'boolean', 127 | default: false 128 | }) 129 | .option('header', { 130 | alias: 'H', 131 | type: 'string', 132 | describe: 'Pass an additional HTTP Header' 133 | }) 134 | .config('config', 'An optional configuration JSON file containing all the options for a single run') 135 | .argv 136 | -------------------------------------------------------------------------------- /lib/utils/embargoedAssets.js: -------------------------------------------------------------------------------- 1 | import jwt from 'jsonwebtoken' 2 | 3 | const SIX_HOURS_IN_MS = 6 * 60 * 60 * 1000 4 | const assetKeyCache = new Map() 5 | 6 | /** 7 | * @param {string} host - The Contentful API host. 8 | * @param {string} accessToken - The access token for the Contentful API. 9 | * @param {string} spaceId - The ID of the Contentful space. 10 | * @param {string} environmentId - The ID of the Contentful environment. 11 | * @param {number} expiresAtMs - The expiration time in milliseconds. 12 | * @param {import('axios').AxiosInstance} httpClient - The HTTP client to use for requests. 13 | */ 14 | function createAssetKey (host, accessToken, spaceId, environmentId, expiresAtMs, httpClient) { 15 | return httpClient(`https://${host}/spaces/${spaceId}/environments/${environmentId}/asset_keys`, { 16 | method: 'POST', 17 | data: JSON.stringify({ 18 | expiresAt: Math.floor(expiresAtMs / 1000) // in seconds 19 | }), 20 | headers: { 21 | Authorization: `Bearer ${accessToken}`, 22 | 'Content-Type': 'application/json' 23 | } 24 | }) 25 | } 26 | 27 | export const shouldCreateNewCacheItem = (cacheItem, currentExpiresAtMs) => 28 | !cacheItem || currentExpiresAtMs - cacheItem.expiresAtMs > SIX_HOURS_IN_MS 29 | 30 | async function createCachedAssetKey (host, accessToken, spaceId, environmentId, minExpiresAtMs, httpClient) { 31 | const cacheKey = `${host}:${spaceId}:${environmentId}` 32 | let cacheItem = assetKeyCache.get(cacheKey) 33 | 34 | if (shouldCreateNewCacheItem(cacheItem, minExpiresAtMs)) { 35 | const expiresAtMs = calculateExpiryTimestamp() 36 | 37 | if (minExpiresAtMs > expiresAtMs) { 38 | throw new Error(`Cannot fetch an asset key so far in the future: ${minExpiresAtMs} > ${expiresAtMs}`) 39 | } 40 | 41 | try { 42 | const assetKeyResponse = await createAssetKey(host, accessToken, spaceId, environmentId, expiresAtMs, httpClient) 43 | cacheItem = { expiresAtMs, result: assetKeyResponse.data } 44 | assetKeyCache.set(cacheKey, cacheItem) 45 | } catch (err) { 46 | // If we encounter an error, make sure to clear the cache item if this is the most recent fetch. 47 | const curCacheItem = assetKeyCache.get(cacheKey) 48 | if (curCacheItem === cacheItem) { 49 | assetKeyCache.delete(cacheKey) 50 | } 51 | 52 | return Promise.reject(err) 53 | } 54 | } 55 | 56 | return cacheItem.result 57 | } 58 | 59 | function generateSignedToken (secret, urlWithoutQueryParams, expiresAtMs) { 60 | // Convert expiresAtMs to seconds, if defined 61 | const exp = expiresAtMs ? Math.floor(expiresAtMs / 1000) : undefined 62 | return jwt.sign({ 63 | sub: urlWithoutQueryParams, 64 | exp 65 | }, secret, { algorithm: 'HS256' }) 66 | } 67 | 68 | function generateSignedUrl (policy, secret, url, expiresAtMs) { 69 | const parsedUrl = new URL(url) 70 | 71 | const urlWithoutQueryParams = parsedUrl.origin + parsedUrl.pathname 72 | const token = generateSignedToken(secret, urlWithoutQueryParams, expiresAtMs) 73 | 74 | parsedUrl.searchParams.set('token', token) 75 | parsedUrl.searchParams.set('policy', policy) 76 | 77 | return parsedUrl.toString() 78 | } 79 | 80 | export function isEmbargoedAsset (url) { 81 | const pattern = /((images)|(assets)|(downloads)|(videos))\.secure\./ 82 | return pattern.test(url) 83 | } 84 | 85 | export function calculateExpiryTimestamp () { 86 | return Date.now() + SIX_HOURS_IN_MS 87 | } 88 | 89 | /** 90 | * @param {string} host - The Contentful API host. 91 | * @param {string} accessToken - The access token for the Contentful API. 92 | * @param {string} spaceId - The ID of the Contentful space. 93 | * @param {string} environmentId - The ID of the Contentful environment. 94 | * @param {string} url - The URL to be signed. 95 | * @param {number} expiresAtMs - The expiration time in milliseconds. 96 | * @param {import('axios').AxiosInstance} httpClient - The HTTP client to use for requests. 97 | */ 98 | export function signUrl (host, accessToken, spaceId, environmentId, url, expiresAtMs, httpClient) { 99 | // handle urls without protocol 100 | if (url.startsWith('//')) { 101 | url = 'https:' + url 102 | } 103 | 104 | return createCachedAssetKey(host, accessToken, spaceId, environmentId, expiresAtMs, httpClient) 105 | .then(({ policy, secret }) => generateSignedUrl(policy, secret, url, expiresAtMs)) 106 | } 107 | -------------------------------------------------------------------------------- /lib/utils/headers.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Turn header option into an object. Invalid header values 3 | * are ignored. 4 | * 5 | * @example 6 | * getHeadersConfig('Accept: Any') 7 | * // -> {Accept: 'Any'} 8 | * 9 | * @example 10 | * getHeadersConfig(['Accept: Any', 'X-Version: 1']) 11 | * // -> {Accept: 'Any', 'X-Version': '1'} 12 | * 13 | * @param value {string|string[]} 14 | */ 15 | export function getHeadersConfig (value) { 16 | if (!value) { 17 | return {} 18 | } 19 | 20 | const values = Array.isArray(value) ? value : [value] 21 | 22 | return values.reduce((headers, value) => { 23 | value = value.trim() 24 | 25 | const separatorIndex = value.indexOf(':') 26 | 27 | // Invalid header format 28 | if (separatorIndex === -1) { 29 | return headers 30 | } 31 | 32 | const headerKey = value.slice(0, separatorIndex).trim() 33 | const headerValue = value.slice(separatorIndex + 1).trim() 34 | 35 | return { 36 | ...headers, 37 | [headerKey]: headerValue 38 | } 39 | }, {}) 40 | } 41 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "contentful-export", 3 | "version": "0.0.0-determined-by-semantic-release", 4 | "description": "this tool allows you to export a space to a JSON dump", 5 | "main": "dist/index.js", 6 | "types": "types.d.ts", 7 | "engines": { 8 | "node": ">=18" 9 | }, 10 | "bin": { 11 | "contentful-export": "./bin/contentful-export" 12 | }, 13 | "scripts": { 14 | "build": "npm run clean && npm run check && babel lib --out-dir dist", 15 | "build:watch": "babel lib --out-dir dist --watch", 16 | "check": "tsc", 17 | "clean": "rimraf dist && rimraf coverage", 18 | "lint": "eslint lib bin/* types.d.ts", 19 | "lint:fix": "npm run lint -- --fix", 20 | "pretest": "npm run lint && npm run build && rimraf ./test/integration/tmp", 21 | "test": "npm run test:unit && npm run test:integration", 22 | "test:unit": "jest --testPathPattern=test/unit --coverage", 23 | "test:unit:debug": "node --inspect-brk ./node_modules/.bin/jest --runInBand --watch --testPathPattern=test/unit", 24 | "test:unit:watch": "npm run test:unit -- --watch", 25 | "test:integration": "jest --testPathPattern=test/integration", 26 | "test:integration:debug": "node --inspect-brk ./node_modules/.bin/jest --runInBand --watch --testPathPattern=test/integration", 27 | "test:integration:watch": "npm run test:integration -- --watch", 28 | "semantic-release": "semantic-release", 29 | "prepublishOnly": "npm run build", 30 | "postpublish": "npm run clean", 31 | "precommit": "npm run lint", 32 | "prepush": "npm run test" 33 | }, 34 | "repository": { 35 | "type": "git", 36 | "url": "https://github.com/contentful/contentful-export.git" 37 | }, 38 | "keywords": [ 39 | "contentful", 40 | "contentful-export" 41 | ], 42 | "author": "Contentful ", 43 | "license": "MIT", 44 | "bugs": { 45 | "url": "https://github.com/contentful/contentful-export/issues" 46 | }, 47 | "dependencies": { 48 | "axios": "^1.8.4", 49 | "bfj": "^8.0.0", 50 | "bluebird": "^3.3.3", 51 | "cli-table3": "^0.6.0", 52 | "contentful": "^11.5.10", 53 | "contentful-batch-libs": "^9.4.1", 54 | "contentful-management": "^11.48.1", 55 | "date-fns": "^4.1.0", 56 | "figures": "^3.2.0", 57 | "jsonwebtoken": "^9.0.0", 58 | "listr": "^0.14.1", 59 | "listr-update-renderer": "^0.5.0", 60 | "listr-verbose-renderer": "^0.6.0", 61 | "lodash.startcase": "^4.4.0", 62 | "mkdirp": "^2.0.0", 63 | "yargs": "^18.0.0" 64 | }, 65 | "devDependencies": { 66 | "@babel/cli": "^7.0.0", 67 | "@babel/core": "^7.0.0", 68 | "@babel/plugin-proposal-object-rest-spread": "^7.0.0", 69 | "@babel/preset-env": "^7.0.0", 70 | "@babel/template": "^7.0.0", 71 | "@babel/types": "^7.0.0", 72 | "@types/jest": "^29.0.0", 73 | "@typescript-eslint/eslint-plugin": "^8.2.0", 74 | "babel-jest": "^29.0.0", 75 | "babel-plugin-add-module-exports": "^1.0.2", 76 | "cz-conventional-changelog": "^3.3.0", 77 | "eslint": "^8.27.0", 78 | "eslint-config-standard": "^17.0.0", 79 | "eslint-plugin-import": "^2.12.0", 80 | "eslint-plugin-jest": "^28.5.0", 81 | "eslint-plugin-node": "^11.1.0", 82 | "eslint-plugin-promise": "^6.0.0", 83 | "eslint-plugin-standard": "^5.0.0", 84 | "https-proxy-agent": "^7.0.0", 85 | "husky": "^4.3.8", 86 | "jest": "^29.0.0", 87 | "nixt": "^0.5.0", 88 | "nock": "^14.0.0", 89 | "opener": "^1.4.1", 90 | "rimraf": "^4.0.7", 91 | "semantic-release": "^22.0.12" 92 | }, 93 | "files": [ 94 | "bin", 95 | "dist", 96 | "example-config.json", 97 | "index.js", 98 | "types.d.ts" 99 | ], 100 | "config": { 101 | "commitizen": { 102 | "path": "./node_modules/cz-conventional-changelog" 103 | } 104 | }, 105 | "release": { 106 | "branches": [ 107 | "main", 108 | { 109 | "name": "beta", 110 | "channel": "beta", 111 | "prerelease": true 112 | } 113 | ], 114 | "plugins": [ 115 | [ 116 | "@semantic-release/commit-analyzer", 117 | { 118 | "releaseRules": [ 119 | { 120 | "type": "build", 121 | "scope": "deps", 122 | "release": "patch" 123 | } 124 | ] 125 | } 126 | ], 127 | "@semantic-release/release-notes-generator", 128 | "@semantic-release/npm", 129 | "@semantic-release/github" 130 | ] 131 | }, 132 | "jest": { 133 | "testEnvironment": "node", 134 | "collectCoverageFrom": [ 135 | "lib/**/*.js" 136 | ], 137 | "coveragePathIgnorePatterns": [ 138 | "usageParams.js" 139 | ] 140 | }, 141 | "overrides": { 142 | "cross-spawn": "^7.0.6" 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /test/integration/export-lib.test.js: -------------------------------------------------------------------------------- 1 | import { join } from 'path' 2 | import fs from 'fs' 3 | 4 | import mkdirp from 'mkdirp' 5 | import rimraf from 'rimraf' 6 | 7 | import runContentfulExport from '../../dist/index' 8 | 9 | const fsPromises = fs.promises 10 | 11 | jest.setTimeout(15000) 12 | 13 | const tmpFolder = join(__dirname, 'tmp-lib') 14 | const spaceId = process.env.EXPORT_SPACE_ID 15 | const managementToken = process.env.MANAGEMENT_TOKEN 16 | const deliveryToken = process.env.DELIVERY_TOKEN 17 | 18 | const spaceIdEmbargoedAssets = process.env.EXPORT_SPACE_ID_EMBARGOED_ASSETS 19 | 20 | beforeAll(() => { 21 | mkdirp.sync(tmpFolder) 22 | }) 23 | 24 | afterAll(() => { 25 | rimraf.sync(tmpFolder) 26 | }) 27 | 28 | test('It should export space when used as a library', () => { 29 | return runContentfulExport({ spaceId, managementToken, saveFile: false, exportDir: tmpFolder }) 30 | .catch((multierror) => { 31 | const errors = multierror.errors.filter((error) => Object.prototype.hasOwnProperty.call(error, 'error')) 32 | expect(errors).toHaveLength(0) 33 | }) 34 | .then((content) => { 35 | expect(content).toBeTruthy() 36 | expect(content.contentTypes).toHaveLength(2) 37 | expect(content.editorInterfaces).toHaveLength(2) 38 | expect(content.entries).toHaveLength(4) 39 | expect(content.assets).toHaveLength(4) 40 | expect(content.locales).toHaveLength(1) 41 | expect(content.tags).toHaveLength(4) 42 | expect(content.webhooks).toHaveLength(0) 43 | expect(content.roles).toHaveLength(7) 44 | // make sure entries are delivered from CMA 45 | expect(content.entries[0].sys).toHaveProperty('publishedVersion') 46 | }) 47 | }) 48 | 49 | test('It should export environment when used as a library', () => { 50 | return runContentfulExport({ spaceId, environmentId: 'staging', managementToken, saveFile: false, exportDir: tmpFolder }) 51 | .catch((multierror) => { 52 | const errors = multierror.errors.filter((error) => Object.prototype.hasOwnProperty.call(error, 'error')) 53 | expect(errors).toHaveLength(0) 54 | }) 55 | .then((content) => { 56 | expect(content).toBeTruthy() 57 | expect(content.contentTypes).toHaveLength(2) 58 | expect(content.editorInterfaces).toHaveLength(2) 59 | expect(content.entries).toHaveLength(4) 60 | expect(content.assets).toHaveLength(4) 61 | expect(content.locales).toHaveLength(1) 62 | expect(content.tags).toHaveLength(2) 63 | expect(content).not.toHaveProperty('webhooks') 64 | expect(content).not.toHaveProperty('roles') 65 | }) 66 | }) 67 | 68 | test('It should export space when used as a library, with deliveryToken', () => { 69 | return runContentfulExport({ spaceId, managementToken, deliveryToken, saveFile: false, exportDir: tmpFolder }) 70 | .catch((multierror) => { 71 | const errors = multierror.errors.filter((error) => Object.prototype.hasOwnProperty.call(error, 'error')) 72 | expect(errors).toHaveLength(0) 73 | }) 74 | .then((content) => { 75 | expect(content).toBeTruthy() 76 | expect(content.contentTypes).toHaveLength(2) 77 | expect(content.editorInterfaces).toHaveLength(2) 78 | expect(content.entries).toHaveLength(4) 79 | expect(content.assets).toHaveLength(4) 80 | expect(content.locales).toHaveLength(1) 81 | expect(content.tags).toHaveLength(4) 82 | expect(content.webhooks).toHaveLength(0) 83 | expect(content.roles).toHaveLength(7) 84 | }) 85 | }) 86 | 87 | test('It should export embargoed assets space when used as a library', () => { 88 | return runContentfulExport({ 89 | spaceId: spaceIdEmbargoedAssets, 90 | managementToken, 91 | saveFile: true, 92 | downloadAssets: true, 93 | exportDir: tmpFolder, 94 | host: 'api.contentful.com' 95 | }) 96 | .catch((multierror) => { 97 | const errors = multierror.errors.filter((error) => Object.prototype.hasOwnProperty.call(error, 'error')) 98 | expect(errors).toHaveLength(0) 99 | }) 100 | .then(async (content) => { 101 | expect(content.assets).toHaveLength(1) 102 | 103 | // This code ensures that the protected/embargoed asset has actually been downloaded 104 | const files = await fsPromises.readdir(tmpFolder, { withFileTypes: true }) 105 | const directories = files.filter(f => f.isDirectory()) 106 | expect(directories).toHaveLength(1) 107 | }) 108 | }) 109 | -------------------------------------------------------------------------------- /test/unit/index.test.js: -------------------------------------------------------------------------------- 1 | import { resolve } from 'path' 2 | 3 | import bfj from 'bfj' 4 | import fs from 'fs' 5 | import mkdirp from 'mkdirp' 6 | 7 | import { 8 | setupLogging, 9 | displayErrorLog, 10 | writeErrorLogFile 11 | } from 'contentful-batch-libs' 12 | 13 | import { mockDownloadAssets } from './mocks/download-assets' 14 | import { mockGetSpaceData } from './mocks/get-space-data' 15 | 16 | import downloadAssets from '../../lib/tasks/download-assets' 17 | import getSpaceData from '../../lib/tasks/get-space-data' 18 | import initClient from '../../lib/tasks/init-client' 19 | import runContentfulExport from '../../lib/index' 20 | 21 | jest.spyOn(global.console, 'log') 22 | jest.mock('../../lib/tasks/init-client') 23 | jest.mock('../../lib/tasks/download-assets', () => jest.fn(() => mockDownloadAssets)) 24 | jest.mock('../../lib/tasks/get-space-data', () => jest.fn(mockGetSpaceData)) 25 | 26 | jest.mock('fs', () => ({ access: jest.fn((path, cb) => cb()) })) 27 | jest.mock('mkdirp', () => jest.fn()) 28 | jest.mock('bfj', () => ({ write: jest.fn().mockResolvedValue() })) 29 | jest.mock('contentful-batch-libs/dist/logging', () => ({ 30 | setupLogging: jest.fn(), 31 | displayErrorLog: jest.fn(), 32 | logToTaskOutput: () => jest.fn(), 33 | writeErrorLogFile: jest.fn((destination, errorLog) => { 34 | const multiError = new Error('Errors occured') 35 | multiError.name = 'ContentfulMultiError' 36 | multiError.errors = errorLog 37 | throw multiError 38 | }) 39 | })) 40 | 41 | afterEach(() => { 42 | initClient.mockClear() 43 | getSpaceData.mockClear() 44 | setupLogging.mockClear() 45 | displayErrorLog.mockClear() 46 | fs.access.mockClear() 47 | mkdirp.mockClear() 48 | bfj.write.mockClear() 49 | writeErrorLogFile.mockClear() 50 | downloadAssets.mockClear() 51 | global.console.log.mockClear() 52 | }) 53 | 54 | test('Runs Contentful Export with default config', async () => { 55 | await runContentfulExport({ 56 | errorLogFile: 'errorlogfile', 57 | spaceId: 'someSpaceId', 58 | managementToken: 'someManagementToken' 59 | }) 60 | 61 | expect(initClient.mock.calls).toHaveLength(1) 62 | expect(getSpaceData.mock.calls).toHaveLength(1) 63 | expect(setupLogging.mock.calls).toHaveLength(1) 64 | expect(downloadAssets.mock.calls).toHaveLength(1) 65 | expect(displayErrorLog.mock.calls).toHaveLength(1) 66 | expect(fs.access.mock.calls).toHaveLength(1) 67 | expect(mkdirp.mock.calls).toHaveLength(1) 68 | expect(bfj.write.mock.calls).toHaveLength(1) 69 | expect(writeErrorLogFile.mock.calls).toHaveLength(0) 70 | const exportedTable = global.console.log.mock.calls.find((call) => call[0].match(/Exported entities/)) 71 | expect(exportedTable).not.toBeUndefined() 72 | expect(exportedTable[0]).toMatch(/Exported entities/) 73 | expect(exportedTable[0]).toMatch(/Content Types.+0/) 74 | expect(exportedTable[0]).toMatch(/Entries.+0/) 75 | expect(exportedTable[0]).toMatch(/Assets.+2/) 76 | expect(exportedTable[0]).toMatch(/Locales.+0/) 77 | const assetsTable = global.console.log.mock.calls.find((call) => call[0].match(/Asset file download results/)) 78 | expect(assetsTable).toBeUndefined() 79 | }) 80 | 81 | test('Runs Contentful Export and downloads assets', async () => { 82 | await runContentfulExport({ 83 | errorLogFile: 'errorlogfile', 84 | spaceId: 'someSpaceId', 85 | managementToken: 'someManagementToken', 86 | downloadAssets: true 87 | }) 88 | 89 | expect(initClient.mock.calls).toHaveLength(1) 90 | expect(getSpaceData.mock.calls).toHaveLength(1) 91 | expect(setupLogging.mock.calls).toHaveLength(1) 92 | expect(downloadAssets.mock.calls).toHaveLength(1) 93 | expect(displayErrorLog.mock.calls).toHaveLength(1) 94 | expect(fs.access.mock.calls).toHaveLength(1) 95 | expect(mkdirp.mock.calls).toHaveLength(1) 96 | expect(bfj.write.mock.calls).toHaveLength(1) 97 | expect(writeErrorLogFile.mock.calls).toHaveLength(0) 98 | const exportedTable = global.console.log.mock.calls.find((call) => call[0].match(/Exported entities/)) 99 | expect(exportedTable).not.toBeUndefined() 100 | expect(exportedTable[0]).toMatch(/Exported entities/) 101 | expect(exportedTable[0]).toMatch(/Content Types.+0/) 102 | expect(exportedTable[0]).toMatch(/Entries.+0/) 103 | expect(exportedTable[0]).toMatch(/Assets.+2/) 104 | expect(exportedTable[0]).toMatch(/Locales.+0/) 105 | const assetsTable = global.console.log.mock.calls.find((call) => call[0].match(/Asset file download results/)) 106 | expect(assetsTable).not.toBeUndefined() 107 | expect(assetsTable[0]).toMatch(/Asset file download results/) 108 | expect(assetsTable[0]).toMatch(/Successful.+3/) 109 | expect(assetsTable[0]).toMatch(/Warnings.+2/) 110 | expect(assetsTable[0]).toMatch(/Errors.+1/) 111 | }) 112 | 113 | test('Creates a valid and correct opts object', async () => { 114 | const errorLogFile = 'errorlogfile' 115 | const { default: exampleConfig } = await import('../../example-config.test.json') 116 | 117 | await runContentfulExport({ 118 | errorLogFile, 119 | config: resolve(__dirname, '..', '..', 'example-config.test.json') 120 | }) 121 | 122 | expect(initClient.mock.calls[0][0].skipContentModel).toBeFalsy() 123 | expect(initClient.mock.calls[0][0].skipEditorInterfaces).toBeFalsy() 124 | expect(initClient.mock.calls[0][0].skipTags).toBeFalsy() 125 | expect(initClient.mock.calls[0][0].stripTags).toBeFalsy() 126 | expect(initClient.mock.calls[0][0].errorLogFile).toBe(resolve(process.cwd(), errorLogFile)) 127 | expect(initClient.mock.calls[0][0].spaceId).toBe(exampleConfig.spaceId) 128 | expect(initClient.mock.calls).toHaveLength(1) 129 | expect(getSpaceData.mock.calls).toHaveLength(1) 130 | expect(setupLogging.mock.calls).toHaveLength(1) 131 | expect(downloadAssets.mock.calls).toHaveLength(1) 132 | expect(displayErrorLog.mock.calls).toHaveLength(1) 133 | expect(fs.access.mock.calls).toHaveLength(1) 134 | expect(mkdirp.mock.calls).toHaveLength(1) 135 | expect(bfj.write.mock.calls).toHaveLength(1) 136 | expect(writeErrorLogFile.mock.calls).toHaveLength(0) 137 | const exportedTable = global.console.log.mock.calls.find((call) => call[0].match(/Exported entities/)) 138 | expect(exportedTable).not.toBeUndefined() 139 | expect(exportedTable[0]).toMatch(/Exported entities/) 140 | expect(exportedTable[0]).toMatch(/Content Types.+0/) 141 | expect(exportedTable[0]).toMatch(/Entries.+0/) 142 | expect(exportedTable[0]).toMatch(/Assets.+2/) 143 | expect(exportedTable[0]).toMatch(/Locales.+0/) 144 | }) 145 | 146 | test('Run Contentful export fails due to rejection', async () => { 147 | const rejectError = new Error() 148 | rejectError.request = { uri: 'erroruri' } 149 | getSpaceData.mockImplementation(() => Promise.reject(rejectError)) 150 | 151 | await expect(runContentfulExport({ 152 | errorLogFile: 'errorlogfile', 153 | spaceId: 'someSpaceId', 154 | managementToken: 'someManagementToken' 155 | })).rejects.toThrow() 156 | 157 | expect(initClient.mock.calls).toHaveLength(1) 158 | expect(getSpaceData.mock.calls).toHaveLength(1) 159 | expect(setupLogging.mock.calls).toHaveLength(1) 160 | expect(downloadAssets.mock.calls).toHaveLength(1) 161 | expect(displayErrorLog.mock.calls).toHaveLength(1) 162 | expect(fs.access.mock.calls).toHaveLength(0) 163 | expect(mkdirp.mock.calls).toHaveLength(0) 164 | expect(bfj.write.mock.calls).toHaveLength(0) 165 | expect(writeErrorLogFile.mock.calls).toHaveLength(1) 166 | }) 167 | -------------------------------------------------------------------------------- /test/unit/mocks/download-assets.js: -------------------------------------------------------------------------------- 1 | export const mockDownloadAssets = async (ctx) => { 2 | ctx.assetDownloads = { 3 | successCount: 3, 4 | warningCount: 2, 5 | errorCount: 1 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /test/unit/mocks/get-space-data.js: -------------------------------------------------------------------------------- 1 | import Listr from 'listr' 2 | 3 | export const mockGetSpaceData = () => { 4 | return new Listr([ 5 | { 6 | title: 'mocked get full source space', 7 | task: (ctx) => { 8 | ctx.data = { 9 | contentTypes: [], 10 | entries: [], 11 | assets: [ 12 | { 13 | sys: { id: 'someValidAsset' }, 14 | fields: { 15 | file: { 16 | 'en-US': { 17 | url: '//images.contentful.com/kq9lln4hyr8s/2MTd2wBirYikEYkIIc0YSw/7aa4c06f3054996e45bb3f13964cb254/rocka-nutrition.png' 18 | } 19 | } 20 | } 21 | }, 22 | { 23 | sys: { id: 'someBrokenAsset' }, 24 | fields: {} 25 | } 26 | ], 27 | locales: [] 28 | } 29 | } 30 | } 31 | ]) 32 | } 33 | -------------------------------------------------------------------------------- /test/unit/parseOptions.test.js: -------------------------------------------------------------------------------- 1 | import { HttpsProxyAgent } from 'https-proxy-agent' 2 | import { basename, isAbsolute, resolve, sep } from 'path' 3 | import parseOptions from '../../lib/parseOptions' 4 | 5 | const spaceId = 'foo' 6 | const managementToken = 'someManagementToken' 7 | const basePath = resolve(__dirname, '..', '..') 8 | 9 | const toBeAbsolutePathWithPattern = (received, pattern) => { 10 | const escapedPattern = [basename(basePath), pattern].join(`\\${sep}`) 11 | 12 | return (!isAbsolute(received) || !RegExp(`/${escapedPattern}$/`).test(received)) 13 | } 14 | 15 | test('parseOptions sets requires spaceId', () => { 16 | expect( 17 | () => parseOptions({}) 18 | ).toThrow('The `spaceId` option is required.') 19 | }) 20 | 21 | test('parseOptions sets requires managementToken', () => { 22 | expect( 23 | () => parseOptions({ 24 | spaceId: 'someSpaceId' 25 | }) 26 | ).toThrow('The `managementToken` option is required.') 27 | }) 28 | 29 | test('parseOptions sets correct default options', async () => { 30 | const { default: packageJson } = await import(resolve(basePath, 'package.json')) 31 | const version = packageJson.version 32 | 33 | const options = parseOptions({ spaceId, managementToken }) 34 | 35 | const contentFileNamePattern = `contentful-export-${spaceId}-master-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}\\.json` 36 | const errorFileNamePattern = `contentful-export-error-log-${spaceId}-master-[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}-[0-9]{2}-[0-9]{2}\\.json` 37 | 38 | expect(options.contentFile).toMatch(new RegExp(`^${contentFileNamePattern}$`)) 39 | expect(toBeAbsolutePathWithPattern(options.errorLogFile, errorFileNamePattern)).toBe(true) 40 | expect(options.exportDir).toBe(basePath) 41 | expect(options.includeDrafts).toBe(false) 42 | expect(options.includeArchived).toBe(false) 43 | expect(toBeAbsolutePathWithPattern(options.logFilePath, contentFileNamePattern)).toBe(true) 44 | expect(options.application).toBe(`contentful.export/${version}`) 45 | expect(options.feature).toBe('library-export') 46 | expect(options.accessToken).toBe(managementToken) 47 | expect(options.maxAllowedLimit).toBe(1000) 48 | expect(options.saveFile).toBe(true) 49 | expect(options.skipContent).toBe(false) 50 | expect(options.skipContentModel).toBe(false) 51 | expect(options.skipEditorInterfaces).toBe(false) 52 | expect(options.skipRoles).toBe(false) 53 | expect(options.skipWebhooks).toBe(false) 54 | expect(options.skipTags).toBe(false) 55 | expect(options.stripTags).toBe(false) 56 | expect(options.spaceId).toBe(spaceId) 57 | expect(options.startTime).toBeInstanceOf(Date) 58 | expect(options.useVerboseRenderer).toBe(false) 59 | expect(options.deliveryToken).toBeUndefined() 60 | }) 61 | 62 | test('parseOption accepts config file', async () => { 63 | const configFileName = 'example-config.test.json' 64 | const { default: config } = await import(resolve(basePath, configFileName)) 65 | 66 | const options = parseOptions({ config: configFileName }) 67 | Object.keys(config).forEach((key) => { 68 | expect(options[key]).toBe(config[key]) 69 | }) 70 | }) 71 | 72 | test('parseOption overwrites errorLogFile', () => { 73 | const errorLogFile = 'error.log' 74 | const options = parseOptions({ 75 | spaceId, 76 | managementToken, 77 | errorLogFile 78 | }) 79 | expect(options.errorLogFile).toBe(resolve(basePath, errorLogFile)) 80 | }) 81 | 82 | test('parseOption throws with invalid proxy', () => { 83 | expect(() => parseOptions({ 84 | spaceId, 85 | managementToken, 86 | proxy: 'invalid' 87 | })).toThrow('Please provide the proxy config in the following format:\nhost:port or user:password@host:port') 88 | }) 89 | 90 | test('parseOption accepts proxy config as string', () => { 91 | const options = parseOptions({ 92 | spaceId, 93 | managementToken, 94 | proxy: 'localhost:1234' 95 | }) 96 | expect(options).not.toHaveProperty('proxy') 97 | expect(options.httpsAgent).toBeInstanceOf(HttpsProxyAgent) 98 | }) 99 | 100 | test('parseOption accepts proxy config as object', () => { 101 | const options = parseOptions({ 102 | spaceId, 103 | managementToken, 104 | proxy: { 105 | host: 'localhost', 106 | port: 1234, 107 | user: 'foo', 108 | password: 'bar' 109 | } 110 | }) 111 | expect(options).not.toHaveProperty('proxy') 112 | expect(options.httpsAgent).toBeInstanceOf(HttpsProxyAgent) 113 | }) 114 | 115 | test('parseOptions parses queryEntries option', () => { 116 | const options = parseOptions({ 117 | spaceId, 118 | managementToken, 119 | queryEntries: [ 120 | 'someParam=someValue', 121 | 'someOtherParam=someOtherValue' 122 | ] 123 | }) 124 | expect(options.queryEntries).toMatchObject({ 125 | someParam: 'someValue', 126 | someOtherParam: 'someOtherValue' 127 | }) 128 | }) 129 | 130 | test('parseOptions parses queryAssets option', () => { 131 | const options = parseOptions({ 132 | spaceId, 133 | managementToken, 134 | queryAssets: [ 135 | 'someParam=someValue', 136 | 'someOtherParam=someOtherValue' 137 | ] 138 | }) 139 | expect(options.queryAssets).toMatchObject({ 140 | someParam: 'someValue', 141 | someOtherParam: 'someOtherValue' 142 | }) 143 | }) 144 | 145 | test('parseOptions sets correct options given contentOnly', () => { 146 | const options = parseOptions({ 147 | spaceId, 148 | managementToken, 149 | contentOnly: true 150 | }) 151 | expect(options.skipRoles).toBe(true) 152 | expect(options.skipContentModel).toBe(true) 153 | expect(options.skipWebhooks).toBe(true) 154 | }) 155 | 156 | test('parseOptions accepts custom application & feature', () => { 157 | const managementApplication = 'managementApplicationMock' 158 | const managementFeature = 'managementFeatureMock' 159 | 160 | const options = parseOptions({ 161 | spaceId, 162 | managementToken, 163 | managementApplication, 164 | managementFeature 165 | }) 166 | 167 | expect(options.application).toBe(managementApplication) 168 | expect(options.feature).toBe(managementFeature) 169 | }) 170 | 171 | test('parseOption parses deliveryToken option', () => { 172 | const options = parseOptions({ 173 | spaceId, 174 | managementToken, 175 | deliveryToken: 'testDeliveryToken' 176 | }) 177 | expect(options.accessToken).toBe(managementToken) 178 | expect(options.spaceId).toBe(spaceId) 179 | expect(options.deliveryToken).toBe('testDeliveryToken') 180 | }) 181 | 182 | test('parseOption parses headers option', () => { 183 | const options = parseOptions({ 184 | spaceId, 185 | managementToken, 186 | headers: { 187 | header1: '1', 188 | header2: '2' 189 | } 190 | }) 191 | expect(options.headers).toEqual({ 192 | header1: '1', 193 | header2: '2', 194 | 'CF-Sequence': expect.any(String) 195 | }) 196 | }) 197 | 198 | test('parses params.header if provided', function () { 199 | const config = parseOptions({ 200 | spaceId, 201 | managementToken, 202 | header: ['Accept : application/json ', ' X-Header: 1'] 203 | }) 204 | expect(config.headers).toEqual({ Accept: 'application/json', 'X-Header': '1', 'CF-Sequence': expect.any(String) }) 205 | }) 206 | -------------------------------------------------------------------------------- /test/unit/tasks/download-assets.test.js: -------------------------------------------------------------------------------- 1 | import { promises as fs, rmSync } from 'fs' 2 | import { tmpdir } from 'os' 3 | import { resolve } from 'path' 4 | 5 | import nock from 'nock' 6 | 7 | import downloadAssets from '../../../lib/tasks/download-assets' 8 | 9 | const tmpDirectory = resolve(tmpdir(), 'contentful-import-test') 10 | 11 | const BASE_PATH = '//images.contentful.com' 12 | const BASE_PATH_SECURE = '//images.secure.contentful.com' 13 | const EXISTING_ASSET_URL = '/kq9lln4hyr8s/2MTd2wBirYikEYkIIc0YSw/7aa4c06f3054996e45bb3f13964cb254/rocka-nutrition.png' 14 | const EMBARGOED_ASSET_URL = '/kq9lln4hyr8s/2MTd2wBirYikEYkIIc0YSw/7aa4c06f3054996e45bb3f13964cb254/space-dog.png' 15 | const NON_EXISTING_URL = '/does-not-exist.png' 16 | 17 | const API_HOST = 'api.contentful.com' 18 | const SPACE_ID = 'kq9lln4hyr8s' 19 | const ACCESS_TOKEN = 'abc' 20 | const ENVIRONMENT_ID = 'master' 21 | const POLICY = 'eyJhbG.eyJMDIyfQ.SflKx5c' 22 | const SECRET = 's3cr3t' 23 | 24 | let taskProxy 25 | let output 26 | 27 | nock(`https:${BASE_PATH}`) 28 | .get(EXISTING_ASSET_URL) 29 | .times(6) 30 | .reply(200) 31 | 32 | nock(`https:${BASE_PATH}`) 33 | .get(NON_EXISTING_URL) 34 | .reply(404) 35 | 36 | // Mock downloading assets using signed URLs 37 | nock(`https:${BASE_PATH_SECURE}`) 38 | .get(EMBARGOED_ASSET_URL) 39 | .query({ policy: POLICY, token: /.+/i }) 40 | .times(2) 41 | .reply(200) 42 | 43 | // Mock asset-key creation for embargoed assets 44 | nock(`https://${API_HOST}`) 45 | .post(`/spaces/${SPACE_ID}/environments/${ENVIRONMENT_ID}/asset_keys`, { 46 | expiresAt: /.+/i 47 | }) 48 | .times(1) 49 | .reply(200, { policy: POLICY, secret: SECRET }) 50 | 51 | function getAssets ({ existing = 0, nonExisting = 0, missingUrl = 0, embargoed = 0 } = {}) { 52 | const existingUrl = `${BASE_PATH}${EXISTING_ASSET_URL}` 53 | const embargoedUrl = `${BASE_PATH_SECURE}${EMBARGOED_ASSET_URL}` 54 | const nonExistingUrl = `${BASE_PATH}${NON_EXISTING_URL}` 55 | const assets = [] 56 | for (let i = 0; i < nonExisting; i++) { 57 | assets.push({ 58 | sys: { 59 | id: `Non existing asset ${i}` 60 | }, 61 | fields: { 62 | file: { 63 | 'en-US': { 64 | url: nonExistingUrl, 65 | upload: '//file-stack-url-do-not-use-me.png' 66 | }, 67 | 'de-DE': { 68 | url: nonExistingUrl, 69 | upload: '//file-stack-url-do-not-use-me.png' 70 | } 71 | } 72 | } 73 | }) 74 | } 75 | for (let i = 0; i < existing; i++) { 76 | assets.push({ 77 | sys: { 78 | id: `existing asset ${i}` 79 | }, 80 | fields: { 81 | file: { 82 | 'en-US': { 83 | url: existingUrl, 84 | upload: '//file-stack-url-do-not-use-me.png' 85 | }, 86 | 'de-DE': { 87 | url: existingUrl, 88 | upload: '//file-stack-url-do-not-use-me.png' 89 | } 90 | } 91 | } 92 | }) 93 | } 94 | for (let i = 0; i < embargoed; i++) { 95 | assets.push({ 96 | sys: { 97 | id: `embargoed asset ${i}` 98 | }, 99 | fields: { 100 | file: { 101 | 'en-US': { 102 | url: embargoedUrl, 103 | upload: '//file-stack-url-do-not-use-me.png' 104 | }, 105 | 'de-DE': { 106 | url: embargoedUrl, 107 | upload: '//file-stack-url-do-not-use-me.png' 108 | } 109 | } 110 | } 111 | }) 112 | } 113 | for (let i = 0; i < missingUrl; i++) { 114 | assets.push({ 115 | sys: { 116 | id: `missing file url ${i}` 117 | }, 118 | fields: { 119 | file: { 120 | 'en-US': { 121 | upload: '//file-stack-url-do-not-use-me.png' 122 | }, 123 | 'de-DE': { 124 | upload: '//file-stack-url-do-not-use-me.png' 125 | } 126 | } 127 | } 128 | }) 129 | } 130 | return assets 131 | } 132 | 133 | beforeEach(() => { 134 | output = jest.fn() 135 | taskProxy = new Proxy({}, { 136 | set: (obj, prop, value) => { 137 | if (prop === 'output') { 138 | output(value) 139 | return value 140 | } 141 | throw new Error(`It should not access task property ${String(prop)} (value: ${value})`) 142 | } 143 | }) 144 | }) 145 | beforeAll(async () => { 146 | await fs.mkdir(tmpDirectory, { recursive: true }) 147 | }) 148 | 149 | afterAll(() => { 150 | // Couldn't get `fs.promises.rm` to work without permissions issues 151 | rmSync(tmpDirectory, { recursive: true, force: true }) 152 | 153 | if (!nock.isDone()) { 154 | throw new Error(`pending mocks: ${nock.pendingMocks().join(', ')}`) 155 | } 156 | 157 | nock.cleanAll() 158 | nock.restore() 159 | }) 160 | 161 | test('Downloads assets and properly counts failed attempts', () => { 162 | const task = downloadAssets({ 163 | exportDir: tmpDirectory 164 | }) 165 | const ctx = { 166 | data: { 167 | assets: [ 168 | ...getAssets({ existing: 1, nonExisting: 1 }), 169 | { 170 | sys: { 171 | id: 'corrupt asset [warning]' 172 | }, 173 | fields: {} 174 | } 175 | ] 176 | } 177 | } 178 | 179 | return task(ctx, taskProxy) 180 | .then(() => { 181 | expect(ctx.assetDownloads).toEqual({ 182 | successCount: 2, 183 | warningCount: 1, 184 | errorCount: 2 185 | }) 186 | expect(output.mock.calls).toHaveLength(5) 187 | }) 188 | }) 189 | 190 | test('Downloads embargoed assets', () => { 191 | const task = downloadAssets({ 192 | exportDir: tmpDirectory, 193 | host: API_HOST, 194 | accessToken: ACCESS_TOKEN, 195 | spaceId: SPACE_ID, 196 | environmentId: ENVIRONMENT_ID 197 | }) 198 | const ctx = { 199 | data: { 200 | assets: [ 201 | ...getAssets({ embargoed: 1 }) 202 | ] 203 | } 204 | } 205 | 206 | return task(ctx, taskProxy) 207 | .then(() => { 208 | expect(ctx.assetDownloads).toEqual({ 209 | successCount: 2, 210 | warningCount: 0, 211 | errorCount: 0 212 | }) 213 | expect(output.mock.calls).toHaveLength(2) 214 | }) 215 | }) 216 | 217 | test('it doesn\'t use fileStack url as fallback for the file url and throws a warning output', () => { 218 | const task = downloadAssets({ 219 | exportDir: tmpDirectory 220 | }) 221 | const ctx = { 222 | data: { 223 | assets: [ 224 | ...getAssets({ existing: 2, missingUrl: 1 }) 225 | ] 226 | } 227 | } 228 | 229 | return task(ctx, taskProxy) 230 | .then(() => { 231 | expect(ctx.assetDownloads).toEqual({ 232 | successCount: 4, 233 | warningCount: 0, 234 | errorCount: 2 235 | }) 236 | expect(output.mock.calls).toHaveLength(6) 237 | 238 | const missingUrlsOutputCount = output.mock.calls.filter(call => 239 | call[0]?.endsWith('asset.fields.file[en-US].url') || 240 | call[0]?.endsWith('asset.fields.file[de-DE].url')) 241 | 242 | expect(missingUrlsOutputCount).toHaveLength(2) 243 | }) 244 | }) 245 | -------------------------------------------------------------------------------- /test/unit/tasks/get-space-data.test.js: -------------------------------------------------------------------------------- 1 | import getSpaceData from '../../../lib/tasks/get-space-data' 2 | 3 | const maxAllowedLimit = 100 4 | const resultItemCount = 420 5 | 6 | function pagedResult (query, maxItems, mock = {}) { 7 | const { skip, limit } = query 8 | const cnt = maxItems - skip > limit ? limit : maxItems - skip 9 | return { 10 | items: Array.from({ length: cnt}, (n) => { 11 | const id = n * skip + 1 12 | return Object.assign({ sys: { id }}, mock) 13 | }), 14 | total: maxItems 15 | } 16 | } 17 | 18 | function pagedContentResult (query, maxItems, mock = {}) { 19 | const result = pagedResult(query, maxItems, mock) 20 | result.items.map((item, index) => { 21 | item.sys.publishedVersion = index % 2 22 | return item 23 | }) 24 | return result 25 | } 26 | 27 | const mockSpace = {} 28 | 29 | const mockEnvironment = {} 30 | 31 | const mockClient = {} 32 | 33 | const getEditorInterface = jest.fn() 34 | 35 | const mockAsset = { metadata: { tags: [{}] } } 36 | 37 | const mockEntry = { metadata: { tags: [{}] } } 38 | 39 | function setupMocks () { 40 | mockClient.getSpace = jest.fn(() => Promise.resolve(mockSpace)) 41 | mockSpace.getEnvironment = jest.fn(() => Promise.resolve(mockEnvironment)) 42 | mockEnvironment.getContentTypes = jest.fn((query) => { 43 | return Promise.resolve(pagedResult(query, resultItemCount, { 44 | getEditorInterface 45 | })) 46 | }) 47 | mockEnvironment.getEntries = jest.fn((query) => { 48 | return Promise.resolve(pagedContentResult(query, resultItemCount, mockEntry)) 49 | }) 50 | mockEnvironment.getAssets = jest.fn((query) => { 51 | return Promise.resolve(pagedContentResult(query, resultItemCount, mockAsset)) 52 | }) 53 | mockEnvironment.getLocales = jest.fn((query) => { 54 | return Promise.resolve(pagedResult(query, resultItemCount)) 55 | }) 56 | mockEnvironment.getTags = jest.fn((query) => { 57 | return Promise.resolve(pagedResult(query, resultItemCount)) 58 | }) 59 | mockSpace.getWebhooks = jest.fn((query) => { 60 | return Promise.resolve(pagedResult(query, resultItemCount)) 61 | }) 62 | mockSpace.getRoles = jest.fn((query) => { 63 | return Promise.resolve(pagedResult(query, resultItemCount)) 64 | }) 65 | getEditorInterface.mockImplementation(() => Promise.resolve({})) 66 | } 67 | 68 | beforeEach(setupMocks) 69 | 70 | afterEach(() => { 71 | mockClient.getSpace.mockClear() 72 | mockEnvironment.getContentTypes.mockClear() 73 | mockEnvironment.getEntries.mockClear() 74 | mockEnvironment.getAssets.mockClear() 75 | mockEnvironment.getLocales.mockClear() 76 | mockEnvironment.getTags.mockClear() 77 | mockSpace.getWebhooks.mockClear() 78 | mockSpace.getRoles.mockClear() 79 | getEditorInterface.mockClear() 80 | }) 81 | 82 | test('Gets whole destination content', () => { 83 | return getSpaceData({ 84 | client: mockClient, 85 | spaceId: 'spaceid', 86 | maxAllowedLimit 87 | }) 88 | .run({ 89 | data: {} 90 | }) 91 | .then((response) => { 92 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 93 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 94 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 95 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 96 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 97 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 98 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 99 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 100 | expect(mockSpace.getRoles.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 101 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 102 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 103 | expect(response.data.entries).toHaveLength(resultItemCount / 2) 104 | expect(response.data.assets).toHaveLength(resultItemCount / 2) 105 | expect(response.data.locales).toHaveLength(resultItemCount) 106 | expect(response.data.tags).toHaveLength(resultItemCount) 107 | expect(response.data.webhooks).toHaveLength(resultItemCount) 108 | expect(response.data.roles).toHaveLength(resultItemCount) 109 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 110 | }) 111 | }) 112 | 113 | test('Gets whole destination content without content model', () => { 114 | return getSpaceData({ 115 | client: mockClient, 116 | spaceId: 'spaceid', 117 | maxAllowedLimit, 118 | skipContentModel: true 119 | }) 120 | .run({ 121 | data: {} 122 | }) 123 | .then((response) => { 124 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 125 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 126 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(0) 127 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 128 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 129 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(0) 130 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 131 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 132 | expect(mockSpace.getRoles.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 133 | expect(getEditorInterface.mock.calls).toHaveLength(0) 134 | expect(response.data.contentTypes).toBeUndefined() 135 | expect(response.data.entries).toHaveLength(resultItemCount / 2) 136 | expect(response.data.assets).toHaveLength(resultItemCount / 2) 137 | expect(response.data.locales).toBeUndefined() 138 | expect(response.data.tags).toHaveLength(resultItemCount) 139 | expect(response.data.webhooks).toHaveLength(resultItemCount) 140 | expect(response.data.roles).toHaveLength(resultItemCount) 141 | expect(response.data.editorInterfaces).toBeUndefined() 142 | }) 143 | }) 144 | 145 | test('Gets whole destination content without content', () => { 146 | return getSpaceData({ 147 | client: mockClient, 148 | spaceId: 'spaceid', 149 | maxAllowedLimit, 150 | skipContent: true 151 | }) 152 | .run({ 153 | data: {} 154 | }) 155 | .then((response) => { 156 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 157 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 158 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 159 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(0) 160 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(0) 161 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 162 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 163 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 164 | expect(mockSpace.getRoles.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 165 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 166 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 167 | expect(response.data.entries).toBeUndefined() 168 | expect(response.data.assets).toBeUndefined() 169 | expect(response.data.locales).toHaveLength(resultItemCount) 170 | expect(response.data.tags).toHaveLength(resultItemCount) 171 | expect(response.data.webhooks).toHaveLength(resultItemCount) 172 | expect(response.data.roles).toHaveLength(resultItemCount) 173 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 174 | }) 175 | }) 176 | 177 | test('Gets whole destination content without webhooks', () => { 178 | return getSpaceData({ 179 | client: mockClient, 180 | spaceId: 'spaceid', 181 | maxAllowedLimit, 182 | skipWebhooks: true 183 | }) 184 | .run({ 185 | data: {} 186 | }) 187 | .then((response) => { 188 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 189 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 190 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 191 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 192 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 193 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 194 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 195 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(0) 196 | expect(mockSpace.getRoles.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 197 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 198 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 199 | expect(response.data.entries).toHaveLength(resultItemCount / 2) 200 | expect(response.data.assets).toHaveLength(resultItemCount / 2) 201 | expect(response.data.locales).toHaveLength(resultItemCount) 202 | expect(response.data.tags).toHaveLength(resultItemCount) 203 | expect(response.data.webhooks).toBeUndefined() 204 | expect(response.data.roles).toHaveLength(resultItemCount) 205 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 206 | }) 207 | }) 208 | 209 | test('Gets whole destination content without roles', () => { 210 | return getSpaceData({ 211 | client: mockClient, 212 | spaceId: 'spaceid', 213 | maxAllowedLimit, 214 | skipRoles: true 215 | }) 216 | .run({ 217 | data: {} 218 | }) 219 | .then((response) => { 220 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 221 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 222 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 223 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 224 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 225 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 226 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 227 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 228 | expect(mockSpace.getRoles.mock.calls).toHaveLength(0) 229 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 230 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 231 | expect(response.data.entries).toHaveLength(resultItemCount / 2) 232 | expect(response.data.assets).toHaveLength(resultItemCount / 2) 233 | expect(response.data.locales).toHaveLength(resultItemCount) 234 | expect(response.data.tags).toHaveLength(resultItemCount) 235 | expect(response.data.webhooks).toHaveLength(resultItemCount) 236 | expect(response.data.roles).toBeUndefined() 237 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 238 | }) 239 | }) 240 | 241 | test('Gets whole destination content without editor interfaces', () => { 242 | return getSpaceData({ 243 | client: mockClient, 244 | spaceId: 'spaceid', 245 | maxAllowedLimit, 246 | skipEditorInterfaces: true 247 | }) 248 | .run({ 249 | data: {} 250 | }) 251 | .then((response) => { 252 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 253 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 254 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 255 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 256 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 257 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 258 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 259 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 260 | expect(mockSpace.getRoles.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 261 | expect(getEditorInterface.mock.calls).toHaveLength(0) 262 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 263 | expect(response.data.entries).toHaveLength(resultItemCount / 2) 264 | expect(response.data.assets).toHaveLength(resultItemCount / 2) 265 | expect(response.data.locales).toHaveLength(resultItemCount) 266 | expect(response.data.tags).toHaveLength(resultItemCount) 267 | expect(response.data.webhooks).toHaveLength(resultItemCount) 268 | expect(response.data.roles).toHaveLength(resultItemCount) 269 | expect(response.data.editorInterfaces).toBeUndefined() 270 | }) 271 | }) 272 | 273 | test('Gets whole destination content without tags', () => { 274 | return getSpaceData({ 275 | client: mockClient, 276 | spaceId: 'spaceid', 277 | maxAllowedLimit, 278 | skipTags: true 279 | }) 280 | .run({ 281 | data: {} 282 | }) 283 | .then((response) => { 284 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 285 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 286 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 287 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 288 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 289 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 290 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(0) 291 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 292 | expect(mockSpace.getRoles.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 293 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 294 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 295 | expect(response.data.entries).toHaveLength(resultItemCount / 2) 296 | expect(response.data.assets).toHaveLength(resultItemCount / 2) 297 | expect(response.data.locales).toHaveLength(resultItemCount) 298 | expect(response.data.tags).toBeUndefined() 299 | expect(response.data.webhooks).toHaveLength(resultItemCount) 300 | expect(response.data.roles).toHaveLength(resultItemCount) 301 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 302 | }) 303 | }) 304 | 305 | test('Gets whole destination content with drafts', () => { 306 | return getSpaceData({ 307 | client: mockClient, 308 | spaceId: 'spaceid', 309 | maxAllowedLimit, 310 | includeDrafts: true 311 | }) 312 | .run({ 313 | data: {} 314 | }) 315 | .then((response) => { 316 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 317 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 318 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 319 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 320 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 321 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 322 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 323 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 324 | expect(mockSpace.getRoles.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 325 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 326 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 327 | expect(response.data.entries).toHaveLength(resultItemCount) 328 | expect(response.data.assets).toHaveLength(resultItemCount) 329 | expect(response.data.locales).toHaveLength(resultItemCount) 330 | expect(response.data.tags).toHaveLength(resultItemCount) 331 | expect(response.data.webhooks).toHaveLength(resultItemCount) 332 | expect(response.data.roles).toHaveLength(resultItemCount) 333 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 334 | }) 335 | }) 336 | 337 | test('Gets whole destination content with archived entries', () => { 338 | return getSpaceData({ 339 | client: mockClient, 340 | spaceId: 'spaceid', 341 | maxAllowedLimit, 342 | includeDrafts: true, 343 | includeArchived: true 344 | }) 345 | .run({ 346 | data: {} 347 | }) 348 | .then((response) => { 349 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 350 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 351 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 352 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 353 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 354 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 355 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 356 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 357 | expect(mockSpace.getRoles.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 358 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 359 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 360 | expect(response.data.entries).toHaveLength(resultItemCount) 361 | expect(response.data.assets).toHaveLength(resultItemCount) 362 | expect(response.data.locales).toHaveLength(resultItemCount) 363 | expect(response.data.tags).toHaveLength(resultItemCount) 364 | expect(response.data.webhooks).toHaveLength(resultItemCount) 365 | expect(response.data.roles).toHaveLength(resultItemCount) 366 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 367 | }) 368 | }) 369 | 370 | test('Skips webhooks & roles for non-master environments', () => { 371 | return getSpaceData({ 372 | client: mockClient, 373 | spaceId: 'spaceid', 374 | environmentId: 'staging', 375 | maxAllowedLimit, 376 | includeDrafts: true 377 | }) 378 | .run({ 379 | data: {} 380 | }) 381 | .then((response) => { 382 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 383 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 384 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 385 | expect(mockEnvironment.getEntries.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 386 | expect(mockEnvironment.getAssets.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 387 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 388 | expect(mockEnvironment.getTags.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 389 | expect(mockSpace.getWebhooks.mock.calls).toHaveLength(0) 390 | expect(mockSpace.getRoles.mock.calls).toHaveLength(0) 391 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 392 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 393 | expect(response.data.entries).toHaveLength(resultItemCount) 394 | expect(response.data.assets).toHaveLength(resultItemCount) 395 | expect(response.data.locales).toHaveLength(resultItemCount) 396 | expect(response.data.tags).toHaveLength(resultItemCount) 397 | expect(response.data).not.toHaveProperty('webhooks') 398 | expect(response.data).not.toHaveProperty('roles') 399 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 400 | }) 401 | }) 402 | 403 | test('Gets whole destination content and detects missing editor interfaces', () => { 404 | getEditorInterface.mockImplementation(() => Promise.reject(new Error('No editor interface found'))) 405 | 406 | return getSpaceData({ 407 | client: mockClient, 408 | spaceId: 'spaceid', 409 | maxAllowedLimit, 410 | skipContent: true, 411 | skipWebhooks: true, 412 | skipRoles: true 413 | }) 414 | .run({ 415 | data: {} 416 | }) 417 | .then((response) => { 418 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 419 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 420 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(Math.ceil(resultItemCount / maxAllowedLimit)) 421 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 422 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 423 | expect(response.data.editorInterfaces).toHaveLength(0) 424 | }) 425 | }) 426 | 427 | test('Skips editor interfaces since no content types are found', () => { 428 | mockEnvironment.getContentTypes.mockImplementation(() => Promise.resolve({ 429 | items: [], 430 | total: 0 431 | })) 432 | 433 | return getSpaceData({ 434 | client: mockClient, 435 | spaceId: 'spaceid', 436 | maxAllowedLimit, 437 | skipContent: true, 438 | skipWebhooks: true, 439 | skipRoles: true 440 | }) 441 | .run({ 442 | data: {} 443 | }) 444 | .then((response) => { 445 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 446 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 447 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(1) 448 | expect(getEditorInterface.mock.calls).toHaveLength(0) 449 | expect(response.data.contentTypes).toHaveLength(0) 450 | expect(response.data.editorInterfaces).toBeUndefined() 451 | }) 452 | }) 453 | 454 | test('Loads 1000 items per page by default', () => { 455 | return getSpaceData({ 456 | client: mockClient, 457 | spaceId: 'spaceid', 458 | skipContent: true, 459 | skipWebhooks: true, 460 | skipRoles: true 461 | }) 462 | .run({ 463 | data: {} 464 | }) 465 | .then((response) => { 466 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 467 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 468 | expect(mockEnvironment.getContentTypes.mock.calls).toHaveLength(1) 469 | expect(mockEnvironment.getContentTypes.mock.calls[0][0].limit).toBe(1000) 470 | expect(getEditorInterface.mock.calls).toHaveLength(resultItemCount) 471 | expect(response.data.contentTypes).toHaveLength(resultItemCount) 472 | expect(response.data.editorInterfaces).toHaveLength(resultItemCount) 473 | }) 474 | }) 475 | 476 | test('Query entry/asset respect limit query param', () => { 477 | // overwrite the getAssets mock so maxItems is larger than default page size in pagedGet (get-space-data.js) 478 | mockEnvironment.getAssets = jest.fn((query) => { 479 | return Promise.resolve(pagedContentResult(query, 2000, mockEntry)) 480 | }) 481 | return getSpaceData({ 482 | client: mockClient, 483 | spaceId: 'spaceid', 484 | skipContentModel: true, 485 | skipWebhooks: true, 486 | skipRoles: true, 487 | includeDrafts: true, 488 | queryEntries: { limit: 20 }, // test limit < pageSize 489 | queryAssets: { limit: 1001 } // test limit > pageSize 490 | }) 491 | .run({ 492 | data: {} 493 | }) 494 | .then((response) => { 495 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 496 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 497 | expect(mockEnvironment.getEntries.mock.calls[0][0].limit).toBe(20) 498 | expect(mockEnvironment.getAssets.mock.calls[0][0].limit).toBe(1000) // assets should be called 2x 499 | expect(mockEnvironment.getAssets.mock.calls[1][0].limit).toBe(1) // because it has to fetch the final item in the second page 500 | expect(response.data.assets).toHaveLength(1001) 501 | expect(response.data.entries).toHaveLength(20) 502 | }) 503 | }) 504 | 505 | test('only skips fetched items', () => { 506 | // overwrite the getLocales only returns 20 items in pages of 10 507 | mockEnvironment.getLocales = jest.fn() 508 | .mockResolvedValueOnce({ 509 | items: Array.from({ length: 10 }, (n) => { 510 | const id = n + 1 511 | return Object.assign({ sys: { id } }) 512 | }), 513 | total: 20 514 | }) 515 | .mockResolvedValueOnce({ 516 | items: Array.from({ length: 7 }, (n) => { 517 | const id = n + 11 518 | return Object.assign({ sys: { id } }) 519 | }), 520 | total: 17 521 | }) 522 | return getSpaceData({ 523 | client: mockClient, 524 | spaceId: 'spaceid', 525 | skipContent: true, 526 | skipWebhooks: true, 527 | skipRoles: true 528 | }) 529 | .run({ 530 | data: {} 531 | }) 532 | .then(() => { 533 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 534 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 535 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(2) 536 | expect(mockEnvironment.getLocales.mock.calls[0][0].limit).toBe(1000) 537 | expect(mockEnvironment.getLocales.mock.calls[0][0].skip).toBe(0) 538 | expect(mockEnvironment.getLocales.mock.calls[1][0].limit).toBe(1000) 539 | expect(mockEnvironment.getLocales.mock.calls[1][0].skip).toBe(10) 540 | }) 541 | }) 542 | 543 | test('halts fetching when no items in page', () => { 544 | // overwrite the getLocales returns 0 items 545 | mockEnvironment.getLocales = jest.fn() 546 | .mockResolvedValueOnce({ 547 | items: [], 548 | total: 20 549 | }) 550 | return getSpaceData({ 551 | client: mockClient, 552 | spaceId: 'spaceid', 553 | skipContent: true, 554 | skipWebhooks: true, 555 | skipRoles: true 556 | }) 557 | .run({ 558 | data: {} 559 | }) 560 | .then(() => { 561 | expect(mockClient.getSpace.mock.calls).toHaveLength(1) 562 | expect(mockSpace.getEnvironment.mock.calls).toHaveLength(1) 563 | expect(mockEnvironment.getLocales.mock.calls).toHaveLength(1) 564 | expect(mockEnvironment.getLocales.mock.calls[0][0].limit).toBe(1000) 565 | expect(mockEnvironment.getLocales.mock.calls[0][0].skip).toBe(0) 566 | }) 567 | }) 568 | 569 | test('Strips tags from entries and assets', () => { 570 | return getSpaceData({ 571 | client: mockClient, 572 | spaceId: 'spaceid', 573 | maxAllowedLimit, 574 | stripTags: true 575 | }) 576 | .run({ 577 | data: {} 578 | }) 579 | .then((response) => { 580 | expect(response.data.entries).toHaveLength(resultItemCount / 2) 581 | const hasAssetsWithTags = response.data.assets.some(asset => asset.metadata?.tags?.length > 0) 582 | expect(hasAssetsWithTags).toBe(false) 583 | const hasEntryWithTags = response.data.entries.some(entry => entry.metadata?.tags?.length > 0) 584 | expect(hasEntryWithTags).toBe(false) 585 | }) 586 | }) 587 | -------------------------------------------------------------------------------- /test/unit/tasks/init-client.test.js: -------------------------------------------------------------------------------- 1 | import initClient from '../../../lib/tasks/init-client' 2 | 3 | import contentfulManagement from 'contentful-management' 4 | import contentful from 'contentful' 5 | import { logEmitter } from 'contentful-batch-libs' 6 | 7 | jest.mock('contentful-management', () => { 8 | return { 9 | createClient: jest.fn(() => 'cmaClient') 10 | } 11 | }) 12 | 13 | jest.mock('contentful', () => { 14 | return { 15 | createClient: jest.fn(() => 'cdaClient') 16 | } 17 | }) 18 | 19 | jest.mock('contentful-batch-libs', () => { 20 | return { 21 | logEmitter: { 22 | emit: jest.fn() 23 | } 24 | } 25 | }) 26 | 27 | test('does create clients and passes custom logHandler', () => { 28 | const opts = { 29 | httpAgent: 'httpAgent', 30 | httpsAgent: 'httpsAgent', 31 | application: 'application', 32 | headers: 'headers', 33 | host: 'host', 34 | insecure: 'insecure', 35 | integration: 'integration', 36 | port: 'port', 37 | proxy: 'proxy', 38 | accessToken: 'accessToken', 39 | spaceId: 'spaceId' 40 | } 41 | 42 | initClient(opts) 43 | 44 | expect(contentfulManagement.createClient.mock.calls[0][0]).toMatchObject({ 45 | accessToken: opts.accessToken, 46 | host: opts.host, 47 | port: opts.port, 48 | headers: opts.headers, 49 | insecure: opts.insecure, 50 | proxy: opts.proxy, 51 | httpAgent: opts.httpAgent, 52 | httpsAgent: opts.httpsAgent, 53 | application: opts.application, 54 | integration: opts.integration 55 | }) 56 | expect(contentfulManagement.createClient.mock.calls[0][0]).toHaveProperty('logHandler') 57 | expect(contentfulManagement.createClient.mock.calls[0][0].timeout).toEqual(10000) 58 | expect(contentfulManagement.createClient.mock.calls).toHaveLength(1) 59 | expect(contentful.createClient.mock.calls).toHaveLength(0) 60 | 61 | // Call passed log handler 62 | contentfulManagement.createClient.mock.calls[0][0].logHandler('level', 'logMessage') 63 | 64 | expect(logEmitter.emit.mock.calls[0][0]).toBe('level') 65 | expect(logEmitter.emit.mock.calls[0][1]).toBe('logMessage') 66 | }) 67 | 68 | test('does create both clients when deliveryToken is set', () => { 69 | const opts = { 70 | httpAgent: 'httpAgent', 71 | httpsAgent: 'httpsAgent', 72 | application: 'application', 73 | headers: 'headers', 74 | host: 'host', 75 | insecure: 'insecure', 76 | integration: 'integration', 77 | port: 'port', 78 | proxy: 'proxy', 79 | accessToken: 'accessToken', 80 | spaceId: 'spaceId', 81 | deliveryToken: 'deliveryToken', 82 | hostDelivery: 'hostDelivery' 83 | } 84 | 85 | initClient(opts, true) 86 | 87 | expect(contentfulManagement.createClient.mock.calls[0][0]).toMatchObject({ 88 | accessToken: opts.accessToken, 89 | host: opts.host, 90 | port: opts.port, 91 | headers: opts.headers, 92 | insecure: opts.insecure, 93 | proxy: opts.proxy, 94 | httpAgent: opts.httpAgent, 95 | httpsAgent: opts.httpsAgent, 96 | application: opts.application, 97 | integration: opts.integration 98 | }) 99 | expect(contentful.createClient.mock.calls[0][0]).toMatchObject({ 100 | space: opts.spaceId, 101 | accessToken: opts.deliveryToken, 102 | host: opts.hostDelivery, 103 | port: opts.port, 104 | headers: opts.headers, 105 | insecure: opts.insecure, 106 | proxy: opts.proxy, 107 | httpAgent: opts.httpAgent, 108 | httpsAgent: opts.httpsAgent, 109 | application: opts.application, 110 | integration: opts.integration 111 | }) 112 | expect(contentfulManagement.createClient.mock.calls).toHaveLength(1) 113 | expect(contentful.createClient.mock.calls).toHaveLength(1) 114 | }) 115 | -------------------------------------------------------------------------------- /test/unit/utils/embargoedAssets.test.js: -------------------------------------------------------------------------------- 1 | import { shouldCreateNewCacheItem } from '../../../lib/utils/embargoedAssets' 2 | const SIX_HOURS_IN_MS = 6 * 60 * 60 * 1000 3 | 4 | test('only returns true for expiry time difference greater than 6 hours', () => { 5 | expect(shouldCreateNewCacheItem({ expiresAtMs: 1 }, SIX_HOURS_IN_MS - 2)).toBe(false) 6 | expect(shouldCreateNewCacheItem({ expiresAtMs: 1 }, SIX_HOURS_IN_MS + 2)).toBe(true) 7 | }) 8 | -------------------------------------------------------------------------------- /test/unit/utils/headers.test.js: -------------------------------------------------------------------------------- 1 | import { getHeadersConfig } from '../../../lib/utils/headers' 2 | 3 | test('getHeadersConfig returns empty object when value is undefined', () => { 4 | expect(getHeadersConfig(undefined)).toEqual({}) 5 | }) 6 | 7 | test('getHeadersConfig accepts single or multiple values', () => { 8 | expect(getHeadersConfig('Accept: Any')).toEqual({ Accept: 'Any' }) 9 | expect(getHeadersConfig(['Accept: Any', 'X-Version: 1'])).toEqual({ 10 | Accept: 'Any', 11 | 'X-Version': '1' 12 | }) 13 | }) 14 | 15 | test('getHeadersConfig ignores invalid headers', () => { 16 | expect( 17 | getHeadersConfig(['Accept: Any', 'X-Version: 1', 'invalid']) 18 | ).toEqual({ 19 | Accept: 'Any', 20 | 'X-Version': '1' 21 | }) 22 | }) 23 | 24 | test('getHeadersConfig trims spacing around keys & values', () => { 25 | expect( 26 | getHeadersConfig([ 27 | ' Accept: Any ', 28 | ' X-Version :1 ', 29 | 'invalid' 30 | ]) 31 | ).toEqual({ 32 | Accept: 'Any', 33 | 'X-Version': '1' 34 | }) 35 | }) 36 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ESNext", 4 | "module": "commonjs", 5 | "esModuleInterop": true, 6 | "forceConsistentCasingInFileNames": false, 7 | "strict": false, 8 | "skipLibCheck": true, 9 | "allowJs": true, 10 | "checkJs": true, 11 | "resolveJsonModule": true, 12 | "noEmit": true 13 | }, 14 | "include": ["./lib", "./bin"] 15 | } 16 | -------------------------------------------------------------------------------- /types.d.ts: -------------------------------------------------------------------------------- 1 | export interface Options { 2 | managementToken: string; 3 | spaceId: string; 4 | contentFile?: string; 5 | contentOnly?: boolean; 6 | deliveryToken?: string; 7 | downloadAssets?: boolean; 8 | environmentId?: string; 9 | errorLogFile?: string; 10 | exportDir?: string; 11 | headers?: string[]; 12 | host?: string; 13 | includeArchived?: boolean; 14 | includeDrafts?: boolean; 15 | limit?: number; 16 | managementApplication?: string; 17 | managementFeature?: string; 18 | maxAllowedLimit?: number; 19 | proxy?: string; 20 | queryEntries?: string[]; 21 | queryAssets?: string[]; 22 | rawProxy?: boolean; 23 | saveFile?: boolean; 24 | skipContent?: boolean; 25 | skipContentModel?: boolean; 26 | skipEditorInterfaces?: boolean; 27 | skipRoles?: boolean; 28 | skipWebhooks?: boolean; 29 | skipTags?: boolean; 30 | useVerboseRenderer?: boolean; 31 | } 32 | 33 | type ContentfulExportField = 'contentTypes' | 'entries' | 'assets' | 'locales' | 'tags' | 'webhooks' | 'roles' | 'editorInterfaces'; 34 | 35 | declare const runContentfulExport: (params: Options) => Promise> 36 | export default runContentfulExport 37 | --------------------------------------------------------------------------------