├── .babelrc ├── .eslintignore ├── .eslintrc.json ├── .github └── workflows │ └── node.js.yml ├── .gitignore ├── .npmignore ├── .releaserc.json ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── UPGRADE_GUIDE.md ├── doc └── aem-fastingest-nui-architecture-overview.png ├── e2e ├── .env_example ├── e2eutils.js ├── edge-case-images │ └── zero-byte.jpg ├── filesystem-upload.test.js └── images │ ├── Dir 1 │ ├── .skipdir │ │ └── freeride.jpg │ ├── folder_♂♀°′″℃＄￡‰§№￠℡㈱ │ │ └── 郎礼.jpg │ ├── freeride-steep.jpg │ ├── freeride.jpg │ ├── ice-climbing.jpg │ ├── subdir1 │ │ ├── .freeride.jpg │ │ ├── ski touring.jpg │ │ ├── skiing_1.jpg │ │ └── skiing_2.jpg │ ├── subdir2 │ │ └── .emptydir │ └── 이두吏讀.jpg │ ├── Freeride#extreme.jpg │ ├── climber-ferrata-la-torre-di-toblin.jpg │ └── freeride-siberia.jpg ├── index.js ├── package-lock.json ├── package.json ├── src ├── constants.js ├── create-directory-result.js ├── direct-binary-upload-options.js ├── direct-binary-upload-process.js ├── direct-binary-upload.js ├── error-codes.js ├── exports.js ├── file-upload-results.js ├── filesystem-upload-asset.js ├── filesystem-upload-directory.js ├── filesystem-upload-item-manager.js ├── filesystem-upload-options.js ├── filesystem-upload-utils.js ├── filesystem-upload.js ├── fs-promise.js ├── http-result.js ├── http-utils.js ├── upload-base.js ├── upload-error.js ├── upload-file.js ├── upload-options-base.js ├── upload-result.js └── utils.js └── test ├── create-directory-result.test.js ├── direct-binary-upload-options.test.js ├── direct-binary-upload-process.test.js ├── direct-binary-upload.test.js ├── exports.test.js ├── filesystem-upload-directory.test.js ├── filesystem-upload-item-manager.test.js ├── filesystem-upload-options.test.js ├── filesystem-upload-utils.test.js ├── filesystem-upload.test.js ├── http-utils.test.js ├── mock-aem-upload.js ├── mock-blob.js ├── mock-httptransfer-adapter.js ├── testutils.js ├── upload-file.test.js ├── upload-result.test.js └── utils.test.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | ["@babel/env", { 4 | "targets": { 5 | "ie": "9" 6 | } 7 | }] 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | dist/* 2 | .vscode/* 3 | coverage/* 4 | doc/* 5 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "airbnb-base" 3 | } -------------------------------------------------------------------------------- /.github/workflows/node.js.yml: -------------------------------------------------------------------------------- 1 | # This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions 3 | 4 | name: Node.js CI 5 | 6 | on: 7 | push: 8 | branches: [ master ] 9 | pull_request: 10 | branches: [ master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | 17 | strategy: 18 | matrix: 19 | node-version: [14.21, 16.20, 18.16] 20 | 21 | steps: 22 | - uses: actions/checkout@v2 23 | - name: Setup unit test environment 24 | run: sudo apt-get install librsvg2-bin imagemagick exiftool 25 | - name: Use Node.js ${{ matrix.node-version }} 26 | uses: actions/setup-node@v1 27 | with: 28 | node-version: ${{ matrix.node-version }} 29 | - name: Log used OS 30 | run: uname -a 31 | - name: Install dependencies (all) 32 | run: npm install 33 | - name: Build 34 | run: npm run build 35 | - name: Run unit tests 36 | run: npm test 37 | 38 | sizewatcher: 39 | 40 | runs-on: ubuntu-latest 41 | 42 | steps: 43 | - uses: actions/checkout@v2 44 | - run: npx @adobe/sizewatcher 45 | semantic-release: 46 | runs-on: ubuntu-latest 47 | needs: [build] 48 | if: ${{ !contains(github.event.head_commit.message, '[ci skip]') && github.ref == 'refs/heads/master' }} 49 | steps: 50 | - uses: actions/checkout@v2 51 | with: 52 | persist-credentials: false 53 | - name: Use Node.js 18.16 54 | uses: actions/setup-node@v1 55 | with: 56 | node-version: '18.16' 57 | - run: npm install 58 | - run: npm run build 59 | - run: npm run semantic-release 60 | env: 61 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 62 | NPM_TOKEN: ${{ secrets.ADOBE_BOT_NPM_TOKEN }} 63 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | 63 | *.html 64 | 65 | dist 66 | browser 67 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | *.html 2 | *.log 3 | doc/ 4 | browser/ 5 | *.tgz 6 | .eslintrc.json 7 | .babelrc 8 | webpack.config.js 9 | e2e/ 10 | .github/ 11 | test/ 12 | .nyc_output/ 13 | -------------------------------------------------------------------------------- /.releaserc.json: -------------------------------------------------------------------------------- 1 | { 2 | "plugins": [ 3 | "@semantic-release/commit-analyzer", 4 | "@semantic-release/release-notes-generator", 5 | ["@semantic-release/changelog", { 6 | "changelogFile": "CHANGELOG.md" 7 | }], 8 | ["@semantic-release/npm"], 9 | ["@semantic-release/git", { 10 | "assets": ["package.json", "package-lock.json", "CHANGELOG.md"], 11 | "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" 12 | }], 13 | "@semantic-release/github" 14 | ], 15 | "branches": ["master"] 16 | } 17 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## [2.0.3](https://github.com/adobe/aem-upload/compare/v2.0.2...v2.0.3) (2024-01-29) 2 | 3 | 4 | ### Bug Fixes 5 | 6 | * improve support for handling eventual consistency when uploading. ([#118](https://github.com/adobe/aem-upload/issues/118)) ([0372ef4](https://github.com/adobe/aem-upload/commit/0372ef460dacb9c837a698ac9826a254e28adfe1)) 7 | 8 | ## [2.0.2](https://github.com/adobe/aem-upload/compare/v2.0.1...v2.0.2) (2023-08-04) 9 | 10 | 11 | ### Bug Fixes 12 | 13 | * pass request options to cors requests ([#113](https://github.com/adobe/aem-upload/issues/113)) ([528b654](https://github.com/adobe/aem-upload/commit/528b65497de300f6d22a4b81d54dd49f907e47f3)) 14 | 15 | ## [2.0.1](https://github.com/adobe/aem-upload/compare/v2.0.0...v2.0.1) (2023-05-26) 16 | 17 | 18 | ### Bug Fixes 19 | 20 | * update to latest node-httptransfer ([#108](https://github.com/adobe/aem-upload/issues/108)) ([33f4e01](https://github.com/adobe/aem-upload/commit/33f4e01ca22dfcf759954a9373c6d980ad6f2430)) 21 | 22 | # [2.0.0](https://github.com/adobe/aem-upload/compare/v1.5.0...v2.0.0) (2023-04-21) 23 | 24 | 25 | ### Features 26 | 27 | * use fetch instead of axios, and expose http options ([#107](https://github.com/adobe/aem-upload/issues/107)) ([1650a94](https://github.com/adobe/aem-upload/commit/1650a940279f41d420487c58da66b2c31eec0804)) 28 | 29 | 30 | ### BREAKING CHANGES 31 | 32 | * several classes and underlying functionality changed. See upgrade guide for details. 33 | 34 | # AEM Upload Change Log 35 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Adobe Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at Grp-opensourceoffice@adobe.com. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [https://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: https://contributor-covenant.org 74 | [version]: https://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thanks for choosing to contribute! 4 | 5 | The following are a set of guidelines to follow when contributing to this project. 6 | 7 | ## Code Of Conduct 8 | 9 | This project adheres to the Adobe [code of conduct](CODE_OF_CONDUCT.md). By participating, 10 | you are expected to uphold this code. Please report unacceptable behavior to 11 | [Grp-opensourceoffice@adobe.com](mailto:Grp-opensourceoffice@adobe.com). 12 | 13 | ## Have A Question? 14 | 15 | Start by filing an issue. The existing committers on this project work to reach 16 | consensus around project direction and issue solutions within issue threads 17 | (when appropriate). 18 | 19 | ## Contributor License Agreement 20 | 21 | All third-party contributions to this project must be accompanied by a signed contributor 22 | license agreement. This gives Adobe permission to redistribute your contributions 23 | as part of the project. [Sign our CLA](https://opensource.adobe.com/cla.html). You 24 | only need to submit an Adobe CLA one time, so if you have submitted one previously, 25 | you are good to go! 26 | 27 | ## Code Reviews 28 | 29 | All submissions should come in the form of pull requests and need to be reviewed 30 | by project committers. Read [GitHub's pull request documentation](https://help.github.com/articles/about-pull-requests/) 31 | for more information on sending pull requests. 32 | 33 | Lastly, please follow the [pull request template](PULL_REQUEST_TEMPLATE.md) when 34 | submitting a pull request! 35 | 36 | ## From Contributor To Committer 37 | 38 | We love contributions from our community! If you'd like to go a step beyond contributor 39 | and become a committer with full write access and a say in the project, you must 40 | be invited to the project. The existing committers employ an internal nomination 41 | process that must reach lazy consensus (silence is approval) before invitations 42 | are issued. If you feel you are qualified and want to get more deeply involved, 43 | feel free to reach out to existing committers to have a conversation about that. 44 | 45 | ## Security Issues 46 | 47 | Security issues shouldn't be reported on this issue tracker. Instead, [file an issue to our security experts](https://helpx.adobe.com/security/alertus.html) 48 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Adobe 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | 6 | 7 | ## Related Issue 8 | 9 | 10 | 11 | 12 | 13 | 14 | ## Motivation and Context 15 | 16 | 17 | 18 | ## How Has This Been Tested? 19 | 20 | 21 | 22 | 23 | 24 | ## Screenshots (if appropriate): 25 | 26 | ## Types of changes 27 | 28 | 29 | 30 | - [ ] Bug fix (non-breaking change which fixes an issue) 31 | - [ ] New feature (non-breaking change which adds functionality) 32 | - [ ] Breaking change (fix or feature that would cause existing functionality to change) 33 | 34 | ## Checklist: 35 | 36 | 37 | 38 | 39 | - [ ] I have signed the [Adobe Open Source CLA](https://opensource.adobe.com/cla.html). 40 | - [ ] My code follows the code style of this project. 41 | - [ ] My change requires a change to the documentation. 42 | - [ ] I have updated the documentation accordingly. 43 | - [ ] I have read the **CONTRIBUTING** document. 44 | - [ ] I have added tests to cover my changes. 45 | - [ ] All new and existing tests passed. 46 | -------------------------------------------------------------------------------- /UPGRADE_GUIDE.md: -------------------------------------------------------------------------------- 1 | # Upgrade Guide 2 | 3 | This document will provide instructions for upgrading between breaking versions of the library. We hope to keep breaking changes to a minimum, but they will happen. Please feel free to submit an issue if we've missed anything. 4 | 5 | ## Upgrading to 2.x.x 6 | 7 | Here are a few things to know when upgrading to version 2 of the library. 8 | 9 | ### HTTP API Change 10 | 11 | Version 2 now uses the Fetch API when submitting HTTP requests (it previously used Axios). Note that this only applies to the process of creating folders; file uploads were already going through the Fetch API. This change brings the two operations inline with one another, and will allow consumers to more directly control the options that are used when the library submits HTTP requests. 12 | 13 | The options used by Fetch can be controlled through `DirectBinaryUploadOptions.withHttpOptions()`. All values provided through this method will be passed directly to the Fetch API. 14 | 15 | ### Upload Result 16 | 17 | The structure of the upload results provided by the library has changed. Previously, the result was a class instance containing various metrics about the upload process. In version 2, the class was changed to a simple javascript `object`. 18 | 19 | The item in question is illustrated with the following code: 20 | 21 | ``` 22 | const result = await upload.uploadFiles(options); 23 | ``` 24 | 25 | The contents of `result` has changed in version 2. Previously, there would have been methods like: 26 | 27 | ``` 28 | result.getTotalFiles(); 29 | result.getElapesedTime(); 30 | result.getTotalSize(); 31 | ``` 32 | 33 | After version 2, the contents of `result` will be a simple object: 34 | 35 | ``` 36 | { 37 | ... 38 | totalFiles: 2, 39 | totalTime: 2367, 40 | totalSize: 127385 41 | ... 42 | } 43 | ``` 44 | 45 | (Note that this is only a sampling of the result data; the actual contents will be more extensive). 46 | 47 | In addition, the result will contain less detail than it did prior to version 2. For example, information about individual file parts, time spent initializing the upload, and some other file upload related metrics are no longer available. 48 | 49 | ### Exported Code 50 | 51 | The library originally exported a transpiled version of its code; this was intended to increase the number of Node.JS versions and browser versions that the library could support. Version 2 has been updated so that the library's primary exports will consist of the code as-is. This means that older versions of Node.JS will no longer be supported through the primary exports. The transpiled code is still available in the `dist` directory, which is also set as the library's `browser` target. 52 | 53 | In addition, the library no longer uses module-like syntax. This slightly changes the way that non-modules will need to consume the library's exports. 54 | 55 | For modules, the `import` statement will continue to work; no changes required: 56 | 57 | ``` 58 | // this will still work 59 | import AemUpload from '@adobe/aem-upload'; 60 | ``` 61 | 62 | For non-modules, there will need to be changes if using the `default` export: 63 | 64 | ``` 65 | // this will no longer work: 66 | const AemUpload = require('@adobe/aem-upload').default; 67 | ``` 68 | 69 | Instead, remove `default`: 70 | 71 | ``` 72 | // this will work: 73 | const AemUpload = require('@adobe/aem-upload'); 74 | ``` 75 | 76 | ### Class Signature Changes 77 | 78 | The following classes have had breaking changes to their signatures. 79 | 80 | * DirectBinaryUpload 81 | * `canUpload()` has been removed. The library will now automatically determine the type of upload that a target AEM instance requires, and use the appropriate algorithm. 82 | * DirectBinaryUploadOptions 83 | * `withHeaders()` has been removed. Instead, consumers should use `withHttpOptions()` and include a `headers` property as described by the Fetch API. 84 | * `withCookies()` has been removed. Instead, consumers should use `withHttpOptions()` and include a `Cookie` header in the `headers` property as described by the Fetch API. 85 | * `withBasicAuth()` has been removed. Instead, consumers should use `withHttpOptions()` and include an `Authorization` header in the `headers` property as described by the Fetch API. 86 | * `withAddContentLengthHeader()` has been removed. This method was deprecated, and has been non-functional and unneeded for a long time. 87 | * `withHttpProxy()` has been removed. Instead, consumers should use `withHttpOptions()` and include proxy information (probably through the `agent` property) as described by the Fetch API. 88 | * `getHeaders()` has been removed. Consumers can access the headers through the `getHttpOptions()` method. 89 | * `addContentLengthHeader()` has been removed. This method was deprecated, and has been non-functional and unneeded for a long time. 90 | * `getController()` has been removed. This method a means for cancelling in-progress uploads, but the cancel functionality has been non-functional for a long time. If your case requires the ability to cancel an upload, please submit an issue and provide details. 91 | * `getHttpProxy()` has been removed. Consumers can access the proxy through the `getHttpOptions()` method. 92 | * HttpProxy 93 | * This class has been removed and is no longer exported. Consumers are responsible for defining their own proxy configuration and providing it to the Fetch API through `DirectBinaryUploadOptions.withHttpOptions()`. 94 | -------------------------------------------------------------------------------- /doc/aem-fastingest-nui-architecture-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/doc/aem-fastingest-nui-architecture-overview.png -------------------------------------------------------------------------------- /e2e/.env_example: -------------------------------------------------------------------------------- 1 | # add the URL to the AEM instance that the e2e tests should use. 2 | # note that the tests WILL create additional content on the 3 | # target, so take care when specifying the value. 4 | # 5 | # EXAMPLE: http://localhost:4502 6 | AEM_ENDPOINT="" 7 | 8 | # The tests require authentication in order to run. EITHER 9 | # BASIC_AUTH or LOGIN_TOKEN must be set. 10 | 11 | # if set, the tests will use basic HTTP authentication to 12 | # connect to the instance. the advantage to this approach 13 | # is that the credentials will only need to change if the 14 | # user's password changes. the disadvantage is that this 15 | # will not work for SSO-enabled instances. 16 | # 17 | # EXAMPLE: username:password 18 | #BASIC_AUTH="" 19 | 20 | # if set, the tests will send the given token in the Cookie 21 | # header when sending requests to AEM. the advantage to this 22 | # approach is that it will support SSO-enabled instances. The 23 | # disadvantage is that the token will need to be refreshed 24 | # when it expires. 25 | # 26 | # EXAMPLE: login-token=1234567 27 | #LOGIN_TOKEN -------------------------------------------------------------------------------- /e2e/e2eutils.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const Path = require('path'); 14 | const fetch = require('node-fetch'); 15 | // eslint-disable-next-line import/no-extraneous-dependencies 16 | const should = require('should'); 17 | 18 | const testutils = require('../test/testutils'); 19 | 20 | // load .env values in the e2e folder, if any 21 | // eslint-disable-next-line import/no-extraneous-dependencies 22 | require('dotenv').config({ path: Path.join(__dirname, '.env') }); 23 | 24 | module.exports = testutils; 25 | 26 | /** 27 | * Retrieves the root URL of the AEM endpoint that the tests should 28 | * use. 29 | * @returns {string} URL for an AEM instance. 30 | */ 31 | module.exports.getAemEndpoint = () => { 32 | const endpoint = process.env.AEM_ENDPOINT; 33 | 34 | if (!endpoint) { 35 | throw new Error('AEM_ENDPOINT environment variable must be supplied'); 36 | } 37 | 38 | return endpoint; 39 | }; 40 | 41 | /** 42 | * Updates the given options to include authentication information required 43 | * to communicate with AEM. 44 | * @param {import('../src/direct-binary-upload-options')} uploadOptions Will be updated with 45 | * auth info. 46 | */ 47 | module.exports.setCredentials = (uploadOptions) => { 48 | const basic = process.env.BASIC_AUTH; 49 | const token = process.env.LOGIN_TOKEN; 50 | const options = {}; 51 | 52 | if (!basic && !token) { 53 | throw new Error('Either BASIC_AUTH or LOGIN_TOKEN env variable must be set'); 54 | } 55 | 56 | if (basic) { 57 | options.headers = { 58 | Authorization: `Basic ${Buffer.from(basic).toString('base64')}`, 59 | }; 60 | } else { 61 | options.headers = { 62 | Cookie: token, 63 | }; 64 | } 65 | 66 | return uploadOptions.withHttpOptions(options); 67 | }; 68 | 69 | /** 70 | * Retrieves the full URL to the folder to use when interacting with AEM. 71 | * @returns {string} A full URL. 72 | */ 73 | module.exports.getTargetFolder = () => `${module.exports.getAemEndpoint()}/content/dam/aem-upload-e2e/test_${new Date().getTime()}`; 74 | 75 | /** 76 | * Uses fetch to submit an HTTP request and provide a response. 77 | * @param {string} url Full URL to which the request will be submitted. 78 | * @param {import('../src/direct-binary-upload-options')} uploadOptions Options whose 79 | * information will be used to augment the request. 80 | * @param {*} httpOptions Raw options to pass to fetch. 81 | * @returns {Promise} Response to the request. 82 | */ 83 | async function submitRequest(url, uploadOptions, httpOptions = {}) { 84 | const { headers: uploadHeaders = {} } = uploadOptions.getHttpOptions(); 85 | const { headers: optionHeaders = {} } = httpOptions; 86 | const fetchOptions = { 87 | ...uploadOptions.getHttpOptions(), 88 | ...httpOptions, 89 | headers: { 90 | ...uploadHeaders, 91 | ...optionHeaders, 92 | }, 93 | }; 94 | const response = await fetch(url, fetchOptions); 95 | if (!response.ok) { 96 | throw new Error(`Unexpected status code ${response.status}`); 97 | } 98 | return response; 99 | } 100 | 101 | /** 102 | * Determines whether or not a given path exists in the target AEM endpoint. 103 | * @param {DirectBinaryUploadOptions} uploadOptions The options' URL will be used 104 | * when querying AEM. 105 | * @param {string} relativePath Relative path (from the options's URL) to the item 106 | * to check. Example: /folder/myasset.jpg. 107 | * @returns {boolean} True if the path exists, false otherwise. 108 | */ 109 | module.exports.doesAemPathExist = async (uploadOptions, relativePath) => { 110 | const headUrl = `${uploadOptions.getUrl().replace('/content/dam', '/api/assets')}${encodeURI(relativePath)}.json`; 111 | 112 | try { 113 | await submitRequest(headUrl, uploadOptions, { 114 | method: 'HEAD', 115 | }); 116 | } catch (e) { 117 | return false; 118 | } 119 | return true; 120 | }; 121 | 122 | /** 123 | * Retrieves the jcr:title property value for a given path. 124 | * @param {DirectBinaryUploadOptions} uploadOptions The options' URL will be used 125 | * when querying AEM. 126 | * @param {string} relativePath Relative path (from the options's URL) to the item 127 | * to check. Example: /folder/myasset.jpg. 128 | * @returns {string} Value of the path's jcr:title property, or empty string if none 129 | * found. 130 | */ 131 | module.exports.getPathTitle = async (uploadOptions, relativePath) => { 132 | const infoUrl = `${uploadOptions.getUrl().replace('/content/dam', '/api/assets')}${encodeURI(relativePath)}.json?showProperty=jcr:title`; 133 | 134 | const response = await submitRequest(infoUrl, uploadOptions); 135 | const { properties = {} } = await response.json(); 136 | 137 | return properties['jcr:title'] || ''; 138 | }; 139 | 140 | /** 141 | * Deletes a path from the target AEM instance. 142 | * @param {DirectBinaryUploadOptions} uploadOptions The options' URL will be used 143 | * when deleting the path. 144 | * @param {string} relativePath Relative path (from the options's URL) to the item 145 | * to delete. Example: /folder/myasset.jpg. 146 | */ 147 | module.exports.deleteAemPath = async (uploadOptions, relativePath = '') => { 148 | const deleteUrl = `${uploadOptions.getUrl().replace('/content/dam', '/api/assets')}${relativePath}`; 149 | 150 | return submitRequest(deleteUrl, uploadOptions, { method: 'DELETE' }); 151 | }; 152 | 153 | module.exports.createAemFolder = async (uploadOptions, folderName) => { 154 | const createUrl = `${uploadOptions.getUrl().replace('/content/dam', `/api/assets/${encodeURIComponent(folderName)}`)}`; 155 | 156 | const data = { 157 | class: 'assetFolder', 158 | properties: { 159 | title: 'Test Folder', 160 | }, 161 | }; 162 | 163 | return submitRequest(createUrl, uploadOptions, { 164 | method: 'POST', 165 | headers: { 166 | 'content-type': 'application/json', 167 | }, 168 | body: JSON.stringify(data), 169 | }); 170 | }; 171 | 172 | /** 173 | * Validates that the result of an upload operation is expected, given an e2e upload 174 | * process. 175 | * @param {string} targetFolder Full URL of the target directory in AEM to which the 176 | * upload was targeted. 177 | * @param {*} result Result as provided by the upload process. 178 | * @param {*} expected Expected data that should be in the result. 179 | */ 180 | module.exports.verifyE2eResult = (targetFolder, result, expected) => { 181 | const targetFolderPath = new URL(targetFolder).pathname; 182 | const rootFolderPath = Path.posix.dirname(targetFolderPath); 183 | const toVerify = { ...result }; 184 | const { createdFolders = [] } = expected; 185 | createdFolders.splice(0, 0, { 186 | elapsedTime: toVerify.createdFolders[0].elapsedTime, 187 | folderPath: rootFolderPath, 188 | folderTitle: Path.basename(rootFolderPath), 189 | retryErrors: [], 190 | }); 191 | createdFolders.splice(0, 0, { 192 | elapsedTime: toVerify.createdFolders[1].elapsedTime, 193 | folderPath: targetFolderPath, 194 | folderTitle: Path.basename(targetFolderPath), 195 | retryErrors: [], 196 | }); 197 | 198 | if (toVerify.createdFolders[0].error) { 199 | should(toVerify.createdFolders[0].error.code).be.exactly('EALREADYEXISTS'); 200 | delete toVerify.createdFolders[0].error; 201 | } 202 | 203 | testutils.verifyResult(toVerify, { 204 | ...expected, 205 | createdFolders, 206 | }); 207 | }; 208 | -------------------------------------------------------------------------------- /e2e/edge-case-images/zero-byte.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/edge-case-images/zero-byte.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/.skipdir/freeride.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/.skipdir/freeride.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/folder_♂♀°′″℃＄￡‰§№￠℡㈱/郎礼.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/folder_♂♀°′″℃＄￡‰§№￠℡㈱/郎礼.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/freeride-steep.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/freeride-steep.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/freeride.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/freeride.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/ice-climbing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/ice-climbing.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/subdir1/.freeride.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/subdir1/.freeride.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/subdir1/ski touring.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/subdir1/ski touring.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/subdir1/skiing_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/subdir1/skiing_1.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/subdir1/skiing_2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/subdir1/skiing_2.jpg -------------------------------------------------------------------------------- /e2e/images/Dir 1/subdir2/.emptydir: -------------------------------------------------------------------------------- 1 | This directory intentionally left blank. -------------------------------------------------------------------------------- /e2e/images/Dir 1/이두吏讀.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Dir 1/이두吏讀.jpg -------------------------------------------------------------------------------- /e2e/images/Freeride#extreme.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/Freeride#extreme.jpg -------------------------------------------------------------------------------- /e2e/images/climber-ferrata-la-torre-di-toblin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/climber-ferrata-la-torre-di-toblin.jpg -------------------------------------------------------------------------------- /e2e/images/freeride-siberia.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/adobe/aem-upload/64d9bcac9fda50109c0fae8a77079f485425e131/e2e/images/freeride-siberia.jpg -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | module.exports = require('./src/exports'); 14 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@adobe/aem-upload", 3 | "version": "2.0.3", 4 | "description": "AEM Assets direct binary access uploading tool", 5 | "main": "index.js", 6 | "browser": "./dist/exports.js", 7 | "license": "Apache-2.0", 8 | "repository": "adobe/aem-upload", 9 | "scripts": { 10 | "test": "npm run lint && npm run testOnly", 11 | "testOnly": "mocha --recursive ./test", 12 | "build": "rimraf dist && babel ./src --out-dir dist", 13 | "prepublishOnly": "npm test && npm run build", 14 | "coverage": "nyc npm run test", 15 | "lint": "eslint .", 16 | "lint:fix": "eslint . --fix", 17 | "e2e": "mocha --recursive ./e2e", 18 | "semantic-release": "semantic-release" 19 | }, 20 | "author": "Adobe", 21 | "contributors": [ 22 | "Jun Zhang", 23 | "Mark Frisbey" 24 | ], 25 | "bugs": "https://github.com/adobe/aem-upload", 26 | "dependencies": { 27 | "@adobe/cloud-service-client": "^1.1.0", 28 | "@adobe/httptransfer": "^3.4.1", 29 | "async": "^3.2.0", 30 | "async-lock": "^1.2.8", 31 | "filesize": "^4.2.1", 32 | "node-fetch": "^2.6.9", 33 | "uuid": "^3.3.2" 34 | }, 35 | "devDependencies": { 36 | "@babel/cli": "^7.16.0", 37 | "@babel/core": "^7.9.0", 38 | "@babel/polyfill": "^7.8.7", 39 | "@babel/preset-env": "^7.9.0", 40 | "@babel/preset-stage-2": "^7.8.3", 41 | "@semantic-release/changelog": "^6.0.3", 42 | "@semantic-release/git": "^10.0.1", 43 | "conventional-changelog-eslint": "^3.0.9", 44 | "dotenv": "^8.2.0", 45 | "eslint": "^8.38.0", 46 | "eslint-config-airbnb-base": "^15.0.0", 47 | "eslint-plugin-import": "^2.27.5", 48 | "json-loader": "^0.5.7", 49 | "mime": "^2.4.4", 50 | "mocha": "^10.2.0", 51 | "mock-fs": "^4.13.0", 52 | "nock": "^13.3.0", 53 | "nyc": "14.1.1", 54 | "rimraf": "^3.0.2", 55 | "should": "^13.2.3", 56 | "sinon": "^9.2.3" 57 | }, 58 | "optionalDependencies": { 59 | "semantic-release": "^21.0.1" 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/constants.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /** 14 | * Constant values that are used as defaults. 15 | */ 16 | module.exports.DefaultValues = { 17 | /** 18 | * The default number of maximum concurrent HTTP requests allowed by the library. 19 | */ 20 | MAX_CONCURRENT: 5, 21 | 22 | /** 23 | * The default number of times the process will attempt submitting an HTTP request 24 | * before giving up and reporting a failure. 25 | */ 26 | RETRY_COUNT: 3, 27 | 28 | /** 29 | * The amount of time, in milliseconds, that the process will wait between retries 30 | * of the same HTTP request. The delay will increase itself by this value for 31 | * each retry. 32 | */ 33 | RETRY_DELAY: 5000, 34 | 35 | /** 36 | * Default timeout for HTTP requests: 1 minute. 37 | */ 38 | REQUEST_TIMEOUT: 60000, 39 | 40 | /** 41 | * Maximum number of files allowed to be uploaded as part of the file system 42 | * upload process. 43 | */ 44 | MAX_FILE_UPLOAD: 1000, 45 | }; 46 | 47 | module.exports.RegularExpressions = { 48 | /** 49 | * Will match values that contain characters that are invalid in AEM node 50 | * names. 51 | */ 52 | INVALID_CHARACTERS_REGEX: /[/:[\]|*\\]/g, 53 | 54 | /** 55 | * Will match values that contain characters that are invalid in AEM folder node 56 | * names. 57 | */ 58 | INVALID_FOLDER_CHARACTERS_REGEX: /[.%;#+?^{}\s"&]/g, 59 | 60 | /** 61 | * Will match values that contain characters that are invalid in AEM asset node 62 | * names. 63 | */ 64 | INVALID_ASSET_CHARACTERS_REGEX: /[#%{}?&]/g, 65 | }; 66 | 67 | module.exports.HttpMethods = { 68 | POST: 'POST', 69 | }; 70 | -------------------------------------------------------------------------------- /src/create-directory-result.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const HttpResult = require('./http-result'); 14 | 15 | /** 16 | * Represents the results of the creation of a directory. These results contain information such as 17 | * the amount of time it took to create, and any error that may have occurred. 18 | */ 19 | class CreateDirectoryResult extends HttpResult { 20 | /** 21 | * Constructs a new instance using the provided information. Can then be used to provide 22 | * additional details as needed. 23 | * 24 | * @param {object} options Options as provided when the upload instance was instantiated. 25 | * @param {DirectBinaryUploadOptions} uploadOptions Options as provided when the upload was 26 | * initiated. 27 | * @param {string} folderPath Full path of the folder that was created. 28 | * @param {string} folderTitle Full title of the folder that was created. 29 | * @param {*} response Response to the create request from the underlying client. 30 | */ 31 | constructor(options, uploadOptions, folderPath, folderTitle) { 32 | super(options, uploadOptions); 33 | 34 | this.folderPath = folderPath; 35 | this.folderTitle = folderTitle; 36 | this.response = false; 37 | this.error = false; 38 | } 39 | 40 | /** 41 | * Sets the response to the create request. 42 | * 43 | * @param {*} response Response to the create request from the underlying client. 44 | */ 45 | setCreateResponse(response) { 46 | this.response = response; 47 | } 48 | 49 | /** 50 | * Sets the error that was the result of the create request. 51 | * 52 | * @param {import('./upload-error')} error Error to the create request. 53 | */ 54 | setCreateError(error) { 55 | this.error = error; 56 | } 57 | 58 | /** 59 | * Retrieves the full path of the folder as it was created in AEM. 60 | * 61 | * @returns {string} Path of a folder. 62 | */ 63 | getFolderPath() { 64 | return this.folderPath; 65 | } 66 | 67 | /** 68 | * Retrieves the title of the folder as it was created in AEM. 69 | * 70 | * @returns {string} Title of a folder. 71 | */ 72 | getFolderTitle() { 73 | return this.folderTitle; 74 | } 75 | 76 | /** 77 | * Retrieves the amount of time, in milliseconds, it took to create the folder. 78 | * 79 | * @returns {number} Time span in milliseconds. 80 | */ 81 | getCreateTime() { 82 | if (this.response && this.response.cloudClient) { 83 | return this.response.cloudClient.requestTime; 84 | } 85 | return 0; 86 | } 87 | 88 | /** 89 | * Converts the result instance into a simple object containing all result data. 90 | * 91 | * @returns {object} Result data in a simple format. 92 | */ 93 | toJSON() { 94 | const json = { 95 | elapsedTime: this.getCreateTime(), 96 | folderPath: this.getFolderPath(), 97 | folderTitle: this.getFolderTitle(), 98 | ...super.toJSON(), 99 | }; 100 | 101 | if (this.error) { 102 | json.error = this.error.toJSON(); 103 | } 104 | return json; 105 | } 106 | } 107 | 108 | module.exports = CreateDirectoryResult; 109 | -------------------------------------------------------------------------------- /src/direct-binary-upload-options.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const URL = require('url'); 14 | 15 | const { trimRight } = require('./utils'); 16 | const { DefaultValues } = require('./constants'); 17 | 18 | /** 19 | * Options that generally control how a direct binary upload will behave. The class contains 20 | * several optional configurations, but the minimal setup will require at least the following: 21 | * 22 | * ```js 23 | * const options = new DirectBinaryUploadOptions() 24 | * .withUrl(uploadTargetUrl) // URL of the target of the upload 25 | * .withUploadFiles(fileList) // list of files to upload 26 | * ``` 27 | * 28 | * All other options are optional. 29 | */ 30 | class DirectBinaryUploadOptions { 31 | constructor() { 32 | this.options = { 33 | maxConcurrent: DefaultValues.MAX_CONCURRENT, 34 | retryCount: DefaultValues.RETRY_COUNT, 35 | retryDelay: DefaultValues.RETRY_DELAY, 36 | requestTimeout: DefaultValues.REQUEST_TIMEOUT, 37 | }; 38 | } 39 | 40 | /** 41 | * Sets the URL to which binaries will be uploaded. The URL should include the path to 42 | * a directory. Depending on the context, this could be either a relative or absolute URL. 43 | * For example, running from a node.js process will require an absolute URL, whereas running 44 | * from a browser will allow relative URLs. 45 | * 46 | * @param {string} url The URL to which binaries will be uploaded. 47 | * @returns {DirectBinaryUploadOptions} The current options instance. Allows for chaining. 48 | */ 49 | withUrl(url) { 50 | this.options.url = url; 51 | return this; 52 | } 53 | 54 | /** 55 | * An array of object instances representing the files that will be uploaded 56 | * to the target URL. 57 | * 58 | * @param {Array} uploadFiles Files to upload to the target. Each file must contain 59 | * at least the following properties: fileName, fileSize, and either filePath OR blob. 60 | * See UploadFile for more information. 61 | * @returns {DirectBinaryUploadOptions} The current options instance. Allows for chaining. 62 | */ 63 | withUploadFiles(uploadFiles) { 64 | this.options.uploadFiles = uploadFiles; 65 | return this; 66 | } 67 | 68 | /** 69 | * Specifies whether or not the process will upload the files concurrently. If false, the 70 | * process will upload one file at a time. If true, the process will upload all files in 71 | * a concurrent "thread"-like manner. Default value is true. 72 | * 73 | * This is a convenience function that is wrapped around setting withMaxConcurrent() to 74 | * either 1 (if isConcurrent is false) or its default value (if isConcurrent is true). 75 | * 76 | * @param {boolean} isConcurrent True if the process should upload files concurrently, false if 77 | * they should be uploaded serially. 78 | * @returns {DirectBinaryUploadOptions} The current options instance. Allows for chaining. 79 | */ 80 | withConcurrent(isConcurrent) { 81 | if (isConcurrent) { 82 | return this.withMaxConcurrent(5); 83 | } 84 | return this.withMaxConcurrent(1); 85 | } 86 | 87 | /** 88 | * Specifies the maximum number of HTTP requests to invoke at one time. Once this number of 89 | * pending requests is reached, no more requests will be submitted until at least one of the 90 | * pending requests finishes. A value less than 2 indicates that only one request at a time 91 | * is allowed, meaning that files will be uploaded serially instead of concurrently. Default 92 | * value is 5. 93 | * 94 | * @param {number} maxConcurrent Maximum number of pending HTTP requests to allow. 95 | * @returns {DirectBinaryUploadOptions} The current options instance. Allows for chaining. 96 | */ 97 | withMaxConcurrent(maxConcurrent) { 98 | this.options.maxConcurrent = maxConcurrent; 99 | return this; 100 | } 101 | 102 | /** 103 | * Specifies the options that will be used when submitting HTTP requests. The method will 104 | * merge the given options with any options provided in previous calls to the method, 105 | * and will pass the merged options as-is to fetch. 106 | * 107 | * @param {*} options Fetch options. 108 | * @returns {DirectBinaryUploadOptions} The current options instance, for chaining. 109 | */ 110 | withHttpOptions(options) { 111 | const { headers: currHeaders } = this.getHttpOptions(); 112 | const { headers: newHeaders } = options; 113 | 114 | const newOptions = { 115 | ...this.getHttpOptions(), 116 | ...options, 117 | }; 118 | 119 | if (currHeaders && newHeaders) { 120 | newOptions.headers = { 121 | ...currHeaders, 122 | ...newHeaders, 123 | }; 124 | } 125 | 126 | this.options.httpOptions = newOptions; 127 | 128 | return this; 129 | } 130 | 131 | /** 132 | * Specifies the number of times the process will attempt retrying a failed HTTP request before 133 | * giving up and reporting an error. Default: 3. 134 | * 135 | * @param {number} retryCount Number of times to resubmit a request. 136 | * @returns {DirectBinaryUploadOptions} The current options instance. Allows for chaining. 137 | */ 138 | withHttpRetryCount(retryCount) { 139 | this.options.retryCount = retryCount; 140 | return this; 141 | } 142 | 143 | /** 144 | * Sets the amount of time, in milliseconds, that the process will wait before retrying a 145 | * failed HTTP request. The delay will increase itself by this value each time the failed 146 | * request is resubmitted. For example, if the delay is 5,000 then the process will wait 147 | * 5,000 milliseconds for the first retry, then 10,000, then 15,000, etc. Default: 5,000. 148 | * 149 | * @param {number} retryDelay A timespan in milliseconds. 150 | * @returns {DirectBinaryUploadOptions} The current options instance. Allows for chaining. 151 | */ 152 | withHttpRetryDelay(retryDelay) { 153 | this.options.retryDelay = retryDelay; 154 | return this; 155 | } 156 | 157 | /** 158 | * Sets the maximum amount of time the module will wait for an HTTP request to complete 159 | * before timing out. Default: 1 minute. 160 | * @param {number} timeout Timeout duration, in milliseconds. 161 | * @returns {DirectBinaryUploadOptions} The current options instance. Allows for chaining. 162 | */ 163 | withHttpRequestTimeout(timeout) { 164 | this.options.requestTimeout = timeout; 165 | return this; 166 | } 167 | 168 | /** 169 | * Retrieves the target URL to which files will be uploaded. 170 | * 171 | * @returns {string} Target URL as provided to the options instance. 172 | */ 173 | getUrl() { 174 | return trimRight(this.options.url, ['/']) || '/'; 175 | } 176 | 177 | /** 178 | * Retrieves the path to the folder where the files will be uploaded. This value 179 | * is based on the URL that was provided to the options. 180 | * 181 | * The path value will not be URL encoded. 182 | * 183 | * @returns {string} Full path to a folder on the target. 184 | */ 185 | getTargetFolderPath() { 186 | const { pathname } = URL.parse(this.getUrl()); 187 | return decodeURIComponent(pathname); 188 | } 189 | 190 | /** 191 | * Retrieves the target URL's prefix, which is everything in the URL up to the target path. 192 | * Will be empty if there is no prefix. 193 | * 194 | * @returns {string} The target URL's prefix. 195 | */ 196 | getUrlPrefix() { 197 | const { 198 | protocol, 199 | host, 200 | } = URL.parse(this.getUrl()); 201 | 202 | return host ? `${protocol}//${host}` : ''; 203 | } 204 | 205 | /** 206 | * Retrieves the list of files that will be uploaded. 207 | * 208 | * @returns {Array} List of UploadFile instances as provided to the options instance. 209 | */ 210 | getUploadFiles() { 211 | return this.options.uploadFiles || []; 212 | } 213 | 214 | /** 215 | * Retrieves a value indicating whether or not the upload process will transfer files 216 | * concurrently. 217 | * 218 | * @returns {boolean} The value as provided to the options instance. 219 | */ 220 | isConcurrent() { 221 | return this.getMaxConcurrent() > 1; 222 | } 223 | 224 | /** 225 | * Retrieves the maximum number of concurrent HTTP requests that should be allowed. 226 | * 227 | * @returns {number} Maximum number. 228 | */ 229 | getMaxConcurrent() { 230 | return this.options.maxConcurrent; 231 | } 232 | 233 | /** 234 | * Retrieves the HTTP options that the upload process will use in each HTTP request that 235 | * it sends with fetch. 236 | * @returns {*} Fetch options. 237 | */ 238 | getHttpOptions() { 239 | return this.options.httpOptions || {}; 240 | } 241 | 242 | /** 243 | * Retrieves the number of times the process will attempt to resubmit a failed HTTP request. 244 | * 245 | * @returns {number} Retry count. 246 | */ 247 | getHttpRetryCount() { 248 | return this.options.retryCount; 249 | } 250 | 251 | /** 252 | * Retrieves the amount of time, in milliseconds, the process wil wait between resubmitting 253 | * the same failed HTTP request. 254 | * 255 | * @returns {number} Timespan in milliseconds. 256 | */ 257 | getHttpRetryDelay() { 258 | return this.options.retryDelay; 259 | } 260 | 261 | /** 262 | * Retrieves the maximum amount of time that the module will wait for an HTTP request to 263 | * complete before timing out. 264 | * 265 | * @returns {number} Timeout duration, in milliseconds. 266 | */ 267 | getHttpRequestTimeout() { 268 | return this.options.requestTimeout; 269 | } 270 | 271 | /** 272 | * Overridden to return an object appropriate for representing this class as a 273 | * JSON object. 274 | * 275 | * @returns {object} The class's JSON representation. 276 | */ 277 | toJSON() { 278 | const json = { 279 | ...this.options, 280 | }; 281 | 282 | return json; 283 | } 284 | } 285 | 286 | module.exports = DirectBinaryUploadOptions; 287 | -------------------------------------------------------------------------------- /src/direct-binary-upload-process.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const { 14 | AEMUpload, 15 | } = require('@adobe/httptransfer'); 16 | const httpTransferLogger = require('@adobe/httptransfer/lib/logger'); 17 | const { v4: uuid } = require('uuid'); 18 | 19 | const UploadOptionsBase = require('./upload-options-base'); 20 | const FileUploadResults = require('./file-upload-results'); 21 | const { 22 | getHttpTransferOptions, 23 | } = require('./http-utils'); 24 | 25 | /** 26 | * Contains all logic for the process that uploads a set of files using direct binary access. 27 | * 28 | * The process will send events as a file is uploaded. Each event will be sent with 29 | * an object containing the following elements: 30 | * * {string} fileName: Name of the file being uploaded. 31 | * * {number} fileSize: Total size of the file, in bytes. 32 | * * {string} targetFolder: Full path to the AEM folder where the file is being 33 | * uploaded. 34 | * * {string} targetFile: Full path to the file in AEM. 35 | * * {string} mimeType: Mime type of the file being uploaded. 36 | * 37 | * Some events may send additional pieces of information, which will be noted 38 | * in the event itself. The events are: 39 | * 40 | * * filestart: Sent when the first part of a file begins to upload. 41 | * * fileprogress: Sent periodically to report on how much of the file has 42 | * uploaded. The event data will include these additional elements: 43 | * * {number} transferred: The total number of bytes that have transferred for 44 | * the file so far. 45 | * * fileend: Sent after the last part of a file has uploaded successfully. Will only 46 | * be sent if the file transfers completely. 47 | * * filecancelled: Sent if the file was cancelled before it could finish. 48 | * * fileerror: Sent if one of the file's parts failed to transfer. The event data 49 | * will include these additional elements: 50 | * * {Array} errors: List of errors that occurred to prevent the upload. 51 | */ 52 | class DirectBinaryUploadProcess extends UploadOptionsBase { 53 | /** 54 | * Constructs a new instance of the upload process for a single directory. 55 | * @param {object} options Overall direct binary process options. 56 | * @param {DirectBinaryUploadOptions} uploadOptions Options specific to the 57 | * current upload. 58 | */ 59 | constructor(options, uploadOptions) { 60 | super(options, uploadOptions); 61 | 62 | this.fileResults = {}; 63 | this.fileEvents = {}; 64 | this.fileTransfer = {}; 65 | this.completeUri = ''; 66 | this.uploadId = uuid(); 67 | 68 | const { log } = options; 69 | if (log) { 70 | httpTransferLogger.debug = (...args) => log.debug(...args); 71 | httpTransferLogger.info = (...args) => log.info(...args); 72 | httpTransferLogger.warn = (...args) => log.warn(...args); 73 | httpTransferLogger.error = (...args) => log.error(...args); 74 | } 75 | } 76 | 77 | /** 78 | * Retrieves a unique identifier that can be used to identify this particular upload. 79 | * 80 | * @returns {string} ID representing the upload. 81 | */ 82 | getUploadId() { 83 | return this.uploadId; 84 | } 85 | 86 | /** 87 | * Retrieves the total size of the upload, which is determined based on the file size 88 | * specified on each upload file. 89 | * 90 | * @returns {number} Total size, in bytes. 91 | */ 92 | getTotalSize() { 93 | let totalSize = 0; 94 | this.getUploadOptions().getUploadFiles().forEach((uploadFile) => { 95 | const { fileSize = 0 } = uploadFile; 96 | totalSize += fileSize; 97 | }); 98 | return totalSize; 99 | } 100 | 101 | /** 102 | * Does the work of uploading all files based on the upload options provided to the process. 103 | * 104 | * @param {import('./upload-result')} uploadResult Result to which information about 105 | * the upload will be added. 106 | * @returns {Promise} Resolves when all files have been uploaded. 107 | */ 108 | async upload(uploadResult) { 109 | const aemUploadOptions = getHttpTransferOptions(this.getOptions(), this.getUploadOptions()); 110 | const fileResults = new FileUploadResults(this.getOptions(), this.getUploadOptions()); 111 | fileResults.addHttpTransferOptions(aemUploadOptions); 112 | const aemUpload = new AEMUpload(); 113 | aemUpload.on('filestart', (data) => { 114 | this.logInfo(`Upload START '${data.fileName}': ${data.fileSize} bytes`); 115 | this.emit('filestart', data); 116 | }); 117 | aemUpload.on('fileprogress', (data) => { 118 | this.logInfo(`Upload PROGRESS '${data.fileName}': ${data.transferred} of ${data.fileSize} bytes`); 119 | this.emit('fileprogress', data); 120 | }); 121 | aemUpload.on('fileend', (data) => { 122 | this.logInfo(`Upload COMPLETE '${data.fileName}': ${data.fileSize} bytes`); 123 | fileResults.addFileEventResult(data); 124 | this.emit('fileend', data); 125 | }); 126 | aemUpload.on('fileerror', (data) => { 127 | this.logError(`Upload FAILED '${data.fileName}': '${data.errors[0].message}'`); 128 | fileResults.addFileEventResult(data); 129 | this.emit('fileerror', data); 130 | }); 131 | 132 | const fileCount = aemUploadOptions.uploadFiles.length; 133 | 134 | uploadResult.startTimer(); 135 | 136 | this.logInfo(`sending ${fileCount} files to httptransfer`); 137 | await aemUpload.uploadFiles(aemUploadOptions); 138 | this.logInfo('successfully uploaded files with httptransfer'); 139 | 140 | uploadResult.setFileUploadResults(fileResults); 141 | uploadResult.stopTimer(); 142 | 143 | // output json result to logger 144 | this.logInfo(`Uploading result in JSON: ${JSON.stringify(uploadResult, null, 4)}`); 145 | } 146 | } 147 | 148 | module.exports = DirectBinaryUploadProcess; 149 | -------------------------------------------------------------------------------- /src/direct-binary-upload.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const UploadBase = require('./upload-base'); 14 | const DirectBinaryUploadProcess = require('./direct-binary-upload-process'); 15 | const UploadResult = require('./upload-result'); 16 | 17 | /** 18 | * Provides capabilities for uploading assets to an AEM instance configured with 19 | * direct binary access. 20 | */ 21 | class DirectBinaryUpload extends UploadBase { 22 | /** 23 | * Uploads multiple files to a target AEM instance. Through configuration, 24 | * supports various potential sources, including a node.js process or a 25 | * browser. 26 | * 27 | * @param {DirectBinaryUploadOptions} options Controls how the upload will behave. See class 28 | * documentation for more details. 29 | * @returns {Promise} Will be resolved when all the files have been uploaded. The data 30 | * passed in successful resolution will be an instance of UploadResult. 31 | */ 32 | async uploadFiles(options) { 33 | const uploadProcess = new DirectBinaryUploadProcess(this.getOptions(), options); 34 | const uploadResult = new UploadResult(this.getOptions(), options); 35 | 36 | this.beforeUploadProcess(uploadProcess); 37 | await this.executeUploadProcess(uploadProcess, uploadResult); 38 | this.afterUploadProcess(uploadProcess, uploadResult); 39 | 40 | return uploadResult.toJSON(); 41 | } 42 | } 43 | 44 | module.exports = DirectBinaryUpload; 45 | -------------------------------------------------------------------------------- /src/error-codes.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /** 14 | * The error codes that the upload process might provide to the consumer. 15 | */ 16 | module.exports = { 17 | /** 18 | * The "catch all" error code that is used in cases where the specific error type cannot 19 | * be determined. 20 | */ 21 | UNKNOWN: 'EUNKNOWN', 22 | 23 | /** 24 | * Used when some entity in the upload process could not be located. 25 | */ 26 | NOT_FOUND: 'ENOTFOUND', 27 | 28 | /** 29 | * Used when the target instance does not support direct binary upload. 30 | */ 31 | NOT_SUPPORTED: 'ENOTSUPPORTED', 32 | 33 | /** 34 | * Used when the options provided by the consumer were insufficient to perform the upload. 35 | */ 36 | INVALID_OPTIONS: 'EINVALIDOPTIONS', 37 | 38 | /** 39 | * Sent when the consumer has insufficient access to perform the upload. 40 | */ 41 | NOT_AUTHORIZED: 'ENOTAUTHORIZED', 42 | 43 | /** 44 | * Indicates an unexpected state in the target API. 45 | */ 46 | UNEXPECTED_API_STATE: 'EUNEXPECTEDAPISTATE', 47 | 48 | /** 49 | * An attempt was made to create an item that already exists. 50 | */ 51 | ALREADY_EXISTS: 'EALREADYEXISTS', 52 | 53 | /** 54 | * The user is forbidden from modifying the requested target. 55 | */ 56 | FORBIDDEN: 'EFORBIDDEN', 57 | 58 | /** 59 | * The user cancelled an operation. 60 | */ 61 | USER_CANCELLED: 'EUSERCANCELLED', 62 | 63 | /** 64 | * Payload provided by user is too large. 65 | */ 66 | TOO_LARGE: 'ETOOLARGE', 67 | }; 68 | -------------------------------------------------------------------------------- /src/exports.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | you may not use this file except in compliance with the License. You may obtain a copy 4 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const DirectBinaryUpload = require('./direct-binary-upload'); 14 | const DirectBinaryUploadOptions = require('./direct-binary-upload-options'); 15 | const DirectBinaryUploadErrorCodes = require('./error-codes'); 16 | const FileSystemUpload = require('./filesystem-upload'); 17 | const FileSystemUploadOptions = require('./filesystem-upload-options'); 18 | 19 | module.exports = { 20 | DirectBinaryUpload, 21 | DirectBinaryUploadOptions, 22 | DirectBinaryUploadErrorCodes, 23 | FileSystemUpload, 24 | FileSystemUploadOptions, 25 | }; 26 | -------------------------------------------------------------------------------- /src/file-upload-results.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const { getAverage } = require('./utils'); 14 | const UploadOptionsBase = require('./upload-options-base'); 15 | 16 | class FileUploadResults extends UploadOptionsBase { 17 | /** 18 | * Constructs a new instance using the provided information. 19 | * 20 | * @param {object} options Options as provided when the direct binary object was instantiated. 21 | * @param {import('./direct-binary-upload-options')} uploadOptions Options as provided 22 | * when the direct binary upload process was initiated. 23 | */ 24 | constructor(options, uploadOptions) { 25 | super(options, uploadOptions); 26 | this.fileLookup = {}; 27 | } 28 | 29 | /** 30 | * Retrieves the total number of files that were included in the upload. 31 | * @returns {number} File count. 32 | */ 33 | getTotalFileCount() { 34 | return Object.keys(this.fileLookup).length; 35 | } 36 | 37 | /** 38 | * Sets the node-httptransfer options that were used to upload a given file. 39 | * @param {*} transferOptions Options for node-httptransfer. 40 | */ 41 | addHttpTransferOptions(transferOptions) { 42 | transferOptions.uploadFiles.forEach((uploadFile) => { 43 | const { fileUrl } = uploadFile; 44 | const targetPath = decodeURI(new URL(fileUrl).pathname); 45 | 46 | const fileInfo = { ...uploadFile }; 47 | if (fileInfo.blob) { 48 | fileInfo.blob = ''; 49 | } 50 | this.fileLookup[targetPath] = fileInfo; 51 | }); 52 | } 53 | 54 | /** 55 | * Adds the result of a file transfer. Will be associated with the options 56 | * previously specified for a file through addHttpTransferOptions(). 57 | * @param {*} data Event data as received from a node-httptransfer event. 58 | */ 59 | addFileEventResult(data) { 60 | const { targetFile } = data; 61 | if (this.fileLookup[targetFile]) { 62 | this.fileLookup[targetFile].result = data; 63 | } 64 | } 65 | 66 | /** 67 | * Retrieves the total size, in bytes, of all files that were uploaded. 68 | * @returns {number} Size, in bytes. 69 | */ 70 | getTotalSize() { 71 | return Object.keys(this.fileLookup) 72 | .map((file) => this.fileLookup[file].fileSize) 73 | .reduce((a, b) => a + b); 74 | } 75 | 76 | /** 77 | * Retrieves the average size, in bytes, of all files that were 78 | * uploaded. 79 | * @returns {number} Size, in bytes. 80 | */ 81 | getAverageSize() { 82 | return getAverage( 83 | Object.keys(this.fileLookup) 84 | .map((file) => this.fileLookup[file].fileSize), 85 | ); 86 | } 87 | 88 | /** 89 | * Retrieves the total number of files that uploaded successfully. 90 | * @returns {number} File count. 91 | */ 92 | getSuccessCount() { 93 | let count = 0; 94 | Object.keys(this.fileLookup).forEach((file) => { 95 | const { result } = this.fileLookup[file]; 96 | if (result) { 97 | const { errors } = result; 98 | if (errors === undefined) { 99 | count += 1; 100 | } 101 | } 102 | }); 103 | return count; 104 | } 105 | 106 | /** 107 | * Retrieves an array of _all_ errors that were encountered as 108 | * files were transferred. 109 | * @returns {Array} Array of error information. 110 | */ 111 | getErrors() { 112 | const allErrors = []; 113 | Object.keys(this.fileLookup).forEach((file) => { 114 | const { result } = file; 115 | if (result) { 116 | const { errors = [] } = result; 117 | errors.forEach((error) => allErrors.push(error)); 118 | } 119 | }); 120 | return allErrors; 121 | } 122 | 123 | /** 124 | * Converts the result into a simple javascript object containing all 125 | * of the result's information. 126 | * @returns Simple object. 127 | */ 128 | toJSON() { 129 | return Object.keys(this.fileLookup).map((path) => this.fileLookup[path]); 130 | } 131 | } 132 | 133 | module.exports = FileUploadResults; 134 | -------------------------------------------------------------------------------- /src/filesystem-upload-asset.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /** 14 | * Represents an asset that will be created in the target AEM 15 | * instance. Consist of functionality for ensuring that the 16 | * remote path of the asset is consistent with the configuration 17 | * of the upload. 18 | */ 19 | 20 | const FileSystemUploadDirectory = require('./filesystem-upload-directory'); 21 | 22 | class FileSystemUploadAsset extends FileSystemUploadDirectory { 23 | /** 24 | * Constructs a new instance of the class using the given values. 25 | * @param {FileSystemUploadOptions} uploadOptions URL of the options 26 | * will be used to build the asset's remote path. 27 | * @param {string} localPath Full path to the local asset. 28 | * @param {string} remoteNodeName Name of the asset's node as it should 29 | * appear in AEM. 30 | * @param {number} size Size, in bytes, of the asset. 31 | * @param {FileSystemUploadDirectory} [directory] If provided, the 32 | * directory to which the asset belongs. If not provided then the 33 | * asset will be treated as the root. 34 | */ 35 | constructor(uploadOptions, localPath, remoteNodeName, size, directory) { 36 | super(uploadOptions, localPath, remoteNodeName, directory); 37 | this.size = size; 38 | } 39 | 40 | /** 41 | * Retrieves the size of the asset, as provided in the constructor. 42 | * @returns {number} Size in bytes. 43 | */ 44 | getSize() { 45 | return this.size; 46 | } 47 | } 48 | 49 | module.exports = FileSystemUploadAsset; 50 | -------------------------------------------------------------------------------- /src/filesystem-upload-directory.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const Path = require('path'); 14 | 15 | /** 16 | * Represents a directory that will be created in the target AEM 17 | * instance. Consist of functionality for ensuring that the 18 | * remote path of the folder is consistent with the configuration 19 | * of the upload. 20 | */ 21 | class FileSystemUploadDirectory { 22 | /** 23 | * Constructs a new directory instance with the given values. 24 | * @param {FileSystemUploadOptions} uploadOptions The URL from 25 | * the options will be used to build the remote URL of the 26 | * directory. 27 | * @param {string} localPath Full local path to the directory. 28 | * @param {string} remoteNodeName The name of the folder's node 29 | * as it should appear in AEM. 30 | * @param {FileSystemUploadDirectory} [parent] Parent directory 31 | * for this directory. If not supplied then the directory will 32 | * be treated as the root of the upload. 33 | */ 34 | constructor(uploadOptions, localPath, remoteNodeName, parent) { 35 | this.uploadOptions = uploadOptions; 36 | this.localPath = localPath; 37 | this.remoteName = remoteNodeName; 38 | this.parent = parent; 39 | } 40 | 41 | /** 42 | * Retrieves the full, local path of the directory, as provided in 43 | * the constructor. 44 | * @returns {string} Local directory path. 45 | */ 46 | getLocalPath() { 47 | return this.localPath; 48 | } 49 | 50 | /** 51 | * Retrieves the full, remote path (only) of the item. Will be built 52 | * using the remote node name provided in the constructor. 53 | * 54 | * The value will not be URL encoded. 55 | * 56 | * @returns {string} Path ready for use in a URL. 57 | */ 58 | getRemotePath() { 59 | const prefix = this.parent 60 | ? this.parent.getRemotePath() 61 | : this.uploadOptions.getTargetFolderPath(); 62 | return `${prefix}/${this.getRemoteNodeName()}`; 63 | } 64 | 65 | /** 66 | * Retrieves the remote URL of the item's parent. 67 | * @returns {string} The item parent's URL. 68 | */ 69 | getParentRemoteUrl() { 70 | if (!this.parentRemoteUrl) { 71 | const path = this.parent 72 | ? this.parent.getRemotePath() 73 | : this.uploadOptions.getTargetFolderPath(); 74 | this.parentRemoteUrl = `${this.uploadOptions.getUrlPrefix()}${encodeURI(path)}`; 75 | } 76 | return this.parentRemoteUrl; 77 | } 78 | 79 | /** 80 | * Retrieves the remote node name of the item, as provided in the 81 | * constructor. 82 | * @returns {string} A node name. 83 | */ 84 | getRemoteNodeName() { 85 | return this.remoteName; 86 | } 87 | 88 | /** 89 | * The name of the item as it was originally provided in the local 90 | * path. 91 | * @returns {string} Item name. 92 | */ 93 | getName() { 94 | return Path.basename(this.localPath); 95 | } 96 | } 97 | 98 | module.exports = FileSystemUploadDirectory; 99 | -------------------------------------------------------------------------------- /src/filesystem-upload-item-manager.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const Path = require('path'); 14 | 15 | const { normalizePath } = require('./utils'); 16 | const FileSystemUploadDirectory = require('./filesystem-upload-directory'); 17 | const FileSystemUploadAsset = require('./filesystem-upload-asset'); 18 | const { 19 | cleanFolderName, 20 | cleanAssetName, 21 | getItemManagerParent, 22 | } = require('./filesystem-upload-utils'); 23 | 24 | /** 25 | * Keeps track of FileSystemUploadDirectory and FileSystemUploadAsset 26 | * instances that have been retrieved using the manager. Its primary 27 | * purpose is to ensure that an instance for any given path is only 28 | * created once, then reused thereafter. It will also automatically 29 | * create necessary instances for parent directories up until the 30 | * manager's root path. 31 | */ 32 | class FileSystemUploadItemManager { 33 | /** 34 | * Constructs a new, empty instance of the manager that will use 35 | * a the given information. 36 | * @param {FileSystemUploadOptions} uploadOptions Will be given to 37 | * each FileSystemUploadDirectory instance that is created. 38 | * @param {string} rootPath The top-most path that the manager 39 | * will track. 40 | * @param {boolean} [keepFlat] If true, assets retrieved through 41 | * the item manager will be kept at the root of the upload instead 42 | * of inside its parent directory. Default: false. 43 | */ 44 | constructor(uploadOptions, rootPath, keepFlat = false) { 45 | this.uploadOptions = uploadOptions; 46 | this.directories = new Map(); 47 | this.assets = new Map(); 48 | this.rootPath = normalizePath(rootPath); 49 | this.keepFlat = keepFlat; 50 | } 51 | 52 | /** 53 | * Retrieves an instance of FileSystemUploadDirectory for a given local path. 54 | * The method will cache the instance, if necessary, and will also ensure that 55 | * all parent instances up to the manager's root are also cached. 56 | * @param {string} localPath Directory path whose information will be used 57 | * to create the FileSystemUploadDirectory instance. 58 | * @returns {Promise} Resolved with a FileSystemUploadDirectory instance 59 | * representing of the given directory path. 60 | */ 61 | async getDirectory(localPath) { 62 | const normalizedPath = normalizePath(localPath); 63 | const parent = await getItemManagerParent(this, this.rootPath, localPath); 64 | 65 | if (!this.directories.has(normalizedPath)) { 66 | const nodeName = await cleanFolderName(this.uploadOptions, Path.basename(normalizedPath)); 67 | this.directories.set( 68 | normalizedPath, 69 | new FileSystemUploadDirectory( 70 | this.uploadOptions, 71 | normalizedPath, 72 | nodeName, 73 | parent, 74 | ), 75 | ); 76 | } 77 | 78 | return this.directories.get(normalizedPath); 79 | } 80 | 81 | /** 82 | * Retrieves an instance of FileSystemUploadAsset for a given local path. 83 | * The method will cache the instance, if necessary, and will also ensure that 84 | * all parent instances up to the manager's root are also cached. 85 | * @param {string} localPath Asset path whose information will be used 86 | * to create the FileSystemUploadAsset instance. 87 | * @param {number} size Size, in bytes, of the asset. 88 | * @returns {Promise} Resolved with a FileSystemUploadAsset instance 89 | * representing of the given asset path. 90 | */ 91 | async getAsset(localPath, size) { 92 | const normalizedPath = normalizePath(localPath); 93 | const parent = !this.keepFlat 94 | ? await getItemManagerParent(this, this.rootPath, localPath) 95 | : undefined; 96 | 97 | if (!this.assets.has(normalizedPath)) { 98 | const nodeName = await cleanAssetName(this.uploadOptions, Path.basename(normalizedPath)); 99 | this.assets.set( 100 | normalizedPath, 101 | new FileSystemUploadAsset( 102 | this.uploadOptions, 103 | normalizedPath, 104 | nodeName, 105 | size, 106 | parent, 107 | ), 108 | ); 109 | } 110 | 111 | return this.assets.get(normalizedPath); 112 | } 113 | 114 | /** 115 | * Retrieves a value indicating whether the given local directory path 116 | * has been cached in the manager. 117 | * @param {string} localPath Full path to a local directory. 118 | * @returns {boolean} True if the directory is cached, false otherwise. 119 | */ 120 | hasDirectory(localPath) { 121 | return this.directories.has(normalizePath(localPath)); 122 | } 123 | 124 | /** 125 | * Retrieves a value indicating whether the given local asset path 126 | * has been cached in the manager. 127 | * @param {string} localPath Full path to a local asset. 128 | * @returns {boolean} True if the asset is cached, false otherwise. 129 | */ 130 | hasAsset(localPath) { 131 | return this.assets.has(normalizePath(localPath)); 132 | } 133 | } 134 | 135 | module.exports = FileSystemUploadItemManager; 136 | -------------------------------------------------------------------------------- /src/filesystem-upload-options.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const DirectBinaryUploadOptions = require('./direct-binary-upload-options'); 14 | const { DefaultValues, RegularExpressions } = require('./constants'); 15 | const UploadError = require('./upload-error'); 16 | const ErrorCodes = require('./error-codes'); 17 | 18 | /** 19 | * Options specific to a file system upload. Also supports all options defined by 20 | * DirectBinaryUploadOptions. 21 | */ 22 | class FileSystemUploadOptions extends DirectBinaryUploadOptions { 23 | constructor() { 24 | super(); 25 | this.replaceValue = '-'; 26 | this.folderNodeProcessor = async (name) => name.replace( 27 | RegularExpressions.INVALID_FOLDER_CHARACTERS_REGEX, 28 | this.replaceValue, 29 | ).toLowerCase(); 30 | 31 | this.assetNodeProcessor = async (name) => name.replace( 32 | RegularExpressions.INVALID_ASSET_CHARACTERS_REGEX, 33 | this.replaceValue, 34 | ); 35 | } 36 | 37 | /** 38 | * Creates a new FileSystemUploadOptions instance that will have the same options 39 | * as an existing options instance. 40 | * @param {DirectBinaryUploadOptions} uploadOptions Options whose value should 41 | * be copied. 42 | */ 43 | static fromOptions(uploadOptions) { 44 | const newOptions = new FileSystemUploadOptions(); 45 | newOptions.options = { ...uploadOptions.options }; 46 | newOptions.controller = uploadOptions.controller; 47 | 48 | if (uploadOptions.proxy) { 49 | newOptions.proxy = uploadOptions.proxy; 50 | } 51 | 52 | if (typeof (uploadOptions.replaceValue) === 'string') { 53 | newOptions.replaceValue = uploadOptions.replaceValue; 54 | } 55 | 56 | if (typeof (uploadOptions.folderNodeProcessor) === 'function') { 57 | newOptions.folderNodeProcessor = uploadOptions.folderNodeProcessor; 58 | } 59 | 60 | if (typeof (uploadOptions.assetNodeProcessor) === 'function') { 61 | newOptions.assetNodeProcessor = uploadOptions.assetNodeProcessor; 62 | } 63 | 64 | if (typeof (uploadOptions.uploadFileOptions) === 'object') { 65 | newOptions.uploadFileOptions = uploadOptions.uploadFileOptions; 66 | } 67 | 68 | return newOptions; 69 | } 70 | 71 | /** 72 | * Sets the maximum number of files that can be uploaded using the module at one time. If 73 | * the total number of files exceeds this number then the library will throw an exception 74 | * with code TOO_LARGE. 75 | * @param {*} maxFileCount 76 | * @returns {FileSystemUploadOptions} The current options instance. Allows for chaining. 77 | */ 78 | withMaxUploadFiles(maxFileCount) { 79 | this.options.maxUploadFiles = maxFileCount; 80 | return this; 81 | } 82 | 83 | /** 84 | * Sets a value indicating whether or not the process should upload all descendent 85 | * directories and files within the given directory. 86 | * @param {boolean} deepUpload True if the upload should be deep, false otherwise. 87 | * @returns {FileSystemUploadOptions} The current options instance. Allows for chaining. 88 | */ 89 | withDeepUpload(deepUpload) { 90 | this.options.deepUpload = deepUpload; 91 | return this; 92 | } 93 | 94 | /** 95 | * Sets a function that will be called before a folder is created in AEM. The given function 96 | * argument will be given the name of the folder as it appears on the file system, and should 97 | * return the name to use as the folder's node name in AEM. 98 | * 99 | * Regardless of the return value of the processor function, certain illegal characters will 100 | * always be removed from the node name. These include /\.[]*:|. The characters 101 | * will be replaced by the value set using withInvalidCharacterReplaceValue(). 102 | * 103 | * The original folder name will be used as the AEM folder's title. 104 | * 105 | * The default behavior is to replace characters %;#,+?^{}" (and whitespace) with 106 | * the value set using withInvalidCharacterReplaceValue(), and to convert the name to lower 107 | * case. 108 | * @param {function} processorFunction Function that will receive a single argument - the 109 | * name of a folder. Should return a Promise that resolves with the node name to use for 110 | * the folder. 111 | * @returns {FileSystemUploadOptions} The current options instance. Allows for chaining. 112 | */ 113 | withFolderNodeNameProcessor(processorFunction) { 114 | this.folderNodeProcessor = processorFunction; 115 | return this; 116 | } 117 | 118 | /** 119 | * Sets a function that will be called before an asset is created in AEM. The given function 120 | * argument will be given the name of the asset as it appears on the file system, and should 121 | * return the name to use as the asset's node name in AEM. 122 | * 123 | * Regardless of the return value of the processor function, certain illegal characters will 124 | * always be removed from the node name. These include /\.[]*:|. The characters 125 | * will be replaced by the value set using withInvalidCharacterReplaceValue(). 126 | * 127 | * The default behavior is to replace characters #%{}?& with the 128 | * value set using withInvalidCharacterReplaceValue(). 129 | * @param {function} processorFunction Function that will receive a single argument - the 130 | * name of an asset. Should return a Promise that resolves with the node name to use for 131 | * the asset. 132 | * @returns {FileSystemUploadOptions} The current options instance. Allows for chaining. 133 | */ 134 | withAssetNodeNameProcessor(processorFunction) { 135 | this.assetNodeProcessor = processorFunction; 136 | return this; 137 | } 138 | 139 | /** 140 | * The value to use when replacing invalid characters in folder or asset node names. 141 | * @param {string} replaceValue Value to use when replacing invalid node name characters. 142 | * Must not contain any of the invalid characters. 143 | * @returns {FileSystemUploadOptions} The current options instance. Allows for chaining. 144 | */ 145 | withInvalidCharacterReplaceValue(replaceValue) { 146 | if (RegularExpressions.INVALID_CHARACTERS_REGEX.test(replaceValue)) { 147 | throw new UploadError( 148 | 'Invalid character replace value contains invalid characters', 149 | ErrorCodes.INVALID_OPTIONS, 150 | ); 151 | } 152 | 153 | this.replaceValue = replaceValue; 154 | return this; 155 | } 156 | 157 | /** 158 | * Upload file options that will be applied to each file that is uploaded as 159 | * part of the file system upload. Most options that can be passed as part of 160 | * a single file upload using DirectBinaryUploadOptions.withUploadFiles() are 161 | * valid. The only exceptions are "fileName", "fileSize", "filePath", and 162 | * "blob", which will be ignored. 163 | * 164 | * @param {object} options Upload file options to apply to each file. 165 | */ 166 | withUploadFileOptions(options) { 167 | this.uploadFileOptions = options; 168 | return this; 169 | } 170 | 171 | /** 172 | * Retrieves the maximum number of files that the module can upload in a single upload 173 | * request. 174 | * 175 | * @returns {number} Maximum file count. 176 | */ 177 | getMaxUploadFiles() { 178 | return this.options.maxUploadFiles || DefaultValues.MAX_FILE_UPLOAD; 179 | } 180 | 181 | /** 182 | * Retrieves a value indicating whether the process should upload all descendent 183 | * directories and files within the given directory. 184 | * @returns {boolean} True for a deep upload, false otherwise. 185 | */ 186 | getDeepUpload() { 187 | return !!this.options.deepUpload; 188 | } 189 | 190 | /** 191 | * Retrieves the function to use to get the node name for a folder to create 192 | * in AEM. 193 | * @returns {function} Function that expects a single folder name argument, and 194 | * returns a Promise that will be resolved with a node name. 195 | */ 196 | getFolderNodeNameProcessor() { 197 | return this.folderNodeProcessor; 198 | } 199 | 200 | /** 201 | * Retrieves the function to use to get the node name for an asset to create 202 | * in AEM. 203 | * @returns {function} Function that expects a single asset name argument, and 204 | * returns a Promise that will be resolved with a node name. 205 | */ 206 | getAssetNodeNameProcessor() { 207 | return this.assetNodeProcessor; 208 | } 209 | 210 | /** 211 | * Retrieves the value to use when replacing invalid characters in node names. 212 | * @returns {string} Replace value. 213 | */ 214 | getInvalidCharacterReplaceValue() { 215 | return this.replaceValue; 216 | } 217 | 218 | /** 219 | * Retrieves the upload file options that will be applied to each file uploaded 220 | * through the module. 221 | * 222 | * @returns {object} Upload file options. 223 | */ 224 | getUploadFileOptions() { 225 | return this.uploadFileOptions || {}; 226 | } 227 | } 228 | 229 | module.exports = FileSystemUploadOptions; 230 | -------------------------------------------------------------------------------- /src/filesystem-upload-utils.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const Path = require('path'); 14 | 15 | const { DefaultValues, RegularExpressions } = require('./constants'); 16 | const { normalizePath } = require('./utils'); 17 | const UploadError = require('./upload-error'); 18 | const ErrorCodes = require('./error-codes'); 19 | 20 | /** 21 | * Retrieves the option indicating whether or not the upload is deep. Takes 22 | * into account that the options might not be FileSystemUploadOptions. 23 | * @param {FileSystemUploadOptions|DirectBinaryUploadOptions} uploadOptions Options 24 | * to retrieve value from. 25 | * @returns {boolean} True if it's a deep upload, false otherwise. 26 | */ 27 | function isDeepUpload(uploadOptions) { 28 | if (!uploadOptions.getDeepUpload) { 29 | // default to false if the class received an options instance 30 | // not of type FileSystemUploadOptions. 31 | return false; 32 | } 33 | return uploadOptions.getDeepUpload(); 34 | } 35 | 36 | /** 37 | * Retrieves the option specifying the maximum number of files that can be 38 | * uploaded at once. Takes into account that the options might not be 39 | * FileSystemUploadOptions. 40 | * @param {FileSystemUploadOptions|DirectBinaryUploadOptions} uploadOptions Options 41 | * to retrieve value from. 42 | * @returns {number} Maximum number of files to upload. 43 | */ 44 | function getMaxFileCount(uploadOptions) { 45 | if (!uploadOptions.getMaxUploadFiles) { 46 | return DefaultValues.MAX_FILE_UPLOAD; 47 | } 48 | return uploadOptions.getMaxUploadFiles(); 49 | } 50 | 51 | /** 52 | * Uses a processor function to clean a node name, then cleans generally disallowed characters 53 | * from the name. 54 | * @param {FileSystemUploadOptions} uploadOptions Used to retrieve the value to use when replacing 55 | * invalid characters. 56 | * @param {function} processorFunction Will be given the provided node name as a single argument. 57 | * Expected to return a Promise that will be resolved with the clean node name value. 58 | * @param {string} nodeName Value to be cleaned of invalid characters. 59 | * @returns {Promise} Will be resolved with the cleaned node name. 60 | */ 61 | async function cleanNodeName(uploadOptions, processorFunction, nodeName) { 62 | const processedName = await processorFunction(nodeName); 63 | return processedName.replace( 64 | RegularExpressions.INVALID_CHARACTERS_REGEX, 65 | uploadOptions.getInvalidCharacterReplaceValue(), 66 | ); 67 | } 68 | 69 | /** 70 | * Uses the given options to clean a folder name, then cleans generally disallowed characters 71 | * from the name. 72 | * @param {FileSystemUploadOptions} uploadOptions Used to retrieve the value to use when replacing 73 | * invalid characters, and the function to call to clean the folder name. 74 | * @param {string} folderName Value to be cleaned of invalid characters. 75 | * @returns {Promise} Will be resolved with the clean name. 76 | */ 77 | async function cleanFolderName(uploadOptions, folderName) { 78 | return cleanNodeName(uploadOptions, uploadOptions.getFolderNodeNameProcessor(), folderName); 79 | } 80 | 81 | /** 82 | * Uses the given options to clean an asset name, then cleans generally disallowed characters 83 | * from the name. 84 | * @param {FileSystemUploadOptions} uploadOptions Used to retrieve the value to use when 85 | * replacing invalid characters, and the function to call to clean the asset name. 86 | * @param {string} folderName Value to be cleaned of invalid characters. 87 | * @returns {Promise} Will be resolved with the clean name. 88 | */ 89 | async function cleanAssetName(uploadOptions, assetName) { 90 | const { 91 | name: assetNameOnly, 92 | ext, 93 | } = Path.parse(assetName); 94 | const cleanName = await cleanNodeName( 95 | uploadOptions, 96 | uploadOptions.getAssetNodeNameProcessor(), 97 | assetNameOnly, 98 | ); 99 | return `${cleanName}${ext}`; 100 | } 101 | 102 | async function getItemManagerParent(itemManager, rootPath, localPath) { 103 | const normalizedPath = normalizePath(localPath); 104 | let parent; 105 | 106 | if (normalizedPath !== rootPath && !String(normalizedPath).startsWith(`${rootPath}/`)) { 107 | throw new UploadError('directory to upload is outside expected root', ErrorCodes.INVALID_OPTIONS); 108 | } 109 | 110 | if (normalizedPath !== rootPath) { 111 | parent = await itemManager.getDirectory(normalizedPath.substr(0, normalizedPath.lastIndexOf('/'))); 112 | } 113 | 114 | return parent; 115 | } 116 | 117 | module.exports = { 118 | isDeepUpload, 119 | getMaxFileCount, 120 | cleanFolderName, 121 | cleanAssetName, 122 | getItemManagerParent, 123 | }; 124 | -------------------------------------------------------------------------------- /src/fs-promise.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const fs = require('fs'); 14 | const UploadError = require('./upload-error'); 15 | const ErrorCodes = require('./error-codes'); 16 | 17 | const unsupportedError = () => { 18 | throw new UploadError('filesystem operations are not permitted in a browser', ErrorCodes.INVALID_OPTIONS); 19 | }; 20 | 21 | let stat = unsupportedError; 22 | let readdir = unsupportedError; 23 | let createReadStream = unsupportedError; 24 | 25 | // fs module is not supported in browsers 26 | if (fs) { 27 | // doing this manually and not using promisify to support older 28 | // versions of node 29 | stat = (path) => new Promise((res, rej) => { 30 | fs.stat(path, (err, stats) => { 31 | if (err) { 32 | rej(err); 33 | return; 34 | } 35 | res(stats); 36 | }); 37 | }); 38 | readdir = (path) => new Promise((res, rej) => { 39 | fs.readdir(path, (err, result) => { 40 | if (err) { 41 | rej(err); 42 | return; 43 | } 44 | res(result); 45 | }); 46 | }); 47 | createReadStream = fs.createReadStream; 48 | } 49 | 50 | module.exports = { 51 | stat, 52 | readdir, 53 | createReadStream, 54 | }; 55 | -------------------------------------------------------------------------------- /src/http-result.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const UploadOptionsBase = require('./upload-options-base'); 14 | const UploadError = require('./upload-error'); 15 | 16 | class HttpResult extends UploadOptionsBase { 17 | /** 18 | * Constructs a new instance of the results, which can be used to add more information. 19 | */ 20 | constructor(options, uploadOptions) { 21 | super(options, uploadOptions); 22 | 23 | this.retryErrors = []; 24 | } 25 | 26 | /** 27 | * Adds an error to the result's list of retry errors. 28 | * 29 | * @param {Error|string} e An error that occurred. 30 | */ 31 | addRetryError(e) { 32 | this.retryErrors.push(UploadError.fromError(e)); 33 | } 34 | 35 | /** 36 | * Retrieves the list of retry errors that occurred within the result's scope. 37 | * 38 | * @returns {Array} List of UploadError instances. 39 | */ 40 | getRetryErrors() { 41 | return this.retryErrors; 42 | } 43 | 44 | /** 45 | * Converts the result to its JSON string representation. 46 | * 47 | * @returns {string} The result as a string. 48 | */ 49 | toString() { 50 | return JSON.stringify(this.toJSON()); 51 | } 52 | 53 | /** 54 | * Converts the result to a simple object. 55 | * 56 | * @returns {object} Result information. 57 | */ 58 | toJSON() { 59 | return { 60 | retryErrors: this.retryErrors, 61 | }; 62 | } 63 | } 64 | 65 | module.exports = HttpResult; 66 | -------------------------------------------------------------------------------- /src/http-utils.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const originalFetch = require('node-fetch'); 14 | const { fetchClient } = require('@adobe/cloud-service-client'); 15 | 16 | const UploadFile = require('./upload-file'); 17 | 18 | const fetch = fetchClient(originalFetch, { 19 | handleCookies: true, 20 | }); 21 | 22 | /** 23 | * Submits an HTTP request using fetch, then provides the response. 24 | * @param {string} url HTTP URL to which request will be submitted. 25 | * @param {*} options Raw options that will be passed directly to fetch. 26 | * @returns {*} A fetch HTTP response. 27 | */ 28 | function submitRequest(url, options = {}) { 29 | return fetch(url, options); 30 | } 31 | 32 | /** 33 | * Converts options provided in a DirectBinaryUploadOptions instance to a format 34 | * suitable to pass to the httptransfer module. 35 | * @param {object} options General upload object options. 36 | * @param {import('./direct-binary-upload-options')} directBinaryUploadOptions Options 37 | * to convert. 38 | */ 39 | function getHttpTransferOptions(options, directBinaryUploadOptions) { 40 | // the httptransfer module accepts a full fileUrl instead of a single 41 | // url with individual file names. if needed, convert the format with a 42 | // single url and individual file names to the fileUrl format. 43 | const convertedFiles = directBinaryUploadOptions.getUploadFiles().map((uploadFile) => { 44 | const uploadFileInstance = new UploadFile(options, directBinaryUploadOptions, uploadFile); 45 | const transferOptions = uploadFileInstance.toJSON(); 46 | if (uploadFile.blob) { 47 | // ensure blob is passed through to transfer options 48 | transferOptions.blob = uploadFile.blob; 49 | } 50 | return transferOptions; 51 | }); 52 | 53 | let headers = {}; 54 | const requestOptions = { ...directBinaryUploadOptions.getHttpOptions() }; 55 | if (requestOptions.headers) { 56 | // passing raw request options to node-httptransfer is somewhat limited because the 57 | // options will be used by init/complete requests to AEM, and inidividual part 58 | // transfers to blob storage. some options interfere with blob storage when included, 59 | // so removing headers here so that they won't be sent to blob storage. 60 | headers = { ...requestOptions.headers }; 61 | delete requestOptions.headers; 62 | } 63 | 64 | const retryOptions = { 65 | retryInitialDelay: directBinaryUploadOptions.getHttpRetryDelay(), 66 | retryMaxCount: directBinaryUploadOptions.getHttpRetryCount(), 67 | retryAllErrors: false, 68 | }; 69 | if (requestOptions.cloudClient) { 70 | retryOptions.retryAllErrors = requestOptions.cloudClient.eventuallyConsistentCreate || false; 71 | delete requestOptions.cloudClient; 72 | } 73 | 74 | const transferOptions = { 75 | uploadFiles: convertedFiles, 76 | concurrent: directBinaryUploadOptions.isConcurrent(), 77 | maxConcurrent: directBinaryUploadOptions.getMaxConcurrent(), 78 | timeout: directBinaryUploadOptions.getHttpRequestTimeout(), 79 | headers, 80 | requestOptions: { 81 | retryOptions, 82 | ...requestOptions, 83 | }, 84 | }; 85 | 86 | return transferOptions; 87 | } 88 | 89 | module.exports = { 90 | submitRequest, 91 | getHttpTransferOptions, 92 | }; 93 | -------------------------------------------------------------------------------- /src/upload-base.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const { EventEmitter } = require('events'); 14 | 15 | /** 16 | * Base class providing common functionality for an upload based on options 17 | * provided to a direct binary access-related instance. 18 | */ 19 | class UploadBase extends EventEmitter { 20 | /** 21 | * Initializes a new upload instance with the given options. 22 | * 23 | * @param {object} [options] Options controlling the upload. 24 | * @param {Object} [options.log] The object to use for logging messages during the upload 25 | * process. If specified, the object should contain methods info(), warn(), debug(), and 26 | * error(). Log information will be passed as parameters to these methods. 27 | */ 28 | constructor(options = {}) { 29 | super(); 30 | this.options = options; 31 | this.log = options.log; 32 | } 33 | 34 | /** 35 | * Retrieves the options as passed to the upload process instance. 36 | * 37 | * @returns {object} Raw options. 38 | */ 39 | getOptions() { 40 | return this.options; 41 | } 42 | 43 | /** 44 | * Uses the info() method of the provided logger to log information about the upload. 45 | */ 46 | logInfo(...theArguments) { 47 | if (this.log) { 48 | this.log.info(...theArguments); 49 | } 50 | } 51 | 52 | /** 53 | * Uses the warn() method of the provided logger to log information about the upload. 54 | */ 55 | logWarn(...theArguments) { 56 | if (this.log) { 57 | this.log.warn(...theArguments); 58 | } 59 | } 60 | 61 | /** 62 | * Uses the debug() method of the provided logger to log information about the upload. 63 | */ 64 | logDebug(...theArguments) { 65 | if (this.log) { 66 | this.log.debug(...theArguments); 67 | } 68 | } 69 | 70 | /** 71 | * Uses the error() method of the provided logger to log information about the upload. 72 | */ 73 | logError(...theArguments) { 74 | if (this.log) { 75 | this.log.error(...theArguments); 76 | } 77 | } 78 | 79 | /** 80 | * Sends an event to external consumers. 81 | * 82 | * @param {string} eventName The name of the event to send. 83 | * @param {object} eventData Will be included as the event's data. 84 | */ 85 | sendEvent(eventName, eventData) { 86 | this.emit(eventName, eventData); 87 | } 88 | 89 | /** 90 | * Builds information about an upload, which will be included in upload-level 91 | * events sent by the uploader process. 92 | * 93 | * @param {import('./direct-binary-upload-process')} uploadProcess The 94 | * upload process that will be performing the work of the upload. 95 | * @param {number} [directoryCount=0] If specified, the number of directories 96 | * that will be created by the upload process. 97 | */ 98 | // eslint-disable-next-line class-methods-use-this 99 | getUploadEventData(uploadProcess, directoryCount = 0) { 100 | return { 101 | uploadId: uploadProcess.getUploadId(), 102 | fileCount: uploadProcess.getUploadOptions().getUploadFiles().length, 103 | totalSize: uploadProcess.getTotalSize(), 104 | directoryCount, 105 | }; 106 | } 107 | 108 | /** 109 | * Sends an event that will inform consumers that items are about to be uploaded. 110 | * 111 | * @param {import('./direct-binary-upload-process')} uploadProcess The 112 | * upload process that will be performing the work of the upload. 113 | * @param {number} [directoryCount=0] If specified, the number of directories 114 | * that will be created by the upload process. 115 | */ 116 | beforeUploadProcess(uploadProcess, directoryCount = 0) { 117 | this.sendEvent('fileuploadstart', this.getUploadEventData(uploadProcess, directoryCount)); 118 | } 119 | 120 | /** 121 | * Sends an event that will inform consumers that items have finished uploading. 122 | * 123 | * @param {import('./direct-binary-upload-process')} uploadProcess The 124 | * upload process that will be performing the work of the upload. 125 | * @param {import('./upload-result')} uploadResult Result information 126 | * about the upload. 127 | * @param {number} [directoryCount=0] If specified, the number of directories 128 | * that will be created by the upload process. 129 | */ 130 | afterUploadProcess(uploadProcess, uploadResult, directoryCount = 0) { 131 | this.sendEvent('fileuploadend', { 132 | ...this.getUploadEventData(uploadProcess, directoryCount), 133 | result: uploadResult.toJSON(), 134 | }); 135 | } 136 | 137 | /** 138 | * Does the work of executing an upload of one or more files to AEM. 139 | * 140 | * @param {import('./direct-binary-upload-process')} uploadProcess The 141 | * upload process that will be performing the work of the upload. 142 | * @param {import('./upload-result')} uploadResult Result information 143 | * about the upload. 144 | * @returns {Promise} Resolves when the upload has finished. 145 | */ 146 | async executeUploadProcess(uploadProcess, uploadResult) { 147 | uploadProcess.on('filestart', (data) => this.sendEvent('filestart', data)); 148 | uploadProcess.on('fileprogress', (data) => this.sendEvent('fileprogress', data)); 149 | uploadProcess.on('fileend', (data) => this.sendEvent('fileend', data)); 150 | uploadProcess.on('fileerror', (data) => this.sendEvent('fileerror', data)); 151 | uploadProcess.on('filecancelled', (data) => this.sendEvent('filecancelled', data)); 152 | 153 | try { 154 | await uploadProcess.upload(uploadResult); 155 | } catch (uploadError) { 156 | uploadResult.addUploadError(uploadError); 157 | } 158 | } 159 | } 160 | 161 | module.exports = UploadBase; 162 | -------------------------------------------------------------------------------- /src/upload-error.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const errorCodes = require('./error-codes'); 14 | 15 | /** 16 | * Concatenates to message values together if both are provided. 17 | * 18 | * @param {string} overallMessage Will be prepended to specificMessage, delimited with a colon, if 19 | * supplied. 20 | * @param {string} specificMessage Will be concatenated with overallMessage, if supplied. Otherwise 21 | * the return value of the method will be specificMessage as-is. 22 | * @returns {string} A message value. 23 | */ 24 | function getFullMessage(overallMessage, specificMessage) { 25 | if (overallMessage) { 26 | return `${overallMessage}: ${specificMessage}`; 27 | } 28 | return specificMessage; 29 | } 30 | 31 | /** 32 | * Custom Error class containing additional information specific to the upload process. This 33 | * primarily consists of an error code, which can be used by consumers to provide more specific 34 | * information about the nature of an error. 35 | */ 36 | class UploadError extends Error { 37 | /** 38 | * Constructs a new UploadError instance out of a given error message. The method will attempt 39 | * to create the most specific type of error it can based on what it receives. 40 | * 41 | * @param {*} error Object from which to create the UploadError instance. Can be several things, 42 | * including an UploadError instance, an error as thrown by axios, a string, or another Error 43 | * instance. 44 | * @param {string} errorMessage Will appear in the error's "message" value. 45 | * @returns {UploadError} An upload error instance. 46 | */ 47 | static fromError(error, errorMessage = '') { 48 | const { 49 | message, 50 | code, 51 | uploadError, 52 | response, 53 | stack, 54 | } = error; 55 | 56 | if (uploadError) { 57 | return error; 58 | } 59 | 60 | if (response) { 61 | const { status } = response; 62 | 63 | let httpCode = errorCodes.UNKNOWN; 64 | if (status === 409) { 65 | httpCode = errorCodes.ALREADY_EXISTS; 66 | } else if (status === 403) { 67 | httpCode = errorCodes.FORBIDDEN; 68 | } else if (status === 400) { 69 | httpCode = errorCodes.INVALID_OPTIONS; 70 | } else if (status === 401) { 71 | httpCode = errorCodes.NOT_AUTHORIZED; 72 | } else if (status === 404) { 73 | httpCode = errorCodes.NOT_FOUND; 74 | } else if (status === 501) { 75 | httpCode = errorCodes.NOT_SUPPORTED; 76 | } 77 | return new UploadError(`Request failed with status code ${status}`, httpCode, stack); 78 | } 79 | 80 | if (message && code) { 81 | return new UploadError(getFullMessage(errorMessage, message), code, stack); 82 | } 83 | 84 | if (message) { 85 | return new UploadError(getFullMessage(errorMessage, message), errorCodes.UNKNOWN, stack); 86 | } 87 | 88 | if (typeof error === 'string') { 89 | return new UploadError(getFullMessage(errorMessage, error), errorCodes.UNKNOWN); 90 | } 91 | 92 | try { 93 | return new UploadError( 94 | getFullMessage( 95 | errorMessage, 96 | JSON.stringify(error), 97 | ), 98 | errorCodes.UNKNOWN, 99 | stack, 100 | ); 101 | } catch (e) { 102 | return new UploadError(getFullMessage(errorMessage, error), errorCodes.UNKNOWN, stack); 103 | } 104 | } 105 | 106 | /** 107 | * Constructs a new instance containing the provided information. 108 | * 109 | * @param {string} message The message that will appear with the Error instance. 110 | * @param {string} code The code indicating the specific type of error. 111 | * @param {string} [innerStack] Additional stack information if the UploadError instance 112 | * originated from another Error. 113 | */ 114 | constructor(message, code, innerStack = '') { 115 | super(message); 116 | this.code = code; 117 | this.innerStack = innerStack; 118 | this.uploadError = true; 119 | } 120 | 121 | /** 122 | * Retrieves the error code representing the specific type of error. See ErrorCodes for more 123 | * information. 124 | * 125 | * @returns {string} An error code value. 126 | */ 127 | getCode() { 128 | return this.code; 129 | } 130 | 131 | /** 132 | * Retrieves the upload error's status as an HTTP status code. 133 | * 134 | * @returns {number} An HTTP status code. 135 | */ 136 | getHttpStatusCode() { 137 | const code = this.getCode(); 138 | 139 | if (code === errorCodes.ALREADY_EXISTS) { 140 | return 409; 141 | } if (code === errorCodes.FORBIDDEN) { 142 | return 403; 143 | } if (code === errorCodes.INVALID_OPTIONS) { 144 | return 400; 145 | } if (code === errorCodes.NOT_AUTHORIZED) { 146 | return 401; 147 | } if (code === errorCodes.NOT_FOUND) { 148 | return 404; 149 | } if (code === errorCodes.TOO_LARGE) { 150 | return 413; 151 | } if (code === errorCodes.NOT_SUPPORTED) { 152 | return 501; 153 | } 154 | return 500; 155 | } 156 | 157 | /** 158 | * Retrieves a message describing the error. 159 | * 160 | * @returns {string} The error's message. 161 | */ 162 | getMessage() { 163 | return this.message; 164 | } 165 | 166 | /** 167 | * Retrieves the inner stack of the error, as provided to the constructor. 168 | * 169 | * @returns {string} The error's inner stack. 170 | */ 171 | getInnerStack() { 172 | return this.innerStack; 173 | } 174 | 175 | /** 176 | * Converts the error instance into a simplified object form. 177 | * 178 | * @returns {object} Simple object representation of the error. 179 | */ 180 | toJSON() { 181 | const json = { 182 | message: this.message, 183 | code: this.code, 184 | }; 185 | 186 | if (this.innerStack) { 187 | json.innerStack = this.innerStack; 188 | } 189 | 190 | return json; 191 | } 192 | 193 | /** 194 | * Converts the error to a string, which will be a stringified version of the error's toJSON() 195 | * method. 196 | * 197 | * @returns {string} String representation of the error. 198 | */ 199 | toString() { 200 | return JSON.stringify(this); 201 | } 202 | } 203 | 204 | module.exports = UploadError; 205 | -------------------------------------------------------------------------------- /src/upload-file.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const fs = require('fs'); 14 | const Path = require('path'); 15 | 16 | const UploadOptionsBase = require('./upload-options-base'); 17 | const UploadError = require('./upload-error'); 18 | const ErrorCodes = require('./error-codes'); 19 | 20 | /** 21 | * Analyzes the given file options and determines if there is sufficient information 22 | * to upload the file. If insufficient, the method will throw an exception. 23 | * 24 | * @param {object} options Information about the file to upload. 25 | */ 26 | function ensureRequiredOptions(options) { 27 | if ( 28 | (!options.fileName && !options.fileUrl) 29 | || (!options.fileSize && options.fileSize !== 0) 30 | || (!options.filePath 31 | && (!options.blob || !options.blob.slice)) 32 | ) { 33 | throw new UploadError('UploadFile missing required fields. Must have one of fileName or fileUrl, fileSize, and either filePath or blob', ErrorCodes.INVALID_OPTIONS); 34 | } 35 | } 36 | 37 | /** 38 | * Represents a file to upload, as provided in upload options. Includes information like the file's 39 | * name and size. Also provide capabilities for reading chunks of the file. 40 | */ 41 | class UploadFile extends UploadOptionsBase { 42 | /** 43 | * Constructs a new instance based on the given information. 44 | * 45 | * @param {object} options Options as provided when the direct binary upload was instantiated. 46 | * @param {DirectBinaryUploadOptions} uploadOptions Options as provided when the upload was 47 | * initiated. 48 | * @param {object} fileOptions Options for the specific file, as provided in the upload options. 49 | * @param {string} fileOptions.fileName The name of the file as it should appear when uploaded. 50 | * @param {number} fileOptions.fileSize Total size of the file to upload, in bytes. 51 | * @param {string} [fileOptions.filePath] Full path to the local filesystem file to upload. 52 | * Either this value or "blob" must be provided. 53 | * @param {Array} [fileOptions.blob] Full binary content of the file to upload. Either this 54 | * value or "filePath" must be provided. 55 | */ 56 | constructor(options, uploadOptions, fileOptions) { 57 | super(options, uploadOptions); 58 | this.fileOptions = fileOptions; 59 | } 60 | 61 | /** 62 | * Retrieves the full URL of the file, based on the given upload options 63 | * and file options. 64 | * 65 | * @returns {string} URL of the file. 66 | */ 67 | getFileUrl() { 68 | ensureRequiredOptions(this.fileOptions); 69 | 70 | let { fileUrl } = this.fileOptions; 71 | const { fileName } = this.fileOptions; 72 | 73 | if (!fileUrl) { 74 | fileUrl = `${this.getUploadOptions().getUrl()}/${encodeURIComponent(fileName)}`; 75 | } 76 | 77 | return fileUrl; 78 | } 79 | 80 | /** 81 | * Retrieves the name of the file as provided in the options. 82 | * 83 | * @returns {string} Name of the file. 84 | */ 85 | getFileName() { 86 | ensureRequiredOptions(this.fileOptions); 87 | const name = Path.basename(new URL(this.getFileUrl()).pathname); 88 | return decodeURIComponent(name); 89 | } 90 | 91 | /** 92 | * Retrieves the size of the file, in bytes, as provided in the options. 93 | * 94 | * @returns {number} Size of the file. 95 | */ 96 | getFileSize() { 97 | ensureRequiredOptions(this.fileOptions); 98 | return this.fileOptions.fileSize; 99 | } 100 | 101 | /** 102 | * Retrieves a value indicating whether or not a new version of the file should be 103 | * created if it already exists. 104 | * 105 | * @returns {boolean} True if a new version should be created, false otherwise. 106 | */ 107 | shouldCreateNewVersion() { 108 | ensureRequiredOptions(this.fileOptions); 109 | return !!this.fileOptions.createVersion; 110 | } 111 | 112 | /** 113 | * Retrieves the label of the new version should one need to be created. 114 | * 115 | * @returns {string} A version label. 116 | */ 117 | getVersionLabel() { 118 | ensureRequiredOptions(this.fileOptions); 119 | return this.fileOptions.versionLabel; 120 | } 121 | 122 | /** 123 | * Retrieves the comment of the new version should one need to be created. 124 | * 125 | * @returns {string} A version comment. 126 | */ 127 | getVersionComment() { 128 | ensureRequiredOptions(this.fileOptions); 129 | return this.fileOptions.versionComment; 130 | } 131 | 132 | /** 133 | * Retrieves a value indicating whether or not the asset should be replaced if 134 | * it already exists. 135 | * 136 | * @returns {boolean} True if the asset should be replaced, false otherwise. 137 | */ 138 | shouldReplace() { 139 | ensureRequiredOptions(this.fileOptions); 140 | return !!this.fileOptions.replace; 141 | } 142 | 143 | /** 144 | * Retrieves a chunk of the file for processing, based on the start and end 145 | * offset. The type of value returned by this method will vary depending on 146 | * the file options that were provided to the constructor. 147 | * @param {number} start Byte offset, inclusive, within the file where the chunk will begin. 148 | * @param {number} end Byte offset, exclusive, within the file where the chunk will end. 149 | * @returns {Readable|Array} If "filePath" was provided in the file options when constructed, 150 | * then the return value will be a Readable stream. If "blob" was provided in the file options 151 | * then the return value will be an Array. 152 | */ 153 | getFileChunk(start, end) { 154 | ensureRequiredOptions(this.fileOptions); 155 | const { 156 | filePath, 157 | blob, 158 | } = this.fileOptions; 159 | 160 | if (filePath) { 161 | return fs.createReadStream(filePath, { start, end }); 162 | } 163 | return blob.slice(start, end); 164 | } 165 | 166 | /** 167 | * Retrieves the headers that should be included with each part upload request. 168 | * @returns {object} Simple object whose names are header names, and whose values 169 | * are header values. 170 | */ 171 | getPartHeaders() { 172 | ensureRequiredOptions(this.fileOptions); 173 | const { partHeaders = {} } = this.fileOptions; 174 | return partHeaders; 175 | } 176 | 177 | /** 178 | * Converts the class instance into a simple object representation. 179 | * 180 | * @returns {object} Simplified view of the class instance. 181 | */ 182 | toJSON() { 183 | const { 184 | fileSize, 185 | filePath, 186 | } = this.fileOptions; 187 | const json = { 188 | fileUrl: this.getFileUrl(), 189 | fileSize, 190 | }; 191 | 192 | if (this.shouldCreateNewVersion()) { 193 | json.createVersion = true; 194 | } 195 | 196 | if (this.getVersionComment()) { 197 | json.versionComment = this.getVersionComment(); 198 | } 199 | 200 | if (this.getVersionLabel()) { 201 | json.versionLabel = this.getVersionLabel(); 202 | } 203 | 204 | if (this.shouldReplace()) { 205 | json.replace = this.shouldReplace(); 206 | } 207 | 208 | if (filePath) { 209 | json.filePath = filePath; 210 | } 211 | if (Object.keys(this.getPartHeaders()).length) { 212 | json.multipartHeaders = this.getPartHeaders(); 213 | } 214 | return json; 215 | } 216 | } 217 | 218 | module.exports = UploadFile; 219 | -------------------------------------------------------------------------------- /src/upload-options-base.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const UploadBase = require('./upload-base'); 14 | 15 | /** 16 | * Common base class for all classes that work with a DirectBinaryUploadOptions 17 | * instance. 18 | */ 19 | class UploadOptionsBase extends UploadBase { 20 | /** 21 | * Constructs a new instance using the provided information. 22 | * 23 | * @param {object} options Options as provided when the direct binary object was instantiated. 24 | * @param {DirectBinaryUploadOptions} uploadOptions Options as provided when the direct binary 25 | * upload process was initiated. 26 | */ 27 | constructor(options, uploadOptions) { 28 | super(options); 29 | this.uploadOptions = uploadOptions; 30 | } 31 | 32 | /** 33 | * Retrieves the upload options that were provided when the upload was initiated. 34 | * 35 | * @returns {DirectBinaryUploadOptions} Upload options. 36 | */ 37 | getUploadOptions() { 38 | return this.uploadOptions; 39 | } 40 | } 41 | 42 | module.exports = UploadOptionsBase; 43 | -------------------------------------------------------------------------------- /src/upload-result.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const HttpResult = require('./http-result'); 14 | const UploadError = require('./upload-error'); 15 | 16 | /** 17 | * Represents results for the upload process as a whole, which might include multiple files. Results 18 | * include information such as total upload time, total file size, total number of files, etc. 19 | */ 20 | class UploadResult extends HttpResult { 21 | /** 22 | * Constructs a new instance of the results, which can be used to add more information. 23 | * 24 | * @param {object} options Options as provided when the upload instance was instantiated. 25 | * @param {DirectBinaryUploadOptions} uploadOptions Options as provided when the upload was 26 | * initiated. 27 | */ 28 | constructor(options, uploadOptions) { 29 | super(options, uploadOptions); 30 | 31 | this.totalTime = 0; 32 | this.fileUploadResults = false; 33 | this.createDirectoryResults = []; 34 | this.errors = []; 35 | } 36 | 37 | /** 38 | * Starts a timer that will be used to calculate the total amount of time it takes for the 39 | * upload process to complete. 40 | */ 41 | startTimer() { 42 | this.start = new Date().getTime(); 43 | } 44 | 45 | /** 46 | * Stops the timer and calculates the amount of time elapsed since startTimer() was called. 47 | */ 48 | stopTimer() { 49 | if (this.start) { 50 | this.totalTime += new Date().getTime() - this.start; 51 | } 52 | } 53 | 54 | /** 55 | * Adds a new create directory to the overall result. Will be used to calculate various overall 56 | * metrics. 57 | * 58 | * @param {import('./create-directory-result')} createDirectoryResult Result whose 59 | * metrics will be included in the overall result. 60 | */ 61 | addCreateDirectoryResult(createDirectoryResult) { 62 | this.createDirectoryResults.push(createDirectoryResult); 63 | } 64 | 65 | /** 66 | * Retrieves all results for directories that were created as part of the upload. 67 | * 68 | * @returns {Array} Directory results. 69 | */ 70 | getCreateDirectoryResults() { 71 | return this.createDirectoryResults; 72 | } 73 | 74 | /** 75 | * Retrieves the amount of time, in milliseconds, that it took to create any directories for the 76 | * upload. 77 | */ 78 | getTotalFolderCreateTime() { 79 | let createTime = 0; 80 | this.getCreateDirectoryResults().forEach((directoryResult) => { 81 | createTime += directoryResult.getCreateTime(); 82 | }); 83 | return createTime; 84 | } 85 | 86 | /** 87 | * Sets information the individual file upload results that will be included in the final 88 | * output. 89 | * @param {import('./file-upload-results')} fileUploadResults File upload information. 90 | */ 91 | setFileUploadResults(fileUploadResults) { 92 | this.fileUploadResults = fileUploadResults; 93 | } 94 | 95 | /** 96 | * Retrieves the total number of files initially included in the overall upload. 97 | * 98 | * @returns {number} Number of files. 99 | */ 100 | getTotalFiles() { 101 | return this.fileUploadResults ? this.fileUploadResults.getTotalFileCount() : 0; 102 | } 103 | 104 | /** 105 | * Retrieves the number of files that uploaded successfully. 106 | * 107 | * @returns {number} Number of files. 108 | */ 109 | getTotalCompletedFiles() { 110 | return this.fileUploadResults ? this.fileUploadResults.getSuccessCount() : 0; 111 | } 112 | 113 | /** 114 | * Retrieves the total amount of time, in milliseconds, that elapsed between calls to 115 | * startTimer() and stopTimer(). 116 | * 117 | * @returns {number} Time span in milliseconds. 118 | */ 119 | getElapsedTime() { 120 | return this.totalTime; 121 | } 122 | 123 | /** 124 | * Sets the total amount of time, in milliseconds, that it took for the upload to complete. 125 | * 126 | * @param {number} totalTime Time span in milliseconds. 127 | */ 128 | setElapsedTime(totalTime) { 129 | this.totalTime = totalTime; 130 | } 131 | 132 | /** 133 | * Retrieves the total size, in bytes, of all files initially provided to the upload process. 134 | * 135 | * @returns {number} Size in bytes. 136 | */ 137 | getTotalSize() { 138 | return this.fileUploadResults ? this.fileUploadResults.getTotalSize() : 0; 139 | } 140 | 141 | /** 142 | * Retrieves the average size, in bytes, of all files initially provided to the upload process. 143 | * 144 | * @returns {number} Size in bytes. 145 | */ 146 | getAverageFileSize() { 147 | return this.fileUploadResults ? this.fileUploadResults.getAverageSize() : 0; 148 | } 149 | 150 | /** 151 | * Retrieves all the individual file upload results contained in the overall result. 152 | * 153 | * @returns {Array} List of file event infos. 154 | */ 155 | getFileUploadResults() { 156 | return this.fileUploadResults ? this.fileUploadResults.toJSON() : []; 157 | } 158 | 159 | /** 160 | * Retrieves all the errors that occurred in the transfer process. 161 | * 162 | * @returns {Array} List of UploadError instances. 163 | */ 164 | getErrors() { 165 | const errors = [...this.getUploadErrors()]; 166 | const fileErrors = this.fileUploadResults ? this.fileUploadResults.getErrors() : []; 167 | return errors.concat(fileErrors); 168 | } 169 | 170 | /** 171 | * Adds a high-level error that prevented the upload from completing. 172 | * 173 | * @param {*} e An error object. 174 | */ 175 | addUploadError(e) { 176 | this.errors.push(UploadError.fromError(e)); 177 | } 178 | 179 | /** 180 | * Retrieves a list of high-level errors that prevented the upload from 181 | * completing. 182 | * 183 | * @returns {Array} An array of error objects. 184 | */ 185 | getUploadErrors() { 186 | return this.errors; 187 | } 188 | 189 | /** 190 | * Converts the result instance into a simple object containing all result data. 191 | * 192 | * @returns {object} Result data in a simple format. 193 | */ 194 | toJSON() { 195 | return { 196 | host: this.getUploadOptions().getUrlPrefix(), 197 | totalFiles: this.getTotalFiles(), 198 | totalTime: this.getElapsedTime(), 199 | totalCompleted: this.getTotalCompletedFiles(), 200 | totalFileSize: this.getTotalSize(), 201 | folderCreateSpent: this.getTotalFolderCreateTime(), 202 | createdFolders: this.getCreateDirectoryResults().map((result) => result.toJSON()), 203 | detailedResult: this.fileUploadResults ? this.fileUploadResults.toJSON() : [], 204 | errors: this.getUploadErrors().map((error) => error.toJSON()), 205 | ...super.toJSON(), 206 | }; 207 | } 208 | } 209 | 210 | module.exports = UploadResult; 211 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const Path = require('path'); 14 | const Async = require('async'); 15 | const AsyncLock = require('async-lock'); 16 | const fs = require('./fs-promise'); 17 | 18 | const { DefaultValues } = require('./constants'); 19 | const UploadError = require('./upload-error'); 20 | const ErrorCodes = require('./error-codes'); 21 | 22 | const lock = new AsyncLock(); 23 | 24 | const TEMP_PATTERNS = [ 25 | /^\/~(.*)/, // catch all paths starting with ~ 26 | /^\/\.(.*)/, // catch all paths starting with . 27 | ]; 28 | 29 | const TEMP_NAME_PATTERNS = [ 30 | /^[.~]/i, 31 | /^TestFile/, // InDesign: on file open, InDesign creates .dat.nosync* file, renames it TestFile, and deletes it 32 | /\.tmp$/i, // Illustrator: on save, creates one or more *.tmp files, renames them to original file name 33 | /\.~tmp$/i, // some default Windows applications use this file format 34 | // Windows 35 | /^desktop\.ini/i, 36 | /^Thumbs\.db/i, 37 | /^Adobe Bridge Cache\.bc$/i, 38 | /^Adobe Bridge Cache\.bct$/i, 39 | ]; 40 | 41 | /** 42 | * Loops through a given array, concurrently invoking the given callback. The loop will have a 43 | * maximum number of pending itemCallbacks at any one time. For example, if there are 100 items 44 | * in the array and 5 itemCallbacks are currently processing, then no more itemCallbacks will be 45 | * invoked until at least one of the pending itemCallbacks completes. 46 | * 47 | * @param {Array} loopArray Array to loop through. 48 | * @param {number} [maxConcurrent] Optionally specify how many concurrent itemCallbacks are allowed. 49 | * Default is 5. 50 | * @param {function} itemCallback Invoked each time an item from the given array is available. Will 51 | * be invoked with two parameters: the item itself and the index of the item in the array. The 52 | * return value of this callback is expected to be a Promise. 53 | * @returns {Promise} Will be resolved when all Promises returned by the callback have been 54 | * resolved. Will be resolved with an Array of all resolve values from the callback's Promises. 55 | */ 56 | function concurrentLoop(loopArray, maxConcurrent, itemCallback) { 57 | let theMaxConcurrent = maxConcurrent; 58 | let theItemCallback = itemCallback; 59 | if (typeof maxConcurrent === 'function') { 60 | theItemCallback = maxConcurrent; 61 | theMaxConcurrent = DefaultValues.MAX_CONCURRENT; 62 | } 63 | 64 | return new Promise((resolve, reject) => { 65 | Async.eachOfLimit( 66 | loopArray, 67 | theMaxConcurrent, 68 | async (loopItem, index) => theItemCallback(loopItem, index), 69 | (err) => { 70 | if (err) { 71 | reject(err); 72 | return; 73 | } 74 | resolve(); 75 | }, 76 | ); 77 | }); 78 | } 79 | 80 | /** 81 | * Calculates the average of all the numbers in an array. 82 | * 83 | * @param {Array} values List of numbers for which to calculate an average. 84 | * @returns {number} The average value, rounded to zero decimal places. 85 | */ 86 | function getAverage(values) { 87 | if (values.length) { 88 | const sum = values.reduce((x, y) => x + y); 89 | return Math.round(sum / values.length); 90 | } 91 | return 0; 92 | } 93 | 94 | function buildCharRegex(charArray) { 95 | let regex = '['; 96 | 97 | charArray.forEach((char) => { 98 | if (char === '\\' || char === ']') { 99 | regex += '\\'; 100 | } 101 | regex += char; 102 | }); 103 | 104 | regex += ']'; 105 | 106 | return regex; 107 | } 108 | 109 | /** 110 | * Removes a given set of characters from the end of a string. 111 | * 112 | * @param {string} toTrim The value to be trimmed. 113 | * @param {Array} charArray An array of single characters to trim. 114 | */ 115 | function trimRight(toTrim, charArray) { 116 | if (toTrim && toTrim.replace) { 117 | return toTrim.replace(new RegExp(`${buildCharRegex(charArray)}*$`, 'g'), ''); 118 | } 119 | return toTrim; 120 | } 121 | 122 | /** 123 | * Removes a given set of characters from the beginning of a string. 124 | * 125 | * @param {string} toTrim The value to be trimmed. 126 | * @param {Array} charArray An array of single characters to trim. 127 | */ 128 | function trimLeft(toTrim, charArray) { 129 | if (toTrim && toTrim.replace) { 130 | return toTrim.replace(new RegExp(`^${buildCharRegex(charArray)}*`, 'g'), ''); 131 | } 132 | return toTrim; 133 | } 134 | 135 | /** 136 | * Joins a list of values together to form a URL path. Each of the given values 137 | * is guaranteed to be separated from all other values by a forward slash. 138 | * 139 | * @param {...string} theArguments Any number of parameters to join. 140 | */ 141 | function joinUrlPath(...theArguments) { 142 | let path = ''; 143 | 144 | theArguments.forEach((arg) => { 145 | const toJoin = trimRight(trimLeft(arg, ['/']), ['/']); 146 | 147 | if (toJoin) { 148 | path += `/${toJoin}`; 149 | } 150 | }); 151 | 152 | return path; 153 | } 154 | 155 | /** 156 | * Removes "/content/dam" from the beginning of a given path, if its 157 | * present. If the path equals "/content/dam" then the method will 158 | * return an empty string. 159 | * 160 | * @param {string} path The path to trim. 161 | */ 162 | function trimContentDam(path) { 163 | if (!path) { 164 | return path; 165 | } 166 | 167 | if (path === '/content/dam') { 168 | return ''; 169 | } 170 | 171 | let trimmed = String(path); 172 | if (trimmed.startsWith('/content/dam/')) { 173 | trimmed = trimmed.substr('/content/dam'.length); 174 | } 175 | 176 | return trimRight(trimmed, ['/']); 177 | } 178 | 179 | /** 180 | * Normalizes a path by ensuring it only contains forward slashes and does not end with a 181 | * slash. If the given path is falsy then the method will return an empty string. 182 | * @param {string} path An item's full path. 183 | * @returns {string} Normalized version of a path. 184 | */ 185 | function normalizePath(path) { 186 | let normPath = path; 187 | if (normPath) { 188 | normPath = normPath.replace(/\\/g, '/'); 189 | if (normPath.charAt(normPath.length - 1) === '/') { 190 | normPath = normPath.substr(0, normPath.length - 1); 191 | } 192 | } 193 | return normPath || ''; 194 | } 195 | 196 | /** 197 | * Determines whether or not a given path is either a temp file, or in a temp directory. 198 | * @param {string} path A file system-like path. 199 | * @returns {boolean} True if the path is a temp file, false otherwise. 200 | */ 201 | function isTempPath(path) { 202 | const tempPath = normalizePath(path); 203 | 204 | if (tempPath === '/') { 205 | return false; 206 | } 207 | 208 | let isTemp = TEMP_PATTERNS.some((pattern) => pattern.test(tempPath)); 209 | 210 | if (!isTemp) { 211 | const pathName = Path.basename(tempPath); 212 | isTemp = TEMP_NAME_PATTERNS.some((pattern) => pattern.test(pathName)); 213 | } 214 | 215 | return isTemp; 216 | } 217 | 218 | /** 219 | * Concurrently loops through all items in a directory, doing a stat 220 | * on each to determine if it's a directory or file. The method will 221 | * skip temp files and directories. 222 | * @param {string} directoryPath Full path to the directory to iterate. 223 | * @param {Array} directories All of the target directory's valid sub-directories 224 | * will be added to this array. 225 | * @param {Array} files All of the target directory's valid sub-files will be 226 | * added to this array. 227 | * @param {Array} errors Any errors encountered while processing the directory will 228 | * be added to this array. 229 | * @returns {number} Total size, in bytes, of all files in the directory. 230 | */ 231 | async function processDirectory(directoryPath, directories, files, errors) { 232 | let contents = false; 233 | 234 | let totalSize = 0; 235 | 236 | try { 237 | contents = await fs.readdir(directoryPath); 238 | } catch (e) { 239 | errors.push(e); 240 | } 241 | 242 | if (contents) { 243 | await concurrentLoop(contents, async (childPath) => { 244 | const fullChildPath = Path.join(directoryPath, childPath); 245 | if (!isTempPath(fullChildPath)) { 246 | let childStat; 247 | try { 248 | childStat = await fs.stat(fullChildPath); 249 | } catch (e) { 250 | errors.push(e); 251 | return; 252 | } 253 | 254 | if (childStat.isDirectory()) { 255 | directories.push({ path: fullChildPath }); 256 | } else if (childStat.isFile()) { 257 | files.push({ path: fullChildPath, size: childStat.size }); 258 | totalSize += childStat.size; 259 | } 260 | } 261 | }); 262 | } 263 | 264 | return totalSize; 265 | } 266 | 267 | /** 268 | * Walks a directory by retrieving all the directories and files 269 | * in the given path, then walking all those sub directories, then 270 | * all sub directories of those sub directories, etc. The end result 271 | * will be the entire tree, including all descendents, for the 272 | * target directory. 273 | * @param {string} directoryPath Directory to traverse. 274 | * @param {number} [maximumPaths] The maximum number of paths to 275 | * process before the method gives up and throws an exception. 276 | * Default value is 5000. 277 | * @param {boolean} [includeDescendents] If true, the method will walk 278 | * descendent directories. If false, the method will only include files 279 | * immediately below the given directory. Default value is true. 280 | */ 281 | async function walkDirectory(directoryPath, maximumPaths = 5000, includeDescendents = true) { 282 | let processDirectories = [{ path: directoryPath }]; 283 | let allDirectories = []; 284 | const allFiles = []; 285 | const allErrors = []; 286 | let walkedTotalSize = 0; 287 | 288 | // this algorithm avoids recursion to prevent overflows. Instead, 289 | // use a stack to keep track of directories to process. 290 | while (processDirectories.length > 0) { 291 | const { path: toProcess } = processDirectories.shift(); 292 | const directories = []; 293 | // eslint-disable-next-line no-await-in-loop 294 | walkedTotalSize += await processDirectory(toProcess, directories, allFiles, allErrors); 295 | allDirectories = allDirectories.concat(directories); 296 | 297 | if (includeDescendents) { 298 | processDirectories = processDirectories.concat(directories); 299 | } 300 | 301 | if (allDirectories.length + allFiles.length > maximumPaths) { 302 | throw new UploadError(`Walked directory exceeded the maximum number of ${maximumPaths} paths`, ErrorCodes.TOO_LARGE); 303 | } 304 | } 305 | 306 | return { 307 | directories: allDirectories, 308 | files: allFiles, 309 | errors: allErrors, 310 | totalSize: walkedTotalSize, 311 | }; 312 | } 313 | 314 | /** 315 | * Creates a "thread"-specific lock on a given ID. Other threads requesting 316 | * a lock on the same ID won't be able to run unless there are no other 317 | * threads holding the lock. Once a lock is obtained, the given callback is 318 | * invoked; the lock will be released when the callback has finished 319 | * executing. The method itself returns a promise, which will resolve once 320 | * the callback has completed. 321 | * @param {string} lockId ID for which an exclusive lock will be obtained. 322 | * @param {function} callback Invoked when the lock has been obtained. The 323 | * lock will be released when the callback has finished executing. The callback 324 | * can return a Promise, and the method will wait until the Promise resolves 325 | * before releasing the lock. 326 | * @returns {Promise} Resolves after a lock has been obtained and the given 327 | * callback has finished executing. 328 | */ 329 | async function getLock(lockId, callback) { 330 | return lock.acquire(lockId, callback); 331 | } 332 | 333 | module.exports = { 334 | concurrentLoop, 335 | getAverage, 336 | trimRight, 337 | trimLeft, 338 | joinUrlPath, 339 | trimContentDam, 340 | normalizePath, 341 | isTempPath, 342 | walkDirectory, 343 | getLock, 344 | }; 345 | -------------------------------------------------------------------------------- /test/create-directory-result.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const { 18 | getTestOptions, 19 | } = require('./testutils'); 20 | const FileSystemUploadOptions = require('../src/filesystem-upload-options'); 21 | const CreateDirectoryResult = require('../src/create-directory-result'); 22 | const UploadError = require('../src/upload-error'); 23 | const ErrorCodes = require('../src/error-codes'); 24 | 25 | describe('Create Directy Result Tests', () => { 26 | it('test result with response', () => { 27 | const response = { 28 | status: 201, 29 | statusText: 'Created', 30 | cloudClient: { 31 | requestTime: 100, 32 | }, 33 | }; 34 | const directoryResult = new CreateDirectoryResult( 35 | getTestOptions(), 36 | new FileSystemUploadOptions(), 37 | '/testing', 38 | 'testing', 39 | ); 40 | directoryResult.setCreateResponse(response); 41 | should(directoryResult.getFolderPath()).be.exactly('/testing'); 42 | should(directoryResult.getFolderTitle()).be.exactly('testing'); 43 | should(directoryResult.getCreateTime()).be.exactly(100); 44 | should(directoryResult.toJSON()).deepEqual({ 45 | elapsedTime: 100, 46 | folderPath: '/testing', 47 | folderTitle: 'testing', 48 | retryErrors: [], 49 | }); 50 | }); 51 | 52 | it('test result without response', () => { 53 | const directoryResult = new CreateDirectoryResult( 54 | getTestOptions(), 55 | new FileSystemUploadOptions(), 56 | '/testing', 57 | 'testing', 58 | ); 59 | should(directoryResult.getFolderPath()).be.exactly('/testing'); 60 | should(directoryResult.getFolderTitle()).be.exactly('testing'); 61 | should(directoryResult.getCreateTime()).be.exactly(0); 62 | should(directoryResult.toJSON()).deepEqual({ 63 | elapsedTime: 0, 64 | folderPath: '/testing', 65 | folderTitle: 'testing', 66 | retryErrors: [], 67 | }); 68 | }); 69 | 70 | it('test result with error', () => { 71 | const directoryResult = new CreateDirectoryResult( 72 | getTestOptions(), 73 | new FileSystemUploadOptions(), 74 | '/testing', 75 | 'testing', 76 | ); 77 | const uploadError = new UploadError('unit test error', ErrorCodes.ALREADY_EXISTS); 78 | directoryResult.setCreateError(uploadError); 79 | 80 | should(directoryResult.getFolderPath()).be.exactly('/testing'); 81 | should(directoryResult.getFolderTitle()).be.exactly('testing'); 82 | should(directoryResult.getCreateTime()).be.exactly(0); 83 | should(directoryResult.toJSON()).deepEqual({ 84 | elapsedTime: 0, 85 | folderPath: '/testing', 86 | folderTitle: 'testing', 87 | retryErrors: [], 88 | error: { 89 | code: ErrorCodes.ALREADY_EXISTS, 90 | message: 'unit test error', 91 | }, 92 | }); 93 | }); 94 | }); 95 | -------------------------------------------------------------------------------- /test/direct-binary-upload-options.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const DirectBinaryUploadOptions = require('../src/direct-binary-upload-options'); 18 | 19 | describe('DirectBinaryUploadOptionsTest', () => { 20 | it('test url slashes', () => { 21 | const options = new DirectBinaryUploadOptions() 22 | .withUrl('/'); 23 | 24 | should(options.getUrl()).be.exactly('/'); 25 | options.withUrl('/trailing/'); 26 | should(options.getUrl()).be.exactly('/trailing'); 27 | }); 28 | 29 | it('test getTargetFolderPath', () => { 30 | const options = new DirectBinaryUploadOptions() 31 | .withUrl('http://somereallyfakeurlhopefully/content/dam/test%20path/asset.jpg'); 32 | should(options.getTargetFolderPath()).be.exactly('/content/dam/test path/asset.jpg'); 33 | }); 34 | 35 | it('test with http options', () => { 36 | const options = new DirectBinaryUploadOptions() 37 | .withHttpOptions({ 38 | headers: { 39 | header1: 'test1', 40 | header2: 'test2', 41 | }, 42 | method: 'PUT', 43 | proxy: 'testproxy', 44 | }) 45 | .withHttpOptions({ 46 | headers: { 47 | header1: 'test1-1', 48 | header3: 'value3', 49 | }, 50 | method: 'POST', 51 | hello: 'world!', 52 | }); 53 | should(options.getHttpOptions()).deepEqual({ 54 | headers: { 55 | header1: 'test1-1', 56 | header2: 'test2', 57 | header3: 'value3', 58 | }, 59 | method: 'POST', 60 | proxy: 'testproxy', 61 | hello: 'world!', 62 | }); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /test/direct-binary-upload-process.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const { Readable } = require('stream'); 16 | const should = require('should'); 17 | 18 | const { 19 | getTestOptions, 20 | addDirectUpload, 21 | resetHttp, 22 | allHttpUsed, 23 | getDirectBinaryUploads, 24 | parseQuery, 25 | } = require('./testutils'); 26 | const MockBlob = require('./mock-blob'); 27 | const UploadResult = require('../src/upload-result'); 28 | const DirectBinaryUploadProcess = require('../src/direct-binary-upload-process'); 29 | const DirectBinaryUploadOptions = require('../src/direct-binary-upload-options'); 30 | 31 | const HOST = 'http://reallyfakeaemuploadhost'; 32 | 33 | describe('DirectBinaryUploadProcessTest', () => { 34 | beforeEach(() => { 35 | resetHttp(); 36 | }); 37 | 38 | afterEach(() => { 39 | should(allHttpUsed()).be.ok(); 40 | resetHttp(); 41 | }); 42 | 43 | describe('upload', () => { 44 | async function runCompleteTest(createVersion, versionLabel, versionComment, replace) { 45 | const targetFolder = `/content/dam/target/folder-create-version-${new Date().getTime()}`; 46 | addDirectUpload(HOST, targetFolder, ['myasset.jpg']); 47 | const fileData = { 48 | fileName: 'myasset.jpg', 49 | fileSize: 512, 50 | blob: new MockBlob(), 51 | }; 52 | 53 | if (createVersion) { 54 | fileData.createVersion = true; 55 | if (versionLabel) { 56 | fileData.versionLabel = versionLabel; 57 | } 58 | if (versionComment) { 59 | fileData.versionComment = versionComment; 60 | } 61 | } 62 | 63 | if (replace) { 64 | fileData.replace = true; 65 | } 66 | 67 | const options = new DirectBinaryUploadOptions() 68 | .withUrl(`${HOST}${targetFolder}`) 69 | .withUploadFiles([fileData]); 70 | 71 | const process = new DirectBinaryUploadProcess(getTestOptions(), options); 72 | 73 | await process.upload(new UploadResult(getTestOptions(), options)); 74 | 75 | // verify that complete request is correct 76 | const { 77 | inits = [], 78 | parts = [], 79 | completes = [], 80 | } = getDirectBinaryUploads(); 81 | should(inits.length).be.ok(); 82 | const init = inits[inits.length - 1]; 83 | should(init.uri).equal(`${targetFolder}.initiateUpload.json`); 84 | should(parseQuery(init.body)).deepEqual({ 85 | fileName: 'myasset.jpg', 86 | fileSize: '512', 87 | }); 88 | should(parts).deepEqual([{ 89 | uri: `${targetFolder}/myasset.jpg`, 90 | body: '0,512,', 91 | }]); 92 | should(completes.length).equal(1); 93 | should(completes[0].uri).equal(`${targetFolder}.completeUpload.json`); 94 | 95 | const completeInfo = parseQuery(completes[0].body); 96 | should(completeInfo.fileName).equal('myasset.jpg'); 97 | should(completeInfo.fileSize).equal('512'); 98 | should(completeInfo.mimeType).equal('image/jpeg'); 99 | should(completeInfo.uploadToken).be.ok(); 100 | should(completeInfo.uploadDuration).be.ok(); 101 | 102 | if (createVersion) { 103 | should(completeInfo.createVersion).be.ok(); 104 | if (versionLabel) { 105 | should(completeInfo.versionLabel).be.exactly(versionLabel); 106 | } else { 107 | should(completeInfo.versionLabel).not.be.ok(); 108 | } 109 | if (versionComment) { 110 | should(completeInfo.versionComment).be.exactly(versionComment); 111 | } else { 112 | should(versionComment).not.be.ok(); 113 | } 114 | } else { 115 | const { createVersion: completeCreateVersion = 'false' } = completeInfo; 116 | should(completeCreateVersion).equal('false'); 117 | should(completeInfo.versionLabel).not.be.ok(); 118 | should(completeInfo.versionComment).not.be.ok(); 119 | } 120 | 121 | const { replace: completeReplace = 'false' } = completeInfo; 122 | if (replace) { 123 | should(completeReplace).equal('true'); 124 | } else { 125 | should(completeReplace).equal('false'); 126 | } 127 | } 128 | 129 | it('create version only test', async () => { 130 | await runCompleteTest(true); 131 | }); 132 | 133 | it('create version with label and comments', async () => { 134 | await runCompleteTest(true, 'label', 'comment'); 135 | }); 136 | 137 | it('replace test', async () => { 138 | await runCompleteTest(false, 'label', 'comment', true); 139 | }); 140 | 141 | it('replace and create version test', async () => { 142 | await runCompleteTest(true, 'label', 'comment', true); 143 | }); 144 | 145 | it('trailing slash', async () => { 146 | const targetFolder = '/target/folder-trailing-slash'; 147 | addDirectUpload(HOST, targetFolder, ['myasset.jpg']); 148 | 149 | const options = new DirectBinaryUploadOptions() 150 | .withUrl(`${HOST}${targetFolder}/`) 151 | .withUploadFiles([{ 152 | fileName: 'myasset.jpg', 153 | fileSize: 512, 154 | blob: new MockBlob(), 155 | }]); 156 | const process = new DirectBinaryUploadProcess(getTestOptions(), options); 157 | await process.upload(new UploadResult(getTestOptions(), options)); 158 | 159 | const { parts = [] } = getDirectBinaryUploads(); 160 | should(parts).deepEqual([{ 161 | uri: `${targetFolder}/myasset.jpg`, 162 | body: '0,512,', 163 | }]); 164 | }); 165 | 166 | it('file upload smoke', async () => { 167 | const fileSize = 1024; 168 | const targetFolder = '/target/file-upload-smoke'; 169 | addDirectUpload(HOST, targetFolder, ['fileuploadsmoke.jpg']); 170 | const options = new DirectBinaryUploadOptions() 171 | .withUrl(`${HOST}${targetFolder}`) 172 | .withUploadFiles([{ 173 | fileName: 'fileuploadsmoke.jpg', 174 | fileSize, 175 | blob: { 176 | slice: () => { 177 | const s = new Readable(); 178 | // eslint-disable-next-line no-underscore-dangle 179 | s._read = () => {}; 180 | let value = ''; 181 | for (let i = 0; i < fileSize / 2; i += 1) { 182 | value += 'a'; 183 | } 184 | s.push(value); 185 | s.push(value); 186 | s.push(null); 187 | 188 | return s; 189 | }, 190 | }, 191 | }]); 192 | const process = new DirectBinaryUploadProcess({ 193 | ...getTestOptions(), 194 | progressDelay: 0, 195 | }, options); 196 | 197 | await process.upload(new UploadResult(getTestOptions(), options)); 198 | 199 | const { 200 | inits = [], 201 | parts = [], 202 | completes = [], 203 | } = getDirectBinaryUploads(); 204 | should(inits.length > 0).be.ok(); 205 | should(inits[inits.length - 1].uri).equal(`${targetFolder}.initiateUpload.json`); 206 | 207 | should(parts.length).equal(1); 208 | should(parts[0].uri).equal(`${targetFolder}/fileuploadsmoke.jpg`); 209 | should(parts[0].body.length).equal(1024); 210 | 211 | should(completes.length).equal(1); 212 | should(completes[0].uri).equal(`${targetFolder}.completeUpload.json`); 213 | should(completes[0].body).be.ok(); 214 | }); 215 | 216 | it('test total upload size', () => { 217 | const options = new DirectBinaryUploadOptions() 218 | .withUploadFiles([{ 219 | fileName: 'fileuploadsmoke.jpg', 220 | fileSize: 1024, 221 | filePath: '/test/file/path.jpg', 222 | }, { 223 | fileName: 'fileuploadsmoke2.jpg', 224 | fileSize: 2048, 225 | filePath: '/test/file/path2.jpg', 226 | }]); 227 | const process = new DirectBinaryUploadProcess(getTestOptions(), options); 228 | should(process.getTotalSize()).be.exactly(3072); 229 | }); 230 | }); 231 | }); 232 | -------------------------------------------------------------------------------- /test/direct-binary-upload.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const { 18 | getTestOptions, 19 | verifyResult, 20 | addDirectUpload, 21 | getDirectBinaryUploads, 22 | allHttpUsed, 23 | resetHttp, 24 | parseQuery, 25 | } = require('./testutils'); 26 | const MockBlob = require('./mock-blob'); 27 | 28 | const DirectBinaryUpload = require('../src/direct-binary-upload'); 29 | const DirectBinaryUploadOptions = require('../src/direct-binary-upload-options'); 30 | 31 | let blob1; let blob2; let 32 | events; 33 | function getTestUploadFiles() { 34 | blob1 = new MockBlob(); 35 | blob2 = new MockBlob(); 36 | return [{ 37 | fileName: 'targetfile.jpg', 38 | fileSize: 1024, 39 | blob: blob1, 40 | }, { 41 | fileName: 'targetfile2.jpg', 42 | fileSize: 1999, 43 | blob: blob2, 44 | }]; 45 | } 46 | 47 | function verifyFile1Event(eventName, eventData, folderName = 'folder') { 48 | const event = eventData.data; 49 | should(eventData.event).be.exactly(eventName); 50 | should(event.fileName).be.exactly('targetfile.jpg'); 51 | should(event.fileSize).be.exactly(1024); 52 | should(event.targetFolder).be.exactly(`/target/${folderName}`); 53 | should(event.targetFile).be.exactly(`/target/${folderName}/targetfile.jpg`); 54 | 55 | if (eventName !== 'filestart') { 56 | should(event.mimeType).be.exactly('image/jpeg'); 57 | } 58 | if (eventName === 'fileprogress') { 59 | should(event.transferred).be.greaterThan(0); 60 | } 61 | if (eventName === 'fileerror') { 62 | should(event.errors.length).be.greaterThan(0); 63 | } 64 | } 65 | 66 | function verifyFile2Event(eventName, eventData, folderName = 'folder') { 67 | const event = eventData.data; 68 | should(eventData.event).be.exactly(eventName); 69 | should(event.fileName).be.exactly('targetfile2.jpg'); 70 | should(event.fileSize).be.exactly(1999); 71 | should(event.targetFolder).be.exactly(`/target/${folderName}`); 72 | should(event.targetFile).be.exactly(`/target/${folderName}/targetfile2.jpg`); 73 | 74 | if (eventName !== 'filestart') { 75 | should(event.mimeType).be.exactly('image/jpeg'); 76 | } 77 | if (eventName === 'fileprogress') { 78 | should(event.transferred).be.greaterThan(0); 79 | } 80 | if (eventName === 'fileerror') { 81 | should(event.errors.length).be.greaterThan(0); 82 | } 83 | } 84 | 85 | function monitorEvents(upload) { 86 | upload.on('fileuploadstart', (data) => { 87 | events.push({ event: 'fileuploadstart', data }); 88 | }); 89 | upload.on('fileuploadend', (data) => { 90 | events.push({ event: 'fileuploadend', data }); 91 | }); 92 | upload.on('filestart', (data) => { 93 | events.push({ event: 'filestart', data }); 94 | }); 95 | upload.on('fileend', (data) => { 96 | events.push({ event: 'fileend', data }); 97 | }); 98 | upload.on('fileprogress', (data) => { 99 | events.push({ event: 'fileprogress', data }); 100 | }); 101 | upload.on('fileerror', (data) => { 102 | events.push({ event: 'fileerror', data }); 103 | }); 104 | upload.on('filecancelled', (data) => { 105 | events.push({ event: 'filecancelled', data }); 106 | }); 107 | } 108 | 109 | const HOST = 'http://reallyfakehostforaemupload'; 110 | 111 | describe('DirectBinaryUploadTest', () => { 112 | beforeEach(() => { 113 | resetHttp(); 114 | events = []; 115 | }); 116 | 117 | afterEach(() => { 118 | should(allHttpUsed()).be.ok(); 119 | resetHttp(); 120 | }); 121 | 122 | describe('uploadFiles', () => { 123 | it('direct upload smoke test', async () => { 124 | addDirectUpload(HOST, '/target/folder', getTestUploadFiles().map((file) => file.fileName)); 125 | const options = new DirectBinaryUploadOptions() 126 | .withUrl(`${HOST}/target/folder`) 127 | .withUploadFiles(getTestUploadFiles()) 128 | .withConcurrent(false); 129 | 130 | const upload = new DirectBinaryUpload(getTestOptions()); 131 | monitorEvents(upload); 132 | 133 | const result = await upload.uploadFiles(options); 134 | should(result).be.ok(); 135 | 136 | const { 137 | inits = [], 138 | parts = [], 139 | completes = [], 140 | } = getDirectBinaryUploads(); 141 | should(inits.length > 0).be.ok(); 142 | should(inits[inits.length - 1].uri).equal('/target/folder.initiateUpload.json'); 143 | 144 | should(parts).deepEqual([{ 145 | uri: '/target/folder/targetfile.jpg', 146 | body: '0,1024,', 147 | }, { 148 | uri: '/target/folder/targetfile2.jpg', 149 | body: '0,1999,', 150 | }]); 151 | 152 | should(completes.length).equal(2); 153 | should(completes[0].uri).equal('/target/folder.completeUpload.json'); 154 | should(completes[1].uri).equal('/target/folder.completeUpload.json'); 155 | 156 | const complete1 = parseQuery(completes[0].body); 157 | const complete2 = parseQuery(completes[1].body); 158 | should(complete1).deepEqual({ 159 | createVersion: 'false', 160 | fileName: 'targetfile.jpg', 161 | fileSize: '1024', 162 | mimeType: 'image/jpeg', 163 | replace: 'false', 164 | uploadDuration: complete1.uploadDuration, 165 | uploadToken: complete1.uploadToken, 166 | }); 167 | should(complete2).deepEqual({ 168 | createVersion: 'false', 169 | fileName: 'targetfile2.jpg', 170 | fileSize: '1999', 171 | mimeType: 'image/jpeg', 172 | replace: 'false', 173 | uploadDuration: complete2.uploadDuration, 174 | uploadToken: complete2.uploadToken, 175 | }); 176 | should(complete1.uploadDuration).be.ok(); 177 | should(complete1.uploadToken).be.ok(); 178 | should(complete2.uploadDuration).be.ok(); 179 | should(complete2.uploadToken).be.ok(); 180 | 181 | // verify return value 182 | verifyResult(result, { 183 | host: HOST, 184 | totalFiles: 2, 185 | totalTime: result.totalTime, 186 | totalCompleted: 2, 187 | totalFileSize: 3023, 188 | folderCreateSpent: 0, 189 | createdFolders: [], 190 | detailedResult: [{ 191 | fileUrl: `${HOST}/target/folder/targetfile.jpg`, 192 | fileSize: 1024, 193 | blob: '', 194 | result: { 195 | fileName: 'targetfile.jpg', 196 | fileSize: 1024, 197 | targetFolder: '/target/folder', 198 | targetFile: '/target/folder/targetfile.jpg', 199 | mimeType: 'image/jpeg', 200 | sourceFile: '', 201 | sourceFolder: '.', 202 | }, 203 | }, { 204 | fileUrl: `${HOST}/target/folder/targetfile2.jpg`, 205 | fileSize: 1999, 206 | blob: '', 207 | result: { 208 | fileName: 'targetfile2.jpg', 209 | fileSize: 1999, 210 | targetFolder: '/target/folder', 211 | targetFile: '/target/folder/targetfile2.jpg', 212 | mimeType: 'image/jpeg', 213 | sourceFile: '', 214 | sourceFolder: '.', 215 | }, 216 | }], 217 | errors: [], 218 | retryErrors: [], 219 | }); 220 | 221 | // verify that events are correct 222 | should(events.length).be.exactly(8); 223 | should(events[0].event).be.exactly('fileuploadstart'); 224 | verifyFile1Event('filestart', events[1]); 225 | verifyFile2Event('filestart', events[2]); 226 | verifyFile1Event('fileprogress', events[3]); 227 | verifyFile2Event('fileprogress', events[4]); 228 | verifyFile1Event('fileend', events[5]); 229 | verifyFile2Event('fileend', events[6]); 230 | should(events[7].event).be.exactly('fileuploadend'); 231 | }); 232 | 233 | it('progress events', async () => { 234 | const targetFolder = '/target/progress_events'; 235 | addDirectUpload(HOST, targetFolder, getTestUploadFiles().map((file) => file.fileName)); 236 | 237 | const options = new DirectBinaryUploadOptions() 238 | .withUrl(`${HOST}${targetFolder}`) 239 | .withUploadFiles(getTestUploadFiles()) 240 | .withConcurrent(false); 241 | 242 | const upload = new DirectBinaryUpload({ 243 | ...getTestOptions(), 244 | progressDelay: 0, 245 | }); 246 | monitorEvents(upload); 247 | 248 | await upload.uploadFiles(options); 249 | 250 | should(events.length).be.exactly(8); 251 | 252 | should(events[0].event).be.exactly('fileuploadstart'); 253 | should(events[0].data.fileCount).be.exactly(2); 254 | should(events[0].data.totalSize).be.exactly(3023); 255 | should(events[1].event).be.exactly('filestart'); 256 | should(events[1].data.fileName).be.exactly('targetfile.jpg'); 257 | should(events[2].event).be.exactly('filestart'); 258 | should(events[2].data.fileName).be.exactly('targetfile2.jpg'); 259 | should(events[3].event).be.exactly('fileprogress'); 260 | should(events[3].data.fileName).be.exactly('targetfile.jpg'); 261 | should(events[3].data.transferred).be.exactly(1024); 262 | should(events[4].event).be.exactly('fileprogress'); 263 | should(events[4].data.fileName).be.exactly('targetfile2.jpg'); 264 | should(events[4].data.transferred).be.exactly(1999); 265 | should(events[5].event).be.exactly('fileend'); 266 | should(events[5].data.fileName).be.exactly('targetfile.jpg'); 267 | should(events[6].event).be.exactly('fileend'); 268 | should(events[6].data.fileName).be.exactly('targetfile2.jpg'); 269 | should(events[7].event).be.exactly('fileuploadend'); 270 | should(events[7].data.fileCount).be.exactly(2); 271 | should(events[7].data.totalSize).be.exactly(3023); 272 | should(events[7].data.result).be.ok(); 273 | }); 274 | }); 275 | }); 276 | -------------------------------------------------------------------------------- /test/exports.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | const Exports = require('../src/exports'); 17 | const { DirectBinaryUploadOptions } = require('../src/exports'); 18 | 19 | describe('Exports Tests', () => { 20 | it('test exports smoke test', () => { 21 | should(Exports).be.ok(); 22 | should(Exports.DirectBinaryUploadOptions).be.ok(); 23 | should(DirectBinaryUploadOptions).be.ok(); 24 | 25 | should(new Exports.DirectBinaryUploadOptions()).be.ok(); 26 | should(new DirectBinaryUploadOptions()).be.ok(); 27 | }); 28 | }); 29 | -------------------------------------------------------------------------------- /test/filesystem-upload-directory.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const FileSystemUploadDirectory = require('../src/filesystem-upload-directory'); 18 | const DirectBinaryUploadOptions = require('../src/direct-binary-upload-options'); 19 | 20 | describe('FileSystemUploadDirectory Tests', () => { 21 | it('test get remote path', () => { 22 | const options = new DirectBinaryUploadOptions() 23 | .withUrl('http://somereallyfakeunittesturl/content/dam/test%20path'); 24 | const directory = new FileSystemUploadDirectory(options, '/local/directory', 'remote-name'); 25 | should(directory.getRemotePath()).be.exactly('/content/dam/test path/remote-name'); 26 | 27 | const child = new FileSystemUploadDirectory(options, '/local/directory/child', 'child-name', directory); 28 | should(child.getRemotePath()).be.exactly('/content/dam/test path/remote-name/child-name'); 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /test/filesystem-upload-item-manager.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const FileSystemUploadOptions = require('../src/filesystem-upload-options'); 18 | const FileSystemUploadItemManager = require('../src/filesystem-upload-item-manager'); 19 | 20 | describe('FileSystemUploadItemManager Tests', () => { 21 | let options; 22 | 23 | beforeEach(() => { 24 | options = new FileSystemUploadOptions() 25 | .withUrl('http://someunittestfakeurl/content/dam/target'); 26 | }); 27 | 28 | it('test get directory', async () => { 29 | const manager = new FileSystemUploadItemManager(options, '\\fake\\test\\directory\\'); 30 | should(manager.hasDirectory('/fake/test/directory/')).not.be.ok(); 31 | should(manager.hasDirectory('/fake/test/directory/child')).not.be.ok(); 32 | should(manager.hasDirectory('/fake/test')).not.be.ok(); 33 | 34 | const subChild = await manager.getDirectory('/fake/test/directory/Child Dir/Sub Child/'); 35 | should(subChild).be.ok(); 36 | should(subChild.getLocalPath()).be.exactly('/fake/test/directory/Child Dir/Sub Child'); 37 | should(subChild.getRemotePath()).be.exactly('/content/dam/target/directory/child-dir/sub-child'); 38 | should(subChild.getName()).be.exactly('Sub Child'); 39 | should(manager.hasDirectory('/fake/test/directory/')).be.ok(); 40 | should(manager.hasDirectory('/fake/test/directory/Child Dir')).be.ok(); 41 | should(manager.hasDirectory('/fake/test/directory/Child Dir/Sub Child')).be.ok(); 42 | should(manager.hasDirectory('/fake/test')).not.be.ok(); 43 | 44 | const child = await manager.getDirectory('/fake/test/directory/Child Dir'); 45 | should(child).be.ok(); 46 | should(child.getLocalPath()).be.exactly('/fake/test/directory/Child Dir'); 47 | should(child.getRemotePath()).be.exactly('/content/dam/target/directory/child-dir'); 48 | should(child.getName()).be.exactly('Child Dir'); 49 | 50 | should(manager.getDirectory('/fake/test')).be.rejected(); 51 | }); 52 | 53 | it('test get asset', async () => { 54 | const folderPath = '/fake/asset/directory'; 55 | const assetPath = `${folderPath}/Asset #1.jpg`; 56 | const manager = new FileSystemUploadItemManager(options, '/fake/asset/directory'); 57 | should(manager.hasAsset(assetPath)).not.be.ok(); 58 | 59 | const asset = await manager.getAsset(assetPath, 1024); 60 | should(asset).be.ok(); 61 | should(asset.getLocalPath()).be.exactly(assetPath); 62 | should(asset.getRemotePath()).be.exactly('/content/dam/target/directory/Asset -1.jpg'); 63 | should(asset.getSize()).be.exactly(1024); 64 | should(asset.getParentRemoteUrl()).be.exactly('http://someunittestfakeurl/content/dam/target/directory'); 65 | should(manager.hasAsset(assetPath)).be.ok(); 66 | should(manager.hasDirectory(folderPath)).be.ok(); 67 | }); 68 | 69 | it('test get root asset', async () => { 70 | const assetPath = '/fake/asset/directory/Asset #1.jpg'; 71 | const manager = new FileSystemUploadItemManager(options, assetPath); 72 | should(manager.hasAsset(assetPath)).not.be.ok(); 73 | 74 | const asset = await manager.getAsset(assetPath, 1024); 75 | should(asset).be.ok(); 76 | should(asset.getLocalPath()).be.exactly(assetPath); 77 | should(asset.getRemotePath()).be.exactly('/content/dam/target/Asset -1.jpg'); 78 | should(asset.getSize()).be.exactly(1024); 79 | should(asset.getParentRemoteUrl()).be.exactly('http://someunittestfakeurl/content/dam/target'); 80 | should(manager.hasAsset(assetPath)).be.ok(); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /test/filesystem-upload-options.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const FileSystemUploadOptions = require('../src/filesystem-upload-options'); 18 | 19 | describe('FileSystemUploadOptions Tests', () => { 20 | let options; 21 | beforeEach(() => { 22 | options = new FileSystemUploadOptions(); 23 | }); 24 | 25 | it('test accessors', () => { 26 | should(options.getUploadFileOptions()).be.ok(); 27 | const newOptions = options.withUploadFileOptions({ hello: 'world!' }); 28 | should(newOptions).be.ok(); 29 | should(options.getUploadFileOptions().hello).be.exactly('world!'); 30 | }); 31 | 32 | it('test from options', async () => { 33 | let fileOptions = new FileSystemUploadOptions() 34 | .withMaxUploadFiles(20) 35 | .withDeepUpload(true) 36 | .withFolderNodeNameProcessor(async (name) => name) 37 | .withAssetNodeNameProcessor(async (name) => name) 38 | .withInvalidCharacterReplaceValue('_') 39 | .withUploadFileOptions({ hello: 'world' }); 40 | let copiedOptions = FileSystemUploadOptions.fromOptions(fileOptions); 41 | should(copiedOptions).be.ok(); 42 | should(copiedOptions.getInvalidCharacterReplaceValue()).be.exactly('_'); 43 | should(copiedOptions.getMaxUploadFiles()).be.exactly(20); 44 | should(copiedOptions.getDeepUpload()).be.ok(); 45 | should(copiedOptions.getUploadFileOptions().hello).be.exactly('world'); 46 | should(await copiedOptions.getFolderNodeNameProcessor()('folder name')).be.exactly('folder name'); 47 | should(await copiedOptions.getAssetNodeNameProcessor()('asset#name')).be.exactly('asset#name'); 48 | 49 | fileOptions = new FileSystemUploadOptions() 50 | .withFolderNodeNameProcessor('invalid') 51 | .withAssetNodeNameProcessor('invalid') 52 | .withInvalidCharacterReplaceValue(() => {}); 53 | copiedOptions = FileSystemUploadOptions.fromOptions(fileOptions); 54 | should(copiedOptions).be.ok(); 55 | should(copiedOptions.getInvalidCharacterReplaceValue()).be.exactly('-'); 56 | should(await copiedOptions.getFolderNodeNameProcessor()('folder name')).be.exactly('folder-name'); 57 | should(await copiedOptions.getAssetNodeNameProcessor()('asset#name')).be.exactly('asset-name'); 58 | }); 59 | 60 | it('test folder node name processor', async () => { 61 | should(await options.getFolderNodeNameProcessor()('A#b')).be.exactly('a-b'); 62 | should(await options.getFolderNodeNameProcessor()('###')).be.exactly('---'); 63 | options.withInvalidCharacterReplaceValue('_'); 64 | should(await options.getFolderNodeNameProcessor()('A#b')).be.exactly('a_b'); 65 | 66 | options.withFolderNodeNameProcessor(async (folderName) => folderName.replace('A', 'B')); 67 | 68 | should(await options.getFolderNodeNameProcessor()('A#b')).be.exactly('B#b'); 69 | }); 70 | 71 | it('test asset node name processor', async () => { 72 | should(await options.getAssetNodeNameProcessor()('A#b')).be.exactly('A-b'); 73 | options.withInvalidCharacterReplaceValue('_'); 74 | should(await options.getAssetNodeNameProcessor()('A#b')).be.exactly('A_b'); 75 | 76 | options.withAssetNodeNameProcessor(async (assetName) => assetName.replace('A', 'B')); 77 | 78 | should(await options.getAssetNodeNameProcessor()('A#b')).be.exactly('B#b'); 79 | }); 80 | 81 | it('test invalid replace character', () => { 82 | should.throws(() => { 83 | options.withInvalidCharacterReplaceValue(':'); 84 | }); 85 | }); 86 | }); 87 | -------------------------------------------------------------------------------- /test/filesystem-upload-utils.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const { cleanFolderName, cleanAssetName } = require('../src/filesystem-upload-utils'); 18 | const FileSystemUploadOptions = require('../src/filesystem-upload-options'); 19 | 20 | describe('FileSystemUploadUtils Tests', () => { 21 | let options; 22 | beforeEach(() => { 23 | options = new FileSystemUploadOptions(); 24 | }); 25 | 26 | it('test clean folder name', async () => { 27 | should(await cleanFolderName(options, 'A b:c.d')).be.exactly('a-b-c-d'); 28 | options.withFolderNodeNameProcessor(async (folderName) => folderName) 29 | .withInvalidCharacterReplaceValue('_'); 30 | should(await cleanFolderName(options, 'A b:c')).be.exactly('A b_c'); 31 | }); 32 | 33 | it('test clean asset name', async () => { 34 | should(await cleanAssetName(options, 'A #b:c.d.jpg')).be.exactly('A -b-c.d.jpg'); 35 | options.withAssetNodeNameProcessor(async (assetName) => assetName) 36 | .withInvalidCharacterReplaceValue('_'); 37 | should(await cleanAssetName(options, 'A #b:c')).be.exactly('A #b_c'); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /test/http-utils.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const { getTestOptions } = require('./testutils'); 18 | 19 | const DirectBinaryUploadOptions = require('../src/direct-binary-upload-options'); 20 | 21 | const { 22 | getHttpTransferOptions, 23 | } = require('../src/http-utils'); 24 | 25 | describe('HttpUtilsTest', () => { 26 | it('test get http transfer options', () => { 27 | const uploadOptions = new DirectBinaryUploadOptions() 28 | .withUrl('http://localhost/content/dam'); 29 | let httpTransfer = getHttpTransferOptions(getTestOptions(), uploadOptions); 30 | should(httpTransfer).deepEqual({ 31 | requestOptions: { 32 | retryOptions: { 33 | retryAllErrors: false, 34 | retryInitialDelay: 5000, 35 | retryMaxCount: 3, 36 | }, 37 | }, 38 | timeout: 60000, 39 | concurrent: true, 40 | maxConcurrent: 5, 41 | uploadFiles: [], 42 | headers: {}, 43 | }); 44 | 45 | uploadOptions.withConcurrent(false) 46 | .withHttpOptions({ 47 | headers: { 48 | hello: 'world!', 49 | }, 50 | method: 'DELETE', 51 | cloudClient: { 52 | eventuallyConsistentCreate: true, 53 | }, 54 | }) 55 | .withHttpRequestTimeout(30000) 56 | .withHttpRetryCount(2) 57 | .withHttpRetryDelay(500) 58 | .withUploadFiles([{ 59 | fileSize: 1024, 60 | fileName: 'file.jpg', 61 | filePath: '/my/test/file.jpg', 62 | createVersion: true, 63 | versionComment: 'My Comment', 64 | versionLabel: 'Version Label', 65 | replace: true, 66 | partHeaders: { 67 | part: 'header', 68 | }, 69 | }, { 70 | fileSize: 2048, 71 | fileName: 'blob-file.jpg', 72 | blob: [1, 2, 3], 73 | }]); 74 | 75 | // test proxying to http endpoint - requests made with node-httptransfer 76 | httpTransfer = getHttpTransferOptions(getTestOptions(), uploadOptions); 77 | should(httpTransfer).deepEqual({ 78 | headers: { 79 | hello: 'world!', 80 | }, 81 | requestOptions: { 82 | method: 'DELETE', 83 | retryOptions: { 84 | retryAllErrors: true, 85 | retryInitialDelay: 500, 86 | retryMaxCount: 2, 87 | }, 88 | }, 89 | concurrent: false, 90 | maxConcurrent: 1, 91 | timeout: 30000, 92 | uploadFiles: [{ 93 | createVersion: true, 94 | filePath: '/my/test/file.jpg', 95 | fileSize: 1024, 96 | fileUrl: 'http://localhost/content/dam/file.jpg', 97 | multipartHeaders: { 98 | part: 'header', 99 | }, 100 | replace: true, 101 | versionComment: 'My Comment', 102 | versionLabel: 'Version Label', 103 | }, { 104 | blob: [1, 2, 3], 105 | fileSize: 2048, 106 | fileUrl: 'http://localhost/content/dam/blob-file.jpg', 107 | }], 108 | }); 109 | }); 110 | }); 111 | -------------------------------------------------------------------------------- /test/mock-aem-upload.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const { EventEmitter } = require('events'); 14 | const Path = require('path'); 15 | const mime = require('mime'); 16 | 17 | let directUploads = []; 18 | 19 | class MockAemUpload extends EventEmitter { 20 | static reset() { 21 | directUploads = []; 22 | } 23 | 24 | static getDirectUploads() { 25 | return directUploads; 26 | } 27 | 28 | uploadFiles(options) { 29 | directUploads.push(options); 30 | 31 | options.uploadFiles.forEach((uploadFile) => { 32 | const { fileUrl, fileSize } = uploadFile; 33 | const parsedUrl = new URL(fileUrl); 34 | const eventData = { 35 | fileName: decodeURIComponent(Path.basename(parsedUrl.pathname)), 36 | fileSize, 37 | targetFolder: `${decodeURI(Path.dirname(parsedUrl.pathname).replaceAll('\\', '/'))}`, 38 | targetFile: decodeURI(parsedUrl.pathname), 39 | mimeType: mime.getType(fileUrl), 40 | }; 41 | this.emit('filestart', eventData); 42 | this.emit('fileprogress', { 43 | ...eventData, 44 | transferred: 512, 45 | }); 46 | this.emit('fileend', eventData); 47 | }); 48 | 49 | return new Promise((resolve) => { 50 | setTimeout(resolve, 5); 51 | }); 52 | } 53 | } 54 | 55 | module.exports = MockAemUpload; 56 | -------------------------------------------------------------------------------- /test/mock-blob.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const { EventEmitter } = require('events'); 14 | const { Readable } = require('stream'); 15 | 16 | class MockBlob extends EventEmitter { 17 | constructor() { 18 | super(); 19 | this.slices = []; 20 | } 21 | 22 | getSlices() { 23 | return this.slices; 24 | } 25 | 26 | slice(start, end) { 27 | const data = `${start},${end},`; 28 | let called = false; 29 | 30 | this.slices.push({ start, end }); 31 | 32 | const slice = new Readable({ 33 | read() { 34 | if (!called) { 35 | this.push(data); 36 | called = true; 37 | } else { 38 | this.push(null); 39 | } 40 | }, 41 | }); 42 | 43 | slice.mockData = data; 44 | 45 | return slice; 46 | } 47 | } 48 | module.exports = MockBlob; 49 | -------------------------------------------------------------------------------- /test/mock-httptransfer-adapter.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const MockAemUpload = require('./mock-aem-upload'); 14 | 15 | class MockHttpTransferAdapter { 16 | constructor(httpTransfer) { 17 | this.httpTransferInstance = httpTransfer; 18 | this.origAemUpload = httpTransfer.AEMUpload; 19 | // eslint-disable-next-line no-param-reassign 20 | httpTransfer.AEMUpload = MockAemUpload; 21 | this.directUploads = {}; 22 | this.directUploadInvokes = {}; 23 | } 24 | 25 | restore() { 26 | this.httpTransferInstance.AEMUpload = this.origAemUpload; 27 | } 28 | 29 | // eslint-disable-next-line class-methods-use-this 30 | getDirectUploads() { 31 | return MockAemUpload.getDirectUploads(); 32 | } 33 | 34 | // eslint-disable-next-line class-methods-use-this 35 | reset() { 36 | MockAemUpload.reset(); 37 | } 38 | } 39 | 40 | module.exports = MockHttpTransferAdapter; 41 | -------------------------------------------------------------------------------- /test/testutils.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | const nock = require('nock'); 14 | const should = require('should'); 15 | const mime = require('mime'); 16 | 17 | /** 18 | * @typedef {DirectBinaryUploadInfo} 19 | * @property {Array} inits The init calls that were made by the test utils. 20 | * @property {Array} parts The part calls that were made by the test utils. 21 | * @property {Array} completes The complete calls that were made by the test utils. 22 | */ 23 | 24 | /** 25 | * @type {DirectBinaryUploadInfo} 26 | */ 27 | let uploadInfo; 28 | let folderInfo; 29 | let firstCheck = true; 30 | 31 | function initializeUploadInfo() { 32 | firstCheck = true; 33 | uploadInfo = { 34 | inits: [], 35 | parts: [], 36 | completes: [], 37 | }; 38 | folderInfo = []; 39 | } 40 | 41 | initializeUploadInfo(); 42 | 43 | function getConsoleLogger() { 44 | return { 45 | // eslint-disable-next-line no-console 46 | info: console.log, 47 | // eslint-disable-next-line no-console 48 | debug: console.log, 49 | // eslint-disable-next-line no-console 50 | warn: console.log, 51 | // eslint-disable-next-line no-console 52 | error: console.log, 53 | }; 54 | } 55 | 56 | /** 57 | * Retrieves a test logger that will print messages to console.log. 58 | */ 59 | module.exports.getConsoleLogger = getConsoleLogger; 60 | 61 | /** 62 | * Retrieves high level direct binary options that provide the ability 63 | * to output logs based on the state of the AEM_UPLOAD_TEST_LOGGING 64 | * environment variable. 65 | * @returns {object} Options for a direct binary operation. 66 | */ 67 | module.exports.getTestOptions = (addlOptions = {}) => { 68 | if (process.env.AEM_UPLOAD_TEST_LOGGING) { 69 | return { 70 | ...addlOptions, 71 | log: getConsoleLogger(), 72 | }; 73 | } 74 | return addlOptions; 75 | }; 76 | 77 | // stores events for monitorEvents(). 78 | let events = []; 79 | 80 | /** 81 | * Monitors all file-related events for the given upload process. This includes 82 | * "filestart", "fileprogress", "fileend", "fileerror", and "filecancelled". 83 | * @param {EventEmitter} toMonitor Emitter to monitor. 84 | */ 85 | module.exports.monitorEvents = (toMonitor) => { 86 | events = []; 87 | toMonitor.on('filestart', (data) => events.push({ event: 'filestart', data })); 88 | toMonitor.on('fileprogress', (data) => events.push({ event: 'fileprogress', data })); 89 | toMonitor.on('fileend', (data) => events.push({ event: 'fileend', data })); 90 | toMonitor.on('fileerror', (data) => events.push({ event: 'fileerror', data })); 91 | toMonitor.on('filecancelled', (data) => events.push({ event: 'filecancelled', data })); 92 | toMonitor.on('foldercreated', (data) => events.push({ event: 'foldercreated', data })); 93 | }; 94 | 95 | /** 96 | * Determines if an event with a matching "targetFile" value was emitted since the 97 | * last invocation of monitorEvents(). If found, the event's other data values will 98 | * be validated. 99 | * @param {string} eventName Name of the expected event. 100 | * @param {string} targetFile Value of the "targetFile" event data property for the 101 | * expected event. 102 | * @returns {object|boolean} The event's data if found, otherwise false. 103 | */ 104 | module.exports.getEvent = (eventName, targetFile) => { 105 | for (let i = 0; i < events.length; i += 1) { 106 | const { event, data } = events[i]; 107 | 108 | if (event === eventName && data.targetFile === targetFile) { 109 | const { fileName, targetFolder } = data; 110 | const lastSlash = targetFile.lastIndexOf('/'); 111 | const expectedFileName = targetFile.substr(lastSlash + 1); 112 | const expectedTargetFolder = targetFile.substr(0, lastSlash); 113 | should(fileName).be.exactly(expectedFileName); 114 | should(targetFolder).be.exactly(expectedTargetFolder); 115 | 116 | return data; 117 | } 118 | } 119 | return false; 120 | }; 121 | 122 | /** 123 | * Determines if an event with a matching "targetFolder" value was emitted since the 124 | * last invocation of monitorEvents(). If found, the event's other data values will 125 | * be validated. 126 | * @param {string} eventName Name of the expected event. 127 | * @param {string} targetFolder Value of the "targetFolder" event data property for the 128 | * expected event. 129 | * @returns {object|boolean} The event's data if found, otherwise false. 130 | */ 131 | module.exports.getFolderEvent = (eventName, targetFolder) => { 132 | for (let i = 0; i < events.length; i += 1) { 133 | const { event, data } = events[i]; 134 | 135 | if (event === eventName && data.targetFolder === targetFolder) { 136 | const { folderName, targetParent } = data; 137 | const lastSlash = targetFolder.lastIndexOf('/'); 138 | const expectedFolderName = targetFolder.substr(lastSlash + 1); 139 | const expectedTargetParent = targetFolder.substr(0, lastSlash); 140 | should(folderName).be.exactly(expectedFolderName); 141 | should(targetParent).be.exactly(expectedTargetParent); 142 | 143 | return data; 144 | } 145 | } 146 | return false; 147 | }; 148 | 149 | function buildLookup(data, keyField) { 150 | const lookup = {}; 151 | data.forEach((item) => { 152 | lookup[item[keyField]] = item; 153 | }); 154 | return lookup; 155 | } 156 | 157 | /** 158 | * Does the work of verifying the full result output of an upload. Strictly compares all values 159 | * in the result, while ensuring that the order in which folders or files were created will 160 | * not fail the comparison. 161 | * @param {*} result Upload result as provided by the module. 162 | * @param {*} expected Full expected output of the result. 163 | */ 164 | module.exports.verifyResult = (result, expected) => { 165 | const toVerify = { ...result }; 166 | const toCompare = { ...expected }; 167 | 168 | // this is special logic to ensure that order doesn't matter when comparing folder 169 | // and file data. the arrays will be converted into simple objects using a key 170 | // from each item in the array, then the lookups will be compared instead of 171 | // the arrays. 172 | const compareDirLookup = buildLookup(toCompare.createdFolders || [], 'folderPath'); 173 | const compareFileLookup = buildLookup(toCompare.detailedResult || [], 'fileUrl'); 174 | const verifyDirLookup = buildLookup(toVerify.createdFolders || [], 'folderPath'); 175 | const verifyFileLookup = buildLookup(toVerify.detailedResult || [], 'fileUrl'); 176 | delete toCompare.createdFolders; 177 | delete toCompare.detailedResult; 178 | delete toVerify.createdFolders; 179 | delete toVerify.detailedResult; 180 | 181 | should(toVerify).deepEqual(toCompare); 182 | should(toVerify.totalTime !== undefined).be.ok(); 183 | should(toVerify.folderCreateSpent !== undefined).be.ok(); 184 | should(verifyDirLookup).deepEqual(compareDirLookup); 185 | should(verifyFileLookup).deepEqual(compareFileLookup); 186 | 187 | const { createdFolders = [] } = toVerify; 188 | createdFolders.forEach((folder) => should(folder.elapsedTime !== undefined).be.ok()); 189 | }; 190 | 191 | /** 192 | * Clears all HTTP mocks created by the test utils. 193 | */ 194 | module.exports.resetHttp = () => { 195 | initializeUploadInfo(); 196 | nock.cleanAll(); 197 | }; 198 | 199 | /** 200 | * Retrieves a value indicating whether all of the HTTP mocks created by the test 201 | * utils have been used. 202 | * @returns {boolean} True if all mocks have been used, false otherwise. 203 | */ 204 | module.exports.allHttpUsed = () => nock.isDone(); 205 | 206 | /** 207 | * Retrieves information about all of the mocked direct binary uploads that were 208 | * performed through the test utils. 209 | * @returns {DirectBinaryUploadInfo} Upload information. 210 | */ 211 | module.exports.getDirectBinaryUploads = () => uploadInfo; 212 | 213 | /** 214 | * Retrieves information about all of the mocked folder creates that were 215 | * performed through the test utils. 216 | * @returns {Array} Upload information. 217 | */ 218 | module.exports.getFolderCreates = () => folderInfo; 219 | 220 | /** 221 | * Creates mock HTTP requests necessary for successfully uploading using AEM's direct 222 | * binary upload process. 223 | * 224 | * @param {string} host Host on which the direct upload will be registered. 225 | * @param {string} targetFolder Full AEM folder path to which upload will be registered. 226 | * @param {Array} fileNames Names of files to be uploaded. 227 | * @param {object} requests Simple object to which request information will be 228 | * added. 229 | */ 230 | module.exports.addDirectUpload = ( 231 | host, 232 | targetFolder, 233 | fileNames, 234 | ) => { 235 | nock.disableNetConnect(); 236 | 237 | const files = fileNames.map((fileName) => { 238 | const partPath = `${encodeURI(targetFolder)}/${encodeURI(fileName)}`; 239 | const partUrl = `${host}${partPath}`; 240 | 241 | // success reply for part 242 | nock(host) 243 | .put(partPath) 244 | .reply(201, (uri, body) => uploadInfo.parts.push({ uri, body })); 245 | 246 | return { 247 | fileName, 248 | mimeType: mime.getType(fileName), 249 | uploadToken: `upload-token-${targetFolder}`, 250 | uploadURIs: [partUrl], 251 | minPartSize: 256, 252 | maxPartSize: 2048, 253 | }; 254 | }); 255 | 256 | const completeURI = `${encodeURI(targetFolder)}.completeUpload.json`; 257 | const initiatePath = `${encodeURI(targetFolder)}.initiateUpload.json`; 258 | 259 | // success reply for init 260 | nock(host) 261 | .post(initiatePath) 262 | .times(firstCheck ? 2 : 1) // twice for direct binary access enabled check 263 | .reply(201, (uri, body) => { 264 | uploadInfo.inits.push({ uri, body }); 265 | return { 266 | completeURI, 267 | folderPath: targetFolder, 268 | files, 269 | }; 270 | }); 271 | 272 | // success reply for complete 273 | nock(host) 274 | .post(completeURI) 275 | .times(fileNames.length) 276 | .reply(201, (uri, body) => { 277 | uploadInfo.completes.push({ uri, body }); 278 | return {}; 279 | }); 280 | firstCheck = false; 281 | }; 282 | 283 | /** 284 | * Create mock HTTP requests necessary for a directory creation request. 285 | * 286 | * @param {string} host Host on which the directory will be created. 287 | * @param {string} directoryPath Full path to the directory to create. 288 | * @param {number} [status=201] Status code that will be included in the response. 289 | */ 290 | module.exports.addCreateDirectory = (host, directoryPath, status = 201) => { 291 | nock.disableNetConnect(); 292 | 293 | nock(host) 294 | .post(`/api/assets${directoryPath}`) 295 | .reply(status, (uri, body) => folderInfo.push({ uri, body })); 296 | }; 297 | 298 | /** 299 | * Parses a raw query string and converts it to a simple javascript object whose keys 300 | * are param names, and values are the param's value. 301 | * @param {string} query Query value to parse. 302 | * @returns {URLSearchParams} Parsed query parameters. 303 | */ 304 | module.exports.parseQuery = (query) => { 305 | const params = new URLSearchParams(query); 306 | const parsed = {}; 307 | params.forEach((value, key) => { 308 | parsed[key] = value; 309 | }); 310 | return parsed; 311 | }; 312 | -------------------------------------------------------------------------------- /test/upload-file.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2021 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | 17 | const { getTestOptions } = require('./testutils'); 18 | 19 | const DirectBinaryUploadOptions = require('../src/direct-binary-upload-options'); 20 | const UploadFile = require('../src/upload-file'); 21 | 22 | describe('UploadFile Tests', () => { 23 | function ensureFailure(options, uploadOptions, fileOptions) { 24 | const uploadFile = new UploadFile(options, uploadOptions, fileOptions); 25 | try { 26 | uploadFile.getFileUrl(); 27 | } catch (e) { 28 | return; 29 | } 30 | should(false).be.ok(); 31 | } 32 | it('test get file URL', async () => { 33 | const uploadOptions = new DirectBinaryUploadOptions() 34 | .withUrl('http://somefakeunittesturl'); 35 | let uploadFile = new UploadFile(getTestOptions(), uploadOptions, { 36 | fileName: 'testfile.jpg', 37 | fileSize: 1024, 38 | filePath: '/test/file.jpg', 39 | }); 40 | should(uploadFile.getFileUrl()).be.exactly('http://somefakeunittesturl/testfile.jpg'); 41 | uploadFile = new UploadFile(getTestOptions(), uploadOptions, { 42 | fileUrl: 'http://fullfileurl/file.jpg', 43 | fileSize: 0, 44 | blob: [], 45 | }); 46 | should(uploadFile.getFileUrl()).be.exactly('http://fullfileurl/file.jpg'); 47 | uploadFile = new UploadFile(getTestOptions(), uploadOptions, { 48 | fileSize: 0, 49 | blob: [], 50 | }); 51 | ensureFailure(getTestOptions(), uploadOptions, { 52 | fileSize: 0, 53 | blob: [], 54 | }); 55 | ensureFailure(getTestOptions(), uploadOptions, { 56 | fileName: 'testfile.jpg', 57 | blob: [], 58 | }); 59 | ensureFailure(getTestOptions(), uploadOptions, { 60 | fileName: 'testfile.jpg', 61 | fileSize: 1024, 62 | }); 63 | ensureFailure(getTestOptions(), uploadOptions, { 64 | fileName: 'testfile.jpg', 65 | fileSize: 1024, 66 | blob: {}, 67 | }); 68 | }); 69 | 70 | it('test part headers', () => { 71 | const uploadOptions = new DirectBinaryUploadOptions() 72 | .withUrl('http://somefakeunittesturl'); 73 | let uploadFile = new UploadFile(getTestOptions(), uploadOptions, { 74 | fileName: 'testfile.jpg', 75 | fileSize: 1024, 76 | filePath: '/test/file.jpg', 77 | }); 78 | should(uploadFile.getPartHeaders()).be.ok(); 79 | should(uploadFile.getPartHeaders().missing).not.be.ok(); 80 | should(uploadFile.toJSON().multipartHeaders).not.be.ok(); 81 | 82 | uploadFile = new UploadFile(getTestOptions(), uploadOptions, { 83 | fileName: 'testfile.jpg', 84 | fileSize: 1024, 85 | filePath: '/test/file.jpg', 86 | partHeaders: { 87 | hello: 'world!', 88 | }, 89 | }); 90 | should(uploadFile.getPartHeaders()).be.ok(); 91 | should(uploadFile.getPartHeaders().hello).equal('world!'); 92 | should(uploadFile.toJSON().multipartHeaders.hello).equal('world!'); 93 | }); 94 | }); 95 | -------------------------------------------------------------------------------- /test/upload-result.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2020 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | const Sinon = require('sinon'); 17 | 18 | const UploadResult = require('../src/upload-result'); 19 | 20 | describe('UploadResult Tests', () => { 21 | // eslint-disable-next-line func-names 22 | before(function () { 23 | this.clock = Sinon.useFakeTimers(10); 24 | }); 25 | 26 | // eslint-disable-next-line func-names 27 | after(function () { 28 | this.clock.restore(); 29 | }); 30 | 31 | // eslint-disable-next-line func-names 32 | it('test timer', async function () { 33 | const uploadResult = new UploadResult(); 34 | uploadResult.startTimer(); 35 | this.clock.tick(20); 36 | uploadResult.stopTimer(); 37 | should(uploadResult.getElapsedTime() >= 20).be.ok(); 38 | uploadResult.startTimer(); 39 | this.clock.tick(20); 40 | uploadResult.stopTimer(); 41 | should(uploadResult.getElapsedTime() >= 40).be.ok(); 42 | }); 43 | }); 44 | -------------------------------------------------------------------------------- /test/utils.test.js: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2019 Adobe. All rights reserved. 3 | This file is licensed to you under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. You may obtain a copy 5 | of the License at http://www.apache.org/licenses/LICENSE-2.0 6 | 7 | Unless required by applicable law or agreed to in writing, software distributed under 8 | the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS 9 | OF ANY KIND, either express or implied. See the License for the specific language 10 | governing permissions and limitations under the License. 11 | */ 12 | 13 | /* eslint-env mocha */ 14 | 15 | const should = require('should'); 16 | const mock = require('mock-fs'); 17 | 18 | const { 19 | concurrentLoop, 20 | trimRight, 21 | trimLeft, 22 | joinUrlPath, 23 | trimContentDam, 24 | walkDirectory, 25 | } = require('../src/utils'); 26 | const { DefaultValues } = require('../src/constants'); 27 | 28 | describe('UtilsTest', () => { 29 | beforeEach(() => { 30 | mock.restore(); 31 | }); 32 | 33 | afterEach(() => { 34 | mock.restore(); 35 | }); 36 | 37 | describe('concurrentLoop tests', () => { 38 | async function runMaxConcurrentTest(maxValue) { 39 | let currCount = 0; 40 | const testArray = []; 41 | for (let i = 0; i < 10; i += 1) { 42 | testArray[i] = i; 43 | } 44 | 45 | const params = []; 46 | params.push(testArray); 47 | 48 | if (maxValue) { 49 | params.push(maxValue); 50 | } 51 | 52 | params.push(() => new Promise((res) => { 53 | should(currCount).be.lessThan(maxValue || DefaultValues.MAX_CONCURRENT); 54 | currCount += 1; 55 | setTimeout(() => { 56 | currCount -= 1; 57 | res(); 58 | }, 100); 59 | })); 60 | 61 | await concurrentLoop(...params); 62 | } 63 | 64 | it('test max concurrent', async () => { 65 | await runMaxConcurrentTest(); 66 | }); 67 | 68 | it('test max concurrent value', async () => { 69 | await runMaxConcurrentTest(7); 70 | }); 71 | }); 72 | 73 | it('test trim right', () => { 74 | should(trimRight('/test/', ['/'])).be.exactly('/test'); 75 | should(trimRight('/test', ['/'])).be.exactly('/test'); 76 | should(trimRight('/', ['/'])).be.exactly(''); 77 | should(trimRight('', ['/'])).be.exactly(''); 78 | should(trimRight(null, ['/'])).be.exactly(null); 79 | should(trimRight(1, ['/'])).be.exactly(1); 80 | should(trimRight('/trim\\]', ['\\', ']'])).be.exactly('/trim'); 81 | }); 82 | 83 | it('test trim left', () => { 84 | should(trimLeft('/test/', ['/'])).be.exactly('test/'); 85 | should(trimLeft('/test', ['/'])).be.exactly('test'); 86 | should(trimLeft('/', ['/'])).be.exactly(''); 87 | should(trimLeft('', ['/'])).be.exactly(''); 88 | should(trimLeft(null, ['/'])).be.exactly(null); 89 | should(trimLeft(1, ['/'])).be.exactly(1); 90 | should(trimLeft('\\]trim', ['\\', ']'])).be.exactly('trim'); 91 | }); 92 | 93 | it('test join url path', () => { 94 | should(joinUrlPath('1', '2', '3')).be.exactly('/1/2/3'); 95 | should(joinUrlPath('/1', '/2/', '3/')).be.exactly('/1/2/3'); 96 | should(joinUrlPath('/', '1', '')).be.exactly('/1'); 97 | }); 98 | 99 | it('test trim content dam', () => { 100 | should(trimContentDam('/content/dam')).be.exactly(''); 101 | should(trimContentDam('/content/dam/')).be.exactly(''); 102 | should(trimContentDam('/content/dam/test')).be.exactly('/test'); 103 | should(trimContentDam(null)).be.exactly(null); 104 | should(trimContentDam('/')).be.exactly(''); 105 | should(trimContentDam('/content/dame')).be.exactly('/content/dame'); 106 | should(trimContentDam('/content/dame/test')).be.exactly('/content/dame/test'); 107 | should(trimContentDam('/test')).be.exactly('/test'); 108 | should(trimContentDam('/test/')).be.exactly('/test'); 109 | }); 110 | 111 | function getPathIndex(itemList, path) { 112 | for (let i = 0; i < itemList.length; i += 1) { 113 | const { path: comparePath } = itemList[i]; 114 | if (path === comparePath) { 115 | return i; 116 | } 117 | } 118 | return -1; 119 | } 120 | 121 | function createDirectoryStructure() { 122 | mock({ 123 | '/root': { 124 | 'file1.jpg': '1234', 125 | 'file2.jpg': '1234', 126 | dir1: { 127 | 'file3.jpg': '1234', 128 | 'file4.jpg': '1234', 129 | dir2: { 130 | dir3: { 131 | 'file5.jpg': '1234', 132 | 'file6.jpg': '1234', 133 | }, 134 | }, 135 | }, 136 | '.tempdir': {}, 137 | '.tempfile.jpg': '1234', 138 | error: mock.directory({ 139 | uid: 'invalid', 140 | gid: 'invalid', 141 | mode: 0, 142 | }), 143 | 'error.jpg': mock.symlink({ 144 | path: '/invalid-path', 145 | }), 146 | emptydir: { 147 | emptysubdir: {}, 148 | }, 149 | }, 150 | }); 151 | } 152 | 153 | it('test walk directory with descendents', async () => { 154 | createDirectoryStructure(); 155 | 156 | const { 157 | directories, files, totalSize, errors, 158 | } = await walkDirectory('/root'); 159 | should(directories.length).be.exactly(6); 160 | should(files.length).be.exactly(6); 161 | should(totalSize).be.exactly(files.length * 4); 162 | should(errors.length).be.exactly(2); 163 | 164 | const dir1 = getPathIndex(directories, '/root/dir1'); 165 | const dir2 = getPathIndex(directories, '/root/dir1/dir2'); 166 | const dir3 = getPathIndex(directories, '/root/dir1/dir2/dir3'); 167 | const dir4 = getPathIndex(directories, '/root/emptydir'); 168 | const dir5 = getPathIndex(directories, '/root/emptydir/emptysubdir'); 169 | const dir6 = getPathIndex(directories, '/root/error'); 170 | 171 | should(dir1 >= 0 && dir1 > dir2 && dir1 > dir3); 172 | should(dir2 >= 0 && dir2 > dir3); 173 | should(dir3 >= 0); 174 | should(dir6 >= 0); 175 | should(dir4 >= 0); 176 | should(dir5 > dir4); 177 | 178 | const file1 = getPathIndex(files, '/root/file1.jpg'); 179 | const file2 = getPathIndex(files, '/root/file2.jpg'); 180 | const file3 = getPathIndex(files, '/root/dir1/file3.jpg'); 181 | const file4 = getPathIndex(files, '/root/dir1/file4.jpg'); 182 | const file5 = getPathIndex(files, '/root/dir1/dir2/dir3/file5.jpg'); 183 | const file6 = getPathIndex(files, '/root/dir1/dir2/dir3/file6.jpg'); 184 | 185 | should(file1 >= 0 && file1 < file3 && file1 < file4 && file1 < file5 && file1 < file6).be.ok(); 186 | should(file2 >= 0 && file2 < file3 && file2 < file4 && file2 < file5 && file2 < file6).be.ok(); 187 | should(file3 >= 0 && file3 < file5 && file3 < file6).be.ok(); 188 | should(file4 >= 0 && file4 < file5 && file4 < file6).be.ok(); 189 | should(file5 >= 0).be.ok(); 190 | should(file6 >= 0).be.ok(); 191 | }); 192 | 193 | it('test walk directory no descendents', async () => { 194 | createDirectoryStructure(); 195 | 196 | const { 197 | directories, files, totalSize, errors, 198 | } = await walkDirectory('/root', 1000, false); 199 | should(directories.length).be.exactly(3); 200 | 201 | const dir1 = getPathIndex(directories, '/root/dir1'); 202 | const dir2 = getPathIndex(directories, '/root/error'); 203 | const dir3 = getPathIndex(directories, '/root/emptydir'); 204 | should(dir1 >= 0); 205 | should(dir2 >= 0); 206 | should(dir3 >= 0); 207 | 208 | should(files.length).be.exactly(2); 209 | should(totalSize).be.exactly(files.length * 4); 210 | should(errors.length).be.exactly(1); 211 | 212 | const file1 = getPathIndex(files, '/root/file1.jpg'); 213 | const file2 = getPathIndex(files, '/root/file2.jpg'); 214 | 215 | should(file1 >= 0 && file1 < file2).be.ok(); 216 | should(file2 >= 0).be.ok(); 217 | }); 218 | }); 219 | --------------------------------------------------------------------------------