├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── config.example.js ├── index.js ├── lib ├── index.js ├── questions.js ├── schemas.js └── transfer.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # node-waf configuration 21 | .lock-wscript 22 | 23 | # Compiled binary addons (http://nodejs.org/api/addons.html) 24 | build/Release 25 | 26 | # Dependency directory 27 | node_modules 28 | 29 | # Optional npm cache directory 30 | .npm 31 | 32 | # Optional REPL history 33 | .node_repl_history 34 | 35 | .DS_Store 36 | 37 | config.js 38 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '4.3' 4 | after_success: 5 | - bash <(curl -s https://codecov.io/bash) 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 parse-server-modules 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # parse-files-utils [Archive] 2 | 3 | Utilities to list and ~~migrate~~ Parse files. 4 | 5 | **File migration is not available anymore as the hosted Parse service has now shut down** 6 | 7 | This utility will do the following: 8 | 9 | 1. Get all files across all classes in a Parse database. 10 | 2. Print file URLs to console OR transfer to S3, GCS, or filesystem. 11 | 3. Rename files so that [Parse Server](https://github.com/ParsePlatform/parse-server) no longer detects that they are hosted by Parse. 12 | 4. Update MongoDB with new file names. 13 | 14 | ![Use at your own risk](https://github.com/mongodb/support-tools/raw/master/use-at-your-own-risk.jpg) 15 | 16 | [(image from flickr user alykat)](http://www.flickr.com/photos/80081757@N00/4271250480/) 17 | 18 | DISCLAIMER 19 | ---------- 20 | Please note: all tools/ scripts in this repo are released for use "AS IS" **without any warranties of any kind**, 21 | including, but not limited to their installation, use, or performance. We disclaim any and all warranties, either 22 | express or implied, including but not limited to any warranty of noninfringement, merchantability, and/ or fitness 23 | for a particular purpose. We do not warrant that the technology will meet your requirements, that the operation 24 | thereof will be uninterrupted or error-free, or that any errors will be corrected. 25 | 26 | Any use of these scripts and tools is **at your own risk**. There is no guarantee that they have been through 27 | thorough testing in a comparable environment and we are not responsible for any damage or data loss incurred with 28 | their use. 29 | 30 | You are responsible for reviewing and testing any scripts you run *thoroughly* before use in any non-testing 31 | environment. 32 | 33 | Thanks, 34 | The *UNOFFICIAL* parse-server-modules team 35 | 36 | [(this disclaimer was originally published here)](https://github.com/mongodb/support-tools/blob/master/README.md) 37 | 38 | #### \*WARNING\* 39 | As soon as this script transfers files away from Parse.com hosted files (and renames them in the database) 40 | any clients that use api.parse.com will no longer be able to access the files. 41 | See the section titled "5. Files" in the [Parse Migration Guide](https://parse.com/migration) 42 | and Parse Server [issue #1582](https://github.com/ParsePlatform/parse-server/issues/1582). 43 | 44 | ## Installation 45 | 46 | 1. Clone the repo: `git clone git@github.com:parse-server-modules/parse-files-utils.git` 47 | 2. cd into repo: `cd parse-files-utils` 48 | 3. Install dependencies: `npm install` 49 | 50 | ## Usage 51 | 52 | The quickest way to get started is to run `npm start` and follow the command prompts. 53 | 54 | You can optionally specify a js/json configuration file (see [config.example.js](./config.example.js)). 55 | ``` 56 | $ npm start config.js 57 | ``` 58 | 59 | ### Available configuration options 60 | 61 | * `applicationId`: Parse application id. 62 | * `masterKey`: Parse master key. 63 | * `serverURL`: The URL for the Parse server (default: http://api.parse.com/1). 64 | This is used to with `applicationId` and `masterKey` to get the schema and fetch all files/objects. 65 | * `renameFiles` (boolean): Whether or not to rename Parse hosted files. 66 | This removes the "tfss-" or legacy Parse filename prefix before saving with the new file adapter. 67 | * `renameInDatabase` (boolean): Whether or not to rename files in MongoDB. 68 | * `mongoURL`: MongoDB connection url. 69 | Direct access to the database is needed because Parse SDK doesn't allow direct writing to file fields. 70 | * `filesToTransfer`: Which files to transfer. 71 | Accepted options: 72 | * `"parseOnly"`: only process files with a filename that starts with "tfss-" or matches Parse's legacy file name format. 73 | * `"parseServerOnly"`: only process files with a filename that **does not** start with "tfss-" nor match Parse's legacy file name format. 74 | * `"all"`: process all files. 75 | * `filesAdapter`: A Parse Server file adapter with a function for `createFile(filename, data)` 76 | (ie. [parse-server-fs-adapter](https://github.com/parse-server-modules/parse-server-fs-adapter), 77 | [parse-server-s3-adapter](https://github.com/parse-server-modules/parse-server-s3-adapter), 78 | [parse-server-gcs-adapter](https://github.com/parse-server-modules/parse-server-gcs-adapter)). 79 | * `filesystemPath`: The path/directory to save files to when transfering to filesystem. 80 | * `aws_accessKeyId`: AWS access key id. 81 | * `aws_secretAccessKey`: AWS secret access key. 82 | * `aws_bucket`: S3 bucket name. 83 | * `gcs_projectId`: GCS project id. 84 | * `gcs_keyFilename`: GCS key filename (ie. `credentials.json`). 85 | * `gcs_bucket`: GCS bucket name. 86 | * `asyncLimit`: The number of files to process at the same time (default: 5). 87 | 88 | 89 | ## Parse File Migrations 90 | 91 | If you need to migrate files from hosted Parse.com to self-hosted Parse Server, 92 | you should follow one of the below strategies. 93 | Given [ParsePlatform/parse-server#1582](https://github.com/ParsePlatform/parse-server/issues/1582), 94 | there will not be any updates to api.parse.com. This leaves two options for the file migration. 95 | Each one has its own set of advantages and disadvantages. 96 | 97 | **Option 1**: 98 | "Supporting clients that access via api.parse.com is *not* important" 99 | * File utils configuration: 100 | * `filesToTransfer`: 'parseOnly' 101 | * `renameFiles`: true 102 | * `renameInDatabase`: true 103 | * Parse Server configuration: keep using `fileKey` in settings 104 | * After file migration: 105 | * api.parse.com clients: 106 | * can not see all previously uploaded files 107 | * can not see files uploaded by Parse Server clients 108 | * can see new files uploaded by api.parse.com clients 109 | * Parse Server clients: 110 | * can see all previously uploaded files 111 | * can see new files uploaded by api.parse.com clients 112 | * can see new files uploaded by Parse Server clients 113 | * Additional steps required: 114 | * Run file migration again after all clients switch to Parse Server or before Jan 28 115 | 116 | **Option 2**: 117 | "Supporting clients that access via api.parse.com is important" 118 | * File utils configuration: 119 | * `filesToTransfer`: 'parseOnly' 120 | * `renameFiles`: false 121 | * `renameInDatabase`: false 122 | * Parse Server configuration: 123 | * Use version >= 2.2.16 124 | * remove `fileKey` from settings 125 | * After file migration: 126 | * api.parse.com clients: 127 | * can see all previously uploaded files 128 | * can not see files uploaded by Parse Server clients 129 | * Parse Server clients: 130 | * can see all migrated files 131 | * can not see new files uploaded by api.parse.com clients 132 | * Additional steps required: 133 | * Run file migration again after all clients switch to Parse Server or before Jan 28 134 | 135 | -------------------------------------------------------------------------------- /config.example.js: -------------------------------------------------------------------------------- 1 | var FileAdapter = require('parse-server-fs-adapter'); 2 | var S3Adapter = require('parse-server-s3-adapter'); 3 | var GCSAdapter = require('parse-server-gcs-adapter'); 4 | 5 | module.exports = { 6 | applicationId: "PARSE_APPLICATION_ID", 7 | masterKey: "PARSE_MASTER_KEY", 8 | mongoURL: "mongodb://:@mongourl.com:27017/database_name", 9 | serverURL: "https://api.customparseserver.com/parse", 10 | filesToTransfer: 'parseOnly', 11 | renameInDatabase: false, 12 | transferTo: 'filesystem', 13 | 14 | // For filesystem configuration 15 | filesystemPath: './downloaded_files', 16 | 17 | // For S3 configuration 18 | aws_accessKeyId: "ACCESS_KEY_ID", 19 | aws_secretAccessKey: "SECRET_ACCESS_KEY", 20 | aws_bucket: "BUCKET_NAME", 21 | aws_bucketPrefix: "", 22 | 23 | // For GCS configuration 24 | gcs_projectId: "GCS_PROJECT_ID", 25 | gcs_keyFilename: "credentials.json", 26 | gcs_bucket: "BUCKET_NAME", 27 | 28 | // For Azure configuration 29 | azure_account: "STORAGE_ACCOUNT_NAME", 30 | azure_container: "BLOB_CONTAINER", 31 | azure_accessKey: "ACCESS_KEY", 32 | 33 | // Or set filesAdapter to a Parse Server file adapter 34 | // filesAdapter: new FileAdapter({ 35 | // filesSubDirectory: './downloaded_files' 36 | // }), 37 | // filesAdapter: new S3Adapter({ 38 | // accessKey: 'ACCESS_KEY_ID', 39 | // secretKey: 'SECRET_ACCESS_KEY', 40 | // bucket: 'BUCKET_NAME' 41 | // }), 42 | // filesAdapter: new GCSAdapter({ 43 | // projectId: "GCS_PROJECT_ID", 44 | // keyFilename: "credentials.json", 45 | // bucket: "BUCKET_NAME", 46 | // }), 47 | }; 48 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var configFilePath = process.argv[2]; 3 | var config = {}; 4 | 5 | if (configFilePath) { 6 | configFilePath = path.resolve(configFilePath); 7 | 8 | try { 9 | config = require(configFilePath); 10 | } catch(e) { 11 | console.log('Cannot load '+configFilePath); 12 | process.exit(1); 13 | } 14 | } 15 | 16 | var utils = require('./lib')(config); 17 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | var Parse = require('parse/node'); 2 | var inquirer = require('inquirer'); 3 | 4 | var schemas = require('./schemas'); 5 | var transfer = require('./transfer'); 6 | var questions = require('./questions.js'); 7 | 8 | module.exports = initialize; 9 | 10 | function initialize(config) { 11 | questions(config).then(function (answers) { 12 | config = Object.assign(config, answers); 13 | console.log(JSON.stringify(config, null, 2)); 14 | return inquirer.prompt({ 15 | type: 'confirm', 16 | name: 'next', 17 | message: 'About to start the file transfer. Does the above look correct?', 18 | default: true, 19 | }); 20 | }).then(function(answers) { 21 | if (!answers.next) { 22 | console.log('Aborted!'); 23 | process.exit(); 24 | } 25 | Parse.initialize(config.applicationId, null, config.masterKey); 26 | Parse.serverURL = config.serverURL; 27 | return transfer.init(config); 28 | }).then(function() { 29 | return getAllFileObjects(); 30 | }).then(function(objects) { 31 | return transfer.run(objects); 32 | }).then(function() { 33 | console.log('Complete!'); 34 | process.exit(); 35 | }).catch(function(error) { 36 | console.log(error); 37 | process.exit(1); 38 | }); 39 | } 40 | 41 | function getAllFileObjects() { 42 | console.log("Fetching schema..."); 43 | return schemas.get().then(function(res){ 44 | console.log("Fetching all objects with files..."); 45 | var schemasWithFiles = onlyFiles(res); 46 | return Promise.all(schemasWithFiles.map(getObjectsWithFilesFromSchema)); 47 | }).then(function(results) { 48 | var files = results.reduce(function(c, r) { 49 | return c.concat(r); 50 | }, []).filter(function(file) { 51 | return file.fileName !== 'DELETE'; 52 | }); 53 | 54 | return Promise.resolve(files); 55 | }); 56 | } 57 | 58 | function onlyFiles(schemas) { 59 | return schemas.map(function(schema) { 60 | var fileFields = Object.keys(schema.fields).filter(function(key){ 61 | var value = schema.fields[key]; 62 | return value.type == "File"; 63 | }); 64 | if (fileFields.length > 0) { 65 | return { 66 | className: schema.className, 67 | fields: fileFields 68 | } 69 | } 70 | }).filter(function(s){ return s != undefined }) 71 | } 72 | 73 | function getAllObjects(baseQuery) { 74 | var allObjects = []; 75 | var next = function() { 76 | if (allObjects.length) { 77 | baseQuery.greaterThan('createdAt', allObjects[allObjects.length-1].createdAt); 78 | } 79 | return baseQuery.find({useMasterKey: true}).then(function(r){ 80 | allObjects = allObjects.concat(r); 81 | if (r.length == 0) { 82 | return Promise.resolve(allObjects); 83 | } else { 84 | return next(); 85 | } 86 | }); 87 | } 88 | return next(); 89 | } 90 | 91 | function getObjectsWithFilesFromSchema(schema) { 92 | var query = new Parse.Query(schema.className); 93 | query.select(schema.fields.concat('createdAt')); 94 | query.ascending('createdAt'); 95 | query.limit(1000); 96 | 97 | var checks = schema.fields.map(function(field) { 98 | return new Parse.Query(schema.className).exists(field); 99 | }); 100 | query._orQuery(checks); 101 | 102 | return getAllObjects(query).then(function(results) { 103 | return results.reduce(function(current, result){ 104 | return current.concat( 105 | schema.fields.map(function(field){ 106 | var fName = result.get(field) ? result.get(field).name() : 'DELETE'; 107 | var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; 108 | return { 109 | className: schema.className, 110 | objectId: result.id, 111 | fieldName: field, 112 | fileName: fName, 113 | url: fUrl 114 | } 115 | }) 116 | ); 117 | }, []); 118 | }); 119 | } 120 | -------------------------------------------------------------------------------- /lib/questions.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Uses command line prompts to collect necessary info 3 | */ 4 | 5 | var inquirer = require('inquirer'); 6 | module.exports = questions; 7 | 8 | function questions(config) { 9 | return inquirer.prompt([ 10 | // Collect Parse info 11 | { 12 | type: 'input', 13 | name: 'applicationId', 14 | message: 'The applicationId', 15 | when: !config.applicationId 16 | }, { 17 | type: 'input', 18 | name: 'masterKey', 19 | message: 'The masterKey', 20 | when: !config.masterKey 21 | }, { 22 | type: 'input', 23 | name: 'serverURL', 24 | message: 'The Parse serverURL', 25 | when: !config.serverURL, 26 | default: 'https://api.parse.com/1' 27 | }, { 28 | type: 'list', 29 | name: 'filesToTransfer', 30 | message: 'What files would you like to transfer?', 31 | choices: [ 32 | {name: 'Only parse.com hosted files', value: 'parseOnly'}, 33 | {name: 'Only Parse Server (self hosted server) files', value: 'parseServerOnly'}, 34 | {name: 'All files', value: 'all'} 35 | ], 36 | when: (['parseOnly','parseServerOnly', 'all'].indexOf(config.filesToTransfer) == -1) 37 | }, { 38 | type: 'confirm', 39 | name: 'renameFiles', 40 | message: 'Rename Parse hosted file names?', 41 | default: false, 42 | when: function(answers) { 43 | return config.renameFiles == undefined && 44 | (answers.filesToTransfer == 'all' || config.filesToTransfer == 'all' || 45 | config.filesToTransfer == 'parseOnly' || answers.filesToTransfer == 'parseOnly'); 46 | } 47 | }, { 48 | type: 'confirm', 49 | name: 'renameInDatabase', 50 | message: 'Rename Parse hosted files in the database after transfer?', 51 | default: false, 52 | when: function(answers) { 53 | return config.renameInDatabase == undefined && 54 | (answers.renameFiles || config.renameFiles) && 55 | (answers.filesToTransfer == 'all' || config.filesToTransfer == 'all' || 56 | config.filesToTransfer == 'parseOnly' || answers.filesToTransfer == 'parseOnly'); 57 | } 58 | }, { 59 | type: 'input', 60 | name: 'mongoURL', 61 | message: 'MongoDB URL', 62 | default: 'mongodb://localhost:27017/database', 63 | when: function(answers) { 64 | return (config.renameInDatabase || answers.renameInDatabase) && 65 | !config.mongoURL; 66 | } 67 | }, 68 | 69 | // Where to transfer to 70 | { 71 | type: 'list', 72 | name: 'transferTo', 73 | message: 'Where would you like to transfer files to?', 74 | choices: [ 75 | {name: 'Print List of URLs', value: 'print'}, 76 | {name: 'Local File System', value: 'filesystem'}, 77 | {name: 'AWS S3', value: 's3'}, 78 | {name: 'Google Cloud Storage', value: 'gcs'}, 79 | {name: 'Azure Blob Storage', value: 'azure'}, 80 | ], 81 | when: function() { 82 | return (['print','filesystem','s3','gcs'].indexOf(config.transferTo) == -1) && 83 | !config.filesAdapter 84 | } 85 | }, 86 | 87 | // filesystem settings 88 | { 89 | type: 'input', 90 | name: 'filesystemPath', 91 | message: 'Local filesystem path to save files to', 92 | when: function(answers) { 93 | return !config.filesystemPath && 94 | (config.transferTo == 'filesystem' || 95 | answers.transferTo == 'filesystem'); 96 | }, 97 | default: './downloaded_files' 98 | }, 99 | 100 | // S3 settings 101 | { 102 | type: 'input', 103 | name: 'aws_accessKeyId', 104 | message: 'AWS access key id', 105 | when: function(answers) { 106 | return (answers.transferTo == 's3' || config.transferTo == 's3') && 107 | !config.aws_accessKeyId && 108 | !config.aws_profile; 109 | }, 110 | default: process.env.AWS_ACCESS_KEY_ID 111 | }, { 112 | type: 'input', 113 | name: 'aws_secretAccessKey', 114 | message: 'AWS secret access key', 115 | when: function(answers) { 116 | return (answers.transferTo == 's3' || config.transferTo == 's3') && 117 | !config.aws_secretAccessKey && 118 | !config.aws_profile; 119 | }, 120 | default: process.env.AWS_SECRET_ACCESS_KEY 121 | }, { 122 | type: 'input', 123 | name: 'aws_bucket', 124 | message: 'S3 bucket name', 125 | when: function(answers) { 126 | return (answers.transferTo == 's3' || config.transferTo == 's3') && 127 | !config.aws_bucket; 128 | } 129 | }, 130 | { 131 | type: 'input', 132 | name: 'aws_bucketPrefix', 133 | message: 'S3 bucket prefix (optional)', 134 | when: function (answers) { 135 | return (answers.transferTo == 's3' || config.transferTo == 's3') && 136 | !config.aws_bucketPrefix; 137 | } 138 | }, 139 | 140 | // GCS settings 141 | { 142 | type: 'input', 143 | name: 'gcs_projectId', 144 | message: 'GCS project id', 145 | when: function(answers) { 146 | return (answers.transferTo == 'gcs' || config.transferTo == 'gcs') && 147 | !config.gcs_projectId; 148 | } 149 | }, { 150 | type: 'input', 151 | name: 'gcs_keyFilename', 152 | message: 'GCS key filename', 153 | when: function(answers) { 154 | return (answers.transferTo == 'gcs' || config.transferTo == 'gcs') && 155 | !config.gcs_keyFilename; 156 | }, 157 | default: 'credentials.json' 158 | }, { 159 | type: 'input', 160 | name: 'gcs_bucket', 161 | message: 'GCS bucket name', 162 | when: function(answers) { 163 | return (answers.transferTo == 'gcs' || config.transferTo == 'gcs') && 164 | !config.gcs_bucket; 165 | } 166 | }, 167 | 168 | // Azure settings 169 | { 170 | type: 'input', 171 | name: 'azure_account', 172 | message: 'Azure Storage account', 173 | when: function(answers) { 174 | return (answers.transferTo == 'azure' || config.transferTo == 'azure') && 175 | !config.azure_account; 176 | } 177 | }, { 178 | type: 'input', 179 | name: 'azure_container', 180 | message: 'Azure Storage container', 181 | when: function(answers) { 182 | return (answers.transferTo == 'azure' || config.transferTo == 'azure') && 183 | !config.azure_account && 184 | !config.azure_container; 185 | } 186 | }, { 187 | type: 'input', 188 | name: 'azure_accessKey', 189 | message: 'Azure Storage access key', 190 | when: function(answers) { 191 | return (answers.transferTo == 'azure' || config.transferTo == 'azure') && 192 | !config.azure_account && 193 | !config.azure_container && 194 | !config.azure_accessKey; 195 | } 196 | }, 197 | { 198 | type: 'input', 199 | name: 'aws_bucketPrefix', 200 | message: 'S3 bucket prefix (optional)', 201 | when: function (answers) { 202 | return (answers.transferTo == 's3' || config.transferTo == 's3') && 203 | !config.aws_bucketPrefix; 204 | } 205 | }, 206 | ]); 207 | } 208 | -------------------------------------------------------------------------------- /lib/schemas.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var Parse = require('parse/node'); 3 | 4 | function get() { 5 | return new Promise(function(resolve, reject) { 6 | request({ 7 | method: 'GET', 8 | url: Parse.serverURL+"/schemas", 9 | json: true, 10 | headers: { 11 | 'Content-Type': 'application/json', 12 | 'X-Parse-Application-Id': Parse.applicationId, 13 | 'X-Parse-Master-Key': Parse.masterKey 14 | } 15 | }, function(err, res, body) { 16 | if (err) { 17 | return reject(err); 18 | } 19 | if (!body.results) { 20 | return reject(JSON.stringify(body)); 21 | } 22 | resolve(body.results); 23 | }) 24 | }); 25 | } 26 | 27 | module.exports = Object.freeze({ 28 | get: get 29 | }) 30 | -------------------------------------------------------------------------------- /lib/transfer.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var crypto = require('crypto'); 3 | var async = require('async'); 4 | var FilesystemAdapter = require('parse-server-fs-adapter'); 5 | var S3Adapter = require('parse-server-s3-adapter'); 6 | var GCSAdapter = require('parse-server-gcs-adapter'); 7 | var AzureStorageAdapter = require('parse-server-azure-storage').AzureStorageAdapter; 8 | var MongoClient = require('mongodb').MongoClient; 9 | 10 | // regex that matches old legacy Parse hosted files 11 | var legacyFilesPrefixRegex = new RegExp("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}-"); 12 | var migratedFilePrefix = 'mfp_'; 13 | 14 | var db, config; 15 | 16 | module.exports.init = init; 17 | module.exports.run = run; 18 | 19 | function init(options) { 20 | console.log('Initializing transfer configuration...'); 21 | config = options; 22 | return new Promise(function(resolve, reject) { 23 | if (config.renameInDatabase) { 24 | console.log('Connecting to MongoDB'); 25 | MongoClient.connect(config.mongoURL, function(error, database) { 26 | if (error) { 27 | return reject(error); 28 | } 29 | console.log('Successfully connected to MongoDB'); 30 | db = database; 31 | _setup().then(resolve, reject); 32 | }); 33 | } else { 34 | _setup().then(resolve, reject); 35 | } 36 | }); 37 | } 38 | 39 | function _setup() { 40 | config.adapterName = config.transferTo || config.filesAdapter.constructor.name; 41 | console.log('Initializing '+config.adapterName+' adapter'); 42 | if (config.filesAdapter && config.filesAdapter.createFile) { 43 | return Promise.resolve(); 44 | } else if (config.transferTo == 'print') { 45 | return Promise.resolve(); 46 | } else if (config.transferTo == 'filesystem') { 47 | config.filesAdapter = new FilesystemAdapter({ 48 | filesSubDirectory: config.filesystemPath 49 | }); 50 | } else if (config.transferTo == 's3') { 51 | config.filesAdapter = new S3Adapter({ 52 | accessKey: config.aws_accessKeyId, 53 | secretKey: config.aws_secretAccessKey, 54 | bucket: config.aws_bucket, 55 | bucketPrefix: config.aws_bucketPrefix, 56 | directAccess: true 57 | }); 58 | } else if (config.transferTo == 'gcs') { 59 | config.filesAdapter = new GCSAdapter({ 60 | projectId: config.gcs_projectId, 61 | keyFilename: config.gcs_keyFilename, 62 | bucket: config.gcs_bucket, 63 | directAccess: true 64 | }); 65 | } else if (config.transferTo == 'azure') { 66 | 67 | var account = config.azure_account; 68 | var container = config.azure_container; 69 | var options = { 70 | accessKey: config.azure_accessKey, 71 | directAccess: false // If set to true, files will be served by Azure Blob Storage directly 72 | } 73 | config.filesAdapter = new AzureStorageAdapter(account, container, options); 74 | 75 | } else { 76 | return Promise.reject('Invalid files adapter'); 77 | } 78 | return Promise.resolve(); 79 | } 80 | 81 | function run(files) { 82 | console.log('Processing '+files.length+' files'); 83 | console.log('Saving files with '+config.adapterName); 84 | return _processFiles(files); 85 | } 86 | 87 | /** 88 | * Handle error from requests 89 | */ 90 | function _requestErrorHandler(error, response) { 91 | if (error) { 92 | return error; 93 | } else if (response.statusCode >= 300) { 94 | console.log('Failed request ('+response.statusCode+') skipping: '+response.request.href); 95 | return true; 96 | } 97 | return false; 98 | } 99 | 100 | /** 101 | * Converts a file into a non Parse file name 102 | * @param {String} fileName 103 | * @return {String} 104 | */ 105 | function _createNewFileName(fileName) { 106 | if (!config.renameFiles) { 107 | return fileName; 108 | } 109 | if (_isParseHostedFile(fileName)) { 110 | fileName = fileName.replace('tfss-', ''); 111 | var newPrefix = crypto.randomBytes(32/2).toString('hex'); 112 | fileName = newPrefix + fileName.replace(legacyFilesPrefixRegex, ''); 113 | } 114 | return migratedFilePrefix + fileName; 115 | } 116 | 117 | function _isParseHostedFile(fileName) { 118 | if (fileName.indexOf('tfss-') === 0 || legacyFilesPrefixRegex.test(fileName)) { 119 | return true; 120 | } 121 | return false; 122 | } 123 | 124 | /** 125 | * Loops through n files at a time and calls handler 126 | * @param {Array} files Array of files 127 | * @param {Function} handler handler function for file 128 | * @return {Promise} 129 | */ 130 | function _processFiles(files, handler) { 131 | var asyncLimit = config.asyncLimit || 5; 132 | return new Promise(function(resolve, reject) { 133 | async.eachOfLimit(files, asyncLimit, function(file, index, callback) { 134 | process.stdout.write('Processing '+(index+1)+'/'+files.length+'\r'); 135 | file.newFileName = _createNewFileName(file.fileName); 136 | if (_shouldTransferFile(file)) { 137 | _transferFile(file).then(callback, callback); 138 | } else { 139 | process.nextTick(callback); 140 | } 141 | }, function(error) { 142 | if (error) { 143 | return reject(error); 144 | } 145 | resolve('\nComplete!'); 146 | }); 147 | }) 148 | } 149 | 150 | /** 151 | * Changes the file name that is saved in MongoDB 152 | * @param {Object} file the file info 153 | */ 154 | function _changeDBFileField(file) { 155 | return new Promise(function(resolve, reject) { 156 | if (file.fileName == file.newFileName || !config.renameInDatabase) { 157 | return resolve(); 158 | } 159 | var update = {$set:{}}; 160 | update.$set[file.fieldName] = file.newFileName; 161 | db.collection(file.className).update( 162 | { _id : file.objectId }, 163 | update, 164 | function(error, result ) { 165 | if (error) { 166 | return reject(error); 167 | } 168 | resolve(); 169 | } 170 | ); 171 | }); 172 | } 173 | 174 | /** 175 | * Determines if a file should be transferred based on configuration 176 | * @param {Object} file the file info 177 | */ 178 | function _shouldTransferFile(file) { 179 | if (config.filesToTransfer == 'all') { 180 | return true; 181 | } else if ( 182 | config.filesToTransfer == 'parseOnly' && 183 | _isParseHostedFile(file.fileName) 184 | ) { 185 | return true; 186 | } else if ( 187 | config.filesToTransfer == 'parseServerOnly' && 188 | !_isParseHostedFile(file.fileName) 189 | ) { 190 | return true; 191 | } 192 | return false; 193 | } 194 | 195 | function _replaceDomainName(url) { 196 | url = url.replace(/http(s)?:\/\/filescdn\.parsetfss\.com\//, 'https://s3.amazonaws.com/files.parsetfss.com/'); 197 | url = url.replace(/http(s)?:\/\/files\.parsetfss\.com\//, 'https://s3.amazonaws.com/files.parsetfss.com/'); 198 | url = url.replace(/http(s)?:\/\/files\.parse\.com\//, 'https://s3.amazonaws.com/files.parse.com/'); 199 | return url; 200 | } 201 | 202 | /** 203 | * Request file from URL and upload with filesAdapter 204 | * @param {Object} file the file info object 205 | */ 206 | function _transferFile(file) { 207 | return new Promise(function(resolve, reject) { 208 | if (config.transferTo == 'print') { 209 | console.log(file.url); 210 | // Use process.nextTick to avoid max call stack error 211 | return process.nextTick(resolve); 212 | } 213 | request({ url: _replaceDomainName(file.url), encoding: null }, function(error, response, body) { 214 | if (_requestErrorHandler(error, response)) { 215 | return reject(error); 216 | } 217 | config.filesAdapter.createFile( 218 | file.newFileName, body, response.headers['content-type'] 219 | ).then(function() { 220 | return _changeDBFileField(file); 221 | }).then(resolve, reject); 222 | }); 223 | }); 224 | } 225 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "parse-files-utils", 3 | "version": "0.0.1", 4 | "description": "Utilities to list and migrate Parse files", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js", 8 | "test": "echo 'build some tests!' && exit 0", 9 | "posttest": "istanbul cover lib/*" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git+https://github.com/parse-server-modules/parse-files-utils.git" 14 | }, 15 | "keywords": [ 16 | "parse", 17 | "files", 18 | "utils", 19 | "migration" 20 | ], 21 | "author": "Florent Vilmart", 22 | "license": "MIT", 23 | "bugs": { 24 | "url": "https://github.com/parse-server-modules/parse-files-utils/issues" 25 | }, 26 | "homepage": "https://github.com/parse-server-modules/parse-files-utils#readme", 27 | "dependencies": { 28 | "async": "^2.0.0", 29 | "inquirer": "^1.1.2", 30 | "mongodb": "^2.2.4", 31 | "parse": "^1.8.5", 32 | "parse-server-azure-storage": "^1.1.0", 33 | "parse-server-fs-adapter": "^1.0.0", 34 | "parse-server-gcs-adapter": "^1.0.0", 35 | "parse-server-s3-adapter": "^1.0.4", 36 | "request": "^2.72.0" 37 | }, 38 | "devDependencies": { 39 | "istanbul": "^0.4.4" 40 | } 41 | } 42 | --------------------------------------------------------------------------------