├── filelist.txt ├── .gitattributes ├── rclone.conf ├── LICENSE ├── README.md └── app.js /filelist.txt: -------------------------------------------------------------------------------- 1 | 1N7rmP_1y4eo8bc75muJQPHXss-GgR1ja===Rick Roll Troll -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /rclone.conf: -------------------------------------------------------------------------------- 1 | [gdrive] 2 | type = drive 3 | scope = drive 4 | token = {"access_token":"xxxxxx","token_type":"xxxxx","refresh_token":"xxxxx","expiry":"xxxxx"} -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 axzxc1236 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Rclone Gdrive public folder automation 2 | 3 | ## Why this is created 4 | I need to copy many publically shared Google Drive folder with rclone, you might know there is a copy function on Google Drive webpage, why not use that? 5 | 1. I might want to copy these folders to my computer. 6 | 2. I might want to copy these folders to my Google Drive but encrypted. 7 | 3. I might want to copy these folders to another cloud storage providers. (for example Dropbox, FTP Server...) 8 | 9 | ## How to use it? 10 | 1. Install [Node.js](https://nodejs.org/en/download/) 11 | 2. replace rclone.conf file with working one. (use "rclone config file" command to find where it's located.) 12 | 3. Make sure rclone.conf starts with a drive remote, or at least conrains a drive remote, with a working token. 13 | 4. modify filelist.txt with following format (can be a multiline file) 14 | > (public folder ID)===(destination) 15 | 16 | > for example [https://drive.google.com/drive/folders/1N7rmP_1y4eo8bc75muJQPHXss-GgR1ja](https://drive.google.com/drive/folders/1N7rmP_1y4eo8bc75muJQPHXss-GgR1ja) has the folder ID 1N7rmP_1y4eo8bc75muJQPHXss-GgR1ja 17 | > 18 | > (If you are wondering what is the file I linked, it's [a known troll file](https://www.reddit.com/r/DHExchange/comments/ax4or0/psa_beware_of_this_individual_who_claims_to_have/) contains all the Rick rolls you need.) 19 | 20 | > destination can be whatever file path you put into rclone, like "encrypted:Rick roll troll" (encrypted remote must be in rclone.conf you provided for that to work) or just "Rick roll troll", **It doesn't need to have double quotes even if the path contains whitespace** (at least on Windows 10), it looks like Node.JS adds double quotes itself. 21 | 22 | 5. After you configured correctly, run "node app.js" and see if it works. 23 | 24 | ## Note 25 | 26 | If you don't have rclone(.exe) in your path environment variable, you need to change "rcloneCommand" in app.js to something like "C:/rclone/rclone.exe" or wherever rclone is located. 27 | 28 | **Sometimes you will get error messages in rclone and.... it might be fine**, there is [a issue that requires better fix but now have workaround](https://github.com/axzxc1236/Rclone_Gdrive_public_folder_automation/issues/1). -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const fs = require("fs"); 4 | const { spawn } = require('child_process'); 5 | const { EOL } = require('os'); 6 | 7 | const filelist = "filelist.txt" 8 | const original_config = "rclone.conf"; 9 | const modified_config = "rclone_modified.conf"; 10 | const rcloneCommand = "rclone"; //If you are using Windows and rclone.exe is not in path, you want to change that to something like "C:/rclone/rclone.exe" 11 | const writeDebugFile = true; 12 | const debug_file_filename = "debug_file"; 13 | const amounts_of_retries_before_giveup = 1; //It shouldn't need to be larger than 1 14 | 15 | var token = null; 16 | var token_expiry_time = null; 17 | 18 | function readTokenFromOriginalConfig() { 19 | try { 20 | let configContent = fs.readFileSync(original_config, { encoding: "utf-8" }).split(/\r?\n/); 21 | for (let i = 0; i < configContent.length; i++) { 22 | if (configContent[i].startsWith("token = ")) { 23 | token = configContent[i].replace("token = ", ""); 24 | token_expiry_time = Date.parse(JSON.parse(token).expiry) 25 | console.log("token: " + token); 26 | console.log("token expiry time: " + token_expiry_time); 27 | //Debug 28 | if (writeDebugFile) 29 | fs.appendFileSync(debug_file_filename, Date() + " " + token + EOL); 30 | // 31 | } 32 | } 33 | } catch (e) { 34 | console.log("encountered error trying to read token: " + e); 35 | } 36 | 37 | if (token != null) { 38 | //starts downloading 39 | readFileList(); 40 | if (!errorParsingFileList && folderIDList.length != 0) { 41 | console.log("Good to go."); 42 | downloadFile(0); 43 | } 44 | } 45 | } 46 | 47 | var folderIDList = []; 48 | var destinationList = []; 49 | var errorParsingFileList = false; 50 | function readFileList() { 51 | try { 52 | let fileListContent = fs.readFileSync(filelist, { encoding: "utf-8" }).split(/\r?\n/); 53 | for (let i = 0; i < fileListContent.length; i++) { 54 | if (fileListContent[i] != "") { 55 | let array = fileListContent[i].split("==="); 56 | if (array.length != 2) { 57 | console.log("error parsing this line: " + fileListContent[i]); 58 | errorParsingFileList = true; 59 | } else { 60 | folderIDList.push(array[0]); 61 | destinationList.push(array[1]); 62 | } 63 | } 64 | } 65 | } catch (e) { 66 | console.log("encountered error trying to read filelist: " + e); 67 | errorParsingFileList = true; 68 | } 69 | } 70 | 71 | var downloadRetries = 0; 72 | function downloadFile(index) { 73 | console.log("Copying Folder " + folderIDList[index] + " to " + destinationList[index]); 74 | //copy a config file with different folder ID 75 | if (fs.existsSync(modified_config)) 76 | fs.unlinkSync(modified_config) 77 | fs.copyFileSync(original_config, modified_config); 78 | fs.appendFileSync(modified_config, 79 | EOL + "[tmp]" + EOL + 80 | "type = drive" + EOL + 81 | "scope = drive" + EOL + 82 | "root_folder_id = " + folderIDList[index] + EOL + 83 | "token = " + token + EOL); 84 | // 85 | 86 | //spawn rclone process 87 | const rclone = spawn(rcloneCommand, ["--config", modified_config, "-P", "copy", "tmp:", destinationList[index]], { stdio: 'inherit' }); 88 | 89 | rclone.on('close', (code) => { 90 | readTokensFromModifiedConfig(); 91 | console.log("child process exited with code" + code); 92 | if (code != 0) { 93 | if (downloadRetries < amounts_of_retries_before_giveup) { 94 | //Why retry is needed and what is this different from retry built into rclone: 95 | //When rclone updates the token to your Google Drive remote, it didn't update tmp remote's token 96 | //so you will get error messages because it can't read public shared folder... 97 | //need to re-copy your Google Drive token to tmp remote. 98 | downloadRetries++; 99 | downloadFile(index); 100 | } else { 101 | console.log("encountered an error, not continuing."); 102 | } 103 | } else if (index+1 == folderIDList.length) { 104 | console.log("I think we are done."); 105 | } else { 106 | downloadRetries = 0; 107 | downloadFile(index+1); 108 | } 109 | }); 110 | // 111 | } 112 | 113 | function readTokensFromModifiedConfig() { 114 | //This function is used to find renewed token saved in modified config. 115 | //When rclone is running, it might renew a new token and save it in modified_config 116 | //, but modified_config is going to be replaced with content from original_config 117 | //, so we need to find and save the new token. 118 | try { 119 | let modifiedConfigContent = fs.readFileSync(modified_config, { encoding: "utf-8" }).split(/\r?\n/); 120 | for (let i = 0; i < modifiedConfigContent.length; i++) { 121 | if (modifiedConfigContent[i].startsWith("token = ")) { 122 | let tmp_token = modifiedConfigContent[i].replace("token = ", ""); 123 | let tmp_token_expiry_time = Date.parse(JSON.parse(tmp_token).expiry) 124 | if (token != tmp_token && tmp_token_expiry_time > token_expiry_time) { 125 | console.log("I found a new token: " + tmp_token); 126 | console.log("old token is: " + token); 127 | console.log("new token expiry time: " + tmp_token_expiry_time); 128 | //Debug 129 | if (writeDebugFile) 130 | fs.appendFileSync(debug_file_filename, 131 | Date() + " old token is " + token + EOL + 132 | Date() + " new token is " + tmp_token + EOL); 133 | // 134 | //Modify original config to replace token with new one. 135 | let originalConfigFileContent = fs.readFileSync(original_config, { encoding: "utf-8" }); 136 | originalConfigFileContent = originalConfigFileContent.replace(token, tmp_token); 137 | fs.writeFileSync(original_config, originalConfigFileContent); 138 | // 139 | token = tmp_token; 140 | } 141 | } 142 | } 143 | } catch (e) { 144 | console.log("encountered error trying to read token: " + e); 145 | } 146 | } 147 | 148 | //Try to read new token before doing anything incase the script terminated with error last time. 149 | //if (fs.existsSync(modified_config)) 150 | // readTokensFromModifiedConfig(); 151 | readTokenFromOriginalConfig(); //This function should be called once and only once when you start the script. 152 | --------------------------------------------------------------------------------