├── .gitignore ├── .gitpod.dockerfile ├── .gitpod.yml ├── .vscode ├── MAINTAINERS.md ├── arctictern.py ├── client.cnf ├── font_fix.py ├── heroku_config.sh ├── init_tasks.sh ├── launch.json ├── make_url.py ├── mysql.cnf ├── rmdep.sh ├── settings.json ├── start_mysql.sh ├── upgrades.json ├── uptime.sh └── version.txt ├── README.md └── reel2reel.py /.gitignore: -------------------------------------------------------------------------------- 1 | core.Microsoft* 2 | core.mongo* 3 | core.python* 4 | env.py 5 | __pycache__/ 6 | *.py[cod] 7 | node_modules/ 8 | .github/ 9 | cloudinary_python.txt 10 | -------------------------------------------------------------------------------- /.gitpod.dockerfile: -------------------------------------------------------------------------------- 1 | FROM gitpod/workspace-base:latest 2 | 3 | RUN echo "CI version from base" 4 | 5 | ### NodeJS ### 6 | USER gitpod 7 | ENV NODE_VERSION=16.13.0 8 | ENV TRIGGER_REBUILD=1 9 | RUN curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | PROFILE=/dev/null bash \ 10 | && bash -c ". .nvm/nvm.sh \ 11 | && nvm install $NODE_VERSION \ 12 | && nvm alias default $NODE_VERSION \ 13 | && npm install -g typescript yarn node-gyp" \ 14 | && echo ". ~/.nvm/nvm.sh" >> /home/gitpod/.bashrc.d/50-node 15 | ENV PATH=$PATH:/home/gitpod/.nvm/versions/node/v${NODE_VERSION}/bin 16 | 17 | ### Python ### 18 | USER gitpod 19 | RUN sudo install-packages python3-pip 20 | 21 | ENV PATH=$HOME/.pyenv/bin:$HOME/.pyenv/shims:$PATH 22 | RUN curl -fsSL https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash \ 23 | && { echo; \ 24 | echo 'eval "$(pyenv init -)"'; \ 25 | echo 'eval "$(pyenv virtualenv-init -)"'; } >> /home/gitpod/.bashrc.d/60-python \ 26 | && pyenv update \ 27 | && pyenv install 3.8.11 \ 28 | && pyenv global 3.8.11 \ 29 | && python3 -m pip install --no-cache-dir --upgrade pip \ 30 | && python3 -m pip install --no-cache-dir --upgrade \ 31 | setuptools wheel virtualenv pipenv pylint rope flake8 \ 32 | mypy autopep8 pep8 pylama pydocstyle bandit notebook \ 33 | twine \ 34 | && sudo rm -rf /tmp/*USER gitpod 35 | ENV PYTHONUSERBASE=/workspace/.pip-modules \ 36 | PIP_USER=yes 37 | ENV PATH=$PYTHONUSERBASE/bin:$PATH 38 | 39 | # Setup Heroku CLI 40 | RUN curl https://cli-assets.heroku.com/install.sh | sh 41 | 42 | # Setup PostgreSQL 43 | 44 | RUN sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list' && \ 45 | sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 && \ 46 | sudo apt-get update -y && \ 47 | sudo apt-get install -y postgresql-16 48 | 49 | ENV PGDATA="/workspace/.pgsql/data" 50 | 51 | RUN mkdir -p ~/.pg_ctl/bin ~/.pg_ctl/sockets \ 52 | && echo '#!/bin/bash\n[ ! -d $PGDATA ] && mkdir -p $PGDATA && initdb --auth=trust -D $PGDATA\npg_ctl -D $PGDATA -l ~/.pg_ctl/log -o "-k ~/.pg_ctl/sockets" start\n' > ~/.pg_ctl/bin/pg_start \ 53 | && echo '#!/bin/bash\npg_ctl -D $PGDATA -l ~/.pg_ctl/log -o "-k ~/.pg_ctl/sockets" stop\n' > ~/.pg_ctl/bin/pg_stop \ 54 | && chmod +x ~/.pg_ctl/bin/* 55 | 56 | ENV PGDATABASE="postgres" 57 | 58 | ENV PATH="/usr/lib/postgresql/14/bin:/home/gitpod/.nvm/versions/node/v${NODE_VERSION}/bin:$HOME/.pg_ctl/bin:$PATH" 59 | 60 | 61 | # Add aliases 62 | 63 | RUN echo 'alias run="python3 $GITPOD_REPO_ROOT/manage.py runserver 0.0.0.0:8000"' >> ~/.bashrc && \ 64 | echo 'alias heroku_config=". $GITPOD_REPO_ROOT/.vscode/heroku_config.sh"' >> ~/.bashrc && \ 65 | echo 'alias python=python3' >> ~/.bashrc && \ 66 | echo 'alias pip=pip3' >> ~/.bashrc && \ 67 | echo 'alias arctictern="python3 $GITPOD_REPO_ROOT/.vscode/arctictern.py"' >> ~/.bashrc && \ 68 | echo 'alias reel2reel="python3 $GITPOD_REPO_ROOT/reel2reel.py"' >> ~/.bashrc && \ 69 | echo 'alias font_fix="python3 $GITPOD_REPO_ROOT/.vscode/font_fix.py"' >> ~/.bashrc && \ 70 | echo 'alias set_pg="export PGHOSTADDR=127.0.0.1"' >> ~/.bashrc && \ 71 | echo 'alias make_url="python3 $GITPOD_REPO_ROOT/.vscode/make_url.py "' >> ~/.bashrc && \ 72 | echo 'FILE="$GITPOD_REPO_ROOT/.vscode/post_upgrade.sh"' >> ~/.bashrc && \ 73 | echo 'if [ -z "$POST_UPGRADE_RUN" ]; then' >> ~/.bashrc && \ 74 | echo ' if [[ -f "$FILE" ]]; then' >> ~/.bashrc && \ 75 | echo ' . "$GITPOD_REPO_ROOT/.vscode/post_upgrade.sh"' >> ~/.bashrc && \ 76 | echo " fi" >> ~/.bashrc && \ 77 | echo "fi" >> ~/.bashrc 78 | 79 | # Local environment variables 80 | 81 | ENV PORT="8080" 82 | ENV IP="0.0.0.0" 83 | -------------------------------------------------------------------------------- /.gitpod.yml: -------------------------------------------------------------------------------- 1 | image: 2 | file: .gitpod.dockerfile 3 | tasks: 4 | - init: . ${GITPOD_REPO_ROOT}/.vscode/init_tasks.sh 5 | command: /home/gitpod/.pg_ctl/bin/pg_start > /dev/null 6 | - command: . ${GITPOD_REPO_ROOT}/.vscode/uptime.sh & 7 | vscode: 8 | extensions: 9 | - ms-python.python 10 | - formulahendry.auto-close-tag 11 | - eventyret.bootstrap-4-cdn-snippet 12 | - hookyqr.beautify 13 | - matt-rudge.auto-open-preview-panel 14 | - ms-toolsai.jupyter 15 | - ms-toolsai.jupyter-keymap 16 | - ms-toolsai.jupyter-renderers 17 | -------------------------------------------------------------------------------- /.vscode/MAINTAINERS.md: -------------------------------------------------------------------------------- 1 | # Gitpod Template Maintainer’s Guide 2 | 3 | Revision Date: 24th September, 2021 4 | 5 | Author(s): Matt Rudge 6 | 7 | 8 | ## Overview 9 | 10 | As of 23rd September, 2021, the [Gitpod Full Template](https://github.com/Code-Institute-Org/gitpod-full-template) now has versioning features. Versioning capability is provided by the following files, all in the .vscode directory: 11 | 12 | `arctictern.py` - the migration tool to allow the student’s workspace to be updated 13 | 14 | `upgrades.json` - a JSON file containing the changes made since the template was last updated 15 | 16 | `version.txt` - a text file containing the current version number 17 | 18 | 19 | ## File formats 20 | 21 | The upgrades.json file has the following format: 22 | 23 | Key: Version number as a string 24 | 25 | Value: Bash commands to apply the changes made, separated by \n for newlines 26 | 27 | For example, in an earlier version of the template, we removed the `DATABASE_URL` and `PGHOSTADDR` environment variables. To apply these changes to an existing workspace, here is the content of the upgrades.json file: 28 | 29 | ```json 30 | { 31 | 32 | "1.210831": "echo 'unset DATABASE_URL' >> ~/.bashrc\necho 'unset PGHOSTADDR' >> ~/.bashrc" 33 | 34 | } 35 | ``` 36 | 37 | The key is the version number, in this case 1.210831. The value is the commands that will be run. In this case, we are appending two `unset` commands to the `.bashrc` file. Each command should be separated by \n to indicate a newline. 38 | 39 | The version.txt file is a simple text file with one line, which is the version number. 40 | 41 | 42 | ## Version numbering 43 | 44 | Currently, for ease, we are using the major version number 1. The minor version number is the date that the template was changed in YYMMDD format. So, the current version of the template is 1.210923 because it was last modified on September 23rd, 2021. 45 | 46 | 47 | ## Arctic Tern workflow 48 | 49 | Students can run the Arctic Tern migration tool by simply typing arctictern in the terminal. 50 | 51 | When Arctic Tern is run, it performs the following actions: 52 | 53 | 1. Retrieve updated versions of the main files 54 | 2. Check to see if the version number of the template in the student’s workspace is lower than the template’s current version on GitHub 55 | 3. If it is, retrieve and parse the `upgrades.json` file 56 | 4. If the version key in the `upgrades.json` file is greater than the student’s template version then add the commands in the value to the `post_upgrade.sh` file 57 | 58 | When Arctic Tern has finished, the student should add, commit and push the changes to GitHub. If they want the changes to take effect immediately, then they should restart their workspace. 59 | 60 | If the `post_upgrade.sh` file exists in the `.vscode` directory, then it will be run once when the workspace is started. 61 | 62 | 63 | ## Updates for students running prior versions of the template 64 | 65 | Because of a current bug in how Gitpod parses the `.gitpod.yml` file, students who are using a version of the template older than 1.210923 (which is the vast majority) will not be able to take advantage of these features without recreating their workspace. 66 | 67 | The workaround for them is to: 68 | 69 | 1. Download and run `arctictern.py` in their current workspace. 70 | 2. Add, commit and push to GitHub. 71 | 3. Open the workspace using the green Gitpod button from GitHub 72 | 4. Manually copy over any env.py or SQLite databases from the old workspace 73 | 74 | 75 | ## Process for adding changes to the template 76 | 77 | If you need to update or change the template, then please follow this procedure: 78 | 79 | 80 | ### Fork the template 81 | 82 | Create a fork in your own GitHub repository and work from there. Do not work on the template directly in the Org repo. 83 | 84 | 85 | ### Determine the changes that need to be made 86 | 87 | In determining the changes that need to be made, please keep in mind which files will need to be adjusted. 88 | 89 | Some files do not need to have their changes reflected in the upgrades.json file. Use the following table to decide: 90 | 91 | 92 | 93 | 94 | 96 | 100 | 101 | 102 | 104 | 106 | 107 | 108 | 110 | 112 | 113 | 114 | 116 | 118 | 119 | 120 | 122 | 124 | 125 | 126 | 128 | 130 | 131 | 132 | 134 | 136 | 137 | 138 | 140 | 142 | 143 | 144 | 146 | 148 | 149 | 150 | 152 | 154 | 155 | 156 | 158 | 160 | 161 | 162 | 164 | 166 | 167 | 168 | 170 | 172 | 173 | 174 | 176 | 178 | 179 | 180 | 182 | 184 | 185 | 186 | 188 | 190 | 191 | 192 | 194 | 196 | 197 |
Filename 95 | Changes to be 97 |

98 | reflected in upgrade.json? 99 |

.gitignore 103 | No 105 |
.gitpod.yml 109 | No 111 |
.gitpod.dockerfile 115 | Yes 117 |
README.md 121 | No 123 |
.vscode/arctictern.py 127 | No 129 |
.vscode/client.cnf 133 | Yes 135 |
.vscode/font_fix.py 139 | No 141 |
.vscode/heroku_config.sh 145 | No 147 |
.vscode/init_tasks.sh 151 | Yes 153 |
.vscode/launch.json 157 | No 159 |
.vscode/mysql.cnf 163 | Yes 165 |
.vscode/settings.json 169 | No 171 |
.vscode/start_mysql.sh 175 | Yes 177 |
.vscode/upgrades.json 181 | No 183 |
.vscode/uptime.sh 187 | No 189 |
.vscode/version.txt 193 | No 195 |
198 | 199 | 200 | 201 | ### Make and test your changes 202 | 203 | Do this in your local repo and confirm that everything works as expected. 204 | 205 | 206 | ### Change the version number in version.txt 207 | 208 | Update the version number using the format specified above. 209 | 210 | 211 | ### Update the upgrades.json file if necessary 212 | 213 | If the `upgrades.json` file does need to be updated, then add a key at the end of the dictionary with the version number. 214 | 215 | The value should be the commands that will be run at the end of `.bashrc` to apply your changes. Bear in mind that these commands may be different to the ones you have added to the other files. 216 | 217 | 218 | ### Update the README.md file 219 | 220 | Add the updated date, version and any changes that have been made to the change log. Note that if you have opened your forked template in Gitpod then it will have substituted USER_NAME at the top of the README file with your GitHub username, so change it back to USER_NAME before committing. 221 | 222 | 223 | ### Create a PR 224 | 225 | Create a pull request against the original repo and assign at least one reviewer. 226 | 227 | 228 | ### And relax... 229 | -------------------------------------------------------------------------------- /.vscode/arctictern.py: -------------------------------------------------------------------------------- 1 | """ 2 | arctictern.py 3 | A little script that does a big migration 4 | """ 5 | 6 | import json 7 | import os 8 | import requests 9 | import shutil 10 | import subprocess 11 | import sys 12 | from os.path import exists 13 | 14 | BASE_URL = "https://raw.githubusercontent.com/Code-Institute-Org/gitpod-full-template/master/" 15 | 16 | BACKUP = True 17 | MIGRATE = False 18 | CURRENT_VERSION = 1.0 19 | THIS_VERSION = 1.0 20 | 21 | 22 | MIGRATE_FILE_LIST = [{"filename": ".theia/settings.json", 23 | "url": ".vscode/settings.json" 24 | }, 25 | {"filename": ".gitpod.yml", 26 | "url": ".gitpod.yml" 27 | }, 28 | {"filename": ".gitpod.dockerfile", 29 | "url": ".gitpod.dockerfile" 30 | }, 31 | {"filename": ".theia/heroku_config.sh", 32 | "url": ".vscode/heroku_config.sh" 33 | }, 34 | {"filename": ".theia/uptime.sh", 35 | "url": ".vscode/uptime.sh" 36 | }, 37 | {"filename": ".theia/init_tasks.sh", 38 | "url": ".vscode/init_tasks.sh" 39 | }] 40 | 41 | UPGRADE_FILE_LIST = [{"filename": ".vscode/client.cnf", 42 | "url": ".vscode/client.cnf" 43 | }, 44 | {"filename": ".vscode/mysql.cnf", 45 | "url": ".vscode/mysql.cnf" 46 | }, 47 | {"filename": ".vscode/settings.json", 48 | "url": ".vscode/settings.json" 49 | }, 50 | {"filename": ".vscode/launch.json", 51 | "url": ".vscode/launch.json" 52 | }, 53 | {"filename": ".gitpod.yml", 54 | "url": ".gitpod.yml" 55 | }, 56 | {"filename": ".gitpod.dockerfile", 57 | "url": ".gitpod.dockerfile" 58 | }, 59 | {"filename": ".vscode/heroku_config.sh", 60 | "url": ".vscode/heroku_config.sh" 61 | }, 62 | {"filename": ".vscode/init_tasks.sh", 63 | "url": ".vscode/init_tasks.sh" 64 | }, 65 | {"filename": ".vscode/uptime.sh", 66 | "url": ".vscode/uptime.sh" 67 | }, 68 | {"filename": ".vscode/make_url.py", 69 | "url": ".vscode/make_url.py" 70 | }, 71 | {"filename": ".vscode/arctictern.py", 72 | "url": ".vscode/arctictern.py" 73 | }] 74 | 75 | FINAL_LINES = "\nexport POST_UPGRADE_RUN=1\nsource ~/.bashrc\n" 76 | 77 | 78 | def needs_upgrade(): 79 | """ 80 | Checks the version of the current template against 81 | this version. 82 | Returns True if upgrade is needed, False if not. 83 | """ 84 | 85 | if exists(".vscode/version.txt"): 86 | with open(".vscode/version.txt", "r") as f: 87 | THIS_VERSION = float(f.read().strip()) 88 | else: 89 | THIS_VERSION = 1.0 90 | with open(".vscode/version.txt", "w") as f: 91 | f.write(str(THIS_VERSION)) 92 | 93 | r = requests.get(BASE_URL + ".vscode/version.txt") 94 | CURRENT_VERSION = float(r.content) 95 | print(f"Upstream version: {CURRENT_VERSION}") 96 | print(f"Local version: {THIS_VERSION}") 97 | 98 | return CURRENT_VERSION > THIS_VERSION 99 | 100 | 101 | def build_post_upgrade(): 102 | 103 | r = requests.get(BASE_URL + ".vscode/upgrades.json") 104 | upgrades = json.loads(r.content.decode("utf-8")) 105 | content = "" 106 | 107 | for k,v in upgrades.items(): 108 | if float(k) > THIS_VERSION: 109 | print(f"Adding version changes for {k} to post_upgrade.sh") 110 | content += v 111 | 112 | if content: 113 | content += FINAL_LINES 114 | with open(".vscode/post_upgrade.sh", "w") as f: 115 | f.writelines(content) 116 | 117 | print("Built post_upgrade.sh. Restart your workspace for it to take effect") 118 | 119 | 120 | def process(file, suffix): 121 | """ 122 | Replaces and optionally backs up the files that 123 | need to be changed. 124 | Arguments: file - a path and filename 125 | suffix - the suffix to the BASE_URL 126 | """ 127 | 128 | if BACKUP: 129 | try: 130 | shutil.copyfile(file, f"{file}.bak") 131 | except FileNotFoundError: 132 | print(f"{file} not found, a new one will be created") 133 | 134 | with open(file, "wb") as f: 135 | r = requests.get(BASE_URL + suffix) 136 | f.write(r.content) 137 | 138 | 139 | def start_migration(): 140 | """ 141 | Calls the process function and 142 | renames the directory 143 | """ 144 | if not os.path.isdir(".theia") and MIGRATE: 145 | sys.exit("The .theia directory does not exist") 146 | 147 | FILE_LIST = MIGRATE_FILE_LIST if MIGRATE else UPGRADE_FILE_LIST 148 | 149 | if not MIGRATE and not os.path.isdir(".vscode"): 150 | print("Creating .vscode directory") 151 | os.mkdir(".vscode") 152 | 153 | for file in FILE_LIST: 154 | print(f"Processing: {file['filename']}") 155 | process(file["filename"], file["url"]) 156 | 157 | if MIGRATE and os.path.isdir(".vscode"): 158 | print(".vscode directory already exists") 159 | if input("Overwrite? Y/N ").lower() == "y": 160 | shutil.rmtree(".vscode") 161 | else: 162 | print("You will need to manually remove the .theia directory after migration.") 163 | 164 | if MIGRATE and not os.path.isdir(".vscode"): 165 | print("Renaming directory") 166 | os.rename(".theia", ".vscode") 167 | 168 | if not MIGRATE and needs_upgrade(): 169 | build_post_upgrade() 170 | 171 | print("Changes saved.") 172 | print("Please add, commit and push to GitHub.") 173 | print("You may need to stop and restart your workspace for") 174 | print("the changes to take effect.") 175 | 176 | 177 | if __name__ == "__main__": 178 | 179 | BACKUP = "--nobackup" not in sys.argv 180 | MIGRATE = "--migrate" in sys.argv 181 | 182 | print("CI Template Migration Utility 0.2") 183 | print("---------------------------------") 184 | print("The default action is to upgrade the workspace to the latest version.") 185 | print(f"Usage: python3 {sys.argv[0]} [--nobackup --migrate]") 186 | 187 | if not BACKUP: 188 | print("If the --nobackup switch is provided, then changed files will not be backed up.") 189 | if not MIGRATE: 190 | print("If the --migrate switch is provided, the repo will be migrated from Theia to VS Code") 191 | 192 | print() 193 | 194 | if input("Start? Y/N ").lower() == "y": 195 | start_migration() 196 | else: 197 | sys.exit("Migration cancelled by the user") 198 | -------------------------------------------------------------------------------- /.vscode/client.cnf: -------------------------------------------------------------------------------- 1 | [client] 2 | host = localhost 3 | user = root 4 | password = 5 | socket = /var/run/mysqld/mysqld.sock 6 | [mysql_upgrade] 7 | host = localhost 8 | user = root 9 | password = 10 | socket = /var/run/mysqld/mysqld.sock 11 | -------------------------------------------------------------------------------- /.vscode/font_fix.py: -------------------------------------------------------------------------------- 1 | # Fixes the font issue on Brave browser 2 | # Matt Rudge 3 | # August 2021 4 | 5 | import json 6 | import os 7 | 8 | BASE_PATH = os.environ.get("GITPOD_REPO_ROOT") 9 | 10 | with open(f"{BASE_PATH}/.vscode/settings.json", "r+") as f: 11 | content = json.loads(f.read()) 12 | 13 | if "terminal.integrated.fontFamily" not in content: 14 | print("Adding wider and higher font settings") 15 | content["terminal.integrated.lineHeight"] = 1.2 16 | content["terminal.integrated.letterSpacing"] = 2 17 | else: 18 | print("Wider and higher font settings already added!") 19 | 20 | f.seek(0, os.SEEK_SET) 21 | f.write(json.dumps(content)) 22 | f.truncate() 23 | -------------------------------------------------------------------------------- /.vscode/heroku_config.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Script to allow Heroku API key to be pasted 3 | # exported as an environment variable 4 | # 5 | # Matt Rudge, May 2021 6 | 7 | echo Heroku authentication configuration script 8 | echo Code Institute, 2021 9 | echo 10 | echo Get your Heroku API key by going to https://dashboard.heroku.com 11 | echo Go to Account Settings and click on Reveal to view your Heroku API key 12 | echo 13 | 14 | if [[ -z "${HEROKU_API_KEY}" ]]; then 15 | echo Paste your Heroku API key here or press Enter to quit: 16 | read apikey 17 | if [[ -z "${apikey}" ]]; then 18 | return 0 19 | fi 20 | echo export HEROKU_API_KEY=${apikey} >> ~/.bashrc 21 | echo Added the export. Refreshing the terminal. 22 | . ~/.bashrc > /dev/null 23 | echo Done! 24 | else 25 | echo API key is already set. Exiting 26 | fi 27 | -------------------------------------------------------------------------------- /.vscode/init_tasks.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Creates a user record for the current Cloud9 user 4 | # Gives a personalised greeting 5 | # Adds configuration options for SQLite 6 | # Creates run aliases 7 | # Author: Matt Rudge 8 | 9 | echo "Setting the greeting" 10 | sed -i "s/USER_NAME/$GITPOD_GIT_USER_NAME/g" ${GITPOD_REPO_ROOT}/README.md 11 | echo "Creating the gitpod user in MySQL" 12 | RESULT="$(mysql -sse "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = 'gitpod')")" 13 | if [ "$RESULT" = 1 ]; then 14 | echo "gitpod already exists" 15 | else 16 | mysql -e "CREATE USER 'gitpod'@'%' IDENTIFIED BY '';" -u root 17 | echo "Granting privileges" 18 | mysql -e "GRANT ALL PRIVILEGES ON *.* TO 'gitpod'@'%' WITH GRANT OPTION;" -u root 19 | fi 20 | echo "Creating .sqliterc file" 21 | echo ".headers on" > ~/.sqliterc 22 | echo ".mode column" >> ~/.sqliterc 23 | echo "Your workspace is ready to use. Happy coding!" 24 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | "version": "0.2.0", 5 | "configurations": [ 6 | { 7 | "name": "Python: Current File (Integrated Terminal)", 8 | "type": "python", 9 | "request": "launch", 10 | "program": "${file}", 11 | "console": "internalConsole" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /.vscode/make_url.py: -------------------------------------------------------------------------------- 1 | # Simple utility for creating the Cloudinary URL from a 2 | # cloudinary_python.txt file 3 | # Matt Rudge, November 2021 4 | 5 | import re 6 | 7 | with open("cloudinary_python.txt") as f: 8 | content = f.readlines() 9 | 10 | cloud_name = re.findall(r"['](.*?)[']",content[15])[0] 11 | api_key = re.findall(r"['](.*?)[']",content[16])[0] 12 | api_secret = re.findall(r"['](.*?)[']",content[17])[0] 13 | 14 | print(f"cloudinary://{api_key}:{api_secret}@{cloud_name}") 15 | -------------------------------------------------------------------------------- /.vscode/mysql.cnf: -------------------------------------------------------------------------------- 1 | [mysqld_safe] 2 | socket = /var/run/mysqld/mysqld.sock 3 | nice = 0 4 | 5 | [mysqld] 6 | user = gitpod 7 | pid-file = /var/run/mysqld/mysqld.pid 8 | socket = /var/run/mysqld/mysqld.sock 9 | port = 3306 10 | basedir = /usr 11 | datadir = /workspace/mysql 12 | tmpdir = /tmp 13 | lc-messages-dir = /usr/share/mysql 14 | skip-external-locking 15 | 16 | key_buffer_size = 16M 17 | max_allowed_packet = 16M 18 | thread_stack = 192K 19 | thread_cache_size = 8 20 | 21 | myisam-recover-options = BACKUP 22 | 23 | general_log_file = /var/log/mysql/mysql.log 24 | general_log = 1 25 | log_error = /var/log/mysql/error.log 26 | -------------------------------------------------------------------------------- /.vscode/rmdep.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "Removing all Python dependencies" 4 | pip3 uninstall -y -r <(pip3 freeze) > /dev/null 5 | echo "Done!" 6 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "python.linting.pylintEnabled": true, 3 | "python.linting.enabled": true, 4 | "python.linting.pep8Enabled": false, 5 | "python.linting.flake8Enabled": true, 6 | "python.terminal.activateEnvironment": false, 7 | "python.formatting.autopep8Path": "/home/gitpod/.pyenv/shims/autopep8", 8 | "python.linting.flake8Path": "/home/gitpod/.pyenv/shims/flake8", 9 | "cornflakes.linter.executablePath": "/home/gitpod/.pyenv/shims/flake8", 10 | "files.exclude": { 11 | "**/.DS_Store": true, 12 | "**/.git": true, 13 | "**/.github": true, 14 | "**/.gitp*": true, 15 | "**/.hg": true, 16 | "**/.svn": true, 17 | "**/.vscode": true, 18 | "**/core.Microsoft*": true, 19 | "**/core.mongo*": true, 20 | "**/core.python*": true, 21 | "**/CVS": true 22 | }, 23 | "files.autoSave": "off", 24 | "workbench.colorTheme": "Visual Studio Dark", 25 | "terminal.integrated.gpuAcceleration": "canvas", 26 | "editor.defaultFormatter": "HookyQR.beautify" 27 | } 28 | -------------------------------------------------------------------------------- /.vscode/start_mysql.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # this script is intended to be called from .bashrc 4 | # This is a workaround for not having something like supervisord 5 | 6 | if [ ! -e /var/run/mysqld/gitpod-init.lock ] 7 | then 8 | touch /var/run/mysqld/gitpod-init.lock 9 | 10 | # initialize database structures on disk, if needed 11 | [ ! -d /workspace/mysql ] && mysqld --initialize-insecure 12 | 13 | # launch database, if not running 14 | [ ! -e /var/run/mysqld/mysqld.pid ] && mysqld --daemonize 15 | 16 | rm /var/run/mysqld/gitpod-init.lock 17 | fi 18 | -------------------------------------------------------------------------------- /.vscode/upgrades.json: -------------------------------------------------------------------------------- 1 | { 2 | "1.210831": "echo 'unset DATABASE_URL' >> ~/.bashrc\necho 'unset PGHOSTADDR' >> ~/.bashrc\n", 3 | "1.211118": "echo 'alias make_url=\"python3 $GITPOD_REPO_ROOT/.vscode/make_url.py\"' >> ~/.bashrc\n", 4 | } 5 | -------------------------------------------------------------------------------- /.vscode/uptime.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Pings the webhook so that we can gather 4 | # basic usage stats. No personally identifiable 5 | # data is captured here, and it is impossible to 6 | # identify an individual user from the captured data. 7 | # Matt Rudge, April 2021 8 | 9 | UUID=$(cat /proc/sys/kernel/random/uuid) 10 | URL=https://1xthkmzwg3.execute-api.eu-west-1.amazonaws.com/prod/lrsapi/ 11 | API_KEY=jceBCdeGZP9RDeUNCfM4jIQ39Cx0jtG51QgcwDwc 12 | VERB="started" 13 | 14 | clear 15 | 16 | while true; do 17 | 18 | DATA="{\"activity_time\":\"$(date +%Y-%m-%dT%H:%M:%S).000Z\",\"actor\":\"${UUID}\",\"verb\":\"${VERB}\",\"activity_object\":\"Gitpod Workspace\",\"extra_data\":\"{}\"}" 19 | curl -s -X POST -H "x-api-key: ${API_KEY}" -d "${DATA}" ${URL} 1> /dev/null 20 | VERB="running" 21 | sleep 300 22 | 23 | done 24 | -------------------------------------------------------------------------------- /.vscode/version.txt: -------------------------------------------------------------------------------- 1 | 1.211203 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![CI logo](https://codeinstitute.s3.amazonaws.com/fullstack/ci_logo_small.png) 2 | 3 | # Reel2Reel 4 | 5 | ## What is it? 6 | 7 | This is a tool for Code Institute students and alumni. It allows you to dump a database from one Postgres server and upload it to another. 8 | 9 | ## How do I use it? 10 | 11 | You will need to have created a new database using our [Database Creator app](https://dbs.ci-dbs.net). 12 | 13 | 1. Open this repo in your chosen IDE 14 | 2. Run `python3 reel2reel.py` (Note: If you use Gitpod or Codeanywhere/Daytona, you can simply type `reel2reel`) 15 | 3. Paste in the database URL of the source database - the one you want to copy from. It will start with `postgres://` 16 | 4. Now paste in the database URL of the target database, probably the one created by the Database Creator app. Again, it will start with `postgres://` 17 | 5. The data will now be downloaded from the source database and uploaded to your new target database. 18 | 19 | ## Problems 20 | 21 | - Ensure that the URLs are copied correctly. Reel2Reel will throw an error if the URL doesn't start with `postgres://`. 22 | - If you are copying from a Heroky database instance then make sure that you run the process as soon as you copy the Heroku `DATABASE_URL`. Heroku periodically rotates the credentials, which means an old `DATABASE_URL` may not work. 23 | 24 | ## FAQs 25 | 26 | *Can I use this with VSCode?* 27 | 28 | Yes, you will need to install the latest versions of the Postgres command line client `psql`. 29 | 30 | *I'm a power user. Is there an easier way to use this?* 31 | 32 | Absolutely. You can supply the URLs as arguments like so: 33 | 34 | `python3 reel2reel.py ` 35 | 36 | *What does it actually do?* 37 | 38 | 1. Connects to the source database and dumps it to a local `dump.sql` file 39 | 2. Modifies the `dump.sql` file so that it contains the database name and username for the new database 40 | 3. Runs the modified `dump.sql` file against the specified destination database 41 | 42 | Feel free to examine the code and see if you understand how it works. 43 | 44 | ## Reel2Reel?? 45 | 46 | Because we like to move it, move it. 47 | 48 | --- 49 | 50 | Happy coding! 51 | -------------------------------------------------------------------------------- /reel2reel.py: -------------------------------------------------------------------------------- 1 | """ 2 | Reel2Reel - a script to move a database from one Postgres 3 | server to to another because we like to move it, move it. 4 | 5 | Usage: 6 | python3 reel2reel.py 7 | or interactively: 8 | python3 reel2reel.py 9 | 10 | Matt Rudge 11 | Code Institute 12 | October, 2022 13 | """ 14 | import os 15 | import re 16 | import sys 17 | from urllib.parse import urlparse 18 | 19 | 20 | def split_url(db_url): 21 | """ 22 | Performs checks on the URL for completeness and then 23 | parses it using urllib 24 | """ 25 | 26 | if db_url[0:11] != "postgres://" and db_url[0:13] != "postgresql://": 27 | print("Error: The URL seems incorrectly formatted.") 28 | sys.exit(1) 29 | 30 | e = urlparse(db_url) 31 | 32 | if not e.hostname or not e.username or not e.password or not e.path: 33 | print("Error: The URL seems incorrectly formatted.") 34 | sys.exit(1) 35 | 36 | return e 37 | 38 | 39 | def parse_dump(h_user, h_db, e_user, e_db): 40 | """ 41 | Takes the dumped SQL file and replaces the database details 42 | with the new user and database 43 | """ 44 | 45 | with open("dump.sql", "r", encoding="utf8") as f: 46 | 47 | data = f.read() 48 | 49 | data = re.sub(h_user, e_user, data) 50 | data = re.sub(h_db, e_db, data) 51 | 52 | with open("dump.sql", "w", encoding="utf8") as f: 53 | # Why not just open the file as r+ earlier and 54 | # f.seek(0) to the beginning? Because not all of the 55 | # file would get overwritten. 56 | f.write(data) 57 | 58 | 59 | def do_source(s): 60 | """ 61 | Performs the dump of the source data 62 | """ 63 | 64 | os.environ["PGPASSWORD"] = s.password 65 | 66 | print("Extracting the source database.") 67 | 68 | res = os.system(f"pg_dump --host={s.hostname} \ 69 | --username={s.username} --dbname={s.path[1:]} -w > dump.sql") 70 | 71 | if res != 0: 72 | print("Error: Cannot connect to source database server.") 73 | sys.exit(2) 74 | 75 | print("Extraction successful. File saved to dump.sql.") 76 | 77 | 78 | def do_dest(d): 79 | """ 80 | Uploads the modified data to the destination server 81 | """ 82 | 83 | print("Uploading to destination server") 84 | 85 | os.environ["PGPASSWORD"] = d.password 86 | 87 | res = os.system(f"psql --host={d.hostname} --username={d.username} \ 88 | --dbname={d.path[1:]} -w < dump.sql >/dev/null 2>&1") 89 | 90 | if res != 0: 91 | print("Error: Cannot upload the data to destination.") 92 | sys.exit(2) 93 | 94 | print("Upload complete. Please check your destination database.") 95 | 96 | 97 | def main(s_url, d_url): 98 | """ 99 | The main function. Calls other functions to perform the migration 100 | """ 101 | 102 | s = split_url(s_url) 103 | d = split_url(d_url) 104 | 105 | do_source(s) 106 | 107 | print("Modifying the downloaded data.") 108 | 109 | parse_dump(s.username, s.path[1:], d.username, d.path[1:]) 110 | 111 | do_dest(d) 112 | 113 | 114 | if __name__ == "__main__": 115 | print("Reel2Reel - PostgreSQL to PostgreSQL Mover") 116 | print("Code Institute, 2022\n") 117 | 118 | if len(sys.argv) == 2: 119 | print("You can supply the source and destination URLs as arguments") 120 | print("Usage: python3 reel2reel.py ") 121 | sys.exit(1) 122 | if len(sys.argv) > 1: 123 | source = sys.argv[1] 124 | dest = sys.argv[2] 125 | else: 126 | source = input("Paste your source DATABASE_URL here: ") 127 | dest = input("Paste your destination DATABASE_URL here: ") 128 | 129 | main(source, dest) 130 | --------------------------------------------------------------------------------