├── app ├── test │ ├── __init__.py │ ├── gmail_api_test.py │ ├── dropbox_api_test.py │ ├── sftp_to_go_ftp_test.py │ ├── ftp_api_test.py │ ├── sftp_to_go_s3_test.py │ ├── gitlab_api_test.py │ ├── s3_api_test.py │ └── github_api_test.py ├── gmail_api.py ├── dropbox_api.py ├── gitlab_api.py ├── ftp_api.py ├── github_api.py ├── sftp_to_go_ftp.py ├── sftp_to_go_s3.py ├── config.py ├── s3_api.py └── main.py ├── test.txt ├── files ├── file.json └── file2.json ├── .gitignore ├── requirements.txt ├── doc ├── DriveHQ.md ├── Gitlab.md ├── Gmail.md ├── Dropbox.md ├── S3.md └── Github.md ├── README.md ├── .github └── dependabot.yml └── LICENSE /app/test/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test.txt: -------------------------------------------------------------------------------- 1 | push from pygithub -------------------------------------------------------------------------------- /files/file.json: -------------------------------------------------------------------------------- 1 | {name: "Beppe", city: "Amsterdam"}; -------------------------------------------------------------------------------- /files/file2.json: -------------------------------------------------------------------------------- 1 | {name: "BeppeX", city: "Amsterdam"}; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | venv 3 | .idea 4 | __pycache__ 5 | *.bat 6 | docker-compose.yml 7 | docker-compose-local.yml 8 | .env 9 | tmp 10 | debug.log 11 | 12 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Flask==3.0.1 2 | requests==2.31.0 3 | python-dotenv===1.0.1 4 | PyGithub===2.1.1 5 | dropbox===11.36.2 6 | boto3==1.34.33 7 | pyftpdlib===1.5.9 8 | yagmail===0.15.293 9 | python-gitlab===4.4.0 10 | -------------------------------------------------------------------------------- /app/test/gmail_api_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from gmail_api import * 4 | 5 | 6 | class MailApiTest(unittest.TestCase): 7 | 8 | def test_upload(self): 9 | dest = "beppe@example.com" 10 | localfile= 'tmp/attach.txt' 11 | 12 | upload_file(dest, localfile) 13 | -------------------------------------------------------------------------------- /app/test/dropbox_api_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from dropbox_api import * 4 | 5 | 6 | class DropboxApiTest(unittest.TestCase): 7 | 8 | def test_download(self): 9 | filename = '/files/us-cities-demographics.csv' 10 | 11 | download_file(filename) 12 | 13 | def test_upload(self): 14 | filename = '/files/us-cities-demographics.csv' 15 | localfile= 'tmp/us-cities-demographics.csv' 16 | 17 | upload_file(filename, localfile) 18 | -------------------------------------------------------------------------------- /app/test/sftp_to_go_ftp_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from sftp_to_go_ftp import * 4 | 5 | 6 | class SftpToGoFtpTest(unittest.TestCase): 7 | 8 | def test_download(self): 9 | filename = 'doc/S3.md' 10 | tmpfile = '../tmp/S3.md' 11 | content = download_file(filename, tmpfile) 12 | 13 | print(content) 14 | 15 | def test_upload(self): 16 | filename = 'doc/Github.md' 17 | localfile = '../../doc/Github.md' 18 | 19 | upload_file(filename, localfile) 20 | -------------------------------------------------------------------------------- /app/test/ftp_api_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from ftp_api import * 4 | 5 | 6 | class FtpApiTest(unittest.TestCase): 7 | 8 | def test_download(self): 9 | filename = '/files/us-cities-demographics.csv' 10 | tmpfile = 'tmp/us-tmp.csv' 11 | 12 | content = download_file(filename, tmpfile) 13 | 14 | print(content) 15 | 16 | def test_upload(self): 17 | filename = 'doc/Github.md' 18 | localfile = '../../doc/Github.md' 19 | 20 | upload_file(filename, localfile) 21 | -------------------------------------------------------------------------------- /app/gmail_api.py: -------------------------------------------------------------------------------- 1 | import yagmail 2 | 3 | from config import * 4 | 5 | # Send file by email 6 | 7 | yag = yagmail.SMTP(get_gmail_username(), get_gmail_password()) 8 | 9 | 10 | def upload_file(recipient, attachment): 11 | """ 12 | Send email with file 13 | :param recipient: recipient of the email 14 | :param attachment: path of local file to upload 15 | :return: 16 | """ 17 | print(f"attachment {attachment}") 18 | 19 | contents = ['Body of email', attachment] 20 | 21 | yag.send(recipient, 'subject', contents) 22 | -------------------------------------------------------------------------------- /app/test/sftp_to_go_s3_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from sftp_to_go_s3 import * 4 | 5 | 6 | class SftpToGoS3Test(unittest.TestCase): 7 | 8 | def test_download(self): 9 | bucket = get_sftptogo_aws_bucket_name() 10 | key = 'doc/S3.md' 11 | filename = '../tmp/S3.md' 12 | 13 | download_object(bucket, key, filename) 14 | 15 | def test_upload(self): 16 | bucket = get_sftptogo_aws_bucket_name() 17 | key = 'doc/S3.md' 18 | filename = '../../doc/S3.md' 19 | 20 | upload_object(bucket, key, filename) 21 | 22 | -------------------------------------------------------------------------------- /app/test/gitlab_api_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from gitlab_api import * 4 | 5 | 6 | class GitlabApiTest(unittest.TestCase): 7 | 8 | def test_check_file_exist(self): 9 | seq = check_file_exist("files/config.json") 10 | 11 | self.assertTrue(seq) 12 | 13 | def test_check_file_does_not_exist(self): 14 | seq = check_file_exist("files/notfound.json") 15 | 16 | self.assertFalse(seq) 17 | 18 | def test_get_file(self): 19 | seq = get_file("files/config.json") 20 | 21 | self.assertIsNotNone(seq) 22 | 23 | def test_put_file(self): 24 | seq = put_file("files/file.json", "{name: \"Beppe\", city: \"Amsterdam\"};") 25 | -------------------------------------------------------------------------------- /app/test/s3_api_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from s3_api import * 4 | 5 | 6 | class S3ApiTest(unittest.TestCase): 7 | 8 | def test_download(self): 9 | bucket = 'beppe-udacity' 10 | key = 'capstone/us-cities-demographics.csv' 11 | filename = '../tmp/us-cities-demographics.csv' 12 | 13 | download_object(bucket, key, filename) 14 | 15 | def test_upload(self): 16 | bucket = 'beppe-udacity' 17 | key = 'doc/S3.md' 18 | filename = '../../doc/S3.md' 19 | 20 | upload_object(bucket, key, filename) 21 | 22 | def test_bucket_list(self): 23 | 24 | buckets = bucket_list() 25 | print(buckets) 26 | 27 | -------------------------------------------------------------------------------- /doc/DriveHQ.md: -------------------------------------------------------------------------------- 1 | # DriveHQ 2 | 3 | [DriveHQ](https://www.drivehq.com/) is a file server provider supporting FTP and WebDAV. It offers a 5 Free GB Basic Service. 4 | 5 | Using the Python `ftplib` module it is pretty simple to store and fetch files via FTP. 6 | 7 | Notes: 8 | * signup for a DriveHQ Basic Account 9 | * login with the same credentials using the Python api 10 | 11 | 12 | **Example** 13 | 14 | Checkout [sample code](https://github.com/gcatanese/HerokuFiles/tree/main/app/ftp_api.py) in ftp_api.py 15 | 16 | ## Deploy to Heroku 17 | 18 | Before deploying to Heroku make sure to define the AWS credentials as environment variables and create the corresponding Heroku Config Vars 19 | ``` 20 | heroku config:set FTP_HOST=ftp.drivehq.com 21 | heroku config:set FTP_USER=username 22 | heroku config:set FTP_PWD=password 23 | ``` 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /app/test/github_api_test.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from github_api import check_file_exist, get_file, put_file, get_user 4 | 5 | 6 | class GithubApiTest(unittest.TestCase): 7 | 8 | def test_check_file_exist(self): 9 | seq = check_file_exist("/files/file.json") 10 | 11 | self.assertTrue(seq) 12 | 13 | def test_check_file_does_not_exist(self): 14 | seq = check_file_exist("files/notfound.json") 15 | 16 | self.assertFalse(seq) 17 | 18 | def test_get_file(self): 19 | seq = get_file("files/file.json") 20 | 21 | self.assertIsNotNone(seq) 22 | 23 | def test_put_file(self): 24 | seq = put_file("files/file.json", "{name: \"Beppe\", city: \"Amsterdam\"};") 25 | 26 | def test_get_user(self): 27 | user = get_user("perosa") 28 | 29 | print(user.twitter_username) 30 | 31 | 32 | -------------------------------------------------------------------------------- /doc/Gitlab.md: -------------------------------------------------------------------------------- 1 | # Gitlab 2 | 3 | Store files on any of your Gitlab private or public repositories using [Python-Gitlab](https://python-gitlab.readthedocs.io/en/stable/index.html) 4 | 5 | ## Gitlab Personal Access Token 6 | 7 | Generate a Github Personal Access Token: go to `Account -> Settings -> Access Tokens -> Add Personal Access Token` 8 | 9 | Grant the token the necessary privileges (read/write into repositories) and **NEVER** show or share your tokens. 10 | 11 | 12 | ## Code Samples 13 | 14 | 15 | **Example** 16 | 17 | Checkout [sample code](https://github.com/gcatanese/HerokuFiles/tree/main/app/gitlab_api.py) in gitlab_api.py 18 | 19 | ## Deploy to Heroku 20 | 21 | Before deploying to Heroku make sure to define the Access Token as environment variable and create a corresponding Heroku Config Var 22 | ``` 23 | heroku config:set GITLAB_ACCESS_TOKEN=xyz) 24 | ``` 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /app/dropbox_api.py: -------------------------------------------------------------------------------- 1 | import dropbox 2 | 3 | from config import get_dropbox_access_token 4 | 5 | # Store files on Dropbox 6 | 7 | dbx = dropbox.Dropbox(get_dropbox_access_token()) 8 | 9 | 10 | def download_file(filename): 11 | """ 12 | Get file from Dropbox 13 | :param filename: full path of the file 14 | :return: 15 | """ 16 | print(f"filename {filename}") 17 | 18 | f, r = dbx.files_download(filename) 19 | 20 | content = r.content 21 | 22 | return content 23 | 24 | 25 | def upload_file(filename, localfile): 26 | """ 27 | Save file to Dropbox 28 | :param filename: path where to save the file 29 | :param localfile: path of local file to upload 30 | :return: 31 | """ 32 | print(f"filename {filename}") 33 | 34 | with open(localfile, "rb") as f: 35 | dbx.files_upload(f.read(), filename, mute=True) 36 | 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /doc/Gmail.md: -------------------------------------------------------------------------------- 1 | # Gmail 2 | 3 | Send a file by email using your Gmail account and the [yagmail](https://github.com/kootenpv/yagmail) Python library. 4 | 5 | This is a very basic approach but it might be useful if there is the need to save a file somewhere for backup or 6 | for performing a quick analysis 7 | 8 | ## Gmail 9 | 10 | You will need to use your Gmail credentials as well as enabling `Allow less secure apps` in the Gmail settings, see 11 | [here](https://www.google.com/settings/security/lesssecureapps) 12 | 13 | ## Code Samples 14 | 15 | **Example** 16 | 17 | Checkout [sample code](https://github.com/gcatanese/HerokuFiles/tree/main/app/gmail_api.py) in gmail_api.py 18 | 19 | ## Deploy to Heroku 20 | 21 | Before deploying to Heroku make sure to define the credentials as environment variables and create the corresponding Heroku Config Vars 22 | ``` 23 | heroku config:set GMAIL_USERNAME=xyz) 24 | heroku config:set GMAIL_PASSWORD=abc) 25 | ``` 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /app/gitlab_api.py: -------------------------------------------------------------------------------- 1 | import gitlab 2 | 3 | from config import get_gitlab_access_token 4 | 5 | # Store files on Gitlab 6 | 7 | REPO_ID = '23804786' 8 | 9 | gl = gitlab.Gitlab('https://gitlab.com', private_token=get_gitlab_access_token()) 10 | 11 | 12 | def get_file(filename): 13 | 14 | project = gl.projects.get(REPO_ID) 15 | 16 | f = project.files.get(file_path=filename, ref='master') 17 | 18 | return f 19 | 20 | 21 | def check_file_exist(filename): 22 | try: 23 | file = get_file(filename) 24 | except: 25 | return False 26 | 27 | return True 28 | 29 | 30 | def put_file(filename, content): 31 | 32 | project = gl.projects.get(REPO_ID) 33 | 34 | action = 'create' 35 | 36 | if check_file_exist(filename): 37 | action = 'update' 38 | 39 | data = { 40 | 'branch': 'master', 41 | 'commit_message': 'Push file', 42 | 'actions': [ 43 | { 44 | 'action': action, 45 | 'file_path': filename, 46 | 'content': content, 47 | } 48 | ] 49 | } 50 | 51 | commit = project.commits.create(data) 52 | 53 | print(commit) 54 | 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Remote files with Heroku 2 | 3 | Heroku file system is [ephemeral](https://devcenter.heroku.com/articles/dynos#ephemeral-filesystem), meaning Dyno applications can write on the file system but 4 | changes are discarded when the Dyno restarts. 5 | 6 | It is also important to remember that Heroku Dynos restart (at least) every 24 hrs (see [cycling](https://devcenter.heroku.com/articles/dynos#restarting)), 7 | hence deleting all local filesystem changes. 8 | 9 | Applications which need to persist data should rely on an external service like a database or a remote storage. 10 | 11 | ## Free options for remote file storage 12 | 13 | For applications that need to store files there are few remote storage cloud services offering a **free tier**, some of them limiting the size and/or the features available. 14 | 15 | They all provide a secure (tokens, credentials) access via APIs and are suitable for both projects under development or in production (see details of each provider): 16 | 17 | * [Github](doc/Github.md) 18 | * [S3](doc/S3.md) 19 | * [Dropbox](doc/Dropbox.md) 20 | * [DriveHQ (FTP)](doc/DriveHQ.md) 21 | * [Gitlab](doc/Gitlab.md) 22 | * [Gmail](doc/Gmail.md) 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /app/ftp_api.py: -------------------------------------------------------------------------------- 1 | import ftplib 2 | 3 | from config import * 4 | 5 | # Store files on FTP server 6 | 7 | 8 | def download_file(filename, tmpfile): 9 | """ 10 | Get file from FTP server 11 | :param filename: full path of the file 12 | :return: 13 | """ 14 | print(f"filename {filename}") 15 | 16 | session = ftplib.FTP(get_ftp_host(), get_ftp_username(), get_ftp_password()) 17 | 18 | f = open(tmpfile, 'wb') 19 | session.retrbinary('RETR ' + filename, f.write, 1024) 20 | 21 | session.quit() 22 | f.close() 23 | 24 | f = open(tmpfile, 'rb') 25 | content = f.read() 26 | 27 | return content 28 | 29 | 30 | def upload_file(filename, localfile): 31 | """ 32 | Save file to FTP server 33 | :param filename: path where to save the file 34 | :param localfile: path of local file to upload 35 | :return: 36 | """ 37 | print(f"filename {filename}") 38 | 39 | session = ftplib.FTP(get_ftp_host(), get_ftp_username(), get_ftp_password()) 40 | file = open(localfile, 'rb') # file to send 41 | session.storbinary('STOR '+filename, file) # send the file 42 | file.close() # close file and FTP 43 | session.quit() 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /app/github_api.py: -------------------------------------------------------------------------------- 1 | from github import Github 2 | 3 | from config import get_github_access_token 4 | 5 | REPO_NAME = 'HerokuFiles' 6 | 7 | 8 | github = Github(get_github_access_token()) 9 | 10 | 11 | def get_repos(): 12 | for repo in github.get_user().get_repos(): 13 | print(repo.name) 14 | 15 | 16 | def get_file(filename): 17 | repository = github.get_user().get_repo(REPO_NAME) 18 | file = repository.get_contents(filename) 19 | 20 | print(file.url) 21 | 22 | return file 23 | 24 | 25 | def check_file_exist(filename): 26 | try: 27 | file = get_file(filename) 28 | except: 29 | return False 30 | 31 | return True 32 | 33 | 34 | def put_file(filename, content): 35 | 36 | repository = github.get_user().get_repo(REPO_NAME) 37 | 38 | if check_file_exist(filename): 39 | print(f'update_file {filename}') 40 | f = get_file(filename) 41 | f = repository.update_file(filename, "update_file via PyGithub", content, f.sha) 42 | else: 43 | print(f'create_file {filename}') 44 | f = repository.create_file(filename, "create_file via PyGithub", content) 45 | 46 | 47 | def get_user(name): 48 | user = github.get_user(name) 49 | 50 | return user 51 | 52 | -------------------------------------------------------------------------------- /app/sftp_to_go_ftp.py: -------------------------------------------------------------------------------- 1 | import ftplib 2 | 3 | from config import * 4 | 5 | # Store files on SFTP To Go server 6 | 7 | 8 | def download_file(filename, tmpfile): 9 | """ 10 | Get file from FTP server 11 | :param filename: full path of the file 12 | :return: 13 | """ 14 | print(f"filename {filename}") 15 | 16 | session = ftplib.FTP_TLS(get_ftp_host(), get_ftp_username(), get_ftp_password()) 17 | session.prot_p() # data-channel protected by TLS 18 | 19 | f = open(tmpfile, 'wb') 20 | session.retrbinary('RETR ' + filename, f.write, 1024) 21 | 22 | session.quit() 23 | f.close() 24 | 25 | f = open(tmpfile, 'rb') 26 | content = f.read() 27 | 28 | return content 29 | 30 | 31 | def upload_file(filename, localfile): 32 | """ 33 | Save file to SFTP server 34 | :param filename: path where to save the file 35 | :param localfile: path of local file to upload 36 | :return: 37 | """ 38 | print(f"filename {filename}") 39 | 40 | session = ftplib.FTP_TLS(get_ftp_host(), get_ftp_username(), get_ftp_password()) 41 | session.prot_p() # data-channel protected by TLS 42 | 43 | file = open(localfile, 'rb') # file to send 44 | session.storbinary('STOR '+filename, file) # send the file 45 | file.close() # close file and FTP 46 | session.quit() 47 | 48 | -------------------------------------------------------------------------------- /app/sftp_to_go_s3.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | 3 | from config import * 4 | 5 | # Integration with SFTP To Go (S3) 6 | 7 | session = boto3.session.Session() 8 | 9 | 10 | s3 = session.client( 11 | service_name='s3', 12 | aws_access_key_id=get_sftptogo_aws_access_key_id(), 13 | aws_secret_access_key=get_sftptogo_aws_secret_access_key(), 14 | region_name=get_sftptogo_aws_region() 15 | ) 16 | 17 | 18 | def download_object(bucket_name, key, filename): 19 | """ 20 | Get file from S3 21 | :param bucket_name: name of S3 bucket 22 | :param key: key of the file (ie /a/b/mydoc.txt) 23 | :param filename: path where to save the file 24 | :return: 25 | """ 26 | print(f"filename {filename}") 27 | 28 | s3.download_file(Bucket=bucket_name, Key=key, Filename=filename) 29 | 30 | print(f"Object {key} has been downloaded to " + filename) 31 | 32 | return filename 33 | 34 | 35 | def upload_object(bucket_name, key, filename): 36 | """ 37 | Put file onto S3 38 | :param bucket_name: name of S3 bucket 39 | :param key: key of the file (ie /a/b/mydoc.txt) 40 | :param filename: path of the file to upload 41 | :return: 42 | """ 43 | print(f"filename {filename}") 44 | 45 | s3.upload_file(Bucket=bucket_name, Key=key, Filename=filename) 46 | 47 | print(f"File {filename} has been uploaded to {bucket_name}:{key}") 48 | 49 | return key 50 | 51 | 52 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: pip 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | time: "04:00" 8 | open-pull-requests-limit: 10 9 | ignore: 10 | - dependency-name: boto3 11 | versions: 12 | - 1.16.63 13 | - 1.17.0 14 | - 1.17.1 15 | - 1.17.10 16 | - 1.17.11 17 | - 1.17.12 18 | - 1.17.13 19 | - 1.17.14 20 | - 1.17.15 21 | - 1.17.16 22 | - 1.17.17 23 | - 1.17.19 24 | - 1.17.2 25 | - 1.17.20 26 | - 1.17.21 27 | - 1.17.22 28 | - 1.17.23 29 | - 1.17.25 30 | - 1.17.27 31 | - 1.17.28 32 | - 1.17.29 33 | - 1.17.3 34 | - 1.17.30 35 | - 1.17.32 36 | - 1.17.33 37 | - 1.17.34 38 | - 1.17.35 39 | - 1.17.36 40 | - 1.17.39 41 | - 1.17.4 42 | - 1.17.40 43 | - 1.17.41 44 | - 1.17.43 45 | - 1.17.46 46 | - 1.17.48 47 | - 1.17.49 48 | - 1.17.5 49 | - 1.17.50 50 | - 1.17.51 51 | - 1.17.52 52 | - 1.17.53 53 | - 1.17.54 54 | - 1.17.55 55 | - 1.17.56 56 | - 1.17.57 57 | - 1.17.58 58 | - 1.17.59 59 | - 1.17.6 60 | - 1.17.7 61 | - 1.17.8 62 | - 1.17.9 63 | - dependency-name: dropbox 64 | versions: 65 | - 11.4.0 66 | - 11.6.0 67 | - dependency-name: yagmail 68 | versions: 69 | - 0.14.247 70 | - dependency-name: python-dotenv 71 | versions: 72 | - 0.16.0 73 | -------------------------------------------------------------------------------- /app/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from dotenv import load_dotenv, find_dotenv 4 | 5 | load_dotenv(find_dotenv()) 6 | 7 | 8 | def get_github_access_token(): 9 | return os.environ.get("GITHUB_ACCESS_TOKEN", "") 10 | 11 | 12 | def get_aws_access_key_id(): 13 | return os.environ.get('AWS_ACCESS_KEY_ID') 14 | 15 | 16 | def get_aws_secret_access_key(): 17 | return os.environ.get('AWS_SECRET_ACCESS_KEY') 18 | 19 | 20 | def get_dropbox_access_token(): 21 | return os.environ.get("DROPBOX_ACCESS_TOKEN", "") 22 | 23 | 24 | def get_ftp_host(): 25 | return os.environ.get("FTP_HOST", "") 26 | 27 | 28 | def get_ftp_username(): 29 | return os.environ.get("FTP_USERNAME", "") 30 | 31 | 32 | def get_ftp_password(): 33 | return os.environ.get("FTP_PASSWORD", "") 34 | 35 | 36 | def get_gitlab_access_token(): 37 | return os.environ.get("GITLAB_ACCESS_TOKEN", "") 38 | 39 | 40 | def get_gmail_username(): 41 | return os.environ.get("GMAIL_USERNAME", "") 42 | 43 | 44 | def get_gmail_password(): 45 | return os.environ.get("GMAIL_PASSWORD", "") 46 | 47 | 48 | def get_sftptogo_aws_access_key_id(): 49 | return os.environ.get('SFTPTOGO_AWS_ACCESS_KEY_ID') 50 | 51 | 52 | def get_sftptogo_aws_secret_access_key(): 53 | return os.environ.get('SFTPTOGO_AWS_SECRET_ACCESS_KEY') 54 | 55 | 56 | def get_sftptogo_aws_region(): 57 | return os.environ.get('SFTPTOGO_AWS_REGION') 58 | 59 | 60 | def get_sftptogo_aws_bucket_name(): 61 | return os.environ.get('SFTPTOGO_AWS_BUCKET_NAME') 62 | -------------------------------------------------------------------------------- /app/s3_api.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | 3 | from config import get_aws_access_key_id, get_aws_secret_access_key 4 | 5 | # Store files on S3 6 | 7 | session = boto3.session.Session() 8 | 9 | 10 | s3 = session.client( 11 | service_name='s3', 12 | aws_access_key_id=get_aws_access_key_id(), 13 | aws_secret_access_key=get_aws_secret_access_key(), 14 | region_name='eu-west-3' 15 | ) 16 | 17 | 18 | def download_object(bucket_name, key, filename): 19 | """ 20 | Get file from S3 21 | :param bucket_name: name of S3 bucket 22 | :param key: key of the file (ie /a/b/mydoc.txt) 23 | :param filename: path where to save the file 24 | :return: 25 | """ 26 | print(f"filename {filename}") 27 | 28 | s3.download_file(Bucket=bucket_name, Key=key, Filename=filename) 29 | 30 | print(f"Object {key} has been downloaded to " + filename) 31 | 32 | return filename 33 | 34 | 35 | def upload_object(bucket_name, key, filename): 36 | """ 37 | Put file onto S3 38 | :param bucket_name: name of S3 bucket 39 | :param key: key of the file (ie /a/b/mydoc.txt) 40 | :param filename: path of the file to upload 41 | :return: 42 | """ 43 | print(f"filename {filename}") 44 | 45 | s3.upload_file(Bucket=bucket_name, Key=key, Filename=filename) 46 | 47 | print(f"File {filename} has been uploaded to {bucket_name}:{key}") 48 | 49 | return key 50 | 51 | 52 | def bucket_list(): 53 | """ 54 | List buckets 55 | :return: 56 | """ 57 | 58 | buckets = s3.list_buckets() 59 | 60 | return buckets 61 | 62 | 63 | -------------------------------------------------------------------------------- /doc/Dropbox.md: -------------------------------------------------------------------------------- 1 | # Dropbox 2 | 3 | [Dropbx](https://www.dropbox.com/) is a file hosting service that offer various plans at different prices. 4 | The Dropbox Basic account is free and allows storing up to 2GB of files. 5 | 6 | The intuitive Web User Interface can be used to create, edit and download files, there is a nice integration 7 | with Google docs and Office Online, but what you really want to use is the [Python Dropbox SDK](https://www.dropbox.com/developers/documentation/python) 8 | 9 | ## Dropbox Setup 10 | 11 | Sign up for a Dropbox account then register a new app in the `App Console`. After defining the permissions (ie files.content.write, files.content.read) 12 | generate the Access Token. 13 | 14 | 15 | ## Code Samples 16 | 17 | Here are some snippets to show how to read/write files 18 | 19 | **Create a file** 20 | ``` 21 | dbx = dropbox.Dropbox('access_token') 22 | 23 | filename = '/local_files/file.json' 24 | dbx.files_upload(f.read(), filename, mute=True) 25 | ``` 26 | 27 | **Get a file** 28 | ``` 29 | dbx = dropbox.Dropbox('access_token') 30 | 31 | filename = '/dropbox_root/file.json' 32 | f, r = dbx.files_download(filename) 33 | 34 | print(r.content) 35 | ``` 36 | 37 | **Example** 38 | 39 | Checkout [sample code](https://github.com/gcatanese/HerokuFiles/tree/main/app/dropbox_api.py) in dropbox_api.py 40 | 41 | ## Deploy to Heroku 42 | 43 | Before deploying to Heroku make sure to define the Access Token as environment variable and create a corresponding Heroku Config Var 44 | ``` 45 | heroku config:set DROPBOX_ACCESS_TOKEN=xyz) 46 | ``` 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /doc/S3.md: -------------------------------------------------------------------------------- 1 | # Amazon S3 2 | 3 | Amazon S3 (Simple Storage Service) in the remote storage offered by Amazon. It is not entirely free but extremely cheap, 4 | for very low usage it won't even charge. 5 | Using Python [boto3](https://boto3.amazonaws.com/v1/documentation/api/latest/index.html) it is pretty simple to 6 | access and manage any file type. 7 | A great plus of this option is that there are several other S3 compatible storage services, which makes easier 8 | to move to a different provider if needed. 9 | 10 | Notes: 11 | * create AWS credentials from [IAM Console](https://console.aws.amazon.com/iam/home#/home) 12 | 13 | **Get a file** 14 | ``` 15 | session = boto3.session.Session() 16 | 17 | s3 = session.client( 18 | service_name='s3', 19 | aws_access_key_id='xyz', 20 | aws_secret_access_key='abc' 21 | ) 22 | 23 | s3.download_file(Bucket='bucket_name', Key='dir/a.txt', Filename=/tmp/a.txt) 24 | ``` 25 | 26 | **Put a file** 27 | ``` 28 | session = boto3.session.Session() 29 | 30 | s3 = session.client( 31 | service_name='s3', 32 | aws_access_key_id='xyz', 33 | aws_secret_access_key='abc' 34 | ) 35 | 36 | s3.upload_file(Bucket='bucket_name', Key='dir/a.txt', Filename=/tmp/a.txt) 37 | ``` 38 | 39 | 40 | **Example** 41 | 42 | Checkout [sample code](https://github.com/gcatanese/HerokuFiles/tree/main/app/s3_api.py) in s3_api.py 43 | 44 | ## Deploy to Heroku 45 | 46 | Before deploying to Heroku make sure to define the AWS credentials as environment variables and create the corresponding Heroku Config Vars 47 | ``` 48 | heroku config:set AWS_ACCESS_KEY_ID=xyz) 49 | heroku config:set AWS_SECRET_ACCESS_KEY=abc) 50 | ``` 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /app/main.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from flask import Flask, request 5 | 6 | from github_api import * 7 | 8 | try: 9 | app = Flask(__name__) 10 | 11 | logging.basicConfig(level=logging.DEBUG) 12 | logging.getLogger('werkzeug').setLevel(logging.ERROR) 13 | except Exception as e: 14 | logging.exception("Error at startup") 15 | 16 | 17 | @app.route('/ping') 18 | def ping(): 19 | """ 20 | Ping the endpoint 21 | :return: 22 | """ 23 | logging.info('/ping') 24 | return "ping Ok" 25 | 26 | 27 | @app.route('/check') 28 | def check(): 29 | """ 30 | Check file exists 31 | :return: 32 | """ 33 | logging.info('/check') 34 | 35 | # filename (ie files/file.json) 36 | filename = request.args.get('filename') 37 | 38 | ret = check_file_exist(filename) 39 | 40 | return str(ret) 41 | 42 | 43 | @app.route('/get') 44 | def get(): 45 | """ 46 | Get file 47 | :return: 48 | """ 49 | logging.info('/get') 50 | 51 | # filename (ie files/file.json) 52 | filename = request.args.get('filename') 53 | 54 | file = get_file(filename) 55 | 56 | return file.decoded_content.decode() 57 | 58 | 59 | @app.route('/put', methods=['POST']) 60 | def put(): 61 | """ 62 | Store file 63 | :return: 64 | """ 65 | logging.info('/put') 66 | 67 | # filename (ie files/file.json) 68 | filename = request.args.get('filename') 69 | content = str(request.json) 70 | 71 | put_file(filename, content) 72 | 73 | return "Ok" 74 | 75 | 76 | def get_port(): 77 | """ 78 | Retrieves port 79 | :return: 80 | """ 81 | return int(os.environ.get("PORT", 5000)) 82 | 83 | 84 | if __name__ == '__main__': 85 | app.run(debug=True, port=get_port(), host='0.0.0.0') 86 | -------------------------------------------------------------------------------- /doc/Github.md: -------------------------------------------------------------------------------- 1 | # Github 2 | 3 | A good option is to use Github: it is free, reliable and devs are already familiar with it. 4 | 5 | Store files on any of your Github private or public repositories using [PyGithub](https://github.com/PyGithub/PyGithub) 6 | 7 | ## Github Access Token 8 | 9 | Generate a Github Access Token necessary to authenticate the API calls: go to `Account -> Settings -> Developer Settings -> Personal Access Token` 10 | 11 | Grant the token the necessary privileges (read/write into repositories) and **NEVER** show or share your tokens. 12 | 13 | See Github documentation if unsure: [Creating a personal access token 14 | ](https://docs.github.com/en/github/authenticating-to-github/creating-a-personal-access-token) 15 | 16 | ## Code Samples 17 | 18 | Here are some snippets to show how to read/write files 19 | 20 | **Create a file** 21 | ``` 22 | github = Github('access_token) 23 | repository = github.get_user().get_repo('my_repo') 24 | 25 | content = '{\"name\":\"beppe\",\"city\":\"amsterdam\"}' 26 | f = repository.create_file('data.json', "create_file via PyGithub", content) 27 | ``` 28 | 29 | **Get a file** 30 | ``` 31 | github = Github('access_token) 32 | repository = github.get_user().get_repo('my_repo') 33 | 34 | file = repository.get_contents(filename) 35 | print(file.decoded_content.decode()) 36 | ``` 37 | 38 | **Example** 39 | 40 | Checkout [Python app sample code](https://github.com/gcatanese/HerokuFiles/tree/main/app) which provides few Flask 41 | endpoints demonstrating the access to `Github` 42 | 43 | ``` 44 | curl http://localhost:5000/check?filename=files/file.json 45 | 46 | curl http://localhost:5000/get?filename=files/file.json 47 | 48 | curl -i -X POST -H "Content-Type: application/json" -d "{\"name\":\"beppe\",\"city\":\"amsterdam\"}" http://localhost:5000/put?filename=files/file.json 49 | ``` 50 | 51 | 52 | ## Deploy to Heroku 53 | 54 | Before deploying to Heroku make sure to define the Github Access Token as environment variable and create a corresponding Heroku Config Var 55 | ``` 56 | heroku config:set GITHUB_ACCESS_TOKEN=xyz) 57 | ``` 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------