├── LICENSE ├── bin ├── python ├── python3 ├── pip ├── pip3 ├── pip3.5 ├── easy_install ├── easy_install-3.5 ├── activate.csh ├── activate └── activate.fish ├── dev ├── bin │ ├── python │ ├── python3 │ ├── pip │ ├── pip3 │ ├── pip3.5 │ ├── easy_install │ ├── easy_install-3.5 │ ├── activate.csh │ ├── activate │ └── activate.fish └── pyvenv.cfg ├── .cache └── v │ └── cache │ └── lastfailed ├── soviet_art_bot ├── __init__.py ├── bin │ ├── python2 │ ├── python2.7 │ └── python ├── tests │ ├── __init__.py │ ├── .cache │ │ └── v │ │ │ └── cache │ │ │ └── lastfailed │ └── test_download_images.py ├── include │ └── python2.7 ├── event.json └── download_images.py ├── lambda_bundle.zip ├── requirements.txt ├── ecr_credentials.sh ├── .gitignore ├── settings.py ├── README.md ├── setup.py ├── lambda_deploy.sh ├── .travis.yml └── lambda └── lambda_function.py /LICENSE: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bin/python: -------------------------------------------------------------------------------- 1 | python3 -------------------------------------------------------------------------------- /dev/bin/python: -------------------------------------------------------------------------------- 1 | python3 -------------------------------------------------------------------------------- /.cache/v/cache/lastfailed: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /soviet_art_bot/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /bin/python3: -------------------------------------------------------------------------------- 1 | /usr/local/bin/python3 -------------------------------------------------------------------------------- /soviet_art_bot/bin/python2: -------------------------------------------------------------------------------- 1 | python -------------------------------------------------------------------------------- /soviet_art_bot/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dev/bin/python3: -------------------------------------------------------------------------------- 1 | /usr/local/bin/python3 -------------------------------------------------------------------------------- /soviet_art_bot/bin/python2.7: -------------------------------------------------------------------------------- 1 | python -------------------------------------------------------------------------------- /soviet_art_bot/include/python2.7: -------------------------------------------------------------------------------- 1 | /Users/vboykis/anaconda/include/python2.7 -------------------------------------------------------------------------------- /lambda_bundle.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/soviet-art-bot/HEAD/lambda_bundle.zip -------------------------------------------------------------------------------- /dev/pyvenv.cfg: -------------------------------------------------------------------------------- 1 | home = /usr/local/bin 2 | include-system-site-packages = false 3 | version = 3.5.1 4 | -------------------------------------------------------------------------------- /soviet_art_bot/bin/python: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/veekaybee/soviet-art-bot/HEAD/soviet_art_bot/bin/python -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | boto3==1.4.4 2 | requests==2.14.2 3 | Pillow==5.0.0 4 | setuptools==38.5.1 5 | twython==3.6.0 6 | pytest==3.4.0 7 | -------------------------------------------------------------------------------- /soviet_art_bot/tests/.cache/v/cache/lastfailed: -------------------------------------------------------------------------------- 1 | { 2 | "download_test.py": true, 3 | "test_download_images.py": true, 4 | "test_download_images.py::test_download_images": true 5 | } -------------------------------------------------------------------------------- /ecr_credentials.sh: -------------------------------------------------------------------------------- 1 | mkdir -p ~/.aws 2 | 3 | echo "Mkdir" 4 | 5 | cat << EOF > ~/.aws/credentials 6 | [default] 7 | aws_access_key_id = ${AWS_ACCESS_KEY_ID} 8 | aws_secret_access_key = ${AWS_SECRET_ACCESS_KEY} 9 | EOF 10 | 11 | cat ~/.aws/credentials -------------------------------------------------------------------------------- /bin/pip: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from pip import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /bin/pip3: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from pip import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /bin/pip3.5: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from pip import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /dev/bin/pip: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/dev/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from pip import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /dev/bin/pip3: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/dev/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from pip import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /dev/bin/pip3.5: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/dev/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from pip import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /bin/easy_install: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from setuptools.command.easy_install import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /bin/easy_install-3.5: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from setuptools.command.easy_install import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /dev/bin/easy_install: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/dev/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from setuptools.command.easy_install import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /dev/bin/easy_install-3.5: -------------------------------------------------------------------------------- 1 | #!/Users/vboykis/Desktop/soviet_art_bot/dev/bin/python3 2 | 3 | # -*- coding: utf-8 -*- 4 | import re 5 | import sys 6 | 7 | from setuptools.command.easy_install import main 8 | 9 | if __name__ == '__main__': 10 | sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) 11 | sys.exit(main()) 12 | -------------------------------------------------------------------------------- /soviet_art_bot/event.json: -------------------------------------------------------------------------------- 1 | { 2 | "account": "123456789012", 3 | "region": "us-east-1", 4 | "detail": {}, 5 | "detail-type": "Scheduled Event", 6 | "source": "aws.events", 7 | "time": "1970-01-01T00:00:00Z", 8 | "id": "cdc73f9d-aea9-11e3-9d5a-835b769c0d9c", 9 | "resources": [ 10 | "arn:aws:events:us-east-1:123456789012:rule/my-schedule" 11 | ] 12 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Images 4 | *.jpg 5 | *.jpeg 6 | *.png 7 | 8 | # Environment Files 9 | .env 10 | 11 | # Byte-compiled / optimized / DLL files 12 | __pycache__/ 13 | *.py[cod] 14 | *$py.class 15 | 16 | #PyCharm 17 | .idea/ 18 | 19 | # C extensions 20 | *.so 21 | 22 | # Distribution / packaging 23 | .Python 24 | build/ 25 | develop-eggs/ 26 | dist/ 27 | downloads/ 28 | eggs/ 29 | .eggs/ 30 | lib/ 31 | lib64/ 32 | parts/ 33 | sdist/ 34 | var/ 35 | wheels/ 36 | *.egg-info/ 37 | .installed.cfg 38 | *.egg 39 | MANIFEST 40 | -------------------------------------------------------------------------------- /settings.py: -------------------------------------------------------------------------------- 1 | from pathlib import Path 2 | from dotenv import load_dotenv 3 | 4 | # Create .env file path. 5 | dotenv_path = join(dirname(__file__), '.env') 6 | 7 | # Load file from the path. 8 | load_dotenv(dotenv_path) 9 | 10 | # Twitter Keys 11 | 12 | CONSUMER_KEY = os.getenv('CONSUMER_KEY') 13 | CONSUMER_SECRET = os.getenv('CONSUMER_SECRET') 14 | ACCESS_TOKEN = os.getenv('ACCESS_TOKEN') 15 | ACCESS_SECRET = os.getenv('ACCESS_SECRET') 16 | 17 | 18 | #Wikiart 19 | BASE_URL= "https://www.wikiart.org/" 20 | STYLE_URL = "en/paintings-by-style/socialist-realism?json=2" 21 | PAGINATION_URL = "page=" 22 | 23 | 24 | #timeout for WikiArt 25 | # SEE https://github.com/lucasdavid/wikiart/blob/master/wikiart/settings.py 26 | METADATA_REQUEST_TIMEOUT = 2 * 60 27 | PAINTINGS_REQUEST_TIMEOUT = 5 * 60 28 | 29 | #Local filepaths 30 | TOP_LEVEL_PATH = Path('/Users/vboykis/Desktop/soviet_art_bot/') 31 | ASSET_PATH = TOP_LEVEL_PATH/ 'assets' 32 | 33 | # Metadata filename 34 | 35 | METADATA_FILENAME = 'art_metadata.json' 36 | MEDTADATA_FILE = ASSET_PATH.joinpath(METADATA_FILENAME) 37 | 38 | 39 | #AWS Locations 40 | 41 | BASE_BUCKET = 'soviet-art-bot' 42 | 43 | 44 | -------------------------------------------------------------------------------- /dev/bin/activate.csh: -------------------------------------------------------------------------------- 1 | # This file must be used with "source bin/activate.csh" *from csh*. 2 | # You cannot run it directly. 3 | # Created by Davide Di Blasi . 4 | # Ported to Python 3.3 venv by Andrew Svetlov 5 | 6 | alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' 7 | 8 | # Unset irrelavent variables. 9 | deactivate nondestructive 10 | 11 | setenv VIRTUAL_ENV "/Users/vboykis/Desktop/soviet_art_bot/dev" 12 | 13 | set _OLD_VIRTUAL_PATH="$PATH" 14 | setenv PATH "$VIRTUAL_ENV/bin:$PATH" 15 | 16 | 17 | set _OLD_VIRTUAL_PROMPT="$prompt" 18 | 19 | if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then 20 | if ("dev" != "") then 21 | set env_name = "dev" 22 | else 23 | if (`basename "VIRTUAL_ENV"` == "__") then 24 | # special case for Aspen magic directories 25 | # see http://www.zetadev.com/software/aspen/ 26 | set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` 27 | else 28 | set env_name = `basename "$VIRTUAL_ENV"` 29 | endif 30 | endif 31 | set prompt = "[$env_name] $prompt" 32 | unset env_name 33 | endif 34 | 35 | alias pydoc python -m pydoc 36 | 37 | rehash 38 | -------------------------------------------------------------------------------- /bin/activate.csh: -------------------------------------------------------------------------------- 1 | # This file must be used with "source bin/activate.csh" *from csh*. 2 | # You cannot run it directly. 3 | # Created by Davide Di Blasi . 4 | # Ported to Python 3.3 venv by Andrew Svetlov 5 | 6 | alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; test "\!:*" != "nondestructive" && unalias deactivate' 7 | 8 | # Unset irrelavent variables. 9 | deactivate nondestructive 10 | 11 | setenv VIRTUAL_ENV "/Users/vboykis/Desktop/soviet_art_bot" 12 | 13 | set _OLD_VIRTUAL_PATH="$PATH" 14 | setenv PATH "$VIRTUAL_ENV/bin:$PATH" 15 | 16 | 17 | set _OLD_VIRTUAL_PROMPT="$prompt" 18 | 19 | if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then 20 | if ("soviet_art_bot" != "") then 21 | set env_name = "soviet_art_bot" 22 | else 23 | if (`basename "VIRTUAL_ENV"` == "__") then 24 | # special case for Aspen magic directories 25 | # see http://www.zetadev.com/software/aspen/ 26 | set env_name = `basename \`dirname "$VIRTUAL_ENV"\`` 27 | else 28 | set env_name = `basename "$VIRTUAL_ENV"` 29 | endif 30 | endif 31 | set prompt = "[$env_name] $prompt" 32 | unset env_name 33 | endif 34 | 35 | alias pydoc python -m pydoc 36 | 37 | rehash 38 | -------------------------------------------------------------------------------- /soviet_art_bot/tests/test_download_images.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | import os 3 | 4 | import soviet_art_bot.download_images as dl 5 | import settings 6 | 7 | 8 | # Test JSON 9 | single_painting = [{'artistName': 'Aleksandr Deyneka', 'images': None, 'width': 634, 'artistUrl': '/en/aleksandr-deyneka', 'albums': None, 'title': 'Winter in Kursk', 'id': '577271cfedc2cb3880c2de61', 'year': '1916', 'height': 750, 'paintingUrl': '/en/aleksandr-deyneka/winter-in-kursk-1916', 'flags': 2, 'image': 'https://uploads8.wikiart.org/images/aleksandr-deyneka/winter-in-kursk-1916.jpg', 'map': '0123**67*'}] 10 | add_painting = [{'artistName': 'Konstantin Yuon', 'images': None, 'width': 776, 'artistUrl': '/en/konstantin-yuon', 'albums': None, 'title': 'The Symphony of Action', 'id': '57727792edc2cb3880d4f131', 'year': '1922', 'height': 650, 'paintingUrl': '/en/konstantin-yuon/the-symphony-of-action-1922', 'flags': 2, 'image': 'https://uploads4.wikiart.org/images/konstantin-yuon/the-symphony-of-action-1922.jpg', 'map': '0123**67*'}] 11 | 12 | 13 | def test_build_url(): 14 | url = settings.BASE_URL + settings.STYLE_URL + "&" + settings.PAGINATION_URL + str(1) 15 | assert url == "https://www.wikiart.org/en/paintings-by-style/socialist-realism?json=2&page=1" 16 | 17 | 18 | # Tests that paintings are extracted and added to a list 19 | def test_parse_data(): 20 | dl.parse_data(single_painting, add_painting) 21 | assert len(single_painting) == 2 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | # Soviet Art Bot v.20 4 | 5 | 6 | [![Build Status](https://travis-ci.org/veekaybee/soviet-art-bot.svg?branch=master)](https://travis-ci.org/veekaybee/soviet-art-bot) 7 | [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/dwyl/esta/issues) 8 | 9 | Latest Updates: 10 | --- 11 | 12 | + Fixing the html escape issue where apostrophes get rendered into hideous HTML representations by adding some [unescapes](https://github.com/veekaybee/soviet-art-bot/blob/35d13d788e1f8237b562176bb65de20ad5d3d598/lambda/lambda_function.py#L41-L42) 13 | + Adding some 12-factor principles by not storing my Twitter credentials in the app, but in SSM instead and pulling them in through [Travis-CI](https://github.com/veekaybee/soviet-art-bot/blob/master/build_env.sh) 14 | 15 | 16 | 17 | ![](https://raw.githubusercontent.com/veekaybee/soviet-art-bot/gh-pages/static/in_peaceful_fields.jpg) 18 | 19 | A bot that finds socialist realism paintings and tweets them out. For the official webpage, [check out here.](https://veekaybee.github.io/soviet-art-bot/) 20 | 21 | To see it in action, follow [@SovietArtBot](https://twitter.com/SovietArtBot) 22 | 23 | ## Why Soviet Art Bot 24 | 25 | The TL;DR: Putting art on Twitter is fun and a great way to learn more about AWS. 26 | 27 | To read a really long post on Soviet Art Bot, [click here.](http://veekaybee.github.io/2018/02/19/creating-a-twitter-art-bot/) 28 | 29 | ## Architecture 30 | 31 | ![High-level](https://raw.githubusercontent.com/veekaybee/veekaybee.github.io/master/images/high-level-flow.png) 32 | 33 | Low-Level 34 | ![architecture](https://raw.githubusercontent.com/veekaybee/veekaybee.github.io/master/images/architecture.png) 35 | 36 | ## Installation 37 | 38 | In progress. 39 | 40 | ## Contributing [![contributions welcome](https://img.shields.io/badge/contributions-welcome-brightgreen.svg?style=flat)](https://github.com/dwyl/esta/issues) 41 | 42 | It's really the alpha Wild Wild West stages of this project so I don't have any specific guidelines. That said, if you see something, say something. Pull requests welcome. 43 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | 5 | import io 6 | import os 7 | import sys 8 | from shutil import rmtree 9 | 10 | from setuptools import find_packages, setup, Command 11 | 12 | # Package meta-data. 13 | NAME = 'soviet-art-bot' 14 | DESCRIPTION = 'A bot that tweets out socialist realism paintings every so often. ' 15 | URL = 'https://github.com/veekaybee/soviet-art-bot' 16 | EMAIL = 'vickiboykis@gmail.com' 17 | AUTHOR = 'Vicki Boykis' 18 | 19 | # What packages are required for this module to be executed? 20 | REQUIRED = [ 21 | # 'requests', 'maya', 'records', 22 | ] 23 | 24 | # The rest you shouldn't have to touch too much :) 25 | # ------------------------------------------------ 26 | # Except, perhaps the License and Trove Classifiers! 27 | # If you do change the License, remember to change the Trove Classifier for that! 28 | 29 | here = os.path.abspath(os.path.dirname(__file__)) 30 | 31 | 32 | # Where the magic happens: 33 | setup( 34 | name=NAME, 35 | #version=about['__version__'], 36 | description=DESCRIPTION, 37 | # long_description=long_description, 38 | author=AUTHOR, 39 | author_email=EMAIL, 40 | url=URL, 41 | packages=find_packages(exclude=('tests',)), 42 | # If your package is a single module, use this instead of 'packages': 43 | # py_modules=['mypackage'], 44 | 45 | # entry_points={ 46 | # 'console_scripts': ['mycli=mymodule:cli'], 47 | # }, 48 | install_requires=REQUIRED, 49 | include_package_data=True, 50 | license='MIT', 51 | classifiers=[ 52 | # Trove classifiers 53 | # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers 54 | 'License :: OSI Approved :: MIT License', 55 | 'Programming Language :: Python', 56 | 'Programming Language :: Python :: 2.6', 57 | 'Programming Language :: Python :: 2.7', 58 | 'Programming Language :: Python :: 3', 59 | 'Programming Language :: Python :: 3.3', 60 | 'Programming Language :: Python :: 3.4', 61 | 'Programming Language :: Python :: 3.5', 62 | 'Programming Language :: Python :: 3.6', 63 | 'Programming Language :: Python :: Implementation :: CPython', 64 | 'Programming Language :: Python :: Implementation :: PyPy' 65 | ], 66 | 67 | ) 68 | -------------------------------------------------------------------------------- /dev/bin/activate: -------------------------------------------------------------------------------- 1 | # This file must be used with "source bin/activate" *from bash* 2 | # you cannot run it directly 3 | 4 | deactivate () { 5 | # reset old environment variables 6 | if [ -n "$_OLD_VIRTUAL_PATH" ] ; then 7 | PATH="$_OLD_VIRTUAL_PATH" 8 | export PATH 9 | unset _OLD_VIRTUAL_PATH 10 | fi 11 | if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then 12 | PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" 13 | export PYTHONHOME 14 | unset _OLD_VIRTUAL_PYTHONHOME 15 | fi 16 | 17 | # This should detect bash and zsh, which have a hash command that must 18 | # be called to get it to forget past commands. Without forgetting 19 | # past commands the $PATH changes we made may not be respected 20 | if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then 21 | hash -r 22 | fi 23 | 24 | if [ -n "$_OLD_VIRTUAL_PS1" ] ; then 25 | PS1="$_OLD_VIRTUAL_PS1" 26 | export PS1 27 | unset _OLD_VIRTUAL_PS1 28 | fi 29 | 30 | unset VIRTUAL_ENV 31 | if [ ! "$1" = "nondestructive" ] ; then 32 | # Self destruct! 33 | unset -f deactivate 34 | fi 35 | } 36 | 37 | # unset irrelavent variables 38 | deactivate nondestructive 39 | 40 | VIRTUAL_ENV="/Users/vboykis/Desktop/soviet_art_bot/dev" 41 | export VIRTUAL_ENV 42 | 43 | _OLD_VIRTUAL_PATH="$PATH" 44 | PATH="$VIRTUAL_ENV/bin:$PATH" 45 | export PATH 46 | 47 | # unset PYTHONHOME if set 48 | # this will fail if PYTHONHOME is set to the empty string (which is bad anyway) 49 | # could use `if (set -u; : $PYTHONHOME) ;` in bash 50 | if [ -n "$PYTHONHOME" ] ; then 51 | _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" 52 | unset PYTHONHOME 53 | fi 54 | 55 | if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then 56 | _OLD_VIRTUAL_PS1="$PS1" 57 | if [ "x(dev) " != x ] ; then 58 | PS1="(dev) $PS1" 59 | else 60 | if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then 61 | # special case for Aspen magic directories 62 | # see http://www.zetadev.com/software/aspen/ 63 | PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" 64 | else 65 | PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" 66 | fi 67 | fi 68 | export PS1 69 | fi 70 | 71 | # This should detect bash and zsh, which have a hash command that must 72 | # be called to get it to forget past commands. Without forgetting 73 | # past commands the $PATH changes we made may not be respected 74 | if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then 75 | hash -r 76 | fi 77 | -------------------------------------------------------------------------------- /bin/activate: -------------------------------------------------------------------------------- 1 | # This file must be used with "source bin/activate" *from bash* 2 | # you cannot run it directly 3 | 4 | deactivate () { 5 | # reset old environment variables 6 | if [ -n "$_OLD_VIRTUAL_PATH" ] ; then 7 | PATH="$_OLD_VIRTUAL_PATH" 8 | export PATH 9 | unset _OLD_VIRTUAL_PATH 10 | fi 11 | if [ -n "$_OLD_VIRTUAL_PYTHONHOME" ] ; then 12 | PYTHONHOME="$_OLD_VIRTUAL_PYTHONHOME" 13 | export PYTHONHOME 14 | unset _OLD_VIRTUAL_PYTHONHOME 15 | fi 16 | 17 | # This should detect bash and zsh, which have a hash command that must 18 | # be called to get it to forget past commands. Without forgetting 19 | # past commands the $PATH changes we made may not be respected 20 | if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then 21 | hash -r 22 | fi 23 | 24 | if [ -n "$_OLD_VIRTUAL_PS1" ] ; then 25 | PS1="$_OLD_VIRTUAL_PS1" 26 | export PS1 27 | unset _OLD_VIRTUAL_PS1 28 | fi 29 | 30 | unset VIRTUAL_ENV 31 | if [ ! "$1" = "nondestructive" ] ; then 32 | # Self destruct! 33 | unset -f deactivate 34 | fi 35 | } 36 | 37 | # unset irrelavent variables 38 | deactivate nondestructive 39 | 40 | VIRTUAL_ENV="/Users/vboykis/Desktop/soviet_art_bot" 41 | export VIRTUAL_ENV 42 | 43 | _OLD_VIRTUAL_PATH="$PATH" 44 | PATH="$VIRTUAL_ENV/bin:$PATH" 45 | export PATH 46 | 47 | # unset PYTHONHOME if set 48 | # this will fail if PYTHONHOME is set to the empty string (which is bad anyway) 49 | # could use `if (set -u; : $PYTHONHOME) ;` in bash 50 | if [ -n "$PYTHONHOME" ] ; then 51 | _OLD_VIRTUAL_PYTHONHOME="$PYTHONHOME" 52 | unset PYTHONHOME 53 | fi 54 | 55 | if [ -z "$VIRTUAL_ENV_DISABLE_PROMPT" ] ; then 56 | _OLD_VIRTUAL_PS1="$PS1" 57 | if [ "x(soviet_art_bot) " != x ] ; then 58 | PS1="(soviet_art_bot) $PS1" 59 | else 60 | if [ "`basename \"$VIRTUAL_ENV\"`" = "__" ] ; then 61 | # special case for Aspen magic directories 62 | # see http://www.zetadev.com/software/aspen/ 63 | PS1="[`basename \`dirname \"$VIRTUAL_ENV\"\``] $PS1" 64 | else 65 | PS1="(`basename \"$VIRTUAL_ENV\"`)$PS1" 66 | fi 67 | fi 68 | export PS1 69 | fi 70 | 71 | # This should detect bash and zsh, which have a hash command that must 72 | # be called to get it to forget past commands. Without forgetting 73 | # past commands the $PATH changes we made may not be respected 74 | if [ -n "$BASH" -o -n "$ZSH_VERSION" ] ; then 75 | hash -r 76 | fi 77 | -------------------------------------------------------------------------------- /dev/bin/activate.fish: -------------------------------------------------------------------------------- 1 | # This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) 2 | # you cannot run it directly 3 | 4 | function deactivate -d "Exit virtualenv and return to normal shell environment" 5 | # reset old environment variables 6 | if test -n "$_OLD_VIRTUAL_PATH" 7 | set -gx PATH $_OLD_VIRTUAL_PATH 8 | set -e _OLD_VIRTUAL_PATH 9 | end 10 | if test -n "$_OLD_VIRTUAL_PYTHONHOME" 11 | set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME 12 | set -e _OLD_VIRTUAL_PYTHONHOME 13 | end 14 | 15 | if test -n "$_OLD_FISH_PROMPT_OVERRIDE" 16 | functions -e fish_prompt 17 | set -e _OLD_FISH_PROMPT_OVERRIDE 18 | . ( begin 19 | printf "function fish_prompt\n\t#" 20 | functions _old_fish_prompt 21 | end | psub ) 22 | functions -e _old_fish_prompt 23 | end 24 | 25 | set -e VIRTUAL_ENV 26 | if test "$argv[1]" != "nondestructive" 27 | # Self destruct! 28 | functions -e deactivate 29 | end 30 | end 31 | 32 | # unset irrelavent variables 33 | deactivate nondestructive 34 | 35 | set -gx VIRTUAL_ENV "/Users/vboykis/Desktop/soviet_art_bot/dev" 36 | 37 | set -gx _OLD_VIRTUAL_PATH $PATH 38 | set -gx PATH "$VIRTUAL_ENV/bin" $PATH 39 | 40 | # unset PYTHONHOME if set 41 | if set -q PYTHONHOME 42 | set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME 43 | set -e PYTHONHOME 44 | end 45 | 46 | if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" 47 | # fish uses a function instead of an env var to generate the prompt. 48 | 49 | # save the current fish_prompt function as the function _old_fish_prompt 50 | . ( begin 51 | printf "function _old_fish_prompt\n\t#" 52 | functions fish_prompt 53 | end | psub ) 54 | 55 | # with the original prompt function renamed, we can override with our own. 56 | function fish_prompt 57 | # Prompt override? 58 | if test -n "(dev) " 59 | printf "%s%s%s" "(dev) " (set_color normal) (_old_fish_prompt) 60 | return 61 | end 62 | # ...Otherwise, prepend env 63 | set -l _checkbase (basename "$VIRTUAL_ENV") 64 | if test $_checkbase = "__" 65 | # special case for Aspen magic directories 66 | # see http://www.zetadev.com/software/aspen/ 67 | printf "%s[%s]%s %s" (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) (_old_fish_prompt) 68 | else 69 | printf "%s(%s)%s%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) (_old_fish_prompt) 70 | end 71 | end 72 | 73 | set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" 74 | end 75 | -------------------------------------------------------------------------------- /bin/activate.fish: -------------------------------------------------------------------------------- 1 | # This file must be used with ". bin/activate.fish" *from fish* (http://fishshell.org) 2 | # you cannot run it directly 3 | 4 | function deactivate -d "Exit virtualenv and return to normal shell environment" 5 | # reset old environment variables 6 | if test -n "$_OLD_VIRTUAL_PATH" 7 | set -gx PATH $_OLD_VIRTUAL_PATH 8 | set -e _OLD_VIRTUAL_PATH 9 | end 10 | if test -n "$_OLD_VIRTUAL_PYTHONHOME" 11 | set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME 12 | set -e _OLD_VIRTUAL_PYTHONHOME 13 | end 14 | 15 | if test -n "$_OLD_FISH_PROMPT_OVERRIDE" 16 | functions -e fish_prompt 17 | set -e _OLD_FISH_PROMPT_OVERRIDE 18 | . ( begin 19 | printf "function fish_prompt\n\t#" 20 | functions _old_fish_prompt 21 | end | psub ) 22 | functions -e _old_fish_prompt 23 | end 24 | 25 | set -e VIRTUAL_ENV 26 | if test "$argv[1]" != "nondestructive" 27 | # Self destruct! 28 | functions -e deactivate 29 | end 30 | end 31 | 32 | # unset irrelavent variables 33 | deactivate nondestructive 34 | 35 | set -gx VIRTUAL_ENV "/Users/vboykis/Desktop/soviet_art_bot" 36 | 37 | set -gx _OLD_VIRTUAL_PATH $PATH 38 | set -gx PATH "$VIRTUAL_ENV/bin" $PATH 39 | 40 | # unset PYTHONHOME if set 41 | if set -q PYTHONHOME 42 | set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME 43 | set -e PYTHONHOME 44 | end 45 | 46 | if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" 47 | # fish uses a function instead of an env var to generate the prompt. 48 | 49 | # save the current fish_prompt function as the function _old_fish_prompt 50 | . ( begin 51 | printf "function _old_fish_prompt\n\t#" 52 | functions fish_prompt 53 | end | psub ) 54 | 55 | # with the original prompt function renamed, we can override with our own. 56 | function fish_prompt 57 | # Prompt override? 58 | if test -n "(soviet_art_bot) " 59 | printf "%s%s%s" "(soviet_art_bot) " (set_color normal) (_old_fish_prompt) 60 | return 61 | end 62 | # ...Otherwise, prepend env 63 | set -l _checkbase (basename "$VIRTUAL_ENV") 64 | if test $_checkbase = "__" 65 | # special case for Aspen magic directories 66 | # see http://www.zetadev.com/software/aspen/ 67 | printf "%s[%s]%s %s" (set_color -b blue white) (basename (dirname "$VIRTUAL_ENV")) (set_color normal) (_old_fish_prompt) 68 | else 69 | printf "%s(%s)%s%s" (set_color -b blue white) (basename "$VIRTUAL_ENV") (set_color normal) (_old_fish_prompt) 70 | end 71 | end 72 | 73 | set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" 74 | end 75 | -------------------------------------------------------------------------------- /lambda_deploy.sh: -------------------------------------------------------------------------------- 1 | lambda_project_home="$(pwd)" 2 | dist_dir_name="dist" 3 | proj_file_names=("lambda") 4 | pip_env_dir_name="dev" 5 | n_libs_dir_name="native_libs" 6 | 7 | 8 | # Use lambda versioning 9 | if [[ $TRAVIS_BRANCH == 'dev' ]]; then 10 | lambda_function_name="soviet_lambda_$TRAVIS_BRANCH" 11 | s3_deploy_bucket="soviet-art-bot-$TRAVIS_BRANCH" 12 | deploy_bundle_name="lambda_bundle_$TRAVIS_BRANCH.zip" 13 | s3_deploy_key=${deploy_bundle_name} 14 | IFS=$'\n' read -d '' -r -a lines < ${lambda_project_home}/.env 15 | elif [[ $TRAVIS_BRANCH == 'master' ]]; then 16 | lambda_function_name="soviet_lambda_$TRAVIS_BRANCH" 17 | s3_deploy_bucket="soviet-art-bot-$TRAVIS_BRANCH" 18 | deploy_bundle_name="lambda_bundle_$TRAVIS_BRANCH.zip" 19 | s3_deploy_key=${deploy_bundle_name} 20 | IFS=$'\n' read -d '' -r -a lines < ${lambda_project_home}/.env 21 | fi 22 | 23 | 24 | 25 | if [ -z "${AWS_CLI_PROFILE}" ]; then 26 | aws_cli_profile="" 27 | else 28 | aws_cli_profile="--profile ${AWS_CLI_PROFILE}" 29 | fi 30 | 31 | dist_path=${lambda_project_home}/${dist_dir_name} 32 | 33 | echo "Cleaning up dist dir ..." 34 | rm -rf ${dist_path}/* 35 | 36 | echo "Adding source files ..." 37 | 38 | for sf in "${proj_file_names[@]}" 39 | do 40 | proj_path=${lambda_project_home}/${sf} 41 | cp -rf ${proj_path} ${dist_path} 42 | done 43 | 44 | 45 | echo "Adding pip libs ..." 46 | 47 | env_path=${lambda_project_home}/${pip_env_dir_name} 48 | cp -rf ${env_path}/lib/python3.5/site-packages/* ${dist_path} 49 | 50 | if [ -z "$n_libs_dir_name" ]; then 51 | echo "n_libs_dir_name is unset"; 52 | else 53 | n_libs_path=${lambda_project_home}/${n_libs_dir_name} 54 | echo "Adding native libs" 55 | cp -rf ${n_libs_path}/* ${dist_path} 56 | fi 57 | 58 | echo "Create deployment package folder ..." 59 | 60 | cd ${dist_path} 61 | 62 | zip -q -r ${deploy_bundle_name} . 63 | 64 | mv ${deploy_bundle_name} ${lambda_project_home}/ 65 | 66 | cd - 67 | 68 | echo "Uploading deployment package to S3 ..." 69 | 70 | deploy_bundle_path=${lambda_project_home}/${deploy_bundle_name} 71 | aws s3 cp ${deploy_bundle_path} s3://${s3_deploy_bucket}/${s3_deploy_key} \ 72 | ${aws_cli_profile} \ 73 | && echo "Successful Upload" || (echo "Failed" && exit 1) 74 | 75 | echo "Updating Lambda functions ..." 76 | 77 | aws lambda update-function-code --function-name ${lambda_function_name} \ 78 | --region 'us-east-1' \ 79 | --s3-bucket ${s3_deploy_bucket} --s3-key ${s3_deploy_key} \ 80 | --publish ${aws_cli_profile} \ 81 | && echo "Deployment completed successfully" || (echo "Failed" && exit 1) 82 | 83 | aws lambda update-function-configuration \ 84 | --region 'us-east-1' \ 85 | --function-name ${lambda_function_name} \ 86 | --environment "Variables={CONSUMER_KEY=${lines[2]},CONSUMER_SECRET=${lines[3]},ACCESS_TOKEN=${lines[1]},ACCESS_SECRET=${lines[0]}}" \ 87 | && echo "Variable Update Completed Successfully" || (echo "Failed" && exit 1) 88 | 89 | echo "Functions and variables updated ..." -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: 3 | - 3.6 4 | install: 5 | - pip install . 6 | - pip install -r requirements.txt 7 | - pip install awscli 8 | - pip install twython -t lambda/ 9 | before_script: export PYTHONPATH=$PYTHONPATH:$(pwd) 10 | script: 11 | - bash build_env.sh 12 | - bash ecr_credentials.sh 13 | - bash lambda_deploy.sh 14 | branches: 15 | only: 16 | - master 17 | - dev 18 | 19 | env: 20 | global: 21 | - secure: WazFdG9OdKY2MU/4KXbjS8adFSbdxsji27C5M41cSIVBfcEUrOvManMhOP6VpykdHcUXg/IYPTMqGs33dUXq+abZ0fxARdAZjpfAAY8lx95vKMmQxsgoqExWTzjZoXC+uEYSGy9gnSJs03sFkwmz0dfB/FLqXrgHa00oucQqFSzdruJW0Tsnq0kw1uVwFSUItfztsoEwrxQAsyy+pAW9Gs3pLVeUVzKDPXd/YVc1bOuufuMSw5nUQaeI6Ah8IAxQom5uws1X9QfDMRbIiaY2FHInusJP9bQPlVfiqzhHsLNLM1L4z7/3EIuI9zLJj3b9MJCmyuTsWQ4FRkeQ8ZZYDPHw/B2DlSDMXK1xfHfPZYPDBNxdq//dDJwB7rmWjB1fftNpZEqf4GvErT4iAj/hEezAyFT9A4aLO8BqqcSanahBT8nxWDykTM3DjYEEXCFoqZnoAMHYZ1dUZfPfKI7qTJHm00YZ/XyClto9FsFz8GvyVvSB9v3BUi/K8lMT8TY5+ejKFSKomC3JS5DhKB0Pnig/y+sjRN8BzDZA1DrbiyFwrgYXjw5GKNwAq1luPJ397BapIMI52QY469NrjRhtvGc+KSapJm7BZPIWB3TTs4yLDy8BvTcLd9JPp1Nd1bBuwxQ5dC/7un8xM7lzKYvmZNNF2YZynsl23mEg+cuPyxw= 22 | - secure: KS/2FcbZg57BVX+iyVfaZIkWoxyO0cg3qOiYJccRc9oKVS7mBfnZ0KRgetfBrNl86CeGa+DEIdywrTZWle1Bfc38N9MgZuzYe831q/JlABcKgH7EC7XMM0PBq/d3rus4p73i4mcvvyq3Q8XZSI/7GitjteUZ9ILL+a/YmOtJEcdyy1wUz6ncRTJOVzWZfclOM+LDJijQritVJmCFCqKacuRGLRdbVE0AZ4RWJUdworXTghzRt9q+Eoi5E4CNg6QQkUi0d403YTX4kJWiIa/yXBEE6UjhdXLQXwuRJj/0sjLzIcYQ3mElRmKNkrVzKf/jA6iHqLVYBTHymXfkY7Cfz1AhNTo7DPctd9HMy5JQIu4aXQ0Maj0IOUqIUUJJ3Pj2+m7tQA3LSk0iUnIVsUit6n0zW/Tfh2ztpT0y0UTejZsXYcstk7nABX/5AqdUKkBCwbffjLXwyDozZ1Y+cMcyRvhjWEYv1HTALo1CRoYVoDkAtd8h3RY79K40yBcpHy5WqJfVAXscng1FVVC/TlmYXg0H0A486NoJ5JixvWYZfqtQXgYXxvEKrp6a09MKe6fCHdCZQ4mGl5t0k45jjgKrXcaGXX4OoAlQyhlY+6f4r1FPOO2UxpLqjlfNlDXZqbD7Yk5LuzGrnCSfOH5XMnP3iadetSdU8iqEG0BDYeW1SaI= 23 | - secure: uOSSOBVwy2SaTFJhsi0LL4l/qqAwU8B3cgn8NEOCIV1Yo9bRE54New06anTDW8u5wW2c1XESfq/Fr8ROOUfuceaj1NuaEoqWh3eEnjCLjhT4HL8QIdwVpT6t47G+Ug70yWkqHC3KQCbwYHAtWGqhdicxDTu1AiirWesc/vHt1WGjrwvnCY3V5pJ0GKL125zkBoeapqqGruMFaMIAJudzmgCm3TXWHpBMOi2/f65htecVA8nkEj/iPRD/XnR09JwSoIRcSASk0DWrDXd5DJHJkLTDKapwKC0U158LHGP59Lhf+1s4lKtrKnhKy39AW5yR9lWRw1TiFswI0r18kiWlNQRRE+SmLfiZz4BBazsZxAMsJIIMfqCCyVwQ9TpqghRm0B3Hptnhh2CWDAHQB5I/NGeAwsS5fJ4fjuvZmw9CzVerXHSAK4ETAGcphmACP2wYssrWvZ3epPLMlJbjy9t7sy9VAolW5RdJ38q7PzgU3wYWDMqgD3hhtBoWDx9FtftIIPk8OXt2J0rQhAE7KrkJnhGUvOKA4lZ9xr4j1BwrOhrmGobe4uYK0uUS42B5dfyg6g4jNiFdORJ/u4z14F9rHHXUfZD2BHVOrU5BHKY55SIqZdXtmexVhM3UGnTafWRpUVMQIm1nkDd02sMu2ZSsVeOmxE2MpV5oMJrbUy+5uQ8= 24 | - secure: OdkdcA+CQv1XpdsnIGnQnBUO1hN4TxSXCxCYQBnXRuN4lc289lokKWfoJpjJH5Jy8s/3fPj9xkDcmiFVY3/vdR6qcdoGy346Eq3qN/L1XVpJNa2DlvAM6cj2bmmmWzfmhf1zcSMaDbXtnIVXVthIt1VgbFPZYQjJQa99HEmNk6vH2yUK2YbDAwKhgK5f6cruVbMwMdieYPzSl08o61GJHR8OrQqm1GLn4BA2xVgDCD6N6y2goIJvbfB31Xcn7AgphTfpVmUUgbJwfpRinZzLf+o/B6/sTSwCV6/lcYgazNchUPMYrC1qV5UIMjFUPBVGxfrs8s3Wglh0zzqSEQpw8m5sACi7KoiDQyiee1Pjx+SL+N1rUFSGgLmPx+01DJQ2suhwX1OVBDeqjohBeU81wyCYbJA8A3S8k4N1iaM3TM++UrXf8ZRHTd43z6Hc4pBeyJi+TRY1UJEcc83W0TDmonY28xHP178fvjCDwTxagB7zZxHPuQNfgyrt2UCeZu2GLtG2+frhG/bu1Zb04CazmUk2oQDXqL6d5RNVCaSFPB+u9Uqm5LjEssVLHzsOOyhHfc9zeVsNX3TR3GXgyMEiSMCSut2HSTFxCa9+1E2t6O0ZqZu+2qGcf0vALGqngQ55X0Hq7IcXoWL9fQXUEED/lYzV7CV9R9W2QTS7SA86EtA= -------------------------------------------------------------------------------- /lambda/lambda_function.py: -------------------------------------------------------------------------------- 1 | #system imports 2 | import json 3 | import boto3 4 | from botocore.exceptions import ClientError 5 | import tempfile 6 | import os 7 | import subprocess 8 | from six.moves.html_parser import HTMLParser 9 | import random 10 | from collections import defaultdict 11 | from twython import Twython, TwythonError 12 | import html 13 | 14 | 15 | session = boto3.Session() 16 | s3_resource = boto3.resource('s3', region_name='us-east-1') 17 | s3 = boto3.client('s3') 18 | ssm = boto3.client('ssm') 19 | h = HTMLParser() 20 | 21 | 22 | def lambda_handler(event, context): 23 | bucket_name = 'soviet-art-bot' 24 | key = 'art_metadata.json' 25 | 26 | try: 27 | # load json metadata from S3 bucket into JSON 28 | data = s3.get_object(Bucket=bucket_name, Key=key) 29 | json_data = json.loads(data['Body'].read().decode('utf-8')) 30 | except Exception as e: 31 | print(e) 32 | raise e 33 | 34 | print("Got keys") 35 | 36 | indexed_json = defaultdict() 37 | 38 | for value in json_data: 39 | artist = value['artistName'] 40 | title = value['title'] 41 | title = html.unescape(title) 42 | title = html.unescape(title) 43 | year = value['year'] 44 | values = [artist, title, year] 45 | 46 | # return only image name at end of URL 47 | find_index = value['image'].rfind('/') 48 | img_suffix = value['image'][find_index + 1:] 49 | img_link = img_suffix 50 | 51 | try: 52 | indexed_json[img_link].append(values) 53 | except KeyError: 54 | indexed_json[img_link] = (values) 55 | 56 | # Shuffle images 57 | single_image_metadata = random.choice(list(indexed_json.items())) 58 | 59 | url = single_image_metadata[0] 60 | painter = single_image_metadata[1][0] 61 | title= single_image_metadata[1][1] 62 | year = single_image_metadata[1][2] 63 | 64 | print(url, painter, title,year) 65 | 66 | # Connect to Twitter via Twython 67 | 68 | CONSUMER_KEY = os.environ['CONSUMER_KEY'] 69 | CONSUMER_SECRET = os.environ['CONSUMER_SECRET'] 70 | ACCESS_TOKEN = os.environ['ACCESS_TOKEN'] 71 | ACCESS_SECRET = os.environ['ACCESS_SECRET'] 72 | 73 | try: 74 | twitter = Twython(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_SECRET) 75 | print(twitter) 76 | except TwythonError as e: 77 | print(e) 78 | 79 | #Try tweeting 80 | try: 81 | 82 | tmp_dir = tempfile.gettempdir() 83 | # subprocess.call('rm -rf /tmp/*', shell=True) 84 | path = os.path.join(tmp_dir, url) 85 | print(path) 86 | 87 | # Try to match URL in filepath to URL in metadata; if it doesn't work, try another one 88 | for i in range(0, 3): 89 | try: 90 | x = s3_resource.Bucket(bucket_name).download_file(url, path) 91 | print("file moved to /tmp") 92 | print(os.listdir(tmp_dir)) 93 | 94 | with open(path, 'rb') as img: 95 | print("Path", path) 96 | twit_resp = twitter.upload_media(media=img) 97 | twitter.update_status(status="\"%s\"\n%s, %s" % (title, painter, year), 98 | media_ids=twit_resp['media_id']) 99 | except ClientError as e: 100 | if e.response['Error']['Code'] == 'ResourceNotFoundException': 101 | continue 102 | break 103 | 104 | 105 | except TwythonError as e: 106 | print(e) 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /soviet_art_bot/download_images.py: -------------------------------------------------------------------------------- 1 | """ 2 | This code queries WikiArt and returns all paintings and metadata 3 | in the Socialist Realism category 4 | """ 5 | 6 | import json 7 | import shutil 8 | import sys 9 | from glob import glob 10 | import boto3 11 | import requests 12 | 13 | import PIL 14 | from PIL import Image 15 | 16 | import settings 17 | 18 | # intialize connection to S3 resources 19 | s3 = boto3.resource('s3') 20 | s3_client = boto3.client('s3', 'us-east-1') 21 | 22 | def parse_data(paints_list,data): 23 | """ 24 | Extends a list of paintings 25 | :param paints_list: 26 | :param data: 27 | :return: 28 | """ 29 | paints_list.extend(data) 30 | 31 | def get_json(): 32 | """ 33 | Get JSON with art name and location from WikiArt site 34 | :return: dictionary of filenames from WikiArt 35 | """ 36 | data_list = [] 37 | 38 | for page in range(1,10): 39 | url = settings.BASE_URL + settings.STYLE_URL + "&" + settings.PAGINATION_URL + str(page) 40 | print(page, "pages processed") 41 | try: 42 | response = requests.get(url, timeout=settings.METADATA_REQUEST_TIMEOUT) 43 | data = response.json()['Paintings'] 44 | parse_data(data_list, data) 45 | except requests.exceptions.RequestException as e: 46 | print(e) 47 | sys.exit(1) 48 | 49 | return data_list 50 | 51 | 52 | def save_json(data): 53 | """ 54 | Converts list to JSON, writes to file 55 | :param data: Data (list) 56 | :return: 57 | """ 58 | data = json.dumps(data) 59 | 60 | with settings.MEDTADATA_FILE.open('w') as outfile: 61 | outfile.write(data) 62 | 63 | def get_image_links(data): 64 | """ 65 | Passes in a list of image links 66 | :param data: Data (list) 67 | :return: List of painting links 68 | """ 69 | 70 | painting_links = [] 71 | 72 | print(data) 73 | 74 | for painting in data: 75 | parse_data(painting_links, painting['image']) 76 | 77 | return painting_links 78 | 79 | 80 | def download_images(links): 81 | """ 82 | Passes in a list of links pointing to image files to download 83 | :param links (list): 84 | :return Images downloaded into the assets folder: 85 | """ 86 | 87 | for link in links: 88 | print("Processing", link) 89 | try: 90 | response = requests.get(link, 91 | timeout=settings.METADATA_REQUEST_TIMEOUT, stream=True) 92 | except requests.exceptions.RequestException as e: 93 | print(e) 94 | sys.exit(1) 95 | 96 | image_name = link.rsplit('/', 1)[1] 97 | 98 | file_location = settings.ASSET_PATH.joinpath(image_name) 99 | 100 | with open(str(file_location), 'wb') as outfile: 101 | shutil.copyfileobj(response.raw, outfile) 102 | 103 | 104 | def upload_images_to_s3(directory): 105 | """ 106 | Upload images to S3 bucket if they end with png or jpg 107 | :param directory: 108 | :return: null 109 | """ 110 | 111 | for f in directory.iterdir(): 112 | if str(f).endswith(('.png', '.jpg', '.jpeg')): 113 | full_file_path = str(f.parent) + "/" + str(f.name) 114 | file_name = str(f.name) 115 | s3_client.upload_file(full_file_path, settings.BASE_BUCKET, file_name) 116 | print(f,"put") 117 | 118 | 119 | def upload_json_to_s3(directory): 120 | """ 121 | Upload metadata json to directory 122 | :param directory: 123 | :return: null 124 | """ 125 | 126 | for f in directory.iterdir(): 127 | if str(f).endswith('.json'): 128 | full_file_path = str(f.parent) + "/" + str(f.name) 129 | file_name = str(f.name) 130 | s3_client.upload_file(full_file_path, settings.BASE_BUCKET, file_name) 131 | 132 | #TODO: CLEAN UP RESIZING UTILITY 133 | 134 | def resize_images(): 135 | pass 136 | # if os.path.getsize(filename) > (max_size * 1024): 137 | # basewidth = 300 138 | # img = Image.open('fullsized_image.jpg') 139 | # wpercent = (basewidth / float(img.size[0])) 140 | # 
hsize = int((float(img.size[1]) * float(wpercent))) 141 | # img = img.resize((basewidth, hsize), PIL.Image.ANTIALIAS) 142 | # 
img.save('resized_image.jpg') 143 | 144 | def main(): 145 | pass 146 | # data = get_json() 147 | # files = save_json(data) 148 | # links = get_image_links(data) 149 | # # download_images(links) 150 | # # upload_images_to_s3(settings.ASSET_PATH) 151 | # # upload_json_to_s3(settings.ASSET_PATH) 152 | 153 | if __name__ == '__main__': 154 | main() 155 | 156 | 157 | 158 | 159 | 160 | --------------------------------------------------------------------------------