├── .gitignore ├── Dockerfile ├── README.md ├── da_project ├── __init__.py ├── analysis │ ├── .gitkeep │ └── __init__.py ├── data │ ├── .gitkeep │ └── __init__.py ├── hello_world.py └── visualization │ └── .gitkeep ├── data ├── .gitignore ├── interim │ └── .gitignore ├── output │ └── .gitignore └── raw │ └── .gitignore ├── deliverables ├── .gitkeep └── figures │ └── .gitkeep ├── docker ├── .gitignore ├── build ├── jupyter └── run ├── guide └── README.md ├── models └── .gitkeep ├── notebooks ├── .gitkeep └── Hello world.ipynb ├── references └── .gitkeep ├── requirements.txt ├── scripts ├── .gitignore └── .gitkeep └── test_environment.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | *.egg-info/ 23 | .installed.cfg 24 | *.egg 25 | 26 | # PyInstaller 27 | # Usually these files are written by a python script from a template 28 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 29 | *.manifest 30 | *.spec 31 | 32 | # Installer logs 33 | pip-log.txt 34 | pip-delete-this-directory.txt 35 | 36 | # Unit test / coverage reports 37 | htmlcov/ 38 | .tox/ 39 | .coverage 40 | .coverage.* 41 | .cache 42 | nosetests.xml 43 | coverage.xml 44 | *,cover 45 | 46 | # Translations 47 | *.mo 48 | *.pot 49 | 50 | # Django stuff: 51 | *.log 52 | 53 | # Sphinx documentation 54 | docs/_build/ 55 | 56 | # PyBuilder 57 | target/ 58 | 59 | # DotEnv configuration 60 | .env 61 | 62 | # Database 63 | *.db 64 | *.rdb 65 | 66 | # Pycharm 67 | .idea 68 | 69 | # VS Code 70 | .vscode/ 71 | 72 | # Spyder 73 | .spyproject/ 74 | 75 | # Jupyter NB Checkpoints 76 | .ipynb_checkpoints/ 77 | 78 | # bash history 79 | .bash_history 80 | 81 | #.swp files 82 | *.swp 83 | 84 | # exclude data from source control by default 85 | # /data/ 86 | 87 | # conda 88 | .conda/ 89 | 90 | .ipython/ 91 | .jupyter/ 92 | .local/ -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.6-slim-stretch 2 | 3 | LABEL maintainer="Simon Kassel " 4 | 5 | ADD requirements.txt /tmp/requirements.txt 6 | 7 | RUN apt-get update && \ 8 | apt-get install -y \ 9 | wget \ 10 | build-essential \ 11 | make \ 12 | gcc \ 13 | locales \ 14 | libgdal20 libgdal-dev \ 15 | python-dev \ 16 | protobuf-compiler \ 17 | libprotobuf-dev \ 18 | libtokyocabinet-dev \ 19 | python-psycopg2 \ 20 | libspatialindex-dev && \ 21 | python -m pip install numpy cython --no-binary numpy,cython && \ 22 | python -m pip install \ 23 | "rasterio>=1.0a12" fiona shapely rtree \ 24 | --pre --no-binary rasterio,fiona,shapely && \ 25 | python -m pip install -r /tmp/requirements.txt && \ 26 | python -m pip uninstall -y cython && \ 27 | rm -r /root/.cache/pip && \ 28 | apt-get remove -y --purge libgdal-dev make gcc build-essential && \ 29 | apt-get autoremove -y && \ 30 | rm -rf /var/lib/apt/lists/* 31 | 32 | RUN dpkg-reconfigure locales && \ 33 | locale-gen C.UTF-8 && \ 34 | /usr/sbin/update-locale LANG=C.UTF-8 35 | 36 | ENV LC_ALL C.UTF-8 37 | ENV PYTHONPATH="$PYTHONPATH:/home/user/" 38 | 39 | # Open Ports for Jupyter 40 | EXPOSE 8888 41 | 42 | RUN useradd -d /home/user -m -s /bin/bash user && echo "user:user" | chpasswd && adduser user sudo 43 | 44 | # start working in the "tester" home directory 45 | WORKDIR /home/user 46 | 47 | # Make the files owned by tester 48 | RUN chown -R user:user /home/user 49 | 50 | # Switch to your new user in the docker image 51 | USER user 52 | 53 | # Run a shell script 54 | CMD ["/bin/bash"] 55 | 56 | 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Azavea Data Analytics team python project template 2 | ============================== 3 | 4 | A file structure template, development environment and rule set for python data analytics projects on the data analytics team 5 | 6 | Getting Started 7 | ------------ 8 | Change the name of folder that contains this whole repo: `python-project-template` -> `{your project name}` 9 | 10 | From within the repo directory, first remove git tracking from the project 11 | 12 | `rm -rf .git` 13 | 14 | The project template uses a placeholder name of 'da-project'. Change that name in the following files/directories (relative to the repo root): 15 | - `da-project/` (change the name of the folder) 16 | - `./docker/run/` 17 | - `./docker/build/` 18 | 19 | If you have not already done so, build the Docker image (you will only need to do this once) 20 | 21 | `docker/build` 22 | 23 | Run a Docker container: 24 | 25 | `docker/run` 26 | 27 | This will open a bash shell within the Docker container. Within the container the 'project' directory on the host machine (as specified as a parameter of `run` above) will map to `/opt/src/` within the container. You can now access the full file structure of this template from within the container. 28 | 29 | Run a Jupyter Notebook within Docker container: 30 | 31 | `docker/jupyter` 32 | 33 | You will need to open the link that is displayed in your terminal. 34 | 35 | To exit: 36 | 37 | `exit` 38 | 39 | Initialize a new git repository: 40 | 41 | `git init` 42 | 43 | Project Organization 44 | ------------ 45 | 46 | ├── README.md <- The top-level README for developers using this project. 47 | ├── data 48 | │   ├── interm <- Intermediate data that has been transformed 49 | │   ├── processed <- The final, canonical data sets for modeling 50 | │   └── raw <- The original, immutable data dump 51 | │ 52 | ├── guide <- A set of markdown files with documented best practices, guidelines and rools for collaborative projects 53 | │ 54 | ├── models <- Trained and serialized models, model predictions, or model summaries 55 | │ 56 | ├── notebooks <- Jupyter notebooks. Naming convention is a number (for ordering), 57 | │ the creator's initials, and a short `-` delimited description, e.g 58 | │ `1.0-jqp-initial-data-exploration` 59 | │ 60 | ├── references <- Data dictionaries, manuals, and all other explanatory materials. 61 | │ 62 | ├── reports <- Generated analysis as HTML, PDF, LaTeX, etc. 63 | │   └── figures <- Generated graphics and figures to be used in reporting 64 | │ 65 | ├── requirements.txt <- The requirements file for reproducing the analysis environment 66 | │ 67 | └── da-project <- Source code for use in this project. 68 | │ 69 |    ├── data <- Scripts to download or generate data 70 |    │   └── make_dataset.py 71 | │ 72 |    ├── features <- Scripts to turn raw data into features for modeling 73 |    │   └── build_features.py 74 | │ 75 |    ├── models <- Scripts to train models and then use trained models to make 76 | │ │ predictions 77 |    │   ├── predict_model.py 78 |    │   └── train_model.py 79 | │ 80 |    └── visualization <- Scripts to create exploratory and results oriented visualizations 81 |    └── visualize.py 82 | 83 | 84 | 85 | -------- 86 | 87 |

Project based on the cookiecutter data science project template. #cookiecutterdatascience

88 | -------------------------------------------------------------------------------- /da_project/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/da_project/__init__.py -------------------------------------------------------------------------------- /da_project/analysis/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/da_project/analysis/.gitkeep -------------------------------------------------------------------------------- /da_project/analysis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/da_project/analysis/__init__.py -------------------------------------------------------------------------------- /da_project/data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/da_project/data/.gitkeep -------------------------------------------------------------------------------- /da_project/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/da_project/data/__init__.py -------------------------------------------------------------------------------- /da_project/hello_world.py: -------------------------------------------------------------------------------- 1 | def hello_world(): 2 | print('Hello world') -------------------------------------------------------------------------------- /da_project/visualization/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/da_project/visualization/.gitkeep -------------------------------------------------------------------------------- /data/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | !raw/ 6 | !interim/ 7 | !output/ 8 | -------------------------------------------------------------------------------- /data/interim/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /data/output/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /data/raw/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /deliverables/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/deliverables/.gitkeep -------------------------------------------------------------------------------- /deliverables/figures/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/deliverables/figures/.gitkeep -------------------------------------------------------------------------------- /docker/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /docker/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker build -t da-project . -------------------------------------------------------------------------------- /docker/jupyter: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # run container 4 | docker run -it --rm \ 5 | -p 8888:8888 \ 6 | -v $(pwd $1):/home/user:rw \ 7 | da-project \ 8 | jupyter notebook --allow-root --port=8888 --ip=0.0.0.0 --notebook-dir=/home/user/notebooks 9 | -------------------------------------------------------------------------------- /docker/run: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # run container 4 | docker run -it --rm \ 5 | -p 8888:8888 \ 6 | -v $(pwd $1):/home/user:rw \ 7 | da-project 8 | 9 | -------------------------------------------------------------------------------- /guide/README.md: -------------------------------------------------------------------------------- 1 | # Rules & Best Practices 2 | 3 | A markdown documentation of rules, guidelines and best practices for working on collaborative data analysis projects on the Data Analytics team 4 | -------------------------------------------------------------------------------- /models/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/models/.gitkeep -------------------------------------------------------------------------------- /notebooks/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/notebooks/.gitkeep -------------------------------------------------------------------------------- /notebooks/Hello world.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "### Load from da_project module" 8 | ] 9 | }, 10 | { 11 | "cell_type": "code", 12 | "execution_count": null, 13 | "metadata": {}, 14 | "outputs": [], 15 | "source": [ 16 | "from da_project.hello_world import hello_world" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "hello_world()" 26 | ] 27 | } 28 | ], 29 | "metadata": { 30 | "kernelspec": { 31 | "display_name": "Python 3", 32 | "language": "python", 33 | "name": "python3" 34 | }, 35 | "language_info": { 36 | "codemirror_mode": { 37 | "name": "ipython", 38 | "version": 3 39 | }, 40 | "file_extension": ".py", 41 | "mimetype": "text/x-python", 42 | "name": "python", 43 | "nbconvert_exporter": "python", 44 | "pygments_lexer": "ipython3", 45 | "version": "3.6.9" 46 | } 47 | }, 48 | "nbformat": 4, 49 | "nbformat_minor": 2 50 | } 51 | -------------------------------------------------------------------------------- /references/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/references/.gitkeep -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | rasterio>=1.0a12 2 | fiona==1.8.* 3 | shapely==1.7.* 4 | pandas==0.25.* 5 | geopandas==0.5.* 6 | pyproj==2.3.* 7 | rasterstats==0.13.* 8 | jupyter==1.0.* 9 | scipy==1.3.* 10 | scikit-learn==0.21.* 11 | seaborn==0.9.* 12 | statsmodels==0.10.* 13 | requests==2.22.* 14 | numpy==1.17.* 15 | descartes==1.1.* 16 | 17 | -------------------------------------------------------------------------------- /scripts/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /scripts/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/azavea/python-project-template/6791d8153a7197052c3f8592c0625f09c7fc12dc/scripts/.gitkeep -------------------------------------------------------------------------------- /test_environment.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | REQUIRED_PYTHON = "python3" 4 | 5 | def test_python_geospatial(): 6 | # TODO: implement this 7 | pass 8 | 9 | def test_python_version(): 10 | system_major = sys.version_info.major 11 | if REQUIRED_PYTHON == "python": 12 | required_major = 2 13 | elif REQUIRED_PYTHON == "python3": 14 | required_major = 3 15 | else: 16 | raise ValueError("Unrecognized python interpreter: {}".format( 17 | REQUIRED_PYTHON)) 18 | 19 | if system_major != required_major: 20 | raise TypeError( 21 | "This project requires Python {}. Found: Python {}".format( 22 | required_major, sys.version)) 23 | else: 24 | print(">>> Development environment passes all tests!") 25 | 26 | 27 | if __name__ == '__main__': 28 | test_python_geospatial() 29 | test_python_version() 30 | --------------------------------------------------------------------------------