├── .devcontainer ├── Dockerfile ├── devcontainer.env ├── devcontainer.json ├── install_dependencies.sh └── requirements.txt ├── README.md ├── diagrams ├── .gitignore └── docker-workflow.drawio ├── examples ├── ex-1 │ └── Dockerfile ├── ex-2 │ └── Dockerfile └── ex-3 │ └── .devcontainer │ ├── Dockerfile │ └── devcontainer.json ├── images ├── command-palette.png ├── dev_container_symbol.png ├── docker layers 1.png ├── docker layers 2.png ├── docker-architecture.png ├── docker-install.png ├── docker-layers.png ├── docker-workflow.png ├── dockerfile to container.png ├── open_dev_container.gif ├── open_dev_container.mov ├── open_dev_container2.gif ├── open_dev_container2a.mov ├── python.png ├── vscode-download.png └── vscode-extensions.png └── tests └── test1.py /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | # Arguments 4 | ARG PYTHON_VER 5 | ARG ENV_NAME 6 | 7 | # Environment variables 8 | ENV ENV_NAME=$ENV_NAME 9 | ENV PYTHON_VER=$PYTHON_VER 10 | 11 | # Copy files 12 | RUN mkdir requirements 13 | COPY requirements.txt requirements/ 14 | COPY install_dependencies.sh requirements/ 15 | 16 | # Install dependencies 17 | RUN bash requirements/install_dependencies.sh $ENV_NAME $PYTHON_VER -------------------------------------------------------------------------------- /.devcontainer/devcontainer.env: -------------------------------------------------------------------------------- 1 | VAR1=var_1 2 | VAR2=var_2 -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "${localEnv:ENV_NAME}", 3 | "build": { 4 | "dockerfile": "Dockerfile", 5 | "args": {"ENV_NAME": "${localEnv:ENV_NAME}", 6 | "PYTHON_VER": "${localEnv:PYTHON_VER}"}, 7 | "context": "." 8 | }, 9 | "customizations": { 10 | "settings": { 11 | "python.defaultInterpreterPath": "/opt/conda/envs/${localEnv:ENV_NAME}/bin/python" 12 | }, 13 | "vscode": { 14 | "extensions": [ 15 | "quarto.quarto", 16 | "ms-azuretools.vscode-docker", 17 | "ms-python.python", 18 | "ms-vscode-remote.remote-containers", 19 | "yzhang.markdown-all-in-one", 20 | "redhat.vscode-yaml", 21 | "ms-toolsai.jupyter" 22 | ] 23 | } 24 | }, 25 | 26 | "mounts": [ 27 | "source=${localEnv:CSV_PATH},target=/home/csv,type=bind,consistency=cache" 28 | ], 29 | "remoteEnv": { 30 | "MY_VAR": "${localEnv:MY_VAR}" 31 | }, 32 | "runArgs": ["--env-file",".devcontainer/devcontainer.env"], 33 | "postCreateCommand": "python3 tests/test1.py" 34 | } -------------------------------------------------------------------------------- /.devcontainer/install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ENV_NAME=$1 4 | PYTHON_VER=$2 5 | CPU=$(uname -m) 6 | 7 | 8 | # Installing prerequisites 9 | apt-get update && \ 10 | apt-get install -y \ 11 | python3-launchpadlib \ 12 | vim \ 13 | && apt update 14 | 15 | 16 | # Install miniconda 17 | apt update && apt-get install -y --no-install-recommends \ 18 | software-properties-common \ 19 | && add-apt-repository -y ppa:deadsnakes/ppa \ 20 | && apt update 21 | 22 | wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${CPU}.sh -O ~/miniconda.sh \ 23 | && /bin/bash ~/miniconda.sh -b -p /opt/conda \ 24 | && export PATH=/opt/conda/bin:$PATH \ 25 | && conda init bash \ 26 | && conda install conda-build 27 | 28 | # Set environment 29 | . /root/.bashrc \ 30 | && conda create -y --name $ENV_NAME python=$PYTHON_VER 31 | 32 | echo "conda activate $ENV_NAME" >> ~/.bashrc 33 | 34 | conda activate $ENV_NAME 35 | 36 | # # Install the Python packages 37 | pip3 install -r /requirements/requirements.txt -------------------------------------------------------------------------------- /.devcontainer/requirements.txt: -------------------------------------------------------------------------------- 1 | wheel==0.40.0 2 | pandas==2.0.3 3 | plotly==5.15.0 4 | plotly-express==0.4.1 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Setting Python Development Environment with VScode and Docker 2 | 3 | This repo provides a step-by-step guide and a template for setting up a Python 🐍 development environment with VScode and Docker 🐳. Docker is an amazing tool, but it has some learning curve. This tutorial aims to reduce the entry barrier for learning tools such as Docker. 4 | 5 | See also: 6 | - [A Dockerized Python Development Environment Template](https://github.com/RamiKrispin/vscode-python-template) 7 | - [A Dockerized Python Development Environment Template - tutorial](https://medium.com/@rami.krispin/setting-a-dockerized-python-development-environment-template-de2400c4812b) 8 | - [Setting R Development with VScode and Docker](https://github.com/RamiKrispin/vscode-r) 9 | - [Running Python/R with Docker vs. Virtual Environment](https://medium.com/@rami.krispin/running-python-r-with-docker-vs-virtual-environment-4a62ed36900f) 10 | - [Deploy Flexdashboard on Github Pages with Github Actions and Docker](https://github.com/RamiKrispin/deploy-flex-actions) 11 | - [Docker for Data Scientists 🐳](https://github.com/RamiKrispin/Introduction-to-Docker) (WIP) 12 | 13 | 14 |
15 | 16 |
17 | 18 | ## Table of contents 19 | - [Motivation](https://github.com/RamiKrispin/vscode-python/tree/main#motivation) 20 | - [Scope](https://github.com/RamiKrispin/vscode-python/tree/main#scope) 21 | - [Prerequisites](https://github.com/RamiKrispin/vscode-python/tree/main#prerequisites) 22 | - [General Architecture and Workflow](https://github.com/RamiKrispin/vscode-python/tree/main#general-architecture-and-workflow) 23 | - [Getting Started with Docker](https://github.com/RamiKrispin/vscode-python/tree/main#getting-started-with-docker) 24 | - [Running Python on Docker - the Hard Way](https://github.com/RamiKrispin/vscode-python/tree/main#running-python-on-docker---the-hard-way) 25 | - [Setting the Dev Containers Extension](https://github.com/RamiKrispin/vscode-python/tree/main#setting-the-dev-containers-extension) 26 | - [Setting the Python Environment](https://github.com/RamiKrispin/vscode-python/tree/main#setting-the-python-environment) 27 | - [Summary](https://github.com/RamiKrispin/vscode-python/tree/main#summary) 28 | - [Resources](https://github.com/RamiKrispin/vscode-python/tree/main#resources) 29 | - [License](https://github.com/RamiKrispin/vscode-python/tree/main#license) 30 | 31 | ## Motivation 32 | 33 | If you have never used Docker or VScode, this section explains the advantages of each tool and why they work so well together. 34 | 35 | ### Why develop with Docker 🐳? 36 | 37 | Docker is a CI/CD tool that enables seamless code deployment from development to production environments. By creating OS-level virtualization, it can package an application and its dependencies in a virtual container and ship it between different environments. The main advantages of using Docker within your development environment are: 38 | - **Reproducibility** - Docker enables you to seamlessly package your code and its dependencies into a single container and execute, test, share, and deploy it with a high level of consistency 39 | - **Collaboration** - Docker solves the dependencies madness when a team of developers works together on a specific project. Having a unified environment saves a ton of time during the development step. For example, if one developer gets some error, it is easy for other developers to reproduce the error and help debug it 40 | - **Deployment** - Docker simplify the code shipment from the development environment to the production 41 | 42 | 43 | ### Why VScode 💻? 44 | 45 | VScode (aka Visual Studio Code) is a general-purpose free code editor developed by Microsoft. It can be run locally on Windows, macOS, and Linux OSs or on a cloud environment. The main advantages of using VScode as your IDE are: 46 | - **Multi-languages** - VScode supports out-of-the-box JavaScript, TypeScript, and Node.js, and has a variety of extensions that enable running other programming languages such as Python, C++, C#, Go, etc. 47 | - **Extensions** - VScode has a large community support that builds and maintains a variety of extensions that extend the editor's capabilities and functionality. That includes extensions that support programming languages (such as Python, R, and Julia), plugs-in that enables connection with external applications (Docker, Postgres, etc.), and other applications 48 | - **Git integration** - VScode has built-in integration with Git 49 | - **Docker integration** - VScode supports natively Docker integration, which we will dive deep into in the coming sections 50 | 51 | In addition, VScode is free and open source 🌈. 52 | 53 | ### VScode + Docker + Python= ❤️ ❤️ ❤️ 54 | 55 | Docker's high reproducibility, coupled with the seamless integration of Visual Studio Code with Docker, makes it a great development tool, particularly for Python developers. This integration enables developers to easily create, test, and deploy their Python applications in a consistent and reproducible environment, ensuring that their code runs as expected. 56 | 57 | 58 | ## Scope 59 | 60 | This tutorial focuses on setting up a dockerized Python development environment with VScode. We will explore how to integrate VScode with Docker using Microsoft's [Dev Container](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extensions and show various of methods for configuring a Python environment with Docker. However, it's important to note that while this tutorial covers some of the core Docker features, it is not a complete Docker guide. If you don't have previous experience with Docker, I highly recommend taking one of the Docker crash courses. 61 | 62 | ## Prerequisites 63 | 64 | Generally, this tutorial does not require a previous experience with Docker and VScode. In line with the tutorial scope above, the goal is to teach you how to set up a Python development environment using Docker and VScode. That being said, as mentioned above, you will benefit more from and highly utilize this tutorial if taking some Docker intro courses before starting this tutorial. 65 | 66 | The main requirements for this tutorial are setting VScode and Docker Desktop. In addition, you will need to set up an account to Docker Hub. 67 | 68 | ### Installing VScode 69 | 70 | Installing VScode is straightforward - go to the VScode website https://code.visualstudio.com/ and click on the Download button (purple rectangle on the screenshot): 71 |
72 | 73 |
Figure 1 - Visual Studio Code download page
74 |
75 | 76 | 77 | Download the installation file and follow the instructions. 78 | 79 | To set the Python environment with Docker in VScode we will need the following extensions: 80 | 81 | - **Dev Containers** - this extension enables to open a folder and execute a code inside a Docker container (more info available [here](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers)) 82 | - **Python** - the main Python plug-in for VScode, enables to execute, debugging, code navigation, code formatting, etc. (more info available [here](https://marketplace.visualstudio.com/items?itemName=ms-python.python)) 83 | 84 | Here is how to install an extension on VScode: 85 | - Click the Extensions button on the left menu (mark with a purple arrow on the screenshot below) 86 | - Type the extension name on the search menu (see the yellow rectangular). You can see the search results below, and clicking on each extension will open a window with the extension details. 87 | - Last but not least, Click the install button (see the green rectangular) to install the extension 88 |
89 | 90 |
Figure 2 - Steps to install extension on VScode
91 |
92 | 93 |
94 | 95 | **Note:** The Dev Containers extension is required to launch the dockerized environment. We will see later in this tutorial how to set and install the necessary extensions for your dockerized environment automatically with the `devcontainer.json` file. 96 | 97 | ### Setting Docker 98 | Various ways exist to build and run Docker images on different operations systems. For the purpose of this guide, we will be utilizing Docker Desktop. It is a user-friendly container management interface that is compatible with MacOS, Windows, and Linux operating systems. 99 | 100 | **Note:** Docker Desktop is free for personal use but requires a license for commercial use. For further information, please refer to https://www.docker.com/pricing/. 101 | 102 | To install Docker Desktop, go to Docker website and follow the installation instructions according to your OS: 103 |
104 | 105 |
Figure 3 - Docker Desktop download page
106 |
107 | 108 |
109 | 110 | ### Docker Hub 111 | 112 | Container Registry has a similar functionality as Github for code, and it uses to store and share images. There are many container registries, and the most common is Docker Hub. We will use throughout the tutorial Docker Hub to pull different images, such as Python built-in images. To register and create an account go to https://hub.docker.com and follow the registration instructions. 113 | 114 | After installing Docker Desktop and setting account on Docker Hub, open Docker Desktop, and from the command line, login to Docker Hub: 115 | 116 | ``` shell 117 | docker login 118 | ``` 119 | 120 | You will have to enter your username and password, and you should expect the following output if the login is successful: 121 | 122 | ``` shell 123 | Login with your Docker ID to push and pull images from Docker Hub. If you don't have a Docker ID, head over to https://hub.docker.com to create one. 124 | Username: rkrispin 125 | Password: 126 | Login Succeeded 127 | 128 | Logging in with your password grants your terminal complete access to your account. 129 | For better security, log in with a limited-privilege personal access token. Learn more at https://docs.docker.com/go/access-tokens/ 130 | ``` 131 | **Note:** Docker Hub is completely public (for the free tier). Any image you push and store there will be available for all other users. Regardless if your container registry is public or not, **NEVER** store credentials, passwords, or any other sensitive information on your Docker images. 132 | 133 | 134 | ### Hello World! 135 | 136 | There is no better way to test if Docker was installed properly than by running whalesay (or 🐳 say) Docker's most `Hello World!` common example. The whalesay is an adaption of the Linux cowsay (🐮 say) game using a whale instead of a cow to print some text. Let's run the below code from the terminal to print `Hello Python Users! 👋 🐍`: 137 | 138 | ``` shell 139 | docker run docker/whalesay cowsay Hello Python Users! 👋 🐍 140 | ``` 141 | If this is the first time you are using Docker or your first time using the `whalesay` image you should expect the following message: 142 | 143 | ``` shell 144 | Unable to find image 'docker/whalesay:latest' locally 145 | ``` 146 | 147 | That is a generic message that notifies that the requested image cannot be found locally, and Docker will try to pull the image from the hub (if specified) and follow by downloading the image: 148 | 149 | 150 | ``` shell 151 | latest: Pulling from docker/whalesay 152 | Image docker.io/docker/whalesay:latest uses outdated schema1 manifest format. Please upgrade to a schema2 image for better future compatibility. More information at https://docs.docker.com/registry/spec/deprecated-schema-v1/ 153 | e190868d63f8: Pull complete 154 | 909cd34c6fd7: Pull complete 155 | 0b9bfabab7c1: Pull complete 156 | a3ed95caeb02: Pull complete 157 | 00bf65475aba: Pull complete 158 | c57b6bcc83e3: Pull complete 159 | 8978f6879e2f: Pull complete 160 | 8eed3712d2cf: Pull complete 161 | Digest: sha256:178598e51a26abbc958b8a2e48825c90bc22e641de3d31e18aaf55f3258ba93b 162 | Status: Downloaded newer image for docker/whalesay:latest 163 | ``` 164 | 165 | And this is the expected output: 166 | 167 | ``` 168 | _______________________________ 169 | < Hello Python Users! 👋 🐍 > 170 | ------------------------------- 171 | \ 172 | \ 173 | \ 174 | ## . 175 | ## ## ## == 176 | ## ## ## ## === 177 | /""""""""""""""""___/ === 178 | ~~~ {~~ ~~~~ ~~~ ~~~~ ~~ ~ / ===- ~~~ 179 | \______ o __/ 180 | \ \ __/ 181 | \____\______/ 182 | ``` 183 | 184 | If you are able to run the `whalesay` app you are ready to get started with Docker. 185 | 186 | ## General Architecture and Workflow 187 | 188 | Before diving into the core functionality of Docker, let's review the generic development workflow architecture with Docker. Docker has similar functionality as Git and Github (or Gitlab, Bitbucket, etc.), enabling shifting your environment and settings (as opposed to code with Git) from one environment to another one (e.g., dev -> staging or dev -> prod) ensuring a high level of reproducibility. As a matter of fact those two (Docker and Git) go together side by side. 189 | 190 | 191 | ### General architecture 192 | 193 | The diagram below describes a high-level architecture of a Dockerized development environment with VScode. It might be overwhelming if you have never used Docker before, and it will make more sense (I hope) by the end of this section. 194 |
195 | 196 |
Figure 4 - Development workflow with VScode and Docker
197 |
198 | 199 |
200 | 201 | 202 | This process includes the following components: 203 | - **Dev Container** - is the VScode extension that enables you to execute your local code inside a dockerized environment seamlessly. By default, it mounts your local folder to the docker environment ensuring your code runs inside the container and lives locally. 204 | - **devcontainer.json** - is the Dev Container configuration file that enables you to highly customize your VScode development environment when using the Dev Container extension. From settings the VScode options (e.g., fonts, list of extensions to install, etc.) to the Docker settings (similar to the docker-compose.yml file functionality) 205 | - **Dockerfile** - is the image manifest or recipe. It provides instructions for the docker engine about which base image to use and what components to install. Typically, you start the build process by importing some base image using the `FROM` command, which we will explain later in this tutorial. The Dev Container extension enables you to build the image on the fly when launching the environment using the Dockerfile or import a built-in image from some image registry such as Docker Hub. 206 | - **Image registry** - has similar functionality as Github / Gitlab / Bitbucket, and it is used to store public images (or private on some enterprise versions). The image registry lets you shift and distribute your images from one environment to another. In this tutorial, we will use the main image registry - Docker Hub. 207 | - **Code registry** - beyond version control, it enables you to shift your code from one environment to another. In this tutorial, we will use Github as our code registry. 208 | 209 | **Note:** Regardless if you are using a private or public image registry, as a good practice, you should **NEVER** store passwords, credentials, or any sensitive information on your images. Rather, add environment variables or load information from a mounted volume during the image run time. 210 | 211 | ### Development workflow 212 | Let's now organize and order this process to a general workflow. The below diagram defines a general order of operation for the development process applying the following steps: 213 | - **Install dependencies** - setting the prerequisites, including installing VScode and the required extensions (e.g., Dev Container, etc.), installing Docker, and setting a Docker Hub account (or equivalent) 214 | - **Set the Dockerfile** - this step is optional, if you wish to build your image on the fly. Alternatively, you can import a built-in image from an image registry and skip this step. In the next section, we will dive into more details about the core functionality of the Dockerfile and how to set it 215 | - **Set the devcontainer.json file** - this file defines the environment and enables you to customize both the VScode functionality and VScode settings. Later in this tutorial, we will see how to set this file 216 | - **Development** - once the `devcontainer.json` file is set, you can launch your dockerized development environment using the Dev Container extension on your local folder. 217 | - **Testing** - this is a recommended intermediate step before shipping your code and environment to deployment. There are multiple approaches to code and environment testing, and the main goal is to ensure that your code sync with the dockerized environment and identify potential problems before deploying it. 218 | - **Deployment** - last but not least, using code and container registry (e.g., Github and Docker Hub), we can deploy our code using the same dockerized environment to some remote server (e.g., Github Actions, AWS, GCP, Azure, etc.) or have your peers run your code in their computer. 219 | 220 |
221 | 222 |
Figure 5 - Development with VScode and Docker workflow
223 |
224 | 225 |
226 | 227 | In the next section, we review Docker basic commands and learn how to set a Dockerfile. 228 | 229 | ## Getting Started with Docker 230 | Generally, the VScode **Dev Container** extension lets you containerize your environment by importing a built-in image. However, often, you may want to add more layers (e.g., install additional dependencies) or build it yourself. This section focuses on the essential Docker requirements that will enable you to build your image and run it inside a container: 231 | 232 | - **Dockerfile** - the image recipe, allows you to add components and customize the dependencies according to the development environment requirements 233 | - **Docker CLI** - core commands to build the image and run it as a containerized environment 234 | 235 |
236 | 237 |
Figure 6 - Docker general workflow
238 |
239 | 240 |
241 | 242 | **Note:** It is important to emphasize that this section covers the basic Docker requirements for this tutorial and is not an alternative to a full Docker tutorial or course. 243 | 244 | 245 | ### The Dockerfile 246 | The `Dockerfile` provides a set of instructions for the Docker engine about how to build the image. You can think about it as the image's recipe. It has its own unique and intuitive syntax using the following structure: 247 | 248 | ``` Dockerfile 249 | COMMAND some instructions 250 | ``` 251 | 252 | For example, the following `Dockerfile` imports the official Python (version 3.10) image as the base image and then using the `apt-get update` and `apt-get install` to install the `curl` library : 253 | 254 | 255 | `./examples/ex-1/Dockerfile` 256 | ``` Dockerfile 257 | FROM python:3.10 258 | 259 | LABEL example=1 260 | 261 | ENV PYTHON_VER=3.10 262 | 263 | RUN apt-get update && apt-get install -y --no-install-recommends curl 264 | ``` 265 | 266 | In a nutshell, we used the `FROM` command to specify the image we want to import from the Docker registry (don't forget to login to the Docker registry service you are using before building the image!). The `LABEL` command is used to set labels or comments, and the `ENV` command is to set environment variables. Last but not least, the `RUN` command is used to run a command on the command line, in this case, to install the `curl` library. 267 | 268 | Let's now review the Dockerfile core commands: 269 | - `FROM` - Defines the base image to use for the image's build. In most cases, unless you are building the image from scratch, you will use some base image with some pre-installed OS and some dependencies. For example, in this tutorial, we will import as our base image the official [Python image](https://hub.docker.com/_/python) 270 | - `LABEL` - Enables to add information about the image to the image's metadata, such as authors, maintainers, license, etc. 271 | - `ENV` - Uses to set environment variables 272 | - `ARG` - Enables to set parameters during the build time 273 | - `RUN` - Allows executing CLI commands (e.g., `pip install ...`, `apt-get ...`, `apt-install...`, `wget...`, etc.) during the build time to add additional components to the base image 274 | - `COPY` - Enables to copy objects (e.g., files and folders) from your local system to the image 275 | - `WORKDIR` - Sets the working directory inside the image 276 | - `EXPOSE` - Defines the port number to expose the image during the run time 277 | - `CMD` - Sets a default command to execute during the run time of the image 278 | - `ENDPOINT` - Allows configuring a container that will run as an executable 279 | 280 | Do not worry if, at this point, you do not fully understand the use cases of some of those commands. It will make more sense when we start to build images in the next section. 281 | 282 | ### Docker Build 283 | 284 | Once the `Dockerfile` is ready, the next step is to build the image using the `docker build` command from the command line. For example, let's build the above `Dockerfile` using the `build` command from this repo root folder: 285 | 286 | ``` shell 287 | docker build . -f ./examples/ex-1/Dockerfile -t rkrispin/vscode-python:ex1 288 | ``` 289 | 290 | Here are the arguments we used with the `build` command: 291 | - The `-f` tag defines the `Dockerfile` path. This argument is optional and should be used if you are calling the `build` function from a different folder than one of the `Dockerfile` 292 | - The `.` symbol defines the context folder of the files system as the one of the `Dockerfile`. Although we did not use the file system in this case, this enables us in other cases to call and copy files from our local folder to the image during the build time 293 | - The `-t` is used to set the image's name and tag (e.g., version). In this case, the image name is `rkrispin/vscode-python` and the tag is `ex1`. 294 | 295 | 296 | You should expect the following output: 297 | 298 | ``` shell 299 | [+] Building 94.2s (6/6) FINISHED 300 | => [internal] load build definition from Dockerfile 0.0s 301 | => => transferring dockerfile: 162B 0.0s 302 | => [internal] load .dockerignore 0.0s 303 | => => transferring context: 2B 0.0s 304 | => [internal] load metadata for docker.io/library/python:3.10 6.0s 305 | => [1/2] FROM docker.io/library/python:3.10@sha256:a8462db480ec3a74499a297b1f8e074944283407b7a417f22f20d8e2e1619782 82.1s 306 | => => resolve docker.io/library/python:3.10@sha256:a8462db480ec3a74499a297b1f8e074944283407b7a417f22f20d8e2e1619782 0.0s 307 | => => sha256:a8462db480ec3a74499a297b1f8e074944283407b7a417f22f20d8e2e1619782 1.65kB / 1.65kB 0.0s 308 | => => sha256:4a1aacea636cab6af8f99f037d1e56a4de97de6025da8eff90b3315591ae3617 2.01kB / 2.01kB 0.0s 309 | => => sha256:23e11cf6844c334b2970fd265fb09cfe88ec250e1e80db7db973d69d757bdac4 7.53kB / 7.53kB 0.0s 310 | => => sha256:bba7bb10d5baebcaad1d68ab3cbfd37390c646b2a688529b1d118a47991116f4 49.55MB / 49.55MB 26.1s 311 | => => sha256:ec2b820b8e87758dde67c29b25d4cbf88377601a4355cc5d556a9beebc80da00 24.03MB / 24.03MB 11.0s 312 | => => sha256:284f2345db055020282f6e80a646f1111fb2d5dfc6f7ee871f89bc50919a51bf 64.11MB / 64.11MB 26.4s 313 | => => sha256:fea23129f080a6e28ebff8124f9dc585b412b1a358bba566802e5441d2667639 211.00MB / 211.00MB 74.5s 314 | => => sha256:7c62c924b8a6474ab5462996f6663e07a515fab7f3fcdd605cae690a64aa01c7 6.39MB / 6.39MB 28.2s 315 | => => extracting sha256:bba7bb10d5baebcaad1d68ab3cbfd37390c646b2a688529b1d118a47991116f4 1.6s 316 | => => sha256:c48db0ed1df2d2df2dccd680323097bafb5decd0b8a08f02684b1a81b339f39b 17.15MB / 17.15MB 31.9s 317 | => => extracting sha256:ec2b820b8e87758dde67c29b25d4cbf88377601a4355cc5d556a9beebc80da00 0.6s 318 | => => sha256:f614a567a40341ac461c855d309737ebccf10a342d9643e94a2cf0e5ff29b6cd 243B / 243B 28.4s 319 | => => sha256:00c5a00c6bc24a1c23f2127a05cfddd90865628124100404f9bf56d68caf17f4 3.08MB / 3.08MB 29.4s 320 | => => extracting sha256:284f2345db055020282f6e80a646f1111fb2d5dfc6f7ee871f89bc50919a51bf 2.5s 321 | => => extracting sha256:fea23129f080a6e28ebff8124f9dc585b412b1a358bba566802e5441d2667639 6.2s 322 | => => extracting sha256:7c62c924b8a6474ab5462996f6663e07a515fab7f3fcdd605cae690a64aa01c7 0.3s 323 | => => extracting sha256:c48db0ed1df2d2df2dccd680323097bafb5decd0b8a08f02684b1a81b339f39b 0.5s 324 | => => extracting sha256:f614a567a40341ac461c855d309737ebccf10a342d9643e94a2cf0e5ff29b6cd 0.0s 325 | => => extracting sha256:00c5a00c6bc24a1c23f2127a05cfddd90865628124100404f9bf56d68caf17f4 0.2s 326 | => [2/2] RUN apt-get update && apt-get install -y --no-install-recommends curl 5.9s 327 | => exporting to image 0.1s 328 | => => exporting layers 0.1s 329 | => => writing image sha256:a8e4c6d06c97e9a331a10128d1ea1fa83f3a525e67c7040c2410940312e946f5 0.0s 330 | => => naming to docker.io/rkrispin/vscode-python:ex1 331 | 332 | ``` 333 | 334 | **Note:** The above output of the build describes the different layers of the image. Don't worry if, at this point, it looks and sounds like gibberish. Reading this output type will be easier after reading the next section, which focuses on the image layers. 335 | 336 | 337 | You can use the `docker images` command to validate that the image was created successfully: 338 | 339 | ``` shell 340 | >docker images 341 | REPOSITORY TAG IMAGE ID CREATED SIZE 342 | rkrispin/vscode-python ex1 a8e4c6d06c97 43 hours ago 1.02GB 343 | ``` 344 | 345 | The next section will focus on the image layers and caching process. 346 | 347 | 348 | ### The image layers 349 | 350 | The build process of Docker's images is based on layers. Depending on the context, the docker engine takes each one of the `Dockerfile` commands during the build time and translates it either into layer or metadata. `Dockerfile` commands, such as `FROM` and `RUN` are translated into a layer, and commands, such as `LABEL`, `ARG`, `ENV`, and `CMD` are translated into metadata. For example, we can observe in the output of the build of `rkrispin/vscode-python` image above that there are two layers: 351 | - The first layer started with `[1/2] FROM...`, corresponding to the `FROM python:3.10` line on the `Dockerfile`, which import the Python 3.10 official image 352 | - The second layer started with `[2/2] RUN apt-get...`, corresponding to the `RUN` command on the `Dockerfile` 353 | 354 |
355 | 356 |
Figure 7 - Example of a build output with respect to the Dockerfile
357 |
358 | 359 |
360 | 361 | The `docker inspect` command returns the image metadata details in a JSON format. That includes the envrioment variables, labels, layers and general metadata. In the following example, we will us [jq](https://jqlang.github.io/jq/) to extract the layers information from the metadata JSON file: 362 | 363 | ``` shell 364 | > docker inspect rkrispin/vscode-python:ex1 | jq '.[] | .RootFS' 365 | { 366 | "Type": "layers", 367 | "Layers": [ 368 | "sha256:332b199f36eb054386cd2931c0824c97c6603903ca252835cc296bacde2913e1", 369 | "sha256:2f98f42985b15cbe098d2979fa9273e562e79177b652f1208ae39f97ff0424d3", 370 | "sha256:964529c819bb33d3368962458c1603ca45b933487b03b4fb2754aa55cc467010", 371 | "sha256:e67fb4bad8f42cca08769ee21bbe15aca61ab97d4a46b181e05fefe3a03ee06d", 372 | "sha256:037f26f869124174b0d6b6d97b95a5f8bdff983131d5a1da6bc28ddbc73531a5", 373 | "sha256:737cec5220379f795b727e6c164e36e8e79a51ac66a85b3e91c3f25394d99224", 374 | "sha256:65f4e45c2715f03ed2547e1a5bdfac7baaa41883450d87d96f877fbe634f41a9", 375 | "sha256:baef981f26963b264913e79bd0a1472bae389441022d71f559e9d186600d2629", 376 | "sha256:88e1d36ff4812423afc93d5f6208f2783df314d5ecf6f961325c65e1dbf891da" 377 | ] 378 | } 379 | 380 | ``` 381 | 382 | As you can see from the image's layers output above, the `rkrispin/vscode-python:ex1` image has nine layers. Each layer is represented by its hash key (e.g., `sha256:...`), and it is cached on the backend. While we saw on the build output that the docker engine triggered two processes from the `FROM` and `RUN` commands, we ended up with nine layers as opposed to two. The main reason for that is related to the fact that when importing the baseline image, we inherited the imported image characteristics, including the layers. In this case, we used the `FROM` to import the official Python image, which included eight layers, and then added the 9th layer by executing the `RUN` commands. You can test it by pulling the baseline image and using the inspect command to review its layers: 383 | 384 | ``` shell 385 | > docker pull python:3.10 386 | 3.10: Pulling from library/python 387 | bba7bb10d5ba: Already exists 388 | ec2b820b8e87: Already exists 389 | 284f2345db05: Already exists 390 | fea23129f080: Already exists 391 | 7c62c924b8a6: Already exists 392 | c48db0ed1df2: Already exists 393 | f614a567a403: Already exists 394 | 00c5a00c6bc2: Already exists 395 | Digest: sha256:a8462db480ec3a74499a297b1f8e074944283407b7a417f22f20d8e2e1619782 396 | Status: Downloaded newer image for python:3.10 397 | docker.io/library/python:3.10 398 | 399 | > docker inspect python:3.10 | jq '.[] | .RootFS' 400 | { 401 | "Type": "layers", 402 | "Layers": [ 403 | "sha256:332b199f36eb054386cd2931c0824c97c6603903ca252835cc296bacde2913e1", 404 | "sha256:2f98f42985b15cbe098d2979fa9273e562e79177b652f1208ae39f97ff0424d3", 405 | "sha256:964529c819bb33d3368962458c1603ca45b933487b03b4fb2754aa55cc467010", 406 | "sha256:e67fb4bad8f42cca08769ee21bbe15aca61ab97d4a46b181e05fefe3a03ee06d", 407 | "sha256:037f26f869124174b0d6b6d97b95a5f8bdff983131d5a1da6bc28ddbc73531a5", 408 | "sha256:737cec5220379f795b727e6c164e36e8e79a51ac66a85b3e91c3f25394d99224", 409 | "sha256:65f4e45c2715f03ed2547e1a5bdfac7baaa41883450d87d96f877fbe634f41a9", 410 | "sha256:baef981f26963b264913e79bd0a1472bae389441022d71f559e9d186600d2629" 411 | ] 412 | } 413 | ``` 414 | 415 | ### Layers caching 416 | 417 | One of the cons of Docker is the image build time. As the level of complexity of the Dockerfile is higher (e.g., a large number of dependencies), the longer the build time. Sometimes, your build won't execute as expected on the first try. Either some requirements are missing, or something breaks during the build time. This is where the use of caching helps in reducing the image rebuild time. Docker has smart mechanization that identifies if each layer should be built from scratch or can leverage a cached layer and save time. For example, let's add to the previous example another command to install the `vim` editor. Generally, we can (and should) add it to the same apt-get we are using to install the `curl` package, but for the purpose of showing the layers caching functionality, we will run it separately: 418 | 419 | 420 | `./examples/ex-2/Dockerfile` 421 | ``` Dockerfile 422 | FROM python:3.10 423 | 424 | LABEL example=1 425 | 426 | ENV PYTHON_VER=3.10 427 | 428 | RUN apt-get update && apt-get install -y --no-install-recommends curl 429 | 430 | RUN apt-get update && apt-get install -y --no-install-recommends vim 431 | ``` 432 | 433 | We will use the below command to build this image and tag it as `rkrispin/vscode-python:ex2`: 434 | 435 | ``` shell 436 | docker build . -f ./examples/ex-2/Dockerfile -t rkrispin/vscode-python:ex2 --progress=plain 437 | ``` 438 | You should expect the following output (if ran the previous build): 439 | 440 | ``` shell 441 | => [internal] load build definition from Dockerfile 0.0s 442 | => => transferring dockerfile: 234B 0.0s 443 | => [internal] load .dockerignore 0.0s 444 | => => transferring context: 2B 0.0s 445 | => [internal] load metadata for docker.io/library/python:3.10 0.0s 446 | => [1/3] FROM docker.io/library/python:3.10 0.0s 447 | => CACHED [2/3] RUN apt-get update && apt-get install -y --no-install-recommends curl 0.0s 448 | => [3/3] RUN apt-get update && apt-get install -y --no-install-recommends vim 34.3s 449 | => exporting to image 0.4s 450 | => => exporting layers 0.4s 451 | => => writing image sha256:be39eb0eb986f083a02974c2315258377321a683d8472bac15e8d5694008df35 0.0s 452 | => => naming to docker.io/rkrispin/vscode-python:ex2 453 | ``` 454 | 455 | 456 | As can be noticed from the above build output, the first and second layers already exist from the previous build. Therefore, the docker engine adds their cached layers to the image (as opposed to building them from scratch), and just builds the 3rd layer and installs the vim editor. 457 | 458 | **Note:** By default, the build output is concise and short. You can get more detailed output during the build time by adding the `progress` argument and setting it to `plain`: 459 | 460 | ``` shell 461 | > docker build . -f ./examples/ex-2/Dockerfile -t rkrispin/vscode-python:ex2 --progress=plain 462 | #1 [internal] load .dockerignore 463 | #1 transferring context: 2B done 464 | #1 DONE 0.0s 465 | 466 | #2 [internal] load build definition from Dockerfile 467 | #2 transferring dockerfile: 234B done 468 | #2 DONE 0.0s 469 | 470 | #3 [internal] load metadata for docker.io/library/python:3.10 471 | #3 DONE 0.0s 472 | 473 | #4 [1/3] FROM docker.io/library/python:3.10 474 | #4 DONE 0.0s 475 | 476 | #5 [2/3] RUN apt-get update && apt-get install -y --no-install-recommends curl 477 | #5 CACHED 478 | 479 | #6 [3/3] RUN apt-get update && apt-get install -y --no-install-recommends vim 480 | #6 CACHED 481 | 482 | #7 exporting to image 483 | #7 exporting layers done 484 | #7 writing image sha256:be39eb0eb986f083a02974c2315258377321a683d8472bac15e8d5694008df35 0.0s done 485 | #7 naming to docker.io/rkrispin/vscode-python:ex2 done 486 | #7 DONE 0.0s 487 | ``` 488 | 489 | Since we already cached the 3rd layer on the previous build, all the layers in the above output are cached, and the run time is less than 1 second. 490 | 491 | When setting your Dockerfile, you should be minded and strategic to the layers caching process. The order of the layers does matter! The following images demonstrate when the docker engine will use cached layers and when to rebuild them. The first image illustrates the initial build: 492 | 493 |
494 | 495 |
Figure 8 - Illustration of initial build of image. The left side represents the Dockerfile's commands and the right one the corresponding layers
496 |
497 | 498 |
499 | 500 | In this case, we have a Dockerfile with four commands that are translated during the build time into four layers. What will happen if we add a fifth command and place it right after the third one? The docker engine will identify that the first and second commands in the Dockerfile did not change and, therefore, will use the corresponding cached layers (one and two), and rebuild the rest of the layers from scratch: 501 | 502 |
503 | 504 |
Figure 9 - Illustration of the caching process during the rebuild of an image
505 |
506 | 507 |
508 | 509 | When planning your Dockerfile, if applicable, a good practice is to place the commands that will most likely stay the same and keep new updates to the end of the file if possible. 510 | 511 | That was just the tip of the iceberg, and there is much more to learn about Docker. The next section will explore different methods to run Python inside a container. 512 | 513 | ## Running Python on Docker - the hard way 514 | 515 | In the previous sections, we saw how to define the image requirements with the `Dockerfile` and build it with the `build` command. This section focuses on running Python inside a container using the `docker run` command. 516 | 517 | ### Docker run 518 | 519 | The `docker run` or `run` command enables us to create and run a new container from an image. Typically, the `run` command is used to launch a dockerized application or server or to execute a code following the below syntax: 520 | 521 | ``` shell 522 | docker run [OPTIONS] IMAGE [COMMAND] [ARG...] 523 | ``` 524 | 525 | For example, we can use the `run` command with the official Python 3.10 image: 526 | 527 | ``` shell 528 | docker run python:3.10 529 | ``` 530 | 531 | Surprisingly (or not), nothing happened. To understand that better, we need to go back to the `Dockerfile`. Generally, images can be used to run: 532 | - Server 533 | - Application 534 | 535 | In both cases, we use the `Dockerfile` to set and enable launching them during the run time. In the case of a server, we use on the `Dockerfile` the `PORT` and `CMD` commands to set the server's port on the image and launch the server, respectively. We then use the `run` command and add the `-p` (or `--publish list`) option to map the server's port with a local port. Similarly, to launch an application, we use the `CMD` command on the `Dockerfile` to define the launch command during the run time and use the `--interactive` and `--tty` options to launch the container in interactive mode, which enables us to access the application. 536 | 537 | Let's now go back to the `python:3.10` image and use the `inspect` command to check if the `CMD` command was defined: 538 | 539 | ``` shell 540 | > docker inspect python:3.10 | jq '.[] | .Config.Cmd' 541 | [ 542 | "python3" 543 | ] 544 | ``` 545 | 546 | **Note:** We used the `jq` library again to parse out from the JSON output the CMD metadata 547 | 548 | As you can see, the `CMD` on the `python:3.10` image is set to run the default Python launch command - `python3`, which launches Python during the run time. Let's now add the `--interactive` and `--tty` options to run the container in an interactive mode: 549 | 550 | ```shell 551 | docker run --interactive --tty python:3.10 552 | ``` 553 | This launches the default Python version on the image. We can then test it by using the `print` command to print `Hello World!`: 554 | 555 | ```python 556 | Python 3.10.12 (main, Jun 14 2023, 18:40:54) [GCC 12.2.0] on linux 557 | Type "help", "copyright", "credits" or "license" for more information. 558 | >>> print("Hello World!") 559 | Hello World! 560 | >>> 561 | ``` 562 | 563 | OK, we have Python running inside a dockerized environment, so why should we not use it? Mainly due to the following reasons: 564 | - This is not a development environment, and it is harder (in my mind) to maintain and develop code from the terminal with respect to Python IDEs such as PyCharm or VScode. 565 | - By default, the `docker run` is an ephemeral process, and therefore, your code is lost when you shut down the container. 566 | 567 | While there are ways to overcome the above issues, it is still convoluted and not as efficient as using VScode. In the next section, we will see how to set and run Python code with VScode and the Dev Containers extension. 568 | 569 | ## Setting the Dev Containers Extension 570 | 571 | So far, we covered the foundation of Docker. We saw how to set and build an image with the `Dockerfile` and the `build` command, respectively, and then run it in a container with the `run` command. This section will focus on setting up a Python development environment with VScode and the [Dev Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) extension. 572 | 573 | If you still need to install the Dev Containers extension or Docker Desktop, follow the installation instruction above. Once the extension is installed, you should expect to see on the far left side the extension status bar symbol (`><` alike): 574 | 575 | 576 |
577 | 578 |
Figure 10 - The Dev Containers extension status bar symbol
579 |
580 |
581 | 582 | ### Setting the devcontainer.json file 583 | 584 | The Dev Containers extension enables to open a local folder inside a containerized environment. This solves the container ephemeral issue and enables you to maintain your code locally while developing and testing it inside a container. 585 | 586 | To set the Dev Containers extension on your local folder, create a folder named `.devcontainer` and add the `devcontainer.json` file. Generally, your project folder should follow the following structure: 587 | 588 | ``` shell 589 | . 590 | ├── .devcontainer 591 | │ └── devcontainer.json 592 | └── Your Projects Files 593 | ``` 594 | 595 | The `devcontainer.json` defines and customizes the container and VScode setting, such as: 596 | 597 | - Image settings - defines the image build method or if to pull an existing one 598 | - Project settings such as extensions to install and command to execute during the launch time of the container 599 | 600 | 601 | 602 | Let's start with a practical example by setting the environment using the same image we build in the first example above (e.g., `rkrispin/vscode-python:ex1`). During the launch time of the environment, the Dev Containers extension follows the instractions in the `devcontainer.json` and sets the environment accordingly: 603 | 604 | `.devcontainer/devcontainer.json` 605 | ``` json 606 | { 607 | "name": "Example 3", 608 | "build":{ 609 | "dockerfile": "Dockerfile", 610 | "context": "../" 611 | }, 612 | "customizations": { 613 | "vscode": { 614 | "extensions": [ 615 | "quarto.quarto", 616 | "ms-azuretools.vscode-docker", 617 | "ms-python.python", 618 | "ms-vscode-remote.remote-containers", 619 | "yzhang.markdown-all-in-one", 620 | "redhat.vscode-yaml", 621 | "ms-toolsai.jupyter", 622 | "hediet.vscode-drawio" 623 | ] 624 | } 625 | } 626 | } 627 | ``` 628 | 629 | **Note:** The `devcontainer.json` file must be stored under the `.devcontainer` folder or in the project root folder. To run the above example, you will have to open the `./examples/exp-3` folder in VScode as the project folder using the `File` -> `Open Folder...` options. 630 | 631 | As can see in the above `devcontainer.json`, the `build` section defines the image build process. The `dockerfile` argument points out to the `Dockerfile` to use for the build. The `context` argument defines the files' system path for the `Dockerfile`. Although, we currently do not use the `context` argument in the build time, we will see its applications later. In addition, the `customizations` section enables you to customize the VScode options, such as extensions to install, default Python interpreter, and files to execute during the container launch. 632 | 633 | ### Launching the folder inside a container 634 | 635 | Once you set the `devcontainer.json`, to launch the folder inside the container, go to the bottom far left side of your VScode screen and click the Dev Containers' status bar ()`><` symbol alike). This will open the VScode Command Palette on the top of the screen, and you should see the Dev Containers extension's common commands. Select the `Reopen in Container` options (see the screenshot below): 636 | 637 |
638 | 639 |
Figure 11 - the Dev Containers extensions Command Palette
640 |
641 | 642 |
643 | 644 | The below video demonstrates the full process of launching the Python environment inside a container with the Dev Containers extension: 645 | 646 | 647 |
648 | 649 |
Figure 12 - Open a folder inside a container with the Dev Containers extension
650 |
651 | 652 |
653 | 654 | The next section focuses on customizing the Python environment. 655 | 656 | ## Setting the Python Environment 657 | 658 | In this section, we will connect all the dots together and build a Python development environment using the following architecture: 659 | - We will start by setting the image using the following files: 660 | - Create a `Dockerfile` to define the environment settings (e.g., Python version, packages to install, etc.). In addition, we will use the following helper files: 661 | - I like to keep the `Dockerfile` as concise as possible by using a helper bash script (`install_dependencies.sh`) to install all the environment dependencies, such as Debian packages, installing and setting the Python environment with Conda (or a similar solution), etc. 662 | - In addition, we will use the `requirements.txt` file to define the list of Python packages to install in the environment 663 | - The next step is to set up the Dev Containers extension: 664 | - We will use the `devcontainer.json` file to define the VScode settings for the development environment. That includes the build method, a list of extensions to set, local volumes to mount, and defining environment variables, etc. 665 | - In addition, we will use the `devcontainer.env` file to set additional environment variables. Note that those variables neither be available during the build time nor can be called by `devcontainer.json` file 666 | 667 | 668 | The `.devcontainer` folder should have the following files: 669 | 670 | ```shell 671 | . 672 | ├── Dockerfile 673 | ├── devcontainer.env 674 | ├── devcontainer.json 675 | ├── install_dependencies.sh 676 | └── requirements.txt 677 | ``` 678 | 679 | 680 | Just before getting started, let's define our environment requirments: 681 | - Python version 3.10 682 | - Install the following packages: 683 | - pandas v2.0.3 684 | - plotly v5.15.0 685 | - Conda for setting the virtual environment 686 | - Install the following extensions: 687 | - Docker support 688 | - Python language support 689 | - Markdown editor 690 | - Quarto editor 691 | - YAML language support 692 | - Jupyter notebook support 693 | - Last but not least, we would like to mount a local folder that is not the project folder to load CSV files 694 | 695 | In addition, to make this setting as customized as possible, we will set locally environment variables that will enable us to modify our settings, if needed. That includes the following variables: 696 | - `ENV_NAME` - we will use this variable to set the Conda environment name and to set the path to the default Python interpreter 697 | - `PYTHON_VER` - set the Python version for the conda environment 698 | - `CSV_PATH` - the local path for a folder with CSV files 699 | - `MY_VAR` - A general variable that we will use as example for setting environment variables 700 | 701 | On Mac and Linux you can use the `.zshrc` file to set those variables: 702 | 703 | ~/.zshrc 704 | ``` 705 | # Setting env variables for VScode 706 | export ENV_NAME=python_tutorial 707 | export PYTHON_VER=3.10 708 | export CSV_PATH=$HOME/Desktop/CSV 709 | export MY_VAR=some_var 710 | ``` 711 | 712 | For Windows users, you can use the `setx` command to the environment vairables: 713 | ``` 714 | setx variable_name "variable_value" 715 | ``` 716 | 717 | **Note:** VScode caches environment variables during the launch time. Therefore, when adding new environment variables during an open session, they won't be available until completely closing VScode and reopening it. 718 | 719 | 720 | ### Setting the image 721 | 722 | We will use the below `Dockerfile` to build our Python environment: 723 | 724 | `.devcontainer/Dockerfile` 725 | ``` Dockerfile 726 | FROM python:3.10 727 | 728 | # Arguments 729 | ARG PYTHON_VER 730 | ARG ENV_NAME 731 | 732 | # Environment variables 733 | ENV ENV_NAME=$ENV_NAME 734 | ENV PYTHON_VER=$PYTHON_VER 735 | 736 | # Copy files 737 | RUN mkdir requirements 738 | COPY requirements.txt requirements/ 739 | COPY install_dependencies.sh requirements/ 740 | 741 | # Install dependencies 742 | RUN bash requirements/install_dependencies.sh $ENV_NAME $PYTHON_VER 743 | 744 | ``` 745 | 746 | To keep the build time fairly short, we use the Python official image as our base image for our environment. The main advantage of using this base image type is that it comes with most of the required dependencies and saves us time (and pain). 747 | 748 | Using arguments enables us to parameterize our environment settings. In this case, the user can modify the Python version and environment name using the `PYTHON_VER` and `ENV_NAME` arguments. I then set those arguments as environment variables for convenience (staying available after the build). 749 | 750 | Last but not least, we create a local folder (`requirements`) inside the image and copy files from our local drive based on the path that was defined by the context argument on the `devcontainer.json` file. The `install_dependencies.sh` is a bash helper script that installs dependencies (conda, vim, etc.), sets the conda environment, and installs Python packages using the list in the `requirements.txt` file. 751 | 752 | 753 | While we set the Python environment with conda, you can modify the script below to other alternatives such as `venv`, and `Poetry` in the below script: 754 | 755 | `.devcontainer/install_dependencies.sh` 756 | ``` bash 757 | #!/bin/bash 758 | 759 | CONDA_ENV=$1 760 | PYTHON_VER=$2 761 | CPU=$(uname -m) 762 | 763 | 764 | # Installing prerequisites 765 | apt-get update && \ 766 | apt-get install -y \ 767 | python3-launchpadlib \ 768 | vim \ 769 | && apt update 770 | 771 | 772 | # Install miniconda 773 | apt update && apt-get install -y --no-install-recommends \ 774 | software-properties-common \ 775 | && add-apt-repository -y ppa:deadsnakes/ppa \ 776 | && apt update 777 | 778 | wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-${CPU}.sh -O ~/miniconda.sh \ 779 | && /bin/bash ~/miniconda.sh -b -p /opt/conda \ 780 | && export PATH=/opt/conda/bin:$PATH \ 781 | && conda init bash \ 782 | && conda install conda-build 783 | 784 | # Set environment 785 | . /root/.bashrc \ 786 | && conda create -y --name $CONDA_ENV python=$PYTHON_VER 787 | 788 | echo "conda activate $CONDA_ENV" >> ~/.bashrc 789 | 790 | conda activate $CONDA_ENV 791 | 792 | # Install the Python packages 793 | pip3 install -r requirements/requirements.txt 794 | 795 | ``` 796 | 797 | **Note:** We use the `uname -m` CLI command to extract the CPU architecture (e.g., Intel, M1/2, etc.) and choose the conda's build accordingly. 798 | 799 | We will set the required packages for the Python environment with the `requirements.txt` file: 800 | 801 | `.devcontainer/requirements.txt` 802 | ```txt 803 | wheel==0.40.0 804 | pandas==2.0.3 805 | plotly==5.15.0 806 | plotly-express==0.4.1 807 | ``` 808 | 809 | ### Setting the devcontainer.json file 810 | 811 | Once we set the image, the next step is setting the `devcontainer.json` file. We incorporated environment variables in the below settings to enable the seamless creation of multiple environments: 812 | 813 | 814 | `.devcontainer/devcontainer.json` 815 | 816 | ```json 817 | { 818 | "name": "${localEnv:ENV_NAME}", 819 | "build": { 820 | "dockerfile": "Dockerfile", 821 | "args": {"ENV_NAME": "${localEnv:ENV_NAME}", 822 | "PYTHON_VER": "${localEnv:PYTHON_VER}"}, 823 | "context": "." 824 | }, 825 | "customizations": { 826 | "settings": { 827 | "python.defaultInterpreterPath": "/opt/conda/envs/${localEnv:ENV_NAME}/bin/python" 828 | }, 829 | "vscode": { 830 | "extensions": [ 831 | "quarto.quarto", 832 | "ms-azuretools.vscode-docker", 833 | "ms-python.python", 834 | "ms-vscode-remote.remote-containers", 835 | "yzhang.markdown-all-in-one", 836 | "redhat.vscode-yaml", 837 | "ms-toolsai.jupyter" 838 | ] 839 | } 840 | }, 841 | 842 | "mounts": [ 843 | "source=${localEnv:CSV_PATH},target=/home/csv,type=bind,consistency=cache" 844 | ], 845 | "remoteEnv": { 846 | "MY_VAR": "${localEnv:MY_VAR}" 847 | }, 848 | "runArgs": ["--env-file",".devcontainer/devcontainer.env"], 849 | "postCreateCommand": "python3 tests/test1.py" 850 | } 851 | ``` 852 | 853 | For the build, we use the `dockerfile`, `args`, and `context` to define the Dockerfile, arguments (e.g., Python version and environment name), and folder contest during the build, respectively. 854 | 855 | We use the `python.defaultInterpreterPath` argument to set the path of the default Python interpreter to the conda environment we set during the build. 856 | 857 | With the `mounts` argument, we mount a local folder (outside the current folder) with a path inside the container (e.g., `home/csv`). Generally, you would use this option when you wish to load a file or data from a folder outside your working folder or make the separation between your data and code. In this case, the `CSV_PATH` environment variable defines the path of the local folder. 858 | 859 | The `remoteEnv` enables to set environment variables with the container. Alternatively, you can add a `.env` file with a list of environment variables using the `runArgs` argument. 860 | 861 | The `postCreateCommand` argument allows for the execution of commands after the build process has finished. In this case, we utilize it to run a basic test script that checks if the packages can be loaded and prints out the message `Hello World!`: 862 | 863 | `tests/test1.py` 864 | ```python 865 | import pandas as pd 866 | import plotly as py 867 | import plotly.express as px 868 | 869 | print("Hello World!") 870 | ``` 871 | 872 | That's it! We are ready to launch the environment with the Dev Containers extension: 873 | 874 | 875 |
876 | 877 |
Figure 13 - Launch the Python Environment with the Dev Containers extension
878 |
879 | 880 |
881 | 882 | 883 | ## Summary 884 | 885 | This tutorial covered the foundation of setting a dockerized development environment for Python with VScode and Docker. We reviewed the process of setting up a Python environment using the Dev Containers extension. While this tutorial does not focus on Docker, it covers the foundation of Docker with the goal of reducing the entry barrier for new users. In addition, we saw how to set up and launch a containerized Python development environment with the Dev Containers extension. 886 | 887 | Using environment variables enables us to parametized the environment and seamlessly modify and costimize it. The example above uses conda to set the Python environment but you can choose any other method that works best for your needs. 888 | 889 | Last but not least, all feedback is welcome! Please feel free to open an issue if you have any feedback about the tutorial or found some code issues. 890 | 891 | ## Resources 892 | 893 | - Docker 894 | - [Docker](https://www.docker.com/) 895 | - [Docker Hub](https://hub.docker.com/) 896 | - [Docker Crash Course for Absolute Beginners](https://www.youtube.com/watch?v=pg19Z8LL06w&ab_channel=TechWorldwithNana) 897 | - [Docker Tutorial for Beginners - A Full DevOps Course on How to Run Applications in Containers](https://www.youtube.com/watch?v=fqMOX6JJhGo&t=2s&ab_channel=freeCodeCamp.org) 898 | 899 | - VScode 900 | - [VScode documentaion](https://code.visualstudio.com/docs) 901 | - [Dev Containers documentation](https://code.visualstudio.com/docs/devcontainers/containers) 902 | 903 | 904 | ## License 905 | 906 | This tutorial is licensed under a [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International](https://creativecommons.org/licenses/by-nc-sa/4.0/) License. 907 | -------------------------------------------------------------------------------- /diagrams/.gitignore: -------------------------------------------------------------------------------- 1 | *.bkp 2 | *.dtmp -------------------------------------------------------------------------------- /examples/ex-1/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | LABEL example=1 4 | 5 | ENV PYTHON_VER=3.10 6 | 7 | RUN apt-get update && apt-get install -y --no-install-recommends curl -------------------------------------------------------------------------------- /examples/ex-2/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | LABEL example=1 4 | 5 | ENV PYTHON_VER=3.10 6 | 7 | RUN apt-get update && apt-get install -y --no-install-recommends curl 8 | 9 | RUN apt-get update && apt-get install -y --no-install-recommends vim -------------------------------------------------------------------------------- /examples/ex-3/.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.10 2 | 3 | LABEL example=1 4 | 5 | ENV PYTHON_VER=3.10 6 | 7 | RUN apt-get update && apt-get install -y --no-install-recommends curl -------------------------------------------------------------------------------- /examples/ex-3/.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Example 3", 3 | "build":{ 4 | "dockerfile": "Dockerfile", 5 | "context": "../" 6 | }, 7 | "customizations": { 8 | "vscode": { 9 | "extensions": [ 10 | "quarto.quarto", 11 | "ms-azuretools.vscode-docker", 12 | "ms-python.python", 13 | "ms-vscode-remote.remote-containers", 14 | "yzhang.markdown-all-in-one", 15 | "redhat.vscode-yaml", 16 | "ms-toolsai.jupyter", 17 | "hediet.vscode-drawio" 18 | ] 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /images/command-palette.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/command-palette.png -------------------------------------------------------------------------------- /images/dev_container_symbol.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/dev_container_symbol.png -------------------------------------------------------------------------------- /images/docker layers 1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/docker layers 1.png -------------------------------------------------------------------------------- /images/docker layers 2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/docker layers 2.png -------------------------------------------------------------------------------- /images/docker-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/docker-architecture.png -------------------------------------------------------------------------------- /images/docker-install.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/docker-install.png -------------------------------------------------------------------------------- /images/docker-layers.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/docker-layers.png -------------------------------------------------------------------------------- /images/docker-workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/docker-workflow.png -------------------------------------------------------------------------------- /images/dockerfile to container.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/dockerfile to container.png -------------------------------------------------------------------------------- /images/open_dev_container.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/open_dev_container.gif -------------------------------------------------------------------------------- /images/open_dev_container.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/open_dev_container.mov -------------------------------------------------------------------------------- /images/open_dev_container2.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/open_dev_container2.gif -------------------------------------------------------------------------------- /images/open_dev_container2a.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/open_dev_container2a.mov -------------------------------------------------------------------------------- /images/python.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/python.png -------------------------------------------------------------------------------- /images/vscode-download.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/vscode-download.png -------------------------------------------------------------------------------- /images/vscode-extensions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RamiKrispin/vscode-python/1c2b41a451b0262c22686d929aef449bcf1ad792/images/vscode-extensions.png -------------------------------------------------------------------------------- /tests/test1.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import plotly as py 3 | import plotly.express as px 4 | 5 | print("Hello World!") --------------------------------------------------------------------------------