├── .gitignore ├── LICENSE ├── README.md ├── base ├── Dockerfile └── utils │ ├── README.md │ ├── custom │ ├── README.md │ └── custom.js │ ├── jupyter │ ├── README.md │ ├── fix-permissions │ ├── jupyter_notebook_config.py │ ├── start-notebook.sh │ ├── start-singleuser.sh │ └── start.sh │ ├── miniconda-md5 │ ├── Miniconda3-4.5.12-Linux-x86_64.sh.md5 │ ├── Miniconda3-4.6.14-Linux-x86_64.sh.md5 │ ├── Miniconda3-4.7.10-Linux-x86_64.sh.md5 │ └── README.md │ ├── nbexamples-sagemaker │ ├── README.md │ ├── __init__.py │ ├── _version.py │ ├── external │ │ ├── __init__.py │ │ └── titlecase.py │ ├── handlers.py │ └── static │ │ ├── examples.css │ │ ├── examples.js │ │ └── main.js │ └── sample-notebooks │ ├── README.md │ └── update_examples.sh ├── build-base-image.sh ├── build-env-image.sh ├── docker-compose.yml ├── envs ├── create_env.sh ├── create_env_file.py ├── create_env_file.sh ├── create_envs.sh ├── docker │ ├── Dockerfile.all │ ├── Dockerfile.all_python2 │ ├── Dockerfile.all_python3 │ ├── Dockerfile.chainer_p27 │ ├── Dockerfile.chainer_p36 │ ├── Dockerfile.mxnet_p27 │ ├── Dockerfile.mxnet_p36 │ ├── Dockerfile.python2 │ ├── Dockerfile.python2_chainer_p27 │ ├── Dockerfile.python2_mxnet_p27 │ ├── Dockerfile.python2_pytorch_p27 │ ├── Dockerfile.python2_tensorflow_p27 │ ├── Dockerfile.python3 │ ├── Dockerfile.python3_chainer_p36 │ ├── Dockerfile.python3_mxnet_p36 │ ├── Dockerfile.python3_pytorch_p36 │ ├── Dockerfile.python3_tensorflow_p36 │ ├── Dockerfile.pytorch_p27 │ ├── Dockerfile.pytorch_p36 │ ├── Dockerfile.tensorflow_p27 │ └── Dockerfile.tensorflow_p36 ├── env_exports │ ├── README.md │ ├── chainer_p27.yml │ ├── chainer_p27_pip.txt │ ├── chainer_p36.yml │ ├── chainer_p36_pip.txt │ ├── environment_list.txt │ ├── get_env_exports.sh │ ├── mxnet_p27.yml │ ├── mxnet_p27_pip.txt │ ├── mxnet_p36.yml │ ├── mxnet_p36_pip.txt │ ├── python2.yml │ ├── python2_pip.txt │ ├── python3.yml │ ├── python3_pip.txt │ ├── pytorch_p27.yml │ ├── pytorch_p27_pip.txt │ ├── pytorch_p36.yml │ ├── pytorch_p36_pip.txt │ ├── tensorflow_p27.yml │ ├── tensorflow_p27_pip.txt │ ├── tensorflow_p36.yml │ └── tensorflow_p36_pip.txt └── include_libraries.txt └── run-python3-container.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | .idea/ -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Quy Tang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon SageMaker Notebook Container 2 | 3 | SageMaker Notebook Container is a sandboxed local environment that replicates the [Amazon Sagemaker Notebook Instance](https://docs.aws.amazon.com/sagemaker/latest/dg/nbi.html), 4 | including installed software and libraries, file structure and permissions, environment variables, context objects and behaviors. 5 | 6 | * [Background](#background) 7 | * [Prerequisites](#prerequisites) 8 | * [Run Container](#run-container) 9 | * [Using `docker`](#using-docker) 10 | * [Using `docker-compose`](#using-docker-compose) 11 | * [Accessing Jupyter Notebook](#accessing-jupyter-notebook) 12 | * [Optional additions](#optional-additions) 13 | * [Docker CLI](#docker-cli) 14 | * [Git integration](#git-integration) 15 | * [Shared `SageMaker` directory](#shared-sagemaker-directory) 16 | * [Sample scripts](#sample-scripts) 17 | 18 | 19 | ## Background 20 | Amazon SageMaker provides an AWS-hosted Notebook instance, a notably convenient way for any data scientists or developers to access a complete server for working with Amazon SageMaker. 21 | 22 | Nonetheless, it costs money, requires all data to be uploaded online, requires Internet access and especially AWS Console sign-in, and can be difficult to customize. 23 | 24 | To overcome these drawbacks, this Docker container has been created to offer a similar setup usable locally on a laptop/desktop. 25 | 26 | The replicated features include full Jupyter Notebook and Lab server, multiple kernels, AWS & SageMaker SDKs, AWS and Docker CLIs, Git integration, Conda and SageMaker Examples Tabs. 27 | 28 | The AWS-hosted instance and the local container aren't mutually exclusive and should be used together to enhance the data science experience. 29 | 30 | A detailed write-up on the rationale behind this container can be found on [Medium](https://towardsdatascience.com/run-amazon-sagemaker-notebook-locally-with-docker-container-8dcc36d8524a). 31 | 32 | #### Why Docker image? 33 | The most important aim is to achieve a repeatable setup that can be replicated in any laptop or server. 34 | 35 | Most features can be replicated by installing and configuring your laptop/desktop directly. 36 | 37 | However, this comes at a cost of maintenance headache, each time the libraries and tools are upgraded, you have to manage those upgrades yourselves. 38 | 39 | Additionally, if you work in a team, different machines are set up differently, leading to incompatibility issues, what works in 1 machine may not work in another. 40 | 41 | 42 | ## Prerequisites 43 | 44 | #### Docker 45 | 46 | At the minimum, you'll need [Docker](https://docs.docker.com/install/#supported-platforms) engine installed. 47 | 48 | #### AWS Credentials 49 | For AWS SDK and CLI to work, you need to have AWS Credentials configured in the notebook. 50 | 51 | It's recommended to have AWS Credentials configured on your local machine 52 | so that you can use the same for the container (via volume mount) 53 | and avoid the need to configure every time the container starts. 54 | 55 | 1. First install [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) on your machine. 56 | 2. Then configure the [AWS Credentials](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration-multi-profiles) using your Access Key ID and Secret Access Key. 57 | It's recommended to specify a profile name when configuring this, in case you have many accounts or many different user profiles in future: 58 | ```bash 59 | aws configure --profile default-api 60 | ``` 61 | 3. Note the profile name you have specified, it should be used as the value of the environment variable AWS_PROFILE for the container. 62 | 63 | When running a container later on, you just need to add this volume mount `-v ~/.aws:/home/ec2-user/.aws:ro` (For Windows, replace `~` with `%USERPROFILE%`). 64 | 65 | 66 | ## Run Container 67 | 68 | #### Using `docker` 69 | The simplest way to start the `sagemaker-notebook-container` is to use `docker run` command: 70 | 71 | **Unix:** 72 | ```bash 73 | export CONTAINER_NAME=sagemaker-notebook-container 74 | export IMAGE_NAME=qtangs/sagemaker-notebook:tensorflow-p36 75 | export WORKDDIR=/home/ec2-user 76 | export AWS_PROFILE=default-api 77 | 78 | docker run -t --name=${CONTAINER_NAME} \ 79 | -p 8888:8888 \ 80 | -e AWS_PROFILE=${AWS_PROFILE} \ 81 | -v ~/.aws:${WORKDDIR}/.aws:ro \ 82 | ${IMAGE_NAME} 83 | ``` 84 | 85 | **Windows:** 86 | ```bat 87 | set CONTAINER_NAME=sagemaker-notebook-container 88 | set IMAGE_NAME=qtangs/sagemaker-notebook:tensorflow-p36 89 | set WORKDDIR=/home/ec2-user 90 | set AWS_PROFILE=default-api 91 | 92 | docker run -t --name=%CONTAINER_NAME% ^ 93 | -p 8888:8888 ^ 94 | -e AWS_PROFILE=%AWS_PROFILE% ^ 95 | -v %USERPROFILE%/.aws:%WORKDDIR%/.aws:ro ^ 96 | %IMAGE_NAME% 97 | ``` 98 | *(Replace `default-api` with the appropriate profile name from your own `~/.aws/credentials`)* 99 | 100 | #### Using `docker-compose` 101 | If you have [Docker Compose](https://docs.docker.com/compose/install/) (already included in [Docker Desktop](https://docs.docker.com/install/#supported-platforms) for Windows and Mac), 102 | you can use `docker-compose.yml` file so that you don't have to type the full docker run command. 103 | 104 | ```yaml 105 | # docker-compose.yml 106 | version: "3" 107 | services: 108 | sagemaker-notebook-container: 109 | image: qtangs/sagemaker-notebook:tensorflow-p36 110 | container_name: sagemaker-notebook-container 111 | ports: 112 | - 8888:8888 113 | environment: 114 | AWS_PROFILE: 'default-api' 115 | volumes: 116 | - ~/.aws:/home/ec2-user/.aws:ro # For AWS Credentials 117 | ``` 118 | *(For Windows, replace `~` with `${USERPROFILE}`)* 119 | 120 | *(Replace `default-api` with the appropriate profile name from your own `~/.aws/credentials`)* 121 | 122 | With that, you can simply run this each time: 123 | ```bash 124 | docker-compose up 125 | ``` 126 | or 127 | ```bash 128 | docker-compose up sagemaker-notebook-container 129 | ``` 130 | 131 | 132 | #### Accessing Jupyter Notebook 133 | You should see the following output, simply click on the `http://127.0.0.1:8888/...` link 134 | (or copy paste to your browser) to access Jupyter: 135 | ```text 136 | [I 03:10:12.757 NotebookApp] Writing notebook server cookie secret to /home/ec2-user/.local/share/jupyter/runtime/notebook_cookie_secret 137 | [I 03:10:13.335 NotebookApp] JupyterLab extension loaded from /home/ec2-user/anaconda3/lib/python3.7/site-packages/jupyterlab 138 | [I 03:10:13.335 NotebookApp] JupyterLab application directory is /home/ec2-user/anaconda3/share/jupyter/lab 139 | [I 03:10:13.352 NotebookApp] [nb_conda] enabled 140 | [I 03:10:13.373 NotebookApp] Serving notebooks from local directory: /home/ec2-user/SageMaker 141 | [I 03:10:13.373 NotebookApp] The Jupyter Notebook is running at: 142 | [I 03:10:13.373 NotebookApp] http://02b8db3c9e73:8888/?token=a22fc474c429a74650cb9ce74faf0ef2eedee182fc2eddec 143 | [I 03:10:13.373 NotebookApp] or http://127.0.0.1:8888/?token=a22fc474c429a74650cb9ce74faf0ef2eedee182fc2eddec 144 | [I 03:10:13.373 NotebookApp] Use Control-C to stop this server and shut down all kernels (twice to skip confirmation). 145 | ``` 146 | 147 | ## Optional additions 148 | 149 | #### Docker CLI 150 | Many SageMaker examples use docker to build custom images for training. 151 | 152 | Instead of installing a full Docker on Docker, which is a complex operation, these images make use of the host's Docker Engine instead. 153 | 154 | To achieve that, the Docker CLI is already installed on the base image and the Docker socket of the host machine is used to connect the host's Docker Engine. 155 | 156 | This is achieved by including when running the container. 157 | 158 | ```bash 159 | -v /var/run/docker.sock:/var/run/docker.sock:ro 160 | ``` 161 | 162 | *(For Windows, update the mount to: `-v //var/run/docker.sock:/var/run/docker.sock:ro`)* 163 | 164 | #### Git Integration 165 | Git is installed on the base image to allow git access directly from the container. 166 | 167 | Furthermore, the [jupyterlab-git](https://github.com/jupyterlab/jupyterlab-git) extension is installed on Jupyter Lab for quick GUI interaction with Git. 168 | 169 | If you use connect to Git repository using SSH, then you need to mount the `.ssh` folder: 170 | ```bash 171 | -v ~/.ssh:/home/ec2-user/.ssh:ro 172 | ``` 173 | 174 | *(For Windows, replace `~` with `${USERPROFILE}`)* 175 | 176 | #### Shared `SageMaker` directory 177 | To save all work created in the container, mount a directory to act as the `SageMaker` directory under `/home/ec2-user`: 178 | ```bash 179 | -v /Users/foobar/projects/SageMaker:/home/ec2-user/SageMaker 180 | ``` 181 | *(Replace `/Users/foobar/projects/SageMaker` with the appropriate folder from your own machine)* 182 | 183 | ## Sample scripts 184 | Following sample scripts have been provided to show an example of running a container using `qtangs/sagemaker-notebook:python3` image: 185 | 1. `run-python3-container.sh`: 186 | ```bash 187 | docker run -t --name=sagemaker-notebook-container && \ 188 | -p 8888:8888 && \ 189 | -e AWS_PROFILE=default-api && \ 190 | -v ~/.aws:/home/ec2-user/.aws:ro && \ 191 | -v ~/.ssh:/home/ec2-user/.ssh:ro && \ 192 | -v /Users/foobar/projects/SageMaker:/home/ec2-user/SageMaker && \ 193 | qtangs/sagemaker-notebook:python3 194 | ``` 195 | 2. `docker-compose.yml`: 196 | ```yaml 197 | version: "3" 198 | services: 199 | sagemaker-notebook-container: 200 | image: qtangs/sagemaker-notebook:python3 201 | container_name: sagemaker-notebook-container 202 | ports: 203 | - 8888:8888 204 | environment: 205 | AWS_PROFILE: "default-api" 206 | volumes: 207 | - ~/.aws:/home/ec2-user/.aws:ro # For AWS Credentials 208 | - ~/.ssh:/home/ec2-user/.ssh:ro # For Git Credentials 209 | - /var/run/docker.sock:/var/run/docker.sock:ro # For Docker CLI 210 | - /Users/foobar/projects/SageMaker:/home/ec2-user/SageMaker # For saving work in a host directory 211 | ``` 212 | *(Replace `/Users/foobar/projects/SageMaker` with the appropriate folder from your own machine)* 213 | 214 | With that, the container can be started using: 215 | ```bash 216 | docker-compose up sagemaker-notebook-container 217 | ``` 218 | -------------------------------------------------------------------------------- /base/Dockerfile: -------------------------------------------------------------------------------- 1 | # ================================================================== 2 | # Container mimicking SageMaker noteboook instance 3 | # ------------------------------------------------------------------ 4 | 5 | ARG UBUNTU_VERSION=18.04 6 | FROM ubuntu:${UBUNTU_VERSION} 7 | 8 | ARG MINICONDA_VERSION=4.5.12 9 | ARG CONDA_VERSION=4.5.12 10 | 11 | 12 | 13 | # ================================================================== 14 | # From Jupyter's base-notebook 15 | # https://hub.docker.com/r/jupyter/base-notebook/dockerfile 16 | # ### Modifications are commented with '###' 17 | # ------------------------------------------------------------------ 18 | 19 | ### Use ec2-user as the main user, just like the SageMaker Notebook instance ### 20 | ENV NB_USER="ec2-user" \ 21 | NB_UID="500" \ 22 | NB_GID="500" 23 | 24 | USER root 25 | 26 | # Install all OS dependencies for notebook server that starts but lacks all 27 | # features (e.g., download as all possible file formats) 28 | ENV DEBIAN_FRONTEND noninteractive 29 | RUN apt-get update && apt-get -yq dist-upgrade \ 30 | && apt-get install -yq --no-install-recommends \ 31 | wget \ 32 | bzip2 \ 33 | ca-certificates \ 34 | sudo \ 35 | locales \ 36 | fonts-liberation \ 37 | run-one \ 38 | && rm -rf /var/lib/apt/lists/* 39 | 40 | RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ 41 | locale-gen 42 | 43 | # Configure environment 44 | ENV CONDA_DIR=/home/$NB_USER/anaconda3 \ 45 | SHELL=/bin/bash \ 46 | NB_USER=$NB_USER \ 47 | NB_UID=$NB_UID \ 48 | NB_GID=$NB_GID \ 49 | LC_ALL=en_US.UTF-8 \ 50 | LANG=en_US.UTF-8 \ 51 | LANGUAGE=en_US.UTF-8 52 | ENV PATH=$CONDA_DIR/bin:$PATH \ 53 | HOME=/home/$NB_USER 54 | 55 | # Add a script that we will use to correct permissions after running certain commands 56 | COPY utils/jupyter/fix-permissions /usr/local/bin/fix-permissions 57 | 58 | # Enable prompt color in the skeleton .bashrc before creating the default NB_USER 59 | RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc 60 | 61 | # Create NB_USER wtih name jovyan user with UID=1000 and in the 'users' group 62 | # and make sure these dirs are writable by the `users` group. 63 | RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ 64 | sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ 65 | sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ 66 | ### Add a group with the same name as the user ### 67 | groupadd -g $NB_GID $NB_USER && \ 68 | useradd -m -s /bin/bash -u $NB_UID -g $NB_GID $NB_USER && \ 69 | mkdir -p $CONDA_DIR && \ 70 | chown $NB_USER:$NB_GID $CONDA_DIR && \ 71 | chmod g+w /etc/passwd && \ 72 | fix-permissions $HOME && \ 73 | fix-permissions "$(dirname $CONDA_DIR)" 74 | 75 | ### Allow NB_USER to use sudo without password ### 76 | RUN echo "$NB_USER ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers.d/$NB_USER 77 | 78 | USER $NB_UID 79 | WORKDIR $HOME 80 | 81 | # Install conda as jovyan and check the md5 sum provided on the download site 82 | ### Take conda version from ARG instead of hardcoding it here ### 83 | #ENV MINICONDA_VERSION=4.6.14 \ 84 | # CONDA_VERSION=4.7.10 85 | 86 | COPY utils/miniconda-md5 /tmp/miniconda-md5/ 87 | 88 | RUN cd /tmp && \ 89 | wget --quiet https://repo.continuum.io/miniconda/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ 90 | ### use the corresponding md5 file in miniconda-md5 folder to check the md5 sum ### 91 | echo "$(cat miniconda-md5/Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh.md5) *Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \ 92 | /bin/bash Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 93 | rm Miniconda3-${MINICONDA_VERSION}-Linux-x86_64.sh && \ 94 | echo "conda ${CONDA_VERSION}" >> $CONDA_DIR/conda-meta/pinned && \ 95 | $CONDA_DIR/bin/conda config --system --prepend channels conda-forge && \ 96 | $CONDA_DIR/bin/conda config --system --set auto_update_conda false && \ 97 | $CONDA_DIR/bin/conda config --system --set show_channel_urls true && \ 98 | $CONDA_DIR/bin/conda install --quiet --yes conda && \ 99 | $CONDA_DIR/bin/conda update --all --quiet --yes && \ 100 | conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \ 101 | conda clean --all -y && \ 102 | rm -rf /home/$NB_USER/.cache/yarn && \ 103 | fix-permissions $CONDA_DIR && \ 104 | fix-permissions /home/$NB_USER 105 | 106 | # Install Tini 107 | RUN conda install --quiet --yes 'tini=0.18.0' && \ 108 | conda list tini | grep tini | tr -s ' ' | cut -d ' ' -f 1,2 >> $CONDA_DIR/conda-meta/pinned && \ 109 | conda clean --all -y && \ 110 | fix-permissions $CONDA_DIR && \ 111 | fix-permissions /home/$NB_USER 112 | 113 | ARG JUPYTER_NB_VERSION=6.0.0 114 | ARG JUPYTER_LAB_VERSION=1.0.4 115 | 116 | # Install Jupyter Notebook, Lab, and Hub 117 | # Generate a notebook server config 118 | # Cleanup temporary files 119 | # Correct permissions 120 | # Do all this in a single RUN command to avoid duplicating all of the 121 | # files across image layers when the permissions change 122 | ### skip Jupyter Hub for now ### 123 | RUN conda install --quiet --yes \ 124 | "notebook=$JUPYTER_NB_VERSION" \ 125 | #'jupyterhub=1.0.0' \ 126 | "jupyterlab=$JUPYTER_LAB_VERSION" && \ 127 | conda clean --all -y && \ 128 | # npm cache clean --force && \ 129 | jupyter notebook --generate-config && \ 130 | rm -rf $CONDA_DIR/share/jupyter/lab/staging && \ 131 | rm -rf /home/$NB_USER/.cache/yarn && \ 132 | fix-permissions $CONDA_DIR && \ 133 | fix-permissions /home/$NB_USER 134 | 135 | 136 | 137 | # ================================================================== 138 | # From Jupyter's minimal-notebook 139 | # https://hub.docker.com/r/jupyter/minimal-notebook/dockerfile 140 | # ### Modifications are commented with '###' 141 | # ------------------------------------------------------------------ 142 | 143 | USER root 144 | 145 | # Install all OS dependencies for fully functional notebook server 146 | ### Comment out large packages and move them to INSTALL_OPTIONAL_PACKAGES section ### 147 | RUN apt-get update && apt-get install -yq --no-install-recommends \ 148 | build-essential \ 149 | # emacs \ 150 | git \ 151 | # inkscape \ 152 | jed \ 153 | libsm6 \ 154 | libxext-dev \ 155 | libxrender1 \ 156 | lmodern \ 157 | netcat \ 158 | pandoc \ 159 | python-dev \ 160 | # texlive-fonts-extra \ 161 | texlive-fonts-recommended \ 162 | texlive-generic-recommended \ 163 | texlive-latex-base \ 164 | texlive-latex-extra \ 165 | texlive-xetex \ 166 | tzdata \ 167 | unzip \ 168 | nano \ 169 | && rm -rf /var/lib/apt/lists/* 170 | 171 | 172 | 173 | # ================================================================== 174 | # Install ffmpeg for matplotlib anim 175 | # Copy from https://hub.docker.com/r/jupyter/scipy-notebook/dockerfile 176 | # ------------------------------------------------------------------ 177 | 178 | USER root 179 | 180 | # ffmpeg for matplotlib anim 181 | RUN apt-get update && \ 182 | apt-get install -y --no-install-recommends ffmpeg && \ 183 | rm -rf /var/lib/apt/lists/* 184 | 185 | 186 | 187 | # ================================================================== 188 | # Set up Conda Tab 189 | # ------------------------------------------------------------------ 190 | 191 | USER $NB_UID 192 | 193 | RUN conda install --yes 'nb_conda=2.2.1' && \ 194 | conda clean --all -y && \ 195 | jupyter nbextension install nb_conda --py --sys-prefix --symlink && \ 196 | jupyter nbextension enable nb_conda --py --sys-prefix && \ 197 | jupyter serverextension enable nb_conda --py --sys-prefix 198 | 199 | # Fix bug: https://github.com/Anaconda-Platform/nb_conda/issues/66 200 | RUN sed -ie "s/\(for env in info.'envs'.\)/\1 if env != root_env['dir']/g" $CONDA_DIR/lib/python*/site-packages/nb_conda/envmanager.py 201 | 202 | # Disable auto detection of new Conda envs 203 | RUN python -m nb_conda_kernels.install --disable 204 | 205 | 206 | 207 | # ================================================================== 208 | # Set up SageMaker Examples Tab 209 | # ------------------------------------------------------------------ 210 | 211 | USER $NB_UID 212 | 213 | RUN conda install --yes 'nbexamples' && \ 214 | conda clean --all -y && \ 215 | jupyter nbextension install --py nbexamples --sys-prefix && \ 216 | jupyter nbextension enable --py nbexamples --sys-prefix && \ 217 | jupyter serverextension enable --py nbexamples --sys-prefix 218 | 219 | # Use nbexamples from SageMaker 220 | COPY utils/nbexamples-sagemaker $CONDA_DIR/lib/python3.7/site-packages/nbexamples/ 221 | COPY utils/nbexamples-sagemaker/static $CONDA_DIR/share/jupyter/nbextensions/nbexamples/ 222 | 223 | ENV SAMPLE_NOTEBOOKS_DIR=$HOME/sample-notebooks 224 | 225 | # Copy script to set up examples 226 | COPY utils/sample-notebooks $SAMPLE_NOTEBOOKS_DIR 227 | 228 | # Fix permissions on $SAMPLE_NOTEBOOKS_DIR as root 229 | USER root 230 | RUN fix-permissions $SAMPLE_NOTEBOOKS_DIR 231 | 232 | USER $NB_UID 233 | 234 | # Download examples 235 | RUN $SAMPLE_NOTEBOOKS_DIR/update_examples.sh 236 | 237 | 238 | 239 | # ================================================================== 240 | # Add Git to JupyterLab 241 | # ------------------------------------------------------------------ 242 | 243 | USER root 244 | 245 | # Install openssh-client for git connection using SSH 246 | RUN apt-get update && \ 247 | apt-get install -y --no-install-recommends openssh-client && \ 248 | rm -rf /var/lib/apt/lists/* 249 | 250 | USER $NB_UID 251 | 252 | RUN conda install --yes "jupyterlab-git" && \ 253 | conda clean --all -y && \ 254 | jupyter labextension install @jupyterlab/git && \ 255 | jupyter serverextension enable --py jupyterlab_git && \ 256 | npm cache clean --force 257 | 258 | 259 | 260 | # ================================================================== 261 | # Install other important tools including docker cli, awscli 262 | # docker installation guide: https://docs.docker.com/install/linux/docker-ce/ubuntu/#install-using-the-repository 263 | # ------------------------------------------------------------------ 264 | 265 | USER $NB_UID 266 | 267 | ARG DOCKER_VERSION=19.03.1 268 | 269 | # Install Docker cli, the actual docker daemon is expected to be running on host with a mount of /var/run/docker.sock 270 | RUN cd /tmp && \ 271 | export CODE_NAME=$(cat /etc/*-release | grep -oP "UBUNTU_CODENAME=\K\w+") && \ 272 | wget https://download.docker.com/linux/ubuntu/dists/${CODE_NAME}/pool/stable/amd64/docker-ce-cli_${DOCKER_VERSION}~3-0~ubuntu-${CODE_NAME}_amd64.deb -O docker-ce-cli.deb && \ 273 | sudo dpkg -i docker-ce-cli.deb && \ 274 | rm -f docker-ce-cli.deb 275 | 276 | # Allow $NB_USER to run docker without having to specify sudo everytime 277 | RUN echo "alias docker='sudo /usr/bin/docker'" >> $HOME/.bashrc 278 | 279 | RUN df -h 280 | 281 | RUN pip install --no-cache-dir awscli 282 | 283 | 284 | 285 | # ================================================================== 286 | # Install optional packages (downside: these result in larger images) 287 | # ------------------------------------------------------------------ 288 | 289 | USER root 290 | 291 | # Optional packages, example: emacs inkscape texlive-fonts-extra 292 | ARG INSTALL_OPTIONAL_PACKAGES 293 | 294 | RUN if [ ! -z $INSTALL_OPTIONAL_PACKAGES]; then \ 295 | apt-get update && apt-get install -yq --no-install-recommends $INSTALL_OPTIONAL_PACKAGES; \ 296 | fi \ 297 | && rm -rf /var/lib/apt/lists/* 298 | 299 | 300 | 301 | # ================================================================== 302 | # Start service 303 | # ------------------------------------------------------------------ 304 | 305 | USER $NB_UID 306 | 307 | ENV NOTEBOOK_DIR=$HOME/SageMaker 308 | 309 | RUN mkdir $NOTEBOOK_DIR && \ 310 | sed -ie "/^#c.NotebookApp.notebook_dir/c\c.NotebookApp.notebook_dir = '$NOTEBOOK_DIR'" ~/.jupyter/jupyter_notebook_config.py 311 | 312 | # Add custom.js 313 | COPY utils/custom $HOME/.jupyter/custom 314 | 315 | # Add local files as late as possible to avoid cache busting 316 | COPY utils/jupyter/start.sh /usr/local/bin/ 317 | COPY utils/jupyter/start-notebook.sh /usr/local/bin/ 318 | COPY utils/jupyter/start-singleuser.sh /usr/local/bin/ 319 | COPY utils/jupyter/jupyter_notebook_config.py /etc/jupyter/ 320 | 321 | # Fix permissions on /etc/jupyter as root 322 | USER root 323 | RUN fix-permissions /etc/jupyter/ 324 | 325 | USER $NB_UID 326 | 327 | EXPOSE 8888 328 | 329 | # Configure container startup 330 | ENTRYPOINT ["tini", "-g", "--"] 331 | CMD ["start-notebook.sh"] -------------------------------------------------------------------------------- /base/utils/README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | These are utility folders used during Docker image build: 3 | 4 | 1. `custom`: Contains `custom.js` to modify Jupyter's Notebook UI, in this case to add **Open JupyterLab button** link. 5 | 2. `envs`: Custom scripts to create 1 or multiple Conda's environments based on a set of YAML files. Currently included: `python2.yml`, `python3.yml`, `mxnet_p36.yml`, `tensorflow_p36.yml`. 6 | 3. `jupyter`: Jupyter's utilities copied from [https://github.com/jupyter/docker-stacks/tree/master/base-notebook] 7 | 4. `miniconda-md5`: Contains md5 signatures of different versions of the file Miniconda-*.sh downloaded from [https://repo.continuum.io/miniconda] 8 | 5. `nbexamples-sagemaker`: Contains custom nbexamples code copied from SageMaker's Notebook instance. These are responsible for the **SageMaker Examples** tab. 9 | 6. `sample-notebooks`: Contains a script to download examples from AWS's [SageMaker examples](https://github.com/awslabs/amazon-sagemaker-examples) and, in the future, other sources. -------------------------------------------------------------------------------- /base/utils/custom/README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | Contains `custom.js` to modify Jupyter's Notebook UI, in this case to add **Open JupyterLab button** link. 3 | -------------------------------------------------------------------------------- /base/utils/custom/custom.js: -------------------------------------------------------------------------------- 1 | /** 2 | * https://github.com/jupyter/notebook/blob/master/notebook/static/custom/custom.js 3 | * Add Open JupyterLab button 4 | */ 5 | define([ 6 | 'jquery' 7 | ], 8 | function ($) { 9 | /** 10 | * Adds the 'Open JupyterLab' link to the upper-right corner of the Tree page. 11 | */ 12 | function addJupyterLabButton() { 13 | // The "Quit" button was recently added to Jupyter, so we know that if this button exists, 14 | // then JupyterLab is also there. 15 | $('#shutdown').before( 16 | '' 24 | ); 25 | $('#open_jupyter_lab').click(function (e) { 26 | e.preventDefault(); 27 | e.stopPropagation(); 28 | window.location.href = '/lab'; 29 | }); 30 | } 31 | 32 | $(document).ready(addJupyterLabButton()); 33 | 34 | if (typeof exports !== 'undefined') { 35 | // Export functions 36 | exports.addJupyterLabButton = addJupyterLabButton; 37 | } 38 | return exports; 39 | }); 40 | -------------------------------------------------------------------------------- /base/utils/jupyter/README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | Jupyter's utilities copied from [https://github.com/jupyter/docker-stacks/tree/master/base-notebook] 3 | -------------------------------------------------------------------------------- /base/utils/jupyter/fix-permissions: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # set permissions on a directory 3 | # after any installation, if a directory needs to be (human) user-writable, 4 | # run this script on it. 5 | # It will make everything in the directory owned by the group $NB_GID 6 | # and writable by that group. 7 | # Deployments that want to set a specific user id can preserve permissions 8 | # by adding the `--group-add users` line to `docker run`. 9 | 10 | # uses find to avoid touching files that already have the right permissions, 11 | # which would cause massive image explosion 12 | 13 | # right permissions are: 14 | # group=$NB_GID 15 | # AND permissions include group rwX (directory-execute) 16 | # AND directories have setuid,setgid bits set 17 | 18 | set -e 19 | 20 | for d in "$@"; do 21 | find "$d" \ 22 | ! \( \ 23 | -group $NB_GID \ 24 | -a -perm -g+rwX \ 25 | \) \ 26 | -exec chgrp $NB_GID {} \; \ 27 | -exec chmod g+rwX {} \; 28 | # setuid,setgid *on directories only* 29 | find "$d" \ 30 | \( \ 31 | -type d \ 32 | -a ! -perm -6000 \ 33 | \) \ 34 | -exec chmod +6000 {} \; 35 | done 36 | -------------------------------------------------------------------------------- /base/utils/jupyter/jupyter_notebook_config.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) Jupyter Development Team. 2 | # Distributed under the terms of the Modified BSD License. 3 | 4 | from jupyter_core.paths import jupyter_data_dir 5 | import subprocess 6 | import os 7 | import errno 8 | import stat 9 | 10 | c = get_config() 11 | c.NotebookApp.ip = '0.0.0.0' 12 | c.NotebookApp.port = 8888 13 | c.NotebookApp.open_browser = False 14 | 15 | # https://github.com/jupyter/notebook/issues/3130 16 | c.FileContentsManager.delete_to_trash = False 17 | 18 | # Generate a self-signed certificate 19 | if 'GEN_CERT' in os.environ: 20 | dir_name = jupyter_data_dir() 21 | pem_file = os.path.join(dir_name, 'notebook.pem') 22 | try: 23 | os.makedirs(dir_name) 24 | except OSError as exc: # Python >2.5 25 | if exc.errno == errno.EEXIST and os.path.isdir(dir_name): 26 | pass 27 | else: 28 | raise 29 | 30 | # Generate an openssl.cnf file to set the distinguished name 31 | cnf_file = os.path.join(os.getenv('CONDA_DIR', '/usr/lib'), 'ssl', 'openssl.cnf') 32 | if not os.path.isfile(cnf_file): 33 | with open(cnf_file, 'w') as fh: 34 | fh.write('''\ 35 | [req] 36 | distinguished_name = req_distinguished_name 37 | [req_distinguished_name] 38 | ''') 39 | 40 | # Generate a certificate if one doesn't exist on disk 41 | subprocess.check_call(['openssl', 'req', '-new', 42 | '-newkey', 'rsa:2048', 43 | '-days', '365', 44 | '-nodes', '-x509', 45 | '-subj', '/C=XX/ST=XX/L=XX/O=generated/CN=generated', 46 | '-keyout', pem_file, 47 | '-out', pem_file]) 48 | # Restrict access to the file 49 | os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) 50 | c.NotebookApp.certfile = pem_file 51 | 52 | # Change default umask for all subprocesses of the notebook server if set in 53 | # the environment 54 | if 'NB_UMASK' in os.environ: 55 | os.umask(int(os.environ['NB_UMASK'], 8)) -------------------------------------------------------------------------------- /base/utils/jupyter/start-notebook.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | wrapper="" 8 | if [[ "${RESTARTABLE}" == "yes" ]]; then 9 | wrapper="run-one-constantly" 10 | fi 11 | 12 | if [[ ! -z "${JUPYTERHUB_API_TOKEN}" ]]; then 13 | # launched by JupyterHub, use single-user entrypoint 14 | exec /usr/local/bin/start-singleuser.sh "$@" 15 | elif [[ ! -z "${JUPYTER_ENABLE_LAB}" ]]; then 16 | . /usr/local/bin/start.sh $wrapper jupyter lab "$@" 17 | else 18 | . /usr/local/bin/start.sh $wrapper jupyter notebook "$@" 19 | fi 20 | -------------------------------------------------------------------------------- /base/utils/jupyter/start-singleuser.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | # set default ip to 0.0.0.0 8 | if [[ "$NOTEBOOK_ARGS $@" != *"--ip="* ]]; then 9 | NOTEBOOK_ARGS="--ip=0.0.0.0 $NOTEBOOK_ARGS" 10 | fi 11 | 12 | # handle some deprecated environment variables 13 | # from DockerSpawner < 0.8. 14 | # These won't be passed from DockerSpawner 0.9, 15 | # so avoid specifying --arg=empty-string 16 | if [ ! -z "$NOTEBOOK_DIR" ]; then 17 | NOTEBOOK_ARGS="--notebook-dir='$NOTEBOOK_DIR' $NOTEBOOK_ARGS" 18 | fi 19 | if [ ! -z "$JPY_PORT" ]; then 20 | NOTEBOOK_ARGS="--port=$JPY_PORT $NOTEBOOK_ARGS" 21 | fi 22 | if [ ! -z "$JPY_USER" ]; then 23 | NOTEBOOK_ARGS="--user=$JPY_USER $NOTEBOOK_ARGS" 24 | fi 25 | if [ ! -z "$JPY_COOKIE_NAME" ]; then 26 | NOTEBOOK_ARGS="--cookie-name=$JPY_COOKIE_NAME $NOTEBOOK_ARGS" 27 | fi 28 | if [ ! -z "$JPY_BASE_URL" ]; then 29 | NOTEBOOK_ARGS="--base-url=$JPY_BASE_URL $NOTEBOOK_ARGS" 30 | fi 31 | if [ ! -z "$JPY_HUB_PREFIX" ]; then 32 | NOTEBOOK_ARGS="--hub-prefix=$JPY_HUB_PREFIX $NOTEBOOK_ARGS" 33 | fi 34 | if [ ! -z "$JPY_HUB_API_URL" ]; then 35 | NOTEBOOK_ARGS="--hub-api-url=$JPY_HUB_API_URL $NOTEBOOK_ARGS" 36 | fi 37 | if [ ! -z "$JUPYTER_ENABLE_LAB" ]; then 38 | NOTEBOOK_BIN="jupyter labhub" 39 | else 40 | NOTEBOOK_BIN="jupyterhub-singleuser" 41 | fi 42 | 43 | . /usr/local/bin/start.sh $NOTEBOOK_BIN $NOTEBOOK_ARGS "$@" 44 | -------------------------------------------------------------------------------- /base/utils/jupyter/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright (c) Jupyter Development Team. 3 | # Distributed under the terms of the Modified BSD License. 4 | 5 | set -e 6 | 7 | # Exec the specified command or fall back on bash 8 | if [ $# -eq 0 ]; then 9 | cmd=( "bash" ) 10 | else 11 | cmd=( "$@" ) 12 | fi 13 | 14 | run-hooks () { 15 | # Source scripts or run executable files in a directory 16 | if [[ ! -d "$1" ]] ; then 17 | return 18 | fi 19 | echo "$0: running hooks in $1" 20 | for f in "$1/"*; do 21 | case "$f" in 22 | *.sh) 23 | echo "$0: running $f" 24 | source "$f" 25 | ;; 26 | *) 27 | if [[ -x "$f" ]] ; then 28 | echo "$0: running $f" 29 | "$f" 30 | else 31 | echo "$0: ignoring $f" 32 | fi 33 | ;; 34 | esac 35 | done 36 | echo "$0: done running hooks in $1" 37 | } 38 | 39 | run-hooks /usr/local/bin/start-notebook.d 40 | 41 | # Handle special flags if we're root 42 | if [ $(id -u) == 0 ] ; then 43 | 44 | # Only attempt to change the jovyan username if it exists 45 | if id jovyan &> /dev/null ; then 46 | echo "Set username to: $NB_USER" 47 | usermod -d /home/$NB_USER -l $NB_USER jovyan 48 | fi 49 | 50 | # Handle case where provisioned storage does not have the correct permissions by default 51 | # Ex: default NFS/EFS (no auto-uid/gid) 52 | if [[ "$CHOWN_HOME" == "1" || "$CHOWN_HOME" == 'yes' ]]; then 53 | echo "Changing ownership of /home/$NB_USER to $NB_UID:$NB_GID with options '${CHOWN_HOME_OPTS}'" 54 | chown $CHOWN_HOME_OPTS $NB_UID:$NB_GID /home/$NB_USER 55 | fi 56 | if [ ! -z "$CHOWN_EXTRA" ]; then 57 | for extra_dir in $(echo $CHOWN_EXTRA | tr ',' ' '); do 58 | echo "Changing ownership of ${extra_dir} to $NB_UID:$NB_GID with options '${CHOWN_EXTRA_OPTS}'" 59 | chown $CHOWN_EXTRA_OPTS $NB_UID:$NB_GID $extra_dir 60 | done 61 | fi 62 | 63 | # handle home and working directory if the username changed 64 | if [[ "$NB_USER" != "jovyan" ]]; then 65 | # changing username, make sure homedir exists 66 | # (it could be mounted, and we shouldn't create it if it already exists) 67 | if [[ ! -e "/home/$NB_USER" ]]; then 68 | echo "Relocating home dir to /home/$NB_USER" 69 | mv /home/jovyan "/home/$NB_USER" 70 | fi 71 | # if workdir is in /home/jovyan, cd to /home/$NB_USER 72 | if [[ "$PWD/" == "/home/jovyan/"* ]]; then 73 | newcwd="/home/$NB_USER/${PWD:13}" 74 | echo "Setting CWD to $newcwd" 75 | cd "$newcwd" 76 | fi 77 | fi 78 | 79 | # Change UID of NB_USER to NB_UID if it does not match 80 | if [ "$NB_UID" != $(id -u $NB_USER) ] ; then 81 | echo "Set $NB_USER UID to: $NB_UID" 82 | usermod -u $NB_UID $NB_USER 83 | fi 84 | 85 | # Set NB_USER primary gid to NB_GID (after making the group). Set 86 | # supplementary gids to NB_GID and 100. 87 | if [ "$NB_GID" != $(id -g $NB_USER) ] ; then 88 | echo "Add $NB_USER to group: $NB_GID" 89 | groupadd -g $NB_GID -o ${NB_GROUP:-${NB_USER}} 90 | usermod -g $NB_GID -aG 100 $NB_USER 91 | fi 92 | 93 | # Enable sudo if requested 94 | if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then 95 | echo "Granting $NB_USER sudo access and appending $CONDA_DIR/bin to sudo PATH" 96 | echo "$NB_USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/notebook 97 | fi 98 | 99 | # Add $CONDA_DIR/bin to sudo secure_path 100 | sed -r "s#Defaults\s+secure_path=\"([^\"]+)\"#Defaults secure_path=\"\1:$CONDA_DIR/bin\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path 101 | 102 | # Exec the command as NB_USER with the PATH and the rest of 103 | # the environment preserved 104 | run-hooks /usr/local/bin/before-notebook.d 105 | echo "Executing the command: ${cmd[@]}" 106 | exec sudo -E -H -u $NB_USER PATH=$PATH XDG_CACHE_HOME=/home/$NB_USER/.cache PYTHONPATH=${PYTHONPATH:-} "${cmd[@]}" 107 | else 108 | if [[ "$NB_UID" == "$(id -u jovyan)" && "$NB_GID" == "$(id -g jovyan)" ]]; then 109 | # User is not attempting to override user/group via environment 110 | # variables, but they could still have overridden the uid/gid that 111 | # container runs as. Check that the user has an entry in the passwd 112 | # file and if not add an entry. 113 | STATUS=0 && whoami &> /dev/null || STATUS=$? && true 114 | if [[ "$STATUS" != "0" ]]; then 115 | if [[ -w /etc/passwd ]]; then 116 | echo "Adding passwd file entry for $(id -u)" 117 | cat /etc/passwd | sed -e "s/^jovyan:/nayvoj:/" > /tmp/passwd 118 | echo "jovyan:x:$(id -u):$(id -g):,,,:/home/jovyan:/bin/bash" >> /tmp/passwd 119 | cat /tmp/passwd > /etc/passwd 120 | rm /tmp/passwd 121 | else 122 | echo 'Container must be run with group "root" to update passwd file' 123 | fi 124 | fi 125 | 126 | # Warn if the user isn't going to be able to write files to $HOME. 127 | if [[ ! -w /home/jovyan ]]; then 128 | echo 'Container must be run with group "users" to update files' 129 | fi 130 | else 131 | # Warn if looks like user want to override uid/gid but hasn't 132 | # run the container as root. 133 | if [[ ! -z "$NB_UID" && "$NB_UID" != "$(id -u)" ]]; then 134 | echo 'Container must be run as root to set $NB_UID' 135 | fi 136 | if [[ ! -z "$NB_GID" && "$NB_GID" != "$(id -g)" ]]; then 137 | echo 'Container must be run as root to set $NB_GID' 138 | fi 139 | fi 140 | 141 | # Warn if looks like user want to run in sudo mode but hasn't run 142 | # the container as root. 143 | if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == 'yes' ]]; then 144 | echo 'Container must be run as root to grant sudo permissions' 145 | fi 146 | 147 | # Execute the command 148 | run-hooks /usr/local/bin/before-notebook.d 149 | echo "Executing the command: ${cmd[@]}" 150 | exec "${cmd[@]}" 151 | fi 152 | -------------------------------------------------------------------------------- /base/utils/miniconda-md5/Miniconda3-4.5.12-Linux-x86_64.sh.md5: -------------------------------------------------------------------------------- 1 | 866ae9dff53ad0874e1d1a60b1ad1ef8 -------------------------------------------------------------------------------- /base/utils/miniconda-md5/Miniconda3-4.6.14-Linux-x86_64.sh.md5: -------------------------------------------------------------------------------- 1 | 718259965f234088d785cad1fbd7de03 -------------------------------------------------------------------------------- /base/utils/miniconda-md5/Miniconda3-4.7.10-Linux-x86_64.sh.md5: -------------------------------------------------------------------------------- 1 | 1c945f2b3335c7b2b15130b1b2dc5cf4 -------------------------------------------------------------------------------- /base/utils/miniconda-md5/README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | Contains md5 signatures of different versions of the file Miniconda-*.sh downloaded from [https://repo.continuum.io/miniconda] 3 | -------------------------------------------------------------------------------- /base/utils/nbexamples-sagemaker/README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | Contains custom nbexamples code copied from SageMaker's Notebook instance. These are responsible for the **SageMaker Examples** tab. -------------------------------------------------------------------------------- /base/utils/nbexamples-sagemaker/__init__.py: -------------------------------------------------------------------------------- 1 | from ._version import get_versions 2 | __version__ = get_versions()['version'] 3 | del get_versions 4 | 5 | 6 | def _jupyter_server_extension_paths(): 7 | """Returns server extension metadata to notebook 4.2+""" 8 | return [{ 9 | 'module': 'nbexamples.handlers' 10 | }] 11 | 12 | 13 | def _jupyter_nbextension_paths(): 14 | """Returns frontend extension metadata to notebook 4.2+""" 15 | return [{ 16 | 'section': 'tree', 17 | 'src': 'static', 18 | 'dest': 'nbexamples', 19 | 'require': 'nbexamples/main' 20 | }] 21 | -------------------------------------------------------------------------------- /base/utils/nbexamples-sagemaker/_version.py: -------------------------------------------------------------------------------- 1 | # This file helps to compute a version number in source trees obtained from 2 | # git-archive tarball (such as those provided by githubs download-from-tag 3 | # feature). Distribution tarballs (built by setup.py sdist) and build 4 | # directories (produced by setup.py build) will contain a much shorter file 5 | # that just contains the computed version number. 6 | 7 | # This file is released into the public domain. Generated by 8 | # versioneer-0.16 (https://github.com/warner/python-versioneer) 9 | 10 | """Git implementation of _version.py.""" 11 | 12 | import errno 13 | import os 14 | import re 15 | import subprocess 16 | import sys 17 | 18 | 19 | def get_keywords(): 20 | """Get the keywords needed to look up the version information.""" 21 | # these strings will be replaced by git during git-archive. 22 | # setup.py/versioneer.py will grep for the variable names, so they must 23 | # each be defined on a line of their own. _version.py will just call 24 | # get_keywords(). 25 | git_refnames = " (HEAD -> master, tag: v0.3.1)" 26 | git_full = "b14421ef9a88828b5a0e76d376043ee0f13f9da8" 27 | keywords = {"refnames": git_refnames, "full": git_full} 28 | return keywords 29 | 30 | 31 | class VersioneerConfig: 32 | """Container for Versioneer configuration parameters.""" 33 | 34 | 35 | def get_config(): 36 | """Create, populate and return the VersioneerConfig() object.""" 37 | # these strings are filled in when 'setup.py versioneer' creates 38 | # _version.py 39 | cfg = VersioneerConfig() 40 | cfg.VCS = "git" 41 | cfg.style = "pep440" 42 | cfg.tag_prefix = "" 43 | cfg.parentdir_prefix = "None" 44 | cfg.versionfile_source = "nbexamples/_version.py" 45 | cfg.verbose = False 46 | return cfg 47 | 48 | 49 | class NotThisMethod(Exception): 50 | """Exception raised if a method is not valid for the current scenario.""" 51 | 52 | 53 | LONG_VERSION_PY = {} 54 | HANDLERS = {} 55 | 56 | 57 | def register_vcs_handler(vcs, method): # decorator 58 | """Decorator to mark a method as the handler for a particular VCS.""" 59 | def decorate(f): 60 | """Store f in HANDLERS[vcs][method].""" 61 | if vcs not in HANDLERS: 62 | HANDLERS[vcs] = {} 63 | HANDLERS[vcs][method] = f 64 | return f 65 | return decorate 66 | 67 | 68 | def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False): 69 | """Call the given command(s).""" 70 | assert isinstance(commands, list) 71 | p = None 72 | for c in commands: 73 | try: 74 | dispcmd = str([c] + args) 75 | # remember shell=False, so use git.cmd on windows, not just git 76 | p = subprocess.Popen([c] + args, cwd=cwd, stdout=subprocess.PIPE, 77 | stderr=(subprocess.PIPE if hide_stderr 78 | else None)) 79 | break 80 | except EnvironmentError: 81 | e = sys.exc_info()[1] 82 | if e.errno == errno.ENOENT: 83 | continue 84 | if verbose: 85 | print("unable to run %s" % dispcmd) 86 | print(e) 87 | return None 88 | else: 89 | if verbose: 90 | print("unable to find command, tried %s" % (commands,)) 91 | return None 92 | stdout = p.communicate()[0].strip() 93 | if sys.version_info[0] >= 3: 94 | stdout = stdout.decode() 95 | if p.returncode != 0: 96 | if verbose: 97 | print("unable to run %s (error)" % dispcmd) 98 | return None 99 | return stdout 100 | 101 | 102 | def versions_from_parentdir(parentdir_prefix, root, verbose): 103 | """Try to determine the version from the parent directory name. 104 | 105 | Source tarballs conventionally unpack into a directory that includes 106 | both the project name and a version string. 107 | """ 108 | dirname = os.path.basename(root) 109 | if not dirname.startswith(parentdir_prefix): 110 | if verbose: 111 | print("guessing rootdir is '%s', but '%s' doesn't start with " 112 | "prefix '%s'" % (root, dirname, parentdir_prefix)) 113 | raise NotThisMethod("rootdir doesn't start with parentdir_prefix") 114 | return {"version": dirname[len(parentdir_prefix):], 115 | "full-revisionid": None, 116 | "dirty": False, "error": None} 117 | 118 | 119 | @register_vcs_handler("git", "get_keywords") 120 | def git_get_keywords(versionfile_abs): 121 | """Extract version information from the given file.""" 122 | # the code embedded in _version.py can just fetch the value of these 123 | # keywords. When used from setup.py, we don't want to import _version.py, 124 | # so we do it with a regexp instead. This function is not used from 125 | # _version.py. 126 | keywords = {} 127 | try: 128 | f = open(versionfile_abs, "r") 129 | for line in f.readlines(): 130 | if line.strip().startswith("git_refnames ="): 131 | mo = re.search(r'=\s*"(.*)"', line) 132 | if mo: 133 | keywords["refnames"] = mo.group(1) 134 | if line.strip().startswith("git_full ="): 135 | mo = re.search(r'=\s*"(.*)"', line) 136 | if mo: 137 | keywords["full"] = mo.group(1) 138 | f.close() 139 | except EnvironmentError: 140 | pass 141 | return keywords 142 | 143 | 144 | @register_vcs_handler("git", "keywords") 145 | def git_versions_from_keywords(keywords, tag_prefix, verbose): 146 | """Get version information from git keywords.""" 147 | if not keywords: 148 | raise NotThisMethod("no keywords at all, weird") 149 | refnames = keywords["refnames"].strip() 150 | if refnames.startswith("$Format"): 151 | if verbose: 152 | print("keywords are unexpanded, not using") 153 | raise NotThisMethod("unexpanded keywords, not a git-archive tarball") 154 | refs = set([r.strip() for r in refnames.strip("()").split(",")]) 155 | # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of 156 | # just "foo-1.0". If we see a "tag: " prefix, prefer those. 157 | TAG = "tag: " 158 | tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) 159 | if not tags: 160 | # Either we're using git < 1.8.3, or there really are no tags. We use 161 | # a heuristic: assume all version tags have a digit. The old git %d 162 | # expansion behaves like git log --decorate=short and strips out the 163 | # refs/heads/ and refs/tags/ prefixes that would let us distinguish 164 | # between branches and tags. By ignoring refnames without digits, we 165 | # filter out many common branch names like "release" and 166 | # "stabilization", as well as "HEAD" and "master". 167 | tags = set([r for r in refs if re.search(r'\d', r)]) 168 | if verbose: 169 | print("discarding '%s', no digits" % ",".join(refs-tags)) 170 | if verbose: 171 | print("likely tags: %s" % ",".join(sorted(tags))) 172 | for ref in sorted(tags): 173 | # sorting will prefer e.g. "2.0" over "2.0rc1" 174 | if ref.startswith(tag_prefix): 175 | r = ref[len(tag_prefix):] 176 | if verbose: 177 | print("picking %s" % r) 178 | return {"version": r, 179 | "full-revisionid": keywords["full"].strip(), 180 | "dirty": False, "error": None 181 | } 182 | # no suitable tags, so version is "0+unknown", but full hex is still there 183 | if verbose: 184 | print("no suitable tags, using unknown + full revision id") 185 | return {"version": "0+unknown", 186 | "full-revisionid": keywords["full"].strip(), 187 | "dirty": False, "error": "no suitable tags"} 188 | 189 | 190 | @register_vcs_handler("git", "pieces_from_vcs") 191 | def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): 192 | """Get version from 'git describe' in the root of the source tree. 193 | 194 | This only gets called if the git-archive 'subst' keywords were *not* 195 | expanded, and _version.py hasn't already been rewritten with a short 196 | version string, meaning we're inside a checked out source tree. 197 | """ 198 | if not os.path.exists(os.path.join(root, ".git")): 199 | if verbose: 200 | print("no .git in %s" % root) 201 | raise NotThisMethod("no .git directory") 202 | 203 | GITS = ["git"] 204 | if sys.platform == "win32": 205 | GITS = ["git.cmd", "git.exe"] 206 | # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] 207 | # if there isn't one, this yields HEX[-dirty] (no NUM) 208 | describe_out = run_command(GITS, ["describe", "--tags", "--dirty", 209 | "--always", "--long", 210 | "--match", "%s*" % tag_prefix], 211 | cwd=root) 212 | # --long was added in git-1.5.5 213 | if describe_out is None: 214 | raise NotThisMethod("'git describe' failed") 215 | describe_out = describe_out.strip() 216 | full_out = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) 217 | if full_out is None: 218 | raise NotThisMethod("'git rev-parse' failed") 219 | full_out = full_out.strip() 220 | 221 | pieces = {} 222 | pieces["long"] = full_out 223 | pieces["short"] = full_out[:7] # maybe improved later 224 | pieces["error"] = None 225 | 226 | # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] 227 | # TAG might have hyphens. 228 | git_describe = describe_out 229 | 230 | # look for -dirty suffix 231 | dirty = git_describe.endswith("-dirty") 232 | pieces["dirty"] = dirty 233 | if dirty: 234 | git_describe = git_describe[:git_describe.rindex("-dirty")] 235 | 236 | # now we have TAG-NUM-gHEX or HEX 237 | 238 | if "-" in git_describe: 239 | # TAG-NUM-gHEX 240 | mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) 241 | if not mo: 242 | # unparseable. Maybe git-describe is misbehaving? 243 | pieces["error"] = ("unable to parse git-describe output: '%s'" 244 | % describe_out) 245 | return pieces 246 | 247 | # tag 248 | full_tag = mo.group(1) 249 | if not full_tag.startswith(tag_prefix): 250 | if verbose: 251 | fmt = "tag '%s' doesn't start with prefix '%s'" 252 | print(fmt % (full_tag, tag_prefix)) 253 | pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" 254 | % (full_tag, tag_prefix)) 255 | return pieces 256 | pieces["closest-tag"] = full_tag[len(tag_prefix):] 257 | 258 | # distance: number of commits since tag 259 | pieces["distance"] = int(mo.group(2)) 260 | 261 | # commit: short hex revision ID 262 | pieces["short"] = mo.group(3) 263 | 264 | else: 265 | # HEX: no tags 266 | pieces["closest-tag"] = None 267 | count_out = run_command(GITS, ["rev-list", "HEAD", "--count"], 268 | cwd=root) 269 | pieces["distance"] = int(count_out) # total number of commits 270 | 271 | return pieces 272 | 273 | 274 | def plus_or_dot(pieces): 275 | """Return a + if we don't already have one, else return a .""" 276 | if "+" in pieces.get("closest-tag", ""): 277 | return "." 278 | return "+" 279 | 280 | 281 | def render_pep440(pieces): 282 | """Build up version string, with post-release "local version identifier". 283 | 284 | Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you 285 | get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty 286 | 287 | Exceptions: 288 | 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] 289 | """ 290 | if pieces["closest-tag"]: 291 | rendered = pieces["closest-tag"] 292 | if pieces["distance"] or pieces["dirty"]: 293 | rendered += plus_or_dot(pieces) 294 | rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) 295 | if pieces["dirty"]: 296 | rendered += ".dirty" 297 | else: 298 | # exception #1 299 | rendered = "0+untagged.%d.g%s" % (pieces["distance"], 300 | pieces["short"]) 301 | if pieces["dirty"]: 302 | rendered += ".dirty" 303 | return rendered 304 | 305 | 306 | def render_pep440_pre(pieces): 307 | """TAG[.post.devDISTANCE] -- No -dirty. 308 | 309 | Exceptions: 310 | 1: no tags. 0.post.devDISTANCE 311 | """ 312 | if pieces["closest-tag"]: 313 | rendered = pieces["closest-tag"] 314 | if pieces["distance"]: 315 | rendered += ".post.dev%d" % pieces["distance"] 316 | else: 317 | # exception #1 318 | rendered = "0.post.dev%d" % pieces["distance"] 319 | return rendered 320 | 321 | 322 | def render_pep440_post(pieces): 323 | """TAG[.postDISTANCE[.dev0]+gHEX] . 324 | 325 | The ".dev0" means dirty. Note that .dev0 sorts backwards 326 | (a dirty tree will appear "older" than the corresponding clean one), 327 | but you shouldn't be releasing software with -dirty anyways. 328 | 329 | Exceptions: 330 | 1: no tags. 0.postDISTANCE[.dev0] 331 | """ 332 | if pieces["closest-tag"]: 333 | rendered = pieces["closest-tag"] 334 | if pieces["distance"] or pieces["dirty"]: 335 | rendered += ".post%d" % pieces["distance"] 336 | if pieces["dirty"]: 337 | rendered += ".dev0" 338 | rendered += plus_or_dot(pieces) 339 | rendered += "g%s" % pieces["short"] 340 | else: 341 | # exception #1 342 | rendered = "0.post%d" % pieces["distance"] 343 | if pieces["dirty"]: 344 | rendered += ".dev0" 345 | rendered += "+g%s" % pieces["short"] 346 | return rendered 347 | 348 | 349 | def render_pep440_old(pieces): 350 | """TAG[.postDISTANCE[.dev0]] . 351 | 352 | The ".dev0" means dirty. 353 | 354 | Eexceptions: 355 | 1: no tags. 0.postDISTANCE[.dev0] 356 | """ 357 | if pieces["closest-tag"]: 358 | rendered = pieces["closest-tag"] 359 | if pieces["distance"] or pieces["dirty"]: 360 | rendered += ".post%d" % pieces["distance"] 361 | if pieces["dirty"]: 362 | rendered += ".dev0" 363 | else: 364 | # exception #1 365 | rendered = "0.post%d" % pieces["distance"] 366 | if pieces["dirty"]: 367 | rendered += ".dev0" 368 | return rendered 369 | 370 | 371 | def render_git_describe(pieces): 372 | """TAG[-DISTANCE-gHEX][-dirty]. 373 | 374 | Like 'git describe --tags --dirty --always'. 375 | 376 | Exceptions: 377 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 378 | """ 379 | if pieces["closest-tag"]: 380 | rendered = pieces["closest-tag"] 381 | if pieces["distance"]: 382 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 383 | else: 384 | # exception #1 385 | rendered = pieces["short"] 386 | if pieces["dirty"]: 387 | rendered += "-dirty" 388 | return rendered 389 | 390 | 391 | def render_git_describe_long(pieces): 392 | """TAG-DISTANCE-gHEX[-dirty]. 393 | 394 | Like 'git describe --tags --dirty --always -long'. 395 | The distance/hash is unconditional. 396 | 397 | Exceptions: 398 | 1: no tags. HEX[-dirty] (note: no 'g' prefix) 399 | """ 400 | if pieces["closest-tag"]: 401 | rendered = pieces["closest-tag"] 402 | rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) 403 | else: 404 | # exception #1 405 | rendered = pieces["short"] 406 | if pieces["dirty"]: 407 | rendered += "-dirty" 408 | return rendered 409 | 410 | 411 | def render(pieces, style): 412 | """Render the given version pieces into the requested style.""" 413 | if pieces["error"]: 414 | return {"version": "unknown", 415 | "full-revisionid": pieces.get("long"), 416 | "dirty": None, 417 | "error": pieces["error"]} 418 | 419 | if not style or style == "default": 420 | style = "pep440" # the default 421 | 422 | if style == "pep440": 423 | rendered = render_pep440(pieces) 424 | elif style == "pep440-pre": 425 | rendered = render_pep440_pre(pieces) 426 | elif style == "pep440-post": 427 | rendered = render_pep440_post(pieces) 428 | elif style == "pep440-old": 429 | rendered = render_pep440_old(pieces) 430 | elif style == "git-describe": 431 | rendered = render_git_describe(pieces) 432 | elif style == "git-describe-long": 433 | rendered = render_git_describe_long(pieces) 434 | else: 435 | raise ValueError("unknown style '%s'" % style) 436 | 437 | return {"version": rendered, "full-revisionid": pieces["long"], 438 | "dirty": pieces["dirty"], "error": None} 439 | 440 | 441 | def get_versions(): 442 | """Get version information or return default if unable to do so.""" 443 | # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have 444 | # __file__, we can work backwards from there to the root. Some 445 | # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which 446 | # case we can only use expanded keywords. 447 | 448 | cfg = get_config() 449 | verbose = cfg.verbose 450 | 451 | try: 452 | return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, 453 | verbose) 454 | except NotThisMethod: 455 | pass 456 | 457 | try: 458 | root = os.path.realpath(__file__) 459 | # versionfile_source is the relative path from the top of the source 460 | # tree (where the .git directory might live) to this file. Invert 461 | # this to find the root from __file__. 462 | for i in cfg.versionfile_source.split('/'): 463 | root = os.path.dirname(root) 464 | except NameError: 465 | return {"version": "0+unknown", "full-revisionid": None, 466 | "dirty": None, 467 | "error": "unable to find root of source tree"} 468 | 469 | try: 470 | pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) 471 | return render(pieces, cfg.style) 472 | except NotThisMethod: 473 | pass 474 | 475 | try: 476 | if cfg.parentdir_prefix: 477 | return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) 478 | except NotThisMethod: 479 | pass 480 | 481 | return {"version": "0+unknown", "full-revisionid": None, 482 | "dirty": None, 483 | "error": "unable to compute version"} 484 | -------------------------------------------------------------------------------- /base/utils/nbexamples-sagemaker/external/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/qtangs/sagemaker-notebook-container/cf6a9e8c58cd5da6b02ea88ab6052bceb291e8a9/base/utils/nbexamples-sagemaker/external/__init__.py -------------------------------------------------------------------------------- /base/utils/nbexamples-sagemaker/external/titlecase.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | Original Perl version by: John Gruber http://daringfireball.net/ 10 May 2008 6 | Python version by Stuart Colville http://muffinresearch.co.uk 7 | License: http://www.opensource.org/licenses/mit-license.php 8 | """ 9 | 10 | from __future__ import unicode_literals 11 | 12 | import argparse 13 | import re 14 | import sys 15 | 16 | __all__ = ['titlecase'] 17 | __version__ = '0.12.0' 18 | 19 | SMALL = 'a|an|and|as|at|but|by|en|for|if|in|of|on|or|the|to|v\.?|via|vs\.?' 20 | PUNCT = r"""!"“#$%&'‘()*+,\-–‒—―./:;?@[\\\]_`{|}~""" 21 | 22 | SMALL_WORDS = re.compile(r'^(%s)$' % SMALL, re.I) 23 | INLINE_PERIOD = re.compile(r'[a-z][.][a-z]', re.I) 24 | UC_ELSEWHERE = re.compile(r'[%s]*?[a-zA-Z]+[A-Z]+?' % PUNCT) 25 | CAPFIRST = re.compile(r"^[%s]*?([A-Za-z])" % PUNCT) 26 | SMALL_FIRST = re.compile(r'^([%s]*)(%s)\b' % (PUNCT, SMALL), re.I) 27 | SMALL_LAST = re.compile(r'\b(%s)[%s]?$' % (SMALL, PUNCT), re.I) 28 | SUBPHRASE = re.compile(r'([:.;?!\-–‒—―][ ])(%s)' % SMALL) 29 | APOS_SECOND = re.compile(r"^[dol]{1}['‘]{1}[a-z]+(?:['s]{2})?$", re.I) 30 | UC_INITIALS = re.compile(r"^(?:[A-Z]{1}\.{1}|[A-Z]{1}\.{1}[A-Z]{1})+$") 31 | MAC_MC = re.compile(r"^([Mm]c|MC)(\w.+)") 32 | 33 | 34 | class Immutable(object): 35 | pass 36 | 37 | 38 | text_type = unicode if sys.version_info < (3,) else str 39 | 40 | 41 | class ImmutableString(text_type, Immutable): 42 | pass 43 | 44 | 45 | class ImmutableBytes(bytes, Immutable): 46 | pass 47 | 48 | 49 | def _mark_immutable(text): 50 | if isinstance(text, bytes): 51 | return ImmutableBytes(text) 52 | return ImmutableString(text) 53 | 54 | 55 | def set_small_word_list(small=SMALL): 56 | global SMALL_WORDS 57 | global SMALL_FIRST 58 | global SMALL_LAST 59 | global SUBPHRASE 60 | SMALL_WORDS = re.compile(r'^(%s)$' % small, re.I) 61 | SMALL_FIRST = re.compile(r'^([%s]*)(%s)\b' % (PUNCT, small), re.I) 62 | SMALL_LAST = re.compile(r'\b(%s)[%s]?$' % (small, PUNCT), re.I) 63 | SUBPHRASE = re.compile(r'([:.;?!][ ])(%s)' % small) 64 | 65 | 66 | def titlecase(text, callback=None, small_first_last=True): 67 | """ 68 | Titlecases input text 69 | 70 | This filter changes all words to Title Caps, and attempts to be clever 71 | about *un*capitalizing SMALL words like a/an/the in the input. 72 | 73 | The list of "SMALL words" which are not capped comes from 74 | the New York Times Manual of Style, plus 'vs' and 'v'. 75 | 76 | """ 77 | 78 | lines = re.split('[\r\n]+', text) 79 | processed = [] 80 | for line in lines: 81 | all_caps = line.upper() == line 82 | words = re.split('[\t ]', line) 83 | tc_line = [] 84 | for word in words: 85 | if callback: 86 | new_word = callback(word, all_caps=all_caps) 87 | if new_word: 88 | # Address #22: If a callback has done something 89 | # specific, leave this string alone from now on 90 | tc_line.append(_mark_immutable(new_word)) 91 | continue 92 | 93 | if all_caps: 94 | if UC_INITIALS.match(word): 95 | tc_line.append(word) 96 | continue 97 | 98 | if APOS_SECOND.match(word): 99 | if len(word[0]) == 1 and word[0] not in 'aeiouAEIOU': 100 | word = word[0].lower() + word[1] + word[2].upper() + word[3:] 101 | else: 102 | word = word[0].upper() + word[1] + word[2].upper() + word[3:] 103 | tc_line.append(word) 104 | continue 105 | 106 | match = MAC_MC.match(word) 107 | if match: 108 | tc_line.append("%s%s" % (match.group(1).capitalize(), 109 | titlecase(match.group(2),callback,small_first_last))) 110 | continue 111 | 112 | if INLINE_PERIOD.search(word) or (not all_caps and UC_ELSEWHERE.match(word)): 113 | tc_line.append(word) 114 | continue 115 | if SMALL_WORDS.match(word): 116 | tc_line.append(word.lower()) 117 | continue 118 | 119 | if "/" in word and "//" not in word: 120 | slashed = map( 121 | lambda t: titlecase(t,callback,False), 122 | word.split('/') 123 | ) 124 | tc_line.append("/".join(slashed)) 125 | continue 126 | 127 | if '-' in word: 128 | hyphenated = map( 129 | lambda t: titlecase(t,callback,small_first_last), 130 | word.split('-') 131 | ) 132 | tc_line.append("-".join(hyphenated)) 133 | continue 134 | 135 | if all_caps: 136 | word = word.lower() 137 | 138 | # Just a normal word that needs to be capitalized 139 | tc_line.append(CAPFIRST.sub(lambda m: m.group(0).upper(), word)) 140 | 141 | if small_first_last and tc_line: 142 | if not isinstance(tc_line[0], Immutable): 143 | tc_line[0] = SMALL_FIRST.sub(lambda m: '%s%s' % ( 144 | m.group(1), 145 | m.group(2).capitalize() 146 | ), tc_line[0]) 147 | 148 | if not isinstance(tc_line[-1], Immutable): 149 | tc_line[-1] = SMALL_LAST.sub( 150 | lambda m: m.group(0).capitalize(), tc_line[-1] 151 | ) 152 | 153 | result = " ".join(tc_line) 154 | 155 | result = SUBPHRASE.sub(lambda m: '%s%s' % ( 156 | m.group(1), 157 | m.group(2).capitalize() 158 | ), result) 159 | 160 | processed.append(result) 161 | 162 | return "\n".join(processed) 163 | 164 | 165 | def cmd(): 166 | '''Handler for command line invocation''' 167 | 168 | # Try to handle any reasonable thing thrown at this. 169 | # Consume '-f' and '-o' as input/output, allow '-' for stdin/stdout 170 | # and treat any subsequent arguments as a space separated string to 171 | # be titlecased (so it still works if people forget quotes) 172 | parser = argparse.ArgumentParser() 173 | in_group = parser.add_mutually_exclusive_group() 174 | in_group.add_argument('string', nargs='*', default=[], 175 | help='String to titlecase') 176 | in_group.add_argument('-f', '--input-file', 177 | help='File to read from to titlecase') 178 | parser.add_argument('-o', '--output-file', 179 | help='File to write titlecased output to)') 180 | 181 | args = parser.parse_args() 182 | 183 | if args.input_file is not None: 184 | if args.input_file == '-': 185 | ifile = sys.stdin 186 | else: 187 | ifile = open(args.input_file) 188 | else: 189 | ifile = sys.stdin 190 | 191 | if args.output_file is not None: 192 | if args.output_file == '-': 193 | ofile = sys.stdout 194 | else: 195 | ofile = open(args.output_file, 'w') 196 | else: 197 | ofile = sys.stdout 198 | 199 | if len(args.string) > 0: 200 | in_string = ' '.join(args.string) 201 | else: 202 | with ifile: 203 | in_string = ifile.read() 204 | 205 | with ofile: 206 | ofile.write(titlecase(in_string)) -------------------------------------------------------------------------------- /base/utils/nbexamples-sagemaker/handlers.py: -------------------------------------------------------------------------------- 1 | """Tornado handlers for nbexamples web service.""" 2 | 3 | # pyhton lib 4 | import datetime 5 | from glob import glob 6 | import json 7 | import os 8 | import shutil 9 | import subprocess as sp 10 | from tornado import web 11 | from traitlets.config import LoggingConfigurable 12 | from nbexamples.external.titlecase import titlecase 13 | 14 | # Jupyter notebook lib 15 | import nbformat 16 | from notebook.utils import url_path_join as ujoin 17 | from notebook.base.handlers import IPythonHandler 18 | 19 | 20 | # See python-titlecase README.md. https://github.com/ppannuto/python-titlecase 21 | def abbreviations(word, **kwargs): 22 | if word.upper() in ('AWS', 'SDK', 'API', 'ML', 'AI'): 23 | return word.upper() 24 | 25 | 26 | class Examples(LoggingConfigurable): 27 | # Path to default customer directory (/home/ec2-user/SageMaker/) 28 | # TODO: Support better local testing for this parameter 29 | base_path = '/home/ec2-user/SageMaker/' 30 | # Path to sample notebook directory (/home/ec2-user/sample-notebooks/) 31 | sample_notebook_dir = '/home/ec2-user/sample-notebooks/' 32 | top_sample_notebook_categories = ['introduction_to_applying_machine_learning', 'introduction_to_amazon_algorithms'] 33 | 34 | def get_supporting_items(self, filepath, notebook_name): 35 | notebook_folder_location = os.path.split(filepath)[0] 36 | items = [] 37 | for root, directories, filenames in os.walk(notebook_folder_location): 38 | root = root.replace(notebook_folder_location, '') 39 | for filename in filenames: 40 | if filename != notebook_name: 41 | items.append(os.path.join(root, filename)) 42 | items.sort() 43 | return items 44 | 45 | def list_examples(self): 46 | all_examples = [] 47 | for category in self.get_categories(): 48 | directory = os.path.join(self.sample_notebook_dir, category) 49 | filepaths = glob(os.path.join(directory, '**', '*.ipynb'), recursive=True) 50 | examples = [{'filepath': os.path.abspath(fp)} for fp in filepaths] 51 | for example in examples: 52 | node = nbformat.read(example['filepath'], nbformat.NO_CONVERT) 53 | example['filename'] = os.path.basename(example['filepath']) 54 | example['metadata'] = node.metadata 55 | example['category'] = category 56 | example['basename'] = os.path.basename(example['filepath']) 57 | example['supporting_items'] = self.get_supporting_items(example['filepath'], example['filename']) 58 | notebook_folder_location = os.path.split(example['filepath'])[0] 59 | example['notebook_folder_name'] = os.path.split(notebook_folder_location)[1] 60 | all_examples.extend(examples) 61 | return all_examples 62 | 63 | def get_sanitized_custom_notebook_name(self, custom_notebook_name): 64 | custom_notebook_basename = os.path.basename(custom_notebook_name) 65 | custom_notebook_name_without_extension = custom_notebook_basename.split('.') 66 | sanitized_custom_notebook_name = custom_notebook_name_without_extension[0].strip() 67 | return sanitized_custom_notebook_name 68 | 69 | def get_customer_notebook_dir_name(self, sample_notebook_location): 70 | notebook_folder_location = os.path.split(sample_notebook_location)[0] 71 | notebook_folder_name = os.path.split(notebook_folder_location)[1] 72 | 73 | date = datetime.datetime.utcnow().strftime('%Y-%m-%d') 74 | destination_location_original = os.path.join(self.base_path, notebook_folder_name) + '_' + date 75 | 76 | # Get non existent version of notebook folder in user space 77 | copy = 1 78 | destination_location = destination_location_original 79 | while os.path.exists(destination_location): 80 | destination_location = destination_location_original + '_Copy' + str(copy) 81 | copy += 1 82 | 83 | return destination_location 84 | 85 | def fetch_example(self, sample_notebook_location, custom_notebook_name): 86 | notebook_folder_location = os.path.split(sample_notebook_location)[0] 87 | destination_location = self.get_customer_notebook_dir_name(sample_notebook_location) 88 | 89 | # Copy entire content inside notebook folder to customer location 90 | shutil.copytree(notebook_folder_location, destination_location, symlinks=False, ignore=None) 91 | 92 | # Rename notebook file with user provided name 93 | original_notebook_name = os.path.join(destination_location, os.path.split(sample_notebook_location)[1]) 94 | sanitized_custom_notebook_name = self.get_sanitized_custom_notebook_name(custom_notebook_name) 95 | if sanitized_custom_notebook_name: 96 | customer_provided_notebook_name = os.path.join(destination_location, sanitized_custom_notebook_name + '.ipynb') 97 | os.rename(original_notebook_name, customer_provided_notebook_name) 98 | else: 99 | customer_provided_notebook_name = original_notebook_name 100 | 101 | # Remove base path from customer_provided_notebook_name 102 | customer_provided_notebook_name = customer_provided_notebook_name.replace(self.base_path, '') 103 | return customer_provided_notebook_name 104 | 105 | def preview_example(self, filepath): 106 | fp = filepath # for brevity 107 | if not os.path.isfile(fp): 108 | raise web.HTTPError(404, "Example not found: %s" % fp) 109 | p = sp.Popen(['jupyter', 'nbconvert', '--to', 'html', '--stdout', fp], 110 | stdout=sp.PIPE, stderr=sp.PIPE) 111 | output, _ = p.communicate() 112 | retcode = p.poll() 113 | if retcode != 0: 114 | raise RuntimeError('nbconvert exited with code {}'.format(retcode)) 115 | return output.decode() 116 | 117 | def get_categories(self): 118 | return [category_name for category_name in os.listdir(self.sample_notebook_dir) 119 | if not category_name.startswith('.') and os.path.isdir(os.path.join(self.sample_notebook_dir, category_name))] 120 | 121 | def get_sanitized_categories(self): 122 | sanitized_categories = [] 123 | for category_name in self.get_categories(): 124 | sanitized_category = { 125 | 'title' : titlecase(category_name.replace('_', ' ').replace('-', ' '), callback=abbreviations), 126 | 'id' : category_name, 127 | 'name' : category_name 128 | } 129 | if category_name in self.top_sample_notebook_categories: 130 | sanitized_categories.insert(0, sanitized_category) 131 | else: 132 | sanitized_categories.append(sanitized_category) 133 | return sanitized_categories 134 | 135 | class BaseExampleHandler(IPythonHandler): 136 | 137 | @property 138 | def manager(self): 139 | return self.settings['example_manager'] 140 | 141 | 142 | class ExampleCategoryHandler(BaseExampleHandler): 143 | @web.authenticated 144 | def get(self): 145 | self.finish(json.dumps(self.manager.get_sanitized_categories())) 146 | 147 | 148 | class ExamplesHandler(BaseExampleHandler): 149 | @web.authenticated 150 | def get(self): 151 | self.finish(json.dumps(self.manager.list_examples())) 152 | 153 | 154 | class ExampleActionHandler(BaseExampleHandler): 155 | 156 | @web.authenticated 157 | def get(self, action): 158 | example_id = self.get_argument('example_id') 159 | if action == 'preview': 160 | self.finish(self.manager.preview_example(example_id)) 161 | elif action == 'fetch': 162 | dest = self.get_argument('dest') 163 | dest = self.manager.fetch_example(example_id, dest) 164 | self.redirect(ujoin(self.base_url, 'notebooks', dest)) 165 | elif action == 'fetchfile': 166 | dest = self.get_argument('dest') 167 | dest = self.manager.fetch_example(example_id, dest) 168 | self.finish(dest) 169 | 170 | 171 | 172 | # ----------------------------------------------------------------------------- 173 | # URL to handler mappings 174 | # ----------------------------------------------------------------------------- 175 | 176 | 177 | _example_action_regex = r"(?Pfetch|fetchfile|preview)" 178 | 179 | default_handlers = [ 180 | (r"/categories", ExampleCategoryHandler), 181 | (r"/examples", ExamplesHandler), 182 | (r"/examples/%s" % _example_action_regex, ExampleActionHandler), 183 | ] 184 | 185 | 186 | def load_jupyter_server_extension(nbapp): 187 | """Load the nbserver""" 188 | webapp = nbapp.web_app 189 | webapp.settings['example_manager'] = Examples(parent=nbapp) 190 | base_url = webapp.settings['base_url'] 191 | 192 | ExampleActionHandler.base_url = base_url # used to redirect after fetch 193 | webapp.add_handlers(".*$", [ 194 | (ujoin(base_url, pat), handler) 195 | for pat, handler in default_handlers 196 | ]) 197 | -------------------------------------------------------------------------------- /base/utils/nbexamples-sagemaker/static/examples.css: -------------------------------------------------------------------------------- 1 | #examples .panel-group .panel { 2 | margin-top: 3px; 3 | margin-bottom: 1em; 4 | } 5 | 6 | #examples .panel-group .panel .panel-heading { 7 | background-color: #eee; 8 | padding-top: 4px; 9 | padding-bottom: 4px; 10 | padding-left: 7px; 11 | padding-right: 7px; 12 | line-height: 22px; 13 | } 14 | 15 | #examples .panel-group .panel .panel-heading a:focus, a:hover { 16 | text-decoration: none; 17 | } 18 | 19 | #examples .panel-group .panel .panel-body { 20 | padding: 0; 21 | } 22 | 23 | #examples .panel-group .panel .panel-body .list_container { 24 | margin-top: 0px; 25 | margin-bottom: 0px; 26 | border: 0px; 27 | border-radius: 0px; 28 | } 29 | 30 | #examples .panel-group .panel .panel-body .list_container .list_item { 31 | border-bottom: 1px solid #ddd; 32 | } 33 | 34 | #examples .panel-group .panel .panel-body .list_container .list_item:last-child { 35 | border-bottom: 0px; 36 | } 37 | 38 | #examples .example-notebooks .list_item { 39 | background-color: inherit !important; 40 | } 41 | 42 | #examples .example-notebooks .list_item:hover { 43 | background-color: #ddd !important; 44 | } 45 | 46 | #examples .example-notebooks .list_item:first-child:hover { 47 | background-color: inherit !important; 48 | } 49 | 50 | #examples .list_item { 51 | padding-top: 4px; 52 | padding-bottom: 4px; 53 | padding-left: 7px; 54 | padding-right: 7px; 55 | line-height: 22px; 56 | } 57 | 58 | #examples .list_item > div { 59 | padding-top: 0; 60 | padding-bottom: 0; 61 | padding-left: 0; 62 | padding-right: 0; 63 | } 64 | 65 | #examples .item_status { 66 | text-align: right; 67 | } 68 | 69 | #examples .item_summary { 70 | color: #bbb; 71 | } 72 | 73 | #examples .item_attribution { 74 | color: #bbb; 75 | margin-left: 10px; 76 | } 77 | 78 | #examples .item_status .btn { 79 | min-width: 13ex; 80 | } 81 | 82 | .modal-dialog .modal-body .control-label { 83 | padding-right: 1em; 84 | } 85 | 86 | #validation-message p { 87 | margin-bottom: 1em; 88 | padding-top: 1em; 89 | } 90 | 91 | #validation-message pre { 92 | margin-left: 1em; 93 | margin-right: 1em; 94 | } 95 | 96 | .right-indent { 97 | padding-right: 1em; 98 | } 99 | 100 | .panel-heading.collapsed .fa-chevron-down, 101 | .panel-heading .fa-chevron-right { 102 | display: none; 103 | } 104 | 105 | .panel-heading.collapsed .fa-chevron-right, 106 | .panel-heading .fa-chevron-down { 107 | display: inline-block; 108 | } 109 | 110 | i.fa { 111 | cursor: pointer; 112 | } 113 | 114 | .collapsed ~ .panel-body { 115 | padding: 0; 116 | } 117 | 118 | .btn-info.aws-custom { 119 | background-color: #fff; 120 | border-color: #545b64; 121 | color: #545b64; 122 | margin-right: 10px; 123 | } 124 | 125 | .btn-info.aws-custom:hover { 126 | background-color: #fafafa; 127 | } 128 | 129 | .btn-success.aws-custom, .btn-primary.aws-custom { 130 | background-color: #ec7211; 131 | border-color: #ec7211; 132 | } 133 | 134 | .btn-success.aws-custom:hover, .btn-primary.aws-custom:hover { 135 | background-color: #eb5f07; 136 | border-color: #eb5f07; 137 | } -------------------------------------------------------------------------------- /base/utils/nbexamples-sagemaker/static/examples.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) Jupyter Development Team. 2 | // Distributed under the terms of the Modified BSD License. 3 | 4 | define([ 5 | 'base/js/namespace', 6 | 'jquery', 7 | 'underscore', 8 | 'base/js/utils', 9 | 'base/js/dialog', 10 | ], function(Jupyter, $, _, utils, dialog) { 11 | "use strict"; 12 | 13 | var Examples = function ( 14 | categories, 15 | options) { 16 | 17 | options = options || {}; 18 | this.options = options; 19 | this.base_url = options.base_url || utils.get_body_data("baseUrl"); 20 | 21 | var category_elements = {}; 22 | categories.forEach(function(category){ 23 | category_elements[category['name']] = $('#' + category['id']) 24 | }); 25 | this.category_elements = category_elements; 26 | 27 | this.dialog_element = this.make_save_as_dialog().appendTo('body'); 28 | this.bind_events(); 29 | }; 30 | 31 | Examples.prototype.bind_events = function () { 32 | var that = this; 33 | $('#refresh_examples_list').click(function () { 34 | that.load_list(); 35 | }); 36 | 37 | // Hide the modal dialog on submit. The declarative attribute does 38 | // not work when form submission is involved. 39 | this.dialog_element.on('submit', '.modal-dialog form', function(evt) { 40 | $(evt.target).closest('.modal').modal('hide'); 41 | }); 42 | 43 | // Show the singleton dialog when the user clicks the use button for any 44 | // example. Set the example ID in the hidden element field. 45 | for (var key in this.category_elements) { 46 | this.category_elements[key].on('click', '[data-filepath]', function(evt) { 47 | var filepath = $(evt.target).data('filepath'); 48 | var basename = $(evt.target).data('basename'); 49 | var notebook_folder_name = $(evt.target).data('notebook_folder_name'); 50 | var supporting_items = _.filter($(evt.target).data('supporting_items').split(',')); 51 | 52 | that.dialog_element 53 | .find('[name="example_id"]') 54 | .val(filepath); 55 | that.dialog_element 56 | .find('[name="dest"]') 57 | .val(basename); 58 | that.show_supporting_items_list(notebook_folder_name, supporting_items); 59 | that.dialog_element.modal('show'); 60 | }); 61 | } 62 | }; 63 | 64 | Examples.prototype.load_list = function () { 65 | var settings = { 66 | processData : false, 67 | cache : false, 68 | type : "GET", 69 | dataType : "json", 70 | success : $.proxy(this.load_list_success, this), 71 | error : utils.log_ajax_error, 72 | }; 73 | var url = utils.url_join_encode(this.base_url, 'examples'); 74 | $.ajax(url, settings); 75 | }; 76 | 77 | Examples.prototype.clear_list = function () { 78 | // remove list items and show placeholders 79 | for (var key in this.category_elements) { 80 | this.category_elements[key].children('.list_item').remove(); 81 | this.category_elements[key].children('.list_placeholder').show(); 82 | } 83 | }; 84 | 85 | Examples.prototype.load_list_success = function (examples, status, xhr) { 86 | this.clear_list(); 87 | examples = _.sortBy(examples, function(example) { 88 | return example.metadata.title || example.basename; 89 | }); 90 | for (var i = 0; i < examples.length; i++) { 91 | var element = $('
'); 92 | new Example(element, 93 | examples[i], 94 | this.options 95 | ); 96 | 97 | this.category_elements[examples[i]['category']].append(element); 98 | this.category_elements[examples[i]['category']].children('.list_placeholder').hide() 99 | } 100 | }; 101 | 102 | Examples.prototype.make_save_as_dialog = function () { 103 | var modal_header = $('
') 104 | .addClass('modal-header') 105 | .append( 106 | $('

') 107 | .attr({id: 'nbexamples-modal-label'}) 108 | .addClass('modal-title') 109 | .text('Create a copy in your home directory') 110 | ); 111 | var modal_footer = $('
') 112 | .addClass('modal-footer') 113 | .append( 114 | $('