├── Dockerfile ├── Dockerfile.alpine ├── README.md ├── logging.conf ├── luigi.conf ├── luigi.jpg ├── luigid.sh └── tests ├── docker-compose.yml ├── maria.conf ├── pgsql.conf └── sqlite.conf /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM phusion/baseimage:0.11 2 | CMD ["/sbin/my_init", "--quiet"] 3 | 4 | LABEL maintainer="Kyle Wilcox " 5 | 6 | ARG DEBIAN_FRONTEND="noninteractive" 7 | 8 | ARG PYTHON_VERSION="3.7" 9 | 10 | ARG CONDA_VERSION="4.8.2" 11 | ARG CONDA_PY_VERSION="37" 12 | ARG CONDA_MD5="87e77f097f6ebb5127c77662dfc3165e" 13 | ARG CONDA_DIR="/opt/conda" 14 | 15 | ARG LUIGI_VERSION="2.8.13" 16 | ARG LUIGI_CONFIG_DIR="/etc/luigi/" 17 | ARG LUIGI_CONFIG_PATH="/etc/luigi/luigi.conf" 18 | ARG LUIGI_STATE_DIR="/luigi/state" 19 | 20 | ENV PATH="$CONDA_DIR/bin:$PATH" 21 | ENV LANG="C.UTF-8" 22 | ENV LUIGI_VERSION="${LUIGI_VERSION}" 23 | 24 | RUN echo "**** install binary packages ****" && \ 25 | install_clean \ 26 | bash \ 27 | binutils \ 28 | build-essential \ 29 | bzip2 \ 30 | ca-certificates \ 31 | libglib2.0-0 \ 32 | libsm6 \ 33 | libxext6 \ 34 | libxrender1 \ 35 | wget \ 36 | && \ 37 | \ 38 | echo "**** install miniconda ****" && \ 39 | mkdir -p "$CONDA_DIR" && \ 40 | wget "https://repo.anaconda.com/miniconda/Miniconda3-py${CONDA_PY_VERSION}_${CONDA_VERSION}-Linux-x86_64.sh" -O miniconda.sh && \ 41 | echo "$CONDA_MD5 miniconda.sh" | md5sum -c && \ 42 | \ 43 | bash miniconda.sh -f -b -p "$CONDA_DIR" && \ 44 | echo "export PATH=$CONDA_DIR/bin:\$PATH" > /etc/profile.d/conda.sh && \ 45 | \ 46 | conda update --all --yes && \ 47 | conda config \ 48 | --set auto_update_conda False \ 49 | --set always_yes yes \ 50 | --set changeps1 no \ 51 | --set show_channel_urls True \ 52 | && \ 53 | conda config \ 54 | --add channels conda-forge \ 55 | && \ 56 | \ 57 | echo "**** install python packages ****" && \ 58 | conda install --yes --freeze-installed \ 59 | python=="${PYTHON_VERSION}*" \ 60 | luigi=="${LUIGI_VERSION}" \ 61 | sqlalchemy \ 62 | psycopg2 \ 63 | mysql-connector-python \ 64 | mysqlclient \ 65 | prometheus_client \ 66 | && \ 67 | \ 68 | echo "**** cleanup ****" && \ 69 | rm -rf /root/.cache /tmp/* /var/tmp/* && \ 70 | rm -f miniconda.sh && \ 71 | conda clean --all --force-pkgs-dirs --yes && \ 72 | find "$CONDA_DIR" -follow -type f \( -iname '*.a' -o -iname '*.pyc' -o -iname '*.js.map' \) -delete && \ 73 | \ 74 | echo "**** finalize ****" && \ 75 | mkdir -p "${LUIGI_CONFIG_DIR}" && \ 76 | mkdir -p "${LUIGI_STATE_DIR}" 77 | 78 | COPY logging.conf "${LUIGI_CONFIG_DIR}" 79 | COPY luigi.conf "${LUIGI_CONFIG_DIR}" 80 | VOLUME ["${LUIGI_CONFIG_DIR}", "${LUIGI_STATE_DIR}"] 81 | 82 | EXPOSE 8082/TCP 83 | 84 | RUN mkdir /etc/service/luigid 85 | COPY luigid.sh /etc/service/luigid/run 86 | -------------------------------------------------------------------------------- /Dockerfile.alpine: -------------------------------------------------------------------------------- 1 | ARG PYTHON_VERSION="3.7" 2 | FROM python:$PYTHON_VERSION-alpine3.10 3 | 4 | LABEL maintainer="Kyle Wilcox " 5 | 6 | ARG LUIGI_VERSION="2.8.13" 7 | ARG LUIGI_CONFIG_DIR="/etc/luigi/" 8 | ARG LUIGI_CONFIG_PATH="/etc/luigi/luigi.conf" 9 | ARG LUIGI_STATE_DIR="/luigi/state" 10 | 11 | ENV LUIGI_VERSION="${LUIGI_VERSION}" 12 | 13 | RUN echo "**** install binary packages ****" && \ 14 | apk add --no-cache --virtual .build-deps \ 15 | build-base \ 16 | gcc \ 17 | musl-dev \ 18 | libc-dev \ 19 | libffi-dev \ 20 | python3-dev \ 21 | py-mysqldb \ 22 | postgresql-dev \ 23 | mariadb-dev \ 24 | mariadb-connector-c-dev \ 25 | && \ 26 | \ 27 | echo "**** install python packages ****" && \ 28 | python3 -m pip install \ 29 | luigi=="${LUIGI_VERSION}" \ 30 | sqlalchemy \ 31 | psycopg2 \ 32 | mysql-connector-python \ 33 | mysqlclient \ 34 | prometheus_client \ 35 | && \ 36 | apk add --virtual \ 37 | mariadb-client-libs \ 38 | mariadb-connector-c \ 39 | postgresql-libs \ 40 | && \ 41 | \ 42 | echo "**** cleanup ****" && \ 43 | apk --purge del .build-deps && \ 44 | \ 45 | echo "**** finalize ****" && \ 46 | mkdir -p "${LUIGI_CONFIG_DIR}" && \ 47 | mkdir -p "${LUIGI_STATE_DIR}" 48 | 49 | COPY logging.conf "${LUIGI_CONFIG_DIR}" 50 | COPY luigi.conf "${LUIGI_CONFIG_DIR}" 51 | VOLUME ["${LUIGI_CONFIG_DIR}", "${LUIGI_STATE_DIR}"] 52 | 53 | EXPOSE 8082/TCP 54 | 55 | COPY luigid.sh /bin/run 56 | ENTRYPOINT ["/bin/run"] 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Luigi on Docker 2 | 3 | In addition to supporting the `sqlite` task history backend, this container includes all of the dependencies to use the `mysql` and `postgresql` backends. 4 | 5 | ![Luigi death stare](luigi.jpg) 6 | 7 | ## Versions 8 | 9 | [All available images](https://hub.docker.com/r/axiom/docker-luigi/tags) 10 | 11 | ### alpine 12 | 13 | Uses `pip` to install `luigi` and dependencies 14 | 15 | * `axiom/docker-luigi:latest-alpine` (2.8.13) 16 | * `axiom/docker-luigi:{version}-alpine` 17 | 18 | ### baseimage (ubuntu xenial) 19 | 20 | Uses `conda` to install `luigi` and dependencies 21 | 22 | * `axiom/docker-luigi:latest` (2.8.13) 23 | * `axiom/docker-luigi:{version}` 24 | 25 | ## Exposed Volumes 26 | 27 | #### `/etc/luigi/` 28 | 29 | Both `luigi` and `luigid` load their configuration from `/etc/luigi/`. 30 | 31 | * Configuration: `/etc/luigi/luigi.conf` 32 | * Logging: `/etc/luigi/logging.conf` 33 | 34 | Mount a directory containing a `luigi.conf` and `logging.conf` file(s) to 35 | `/etc/luigi` to provide your own configuration(s). 36 | 37 | ``` 38 | docker run \ 39 | -v /your/directory:/etc/luigi \ 40 | axiom/docker-luigi 41 | ``` 42 | 43 | Or mount a single configuration file: 44 | 45 | ``` 46 | docker run \ 47 | -v /your/directory/logging.conf:/etc/luigi/logging.conf \ 48 | axiom/docker-luigi 49 | ``` 50 | 51 | The default can be found in the `luigi.conf` and `logging.conf` files in this 52 | repository. Be aware that these specify the paths to the logging configration 53 | and the state persistence database. If you change these values in `luigi.conf` 54 | the examples in this document will not work! 55 | 56 | 57 | #### `/luigi/state` 58 | 59 | Mount a volume at `/luigi/state` for the `luigid` **state to be persisted** 60 | between restarts. Example below uses a named docker volume to persist the state: 61 | 62 | ``` 63 | docker run \ 64 | -v luigistate:/luigi/state \ 65 | axiom/docker-luigi 66 | ``` 67 | 68 | 69 | ### Testing 70 | 71 | ```bash 72 | docker-compose -f tests/docker-compose.yml up -d maria pgsql 73 | # Wait ~10s for databases to start 74 | docker-compose -f tests/docker-compose.yml build 75 | docker-compose -f tests/docker-compose.yml up 76 | docker-compose down 77 | ``` 78 | -------------------------------------------------------------------------------- /logging.conf: -------------------------------------------------------------------------------- 1 | [loggers] 2 | keys=root,tornado,client,scheduler,server 3 | 4 | [logger_root] 5 | level=DEBUG 6 | handlers=console 7 | 8 | [logger_client] 9 | level=DEBUG 10 | handlers=console 11 | qualname=luigi-interface 12 | propagate=0 13 | 14 | [logger_server] 15 | level=DEBUG 16 | handlers=console 17 | qualname=luigi.server 18 | propagate=0 19 | 20 | [logger_scheduler] 21 | level=DEBUG 22 | handlers=console 23 | qualname=luigi.scheduler 24 | propagate=0 25 | 26 | [logger_tornado] 27 | level=DEBUG 28 | handlers=warnconsole 29 | qualname=tornado 30 | propagate=0 31 | 32 | [formatters] 33 | keys=detail 34 | 35 | [formatter_detail] 36 | class=logging.Formatter 37 | format=%(asctime)s %(name)-15s %(levelname)-8s %(message)s 38 | 39 | [handlers] 40 | keys=console,warnconsole 41 | 42 | [handler_console] 43 | level=INFO 44 | class=StreamHandler 45 | args=(sys.stdout,) 46 | formatter=detail 47 | 48 | [handler_warnconsole] 49 | level=WARNING 50 | class=StreamHandler 51 | args=(sys.stdout,) 52 | formatter=detail 53 | -------------------------------------------------------------------------------- /luigi.conf: -------------------------------------------------------------------------------- 1 | [core] 2 | logging_conf_file: /etc/luigi/logging.conf 3 | 4 | [scheduler] 5 | record_task_history: True 6 | state_path: /luigi/state/luigi-state.pickle 7 | 8 | [task_history] 9 | db_connection: sqlite:////luigi/state/luigi-task-history.db 10 | -------------------------------------------------------------------------------- /luigi.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/axiom-data-science/docker-luigi/11fd19adac3cc56a61251fefe297310b68f441a4/luigi.jpg -------------------------------------------------------------------------------- /luigid.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | cat << "EOF" 3 | _____ __ __ _____ _____ _____ 4 | (_ _) ) ) ( ( (_ _) / ___ \ (_ _) 5 | | | ( ( ) ) | | / / \_) | | 6 | | | ) ) ( ( | | ( ( ____ | | 7 | | | __ ( ( ) ) | | ( ( (__ ) | | 8 | __| |___) ) ) \__/ ( _| |__ \ \__/ / _| |__ 9 | \________/ \______/ /_____( \____/ /_____( 10 | EOF 11 | echo "Luigi: $(python -c 'import luigi; print(luigi.__meta__.__version__)')" 12 | echo "$(python -VV)" 13 | 14 | exec luigid 15 | -------------------------------------------------------------------------------- /tests/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | 3 | services: 4 | 5 | maria: 6 | image: mariadb/server:latest 7 | environment: 8 | MYSQL_USER: luigid 9 | MYSQL_PASSWORD: luigid 10 | MYSQL_DATABASE: luigid 11 | MYSQL_RANDOM_ROOT_PASSWORD: "yes" 12 | 13 | pgsql: 14 | image: postgres:latest 15 | environment: 16 | POSTGRES_USER: luigid 17 | POSTGRES_PASS: luigid 18 | POSTGRES_DB: luigid 19 | 20 | luigi-alpine-sqlite: 21 | build: 22 | context: ../ 23 | dockerfile: Dockerfile.alpine 24 | volumes: 25 | - "./sqlite.conf:/etc/luigi/luigi.conf:ro" 26 | environment: 27 | LUIGI_CONFIG_PARSER: conf 28 | LUIGI_CONFIG_PATH: /etc/luigi/luigi.conf 29 | 30 | luigi-alpine-maria: 31 | build: 32 | context: ../ 33 | dockerfile: Dockerfile.alpine 34 | volumes: 35 | - "./maria.conf:/etc/luigi/luigi.conf:ro" 36 | environment: 37 | LUIGI_CONFIG_PARSER: conf 38 | LUIGI_CONFIG_PATH: /etc/luigi/luigi.conf 39 | depends_on: 40 | - maria 41 | 42 | luigi-alpine-pgsql: 43 | build: 44 | context: ../ 45 | dockerfile: Dockerfile.alpine 46 | volumes: 47 | - "./pgsql.conf:/etc/luigi/luigi.conf:ro" 48 | environment: 49 | LUIGI_CONFIG_PARSER: conf 50 | LUIGI_CONFIG_PATH: /etc/luigi/luigi.conf 51 | depends_on: 52 | - pgsql 53 | 54 | luigi-sqlite: 55 | build: 56 | context: ../ 57 | volumes: 58 | - "./sqlite.conf:/etc/luigi/luigi.conf:ro" 59 | environment: 60 | LUIGI_CONFIG_PARSER: conf 61 | LUIGI_CONFIG_PATH: /etc/luigi/luigi.conf 62 | 63 | luigi-maria: 64 | build: 65 | context: ../ 66 | volumes: 67 | - "./maria.conf:/etc/luigi/luigi.conf:ro" 68 | environment: 69 | LUIGI_CONFIG_PARSER: conf 70 | LUIGI_CONFIG_PATH: /etc/luigi/luigi.conf 71 | depends_on: 72 | - maria 73 | 74 | luigi-pgsql: 75 | build: 76 | context: ../ 77 | volumes: 78 | - "./pgsql.conf:/etc/luigi/luigi.conf:ro" 79 | environment: 80 | LUIGI_CONFIG_PARSER: conf 81 | LUIGI_CONFIG_PATH: /etc/luigi/luigi.conf 82 | depends_on: 83 | - pgsql 84 | -------------------------------------------------------------------------------- /tests/maria.conf: -------------------------------------------------------------------------------- 1 | [core] 2 | logging_conf_file: /etc/luigi/logging.conf 3 | 4 | [scheduler] 5 | record_task_history: True 6 | state_path: /luigi/state/luigi-state.pickle 7 | metrics_collector: prometheus 8 | 9 | [task_history] 10 | db_connection: mysql://luigid:luigid@maria:3306/luigid 11 | -------------------------------------------------------------------------------- /tests/pgsql.conf: -------------------------------------------------------------------------------- 1 | [core] 2 | logging_conf_file: /etc/luigi/logging.conf 3 | 4 | [scheduler] 5 | record_task_history: True 6 | state_path: /luigi/state/luigi-state.pickle 7 | metrics_collector: prometheus 8 | 9 | [task_history] 10 | db_connection: postgresql://luigid:luigid@pgsql:5432/luigid 11 | -------------------------------------------------------------------------------- /tests/sqlite.conf: -------------------------------------------------------------------------------- 1 | [core] 2 | logging_conf_file: /etc/luigi/logging.conf 3 | 4 | [scheduler] 5 | record_task_history: True 6 | state_path: /luigi/state/luigi-state.pickle 7 | metrics_collector: prometheus 8 | 9 | [task_history] 10 | db_connection: sqlite:////luigi/state/luigi-task-history.db 11 | --------------------------------------------------------------------------------