├── .deps ├── .docker ├── entrypoint.sh └── env.sh ├── .dockerignore ├── .editorconfig ├── .github └── pull_request_template.md ├── .gitignore ├── .gitlab-ci.yml ├── .gitlab-ci ├── build-test-push ├── signed-off-by-check └── test-image ├── CHANGES.rst ├── CONTRIBUTING.rst ├── Dockerfile ├── LICENSE.txt ├── LICENSES ├── AGPL-3.0 ├── Apache-2.0 └── MARV-License ├── README.rst ├── code ├── marv-api │ ├── LICENSE.txt │ ├── MANIFEST.in │ ├── README.rst │ ├── marv_api │ │ ├── __init__.py │ │ ├── dag.py │ │ ├── decorators.py │ │ ├── deprecation.py │ │ ├── ioctrl.py │ │ ├── iomsgs.py │ │ ├── scanner.py │ │ ├── setid.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_dag.py │ │ │ ├── test_decorators.py │ │ │ ├── test_setid.py │ │ │ ├── test_utils.py │ │ │ └── types.capnp │ │ ├── types │ │ │ └── __init__.py │ │ └── utils.py │ ├── marv_detail │ │ ├── __init__.py │ │ └── types.capnp │ ├── marv_nodes │ │ ├── __init__.py │ │ └── types.capnp │ ├── marv_pycapnp │ │ ├── __init__.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── pythonic.capnp │ │ │ ├── test.py │ │ │ ├── test_wrapper.capnp │ │ │ └── test_wrapper.py │ │ └── types.capnp │ ├── requirements.in │ ├── requirements.txt │ └── setup.py ├── marv-cli │ ├── .gitignore │ ├── LICENSE.txt │ ├── MANIFEST.in │ ├── README.rst │ ├── marv_cli │ │ ├── __init__.py │ │ └── __main__.py │ ├── requirements.in │ ├── requirements.txt │ └── setup.py ├── marv-robotics │ ├── .gitignore │ ├── LICENSE.txt │ ├── MANIFEST.in │ ├── README.rst │ ├── marv_robotics │ │ ├── __init__.py │ │ ├── bag.capnp │ │ ├── bag.py │ │ ├── cam.py │ │ ├── detail.py │ │ ├── fulltext.py │ │ ├── gnss.py │ │ ├── matplotlibrc │ │ ├── motion.py │ │ ├── tests │ │ │ ├── __init__.py │ │ │ ├── test_bag_scan.py │ │ │ └── test_bag_utils.py │ │ └── trajectory.py │ ├── marv_ros │ │ ├── __init__.py │ │ ├── img_tools.py │ │ └── tests │ │ │ └── test_img_tools.py │ ├── requirements.in │ ├── requirements.txt │ └── setup.py └── marv │ ├── .gitignore │ ├── LICENSE.txt │ ├── MANIFEST.in │ ├── README.rst │ ├── marv │ ├── __init__.py │ ├── app │ │ └── __init__.py │ ├── cli.py │ ├── collection.py │ ├── config.py │ ├── db.py │ ├── helpers.py │ ├── model.py │ ├── model_fields.py │ ├── sexp.py │ ├── site.py │ ├── testing.py │ ├── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── data │ │ │ ├── empty_dump.json │ │ │ └── full_dump.json │ │ ├── test_comment_tag.py │ │ ├── test_config.py │ │ ├── test_dataset.py │ │ ├── test_docs.py │ │ ├── test_dump_restore.py │ │ ├── test_listing.py │ │ ├── test_metadata.py │ │ ├── test_persist_without_type.py │ │ ├── test_sexp.py │ │ ├── test_site.py │ │ └── test_users.py │ └── utils.py │ ├── marv_node │ ├── __init__.py │ ├── driver.py │ ├── event.py │ ├── io.py │ ├── mixins.py │ ├── node.py │ ├── run.py │ ├── stream.py │ ├── testing │ │ ├── __init__.py │ │ └── _robotics_tests │ │ │ ├── __init__.py │ │ │ ├── output │ │ │ ├── bagmeta.json │ │ │ ├── bagmeta_table.json │ │ │ ├── connections_section.json │ │ │ ├── fulltext.json │ │ │ ├── gnss_section.json │ │ │ ├── images_section.json │ │ │ ├── summary_keyval.json │ │ │ ├── trajectory_section.json │ │ │ └── video_section.json │ │ │ ├── test_bag.py │ │ │ ├── test_fulltext.py │ │ │ ├── test_gnss_section.py │ │ │ ├── test_group_and_topic.py │ │ │ ├── test_non_existing.py │ │ │ ├── test_optional_input.py │ │ │ ├── test_section_images.py │ │ │ ├── test_section_topics.py │ │ │ ├── test_section_videos.py │ │ │ ├── test_trajectory_section.py │ │ │ ├── test_widget_bagmeta_table.py │ │ │ └── test_widget_summary_keyval.py │ └── tests │ │ ├── __init__.py │ │ ├── test_edge_cases.py │ │ ├── test_node.py │ │ ├── test_optional_input.py │ │ ├── test_push_false_values.py │ │ ├── test_run.py │ │ ├── test_run_combined.py │ │ ├── test_run_create_stream.py │ │ ├── test_run_foreach.py │ │ ├── test_run_foreach_with_header.py │ │ ├── test_run_ondemand_group.py │ │ ├── test_run_ondemand_group_with_restart.py │ │ ├── test_run_ondemand_group_with_restart_and_foreach.py │ │ ├── test_run_one_consumer.py │ │ ├── test_run_one_source.py │ │ ├── test_run_plain_is_error.py │ │ ├── test_run_request_input.py │ │ ├── test_run_substream_subscription.py │ │ └── test_run_two_consumers.py │ ├── marv_store │ ├── __init__.py │ └── streams.py │ ├── marv_webapi │ ├── __init__.py │ ├── api.py │ ├── auth.py │ ├── collection.py │ ├── comment.py │ ├── dataset.py │ ├── delete.py │ ├── rpcs.py │ ├── tag.py │ ├── tests │ │ ├── __init__.py │ │ ├── conftest.py │ │ ├── data │ │ │ ├── empty_listings.json │ │ │ ├── full_details.json │ │ │ └── full_listings.json │ │ ├── test_auth.py │ │ ├── test_collection.py │ │ ├── test_comment.py │ │ ├── test_dataset.py │ │ ├── test_delete.py │ │ ├── test_responses.py │ │ ├── test_rpc_query.py │ │ ├── test_static.py │ │ ├── test_tag.py │ │ └── test_tooling.py │ └── tooling.py │ ├── requirements.in │ ├── requirements.txt │ └── setup.py ├── docs ├── .gitignore ├── CHANGES.rst ├── CONTRIBUTING.rst ├── Makefile ├── _static │ └── .keep ├── accesscontrol.rst ├── api │ └── marv.rst ├── conf.py ├── config.rst ├── config │ ├── custom.js │ ├── etc_marv_marv.conf │ ├── marv.conf │ └── marv_multiple.conf ├── debug.rst ├── deploy.rst ├── favicon-32x32.png ├── httpapi.rst ├── index.rst ├── install │ ├── docker.rst │ ├── index.rst │ └── native.rst ├── maintenance.rst ├── migrate │ ├── 1711-1802-marv.conf.diff │ └── index.rst ├── nodes.rst ├── patterns.rst ├── scan.rst ├── tutorial ├── upload.rst ├── views.rst └── widgets.rst ├── requirements ├── develop.in ├── develop.txt ├── marv-api.in ├── marv-api.txt ├── marv-cli.in ├── marv-cli.txt ├── marv-robotics.in ├── marv-robotics.txt ├── marv.in ├── marv.txt ├── venv.in └── venv.txt ├── scripts ├── build-docs ├── build-image ├── download-test-bags ├── enter-container ├── fetch-deps ├── run-container └── setup-venv ├── setup.cfg ├── sites └── example │ └── marv.conf └── tutorial ├── .gitignore ├── code ├── .gitignore ├── LICENSE.txt ├── marv_tutorial │ └── __init__.py ├── requirements.in └── setup.py ├── docs-only-site ├── marv.conf ├── scanroot │ └── .keep └── sessionkey ├── setup-basic-site.rst ├── setup-basic-site0 ├── marv.conf └── sessionkey ├── setup-basic-site1 ├── marv.conf └── sessionkey ├── write-your-own.rst ├── write-your-own0 ├── marv.conf └── sessionkey ├── write-your-own1 ├── marv.conf └── sessionkey ├── write-your-own2 ├── marv.conf └── sessionkey └── write-your-own3 ├── marv.conf └── sessionkey /.deps: -------------------------------------------------------------------------------- 1 | code/marv/marv/app/static@2@https://gitlab.com/ternaris/marv-frontend-builds/-/archive/sha-1df0481306c148f93a79e1c95191f8408c5f83d01fe9cab0d7bea38005933578/marv-frontend-builds-sha-1df0481306c148f93a79e1c95191f8408c5f83d01fe9cab0d7bea38005933578.tar.gz 2 | -------------------------------------------------------------------------------- /.docker/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2016 - 2020 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | source /etc/profile.d/marv_env.sh 7 | 8 | set -e 9 | 10 | if [[ -n "$DEBUG" ]]; then 11 | set -x 12 | fi 13 | 14 | echo "$TIMEZONE" > /etc/timezone 15 | ln -sf /usr/share/zoneinfo/"$TIMEZONE" /etc/localtime 16 | dpkg-reconfigure -f noninteractive tzdata 17 | 18 | groupadd -g $MARV_GID marv || true 19 | useradd -M -u $MARV_UID -g $MARV_GID --shell /bin/bash marv 20 | chown $MARV_UID:$MARV_GID /home/marv 21 | if [[ $MARV_UID -ne 1000 ]] || [[ $MARV_GID -ne 1000 ]]; then 22 | chown -R $MARV_UID:$MARV_GID $MARV_VENV 23 | fi 24 | 25 | for x in /etc/skel/.*; do 26 | target="/home/marv/$(basename "$x")" 27 | if [[ ! -e "$target" ]]; then 28 | cp -a "$x" "$target" 29 | chown -R $MARV_UID:$MARV_GID "$target" 30 | fi 31 | done 32 | 33 | if [[ -n "$DEVELOP" ]]; then 34 | find "$DEVELOP" -maxdepth 2 -name setup.py \ 35 | -execdir su -c "$MARV_VENV/bin/pip install -e ." marv \; 36 | fi 37 | 38 | export HOME=/home/marv 39 | cd $MARV_SITE 40 | 41 | if [[ -d code ]]; then 42 | find code -maxdepth 2 -name setup.py -execdir su -c "$MARV_VENV/bin/pip install -e ." marv \; 43 | fi 44 | 45 | if [[ -n "$MARV_INIT" ]] || [[ ! -e db ]]; then 46 | su marv -p -c '/opt/marv/bin/marv --config "$MARV_CONFIG" init' 47 | fi 48 | su marv -p -c '/opt/marv/bin/marv --config "${MARV_CONFIG}" serve --host 0.0.0.0 --approot "${MARV_APPLICATION_ROOT:-/}" ${MARV_ARGS}' & 49 | 50 | echo 'Container startup complete.' 51 | exec "$@" 52 | -------------------------------------------------------------------------------- /.docker/env.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2016 - 2020 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | if [ -z "$CENV" ]; then 7 | set -e 8 | export CENV=1 9 | if [[ -n "$ACTIVATE_VENV" ]] && [[ -n "$MARV_VENV" ]]; then 10 | source $MARV_VENV/bin/activate 11 | fi 12 | if [[ -d "/home/marv/site" ]]; then 13 | export MARV_SITE="/home/marv/site" 14 | export MARV_CONFIG="$MARV_SITE/marv.conf" 15 | fi 16 | cd 17 | set +e 18 | fi 19 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | !.docker 3 | !CHANGES.rst 4 | !CONTRIBUTING.rst 5 | !code 6 | !dist 7 | !docs 8 | !requirements 9 | !scripts 10 | !sites/example/marv.conf 11 | !tutorial 12 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_style = space 7 | indent_size = 4 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.py] 12 | max_line_length = 100 13 | 14 | [*.{js,json,yml,yaml}] 15 | indent_size = 2 16 | 17 | [COMMIT_EDITMSG] 18 | max_line_length = 72 19 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | Thank you for intending to contribute to MARV. 2 | 3 | This is a **read-only mirror**. 4 | 5 | Please make your contribution through the main GitLab project and check the contribution guide before making your contribution. 6 | 7 | https://gitlab.com/ternaris/marv-robotics 8 | https://gitlab.com/ternaris/marv-robotics/-/blob/master/CONTRIBUTING.rst 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | *.sw[op] 3 | .*.sw[op] 4 | .coverage* 5 | .cache 6 | *.egg-info/ 7 | .eggs/ 8 | htmlcov/ 9 | .*venv/ 10 | .pytest_cache/ 11 | pytest-report.xml 12 | __pycache__/ 13 | /.deps-fetched 14 | /.image-name 15 | /dist/ 16 | /code/*/build/ 17 | /code/*/dist/ 18 | /venv*/ 19 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | variables: 2 | DOCKER_BUILDKIT: 1 3 | DOCKER_DRIVER: overlay2 4 | TMPIMG: ci-$CI_JOB_ID 5 | 6 | stages: 7 | - Image 8 | - Test 9 | - Publish 10 | - Check 11 | 12 | workflow: 13 | rules: 14 | - if: '$CI_SERVER_HOST != "gitlab.com"' 15 | when: never 16 | - if: $CI_COMMIT_TAG 17 | - if: $CI_COMMIT_BRANCH 18 | 19 | 20 | Sign-off check: 21 | stage: Check 22 | image: python:3.7-alpine 23 | rules: 24 | - if: '$CI_PIPELINE_SOURCE != "schedule"' 25 | needs: [] 26 | script: 27 | - apk add --no-cache git 28 | - git fetch origin master 29 | - .gitlab-ci/signed-off-by-check > $CI_PROJECT_DIR/sign-off-report.xml 30 | artifacts: 31 | reports: 32 | junit: sign-off-report.xml 33 | 34 | 35 | # Used by schedule to keep master and latest version tag up-to-date 36 | Image: 37 | stage: Image 38 | image: docker:latest 39 | needs: [] 40 | services: 41 | - docker:dind 42 | script: 43 | - apk add --no-cache git curl 44 | - ./.gitlab-ci/build-test-push 45 | - | 46 | if [ "$CI_PIPELINE_SOURCE" = "schedule" ]; then 47 | tag="$(git tag -l 'v*+ce' |tail -1)" 48 | git reset --hard $tag; 49 | export CI_COMMIT_TAG=$tag 50 | export CI_COMMIT_SHA=$(git rev-parse $tag) 51 | ./.gitlab-ci/build-test-push 52 | fi 53 | variables: 54 | REPO: $CI_REGISTRY_IMAGE/cache 55 | 56 | 57 | pytest: 58 | stage: Test 59 | image: $CI_REGISTRY_IMAGE/cache/ci:z-$CI_COMMIT_SHA 60 | needs: 61 | - Image 62 | script: 63 | - . /opt/marv/bin/activate 64 | - python3 -m pip install --no-deps -e code/marv-api 65 | - python3 -m pip install --no-deps -e code/marv-cli 66 | - python3 -m pip install --no-deps -e code/marv 67 | - python3 -m pip install --no-deps -e code/marv-robotics 68 | - ./scripts/build-docs 69 | - ./scripts/download-test-bags 70 | - ./scripts/fetch-deps 71 | - pytest --cache-clear 72 | artifacts: 73 | when: always 74 | paths: 75 | - docs/_build/html 76 | - pytest-report.xml 77 | reports: 78 | junit: pytest-report.xml 79 | variables: 80 | TZ: UTC 81 | 82 | 83 | pages: 84 | stage: Publish 85 | image: alpine:latest 86 | rules: 87 | - if: '$CI_PIPELINE_SOURCE != "schedule" && 88 | ($CI_COMMIT_BRANCH == "master" || $CI_COMMIT_TAG =~ /^v\d\d\.\d\d\.\d\+ce$/)' 89 | needs: 90 | - Image 91 | - pytest 92 | dependencies: 93 | - pytest 94 | script: 95 | - mv docs/_build/html public 96 | artifacts: 97 | paths: 98 | - public 99 | variables: 100 | GIT_STRATEGY: none 101 | -------------------------------------------------------------------------------- /.gitlab-ci/build-test-push: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2016 - 2020 Ternaris. 4 | # SPDX-License-Identifier: Apache-2.0 5 | 6 | set -eux 7 | 8 | cd "$(dirname "$(realpath "$0")")"/.. 9 | 10 | # Fetch deps and fix mode, CI has 666 11 | ./scripts/fetch-deps 12 | chmod -R go-w . 13 | 14 | 15 | # Cache from master, latest tag, and current branch or tag 16 | docker login -u gitlab-ci-token -p "$CI_JOB_TOKEN" "$CI_REGISTRY" 17 | docker build --pull ${FORCE:+--no-cache} --build-arg BUILDKIT_INLINE_CACHE=1 \ 18 | --cache-from "$REPO:$CI_COMMIT_REF_NAME" \ 19 | --cache-from "$REPO:latest" \ 20 | --cache-from "$REPO:master" \ 21 | -t "$TMPIMG" \ 22 | . 23 | 24 | ./.gitlab-ci/test-image "$TMPIMG" "$TMPIMG-site" 25 | 26 | 27 | # Derive image for CI jobs 28 | docker build --build-arg BUILDKIT_INLINE_CACHE=1 -t "$TMPIMG-ci" -f - . <&1 & } | grep -q 'Container startup complete.'; then 27 | docker logs "$ID" 28 | docker rm -f "$ID" 29 | exit 1 30 | fi 31 | 32 | # Scan and run first to give marv server to finish startup as well 33 | docker exec "$ID" bash -lc 'marv scan' 34 | docker exec "$ID" bash -lc 'marv run --col=bags' 35 | 36 | # Check some routes 37 | docker exec "$ID" curl -sfI http://localhost:8000/ || (docker logs "$ID"; exit 1) 38 | docker exec "$ID" curl -sfI http://localhost:8000/main-built.js || (docker logs "$ID"; exit 1) 39 | docker exec "$ID" curl -sfI http://localhost:8000/marv/api/meta || (docker logs "$ID"; exit 1) 40 | docker rm -f "$ID" 41 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | FROM ubuntu:focal 5 | 6 | ARG PYTHON=python3.8 7 | 8 | # This warning can simply be ignored: 9 | # debconf: delaying package configuration, since apt-utils is not installed 10 | ARG DEBIAN_FRONTEND=noninteractive 11 | RUN apt-get update && \ 12 | apt-get install -y \ 13 | bash-completion \ 14 | bc \ 15 | capnproto \ 16 | curl \ 17 | ffmpeg \ 18 | iputils-ping \ 19 | jq \ 20 | less \ 21 | libcapnp-dev \ 22 | libffi-dev \ 23 | libfreetype6-dev \ 24 | libjpeg-dev \ 25 | liblz4-dev \ 26 | libpng-dev \ 27 | libssl-dev \ 28 | libz-dev \ 29 | locales \ 30 | lsof \ 31 | man \ 32 | python3-pip \ 33 | ${PYTHON} \ 34 | ${PYTHON}-dev \ 35 | ${PYTHON}-venv \ 36 | rsync \ 37 | sqlite3 \ 38 | ssh \ 39 | strace \ 40 | tzdata \ 41 | unzip \ 42 | vim \ 43 | && rm -rf /var/lib/apt/lists/* 44 | 45 | RUN locale-gen en_US.UTF-8; dpkg-reconfigure -f noninteractive locales 46 | ENV LANG en_US.UTF-8 47 | ENV LANGUAGE en_US.UTF-8 48 | ENV LC_ALL en_US.UTF-8 49 | 50 | ENV PIP_DISABLE_PIP_VERSION_CHECK=1 51 | ENV MARV_VENV=/opt/marv 52 | COPY requirements/* ${MARV_VENV}/requirements/ 53 | RUN cd ${MARV_VENV} && \ 54 | ${PYTHON} -m venv . && \ 55 | ./bin/pip install -U -r requirements/venv.txt && \ 56 | ./bin/pip install -U -c requirements/marv.txt cython && \ 57 | ./bin/pip install -U -r requirements/marv.txt && \ 58 | ./bin/pip install -U -r requirements/marv-robotics.txt && \ 59 | ./bin/pip install opencv-python-headless==4.3.0.36 && \ 60 | ./bin/pip install -U -r requirements/develop.txt && \ 61 | rm -rf /root/.cache/pip /root/.cache/matplotlib && \ 62 | rmdir /root/.cache || (ls -la /root/.cache; exit 1) 63 | 64 | COPY CHANGES.rst ${MARV_VENV}/CHANGES.rst 65 | COPY CONTRIBUTING.rst ${MARV_VENV}/CONTRIBUTING.rst 66 | COPY tutorial ${MARV_VENV}/tutorial 67 | COPY code ${MARV_VENV}/code 68 | COPY docs ${MARV_VENV}/docs 69 | RUN rm ${MARV_VENV}/docs/config/marv.conf 70 | COPY sites/example/marv.conf ${MARV_VENV}/docs/config/marv.conf 71 | COPY scripts ${MARV_VENV}/scripts 72 | 73 | ARG version= 74 | 75 | # For internal use only 76 | ARG dist= 77 | COPY ${dist:-CHANGES.rst} ${MARV_VENV}/dist 78 | 79 | RUN bash -c '\ 80 | set -e; \ 81 | cd ${MARV_VENV}; \ 82 | if [[ -n "${dist}" ]]; then \ 83 | ./bin/pip install --no-index -f ${MARV_VENV}/dist marv=="${version}"; \ 84 | ./bin/pip install --no-index -f ${MARV_VENV}/dist marv-robotics=="${version}"; \ 85 | else \ 86 | rm ${MARV_VENV}/dist; \ 87 | find code -maxdepth 2 -name setup.py -execdir ${MARV_VENV}/bin/pip install --no-deps . \; ; \ 88 | (source ./bin/activate && ./scripts/build-docs); \ 89 | ./bin/pip install -U --no-deps ./code/marv; \ 90 | fi; \ 91 | if [[ -d /root/.cache ]]; then \ 92 | rm -rf /root/.cache/pip /root/.cache/matplotlib; \ 93 | rmdir /root/.cache || (ls -la /root/.cache; exit 1) \ 94 | fi; \ 95 | ' 96 | RUN chown -R 1000:1000 /opt/marv 97 | 98 | COPY .docker/entrypoint.sh /marv_entrypoint.sh 99 | COPY .docker/env.sh /etc/profile.d/marv_env.sh 100 | RUN echo 'source /etc/profile.d/marv_env.sh' >> /etc/bash.bashrc 101 | 102 | ENV ACTIVATE_VENV=1 103 | ENTRYPOINT ["/marv_entrypoint.sh"] 104 | CMD ["/bin/sh", "-c", "trap 'exit 147' TERM; tail -f /dev/null & while wait ${!}; [ $? -ge 128 ]; do true; done"] 105 | -------------------------------------------------------------------------------- /LICENSES/MARV-License: -------------------------------------------------------------------------------- 1 | Please contact sales@ternaris.com to obtain a license. -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | MARV Robotics 3 | ============= 4 | 5 | Welcome to the MARV Robotics Community Edition. 6 | 7 | MARV Robotics is a powerful and extensible data management platform, featuring a rich dynamic web interface, driven by your algorithms, configurable to the core, and integrating well with your tools to supercharge your workflows. 8 | 9 | For more information please see: 10 | 11 | - MARV Robotics `documentation `_ 12 | - MARV Robotics `website `_ 13 | 14 | 15 | Contributing 16 | ============ 17 | 18 | Thank you for considering to contribute to MARV. 19 | 20 | To submit issues or create merge requests please follow the instructions provided in the `contribution guide <./CONTRIBUTING.rst>`_. 21 | 22 | By contributing to MARV you accept and agree to the terms and conditions laid out in there. 23 | -------------------------------------------------------------------------------- /code/marv-api/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.in 2 | include *.rst 3 | include *.txt 4 | recursive-include * *.capnp 5 | -------------------------------------------------------------------------------- /code/marv-api/README.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ternaris/marv-robotics/473ca28d85ac55a7190edaa19347936ce3a6553a/code/marv-api/README.rst -------------------------------------------------------------------------------- /code/marv-api/marv_api/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from .decorators import InputNameCollisionError, input, node, select 5 | from .ioctrl import ( 6 | Abort, 7 | ReaderError, 8 | ResourceNotFoundError, 9 | create_group, 10 | create_stream, 11 | get_logger, 12 | get_requested, 13 | get_resource_path, 14 | make_file, 15 | pull, 16 | pull_all, 17 | push, 18 | set_header, 19 | ) 20 | from .scanner import DatasetInfo 21 | 22 | __all__ = ( 23 | 'Abort', 24 | 'DatasetInfo', 25 | 'InputNameCollisionError', 26 | 'ReaderError', 27 | 'ResourceNotFoundError', 28 | 'create_group', 29 | 'create_stream', 30 | 'get_logger', 31 | 'get_requested', 32 | 'get_resource_path', 33 | 'input', 34 | 'make_file', 35 | 'node', 36 | 'pull_all', 37 | 'pull', 38 | 'push', 39 | 'select', 40 | 'set_header', 41 | ) 42 | -------------------------------------------------------------------------------- /code/marv-api/marv_api/dag.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=too-few-public-methods,invalid-name,no-self-argument,no-self-use 5 | 6 | from typing import Optional, Union # noqa: TC002 7 | 8 | from pydantic import BaseModel, Extra, create_model, validator 9 | 10 | 11 | class Model(BaseModel): 12 | 13 | class Config: 14 | extra = Extra.forbid 15 | allow_mutation = False 16 | 17 | def __hash__(self): 18 | dct = self.__dict__ 19 | # Objects of same class with same values for fields have same hash 20 | return hash( 21 | (self.__class__,) + tuple( 22 | tuple(v) if isinstance(v, list) else v for v in (dct[x] for x in self.__fields__) 23 | ), 24 | ) 25 | 26 | 27 | class Inputs(Model): 28 | """Base class for node input configuration models. 29 | 30 | The fields of its subclasses describe the input parameters to be 31 | passed to a node function. 32 | """ 33 | 34 | @classmethod 35 | def subclass(cls, __module__, **kw): 36 | return create_model('Inputs', __base__=Inputs, __module__=__module__, **kw) 37 | 38 | @validator('*', pre=True) 39 | def streamify(cls, val): # noqa: N805 40 | """Turn Node inputs into streams.""" 41 | if hasattr(val, '__marv_node__'): 42 | return Stream(node=val.__marv_node__) 43 | if isinstance(val, Node): 44 | return Stream(node=val) 45 | return val 46 | 47 | 48 | class Node(Model): # pylint: disable=too-few-public-methods 49 | function: str 50 | inputs: Optional[Inputs] 51 | message_schema: Optional[str] 52 | group: Union[bool, str, None] 53 | version: Optional[int] 54 | foreach: Optional[str] 55 | 56 | @validator('function') 57 | def function_needs_to_be_dotted_path(cls, val): # noqa: N805 58 | if '.' not in val: 59 | raise ValueError(f'Expected dotted path to function, not: {val!r}') 60 | return val 61 | 62 | def clone(self, **kw): 63 | # Send inputs through validation 64 | inputs = self.inputs.dict(exclude_unset=True, exclude_defaults=True) 65 | inputs.update(kw) 66 | return self.copy(update={'inputs': type(self.inputs).parse_obj(inputs)}) 67 | 68 | 69 | class Stream(Model): 70 | node: Node 71 | name: Optional[str] 72 | -------------------------------------------------------------------------------- /code/marv-api/marv_api/deprecation.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from __future__ import annotations 5 | 6 | import functools 7 | import warnings 8 | from dataclasses import dataclass 9 | from typing import TYPE_CHECKING 10 | 11 | if TYPE_CHECKING: 12 | from typing import Any, Optional 13 | 14 | 15 | @dataclass 16 | class Info: 17 | module: str 18 | version: str 19 | obj: Any 20 | msg: Optional[str] = None 21 | 22 | 23 | def make_getattr(module, dct): 24 | assert all(x.module == module for x in dct.values()) 25 | 26 | def __getattr__(name): 27 | info = dct.get(name) 28 | if info is None: 29 | raise AttributeError(f'module {module} has no attribute {name}') 30 | 31 | msg = ( 32 | f'{module}.{name} will be removed in {info.version}; ' 33 | f'{info.msg or "please let us know if this is an issue for you."}' 34 | ) 35 | warnings.warn(msg, FutureWarning, stacklevel=2) 36 | return info.obj 37 | 38 | return __getattr__ 39 | 40 | 41 | def deprecated(version, msg=None, name=None): 42 | """Wrap function to trigger deprecated message upon call.""" 43 | 44 | def deco(func): 45 | 46 | @functools.wraps(func) 47 | def wrapper(*args, **kw): 48 | _msg = ( 49 | f'{func.__module__}.{name or func.__name__} will be removed in {version}; ' 50 | f'{msg or "please let us know if this is an issue for you."}' 51 | ) 52 | warnings.warn(_msg, FutureWarning, stacklevel=2) 53 | return func(*args, **kw) 54 | 55 | return wrapper 56 | 57 | return deco 58 | -------------------------------------------------------------------------------- /code/marv-api/marv_api/iomsgs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from collections import namedtuple 5 | 6 | from marv_node.stream import Handle # noqa: F401,TC001 pylint: disable=unused-import 7 | 8 | CreateStream = namedtuple('CreateStream', 'parent name group header') 9 | GetLogger = namedtuple('GetLogger', '') 10 | GetRequested = namedtuple('GetRequested', '') 11 | GetResourcePath = namedtuple('GetResourcePath', 'name') 12 | MakeFile = namedtuple('MakeFile', 'handle name') 13 | 14 | Pull = namedtuple('Pull', 'handle enumerate') 15 | PullAll = namedtuple('PullAll', 'handles') 16 | Push = namedtuple('Push', 'output') 17 | SetHeader = namedtuple('SetHeader', 'header') 18 | -------------------------------------------------------------------------------- /code/marv-api/marv_api/scanner.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | """Dataset scanner. 4 | 5 | Datasets are created based on information provided by scanners. A 6 | scanner is responsible to group files into named datasets:: 7 | 8 | from marv_api import DatasetInfo 9 | 10 | def scan(dirpath, dirnames, filenames): 11 | return [DatasetInfo(os.path.basename(x), [x]) 12 | for x in filenames 13 | if x.endswith('.csv')] 14 | 15 | Scanners are called for every directory within the configured 16 | scanroots, while files and directories starting with a ``.`` and 17 | directories containing an (empty) ``.marvignore`` file are ignored and 18 | will not be traversed into. 19 | 20 | Further, traversal into subdirectories can be controlled by 21 | altering the :paramref:`.dirnames` list in-place. To block further 22 | traversal, e.g. for a directory-based dataset type, set it to an 23 | empty list -- :py:func:`os.walk` is used behind the scenes:: 24 | 25 | dirnames[:] = [] 26 | 27 | """ 28 | 29 | from collections import namedtuple 30 | 31 | DatasetInfo = namedtuple('DatasetInfo', ('name', 'files')) 32 | -------------------------------------------------------------------------------- /code/marv-api/marv_api/setid.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=invalid-name 5 | 6 | import struct 7 | from base64 import b32decode, b32encode 8 | from random import getrandbits 9 | 10 | 11 | def decode_setid(encoded): 12 | """Decode setid as uint128.""" 13 | try: 14 | lo, hi = struct.unpack('' 13 | assert isinstance(utils.NOTSET, tuple) 14 | empty_tuple = () 15 | assert utils.NOTSET is not empty_tuple 16 | 17 | 18 | def test_popattr(): 19 | 20 | class Foo: 21 | a = 1 22 | bb = 2 23 | 24 | assert utils.popattr(Foo, 'a') == 1 25 | with pytest.raises(AttributeError): 26 | assert utils.popattr(Foo, 'a') 27 | assert utils.popattr(Foo, 'a', None) is None 28 | assert utils.popattr(Foo, 'bb', None) == 2 29 | assert utils.popattr(Foo, 'bb', None) is None 30 | 31 | 32 | def test_exclusive_setitem(): 33 | dct = {} 34 | utils.exclusive_setitem(dct, 'foo', 1) 35 | assert dct['foo'] == 1 36 | with pytest.raises(KeyError): 37 | utils.exclusive_setitem(dct, 'foo', 1) 38 | with pytest.raises(ValueError, match='already in dictionary'): 39 | utils.exclusive_setitem(dct, 'foo', 1, ValueError) 40 | -------------------------------------------------------------------------------- /code/marv-api/marv_api/tests/types.capnp: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | @0xc23325e1f3feb928; 5 | 6 | struct Test { 7 | foo @0 :Void; 8 | } 9 | -------------------------------------------------------------------------------- /code/marv-api/marv_api/types/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import capnp # noqa: F401,TC002 pylint: disable=unused-import 5 | 6 | # pylint: disable=no-name-in-module 7 | from marv_detail.types_capnp import Section, Widget 8 | from marv_nodes.types_capnp import Dataset, File, GeoJson, Words 9 | from marv_pycapnp.types_capnp import ( 10 | BoolValue, 11 | DataValue, 12 | Float32Value, 13 | Float64Value, 14 | Int8Value, 15 | Int16Value, 16 | Int32Value, 17 | Int64Value, 18 | TextValue, 19 | TimedBool, 20 | TimedData, 21 | TimedFloat32, 22 | TimedFloat64, 23 | TimedInt8, 24 | TimedInt16, 25 | TimedInt32, 26 | TimedInt64, 27 | TimedText, 28 | TimedUInt8, 29 | TimedUInt16, 30 | TimedUInt32, 31 | TimedUInt64, 32 | TimelineEvent, 33 | Timeslice, 34 | UInt8Value, 35 | UInt16Value, 36 | UInt32Value, 37 | UInt64Value, 38 | ) 39 | 40 | # pylint: enable=no-name-in-module 41 | 42 | __all__ = ( 43 | 'BoolValue', 44 | 'Dataset', 45 | 'DataValue', 46 | 'File', 47 | 'Float32Value', 48 | 'Float64Value', 49 | 'GeoJson', 50 | 'Int16Value', 51 | 'Int32Value', 52 | 'Int64Value', 53 | 'Int8Value', 54 | 'Section', 55 | 'TextValue', 56 | 'TimedBool', 57 | 'TimedData', 58 | 'TimedFloat32', 59 | 'TimedFloat64', 60 | 'TimedInt16', 61 | 'TimedInt32', 62 | 'TimedInt64', 63 | 'TimedInt8', 64 | 'TimedText', 65 | 'TimedUInt16', 66 | 'TimedUInt32', 67 | 'TimedUInt64', 68 | 'TimedUInt8', 69 | 'TimelineEvent', 70 | 'Timeslice', 71 | 'UInt16Value', 72 | 'UInt32Value', 73 | 'UInt64Value', 74 | 'UInt8Value', 75 | 'Widget', 76 | 'Words', 77 | ) 78 | -------------------------------------------------------------------------------- /code/marv-api/marv_api/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import os 5 | import sys 6 | from contextlib import contextmanager 7 | from importlib import import_module 8 | from shlex import quote 9 | from subprocess import Popen 10 | 11 | NOTSET = type('NOTSET', (tuple,), {'__repr__': lambda x: ''})() 12 | 13 | 14 | def echo(*args, **kw): 15 | """Wrap print to let linter forbid print usage.""" 16 | print(*args, **kw) # noqa: T001 17 | 18 | 19 | def err(*args, exit=None, **kw): 20 | """Print to stderr and optionally exit.""" 21 | print(*args, **kw, file=sys.stderr, flush=True) # noqa: T001 22 | if exit is not None: 23 | sys.exit(exit) 24 | 25 | 26 | def find_obj(objpath, name=False): 27 | try: 28 | modpath, objname = objpath.split(':') 29 | except ValueError: 30 | modpath, objname = objpath.rsplit('.', 1) 31 | mod = import_module(modpath) 32 | obj = getattr(mod, objname) 33 | return (objname, obj) if name else obj 34 | 35 | 36 | def exclusive_setitem(dct, key, value, exc_class=KeyError): 37 | if key in dct: 38 | raise exc_class(f'{key!r} already in dictionary') 39 | dct[key] = value 40 | 41 | 42 | def joincmd(cmd): 43 | """Join list of cmd and args into properly quoted string for shell execution.""" 44 | return ' '.join([quote(x) for x in cmd]) 45 | 46 | 47 | @contextmanager 48 | def launch_pdb_on_exception(launch=True): 49 | """Return contextmanager launching pdb on exception. 50 | 51 | Example: 52 | Toggle launch behavior via env variable:: 53 | 54 | with launch_pdb_on_exception(os.environ.get('PDB')): 55 | cli() 56 | 57 | """ 58 | if launch: 59 | try: 60 | yield 61 | except Exception: # pylint: disable=broad-except 62 | import pdb # pylint: disable=import-outside-toplevel 63 | pdb.xpm() # pylint: disable=no-member 64 | else: 65 | yield 66 | 67 | 68 | def popattr(obj, name, default=NOTSET): 69 | try: 70 | value = getattr(obj, name) 71 | delattr(obj, name) 72 | return value # noqa: R504 73 | except AttributeError: 74 | if default is NOTSET: 75 | raise 76 | return default 77 | 78 | 79 | def popen(*args, env=None, **kw): 80 | env = sanitize_env(os.environ.copy() if env is None else env) 81 | return Popen(*args, env=env, **kw) # pylint: disable=consider-using-with 82 | 83 | 84 | def sanitize_env(env): 85 | ld_library_path = env.get('LD_LIBRARY_PATH') 86 | if ld_library_path: 87 | clean_path = ':'.join( 88 | [x for x in ld_library_path.split(':') if not x.startswith('/tmp/_MEI')], 89 | ) 90 | if clean_path: 91 | env['LD_LIBRARY_PATH'] = clean_path 92 | else: 93 | del env['LD_LIBRARY_PATH'] 94 | return env 95 | -------------------------------------------------------------------------------- /code/marv-api/marv_nodes/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import json 5 | import os 6 | 7 | from capnp.lib.capnp import KjException 8 | 9 | import marv_api as marv 10 | from marv_api.setid import SetID 11 | from marv_api.utils import err 12 | from marv_detail import Widget 13 | from marv_pycapnp import Wrapper 14 | 15 | from .types_capnp import Dataset # pylint: disable=import-error 16 | 17 | 18 | @marv.node(Dataset) 19 | def dataset(): 20 | raise RuntimeError('Datasets are added not run') 21 | yield # pylint: disable=unreachable 22 | 23 | 24 | def load_dataset(setdir, dataset): # pylint: disable=redefined-outer-name 25 | setid = SetID(dataset.setid) 26 | files = [ 27 | { 28 | 'path': x.path, 29 | 'missing': bool(x.missing), 30 | 'mtime': x.mtime * 10**6, 31 | 'size': x.size, 32 | } for x in sorted(dataset.files, key=lambda x: x.idx) 33 | ] 34 | dct = { 35 | 'id': setid, 36 | 'name': dataset.name, 37 | 'files': files, 38 | 'time_added': dataset.time_added * 10**6, 39 | 'timestamp': dataset.timestamp * 10**6, 40 | } 41 | userdata = next( 42 | ( 43 | json.loads(x.payload) 44 | for x in getattr(dataset, 'annotations', ()) 45 | if x.type == 'userdata' 46 | ), 47 | {}, 48 | ) 49 | try: 50 | wrapper = Wrapper.from_dict(Dataset, dct, setdir=setdir, userdata=userdata) 51 | except KjException as e: 52 | from pprint import pformat # pylint: disable=import-outside-toplevel 53 | err( 54 | f'Schema violation for {Dataset.schema.node.displayName} with data:\n' 55 | f'{pformat(dct)}\nschema: {Dataset.schema.node.displayName}', 56 | ) 57 | raise e 58 | return [wrapper] 59 | 60 | 61 | dataset.load = load_dataset 62 | 63 | 64 | @marv.node(Widget) 65 | @marv.input('dataset', default=dataset) 66 | def summary_keyval(dataset): # pylint: disable=redefined-outer-name 67 | dataset = yield marv.pull(dataset) 68 | if len(dataset.files) < 2: 69 | return 70 | yield marv.push( 71 | { 72 | 'keyval': { 73 | 'items': [ 74 | { 75 | 'title': 'size', 76 | 'formatter': 'filesize', 77 | 'list': False, 78 | 'cell': { 79 | 'uint64': sum(x.size for x in dataset.files), 80 | }, 81 | }, 82 | { 83 | 'title': 'files', 84 | 'list': False, 85 | 'cell': { 86 | 'uint64': len(dataset.files), 87 | }, 88 | }, 89 | ], 90 | }, 91 | }, 92 | ) 93 | 94 | 95 | @marv.node(Widget) 96 | @marv.input('dataset', default=dataset) 97 | def meta_table(dataset): # pylint: disable=redefined-outer-name 98 | dataset = yield marv.pull(dataset) 99 | columns = [ 100 | { 101 | 'title': 'Name', 102 | 'formatter': 'rellink', 103 | 'sortkey': 'title', 104 | }, 105 | { 106 | 'title': 'Size', 107 | 'formatter': 'filesize', 108 | }, 109 | ] 110 | # dataset.id is setid here 111 | rows = [ 112 | { 113 | 'id': idx, 114 | 'cells': [ 115 | { 116 | 'link': { 117 | 'href': f'{idx}', 118 | 'title': os.path.basename(f.path), 119 | }, 120 | }, 121 | { 122 | 'uint64': f.size, 123 | }, 124 | ], 125 | } for idx, f in enumerate(dataset.files) 126 | ] 127 | yield marv.push({'table': {'columns': columns, 'rows': rows}}) 128 | -------------------------------------------------------------------------------- /code/marv-api/marv_nodes/types.capnp: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | @0xd985651e70b6d5e4; 5 | 6 | using import "/marv_pycapnp/types.capnp".Timestamp; 7 | 8 | 9 | struct Dataset { 10 | id0 @0 :UInt64; 11 | id1 @1 :UInt64; 12 | name @2 :Text; 13 | files @3 :List(File); 14 | timeAdded @4 :Timestamp; 15 | timestamp @5 :Timestamp; 16 | } 17 | 18 | # TODO: 19 | # - remote files 20 | # - URLs for e.g. git commits 21 | 22 | struct File { 23 | path @0 :Text; 24 | missing @1 :Bool; 25 | mtime @2 :Timestamp; 26 | size @3 :UInt64; 27 | } 28 | 29 | 30 | struct Comment { 31 | author @0 :Text; 32 | text @1 :Text; 33 | timeAdded @2 :Timestamp; 34 | } 35 | 36 | 37 | struct GeoJson { 38 | union { 39 | feature @0 :Feature; 40 | featureCollection @1 :FeatureCollection; 41 | point @2 :Point; 42 | multiPoint @3 :MultiPoint; 43 | lineString @4 :LineString; 44 | multiLineString @5 :MultiLineString; 45 | polygon @6 :Polygon; 46 | multiPolygon @7 :MultiPolygon; 47 | geometryCollection @8 :GeometryCollection; 48 | } 49 | 50 | struct Geometry { 51 | union { 52 | point @0 :Point; 53 | multiPoint @1 :MultiPoint; 54 | lineString @2 :LineString; 55 | multiLineString @3 :MultiLineString; 56 | geometryCollection @4 :GeometryCollection; 57 | polygon @5 :Polygon; 58 | multiPolygon @6 :MultiPolygon; 59 | } 60 | } 61 | 62 | using Position = List(Float64); 63 | 64 | struct Feature { 65 | geometry @0 :Geometry; 66 | properties @1 :Properties; 67 | } 68 | 69 | struct FeatureCollection { 70 | features @0 :List(Feature); 71 | } 72 | 73 | struct Point { 74 | coordinates @0 :Position; 75 | } 76 | 77 | struct MultiPoint { 78 | coordinates @0 :List(Position); 79 | } 80 | 81 | struct LineString { 82 | coordinates @0 :List(Position); 83 | } 84 | 85 | struct MultiLineString { 86 | coordinates @0 :List(List(Position)); 87 | } 88 | 89 | struct LinearRing { 90 | coordinates @0 :List(Position); 91 | } 92 | 93 | struct Polygon { 94 | coordinates @0 :List(List(Position)); 95 | } 96 | 97 | struct MultiPolygon { 98 | coordinates @0 :List(List(List(Position))); 99 | } 100 | 101 | struct GeometryCollection { 102 | geometries @0 :List(Geometry); 103 | } 104 | 105 | struct Properties { 106 | coordinatesystem @0 :Text = "WGS84"; # `WGS84` or `cartesian 107 | color @1 :List(Float32); # global color [rgba] 108 | colors @2 :List(List(Float32)); # per vertex color [[rgba]] 109 | fillcolor @3 :List(Float32); # global fill color [rgba] 110 | fillcolors @4 :List(List(Float32)); # per vertex fill color [[rgba]] 111 | width @5 :Float32; # linewidth / strokewith of filled geometry 112 | timestamps @6 :List(UInt64); # per vertex timestamp used for playback 113 | rotations @7 :List(Float64); # per vertex rotations if markers are used 114 | markervertices @8 :List(Float64); # rotation marker polygon (e.g. `[0, 0, -1, .3, -1, -.3]`) 115 | } 116 | } 117 | 118 | 119 | struct PointCloud { 120 | vertices @0 :List(List(Float64)); 121 | timestamp @1 :Timestamp; 122 | } 123 | 124 | 125 | struct Tag { 126 | text @0 :Text; 127 | } 128 | 129 | 130 | struct Words { 131 | words @0 :List(Text); 132 | } 133 | -------------------------------------------------------------------------------- /code/marv-api/marv_pycapnp/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | -------------------------------------------------------------------------------- /code/marv-api/marv_pycapnp/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import pytest 5 | 6 | 7 | @pytest.fixture() 8 | def tmpdir(tmp_path): 9 | """We don't want pytest's path implementation.""" 10 | return tmp_path 11 | -------------------------------------------------------------------------------- /code/marv-api/marv_pycapnp/tests/pythonic.capnp: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | @0xac17afb710499c42; 5 | 6 | using import "/marv_pycapnp/types.capnp".Datetime; 7 | using import "/marv_pycapnp/types.capnp".Timedelta; 8 | using import "/marv_pycapnp/types.capnp".Map; 9 | 10 | struct Pythonic { 11 | datetime @0 :Datetime; 12 | timedelta @1 :Timedelta; 13 | mapping @2 :Map(Text, Text); 14 | } 15 | -------------------------------------------------------------------------------- /code/marv-api/marv_pycapnp/tests/test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=no-self-use 5 | 6 | import unittest 7 | 8 | 9 | class TestCase(unittest.TestCase): 10 | 11 | def test_(self): 12 | from . import pythonic_capnp # pylint: disable=no-name-in-module,import-outside-toplevel 13 | 14 | # class Pythonic(Base): 15 | # """foo""" 16 | # schema = pythonic_capnp.Pythonic 17 | 18 | builder = pythonic_capnp.Pythonic.new_message() 19 | reader = builder.as_reader() 20 | assert reader 21 | -------------------------------------------------------------------------------- /code/marv-api/marv_pycapnp/tests/test_wrapper.capnp: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | @0x8070aafdd912e760; 5 | 6 | 7 | enum EnumType { 8 | foo @0; 9 | bar @1; 10 | } 11 | 12 | 13 | struct TestStruct { 14 | text @0 :Text; 15 | data @1 :Data; 16 | textList @2 :List(Text); 17 | dataList @3 :List(Data); 18 | textListInList @12 :List(List(Text)); 19 | dataListInList @13 :List(List(Data)); 20 | nestedList @4 :List(TestStruct); 21 | union { 22 | unionText @5 :Text; 23 | unionData @6 :Data; 24 | } 25 | union :union { 26 | text @7 :Text; 27 | data @8 :Data; 28 | } 29 | group :group { 30 | text @9 :Text; 31 | data @10 :Data; 32 | } 33 | enum @11 :EnumType; 34 | } 35 | -------------------------------------------------------------------------------- /code/marv-api/marv_pycapnp/types.capnp: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | @0x8ce54f16698f41e9; 5 | 6 | using Timedelta = UInt64; 7 | using Timestamp = UInt64; 8 | 9 | struct BoolValue { 10 | value @0 :Bool; 11 | } 12 | 13 | struct Int8Value { 14 | value @0 :Int8; 15 | } 16 | 17 | struct Int16Value { 18 | value @0 :Int16; 19 | } 20 | 21 | struct Int32Value { 22 | value @0 :Int32; 23 | } 24 | 25 | struct Int64Value { 26 | value @0 :Int64; 27 | } 28 | 29 | struct UInt8Value { 30 | value @0 :UInt8; 31 | } 32 | 33 | struct UInt16Value { 34 | value @0 :UInt16; 35 | } 36 | 37 | struct UInt32Value { 38 | value @0 :UInt32; 39 | } 40 | 41 | struct UInt64Value { 42 | value @0 :UInt64; 43 | } 44 | 45 | struct Float32Value { 46 | value @0 :Float32; 47 | } 48 | 49 | struct Float64Value { 50 | value @0 :Float64; 51 | } 52 | 53 | struct TextValue { 54 | value @0 :Text; 55 | } 56 | 57 | struct DataValue { 58 | value @0 :Data; 59 | } 60 | 61 | struct TimedBool { 62 | value @0 :Bool; 63 | timestamp @1 :Timestamp; 64 | } 65 | 66 | struct TimedInt8 { 67 | value @0 :Int8; 68 | timestamp @1 :Timestamp; 69 | } 70 | 71 | struct TimedInt16 { 72 | value @0 :Int16; 73 | timestamp @1 :Timestamp; 74 | } 75 | 76 | struct TimedInt32 { 77 | value @0 :Int32; 78 | timestamp @1 :Timestamp; 79 | } 80 | 81 | struct TimedInt64 { 82 | value @0 :Int64; 83 | timestamp @1 :Timestamp; 84 | } 85 | 86 | struct TimedUInt8 { 87 | value @0 :UInt8; 88 | timestamp @1 :Timestamp; 89 | } 90 | 91 | struct TimedUInt16 { 92 | value @0 :UInt16; 93 | timestamp @1 :Timestamp; 94 | } 95 | 96 | struct TimedUInt32 { 97 | value @0 :UInt32; 98 | timestamp @1 :Timestamp; 99 | } 100 | 101 | struct TimedUInt64 { 102 | value @0 :UInt64; 103 | timestamp @1 :Timestamp; 104 | } 105 | 106 | struct TimedFloat32 { 107 | value @0 :Float32; 108 | timestamp @1 :Timestamp; 109 | } 110 | 111 | struct TimedFloat64 { 112 | value @0 :Float64; 113 | timestamp @1 :Timestamp; 114 | } 115 | 116 | struct TimedText { 117 | value @0 :Text; 118 | timestamp @1 :Timestamp; 119 | } 120 | 121 | struct TimedData { 122 | value @0 :Data; 123 | timestamp @1 :Timestamp; 124 | } 125 | 126 | struct Timeslice { 127 | start @0 :Timestamp; 128 | # Start of slice 129 | stop @1 :Timestamp; 130 | # End of slice (exclusive) 131 | } 132 | 133 | struct TimelineEvent { 134 | time @0 :Timestamp; 135 | type @1 :Text; 136 | payload @2 :Text; 137 | } 138 | 139 | 140 | # Below here unused so far 141 | 142 | struct Datetime { 143 | timestamp @0 :Timestamp; 144 | # ns since epoch 145 | 146 | tzoffset @1 :Int16; 147 | # timezone offset in minutes 148 | } 149 | 150 | struct Map(Key, Value) { 151 | items @0 :List(Item); 152 | 153 | struct Item { 154 | key @0 :Key; 155 | value @1 :Value; 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /code/marv-api/requirements.in: -------------------------------------------------------------------------------- 1 | ../../requirements/marv-api.in -------------------------------------------------------------------------------- /code/marv-api/requirements.txt: -------------------------------------------------------------------------------- 1 | ../../requirements/marv-api.txt -------------------------------------------------------------------------------- /code/marv-api/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import io 5 | import os 6 | from collections import OrderedDict 7 | 8 | from setuptools import find_packages, setup 9 | 10 | NAME = 'marv-api' 11 | VERSION = '21.12.0' 12 | DESCRIPTION = 'MARV API to implement MARV nodes' 13 | ENTRY_POINTS = {} # type: ignore 14 | 15 | # Copy/paste block below here 16 | 17 | os.chdir(os.path.abspath(os.path.dirname(__file__))) 18 | 19 | with io.open(os.path.join('README.rst'), 'rt', encoding='utf8') as f: 20 | README = f.read() 21 | 22 | with io.open('requirements.in', 'rt', encoding='utf8') as f: 23 | INSTALL_REQUIRES = [ 24 | # e.g. -r ../path/to/file/package_name.in 25 | f'{os.path.basename(req.split()[1])[:-3]}=={VERSION}' if req.startswith('-r') else req 26 | for req in [line.strip() for line in f.readlines() if not line.startswith('#')] 27 | if req 28 | ] 29 | 30 | setup( 31 | name=NAME, 32 | version=VERSION, 33 | description=DESCRIPTION, 34 | long_description=README, 35 | classifiers=[ 36 | 'Development Status :: 5 - Production/Stable', 37 | 'License :: OSI Approved :: GNU Affero General Public License v3', 38 | 'Operating System :: POSIX :: Linux', 39 | 'Programming Language :: Python :: 3 :: Only', 40 | 'Programming Language :: Python :: 3.8', 41 | 'Programming Language :: Python :: Implementation :: CPython', 42 | 'Programming Language :: Python', 43 | 'Topic :: Scientific/Engineering', 44 | ], 45 | author='Ternaris', 46 | author_email='team@ternaris.com', 47 | maintainer='Ternaris', 48 | maintainer_email='team@ternaris.com', 49 | url='https://ternaris.com/marv-robotics', 50 | project_urls=OrderedDict( 51 | ( 52 | ('Documentation', 'https://ternaris.com/marv-robotics/docs/'), 53 | ('Code', 'https://gitlab.com/ternaris/marv-robotics'), 54 | ('Issue tracker', 'https://gitlab.com/ternaris/marv-robotics/issues'), 55 | ), 56 | ), 57 | license='AGPL-3.0-only', 58 | packages=find_packages(), 59 | include_package_data=True, 60 | zip_safe=False, 61 | python_requires='>=3.8.2', 62 | install_requires=INSTALL_REQUIRES, 63 | tests_require=[ 64 | 'pytest', 65 | 'testfixtures', 66 | ], 67 | setup_requires=['pytest-runner'], 68 | entry_points=ENTRY_POINTS, 69 | ) 70 | -------------------------------------------------------------------------------- /code/marv-cli/.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.py[co] 3 | *~ 4 | .*~ 5 | __pycache__/ 6 | /.coverage 7 | /.eggs 8 | /.venv*/ 9 | /cover 10 | /build 11 | /dist 12 | /docs/_build/ 13 | /venv 14 | -------------------------------------------------------------------------------- /code/marv-cli/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.in 2 | include *.rst 3 | include *.txt 4 | -------------------------------------------------------------------------------- /code/marv-cli/README.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | MARV Robotics 3 | ============= 4 | 5 | This package is part of the MARV Robotics Community Edition. 6 | 7 | MARV Robotics is a powerful and extensible data management platform, 8 | featuring a rich dynamic web interface, driven by your algorithms, 9 | configurable to the core, and integrating well with your tools to 10 | supercharge your workflows. 11 | 12 | For more information please see: 13 | 14 | - MARV Robotics `website `_ 15 | - MARV Robotics `documentation `_ 16 | - MARV Robotics `GitLab project `_ 17 | -------------------------------------------------------------------------------- /code/marv-cli/marv_cli/__main__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from . import cli 5 | 6 | cli() 7 | -------------------------------------------------------------------------------- /code/marv-cli/requirements.in: -------------------------------------------------------------------------------- 1 | ../../requirements/marv-cli.in -------------------------------------------------------------------------------- /code/marv-cli/requirements.txt: -------------------------------------------------------------------------------- 1 | ../../requirements/marv-cli.txt -------------------------------------------------------------------------------- /code/marv-cli/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import io 5 | import os 6 | from collections import OrderedDict 7 | 8 | from setuptools import find_packages, setup 9 | 10 | NAME = 'marv-cli' 11 | VERSION = '21.12.0' 12 | DESCRIPTION = 'Core of the MARV command-line interface' 13 | ENTRY_POINTS = { 14 | 'console_scripts': [ 15 | 'marv = marv_cli:cli', 16 | ], 17 | } 18 | 19 | # Copy/paste block below here 20 | 21 | os.chdir(os.path.abspath(os.path.dirname(__file__))) 22 | 23 | with io.open(os.path.join('README.rst'), 'rt', encoding='utf8') as f: 24 | README = f.read() 25 | 26 | with io.open('requirements.in', 'rt', encoding='utf8') as f: 27 | INSTALL_REQUIRES = [ 28 | # e.g. -r ../path/to/file/package_name.in 29 | f'{os.path.basename(req.split()[1])[:-3]}=={VERSION}' if req.startswith('-r') else req 30 | for req in [line.strip() for line in f.readlines() if not line.startswith('#')] 31 | if req 32 | ] 33 | 34 | setup( 35 | name=NAME, 36 | version=VERSION, 37 | description=DESCRIPTION, 38 | long_description=README, 39 | classifiers=[ 40 | 'Development Status :: 5 - Production/Stable', 41 | 'License :: OSI Approved :: GNU Affero General Public License v3', 42 | 'Operating System :: POSIX :: Linux', 43 | 'Programming Language :: Python :: 3 :: Only', 44 | 'Programming Language :: Python :: 3.8', 45 | 'Programming Language :: Python :: Implementation :: CPython', 46 | 'Programming Language :: Python', 47 | 'Topic :: Scientific/Engineering', 48 | ], 49 | author='Ternaris', 50 | author_email='team@ternaris.com', 51 | maintainer='Ternaris', 52 | maintainer_email='team@ternaris.com', 53 | url='https://ternaris.com/marv-robotics', 54 | project_urls=OrderedDict( 55 | ( 56 | ('Documentation', 'https://ternaris.com/marv-robotics/docs/'), 57 | ('Code', 'https://gitlab.com/ternaris/marv-robotics'), 58 | ('Issue tracker', 'https://gitlab.com/ternaris/marv-robotics/issues'), 59 | ), 60 | ), 61 | license='AGPL-3.0-only', 62 | packages=find_packages(), 63 | include_package_data=True, 64 | zip_safe=False, 65 | python_requires='>=3.8.2', 66 | install_requires=INSTALL_REQUIRES, 67 | tests_require=[ 68 | 'pytest', 69 | 'testfixtures', 70 | ], 71 | setup_requires=['pytest-runner'], 72 | entry_points=ENTRY_POINTS, 73 | ) 74 | -------------------------------------------------------------------------------- /code/marv-robotics/.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.py[co] 3 | *.sw[op] 4 | .*.sw[op] 5 | *~ 6 | .*~ 7 | __pycache__/ 8 | /.coverage 9 | /.eggs 10 | /.venv*/ 11 | /cover 12 | /build 13 | /dist 14 | /venv 15 | -------------------------------------------------------------------------------- /code/marv-robotics/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.in 2 | include *.rst 3 | include *.txt 4 | include marv_robotics/matplotlibrc 5 | recursive-include * *.capnp 6 | -------------------------------------------------------------------------------- /code/marv-robotics/README.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | MARV Robotics 3 | ============= 4 | 5 | This package is part of the MARV Robotics Community Edition. 6 | 7 | MARV Robotics is a powerful and extensible data management platform, 8 | featuring a rich dynamic web interface, driven by your algorithms, 9 | configurable to the core, and integrating well with your tools to 10 | supercharge your workflows. 11 | 12 | For more information please see: 13 | 14 | - MARV Robotics `website `_ 15 | - MARV Robotics `documentation `_ 16 | - MARV Robotics `GitLab project `_ 17 | -------------------------------------------------------------------------------- /code/marv-robotics/marv_robotics/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import os 5 | 6 | os.environ.setdefault('MATPLOTLIBRC', os.path.dirname(__file__)) 7 | -------------------------------------------------------------------------------- /code/marv-robotics/marv_robotics/bag.capnp: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | @0xb7826c391b2ebc91; 5 | 6 | using import "/marv_pycapnp/types.capnp".Timedelta; 7 | using import "/marv_pycapnp/types.capnp".Timestamp; 8 | 9 | struct Bagmeta { 10 | bags @0 :List(Bag); 11 | startTime @1 :Timestamp; 12 | endTime @2 :Timestamp; 13 | duration @3 :Timedelta; 14 | msgCount @4 :UInt64; 15 | msgTypes @5 :List(Text); 16 | topics @6 :List(Text); 17 | connections @7 :List(Connection); 18 | } 19 | 20 | struct Bag { 21 | version @0 :UInt16; 22 | startTime @1 :UInt64; 23 | endTime @2 :UInt64; 24 | duration @3 :UInt64; 25 | msgCount @4 :UInt64; 26 | connections @5 :List(Connection); 27 | } 28 | 29 | struct Connection { 30 | topic @0 :Text; 31 | datatype @1 :Text; 32 | md5sum @2 :Text; 33 | msgDef @3 :Text; 34 | msgCount @4 :UInt64; 35 | latching @5 :Bool; 36 | serializationFormat @6 :Text; 37 | } 38 | 39 | struct MsgType { 40 | name @0 :Text; 41 | md5sum @1 :Text; 42 | msgDef @2 :Text; 43 | } 44 | 45 | struct Topic { 46 | name @0 :Text; 47 | msgCount @1 :UInt64; 48 | msgType @2 :Text; 49 | msgTypeDef @4: Text; 50 | msgTypeMd5sum @5: Text; 51 | latching @3 :Bool; 52 | } 53 | 54 | struct Message { 55 | tidx @0 :UInt32; 56 | # Message belongs to topic with tidx within header.topics list 57 | 58 | data @1 :Data; 59 | # Serialized message data 60 | 61 | timestamp @2 :Timestamp; 62 | } 63 | -------------------------------------------------------------------------------- /code/marv-robotics/marv_robotics/fulltext.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import re 5 | 6 | import marv_api as marv 7 | from marv_api.types import Words 8 | 9 | from .bag import make_deserialize, messages 10 | 11 | WSNULL = re.compile(r'[\s\x00]') 12 | 13 | 14 | @marv.node(Words) 15 | @marv.input('stream', foreach=marv.select(messages, '*:std_msgs/msg/String')) 16 | def fulltext_per_topic(stream): 17 | yield marv.set_header(title=stream.topic) 18 | words = set() 19 | deserialize = make_deserialize(stream) 20 | while msg := (yield marv.pull(stream)): 21 | rosmsg = deserialize(msg.data) 22 | words.update(WSNULL.split(rosmsg.data)) 23 | 24 | if not words: 25 | raise marv.Abort() 26 | yield marv.push({'words': list(words)}) 27 | 28 | 29 | @marv.node(Words) 30 | @marv.input('streams', default=fulltext_per_topic) 31 | def fulltext(streams): 32 | """Extract all text from bag file and store for fulltext search.""" 33 | tmp = [] 34 | while stream := (yield marv.pull(streams)): 35 | tmp.append(stream) 36 | streams = tmp 37 | if not streams: 38 | raise marv.Abort() 39 | 40 | msgs = yield marv.pull_all(*streams) 41 | words = {x for msg in msgs for x in msg.words} 42 | yield marv.push({'words': sorted(words)}) 43 | -------------------------------------------------------------------------------- /code/marv-robotics/marv_robotics/matplotlibrc: -------------------------------------------------------------------------------- 1 | backend: Agg 2 | -------------------------------------------------------------------------------- /code/marv-robotics/marv_robotics/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | -------------------------------------------------------------------------------- /code/marv-robotics/marv_robotics/tests/test_bag_utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from unittest.mock import Mock 5 | 6 | from marv_robotics.bag import make_get_timestamp 7 | 8 | 9 | def test_make_get_timestamp(): 10 | # no header 11 | log = Mock([]) 12 | get_timestamp = make_get_timestamp(log) 13 | rosmsg = Mock([]) 14 | bagmsg = Mock([], timestamp=42) 15 | nanosec = get_timestamp(rosmsg, bagmsg) 16 | assert nanosec == 42 17 | 18 | # below magic 19 | log = Mock([]) 20 | get_timestamp = make_get_timestamp(log) 21 | rosmsg = Mock([], header=Mock(stamp=Mock([], sec=0, nanosec=0))) 22 | bagmsg = Mock([], timestamp=42) 23 | nanosec = get_timestamp(rosmsg, bagmsg) 24 | assert nanosec == 0 25 | 26 | # above magic 27 | log = Mock(['warning']) 28 | get_timestamp = make_get_timestamp(log) 29 | rosmsg = Mock([], header=Mock(stamp=Mock([], sec=0, nanosec=0))) 30 | bagmsg = Mock([], timestamp=601 * 10**9) 31 | nanosec = get_timestamp(rosmsg, bagmsg) 32 | assert nanosec == 601 * 10**9 33 | log.warning.assert_called_once() 34 | 35 | # no fallback 36 | log = Mock([]) 37 | get_timestamp = make_get_timestamp(log) 38 | rosmsg = Mock([], header=Mock(stamp=Mock([], sec=2, nanosec=42))) 39 | bagmsg = Mock([]) 40 | nanosec = get_timestamp(rosmsg, bagmsg) 41 | assert nanosec == 2 * 10**9 + 42 42 | 43 | # decision remains after first message 44 | rosmsg = Mock([], header=Mock(stamp=Mock([], sec=0, nanosec=0))) 45 | bagmsg = Mock([], timestamp=601 * 10**9) 46 | nanosec = get_timestamp(rosmsg, bagmsg) 47 | assert nanosec == 0 48 | -------------------------------------------------------------------------------- /code/marv-robotics/marv_ros/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | -------------------------------------------------------------------------------- /code/marv-robotics/requirements.in: -------------------------------------------------------------------------------- 1 | ../../requirements/marv-robotics.in -------------------------------------------------------------------------------- /code/marv-robotics/requirements.txt: -------------------------------------------------------------------------------- 1 | ../../requirements/marv-robotics.txt -------------------------------------------------------------------------------- /code/marv-robotics/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import io 5 | import os 6 | from collections import OrderedDict 7 | 8 | from setuptools import find_packages, setup 9 | 10 | NAME = 'marv-robotics' 11 | VERSION = '21.12.0' 12 | DESCRIPTION = 'Data management platform for robot logs' 13 | ENTRY_POINTS = {} # type: ignore 14 | 15 | # Copy/paste block below here 16 | 17 | os.chdir(os.path.abspath(os.path.dirname(__file__))) 18 | 19 | with io.open(os.path.join('README.rst'), 'rt', encoding='utf8') as f: 20 | README = f.read() 21 | 22 | with io.open('requirements.in', 'rt', encoding='utf8') as f: 23 | INSTALL_REQUIRES = [ 24 | # e.g. -r ../path/to/file/package_name.in 25 | f'{os.path.basename(req.split()[1])[:-3]}=={VERSION}' if req.startswith('-r') else req 26 | for req in [line.strip() for line in f.readlines() if not line.startswith('#')] 27 | if req 28 | ] 29 | 30 | setup( 31 | name=NAME, 32 | version=VERSION, 33 | description=DESCRIPTION, 34 | long_description=README, 35 | classifiers=[ 36 | 'Development Status :: 5 - Production/Stable', 37 | 'License :: OSI Approved :: GNU Affero General Public License v3', 38 | 'Operating System :: POSIX :: Linux', 39 | 'Programming Language :: Python :: 3 :: Only', 40 | 'Programming Language :: Python :: 3.8', 41 | 'Programming Language :: Python :: Implementation :: CPython', 42 | 'Programming Language :: Python', 43 | 'Topic :: Scientific/Engineering', 44 | ], 45 | author='Ternaris', 46 | author_email='team@ternaris.com', 47 | maintainer='Ternaris', 48 | maintainer_email='team@ternaris.com', 49 | url='https://ternaris.com/marv-robotics', 50 | project_urls=OrderedDict( 51 | ( 52 | ('Documentation', 'https://ternaris.com/marv-robotics/docs/'), 53 | ('Code', 'https://gitlab.com/ternaris/marv-robotics'), 54 | ('Issue tracker', 'https://gitlab.com/ternaris/marv-robotics/issues'), 55 | ), 56 | ), 57 | license='AGPL-3.0-only', 58 | packages=find_packages(), 59 | include_package_data=True, 60 | zip_safe=False, 61 | python_requires='>=3.8.2', 62 | install_requires=INSTALL_REQUIRES, 63 | tests_require=[ 64 | 'pytest', 65 | 'testfixtures', 66 | ], 67 | setup_requires=['pytest-runner'], 68 | entry_points=ENTRY_POINTS, 69 | ) 70 | -------------------------------------------------------------------------------- /code/marv/.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.py[co] 3 | *~ 4 | .*~ 5 | __pycache__/ 6 | /.coverage 7 | /.eggs 8 | /.venv*/ 9 | /cover 10 | /build 11 | /dist 12 | /docs/_build/ 13 | /marv_node/testing/_robotics_tests/data/ 14 | /marv/app/docs/ 15 | /marv/app/static/ 16 | /venv 17 | -------------------------------------------------------------------------------- /code/marv/MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.in 2 | include *.rst 3 | include *.txt 4 | recursive-include marv/app/docs * 5 | recursive-include marv/app/static * 6 | global-exclude *~ 7 | -------------------------------------------------------------------------------- /code/marv/README.rst: -------------------------------------------------------------------------------- 1 | ============= 2 | MARV Robotics 3 | ============= 4 | 5 | This package is part of the MARV Robotics Community Edition. 6 | 7 | MARV Robotics is a powerful and extensible data management platform, 8 | featuring a rich dynamic web interface, driven by your algorithms, 9 | configurable to the core, and integrating well with your tools to 10 | supercharge your workflows. 11 | 12 | For more information please see: 13 | 14 | - MARV Robotics `website `_ 15 | - MARV Robotics `documentation `_ 16 | - MARV Robotics `GitLab project `_ 17 | -------------------------------------------------------------------------------- /code/marv/marv/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | -------------------------------------------------------------------------------- /code/marv/marv/helpers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | 5 | def download_link(file, dataset): 6 | return { 7 | 'href': f'download/{dataset.files.index(file)}', 8 | 'target': '_blank', 9 | 'title': file.relpath, 10 | } 11 | 12 | 13 | def file_status(file): 14 | state = [] 15 | if file and file.missing: 16 | state.append({'icon': 'hdd', 'title': 'This file is missing', 'classes': 'text-danger'}) 17 | return state 18 | -------------------------------------------------------------------------------- /code/marv/marv/model_fields.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2019 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from tortoise.fields import CharField 5 | 6 | from marv_api.setid import SetID 7 | 8 | 9 | class SetIDField(CharField): 10 | 11 | def __init__(self, max_length=32, **kwargs): 12 | super().__init__(max_length, **kwargs) 13 | 14 | def to_db_value(self, value: SetID, _) -> str: 15 | return str(value) 16 | 17 | def to_python_value(self, value: str) -> SetID: 18 | return SetID(value) 19 | -------------------------------------------------------------------------------- /code/marv/marv/testing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import os 5 | import shutil 6 | import tempfile 7 | from contextlib import contextmanager 8 | 9 | 10 | @contextmanager 11 | def chdir(directory): 12 | """Change working directory - NOT THREAD SAFE.""" 13 | cwd = os.getcwd() 14 | os.chdir(directory) 15 | try: 16 | yield directory 17 | finally: 18 | os.chdir(cwd) 19 | 20 | 21 | def make_scanroot(scanroot, names): 22 | if not os.path.exists(scanroot): 23 | os.makedirs(scanroot) 24 | for name in names: 25 | with open(os.path.join(scanroot, name), 'w', encoding='utf-8') as f: 26 | f.write(name) 27 | 28 | 29 | @contextmanager 30 | def temporary_directory(keep=None): 31 | """Create, change into, and cleanup temporary directory.""" 32 | tmpdir = tempfile.mkdtemp() 33 | with chdir(tmpdir): 34 | try: 35 | yield tmpdir 36 | finally: 37 | if not keep: 38 | shutil.rmtree(tmpdir) 39 | 40 | 41 | def decode(data, encoding='utf-8'): 42 | if isinstance(data, str): 43 | return data.decode(encoding) 44 | if isinstance(data, dict): 45 | return {decode(k): decode(v) for k, v in data.items()} 46 | if isinstance(data, list): 47 | return [decode(x) for x in data] 48 | if isinstance(data, tuple): 49 | return tuple(decode(x) for x in data) 50 | return data 51 | -------------------------------------------------------------------------------- /code/marv/marv/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # Create loglevels 5 | import marv_cli # noqa: F401,TC001 pylint: disable=unused-import 6 | -------------------------------------------------------------------------------- /code/marv/marv/tests/data/empty_dump.json: -------------------------------------------------------------------------------- 1 | { 2 | "datasets": {}, 3 | "users": [], 4 | "version": "21.05" 5 | } -------------------------------------------------------------------------------- /code/marv/marv/tests/test_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2019 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import pytest 5 | 6 | from marv.db import MultipleSetidFoundError, NoSetidFoundError 7 | 8 | 9 | async def test_id_helpers(site): 10 | assert not await site.db.resolve_shortids([]) 11 | with pytest.raises(MultipleSetidFoundError): 12 | await site.db.resolve_shortids(['']) 13 | with pytest.raises(NoSetidFoundError): 14 | await site.db.resolve_shortids(['@']) 15 | 16 | sets = await site.db.get_datasets_for_collections(None) 17 | setid1 = sets[0] 18 | setid20 = sets[19] 19 | 20 | assert await site.db.resolve_shortids(['aaaa']) == [setid1] 21 | assert await site.db.resolve_shortids(['aaaa', 'aaa']) == [setid1] 22 | assert await site.db.resolve_shortids(['aaaa', 'cmaa']) == sorted([setid1, setid20]) 23 | 24 | res1 = await site.db.get_datasets_by_setids([setid1, setid20], [], '::') 25 | res2 = await site.db.get_datasets_by_dbids([1, 20], [], '::') 26 | assert {x.setid for x in res1} == {x.setid for x in res2} 27 | 28 | res1 = await site.db.get_datasets_by_setids([setid1, setid20], ['files'], '::') 29 | res2 = await site.db.get_datasets_by_dbids([1, 20], ['files'], '::') 30 | assert {x.files[0].path for x in res1} == {x.files[0].path for x in res2} 31 | 32 | 33 | async def test_lookups(site): 34 | sets = await site.db.get_datasets_for_collections(None) 35 | assert len(sets) == 30 36 | 37 | sets = await site.db.get_datasets_for_collections(['hodge']) 38 | assert len(sets) == 10 39 | 40 | res = await site.db.get_filepath_by_setid_idx(sets[9], 0, '::') 41 | assert res.endswith('hodge/0010') 42 | 43 | 44 | async def test_discard(site): 45 | sets = await site.db.get_datasets_for_collections(None) 46 | 47 | first = sets[0] 48 | await site.db.discard_datasets_by_setids([first]) 49 | rest = await site.db.get_datasets_for_collections(None) 50 | assert set(sets) - set(rest) == {first} 51 | 52 | await site.db.discard_datasets_by_setids([first], False) 53 | rest = await site.db.get_datasets_for_collections(None) 54 | assert sets == rest 55 | 56 | await site.db.discard_datasets_by_dbids([1], True, '::') 57 | rest = await site.db.get_datasets_for_collections(None) 58 | assert set(sets) - set(rest) == {first} 59 | 60 | with pytest.raises(NoSetidFoundError): 61 | await site.db.resolve_shortids([first]) 62 | 63 | res = await site.db.resolve_shortids([first], discarded=True) 64 | assert res == [first] 65 | 66 | await site.cleanup_discarded() 67 | with pytest.raises(NoSetidFoundError): 68 | await site.db.resolve_shortids([first], discarded=True) 69 | 70 | await site.cleanup_discarded() 71 | 72 | 73 | async def test_query(site): 74 | await site.db.discard_datasets_by_dbids([10], True, '::') 75 | res = await site.db.bulk_tag( 76 | [ 77 | ('foo', 1), 78 | ('foo', 2), 79 | ], 80 | [], 81 | '::', 82 | ) 83 | 84 | res = await site.db.query() 85 | assert len(res) == 29 86 | 87 | res2 = await site.db.query(abbrev=True) 88 | assert [str(x)[0:10] for x in res] == [str(x) for x in res2] 89 | 90 | res = await site.db.query(collections=['hodge']) 91 | assert len(res) == 9 92 | 93 | res = await site.db.query(discarded=True) 94 | assert len(res) == 1 95 | 96 | res = await site.db.query(outdated=True) 97 | assert len(res) == 0 98 | 99 | res = await site.db.query(missing=True) 100 | assert len(res) == 0 101 | 102 | res = await site.db.query(path='/dev/null/') 103 | assert len(res) == 29 104 | 105 | res = await site.db.query(tags=['bar', 'baz']) 106 | assert len(res) == 0 107 | 108 | res = await site.db.query(tags=['bar', 'foo']) 109 | assert len(res) == 2 110 | -------------------------------------------------------------------------------- /code/marv/marv/tests/test_docs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | 5 | async def test_docs(client): 6 | res = await client.get('/docs/') 7 | assert res.status == 200 8 | 9 | res = await client.get('/docs//absolute') 10 | assert res.status == 403 11 | -------------------------------------------------------------------------------- /code/marv/marv/tests/test_dump_restore.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import json 5 | from pathlib import Path 6 | 7 | import pytest 8 | 9 | from .conftest import recorded 10 | 11 | DATADIR = Path(__file__).parent / 'data' 12 | 13 | 14 | async def prescan(site): 15 | await site.db.group_add('grp') 16 | await site.db.group_adduser('admin', 'adm') 17 | await site.db.group_adduser('grp', 'test') 18 | 19 | 20 | async def postscan(site): 21 | hodgeids = await site.db.query(['hodge']) 22 | podgeids = await site.db.query(['podge']) 23 | await site.db.update_tags_by_setids(hodgeids, add=['TAG1'], remove=[]) 24 | await site.db.update_tags_by_setids([hodgeids[1], podgeids[1]], add=['TAG2'], remove=[]) 25 | await site.db.comment_by_setids([hodgeids[0]], 'test', 'comment\ntext') 26 | await site.db.comment_by_setids([hodgeids[1], podgeids[1]], 'adm', 'more\ncomment') 27 | 28 | 29 | @pytest.mark.marv(site={'empty': True}) 30 | async def test_dump_empty(site): 31 | dump = await site.Database.dump_database(site.config.marv.dburi) 32 | assert recorded(dump, DATADIR / 'empty_dump.json') 33 | 34 | 35 | @pytest.mark.marv(site={'prescan': prescan, 'postscan': postscan, 'size': 2}) 36 | async def test_dump(site): 37 | dump = await site.Database.dump_database(site.config.marv.dburi) 38 | assert recorded(dump, DATADIR / 'full_dump.json') 39 | 40 | 41 | @pytest.mark.marv(site={'empty': True}) 42 | async def test_restore(site): 43 | full_dump = json.loads((DATADIR / 'full_dump.json').read_text()) 44 | await site.restore_database(**full_dump) 45 | 46 | dump = await site.Database.dump_database(site.config.marv.dburi) 47 | dump = json.loads(json.dumps(dump)) 48 | full_dump = json.loads((DATADIR / 'full_dump.json').read_text()) 49 | assert full_dump == dump 50 | -------------------------------------------------------------------------------- /code/marv/marv/tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from unittest import mock 5 | 6 | import pytest 7 | 8 | from marv.db import Database, DBNotInitializedError, DBVersionError 9 | from marv.site import Site 10 | 11 | 12 | async def test_metadata(tmpdir): 13 | marv_conf = tmpdir / 'marv.conf' 14 | marv_conf.write('[marv]\ncollections=') 15 | 16 | # starting without DB must fail 17 | with pytest.raises(DBNotInitializedError): 18 | site = await Site.create(marv_conf, init=False) 19 | 20 | with mock.patch.object(Database, 'VERSION', '00.01'): 21 | site = await Site.create(marv_conf, init=True) 22 | await site.destroy() 23 | 24 | # reusing with same version should work 25 | site = await Site.create(marv_conf, init=False) 26 | await site.destroy() 27 | 28 | # reusing with current version must fail 29 | with pytest.raises(DBVersionError): 30 | site = await Site.create(marv_conf, init=False) 31 | -------------------------------------------------------------------------------- /code/marv/marv/tests/test_persist_without_type.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2019 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import os 5 | from pathlib import Path 6 | 7 | import pytest 8 | 9 | import marv_api as marv 10 | from marv.config import ConfigError 11 | from marv.site import Site 12 | from marv_api import DatasetInfo 13 | from marv_nodes import dataset as dataset_node 14 | 15 | DATADIR = Path(__file__).parent / 'data' 16 | RECORD = os.environ.get('MARV_TESTING_RECORD') 17 | SETIDS = [ 18 | 'l2vnfhfoe3z7ad7vclkd64tsqy', 19 | 'vdls3sgw5yanuat4uepmhjwcpm', 20 | 'e2oxlhpedjwked2llnnzir4tii', 21 | 'phjg4ncymwbrbl4yx35bciqazq', 22 | ] 23 | 24 | MARV_CONF = """ 25 | [marv] 26 | collections = foo 27 | 28 | [collection foo] 29 | scanner = marv.tests.test_persist_without_type:scanner 30 | scanroots = ./scanroots/foo 31 | nodes = 32 | marv_nodes:dataset 33 | marv_nodes:summary_keyval 34 | marv_nodes:meta_table 35 | marv.tests.test_persist_without_type:notype 36 | listing_columns = 37 | name | Name | route | (detail_route (get "dataset.id") (get "dataset.name")) 38 | size | Size | filesize | (sum (get "dataset.files[:].size")) 39 | status | Status | icon[] | (status) 40 | tags | Tags | pill[] | (tags) 41 | notype | NoType | datetime | (get "notype.value") 42 | """ 43 | 44 | 45 | def scanner(dirpath, dirnames, filenames): # pylint: disable=unused-argument 46 | return [DatasetInfo(x, [x]) for x in filenames] 47 | 48 | 49 | @marv.node() 50 | @marv.input('dataset', default=dataset_node) 51 | def notype(dataset): 52 | dataset = yield marv.pull(dataset) 53 | with open(dataset.files[0].path, encoding='utf-8') as f: 54 | yield marv.push({'value': int(f.read())}) 55 | 56 | 57 | @pytest.fixture() 58 | async def site(loop, tmpdir): # pylint: disable=unused-argument 59 | flag = (tmpdir / 'TEST_SITE') 60 | flag.write('') 61 | 62 | marv_conf = (tmpdir / 'marv.conf') 63 | marv_conf.write(MARV_CONF) 64 | 65 | # make scanroots 66 | for sitename in ('foo',): 67 | for idx, name in enumerate(['a', 'b']): 68 | name = f'{sitename}_{name}' 69 | path = tmpdir / 'scanroots' / sitename / name 70 | path.write(str(idx), ensure=True) 71 | 72 | site_ = await Site.create(marv_conf, init=True) 73 | yield site_ 74 | await site_.destroy() 75 | 76 | 77 | async def test_fail_notype(site): # pylint: disable=redefined-outer-name 78 | with pytest.raises(ConfigError): 79 | await site.scan() 80 | -------------------------------------------------------------------------------- /code/marv/marv/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import os 5 | import re 6 | import sys 7 | import time 8 | from datetime import datetime, timedelta 9 | from datetime import tzinfo as tzinfo_base 10 | from itertools import islice 11 | 12 | from marv_api.utils import NOTSET 13 | 14 | 15 | def chunked(iterable, chunk_size): 16 | itr = iter(iterable) 17 | return iter(lambda: tuple(islice(itr, chunk_size)), ()) 18 | 19 | 20 | def findfirst(predicate, iterable, default=NOTSET): 21 | try: 22 | return next(x for x in iterable if predicate(x)) 23 | except StopIteration: 24 | if default is not NOTSET: 25 | return default 26 | raise ValueError('No item matched predicate!') 27 | 28 | 29 | def mtime(path): 30 | """Wrap os.stat() st_mtime for ease of mocking.""" 31 | return os.stat(path).st_mtime 32 | 33 | 34 | def stat(path): 35 | """Wrap os.stat() for ease of mocking.""" # noqa: D402 36 | # TODO: https://github.com/PyCQA/pydocstyle/issues/284 37 | return os.stat(path) 38 | 39 | 40 | def walk(path): 41 | """Wrap os.walk() for ease of mocking.""" # noqa: D402 42 | # TODO: https://github.com/PyCQA/pydocstyle/issues/284 43 | return os.walk(path) 44 | 45 | 46 | def now(): 47 | """Wrap time.time() for ease of mocking.""" 48 | return time.time() 49 | 50 | 51 | def parse_filesize(string): 52 | val, unit = re.match(r'^\s*([0-9.]+)\s*([kmgtpezy]b?)?\s*$', string, re.I)\ 53 | .groups() 54 | val = float(val) 55 | if unit: 56 | val *= 1 << (10 * (1 + 'kmgtpezy'.index(unit.lower()[0]))) 57 | return int(val) 58 | 59 | 60 | def parse_datetime(string): 61 | 62 | class TZInfo(tzinfo_base): 63 | 64 | def __init__(self, offset=None): 65 | self.offset = offset 66 | 67 | def dst(self, dt): 68 | raise NotImplementedError() 69 | 70 | def tzname(self, dt): 71 | return self.offset 72 | 73 | def utcoffset(self, dt): 74 | if self.offset == 'Z': 75 | hours, minutes = 0, 0 76 | else: 77 | hours, minutes = self.offset[1:].split(':') 78 | offset = timedelta(hours=int(hours), minutes=int(minutes)) 79 | return offset if self.offset[0] == '+' else -offset 80 | 81 | groups = re.match( 82 | r'^(\d\d\d\d)-(\d\d)-(\d\d)T' 83 | r'(\d\d):(\d\d):(\d\d)((?:[+-]\d\d:\d\d)|Z)$', 84 | string, 85 | ).groups() 86 | tzinfo = TZInfo(groups[-1]) 87 | return datetime(*(int(x) for x in groups[:-1]), tzinfo=tzinfo) 88 | 89 | 90 | def parse_timedelta(delta): 91 | match = re.match(r'^\s*(?:(\d+)\s*h)?' r'\s*(?:(\d+)\s*m)?' r'\s*(?:(\d+)\s*s?)?\s*$', delta) 92 | h, m, s = match.groups() if match else (None, None, None) # pylint: disable=invalid-name 93 | return (int(h or 0) * 3600 + int(m or 0) * 60 + int(s or 0)) * 1000 94 | 95 | 96 | def profile(func, sort='cumtime'): 97 | # pylint: disable=import-outside-toplevel 98 | import functools 99 | import pstats 100 | from cProfile import Profile 101 | _profile = Profile() 102 | 103 | @functools.wraps(func) 104 | def profiled(*args, **kw): 105 | _profile.enable() 106 | result = func(*args, **kw) 107 | _profile.disable() 108 | stats = pstats.Stats(_profile).sort_stats(sort) 109 | stats.print_stats() 110 | return result # noqa: R504 111 | 112 | return profiled 113 | 114 | 115 | def underscore_to_camelCase(string): # noqa: N802 pylint: disable=invalid-name 116 | return ''.join(x.capitalize() for x in string.split('_')) 117 | 118 | 119 | def within_pyinstaller_bundle(): 120 | return any(x.endswith('base_library.zip') for x in sys.path) 121 | 122 | 123 | def within_staticx_bundle(): 124 | return bool(os.environ.get('STATICX_PROG_PATH')) 125 | -------------------------------------------------------------------------------- /code/marv/marv_node/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | -------------------------------------------------------------------------------- /code/marv/marv_node/event.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from collections import OrderedDict 5 | 6 | 7 | class DefaultOrderedDict(OrderedDict): 8 | 9 | def __init__(self, factory): 10 | super().__init__() 11 | self._factory = factory 12 | 13 | def __getitem__(self, key): 14 | try: 15 | return super().__getitem__(key) 16 | except KeyError: 17 | self[key] = self._factory() 18 | return self[key] 19 | -------------------------------------------------------------------------------- /code/marv/marv_node/io.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from collections import namedtuple 5 | from numbers import Integral 6 | 7 | from marv_api.iomsgs import CreateStream, GetRequested, MakeFile, Pull, PullAll, Push, SetHeader 8 | 9 | from .mixins import Keyed, Request, Task 10 | 11 | Fork = namedtuple('Fork', 'name inputs group') 12 | GetStream = namedtuple('GetStream', 'setid node name') 13 | 14 | # TODO: Rename 15 | Request.register(Pull) 16 | Request.register(PullAll) 17 | Request.register(Push) 18 | Request.register(SetHeader) 19 | 20 | Request.register(CreateStream) 21 | Request.register(Fork) 22 | Request.register(GetRequested) 23 | Request.register(GetStream) 24 | Request.register(MakeFile) 25 | 26 | 27 | class Signal(Task): # pylint: disable=too-few-public-methods 28 | 29 | def __repr__(self): 30 | return type(self).__name__.upper() 31 | 32 | 33 | class Next(Signal): # pylint: disable=too-few-public-methods 34 | """Instruct to send next pending task.""" 35 | 36 | __slots__ = () 37 | 38 | 39 | class Paused(Signal): # pylint: disable=too-few-public-methods 40 | """Indicate a generator has paused.""" 41 | 42 | __slots__ = () 43 | 44 | 45 | class Resume(Signal): # pylint: disable=too-few-public-methods 46 | """Instruct a generator to resume.""" 47 | 48 | __slots__ = () 49 | 50 | 51 | class TheEnd(Signal): # pylint: disable=too-few-public-methods 52 | """Indicate the end of a stream, resulting in None being sent into consumers.""" 53 | 54 | __slots__ = () 55 | 56 | 57 | NEXT = Next() 58 | PAUSED = Paused() 59 | RESUME = Resume() 60 | THEEND = TheEnd() 61 | 62 | 63 | class MsgRequest(Task, Keyed): 64 | __slots__ = ('_handle', '_idx', '__weakref__') 65 | 66 | @property 67 | def key(self): 68 | return (self._handle, self._idx) 69 | 70 | @property 71 | def handle(self): 72 | return self._handle 73 | 74 | @property 75 | def idx(self): 76 | return self._idx 77 | 78 | def __init__(self, handle, idx, requestor): 79 | assert isinstance(idx, Integral), idx 80 | self._handle = handle 81 | self._idx = idx 82 | self._requestor = requestor 83 | 84 | def __iter__(self): 85 | return iter(self.key) 86 | 87 | def __repr__(self): 88 | return f'MsgRequest({self._handle}, {self._idx!r})' 89 | -------------------------------------------------------------------------------- /code/marv/marv_node/mixins.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from abc import ABCMeta, abstractmethod 5 | from logging import getLogger 6 | 7 | 8 | class Keyed(metaclass=ABCMeta): 9 | 10 | @property 11 | @abstractmethod 12 | def key(self): 13 | return None # pragma: nocoverage 14 | 15 | def __lt__(self, other): 16 | if not isinstance(other, type(self)): 17 | return NotImplemented 18 | return self.key < other.key 19 | 20 | def __le__(self, other): 21 | if not isinstance(other, type(self)): 22 | return NotImplemented 23 | return self.key <= other.key 24 | 25 | def __eq__(self, other): 26 | if not isinstance(other, type(self)): 27 | return NotImplemented 28 | return self.key == other.key 29 | 30 | def __ne__(self, other): 31 | if not isinstance(other, type(self)): 32 | return NotImplemented 33 | return self.key != other.key 34 | 35 | def __ge__(self, other): 36 | if not isinstance(other, type(self)): 37 | return NotImplemented 38 | return self.key >= other.key 39 | 40 | def __gt__(self, other): 41 | if not isinstance(other, type(self)): 42 | return NotImplemented 43 | return self.key > other.key 44 | 45 | def __hash__(self): 46 | return hash((type(self), self.key)) 47 | 48 | def __repr__(self): 49 | return f'<{type(self).__name__} key={self.key!r}>' 50 | 51 | 52 | class AGenWrapperMixin: 53 | _agen = None 54 | 55 | @property 56 | def aclose(self): 57 | return self._agen.aclose 58 | 59 | @property 60 | def __anext__(self): 61 | return self._agen.__anext__ 62 | 63 | @property 64 | def asend(self): 65 | return self._agen.asend 66 | 67 | @property 68 | def athrow(self): 69 | return self._agen.athrow 70 | 71 | 72 | class GenWrapperMixin: 73 | _gen = None 74 | 75 | @property 76 | def close(self): 77 | return self._gen.close 78 | 79 | @property 80 | def __next__(self): 81 | return self._gen.__next__ 82 | 83 | @property 84 | def send(self): 85 | return self._gen.send 86 | 87 | @property 88 | def throw(self): 89 | return self._gen.throw 90 | 91 | 92 | class LoggerMixin: 93 | 94 | @property 95 | def logdebug(self): 96 | return self.log.debug 97 | 98 | @property 99 | def lognoisy(self): 100 | return self.log.noisy 101 | 102 | @property 103 | def logverbose(self): 104 | return self.log.verbose 105 | 106 | @property 107 | def loginfo(self): 108 | return self.log.info 109 | 110 | @property 111 | def logwarn(self): 112 | return self.log.warn 113 | 114 | @property 115 | def logerror(self): 116 | return self.log.error 117 | 118 | @property 119 | def log(self): 120 | logkey = ('marv', type(self).__name__.lower()) 121 | if hasattr(self, 'key'): 122 | logkey += ( 123 | (str(self.setid.abbrev), self.node.name, self.node.specs_hash[:10]) + 124 | tuple(str(x) for x in self.key[2:]) 125 | ) 126 | return getLogger('.'.join(logkey)) 127 | 128 | 129 | class Task(metaclass=ABCMeta): # noqa: SIM119 pylint: disable=too-few-public-methods 130 | __slots__ = () 131 | 132 | def __repr__(self): 133 | return type(self).__name__ 134 | 135 | 136 | class Request(metaclass=ABCMeta): # noqa: SIM119 pylint: disable=too-few-public-methods 137 | __slots__ = () 138 | 139 | def __repr__(self): 140 | return type(self).__name__ 141 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import os 5 | from importlib import import_module 6 | 7 | import marv_robotics 8 | 9 | for mod in (x for x in os.listdir(os.path.dirname(marv_robotics.__file__)) if x.endswith('.py')): 10 | mod = '.' + mod[:-3] 11 | import_module(mod, 'marv_robotics') 12 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/output/bagmeta_table.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_which": "table", 4 | "table": { 5 | "actions": [], 6 | "columns": [ 7 | { 8 | "align": "left", 9 | "formatter": "rellink", 10 | "list": false, 11 | "sortkey": "title", 12 | "title": "Name" 13 | }, 14 | { 15 | "align": "left", 16 | "formatter": "filesize", 17 | "list": false, 18 | "sortkey": "", 19 | "title": "Size" 20 | }, 21 | { 22 | "align": "left", 23 | "formatter": "datetime", 24 | "list": false, 25 | "sortkey": "", 26 | "title": "Start time" 27 | }, 28 | { 29 | "align": "left", 30 | "formatter": "datetime", 31 | "list": false, 32 | "sortkey": "", 33 | "title": "End time" 34 | }, 35 | { 36 | "align": "left", 37 | "formatter": "timedelta", 38 | "list": false, 39 | "sortkey": "", 40 | "title": "Duration" 41 | }, 42 | { 43 | "align": "right", 44 | "formatter": "string", 45 | "list": false, 46 | "sortkey": "", 47 | "title": "Message count" 48 | } 49 | ], 50 | "rows": [ 51 | { 52 | "cells": [ 53 | { 54 | "_which": "link", 55 | "link": { 56 | "download": "", 57 | "href": "0", 58 | "target": "_blank", 59 | "title": "test_0.bag" 60 | } 61 | }, 62 | { 63 | "_which": "uint64", 64 | "uint64": 17468 65 | }, 66 | { 67 | "_which": "timestamp", 68 | "timestamp": 1423137547254540178 69 | }, 70 | { 71 | "_which": "timestamp", 72 | "timestamp": 1423137548218460453 73 | }, 74 | { 75 | "_which": "timedelta", 76 | "timedelta": 963920275 77 | }, 78 | { 79 | "_which": "uint64", 80 | "uint64": 29 81 | } 82 | ], 83 | "id": 0 84 | }, 85 | { 86 | "cells": [ 87 | { 88 | "_which": "link", 89 | "link": { 90 | "download": "", 91 | "href": "1", 92 | "target": "_blank", 93 | "title": "test_1.bag" 94 | } 95 | }, 96 | { 97 | "_which": "uint64", 98 | "uint64": 23190 99 | }, 100 | { 101 | "_which": "timestamp", 102 | "timestamp": 1423137548265413068 103 | }, 104 | { 105 | "_which": "timestamp", 106 | "timestamp": 1423137550224936975 107 | }, 108 | { 109 | "_which": "timedelta", 110 | "timedelta": 1959523907 111 | }, 112 | { 113 | "_which": "uint64", 114 | "uint64": 65 115 | } 116 | ], 117 | "id": 1 118 | } 119 | ], 120 | "sortcolumn": 0, 121 | "sortorder": "ascending" 122 | }, 123 | "title": "" 124 | } 125 | ] -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/output/connections_section.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "title": "Connections", 4 | "widgets": [ 5 | { 6 | "_which": "table", 7 | "table": { 8 | "actions": [], 9 | "columns": [ 10 | { 11 | "align": "left", 12 | "formatter": "string", 13 | "list": false, 14 | "sortkey": "", 15 | "title": "Topic" 16 | }, 17 | { 18 | "align": "left", 19 | "formatter": "string", 20 | "list": false, 21 | "sortkey": "", 22 | "title": "Type" 23 | }, 24 | { 25 | "align": "left", 26 | "formatter": "string", 27 | "list": false, 28 | "sortkey": "", 29 | "title": "MD5" 30 | }, 31 | { 32 | "align": "left", 33 | "formatter": "string", 34 | "list": false, 35 | "sortkey": "", 36 | "title": "Latching" 37 | }, 38 | { 39 | "align": "right", 40 | "formatter": "string", 41 | "list": false, 42 | "sortkey": "", 43 | "title": "Message count" 44 | } 45 | ], 46 | "rows": [ 47 | { 48 | "cells": [ 49 | { 50 | "_which": "text", 51 | "text": "/chatter" 52 | }, 53 | { 54 | "_which": "text", 55 | "text": "std_msgs/msg/String" 56 | }, 57 | { 58 | "_which": "text", 59 | "text": "992ce8a1687cec8c8bd883ec73ca41d1" 60 | }, 61 | { 62 | "_which": "bool", 63 | "bool": false 64 | }, 65 | { 66 | "_which": "uint64", 67 | "uint64": 29 68 | } 69 | ], 70 | "id": 0 71 | }, 72 | { 73 | "cells": [ 74 | { 75 | "_which": "text", 76 | "text": "/rosout" 77 | }, 78 | { 79 | "_which": "text", 80 | "text": "rosgraph_msgs/msg/Log" 81 | }, 82 | { 83 | "_which": "text", 84 | "text": "acffd30cd6b6de30f120938c17c593fb" 85 | }, 86 | { 87 | "_which": "bool", 88 | "bool": true 89 | }, 90 | { 91 | "_which": "uint64", 92 | "uint64": 34 93 | } 94 | ], 95 | "id": 1 96 | }, 97 | { 98 | "cells": [ 99 | { 100 | "_which": "text", 101 | "text": "/rosout_agg" 102 | }, 103 | { 104 | "_which": "text", 105 | "text": "rosgraph_msgs/msg/Log" 106 | }, 107 | { 108 | "_which": "text", 109 | "text": "acffd30cd6b6de30f120938c17c593fb" 110 | }, 111 | { 112 | "_which": "bool", 113 | "bool": false 114 | }, 115 | { 116 | "_which": "uint64", 117 | "uint64": 31 118 | } 119 | ], 120 | "id": 2 121 | } 122 | ], 123 | "sortcolumn": 0, 124 | "sortorder": "ascending" 125 | }, 126 | "title": "" 127 | } 128 | ] 129 | } 130 | ] -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/output/fulltext.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "words": [ 4 | "1423137547.42", 5 | "1423137547.52", 6 | "1423137547.62", 7 | "1423137547.72", 8 | "1423137547.82", 9 | "1423137547.92", 10 | "1423137548.02", 11 | "1423137548.12", 12 | "1423137548.22", 13 | "1423137548.32", 14 | "1423137548.42", 15 | "1423137548.52", 16 | "1423137548.62", 17 | "1423137548.72", 18 | "1423137548.82", 19 | "1423137548.92", 20 | "1423137549.02", 21 | "1423137549.12", 22 | "1423137549.22", 23 | "1423137549.32", 24 | "1423137549.42", 25 | "1423137549.52", 26 | "1423137549.62", 27 | "1423137549.72", 28 | "1423137549.82", 29 | "1423137549.92", 30 | "1423137550.02", 31 | "1423137550.12", 32 | "1423137550.22", 33 | "58", 34 | "59", 35 | "60", 36 | "61", 37 | "62", 38 | "63", 39 | "64", 40 | "65", 41 | "66", 42 | "67", 43 | "68", 44 | "69", 45 | "70", 46 | "71", 47 | "72", 48 | "73", 49 | "74", 50 | "75", 51 | "76", 52 | "77", 53 | "78", 54 | "79", 55 | "80", 56 | "81", 57 | "82", 58 | "83", 59 | "84", 60 | "85", 61 | "86", 62 | "hello", 63 | "world" 64 | ] 65 | } 66 | ] -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/output/gnss_section.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "title": "Position and Orientation", 4 | "widgets": [ 5 | { 6 | "_which": "image", 7 | "image": { 8 | "src": "gnss_plots-1/gps:fix__gps:orientation.jpg" 9 | }, 10 | "title": "/gps/fix with /gps/orientation" 11 | } 12 | ] 13 | } 14 | ] -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/output/images_section.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "title": "Images", 4 | "widgets": [ 5 | { 6 | "_which": "gallery", 7 | "gallery": { 8 | "images": [ 9 | { 10 | "src": "images-1/camB:jai:nir:image_raw-0.jpg" 11 | }, 12 | { 13 | "src": "images-1/camB:jai:nir:image_raw-1.jpg" 14 | } 15 | ] 16 | }, 17 | "title": "/camB/jai/nir/image_raw" 18 | }, 19 | { 20 | "_which": "gallery", 21 | "gallery": { 22 | "images": [ 23 | { 24 | "src": "images-1/camB:jai:rgb:image_raw-0.jpg" 25 | }, 26 | { 27 | "src": "images-1/camB:jai:rgb:image_raw-1.jpg" 28 | } 29 | ] 30 | }, 31 | "title": "/camB/jai/rgb/image_raw" 32 | } 33 | ] 34 | } 35 | ] -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/output/summary_keyval.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_which": "keyval", 4 | "keyval": { 5 | "items": [ 6 | { 7 | "cell": { 8 | "_which": "uint64", 9 | "uint64": 40658 10 | }, 11 | "formatter": "filesize", 12 | "list": false, 13 | "title": "size" 14 | }, 15 | { 16 | "cell": { 17 | "_which": "uint64", 18 | "uint64": 2 19 | }, 20 | "formatter": "string", 21 | "list": false, 22 | "title": "files" 23 | }, 24 | { 25 | "cell": { 26 | "_which": "timestamp", 27 | "timestamp": 1423137547254540178 28 | }, 29 | "formatter": "datetime", 30 | "list": false, 31 | "title": "start time" 32 | }, 33 | { 34 | "cell": { 35 | "_which": "timestamp", 36 | "timestamp": 1423137550224936975 37 | }, 38 | "formatter": "datetime", 39 | "list": false, 40 | "title": "end time" 41 | }, 42 | { 43 | "cell": { 44 | "_which": "timedelta", 45 | "timedelta": 2970396797 46 | }, 47 | "formatter": "timedelta", 48 | "list": false, 49 | "title": "duration" 50 | } 51 | ] 52 | }, 53 | "title": "" 54 | } 55 | ] -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/output/trajectory_section.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "title": "Trajectory", 4 | "widgets": [ 5 | { 6 | "_which": "mapPartial", 7 | "mapPartial": "marv-partial:trajectory_section-1/data.json", 8 | "title": "" 9 | } 10 | ] 11 | } 12 | ] -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/output/video_section.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "title": "Videos", 4 | "widgets": [ 5 | { 6 | "_which": "video", 7 | "title": "/camB/jai/nir/image_raw", 8 | "video": { 9 | "src": "ffmpeg-1/camB_jai_nir_image_raw.webm" 10 | } 11 | }, 12 | { 13 | "_which": "video", 14 | "title": "/camB/jai/rgb/image_raw", 15 | "video": { 16 | "src": "ffmpeg-1/camB_jai_rgb_image_raw.webm" 17 | } 18 | } 19 | ] 20 | } 21 | ] -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_bag.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.bag import bagmeta as node 9 | from marv_store import Store 10 | 11 | # TODO: in what form do we need this test? 12 | 13 | 14 | class TestCase(marv_node.testing.TestCase): 15 | # TODO: Generate bags instead, but with connection info! 16 | BAGS = [ 17 | resource_filename('marv_node.testing._robotics_tests', 'data/test_0.bag'), 18 | resource_filename('marv_node.testing._robotics_tests', 'data/test_1.bag'), 19 | ] 20 | 21 | async def test_node(self): 22 | with temporary_directory() as storedir: 23 | store = Store(storedir, {}) 24 | dataset = make_dataset(self.BAGS) 25 | store.add_dataset(dataset) 26 | streams = await run_nodes(dataset, [node], store) 27 | self.assertNodeOutput(streams[0], node) 28 | # TODO: test also header 29 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_fulltext.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.fulltext import fulltext as node 9 | from marv_store import Store 10 | 11 | 12 | class TestCase(marv_node.testing.TestCase): 13 | # TODO: Generate bags instead, but with connection info! 14 | BAGS = [ 15 | resource_filename('marv_node.testing._robotics_tests', 'data/test_0.bag'), 16 | resource_filename('marv_node.testing._robotics_tests', 'data/test_1.bag'), 17 | ] 18 | 19 | async def test_node(self): 20 | with temporary_directory() as storedir: 21 | store = Store(storedir, {}) 22 | dataset = make_dataset(self.BAGS) 23 | store.add_dataset(dataset) 24 | streams = await run_nodes(dataset, [node], store) 25 | self.assertNodeOutput(streams[0], node) 26 | # TODO: test also header 27 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_gnss_section.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.detail import gnss_section as node 9 | from marv_robotics.gnss import gnss_plots 10 | from marv_store import Store 11 | 12 | PERSIST = {'gnss_plots': gnss_plots} 13 | 14 | 15 | class TestCase(marv_node.testing.TestCase): 16 | # TODO: Generate bags instead, but with connection info! 17 | BAGS = [resource_filename('marv_node.testing._robotics_tests', 'data/cam.bag')] 18 | 19 | async def test_node(self): 20 | with temporary_directory() as storedir: 21 | store = Store(storedir, PERSIST) 22 | dataset = make_dataset(self.BAGS) 23 | store.add_dataset(dataset) 24 | streams = await run_nodes(dataset, [node], store, PERSIST) 25 | self.assertNodeOutput(streams[0], node) 26 | # TODO: test also header 27 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_group_and_topic.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_api as marv 7 | import marv_node.testing 8 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 9 | from marv_robotics.bag import make_deserialize, messages 10 | from marv_robotics.fulltext import fulltext 11 | from marv_store import Store 12 | 13 | 14 | @marv.node() 15 | @marv.input('chatter', default=marv.select(messages, '/chatter')) 16 | def collect(chatter): 17 | deserialize = make_deserialize(chatter) 18 | msg = yield marv.pull(chatter) 19 | assert msg is not None 20 | rosmsg = deserialize(msg.data) 21 | yield marv.push(rosmsg.data) 22 | while True: 23 | msg = yield marv.pull(chatter) 24 | if msg is None: 25 | return 26 | rosmsg = deserialize(msg.data) 27 | yield marv.push(rosmsg.data) 28 | 29 | 30 | class TestCase(marv_node.testing.TestCase): 31 | # TODO: Generate bags instead, but with connection info! 32 | BAGS = [ 33 | resource_filename('marv_node.testing._robotics_tests', 'data/test_0.bag'), 34 | resource_filename('marv_node.testing._robotics_tests', 'data/test_1.bag'), 35 | ] 36 | 37 | async def test_node(self): 38 | with temporary_directory() as storedir: 39 | store = Store(storedir, {}) 40 | dataset = make_dataset(self.BAGS) 41 | store.add_dataset(dataset) 42 | streams = await run_nodes(dataset, [fulltext, collect], store) 43 | assert 'hello' in streams[0][0].words 44 | assert any('hello' in x for x in streams[1]) 45 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_non_existing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.detail import images_section as node 9 | from marv_store import Store 10 | 11 | 12 | class TestCase(marv_node.testing.TestCase): 13 | # TODO: Generate bags instead, but with connection info! 14 | BAGS = [resource_filename('marv_node.testing._robotics_tests', 'data/test_0.bag')] 15 | 16 | async def test_node(self): 17 | with temporary_directory() as storedir: 18 | store = Store(storedir, {}) 19 | dataset = make_dataset(self.BAGS) 20 | store.add_dataset(dataset) 21 | streams = await run_nodes(dataset, [node], store) 22 | assert streams == [None] 23 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_optional_input.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_api as marv 7 | import marv_node.testing 8 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 9 | from marv_robotics.bag import messages 10 | from marv_store import Store 11 | 12 | 13 | @marv.node() 14 | @marv.input('stream', default=marv.select(messages, '/non-existent')) 15 | def nooutput(stream): 16 | yield marv.set_header() 17 | while True: 18 | msg = yield marv.pull(stream) 19 | if msg is None: 20 | return 21 | yield marv.push(msg) 22 | 23 | 24 | @marv.node() 25 | @marv.input('nooutput', default=nooutput) 26 | @marv.input('chatter', default=marv.select(messages, '/chatter')) 27 | def collect(nooutput, chatter): # pylint: disable=redefined-outer-name 28 | msg = yield marv.pull(nooutput) 29 | assert msg is None 30 | msg = yield marv.pull(chatter) 31 | assert msg is not None 32 | yield marv.push('Success') 33 | 34 | 35 | class TestCase(marv_node.testing.TestCase): 36 | # TODO: Generate bags instead, but with connection info! 37 | BAGS = [resource_filename('marv_node.testing._robotics_tests', 'data/test_0.bag')] 38 | 39 | async def test_node(self): 40 | with temporary_directory() as storedir: 41 | store = Store(storedir, {}) 42 | dataset = make_dataset(self.BAGS) 43 | store.add_dataset(dataset) 44 | streams = await run_nodes(dataset, [collect], store) 45 | assert streams == [['Success']] 46 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_section_images.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.cam import images 9 | from marv_robotics.detail import images_section as node 10 | from marv_store import Store 11 | 12 | PERSIST = {'images': images} 13 | 14 | 15 | class TestCase(marv_node.testing.TestCase): 16 | # TODO: Generate bags instead, but with connection info! 17 | BAGS = [resource_filename('marv_node.testing._robotics_tests', 'data/cam.bag')] 18 | 19 | async def test_node(self): 20 | with temporary_directory() as storedir: 21 | store = Store(storedir, PERSIST) 22 | dataset = make_dataset(self.BAGS) 23 | store.add_dataset(dataset) 24 | streams = await run_nodes(dataset, [node], store, PERSIST) 25 | self.assertNodeOutput(streams[0], node) 26 | # TODO: test also header 27 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_section_topics.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.detail import connections_section as node 9 | from marv_store import Store 10 | 11 | 12 | class TestCase(marv_node.testing.TestCase): 13 | # TODO: Generate bags instead, but with connection info! 14 | BAGS = [ 15 | resource_filename('marv_node.testing._robotics_tests', 'data/test_0.bag'), 16 | resource_filename('marv_node.testing._robotics_tests', 'data/test_1.bag'), 17 | ] 18 | 19 | async def test_node(self): 20 | with temporary_directory() as storedir: 21 | store = Store(storedir, {}) 22 | dataset = make_dataset(self.BAGS) 23 | store.add_dataset(dataset) 24 | streams = await run_nodes(dataset, [node], store) 25 | self.assertNodeOutput(streams[0], node) 26 | # TODO: test also header 27 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_section_videos.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.cam import ffmpeg 9 | from marv_robotics.detail import video_section as node 10 | from marv_store import Store 11 | 12 | PERSIST = {'ffmpeg': ffmpeg} 13 | 14 | 15 | class TestCase(marv_node.testing.TestCase): 16 | # TODO: Generate bags instead, but with connection info! 17 | BAGS = [resource_filename('marv_node.testing._robotics_tests', 'data/cam.bag')] 18 | 19 | async def test_node(self): 20 | with temporary_directory() as storedir: 21 | store = Store(storedir, PERSIST) 22 | dataset = make_dataset(self.BAGS) 23 | store.add_dataset(dataset) 24 | streams = await run_nodes(dataset, [node], store, PERSIST) 25 | self.assertNodeOutput(streams[0], node) 26 | # TODO: test also header 27 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_trajectory_section.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.detail import trajectory_section as node 9 | from marv_robotics.trajectory import trajectory 10 | from marv_store import Store 11 | 12 | PERSIST = {'trajectory': trajectory, 'trajectory_section': node} 13 | 14 | 15 | class TestCase(marv_node.testing.TestCase): 16 | # TODO: Generate bags instead, but with connection info! 17 | BAGS = [resource_filename('marv_node.testing._robotics_tests', 'data/navsatfix.bag')] 18 | 19 | async def test_node(self): 20 | with temporary_directory() as storedir: 21 | store = Store(storedir, PERSIST) 22 | dataset = make_dataset(self.BAGS) 23 | store.add_dataset(dataset) 24 | streams = await run_nodes(dataset, [node], store, PERSIST) 25 | self.assertNodeOutput(streams[0], node) 26 | # TODO: test also header 27 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_widget_bagmeta_table.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.detail import bagmeta_table as node 9 | from marv_store import Store 10 | 11 | 12 | class TestCase(marv_node.testing.TestCase): 13 | # TODO: Generate bags instead, but with connection info! 14 | BAGS = [ 15 | resource_filename('marv_node.testing._robotics_tests', 'data/test_0.bag'), 16 | resource_filename('marv_node.testing._robotics_tests', 'data/test_1.bag'), 17 | ] 18 | 19 | async def test_node(self): 20 | with temporary_directory() as storedir: 21 | store = Store(storedir, {}) 22 | dataset = make_dataset(self.BAGS) 23 | store.add_dataset(dataset) 24 | streams = await run_nodes(dataset, [node], store) 25 | self.assertNodeOutput(streams[0], node) 26 | # TODO: test also header 27 | -------------------------------------------------------------------------------- /code/marv/marv_node/testing/_robotics_tests/test_widget_summary_keyval.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pkg_resources import resource_filename 5 | 6 | import marv_node.testing 7 | from marv_node.testing import make_dataset, run_nodes, temporary_directory 8 | from marv_robotics.detail import summary_keyval as node 9 | from marv_store import Store 10 | 11 | 12 | class TestCase(marv_node.testing.TestCase): 13 | # TODO: Generate bags instead, but with connection info! 14 | BAGS = [ 15 | resource_filename('marv_node.testing._robotics_tests', 'data/test_0.bag'), 16 | resource_filename('marv_node.testing._robotics_tests', 'data/test_1.bag'), 17 | ] 18 | 19 | async def test_node(self): 20 | with temporary_directory() as storedir: 21 | store = Store(storedir, {}) 22 | dataset = make_dataset(self.BAGS) 23 | store.add_dataset(dataset) 24 | streams = await run_nodes(dataset, [node], store) 25 | self.assertNodeOutput(streams[0], node) 26 | # TODO: test also header 27 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_edge_cases.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=invalid-name 5 | 6 | from ..testing import make_dataset, marv, run_nodes 7 | 8 | DATASET = make_dataset() 9 | SETID = DATASET.setid 10 | 11 | 12 | @marv.node(group=True) 13 | def source(): 14 | a = yield marv.create_stream('a') 15 | b = yield marv.create_stream('b') 16 | yield a.msg(0) 17 | yield b.msg(10) 18 | 19 | 20 | @marv.node() 21 | @marv.input('stream', foreach=source) 22 | def images(stream): 23 | """Produce 2 streams each with 5 messages.""" 24 | offset = yield marv.pull(stream) 25 | yield marv.set_header(title=offset) 26 | yield marv.push(1 + offset) 27 | yield marv.push(2 + offset) 28 | yield marv.push(3 + offset) 29 | yield marv.push(4 + offset) 30 | yield marv.push(5 + offset) 31 | 32 | 33 | @marv.node() 34 | @marv.input('stream', foreach=images) 35 | def galleries(stream): 36 | """Consume each stream into a list.""" 37 | yield marv.set_header(title=stream.title) # TODO: This is currently needed 38 | _images = [] 39 | while True: 40 | img = yield marv.pull(stream) 41 | if img is None: 42 | break 43 | _images.append(img) 44 | yield marv.push({'images': _images}) 45 | 46 | 47 | @marv.node() 48 | @marv.input('galleries', default=galleries) 49 | def images_section(galleries): # pylint: disable=redefined-outer-name 50 | """Consume both galleries into a list.""" 51 | tmp = [] 52 | while True: 53 | msg = yield marv.pull(galleries) 54 | if msg is None: 55 | break 56 | tmp.append(msg) 57 | galleries = tmp 58 | galleries = yield marv.pull_all(*galleries) 59 | yield marv.push({'galleries': galleries}) 60 | 61 | 62 | async def test_foreach_cascade(): 63 | nodes = [images_section] 64 | streams = await run_nodes(DATASET, nodes) 65 | assert streams == [ 66 | [{ 67 | 'galleries': [ 68 | { 69 | 'images': [1, 2, 3, 4, 5], 70 | }, 71 | { 72 | 'images': [11, 12, 13, 14, 15], 73 | }, 74 | ], 75 | }], 76 | ] 77 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_node.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=blacklisted-name,invalid-name 5 | 6 | import unittest 7 | 8 | from ..node import Node 9 | from ..testing import marv 10 | 11 | 12 | @marv.node() 13 | @marv.input('a', default=1) 14 | def a_orig(a): # pylint: disable=unused-argument 15 | yield 16 | 17 | 18 | @marv.node() 19 | @marv.input('a', default=1) 20 | def b_orig(a): # pylint: disable=unused-argument 21 | yield 22 | 23 | 24 | class TestCase(unittest.TestCase): 25 | 26 | def test_node_repr(self): # pylint: disable=no-self-use 27 | 28 | @marv.node() 29 | def foo(): 30 | yield 31 | 32 | foo() 33 | assert repr(Node.from_dag_node(foo)) == '' 34 | 35 | def test_comparisons(self): # pylint: disable=no-self-use 36 | # pylint: disable=comparison-with-itself 37 | a = Node.from_dag_node(a_orig) 38 | b = Node.from_dag_node(b_orig) 39 | assert type(a) is type(b) 40 | assert isinstance(a, Node) 41 | assert a is a 42 | assert a is not b 43 | 44 | assert a.key < b.key 45 | assert a < b 46 | assert a <= b 47 | assert a <= a 48 | assert a == a 49 | assert a != b 50 | assert b > a 51 | assert b >= a 52 | assert b >= b 53 | 54 | assert a == Node.from_dag_node(a_orig.clone()) 55 | assert a != Node.from_dag_node(a_orig.clone(a=2)) 56 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_optional_input.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=invalid-name,redefined-outer-name 5 | 6 | from ..testing import make_dataset, marv, run_nodes 7 | 8 | DATASET = make_dataset() 9 | SETID = DATASET.setid 10 | 11 | 12 | @marv.node() 13 | def meta(): 14 | """Meta information about a dataset.""" 15 | yield marv.push({'topics': ['b', 'c']}) 16 | 17 | 18 | @marv.node(group='ondemand') 19 | @marv.input('meta', default=meta) 20 | def messages(meta): 21 | """Produce streams for requested topics if they exist.""" 22 | meta = yield marv.pull(meta) 23 | requested = yield marv.get_requested() 24 | all_msgs = {'a': list(range(1)), 'b': list(range(2)), 'c': list(range(3))} 25 | topics = [x.name for x in requested] 26 | streams = {} 27 | for topic in topics: 28 | stream = yield marv.create_stream(topic) 29 | if topic in meta['topics']: 30 | streams[topic] = stream 31 | else: 32 | yield stream.finish() 33 | 34 | while streams: 35 | for topic, stream in list(streams.items()): 36 | msgs = all_msgs[topic] 37 | msg = msgs.pop(0) 38 | yield stream.msg(f'{topic}{msg}') 39 | if not msgs: 40 | del streams[topic] 41 | 42 | 43 | @marv.node() 44 | @marv.input('a', default=marv.select(messages, 'a')) 45 | def node_a(a): 46 | yield marv.set_header(topic='a') 47 | while True: 48 | msg = yield marv.pull(a) 49 | if msg is None: 50 | return 51 | yield marv.push(f'node_a-{msg}') 52 | 53 | 54 | @marv.node() 55 | @marv.input('b', default=marv.select(messages, 'b')) 56 | def node_b(b): 57 | yield marv.set_header(topic='b') 58 | while True: 59 | msg = yield marv.pull(b) 60 | if msg is None: 61 | return 62 | yield marv.push(f'node_b-{msg}') 63 | 64 | 65 | @marv.node() 66 | @marv.input('c', default=marv.select(messages, 'c')) 67 | def node_c(c): 68 | yield marv.set_header(topic='c') 69 | while True: 70 | msg = yield marv.pull(c) 71 | if msg is None: 72 | return 73 | yield marv.push(f'node_c-{msg}') 74 | 75 | 76 | @marv.node() 77 | @marv.input('node_a', default=node_a) 78 | @marv.input('node_b', default=node_b) 79 | @marv.input('node_c', default=node_c) 80 | def collect(node_a, node_b, node_c): 81 | yield marv.set_header() 82 | acc = [] 83 | streams = [node_a, node_b, node_c] 84 | while streams: 85 | for stream in streams[:]: 86 | msg = yield marv.pull(stream) 87 | if msg is None: 88 | streams.remove(stream) 89 | continue 90 | acc.append(msg) 91 | yield marv.push({'acc': acc}) 92 | 93 | 94 | async def test(): 95 | nodes = [collect] 96 | streams = await run_nodes(DATASET, nodes) 97 | assert streams == [ 98 | [{ 99 | 'acc': ['node_b-b0', 'node_c-c0', 'node_b-b1', 'node_c-c1', 'node_c-c2'], 100 | }], 101 | ] 102 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_push_false_values.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=invalid-name 5 | 6 | import pytest 7 | 8 | from ..testing import make_dataset, marv, run_nodes 9 | 10 | 11 | class Falsish: 12 | 13 | def __bool__(self): 14 | return False 15 | 16 | def __eq__(self, other): 17 | return type(other) is type(self) 18 | 19 | def __repr__(self): 20 | return '' 21 | 22 | 23 | FALSISH = Falsish() 24 | 25 | 26 | class DontBoolMe: 27 | 28 | def __bool__(self): 29 | raise Exception 30 | 31 | def __eq__(self, other): 32 | return type(other) is type(self) 33 | 34 | def __repr__(self): 35 | return '' 36 | 37 | 38 | DONTBOOLME = DontBoolMe() 39 | 40 | 41 | @marv.node() 42 | def source(): 43 | # yield marv.push(None) -- So far, this means we're done 44 | yield marv.push(0) 45 | yield marv.push(0.0) 46 | yield marv.push(False) 47 | yield marv.push('') 48 | yield marv.push(FALSISH) 49 | yield marv.push(DONTBOOLME) 50 | 51 | 52 | @marv.node() 53 | @marv.input('stream', default=source) 54 | def consumer1(stream): 55 | yield marv.push(42) 56 | while True: 57 | msg = yield marv.pull(stream) 58 | if msg is None: 59 | break 60 | yield marv.push(msg) 61 | 62 | 63 | @marv.node() 64 | @marv.input('stream', default=source) 65 | def consumer2(stream): 66 | yield marv.push(42) 67 | while True: 68 | msg, = yield marv.pull_all(stream) 69 | if msg is None: 70 | break 71 | yield marv.push(msg) 72 | 73 | 74 | DATASET = make_dataset() 75 | 76 | 77 | async def test(): 78 | with pytest.raises(Exception, match='^$'): 79 | bool(DONTBOOLME) 80 | 81 | nodes = [source, consumer1, consumer2] 82 | streams = await run_nodes(DATASET, nodes) 83 | assert streams == [ 84 | [0, 0.0, False, '', FALSISH, DONTBOOLME], 85 | [42, 0, 0.0, False, '', FALSISH, DONTBOOLME], 86 | [42, 0, 0.0, False, '', FALSISH, DONTBOOLME], 87 | ] 88 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=invalid-name 5 | 6 | from ..testing import make_dataset, marv, run_nodes 7 | 8 | 9 | @marv.node() 10 | @marv.input('offset', default=0) 11 | def root(offset): 12 | yield marv.push(10 + offset) 13 | yield marv.push(20 + offset) 14 | 15 | 16 | @marv.node() 17 | @marv.input('stream_a', default=root) 18 | def square(stream_a): 19 | while True: 20 | a = yield marv.pull(stream_a) 21 | if a is None: 22 | break 23 | yield marv.push(a**2) 24 | 25 | 26 | @marv.node() 27 | @marv.input('stream_a', default=root) 28 | @marv.input('stream_b', default=root.clone(offset=5)) 29 | @marv.input('stream_c', default=square) 30 | def add(stream_a, stream_b, stream_c): 31 | while True: 32 | a = yield marv.pull(stream_a) 33 | b = yield marv.pull(stream_b) 34 | c = yield marv.pull(stream_c) 35 | if a is None: 36 | break 37 | yield marv.push(a + b + c) 38 | 39 | 40 | DATASET = make_dataset() 41 | 42 | 43 | async def test(): 44 | nodes = [root, square, add] 45 | streams = await run_nodes(DATASET, nodes) 46 | assert streams == [ 47 | [10, 20], 48 | [100, 400], 49 | [125, 445], 50 | ] 51 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_combined.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | # pylint: disable=invalid-name 5 | 6 | from ..testing import make_dataset, marv, run_nodes 7 | 8 | 9 | @marv.node() 10 | @marv.input('offset', default=0) 11 | def root(offset): 12 | yield marv.push(10 + offset) 13 | yield marv.push(20 + offset) 14 | 15 | 16 | @marv.node() 17 | @marv.input('stream_a', default=root) 18 | def square(stream_a): 19 | while True: 20 | a = yield marv.pull(stream_a) 21 | if a is None: 22 | break 23 | yield marv.push(a**2) 24 | 25 | 26 | @marv.node() 27 | @marv.input('stream_a', default=root) 28 | @marv.input('stream_b', default=root.clone(offset=5)) 29 | @marv.input('stream_c', default=square) 30 | def add(stream_a, stream_b, stream_c): 31 | while True: 32 | a, b, c = yield marv.pull_all(stream_a, stream_b, stream_c) 33 | if a is None: 34 | break 35 | yield marv.push(a + b + c) 36 | 37 | 38 | DATASET = make_dataset() 39 | 40 | 41 | async def test(): 42 | nodes = [root, square, add] 43 | streams = await run_nodes(DATASET, nodes) 44 | assert streams == [ 45 | [10, 20], 46 | [100, 400], 47 | [125, 445], 48 | ] 49 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_create_stream.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import logging 5 | 6 | from testfixtures import LogCapture 7 | 8 | from ..testing import make_dataset, marv, run_nodes 9 | 10 | DATASET = make_dataset() 11 | 12 | 13 | @marv.node(group=True) 14 | def source(): 15 | yield marv.create_stream('Output 1') 16 | yield marv.create_stream('Output 2', foo=1) 17 | 18 | 19 | @marv.node() 20 | @marv.input('source', source) 21 | def consumer(source): # pylint: disable=redefined-outer-name 22 | logger = yield marv.get_logger() 23 | stream = yield marv.pull(source) 24 | logger.critical((stream.name, stream.header)) 25 | stream = yield marv.pull(source) 26 | logger.critical((stream.name, stream.header)) 27 | 28 | 29 | async def test(): 30 | with LogCapture(level=logging.CRITICAL) as log: 31 | await run_nodes(DATASET, [consumer]) 32 | 33 | assert [x.msg for x in log.records if x.msg] == [ 34 | ('Output 1', {}), 35 | ('Output 2', { 36 | 'foo': 1, 37 | }), 38 | ] 39 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_foreach.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import logging 5 | 6 | from testfixtures import LogCapture 7 | 8 | from ..testing import make_dataset, marv, run_nodes 9 | 10 | DATASET = make_dataset() 11 | SETID = DATASET.setid 12 | 13 | 14 | @marv.node(group=True) 15 | def source(): 16 | out1 = yield marv.create_stream('Output 1', foo=1) 17 | out2 = yield marv.create_stream('Output 2', foo=2) 18 | out3 = yield marv.create_stream('Output 3', foo=3) 19 | yield out1.msg(a=1) 20 | yield out2.msg(b=10) 21 | yield out3.msg(c=100) 22 | yield out1.msg(a=2) 23 | yield out2.msg(b=20) 24 | yield out3.msg(c=200) 25 | 26 | 27 | @marv.node() 28 | @marv.input('stream', foreach=source) 29 | def foreach(stream): 30 | logger = yield marv.get_logger() 31 | while True: 32 | msg = yield marv.pull(stream) 33 | if msg is None: 34 | break 35 | logger.critical((0, stream.name, msg)) 36 | 37 | 38 | @marv.node() 39 | @marv.input('node', default=source) 40 | def withall(node): 41 | logger = yield marv.get_logger() 42 | streams = [] 43 | while True: 44 | stream = yield marv.pull(node) 45 | if stream is None: 46 | break 47 | streams.append(stream) 48 | 49 | while streams: 50 | msgs = yield marv.pull_all(*streams) 51 | logger.critical((1, msgs)) 52 | streams = [stream for stream, msg in zip(streams, msgs) if msg is not None] 53 | 54 | 55 | async def test(): 56 | with LogCapture(level=logging.CRITICAL) as log: 57 | await run_nodes(DATASET, [foreach, withall]) 58 | 59 | assert [x.msg for x in log.records if x.msg[0] == 0] == [ 60 | (0, 'Output 1', { 61 | 'a': 1, 62 | }), 63 | (0, 'Output 2', { 64 | 'b': 10, 65 | }), 66 | (0, 'Output 3', { 67 | 'c': 100, 68 | }), 69 | (0, 'Output 1', { 70 | 'a': 2, 71 | }), 72 | (0, 'Output 2', { 73 | 'b': 20, 74 | }), 75 | (0, 'Output 3', { 76 | 'c': 200, 77 | }), 78 | ] 79 | assert [x.msg for x in log.records if x.msg[0] == 1] == [ 80 | (1, [{ 81 | 'a': 1, 82 | }, { 83 | 'b': 10, 84 | }, { 85 | 'c': 100, 86 | }]), 87 | (1, [{ 88 | 'a': 2, 89 | }, { 90 | 'b': 20, 91 | }, { 92 | 'c': 200, 93 | }]), 94 | (1, [None, None, None]), 95 | ] 96 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_foreach_with_header.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import logging 5 | 6 | from testfixtures import LogCapture 7 | 8 | from ..testing import make_dataset, marv, run_nodes 9 | 10 | DATASET = make_dataset() 11 | SETID = DATASET.setid 12 | 13 | 14 | @marv.node(group=True) 15 | def source(): 16 | out1 = yield marv.create_stream('Output 1', foo=1) 17 | yield out1.msg(a=1) 18 | yield out1.msg(a=2) 19 | 20 | 21 | @marv.node() 22 | @marv.input('stream', foreach=source) 23 | def foreach(stream): 24 | yield marv.set_header(**stream.header) 25 | while True: 26 | msg = yield marv.pull(stream) 27 | if msg is None: 28 | break 29 | msg = yield marv.push(msg) 30 | 31 | 32 | @marv.node() 33 | @marv.input('foreaches', default=foreach) 34 | def consumer(foreaches): 35 | logger = yield marv.get_logger() 36 | while True: 37 | stream = yield marv.pull(foreaches) 38 | if stream is None: 39 | break 40 | logger.critical((stream.name, stream.header)) 41 | while True: 42 | msg = yield marv.pull(stream) 43 | if msg is None: 44 | break 45 | logger.critical(msg) 46 | 47 | 48 | async def test(): 49 | with LogCapture(level=logging.CRITICAL) as log: 50 | await run_nodes(DATASET, [consumer]) 51 | 52 | assert [x.msg for x in log.records if x.msg] == [ 53 | ('0', { 54 | 'foo': 1, 55 | }), 56 | { 57 | 'a': 1, 58 | }, 59 | { 60 | 'a': 2, 61 | }, 62 | ] 63 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_ondemand_group.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from ..testing import make_dataset, marv, run_nodes 5 | 6 | DATASET = make_dataset() 7 | SETID = DATASET.setid 8 | 9 | 10 | @marv.node(group='ondemand') 11 | def source(): 12 | requested = yield marv.get_requested() 13 | assert {x.name for x in requested} == {'evensub', 'oddsub', 'primesub'} 14 | 15 | out = {x.name: marv.create_stream(x.name) for x in requested} 16 | for key, stream in out.items(): 17 | out[key] = yield stream 18 | assert out.keys() == {'evensub', 'oddsub', 'primesub'} 19 | 20 | for idx in range(1, 6): 21 | yield marv.push(out['oddsub' if idx % 2 else 'evensub'].msg(idx)) 22 | if idx in [2, 3, 5]: 23 | yield marv.push(out['primesub'].msg(idx)) 24 | 25 | # How do we output one message for multiple streams? 26 | 27 | 28 | @marv.node() 29 | @marv.input('stream', default=marv.select(source, 'evensub')) 30 | def even(stream): 31 | while True: 32 | msg = yield marv.pull(stream) 33 | if msg is None: 34 | break 35 | msg = yield marv.push(msg) 36 | 37 | 38 | @marv.node() 39 | @marv.input('stream', default=marv.select(source, 'oddsub')) 40 | def odd(stream): 41 | while True: 42 | msg = yield marv.pull(stream) 43 | if msg is None: 44 | break 45 | msg = yield marv.push(msg) 46 | 47 | 48 | @marv.node() 49 | @marv.input('stream', default=marv.select(source, 'primesub')) 50 | def prime(stream): 51 | while True: 52 | msg = yield marv.pull(stream) 53 | if msg is None: 54 | break 55 | msg = yield marv.push(msg) 56 | 57 | 58 | async def test(): 59 | nodes = [even, odd, prime] 60 | streams = await run_nodes(DATASET, nodes) 61 | assert streams == [ 62 | [2, 4], 63 | [1, 3, 5], 64 | [2, 3, 5], 65 | ] 66 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_ondemand_group_with_restart.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import logging 5 | from itertools import product 6 | 7 | from testfixtures import LogCapture 8 | 9 | from ..testing import make_dataset, marv, run_nodes 10 | 11 | DATASET = make_dataset() 12 | SETID = DATASET.setid 13 | 14 | 15 | @marv.node(group='ondemand') 16 | def source(): 17 | logger = yield marv.get_logger() 18 | requested = yield marv.get_requested() 19 | logger.critical([x.name for x in requested]) 20 | creates = [marv.create_stream(x.name) for x in requested] 21 | streams = [] 22 | for create in creates: 23 | stream = yield create 24 | streams.append(stream) 25 | 26 | msgs = list(product(streams, [1, 2])) 27 | for stream, msg in msgs: 28 | yield marv.push(stream.msg(msg)) 29 | 30 | 31 | @marv.node() 32 | @marv.input('stream1', default=marv.select(source, 'a')) 33 | @marv.input('stream2', default=marv.select(source, 'b')) 34 | def consumer(stream1, stream2): 35 | streams = [stream1, stream2] 36 | while streams: 37 | for stream in streams[:]: 38 | msg = yield marv.pull(stream) 39 | if msg is None: 40 | streams.remove(stream) 41 | continue 42 | yield marv.push((stream.name, msg)) 43 | 44 | 45 | async def test(): 46 | nodes = [consumer] 47 | 48 | with LogCapture(level=logging.CRITICAL) as log: 49 | streams = await run_nodes(DATASET, nodes) 50 | 51 | assert [x.msg for x in log.records] == [ 52 | ['b'], 53 | ['a', 'b'], 54 | ] 55 | assert streams == [ 56 | [('a', 1), ('b', 1), ('a', 2), ('b', 2)], 57 | ] 58 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_ondemand_group_with_restart_and_foreach.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import logging 5 | from itertools import product 6 | 7 | from testfixtures import LogCapture 8 | 9 | from ..testing import make_dataset, marv, run_nodes 10 | 11 | DATASET = make_dataset() 12 | SETID = DATASET.setid 13 | 14 | 15 | @marv.node(group='ondemand') 16 | def source(): 17 | logger = yield marv.get_logger() 18 | requested = yield marv.get_requested() 19 | logger.critical([x.name for x in requested]) 20 | creates = [marv.create_stream(x.name) for x in requested] 21 | streams = [] 22 | for create in creates: 23 | stream = yield create 24 | streams.append(stream) 25 | msgs = list(product(streams, [1, 2])) 26 | for stream, msg in msgs: 27 | yield marv.push(stream.msg(msg)) 28 | 29 | 30 | @marv.node() 31 | @marv.input('stream', default=marv.select(source, 'a')) 32 | def stream_a(stream): 33 | while True: 34 | msg = yield marv.pull(stream) 35 | if msg is None: 36 | break 37 | msg = yield marv.push(msg) 38 | 39 | 40 | @marv.node() 41 | @marv.input('stream', default=marv.select(source, 'b')) 42 | def stream_b(stream): 43 | while True: 44 | msg = yield marv.pull(stream) 45 | if msg is None: 46 | break 47 | msg = yield marv.push(msg) 48 | 49 | 50 | @marv.node(group=True) 51 | @marv.input('stream1', default=stream_a) 52 | @marv.input('stream2', default=stream_b) 53 | def merged(stream1, stream2): 54 | yield marv.push(stream1) 55 | yield marv.push(stream2) 56 | 57 | 58 | @marv.node() 59 | @marv.input('stream', foreach=merged) 60 | def consumer(stream): 61 | logger = yield marv.get_logger() 62 | while True: 63 | msg = yield marv.pull(stream) 64 | if msg is None: 65 | break 66 | logger.critical((stream.node.name, stream.name, msg)) 67 | 68 | 69 | async def test(): 70 | nodes = [consumer] 71 | await run_nodes(DATASET, nodes) 72 | 73 | with LogCapture(level=logging.CRITICAL) as log: 74 | await run_nodes(DATASET, nodes) 75 | 76 | assert [x.msg for x in log.records] == [ 77 | ['b'], 78 | ['a', 'b'], 79 | ('stream_a', 'default', 1), 80 | ('stream_b', 'default', 1), 81 | ('stream_a', 'default', 2), 82 | ('stream_b', 'default', 2), 83 | ] 84 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_one_consumer.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from ..testing import make_dataset, marv, run_nodes 5 | 6 | 7 | @marv.node() 8 | def source(): 9 | yield marv.push(1) 10 | yield marv.push(2) 11 | yield marv.push(3) 12 | 13 | 14 | @marv.node() 15 | @marv.input('stream', default=source) 16 | def cubic(stream): 17 | while True: 18 | msg = yield marv.pull(stream) 19 | if msg is None: 20 | break 21 | yield marv.push(msg**3) 22 | 23 | 24 | DATASET = make_dataset() 25 | 26 | 27 | async def test(): 28 | nodes = [cubic] 29 | streams = await run_nodes(DATASET, nodes) 30 | assert streams == [ 31 | [1, 8, 27], 32 | ] 33 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_one_source.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from ..testing import make_dataset, marv, run_nodes 5 | 6 | 7 | @marv.node() 8 | def source(): 9 | yield marv.push(1) 10 | yield marv.push(2) 11 | yield marv.push(3) 12 | 13 | 14 | DATASET = make_dataset() 15 | 16 | 17 | async def test(): 18 | nodes = [source] 19 | streams = await run_nodes(DATASET, nodes) 20 | assert streams == [ 21 | [1, 2, 3], 22 | ] 23 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_plain_is_error.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import pytest 5 | 6 | from ..testing import make_dataset, marv, run_nodes 7 | 8 | 9 | @marv.node() 10 | def source(): 11 | yield 1 12 | 13 | 14 | DATASET = make_dataset() 15 | 16 | 17 | async def test(): 18 | nodes = [source] 19 | with pytest.raises(RuntimeError): # TODO: proper exception 20 | await run_nodes(DATASET, nodes) 21 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_request_input.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import logging 5 | 6 | from testfixtures import LogCapture 7 | 8 | from ..testing import make_dataset, marv, run_nodes 9 | 10 | DATASET = make_dataset() 11 | SETID = DATASET.setid 12 | 13 | 14 | @marv.node(group=True) 15 | def multi(): 16 | logger = yield marv.get_logger() 17 | logger.critical('multi started') 18 | a_out = yield marv.create_stream('a') 19 | b_out = yield marv.create_stream('b') 20 | assert a_out.name == 'a' 21 | assert b_out.name == 'b' 22 | 23 | yield a_out.msg(1) 24 | yield b_out.msg(10) 25 | yield a_out.msg(2) 26 | yield b_out.msg(20) 27 | logger.critical('multi finished') 28 | 29 | 30 | @marv.node() 31 | @marv.input('node', default=multi) 32 | def consumer(node): 33 | logger = yield marv.get_logger() 34 | logger.critical('consumer started') 35 | a_in, b_in = yield marv.pull_all(node, node) 36 | assert a_in.name == 'a' 37 | assert b_in.name == 'b' 38 | # streams = yield marv.io(*node) # like for values 39 | # streams = yield marv.io(node, consume=True) # a bit less voodoo 40 | 41 | values = [] 42 | while True: 43 | msgs = yield marv.pull_all(a_in, b_in) 44 | if msgs == [None, None]: 45 | break 46 | values.append(msgs) 47 | assert values == [[1, 10], [2, 20]] 48 | logger.critical('consumer finished') 49 | 50 | 51 | async def test(): 52 | nodes = [consumer] 53 | with LogCapture(level=logging.CRITICAL) as log: 54 | await run_nodes(DATASET, nodes) 55 | 56 | assert [x.msg for x in log.records] == [ 57 | 'multi started', 58 | 'consumer started', 59 | 'multi finished', 60 | 'consumer finished', 61 | ] 62 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_substream_subscription.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from ..testing import make_dataset, marv, run_nodes 5 | 6 | 7 | @marv.node(group='ondemand') 8 | def source(): 9 | out = yield marv.create_stream('foo') 10 | yield out.msg(1) 11 | 12 | 13 | @marv.node() 14 | @marv.input('handle', default=marv.select(source, 'foo')) 15 | def consumer(handle): 16 | while True: 17 | msg = yield marv.pull(handle) 18 | if msg is None: 19 | break 20 | yield marv.push(msg) 21 | 22 | 23 | DATASET = make_dataset() 24 | 25 | 26 | async def test(): 27 | nodes = [consumer] 28 | streams = await run_nodes(DATASET, nodes) 29 | assert streams == [ 30 | [1], 31 | ] 32 | -------------------------------------------------------------------------------- /code/marv/marv_node/tests/test_run_two_consumers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from ..testing import make_dataset, marv, run_nodes 5 | 6 | 7 | @marv.node() 8 | def source(): 9 | yield marv.push(1) 10 | yield marv.push(2) 11 | yield marv.push(3) 12 | 13 | 14 | @marv.node() 15 | @marv.input('stream', default=source) 16 | def cubic(stream): 17 | while True: 18 | msg = yield marv.pull(stream) 19 | if msg is None: 20 | break 21 | yield marv.push(msg**3) 22 | 23 | 24 | DATASET = make_dataset() 25 | 26 | 27 | async def test(): 28 | nodes = [source, cubic] 29 | streams = await run_nodes(DATASET, nodes) 30 | assert streams == [ 31 | [1, 2, 3], 32 | [1, 8, 27], 33 | ] 34 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from . import api, auth, collection, comment 5 | from . import dataset as dataset_api 6 | from . import delete, rpcs, tag 7 | 8 | __all__ = [ 9 | 'api', 10 | 'auth', 11 | 'collection', 12 | 'comment', 13 | 'dataset_api', 14 | 'delete', 15 | 'rpcs', 16 | 'tag', 17 | ] 18 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/api.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from .tooling import Webapi 5 | 6 | api = Webapi('/marv/api') 7 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/auth.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from json import JSONDecodeError 5 | 6 | from aiohttp import web 7 | 8 | from .api import api 9 | from .tooling import generate_token 10 | 11 | 12 | @api.endpoint('/auth', methods=['POST'], only_anon=True) 13 | async def auth_post(request): 14 | try: 15 | req = await request.json() 16 | username = req['username'] 17 | password = req['password'] 18 | except (JSONDecodeError, KeyError): 19 | raise web.HTTPBadRequest 20 | 21 | if not username or not password: 22 | raise web.HTTPUnprocessableEntity 23 | 24 | if not await request.app['site'].db.authenticate(username, password): 25 | raise web.HTTPUnprocessableEntity 26 | 27 | return web.json_response( 28 | { 29 | 'access_token': generate_token(username, request.app['config']['SECRET_KEY']), 30 | }, 31 | ) 32 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/comment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import time 5 | 6 | from aiohttp import web 7 | 8 | from marv.db import DBPermissionError 9 | 10 | from .api import api 11 | from .tooling import HTTPPermissionError 12 | 13 | 14 | @api.endpoint('/comment', methods=['POST']) 15 | async def comment(request): 16 | username = request['username'] 17 | changes = await request.json() 18 | if not changes: 19 | raise web.HTTPBadRequest() 20 | 21 | now = int(time.time() * 1000) 22 | comments = [ 23 | { 24 | 'dataset_id': id, 25 | 'author': username, 26 | 'time_added': now, 27 | 'text': text, 28 | } for id, ops in changes.items() for text in ops.get('add', []) 29 | ] 30 | try: 31 | await request.app['site'].db.bulk_comment(comments, user=username) 32 | except DBPermissionError: 33 | raise HTTPPermissionError(request) 34 | 35 | return web.json_response({}) 36 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/delete.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from json import JSONDecodeError 5 | 6 | from aiohttp import web 7 | 8 | from marv.db import DBPermissionError 9 | 10 | from .api import api 11 | from .tooling import HTTPPermissionError 12 | 13 | 14 | @api.endpoint('/dataset', methods=['DELETE']) 15 | async def delete(request): 16 | try: 17 | ids = await request.json() 18 | except JSONDecodeError: 19 | raise web.HTTPBadRequest 20 | 21 | if not ids: 22 | raise web.HTTPBadRequest 23 | 24 | try: 25 | await request.app['site'].db.discard_datasets_by_dbids(ids, True, user=request['username']) 26 | except DBPermissionError: 27 | raise HTTPPermissionError(request) 28 | 29 | return web.json_response({}) 30 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/rpcs.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import json 5 | from collections import defaultdict 6 | 7 | from aiohttp import web 8 | from tortoise.exceptions import OperationalError 9 | 10 | from .api import api 11 | 12 | 13 | @api.endpoint('/v1/rpcs', methods=['POST']) 14 | async def rpc_entry(request): # noqa: C901 15 | # pylint: disable=too-many-locals,too-many-branches 16 | 17 | try: 18 | posted = await request.json() 19 | except json.JSONDecodeError: 20 | raise web.HTTPBadRequest(text=json.dumps({'errors': ['request is not JSON']})) 21 | if not posted: 22 | raise web.HTTPBadRequest(text=json.dumps({'errors': ['nothing posted']})) 23 | if 'rpcs' not in posted: 24 | raise web.HTTPBadRequest(text=json.dumps({'errors': ['no rpcs posted']})) 25 | 26 | res = {'data': defaultdict(list)} 27 | 28 | for rpc in posted['rpcs']: 29 | try: 30 | (func, payload), = rpc.items() 31 | except ValueError: 32 | raise web.HTTPBadRequest( 33 | text=json.dumps({ 34 | 'errors': ['rpc should be a single key value pair'], 35 | }), 36 | ) 37 | 38 | if func == 'query': 39 | model = payload.get('model') 40 | filters = payload.get('filters', {}) 41 | attrs = payload.get('attrs', {}) 42 | order = payload.get('order') 43 | limit = payload.get('limit') 44 | offset = payload.get('offset') 45 | # v1 keeps collection 46 | if model == 'dataset' and 'collection' in attrs: 47 | attrs['collection_id'] = attrs.pop('collection') 48 | # /v1 keeps collection 49 | try: 50 | result = await request.app['site'].db.rpc_query( 51 | model, 52 | filters, 53 | attrs, 54 | order, 55 | limit, 56 | offset, 57 | request['username'], 58 | ) 59 | except (OperationalError, ValueError) as err: 60 | raise web.HTTPBadRequest(text=json.dumps({'errors': [str(err)]})) 61 | 62 | aliases = payload.get('aliases', {}) 63 | for key, values in result.items(): 64 | # v1 keeps collection 65 | if key == 'dataset': 66 | collections = { 67 | x['id']: x['name'] for x in ( 68 | await request.app['site'].db. 69 | rpc_query('collection', {}, {}, None, None, None, request['username']) 70 | )['collection'] 71 | } 72 | for value in values: 73 | if 'collection_id' in value: 74 | colid = value.pop('collection_id') 75 | if colid in collections: 76 | value['collection'] = collections[colid] 77 | # /v1 keeps collection 78 | res['data'][aliases.get(key, key)].extend(values) 79 | else: 80 | raise web.HTTPBadRequest(text=f'unknown rpc function "{func}"') 81 | 82 | return web.json_response(res) 83 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tag.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from json import JSONDecodeError 5 | 6 | from aiohttp import web 7 | 8 | from marv.db import DBPermissionError 9 | 10 | from .api import api 11 | from .tooling import HTTPPermissionError 12 | 13 | 14 | @api.endpoint('/tag', methods=['POST']) 15 | async def tag(request): # noqa: C901 16 | try: 17 | changes = await request.json() 18 | except JSONDecodeError: 19 | raise web.HTTPBadRequest 20 | 21 | if not changes: 22 | raise web.HTTPBadRequest 23 | 24 | try: 25 | add = [] 26 | remove = [] 27 | for ops in changes.values(): 28 | for opname, target in (('add', add), ('remove', remove)): 29 | for tagname, ids in ops.pop(opname, {}).items(): 30 | for id in ids: 31 | target.append((tagname, id)) 32 | if ops: 33 | raise web.HTTPBadRequest 34 | except AttributeError: 35 | raise web.HTTPBadRequest 36 | 37 | try: 38 | await request.app['site'].db.bulk_tag(add, remove, user=request['username']) 39 | except DBPermissionError: 40 | raise HTTPPermissionError(request) 41 | return web.json_response({}) 42 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2019 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from marv.tests.conftest import app, client, site # noqa: F401,TC001 pylint: disable=unused-import 5 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/data/full_listings.json: -------------------------------------------------------------------------------- 1 | { 2 | "hodge": [ 3 | { 4 | "setid": "aaaaaaaaaaaaaaaaaaaaaaaaqa", 5 | "tags": [ 6 | "TAG1" 7 | ], 8 | "values": [ 9 | { 10 | "id": "aaaaaaaaaaaaaaaaaaaaaaaaqa", 11 | "route": "detail", 12 | "title": "hodge_0001" 13 | }, 14 | 1, 15 | [], 16 | [ 17 | "TAG1" 18 | ], 19 | 1400000000000, 20 | 1500001000000, 21 | 1 22 | ] 23 | }, 24 | { 25 | "setid": "aeaaaaaaaaaaaaaaaaaaaaaaqa", 26 | "tags": [ 27 | "TAG1", 28 | "TAG2" 29 | ], 30 | "values": [ 31 | { 32 | "id": "aeaaaaaaaaaaaaaaaaaaaaaaqa", 33 | "route": "detail", 34 | "title": "hodge_0002" 35 | }, 36 | 2, 37 | [], 38 | [ 39 | "TAG1", 40 | "TAG2" 41 | ], 42 | 1400043200000, 43 | 1500002000000, 44 | 2 45 | ] 46 | } 47 | ], 48 | "podge": [ 49 | { 50 | "setid": "aiaaaaaaaaaaaaaaaaaaaaaaqa", 51 | "tags": [], 52 | "values": [ 53 | { 54 | "id": "aiaaaaaaaaaaaaaaaaaaaaaaqa", 55 | "route": "detail", 56 | "title": "podge_0001" 57 | }, 58 | 1, 59 | [], 60 | [], 61 | 1400086400000, 62 | 1500001000000 63 | ] 64 | }, 65 | { 66 | "setid": "amaaaaaaaaaaaaaaaaaaaaaaqa", 67 | "tags": [ 68 | "TAG2" 69 | ], 70 | "values": [ 71 | { 72 | "id": "amaaaaaaaaaaaaaaaaaaaaaaqa", 73 | "route": "detail", 74 | "title": "podge_0002" 75 | }, 76 | 2, 77 | [], 78 | [ 79 | "TAG2" 80 | ], 81 | 1400129600000, 82 | 1500002000000 83 | ] 84 | }, 85 | { 86 | "setid": "aqaaaaaaaaaaaaaaaaaaaaaaqa", 87 | "tags": [], 88 | "values": [ 89 | { 90 | "id": "aqaaaaaaaaaaaaaaaaaaaaaaqa", 91 | "route": "detail", 92 | "title": "podge_0003" 93 | }, 94 | 3, 95 | [], 96 | [], 97 | 1400172800000, 98 | 1500003000000 99 | ] 100 | }, 101 | { 102 | "setid": "auaaaaaaaaaaaaaaaaaaaaaaqa", 103 | "tags": [], 104 | "values": [ 105 | { 106 | "id": "auaaaaaaaaaaaaaaaaaaaaaaqa", 107 | "route": "detail", 108 | "title": "podge_0004" 109 | }, 110 | 4, 111 | [], 112 | [], 113 | 1400216000000, 114 | 1500004000000 115 | ] 116 | } 117 | ] 118 | } -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/test_collection.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import json 5 | 6 | 7 | async def test_collection(app, site, client): 8 | await client.authenticate('test', 'test_pw') 9 | 10 | await site.db.bulk_tag( 11 | [ 12 | ('foo', 1), 13 | ('foo', 2), 14 | ], 15 | [], 16 | '::', 17 | ) 18 | 19 | res = await client.get_json('/marv/api/_collection/notexist') 20 | assert res.status == 403 21 | 22 | res = await client.get_json('/marv/api/_collection/hodge') 23 | assert len(res['listing']['widget']['data']['rows']) == 10 24 | 25 | res = await client.get_json( 26 | '/marv/api/_collection/hodge', 27 | params={ 28 | 'filter': json.dumps({ 29 | 'not exist': { 30 | 'val': 42, 31 | 'op': 'eq', 32 | }, 33 | }), 34 | }, 35 | ) 36 | assert res.status == 400 37 | 38 | res = await client.get_json( 39 | '/marv/api/_collection/hodge', 40 | params={ 41 | 'filter': json.dumps({ 42 | 'f_name': { 43 | 'val': 42, 44 | 'op': 'not exist', 45 | }, 46 | }), 47 | }, 48 | ) 49 | assert res.status == 400 50 | 51 | res = await client.get_json('/marv/api/_collection/hodge') 52 | app['debug'] = True 53 | res2 = await client.get_json('/marv/api/_collection/hodge') 54 | assert res == res2 55 | 56 | res = await client.get_json( 57 | '/marv/api/_collection/hodge', 58 | params={ 59 | 'filter': json.dumps({ 60 | 'f_name': { 61 | 'val': 'filtér', 62 | 'op': 'eq', 63 | }, 64 | }), 65 | }, 66 | ) 67 | assert 'listing' in res 68 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/test_comment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2019 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | 5 | async def test_comment(client): 6 | await client.authenticate('test', 'test_pw') 7 | 8 | res = await client.post('/marv/api/comment', headers=client.headers, json={}) 9 | assert res.status == 400 10 | 11 | res = await client.post_json( 12 | '/marv/api/comment', 13 | json={ 14 | '1': { 15 | 'add': ['lorem ipsum', 'dolor'], 16 | }, 17 | }, 18 | ) 19 | assert res == {} 20 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/test_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pathlib import Path 5 | 6 | 7 | async def test_dataset(site, client): 8 | await client.authenticate('test', 'test_pw') 9 | 10 | sets = await site.db.get_datasets_for_collections(None) 11 | 12 | res = await client.get_json('/marv/api/dataset/xxxxxxxxxxxxxxxxxxxxxxxxxx', json={}) 13 | assert res.status == 403 14 | 15 | res = await client.get_json(f'/marv/api/dataset/{sets[0]}') 16 | assert res['title'] == 'hodge_0001' 17 | 18 | # 404 with virtual scanroot 19 | res = await client.get_json(f'/marv/api/dataset/{sets[0]}/0') 20 | assert res.status == 404 21 | 22 | # 403 for absolute file 23 | res = await client.get_json(f'/marv/api/dataset/{sets[0]}//absolute') 24 | assert res.status == 403 25 | 26 | # filelist 27 | res = await client.post_json('/marv/api/file-list', json=[]) 28 | assert res.status == 400 29 | 30 | res = await client.post_json('/marv/api/file-list', json=[1, 2]) 31 | assert [x.split('/')[-1] for x in res['paths']] == ['0001', '0002'] 32 | assert [x.split('/')[-1] for x in res['urls']] == ['0', '0'] 33 | 34 | 35 | async def test_nginx(site, client, mocker): 36 | site.config = site.config.copy( 37 | update={ 38 | 'marv': site.config.marv.copy(update={'reverse_proxy': 'nginx'}), 39 | }, 40 | ) 41 | 42 | sets = await site.db.get_datasets_for_collections(None) 43 | 44 | await client.authenticate('test', 'test_pw') 45 | 46 | async def get_filepath(*args): # pylint: disable=unused-argument 47 | return Path('/dev/null/foo') 48 | 49 | mocker.patch('marv_webapi.dataset._get_filepath', wraps=get_filepath) 50 | 51 | res = await client.get_json(f'/marv/api/dataset/{sets[0]}/0') 52 | assert res.status == 200 53 | assert res.headers['x-accel-redirect'] == '/dev/null/foo' 54 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/test_delete.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2019 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | 5 | async def test_dataset(site, client): 6 | await client.authenticate('adm', 'adm_pw') 7 | 8 | sets = await site.db.get_datasets_for_collections(None) 9 | 10 | res = await client.delete('/marv/api/dataset', headers=client.headers, json=[]) 11 | assert res.status == 400 12 | 13 | res = await client.delete('/marv/api/dataset', headers=client.headers, json=[1]) 14 | assert res.status == 200 15 | 16 | rest = await site.db.get_datasets_for_collections(None) 17 | assert set(sets) - set(rest) == {sets[0]} 18 | 19 | await client.authenticate('test', 'test_pw') 20 | res = await client.delete('/marv/api/dataset', headers=client.headers, json=[2]) 21 | assert res.status == 403 22 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/test_responses.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2021 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pathlib import Path 5 | 6 | import pytest 7 | 8 | from marv.tests.conftest import recorded 9 | from marv.tests.test_dump_restore import postscan 10 | 11 | DATADIR = Path(__file__).parent / 'data' 12 | 13 | 14 | async def get_listings(client): 15 | metadata = await client.get_json('/marv/api/meta') 16 | listings = {} 17 | for colinfo in metadata['collections']: 18 | name = colinfo['name'] 19 | listings[name] = await client.get_json(f'/marv/api/_collection/{name}') 20 | return listings 21 | 22 | 23 | def get_rows(listing): 24 | rows = listing['listing']['widget']['data']['rows'] 25 | lst = [] 26 | for row in sorted(rows, key=lambda x: x['setid']): 27 | del row['id'] 28 | lst.append(row) 29 | return lst 30 | 31 | 32 | @pytest.mark.marv(site={'empty': True}) 33 | async def test_empty_responses(client, site): 34 | await site.db.user_add('test', password='test_pw', realm='marv', realmuid='') 35 | await client.authenticate('test', 'test_pw') 36 | assert recorded(await get_listings(client), DATADIR / 'empty_listings.json') 37 | 38 | 39 | @pytest.mark.marv(site={'postscan': postscan, 'size': 2}) 40 | async def test_full_responses(client, site): 41 | await client.authenticate('test', 'test_pw') 42 | sets = await site.db.get_datasets_for_collections(None) 43 | for setid in sets: 44 | await site.run(setid) 45 | 46 | listings = await get_listings(client) 47 | listings = {k: get_rows(v) for k, v in listings.items()} 48 | assert recorded(listings, DATADIR / 'full_listings.json') 49 | 50 | details = [] 51 | for colname, rows in sorted(listings.items()): 52 | for row in rows: 53 | detail = await client.get_json(f'/marv/api/dataset/{row["setid"]}') 54 | del detail['id'] 55 | assert detail['collection'] == colname 56 | details.append(detail) 57 | assert recorded(details, DATADIR / 'full_details.json') 58 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/test_static.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | 5 | async def test_frontend(client): 6 | res = await client.get('/') 7 | assert res.status == 200 8 | assert res.headers['Content-Type'] == 'text/html; charset=utf-8' 9 | 10 | res = await client.get('/main-built.js') 11 | assert res.status == 200 12 | assert res.headers['Content-Type'] in ('application/javascript', 'application/x-javascript') 13 | data = await res.text(encoding='utf-8') 14 | assert data.startswith('/*') 15 | 16 | res = await client.get('/main-built.css') 17 | assert res.status == 200 18 | assert res.headers['Content-Type'] == 'text/css' 19 | data = await res.text(encoding='utf-8') 20 | assert data.startswith('/*') 21 | 22 | res = await client.get('/custom/not-exist.js') 23 | assert res.status == 404 24 | 25 | res = await client.get('/custom//absolute') 26 | assert res.status == 403 27 | 28 | res = await client.get('/custom/relative//withdoubleslash') 29 | assert res.status == 404 30 | 31 | res = await client.get('/not-exist.js') 32 | assert res.status == 404 33 | 34 | res = await client.get('somedir//absolute') 35 | assert res.status == 404 36 | 37 | # deactivate url validation in test framework, 38 | client.server.skip_url_asserts = True 39 | res = await client.get('////absolute') 40 | assert res.status == 403 41 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/test_tag.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2019 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | 5 | async def test_tag(client, site): 6 | await client.authenticate('test', 'test_pw') 7 | 8 | res = await client.post('/marv/api/tag', headers=client.headers, json={}) 9 | assert res.status == 400 10 | 11 | res = await client.post_json( 12 | '/marv/api/tag', 13 | json={ 14 | 'hodge': { 15 | 'add': { 16 | 'important': [1, 2, 3], 17 | }, 18 | }, 19 | }, 20 | ) 21 | assert res == {} 22 | 23 | res = await site.db.get_datasets_by_setids(await site.db.query(tags=['important']), [], '::') 24 | assert sorted(x.id for x in res) == [1, 2, 3] 25 | -------------------------------------------------------------------------------- /code/marv/marv_webapi/tests/test_tooling.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2020 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | from pathlib import Path 5 | 6 | import pytest 7 | from aiohttp import web 8 | 9 | from marv_webapi.tooling import safejoin 10 | 11 | 12 | def test_safejoin(): 13 | with pytest.raises(web.HTTPForbidden): 14 | safejoin(Path('/base'), Path('/foo')) 15 | 16 | with pytest.raises(web.HTTPForbidden): 17 | safejoin(Path('/base'), Path('../foo')) 18 | 19 | with pytest.raises(web.HTTPForbidden): 20 | safejoin(Path('/base'), Path('foo/../../bar')) 21 | 22 | assert str(safejoin(Path('/base'), Path('foo//bar'))) == '/base/foo/bar' 23 | -------------------------------------------------------------------------------- /code/marv/requirements.in: -------------------------------------------------------------------------------- 1 | ../../requirements/marv.in -------------------------------------------------------------------------------- /code/marv/requirements.txt: -------------------------------------------------------------------------------- 1 | ../../requirements/marv.txt -------------------------------------------------------------------------------- /code/marv/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: AGPL-3.0-only 3 | 4 | import io 5 | import os 6 | from collections import OrderedDict 7 | 8 | from setuptools import find_packages, setup 9 | 10 | NAME = 'marv' 11 | VERSION = '21.12.0' 12 | DESCRIPTION = 'MARV framework' 13 | ENTRY_POINTS = { 14 | 'marv_cli': ['marv = marv.cli'], 15 | 'marv_stream': ['messages = marv_ros_stream:messages'], 16 | } 17 | 18 | # Copy/paste block below here 19 | 20 | os.chdir(os.path.abspath(os.path.dirname(__file__))) 21 | 22 | with io.open(os.path.join('README.rst'), 'rt', encoding='utf8') as f: 23 | README = f.read() 24 | 25 | with io.open('requirements.in', 'rt', encoding='utf8') as f: 26 | INSTALL_REQUIRES = [ 27 | # e.g. -r ../path/to/file/package_name.in 28 | f'{os.path.basename(req.split()[1])[:-3]}=={VERSION}' if req.startswith('-r') else req 29 | for req in [line.strip() for line in f.readlines() if not line.startswith('#')] 30 | if req 31 | ] 32 | 33 | setup( 34 | name=NAME, 35 | version=VERSION, 36 | description=DESCRIPTION, 37 | long_description=README, 38 | classifiers=[ 39 | 'Development Status :: 5 - Production/Stable', 40 | 'License :: OSI Approved :: GNU Affero General Public License v3', 41 | 'Operating System :: POSIX :: Linux', 42 | 'Programming Language :: Python :: 3 :: Only', 43 | 'Programming Language :: Python :: 3.8', 44 | 'Programming Language :: Python :: Implementation :: CPython', 45 | 'Programming Language :: Python', 46 | 'Topic :: Scientific/Engineering', 47 | ], 48 | author='Ternaris', 49 | author_email='team@ternaris.com', 50 | maintainer='Ternaris', 51 | maintainer_email='team@ternaris.com', 52 | url='https://ternaris.com/marv-robotics', 53 | project_urls=OrderedDict( 54 | ( 55 | ('Documentation', 'https://ternaris.com/marv-robotics/docs/'), 56 | ('Code', 'https://gitlan.com/ternaris/marv-robotics'), 57 | ('Issue tracker', 'https://gitlab.com/ternaris/marv-robotics/issues'), 58 | ), 59 | ), 60 | license='AGPL-3.0-only', 61 | packages=find_packages(), 62 | include_package_data=True, 63 | zip_safe=False, 64 | python_requires='>=3.8.2', 65 | install_requires=INSTALL_REQUIRES, 66 | tests_require=[ 67 | 'pytest', 68 | 'testfixtures', 69 | ], 70 | setup_requires=['pytest-runner'], 71 | entry_points=ENTRY_POINTS, 72 | ) 73 | -------------------------------------------------------------------------------- /docs/.gitignore: -------------------------------------------------------------------------------- 1 | /_build 2 | /.venv -------------------------------------------------------------------------------- /docs/CHANGES.rst: -------------------------------------------------------------------------------- 1 | ../CHANGES.rst -------------------------------------------------------------------------------- /docs/CONTRIBUTING.rst: -------------------------------------------------------------------------------- 1 | ../CONTRIBUTING.rst -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = -W 6 | SPHINXBUILD = python -msphinx 7 | SPHINXPROJ = MARV 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile apidoc 16 | 17 | apidoc: 18 | sphinx-apidoc -E -M -f -o ./api ../ 19 | 20 | # Catch-all target: route all unknown targets to Sphinx using the new 21 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 22 | %: Makefile 23 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 24 | -------------------------------------------------------------------------------- /docs/_static/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ternaris/marv-robotics/473ca28d85ac55a7190edaa19347936ce3a6553a/docs/_static/.keep -------------------------------------------------------------------------------- /docs/api/marv.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | marv 5 | ==== 6 | 7 | Creating datasets 8 | ----------------- 9 | 10 | .. automodule:: marv_api.scanner 11 | :members: 12 | :show-inheritance: 13 | 14 | 15 | Declaring nodes 16 | --------------- 17 | 18 | .. automodule:: marv_api 19 | :members: node, input 20 | :undoc-members: 21 | :show-inheritance: 22 | 23 | 24 | Interacting with marv 25 | --------------------- 26 | 27 | .. automodule:: marv_api 28 | :noindex: 29 | :members: 30 | :exclude-members: input, node 31 | :undoc-members: 32 | :show-inheritance: 33 | -------------------------------------------------------------------------------- /docs/config/custom.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * Copyright 2016 - 2018 Ternaris. 3 | * SPDX-License-Identifier: CC0-1.0 4 | */ 5 | 6 | window.marv_extensions = { 7 | formats: { 8 | // replace default datetime formatter, show times in UTC 9 | 'datetime': function(date) { return new Date(date).toUTCString(); } 10 | }, 11 | widgets: { 12 | // rowcount widget displays the number of rows in a table 13 | 'rowcount': [ 14 | /* insert callback, renders the data 15 | 16 | @function insert 17 | @param {HTMLElement} element The parent element 18 | @param {Object} data The data to be rendered 19 | @return {Object} state Any variable, if required by remove 20 | */ 21 | function insert(element, data) { 22 | const doc = element.ownerDocument; 23 | const el = doc.createTextNode(data.rows.length + ' rows'); 24 | element.appendChild(el); 25 | 26 | const state = { el }; 27 | return state; 28 | }, 29 | 30 | /* remove callback, clean up if necessary 31 | 32 | @function remove 33 | @param {Object} state The state object returned by insert 34 | */ 35 | function remove(state) { 36 | state.el.remove(); 37 | } 38 | ] 39 | } 40 | }; 41 | -------------------------------------------------------------------------------- /docs/config/etc_marv_marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | 4 | # keep db/db.sqlite as the suffix! 5 | dburi = sqlite:///var/local/lib/marv/db/db.sqlite 6 | 7 | # store could also be somewhere else 8 | store = /var/local/lib/marv/store 9 | 10 | ... -------------------------------------------------------------------------------- /docs/config/marv.conf: -------------------------------------------------------------------------------- 1 | ../../sites/example/marv.conf -------------------------------------------------------------------------------- /docs/config/marv_multiple.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags bags2 videos 3 | 4 | [collection bags] 5 | scanner = marv_robotics.bag:scan 6 | 7 | ... 8 | 9 | [collection bags2] 10 | scanner = marv_robotics.bag:scan 11 | 12 | ... 13 | 14 | [collection videos] 15 | scanner = my_own_scanner:scan 16 | 17 | ... -------------------------------------------------------------------------------- /docs/debug.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | .. _debug: 5 | 6 | Debugging 7 | ========= 8 | 9 | Given an exception during a node run: 10 | 11 | .. code-block:: console 12 | 13 | $ marv run --node bagmeta b563ng6y6d3 --force 14 | 2018-02-01 09:58:10,552 INFO rospy.topics topicmanager initialized 15 | 2018-02-01 09:58:10,984 INFO marv.run b563ng6y6d.bagmeta.dwz4xbykdt.default (bagmeta) started with force 16 | 2018-02-01 09:58:10,987 ERRO marv.cli Exception occurred for dataset b563ng6y6d3pjf6ycx7t52pqae: 17 | Traceback (most recent call last): 18 | File "/webapp/marv/suite/marv/marv/cli.py", line 405, in marvcli_run 19 | excluded_nodes, cachesize=cachesize) 20 | File "/webapp/marv/suite/marv/marv/site.py", line 351, in run 21 | deps=deps, cachesize=cachesize) 22 | File "/webapp/marv/suite/marv/marv_node/run.py", line 63, in run_nodes 23 | done, send_queue_empty = process_task(current, task) 24 | File "/webapp/marv/suite/marv/marv_node/run.py", line 352, in process_task 25 | return loop() 26 | File "/webapp/marv/suite/marv/marv_node/run.py", line 242, in loop 27 | promise = current.send(send) 28 | File "/webapp/marv/suite/marv/marv_node/driver.py", line 89, in _run 29 | request = gen.send(send) 30 | File "/webapp/marv/suite/marv/marv_node/node.py", line 243, in invoke 31 | send = yield gen.send(send) 32 | File "/webapp/marv/suite/robotics/marv_robotics/bag.py", line 171, in bagmeta 33 | xx 34 | NameError: global name 'xx' is not defined 35 | 2018-02-01 09:58:10,992 ERRO marv.cli Error occurred for dataset b563ng6y6d3pjf6ycx7t52pqae: global name 'xx' is not defined 36 | 37 | one can enter pdbpp by running ``PDB=1 marv`` instead of ``marv``: 38 | 39 | .. code-block:: console 40 | 41 | $ PDB=1 marv run --node bagmeta b563ng6y6d3 --force 42 | 2018-02-01 13:04:41,524 INFO rospy.topics topicmanager initialized 43 | 2018-02-01 13:04:41,979 INFO marv.run b563ng6y6d.bagmeta.dwz4xbykdt.default (bagmeta) started with force 44 | NameError("global name 'xx' is not defined",) 45 | /webapp/venv/lib/python2.7/site-packages/IPython/core/debugger.py:243: DeprecationWarning: The `color_scheme` argument is deprecated since version 5.1 46 | DeprecationWarning) 47 | > /webapp/marv/suite/robotics/marv_robotics/bag.py(171)bagmeta() 48 | 170 end_time = 0 49 | --> 171 xx 50 | 172 connections = {} 51 | 52 | (Pdb++) 53 | 54 | Likewise pdb can be used by placing ``import pdb; pdb.set_trace()`` anywhere in the code. 55 | 56 | .. code-block:: console 57 | 58 | $ marv run --node bagmeta b563ng6y6d3 --force 59 | 2018-02-01 13:08:14,235 INFO rospy.topics topicmanager initialized 60 | 2018-02-01 13:08:14,633 INFO marv.run b563ng6y6d.bagmeta.dwz4xbykdt.default (bagmeta) started with force 61 | /webapp/venv/lib/python2.7/site-packages/IPython/core/debugger.py:243: DeprecationWarning: The `color_scheme` argument is deprecated since version 5.1 62 | DeprecationWarning) 63 | > /webapp/marv/suite/robotics/marv_robotics/bag.py(172)bagmeta() 64 | 171 import pdb; pdb.set_trace() 65 | --> 172 connections = {} 66 | 173 for path in paths: 67 | 68 | (Pdb++) 69 | 70 | For more information see https://github.com/antocuni/pdb 71 | -------------------------------------------------------------------------------- /docs/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ternaris/marv-robotics/473ca28d85ac55a7190edaa19347936ce3a6553a/docs/favicon-32x32.png -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | MARV Robotics 5 | ============= 6 | 7 | Welcome to MARV Robotics! 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | 12 | install/index 13 | tutorial/setup-basic-site 14 | tutorial/write-your-own 15 | CONTRIBUTING 16 | patterns 17 | debug 18 | views 19 | config 20 | scan 21 | nodes 22 | widgets 23 | httpapi 24 | upload 25 | deploy 26 | accesscontrol 27 | maintenance 28 | migrate/index 29 | CHANGES 30 | 31 | 32 | API Reference 33 | ============= 34 | 35 | .. toctree:: 36 | :maxdepth: 2 37 | 38 | api/marv 39 | 40 | 41 | Index 42 | ===== 43 | 44 | * :ref:`genindex` 45 | * :ref:`modindex` 46 | * :ref:`search` 47 | -------------------------------------------------------------------------------- /docs/install/index.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | .. _install: 5 | 6 | Installation 7 | ============ 8 | 9 | There are two modes of installation. One is based on `Docker `_ the other installs MARV Robotics natively on your system. It might be a good idea to read both and decide then. 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | 14 | docker 15 | native 16 | -------------------------------------------------------------------------------- /docs/install/native.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | .. _install_native: 5 | 6 | Native 7 | ====== 8 | 9 | Prerequisites 10 | ------------- 11 | 12 | - Checkout of MARV Robotics Enterprise Edition or `Community Edition 13 | `_ 14 | 15 | 16 | System dependencies 17 | ------------------- 18 | 19 | MARV Robotics needs Python 3.8 and ships all components to open bag files and process ROS messages (ROS1 and ROS2). On top of Ubuntu focal ROS1 and ROS2 releases are available using Python 3.8. Please let us know if you experience any issues or need support for an older version of Python. 20 | 21 | Ubuntu focal 22 | ^^^^^^^^^^^^ 23 | 24 | In general, MARV Robotics works on any Linux system. For Ubuntu focal the following will install the necessary system dependencies. 25 | 26 | .. code-block:: console 27 | 28 | # apt-get install capnproto \ 29 | curl \ 30 | ffmpeg \ 31 | libcapnp-dev \ 32 | libffi-dev \ 33 | libfreetype6-dev \ 34 | libjpeg-dev \ 35 | liblz4-dev \ 36 | libpng-dev \ 37 | libssl-dev \ 38 | libz-dev \ 39 | python3.8 \ 40 | python3.8-dev \ 41 | python3.8-venv 42 | 43 | 44 | 45 | MARV Robotics 46 | ------------- 47 | 48 | For the following commands we assume you are within the directory of your checkout of MARV Robotics. 49 | 50 | Setup MARV Robotics in Python virtual environment and activate it: 51 | 52 | .. code-block:: console 53 | 54 | $ ./scripts/setup-venv requirements/marv-robotics.txt venv 55 | $ source venv/bin/activate 56 | (venv) $ marv --help 57 | 58 | Et voilà, marv is successfully installed. The ``(venv)`` prefix indicates the activated virtualenv. In the following sections we assume that your virtualenv is activated. If ``marv`` cannot be found, chances are that the virtualenv containing MARV Robotics is not activated. For more information see `Virtual Environments `_. 59 | 60 | .. warning:: 61 | 62 | MARV Robotics does not need write access to your bag files. As a safety measure install and run MARV as a user having only read-only access to your bag files. 63 | 64 | 65 | Build and serve documentation 66 | ----------------------------- 67 | 68 | Let's dedicate a terminal to build the documentation and to start a small webserver to serve the documentation; actually, to serve MARV Robotics already, which contains the documentation. 69 | 70 | .. code-block:: console 71 | 72 | (venv) $ ./scripts/build-docs 73 | (venv) $ marv --config tutorial/docs-only-site/marv.conf serve 74 | 75 | Now you have an instance of MARV running at: http://localhost:8000. 76 | 77 | It's documentation is linked in the footer and served at: http://localhost:8000/docs/ 78 | 79 | Let's switch to your `locally served documentation `_. 80 | 81 | 82 | Summary 83 | ------- 84 | 85 | You installed some system dependencies, created a virtual python environment, installed MARV Robotics into it, and started a webserver with marv and its documentation: 86 | 87 | Now you are ready to `setup a basic site <../tutorial/setup-basic-site.html>`_. 88 | -------------------------------------------------------------------------------- /docs/maintenance.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | .. _maintenance: 5 | 6 | Maintenance 7 | =========== 8 | 9 | Cleanup 10 | ------- 11 | 12 | When datasets are discarded (via frontend, API, cli) they are only marked to be removed from marv's database and can be "undiscarded". 13 | 14 | .. code-block:: bash 15 | 16 | marv undiscard --help 17 | 18 | To actually remove these from the database 19 | 20 | .. code-block:: bash 21 | 22 | marv cleanup --discarded 23 | 24 | After deleting datasets manually from the filesystem, you can remove them from the database as follows: 25 | 26 | .. code-block:: bash 27 | 28 | marv scan 29 | marv query --missing | xargs marv discard 30 | marv cleanup --discarded 31 | 32 | When searching for ``subset`` :ref:`cfg_c_filters`, marv presents matching results, including values from meanwhile cleaned-up datasets. Cleanup these filters with: 33 | 34 | .. code-block:: bash 35 | 36 | marv cleanup --filters 37 | 38 | Additional cleanup operations will be added in one of the next releases. 39 | 40 | Backup 41 | ------ 42 | 43 | By default, all data related to a marv site is stored in a site directory. It holds the following information: 44 | 45 | - ``db`` marv's sqlite database 46 | - ``marv.conf`` the marv configuration file 47 | - ``sessionkey`` if it changes, users will have to relogin 48 | - ``store`` of all node output, theoretically possible to recreate by running all nodes, which might not be feasible 49 | - ``gunicorn_cfg.py`` configuration for gunicorn serving the marv application 50 | 51 | Make sure to create regular backups of this site directory or the individual components in case you placed them elsewhere. It is not necessary to stop marv to create backups. 52 | 53 | 54 | Dump/restore 55 | ------------ 56 | 57 | Use ``marv dump`` and ``marv restore`` to dump marv's database to a json format and to restore it in a site with the same configuration. You can use the latest version to dump older databases. 58 | 59 | The dump contains: 60 | 61 | - datasets for all collections with setid, files, tags, and comments 62 | - and users with group memberships. 63 | 64 | Apart from the dump, you need to copy from old site: 65 | 66 | - marv.conf 67 | - gunicorn_cfg.py 68 | - sessionkey (if it changes, users will have to relogin) 69 | - store (if you don't copy the store, all nodes will have to rerun) 70 | 71 | .. code-block:: bash 72 | 73 | cd old-site 74 | marv dump ../dump.json 75 | 76 | cd ../new-site 77 | cp -a ../old-site/marv.conf ./ 78 | cp -a ../old-site/gunicorn_cfg.py ./ 79 | cp -a ../old-site/sessionkey ./ 80 | cp -a ../old-site/store ./ 81 | marv restore ../dump.json 82 | 83 | **DUMP/RESTORE IS NOT A REPLACEMENT FOR MAKING BACKUPS** 84 | -------------------------------------------------------------------------------- /docs/migrate/1711-1802-marv.conf.diff: -------------------------------------------------------------------------------- 1 | diff --git a/docs/config/marv.conf b/docs/config/marv.conf 2 | index 0e44b19..31dfe57 100644 3 | --- a/docs/config/marv.conf 4 | +++ b/docs/config/marv.conf 5 | @@ -24,7 +24,7 @@ nodes = 6 | marv_robotics.detail:summary_keyval 7 | marv_robotics.detail:bagmeta_table 8 | # detail sections 9 | - marv_robotics.detail:topics_section 10 | + marv_robotics.detail:connections_section 11 | marv_robotics.detail:images_section 12 | marv_robotics.detail:video_section 13 | marv_robotics.detail:gnss_section 14 | @@ -43,8 +43,8 @@ filters = 15 | start_time | Start time | lt le eq ne ge gt | datetime | (get "bagmeta.start_time") 16 | end_time | End time | lt le eq ne ge gt | datetime | (get "bagmeta.end_time") 17 | duration | Duration | lt le eq ne ge gt | timedelta | (get "bagmeta.duration") 18 | - topics | Topics | any all | subset | (get "bagmeta.topics[:].name") 19 | - msg_types | Message types | any all | subset | (get "bagmeta.msg_types[:].name") 20 | + topics | Topics | any all | subset | (get "bagmeta.topics") 21 | + msg_types | Message types | any all | subset | (get "bagmeta.msg_types") 22 | 23 | listing_columns = 24 | # id | Heading | formatter | value function 25 | @@ -69,7 +69,7 @@ detail_summary_widgets = 26 | bagmeta_table 27 | 28 | detail_sections = 29 | - topics_section 30 | + connections_section 31 | images_section 32 | video_section 33 | gnss_section 34 | -------------------------------------------------------------------------------- /docs/nodes.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | .. _nodes: 5 | 6 | Nodes 7 | ===== 8 | 9 | Bag metadata 10 | ------------ 11 | 12 | .. autofunction:: marv_robotics.bag.bagmeta() 13 | 14 | 15 | ROS bag messages 16 | ---------------- 17 | 18 | .. autofunction:: marv_robotics.bag.raw_messages() 19 | .. autofunction:: marv_robotics.bag.make_deserialize() 20 | .. autofunction:: marv_robotics.bag.make_get_timestamp() 21 | 22 | 23 | Images 24 | ------ 25 | 26 | .. autofunction:: marv_robotics.cam.images() 27 | 28 | 29 | Video 30 | ----- 31 | 32 | .. autofunction:: marv_robotics.cam.ffmpeg() 33 | 34 | 35 | Fulltext 36 | -------- 37 | 38 | .. autofunction:: marv_robotics.fulltext.fulltext() 39 | 40 | 41 | GNSS 42 | ---- 43 | .. autofunction:: marv_robotics.gnss.positions() 44 | .. autofunction:: marv_robotics.gnss.imus() 45 | .. autofunction:: marv_robotics.gnss.navsatorients() 46 | .. autofunction:: marv_robotics.gnss.orientations() 47 | .. autofunction:: marv_robotics.gnss.gnss_plots() 48 | 49 | 50 | Trajectory 51 | ---------- 52 | 53 | .. autofunction:: marv_robotics.trajectory.navsatfix() 54 | 55 | .. autofunction:: marv_robotics.trajectory.trajectory() 56 | 57 | 58 | .. _widget_nodes: 59 | 60 | Widget nodes 61 | ------------ 62 | 63 | Widgets are used by sections or are rendered in the special summary sections when configured in the :ref:`cfg_c_detail_summary_widgets` config key. 64 | 65 | .. autofunction:: marv_robotics.detail.bagmeta_table() 66 | .. autofunction:: marv_robotics.detail.summary_keyval() 67 | .. autofunction:: marv_robotics.detail.galleries() 68 | 69 | 70 | .. _section_nodes: 71 | 72 | Section nodes 73 | ------------- 74 | 75 | .. autofunction:: marv_robotics.detail.images_section() 76 | .. autofunction:: marv_robotics.detail.connections_section() 77 | .. autofunction:: marv_robotics.detail.trajectory_section() 78 | .. autofunction:: marv_robotics.detail.video_section() 79 | .. autofunction:: marv_robotics.detail.gnss_section() 80 | -------------------------------------------------------------------------------- /docs/scan.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | .. _scan: 5 | 6 | Scanner 7 | ======= 8 | 9 | .. autofunction:: marv_robotics.bag.dirscan 10 | .. autofunction:: marv_robotics.bag.scan 11 | -------------------------------------------------------------------------------- /docs/tutorial: -------------------------------------------------------------------------------- 1 | ../tutorial -------------------------------------------------------------------------------- /docs/views.rst: -------------------------------------------------------------------------------- 1 | .. Copyright 2016 - 2018 Ternaris. 2 | .. SPDX-License-Identifier: CC-BY-SA-4.0 3 | 4 | .. _views: 5 | 6 | Views 7 | ===== 8 | 9 | The frontend shipping with marv features: 10 | 11 | - a listing view served at the root of the application, 12 | - a detail view to introspect individual data sets 13 | 14 | Listing and detail view are specific to the selected collection. The displayed collection is selected via a drop-down in the header. 15 | 16 | 17 | Listing 18 | ------- 19 | 20 | The listing view consists of the following sections: 21 | 22 | - A filter section that determines the data sets loaded from the server into the listing view (see :ref:`cfg_c_filters`). 23 | - A summary on the loaded data sets (see :ref:`cfg_c_listing_summary`). 24 | - A set of fancy filter widgets to refine the search on loaded data sets (EE only). For the default set of filter widgets to appear all of the following :ref:`cfg_c_listing_columns` need to be available: ``size``, ``added``, ``start_time``, ``end_time``, and ``duration``. 25 | - A table listing the matching data sets (see :ref:`cfg_c_listing_columns`) 26 | 27 | 28 | Detail 29 | ------ 30 | 31 | In order to introspect individual data sets, the first listing column is usually configured to route to the detail view. 32 | 33 | The detail view consists of :ref:`cfg_c_detail_sections` organized as tabs. The first section is special in that it always exists. It displays a list of :ref:`cfg_c_detail_summary_widgets` and widgets for tagging and commenting. 34 | -------------------------------------------------------------------------------- /requirements/develop.in: -------------------------------------------------------------------------------- 1 | # building documentation 2 | Sphinx 3 | sphinx-autodoc-typehints 4 | sphinx-rtd-theme 5 | sphinx_paramlinks 6 | 7 | # development 8 | callee 9 | darglint 10 | flake8 11 | flake8-annotations 12 | flake8-bugbear 13 | flake8-commas 14 | flake8-comprehensions 15 | flake8-datetimez 16 | flake8-docstrings 17 | flake8-fixme 18 | flake8-isort 19 | flake8-mutable 20 | flake8-print 21 | flake8-pytest-style 22 | flake8-quotes 23 | flake8-return 24 | flake8-simplify 25 | flake8-type-checking 26 | flake8-use-fstring 27 | ipython 28 | pdbpp 29 | pep8-naming 30 | pip-tools 31 | pylint 32 | pytest 33 | pytest-aiohttp 34 | pytest-cov 35 | pytest-flake8 36 | pytest-mock 37 | pytest-pylint 38 | pytest-runner 39 | pytest-yapf3 40 | termcolor 41 | testfixtures 42 | watchdog 43 | -------------------------------------------------------------------------------- /requirements/marv-api.in: -------------------------------------------------------------------------------- 1 | cython 2 | pycapnp-for-marv==0.6.3 3 | pydantic==1.7.* 4 | -------------------------------------------------------------------------------- /requirements/marv-cli.in: -------------------------------------------------------------------------------- 1 | click 2 | -------------------------------------------------------------------------------- /requirements/marv-cli.txt: -------------------------------------------------------------------------------- 1 | click==7.1.2 \ 2 | --hash=sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a \ 3 | --hash=sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc 4 | -------------------------------------------------------------------------------- /requirements/marv-robotics.in: -------------------------------------------------------------------------------- 1 | -r marv-api.in 2 | lz4 3 | matplotlib==3.2.* # TODO: 3.3 breaks in bundle 4 | mpld3 5 | numpy 6 | pillow 7 | plotly 8 | pycryptodome 9 | rosbags 10 | ruamel.yaml 11 | utm 12 | zstandard 13 | -------------------------------------------------------------------------------- /requirements/marv.in: -------------------------------------------------------------------------------- 1 | -r marv-api.in 2 | -r marv-cli.in 3 | PyJWT 4 | aiohttp 5 | aiosqlite==0.12.0 # TODO: bundle breaks at least with 0.16.0 6 | bcrypt 7 | configparser 8 | gunicorn 9 | jinja2 10 | pendulum!=2.1.0 11 | pycapnp-for-marv==0.6.3 12 | pydantic==1.7.* 13 | pypika==0.37.1 # TODO: tortoise breaks at least with 0.37.16 14 | setproctitle 15 | tortoise-orm==0.15.* 16 | uvloop 17 | -------------------------------------------------------------------------------- /requirements/venv.in: -------------------------------------------------------------------------------- 1 | pip==20.1 2 | setuptools==46.1.3 3 | wheel 4 | -------------------------------------------------------------------------------- /requirements/venv.txt: -------------------------------------------------------------------------------- 1 | wheel==0.37.0 \ 2 | --hash=sha256:21014b2bd93c6d0034b6ba5d35e4eb284340e09d63c59aef6fc14b0f346146fd \ 3 | --hash=sha256:e2ef7239991699e3355d54f8e968a21bb940a1dbf34a4d226741e64462516fad 4 | 5 | # The following packages are considered to be unsafe in a requirements file: 6 | pip==20.1 \ 7 | --hash=sha256:4fdc7fd2db7636777d28d2e1432e2876e30c2b790d461f135716577f73104369 \ 8 | --hash=sha256:572c0f25eca7c87217b21f6945b7192744103b18f4e4b16b8a83b227a811e192 9 | setuptools==46.1.3 \ 10 | --hash=sha256:4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee \ 11 | --hash=sha256:795e0475ba6cd7fa082b1ee6e90d552209995627a2a227a47c6ea93282f4bfb1 12 | -------------------------------------------------------------------------------- /scripts/build-docs: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2016 - 2018 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | set -e 7 | 8 | cd "$(dirname "$(realpath "$0")")"/../docs 9 | 10 | sphinx-build --help >/dev/null 2>&1 || ( 11 | echo "ERROR: sphinx-build is not available, have you activated the virtualenv?" 12 | exit 1 13 | ) 14 | 15 | make clean 16 | make html 17 | rm -rf ../code/marv/marv/app/docs 18 | cp -a ./_build/html ../code/marv/marv/app/docs 19 | -------------------------------------------------------------------------------- /scripts/build-image: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2016 - 2018 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | set -e 7 | 8 | cd "$(dirname "$(realpath "$0")")"/.. 9 | 10 | BASENAME="$(basename "$PWD" |tr '[:upper:]' '[:lower:]')" 11 | 12 | if [[ -n "$MARV_IMAGE" ]]; then 13 | IMAGE="$MARV_IMAGE" 14 | elif [[ -e .image-name ]]; then 15 | IMAGE="$(<.image-name)" 16 | else 17 | IMAGE="$BASENAME" 18 | fi 19 | 20 | ./scripts/fetch-deps 21 | docker build -t "$IMAGE" "$@" . 22 | -------------------------------------------------------------------------------- /scripts/download-test-bags: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2016 - 2018 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | cd "$(dirname "$(realpath "$0")")"/.. 7 | 8 | curl https://ternaris.com/marv-robotics/marv_robotics.tests.data.tar.gz | \ 9 | tar xz -C code/marv/marv_node/testing/_robotics_tests 10 | -------------------------------------------------------------------------------- /scripts/enter-container: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2016 - 2018 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | set -e 7 | 8 | cd "$(dirname "$(realpath "$0")")"/.. 9 | 10 | BASENAME="$(basename "$PWD")" 11 | NAME="${MARV_CONTAINER_NAME:-$BASENAME}" 12 | CONTAINER_USER="${MARV_CONTAINER_USER:-marv}" 13 | 14 | declare -a cmd 15 | 16 | if [[ -n "$1" ]]; then 17 | cmd=( bash -lic "$*" ) 18 | else 19 | cmd=( bash -li ) 20 | fi 21 | 22 | docker exec \ 23 | -e COLORFGBG \ 24 | -e TERM \ 25 | -ti \ 26 | -u "$CONTAINER_USER" \ 27 | "$NAME" \ 28 | "${cmd[@]}" 29 | -------------------------------------------------------------------------------- /scripts/fetch-deps: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | # Copyright 2016 - 2020 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | set -efu 7 | 8 | cd "$(dirname "$(realpath "$0")")"/.. 9 | 10 | 11 | fetch() { 12 | OIFS=$IFS 13 | IFS=@ 14 | # shellcheck disable=SC2086 15 | set -- $1 16 | IFS=$OIFS 17 | target="${1:?target is missing}" 18 | strip="${2:?strip is missing}" 19 | url="${3:?url is missing}" 20 | echo "Fetching $target" 21 | rm -rf "$target" 22 | mkdir "$target" 23 | curl -sL "$url" | tar xz --strip-components "$strip" -C "$target" 24 | } 25 | 26 | if cmp -s .deps .deps-fetched; then 27 | exit 0 28 | fi 29 | rm -f .deps-fetched 30 | 31 | while read -r dep; do 32 | fetch "$dep" 33 | done < .deps 34 | 35 | cp .deps .deps-fetched 36 | -------------------------------------------------------------------------------- /scripts/run-container: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2016 - 2020 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | set -e 7 | 8 | usage() { 9 | echo 10 | echo "Usage: run-container SITE SCANROOT [EXTRA_OPTS ...]" 11 | echo 12 | echo "The site is expected to contain marv.conf." 13 | echo "It is mounted inside the container at /home/marv/site." 14 | echo 15 | echo "The scanroot contains the log files for one or more collections." 16 | echo "It is mounted read-only into the container at /scanroot." 17 | echo 18 | echo "All additional arguments are passed as options to docker run." 19 | echo 20 | exit 1 21 | } 22 | 23 | if [[ -n "$DEBUG" ]]; then 24 | set -x 25 | fi 26 | 27 | SITE="$1"; shift || usage 28 | [[ -e "$SITE"/marv.conf ]] || usage 29 | SCANROOT="$1"; shift || usage 30 | [[ -d "$SCANROOT" ]] || usage 31 | SITE="$(realpath "$SITE")" 32 | SCANROOT="$(realpath "$SCANROOT")" 33 | 34 | cd "$(dirname "$(realpath "$0")")"/.. 35 | 36 | BASENAME="$(basename "$PWD" |tr '[:upper:]' '[:lower:]')" 37 | NAME="${MARV_CONTAINER_NAME:-$BASENAME}" 38 | HOSTNAME="${NAME//./-}" 39 | HTTP="${MARV_HTTP:-127.0.0.1:8000}" 40 | 41 | if [[ -n "$MARV_IMAGE" ]]; then 42 | IMAGE="$MARV_IMAGE" 43 | elif [[ -e .image-name ]]; then 44 | IMAGE="$(<.image-name)" 45 | else 46 | IMAGE="$BASENAME" 47 | fi 48 | 49 | get_timezone() { 50 | tz=$(cat /etc/timezone 2>/dev/null|| true) 51 | if [[ -z "$tz" ]]; then 52 | link=$(realpath /etc/localtime) 53 | tz="$(basename "$(dirname "$link")")/$(basename "$link")" 54 | fi 55 | echo $tz 56 | } 57 | 58 | export TIMEZONE="${MARV_TIMEZONE:-$(get_timezone)}" 59 | 60 | ./scripts/fetch-deps 61 | 62 | docker stop "$NAME" || true 63 | docker rm "$NAME" || true 64 | docker run \ 65 | --name "$NAME" \ 66 | --hostname "$HOSTNAME" \ 67 | --restart unless-stopped \ 68 | -e COLORFGBG \ 69 | -e "DEVELOP=${DEVELOP:+/home/marv/code}" \ 70 | -e DEBUG \ 71 | -e MARV_APPLICATION_ROOT \ 72 | -e MARV_ARGS \ 73 | -e MARV_INIT \ 74 | -e MARV_UID="${MARV_UID:-$(id -u)}" \ 75 | -e MARV_GID="${MARV_GID:-$(id -g)}" \ 76 | -e TERM \ 77 | -e TIMEZONE \ 78 | -p "$HTTP:8000" \ 79 | -v "$PWD/.docker/entrypoint.sh:/marv_entrypoint.sh" \ 80 | -v "$PWD/code:/home/marv/code" \ 81 | -v "$PWD/docs:/home/marv/docs" \ 82 | -v "$PWD/requirements:/home/marv/requirements" \ 83 | -v "$PWD/scripts:/home/marv/scripts" \ 84 | -v "$PWD/tutorial:/home/marv/tutorial" \ 85 | -v "$PWD/CHANGES.rst:/home/marv/CHANGES.rst" \ 86 | -v "$PWD/CONTRIBUTING.rst:/home/marv/CONTRIBUTING.rst" \ 87 | -v "$(realpath "$SITE"):/home/marv/site" \ 88 | -v "$(realpath "$SCANROOT"):/scanroot:ro" \ 89 | "$@" \ 90 | "$IMAGE" 91 | -------------------------------------------------------------------------------- /scripts/setup-venv: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2016 - 2018 Ternaris. 4 | # SPDX-License-Identifier: AGPL-3.0-only 5 | 6 | set -e 7 | 8 | usage() { 9 | echo 10 | echo "Usage: setup-venv REQUIREMENTS TARGET" 11 | echo 12 | echo "Example: ./scripts/setup-venv requirements/marv-robotics.txt venv" 13 | echo 14 | exit 1 15 | } 16 | 17 | REQUIREMENTS="$1"; shift || usage 18 | TARGET="$1"; shift || usage 19 | EXTRA_CODE="$1" # intentionally undocumented and subject to change 20 | if [[ -n "$EXTRA_CODE" ]]; then 21 | EXTRA_CODE="$(realpath "$EXTRA_CODE")" 22 | fi 23 | [[ ! -d "$TARGET" ]] || (echo "The target directory must not exist, yet."; exit 1) 24 | PIP_ARGS=${PIP_ARGS:--e} # develop install by default 25 | 26 | REQUIREMENTS="$(realpath "$REQUIREMENTS")" 27 | TARGET="$(realpath "$TARGET")" 28 | 29 | cd "$(dirname "$(realpath "$0")")"/.. 30 | 31 | ./scripts/fetch-deps 32 | 33 | export PIP_DISABLE_PIP_VERSION_CHECK=1 34 | 35 | python3.8 -m venv "$TARGET" 36 | "$TARGET"/bin/pip install -Ur requirements/venv.txt 37 | "$TARGET"/bin/pip install -Uc requirements/marv.txt cython 38 | "$TARGET"/bin/pip install -Ur requirements/marv.txt 39 | "$TARGET"/bin/pip install -Ur "$REQUIREMENTS" 40 | "$TARGET"/bin/pip install opencv-python-headless==4.3.0.36 41 | "$TARGET"/bin/pip install -Ur requirements/develop.txt 42 | 43 | # Install all python distributions directly in code 44 | find code -maxdepth 2 -name setup.py -execdir "$TARGET"/bin/pip install --no-deps $PIP_ARGS . \; 45 | 46 | if [[ -n "$EXTRA_CODE" ]]; then 47 | find "$EXTRA_CODE" -maxdepth 2 -name setup.py -execdir "$TARGET"/bin/pip install --no-deps $PIP_ARGS . \; 48 | fi 49 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [aliases] 2 | test=pytest 3 | 4 | [coverage:report] 5 | exclude_lines = 6 | pragma: no cover 7 | if TYPE_CHECKING: 8 | if __name__ == '__main__': 9 | 10 | [flake8] 11 | avoid-escape = False 12 | docstring_convention = google 13 | docstring_style = google 14 | extend-select = 15 | # docstrings 16 | D204, 17 | D400, 18 | D401, 19 | D404, 20 | D413, 21 | ignore = 22 | # do not enforce check error first 23 | SIM106, 24 | # allow line break after binary operator 25 | W504, 26 | # 27 | # TODO: do not check annotations for now 28 | ANN, 29 | # TODO: Missing docstrings ok for now 30 | D1, 31 | # TODO: rely on pylint line length check for now 32 | E501, 33 | # TODO: disable upcase variable check for now 34 | N806, 35 | # TODO: allow TODO for now 36 | T101, 37 | max-line-length = 100 38 | strictness = long 39 | suppress-none-returning = True 40 | 41 | [isort] 42 | include_trailing_comma = True 43 | line_length = 100 44 | multi_line_output = 3 45 | known_first_party = 46 | marv, 47 | marv_api, 48 | marv_cli, 49 | marv_detail, 50 | marv_node, 51 | marv_nodes, 52 | marv_pycapnp, 53 | marv_robotics, 54 | marv_ros, 55 | marv_store, 56 | marv_webapi, 57 | 58 | [mypy] 59 | ignore_missing_imports = True 60 | 61 | [pydocstyle] 62 | convention = google 63 | add-select = D204,D400,D401,D404,D413 64 | 65 | [pylint.MASTER] 66 | max-line-length = 100 67 | disable = 68 | duplicate-code, 69 | ungrouped-imports, 70 | # 71 | consider-using-f-string, 72 | cyclic-import, 73 | fixme, 74 | missing-class-docstring, 75 | missing-function-docstring, 76 | missing-module-docstring, 77 | raise-missing-from, 78 | redefined-builtin, 79 | exclude-protected = 80 | _asdict, 81 | _meta, 82 | _table_name, 83 | good-names = 84 | _, 85 | e, 86 | ep, 87 | f, 88 | fd, 89 | id, 90 | kw, 91 | log, 92 | np, 93 | pd, 94 | rv, 95 | ts 96 | max-args = 6 97 | ignored-modules = 98 | capnp.lib.capnp, 99 | cv2, 100 | marv_detail.types_capnp, 101 | marv_nodes.types_capnp, 102 | marv_pycapnp.types_capnp, 103 | pydantic, 104 | 105 | [yapf] 106 | based_on_style = google 107 | column_limit = 100 108 | allow_split_before_dict_value = false 109 | dedent_closing_brackets = true 110 | indent_dictionary_value = false 111 | 112 | [tool:pytest] 113 | addopts = 114 | -v 115 | --flake8 116 | --pylint 117 | --pylint-jobs=0 118 | --yapf 119 | --cov=code 120 | --cov-branch 121 | --cov-report=html 122 | --cov-report=term 123 | --no-cov-on-fail 124 | --ignore=docs/conf.py 125 | # exclude the symlinked tutorials 126 | --ignore-glob=docs/tutorial/** 127 | --ignore-glob='flycheck_*' 128 | --ignore-glob='**/flycheck_*' 129 | --junitxml=pytest-report.xml 130 | junit_family = xunit2 131 | markers = 132 | marv 133 | marv_conf 134 | testpaths = 135 | code 136 | docs 137 | tutorial 138 | -------------------------------------------------------------------------------- /sites/example/marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | # Use next line to run behind nginx 4 | # reverse_proxy = nginx 5 | 6 | 7 | [collection bags] 8 | scanner = marv_robotics.bag:scan 9 | 10 | scanroots = 11 | /scanroot 12 | 13 | nodes = 14 | marv_nodes:dataset 15 | marv_robotics.bag:bagmeta 16 | marv_robotics.cam:ffmpeg 17 | marv_robotics.cam:images 18 | marv_robotics.detail:bagmeta_table 19 | marv_robotics.detail:connections_section 20 | marv_robotics.detail:gnss_section 21 | marv_robotics.detail:images_section 22 | marv_robotics.detail:summary_keyval 23 | marv_robotics.detail:trajectory_section 24 | marv_robotics.detail:video_section 25 | # marv_robotics.fulltext:fulltext 26 | marv_robotics.gnss:gnss_plots 27 | marv_robotics.motion:acceleration 28 | marv_robotics.motion:distance_gps 29 | marv_robotics.motion:motion_section 30 | marv_robotics.motion:speed 31 | marv_robotics.trajectory:trajectory 32 | 33 | filters = 34 | # id | Display Name | operators | value type | value function 35 | name | Name | substring | string | (get "dataset.name") 36 | setid | Set Id | startswith | string | (get "dataset.id") 37 | size | Size | lt le eq ne ge gt | filesize | (sum (get "dataset.files[:].size")) 38 | status | Status | any all | subset | (status) 39 | tags | Tags | any all | subset | (tags) 40 | comments | Comments | substring | string | (comments) 41 | # fulltext | Fulltext | words | words | (get "fulltext.words") 42 | files | File paths | substring_any | string[] | (get "dataset.files[:].path") 43 | added_time | Added | lt le eq ne ge gt | datetime | (get "dataset.time_added") 44 | start_time | Start time | lt le eq ne ge gt | datetime | (get "bagmeta.start_time") 45 | end_time | End time | lt le eq ne ge gt | datetime | (get "bagmeta.end_time") 46 | duration | Duration | lt le eq ne ge gt | timedelta | (get "bagmeta.duration") 47 | topics | Topics | any all | subset | (get "bagmeta.topics") 48 | msg_types | Message types | any all | subset | (get "bagmeta.msg_types") 49 | 50 | listing_columns = 51 | # id | Heading | formatter | value function 52 | name | Name | route | (detail_route (get "dataset.id") (get "dataset.name")) 53 | size | Size | filesize | (sum (get "dataset.files[:].size")) 54 | tags | Tags | pill[] | (tags) 55 | added | Added | datetime | (get "dataset.time_added") 56 | start_time | Start time | datetime | (get "bagmeta.start_time") 57 | duration | Duration | timedelta | (get "bagmeta.duration") 58 | max_speed | Max speed | speed | (max (get "speed[:].value")) 59 | distance | Distance | distance | (sum (get "distance_gps[:].value")) 60 | 61 | listing_sort = start_time | descending 62 | 63 | listing_summary = 64 | # id | Title | formatter | extractor 65 | datasets | datasets | int | (len (rows)) 66 | size | size | filesize | (sum (rows "size" 0)) 67 | duration | duration | timedelta | (sum (rows "duration" 0)) 68 | distance | distance | distance | (sum (rows "distance" 0)) 69 | 70 | detail_summary_widgets = 71 | summary_keyval 72 | bagmeta_table 73 | 74 | detail_sections = 75 | connections_section 76 | images_section 77 | video_section 78 | gnss_section 79 | motion_section 80 | trajectory_section 81 | -------------------------------------------------------------------------------- /tutorial/.gitignore: -------------------------------------------------------------------------------- 1 | /scanroot 2 | /*/db/ 3 | /*/store 4 | -------------------------------------------------------------------------------- /tutorial/code/.gitignore: -------------------------------------------------------------------------------- 1 | *.egg-info 2 | *.py[co] 3 | -------------------------------------------------------------------------------- /tutorial/code/requirements.in: -------------------------------------------------------------------------------- 1 | -r ../marv/requirements-develop.txt 2 | matplotlib 3 | mpld3 4 | pillow 5 | plotly 6 | -------------------------------------------------------------------------------- /tutorial/code/setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2016 - 2018 Ternaris. 2 | # SPDX-License-Identifier: CC0-1.0 3 | 4 | from setuptools import setup 5 | 6 | setup( 7 | name='marv-tutorial-code', 8 | version='1.0', 9 | description='MARV Tutorial Code', 10 | url='', 11 | author='Ternaris', 12 | author_email='team@ternaris.com', 13 | license='CC0-1.0', 14 | packages=['marv_tutorial'], 15 | install_requires=[ 16 | 'marv', 17 | 'marv-robotics', 18 | 'matplotlib', 19 | 'mpld3', 20 | 'plotly', 21 | ], 22 | include_package_data=True, 23 | zip_safe=False, 24 | ) 25 | -------------------------------------------------------------------------------- /tutorial/docs-only-site/marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | 4 | [collection bags] 5 | scanner = marv_robotics.bag:scan 6 | scanroots = 7 | scanroot 8 | -------------------------------------------------------------------------------- /tutorial/docs-only-site/scanroot/.keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ternaris/marv-robotics/473ca28d85ac55a7190edaa19347936ce3a6553a/tutorial/docs-only-site/scanroot/.keep -------------------------------------------------------------------------------- /tutorial/docs-only-site/sessionkey: -------------------------------------------------------------------------------- 1 | bfb823c0-0716-472f-b554-ffd08a4e4cca -------------------------------------------------------------------------------- /tutorial/setup-basic-site0/marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | 4 | [collection bags] 5 | scanner = marv_robotics.bag:scan 6 | scanroots = 7 | ./scanroot 8 | -------------------------------------------------------------------------------- /tutorial/setup-basic-site0/sessionkey: -------------------------------------------------------------------------------- 1 | 8448745b-f301-4c68-8084-5b2728f0cc7e -------------------------------------------------------------------------------- /tutorial/setup-basic-site1/marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | 4 | [collection bags] 5 | scanner = marv_robotics.bag:scan 6 | scanroots = 7 | ./scanroot 8 | 9 | nodes = 10 | marv_nodes:dataset 11 | marv_robotics.bag:bagmeta 12 | marv_robotics.detail:bagmeta_table 13 | marv_robotics.detail:connections_section 14 | 15 | detail_summary_widgets = 16 | bagmeta_table 17 | 18 | detail_sections = 19 | connections_section 20 | -------------------------------------------------------------------------------- /tutorial/setup-basic-site1/sessionkey: -------------------------------------------------------------------------------- 1 | 8448745b-f301-4c68-8084-5b2728f0cc7e -------------------------------------------------------------------------------- /tutorial/write-your-own0/marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | 4 | [collection bags] 5 | scanner = marv_robotics.bag:scan 6 | scanroots = 7 | ./scanroot 8 | 9 | nodes = 10 | marv_nodes:dataset 11 | marv_robotics.bag:bagmeta 12 | marv_robotics.detail:bagmeta_table 13 | marv_robotics.detail:connections_section 14 | marv_tutorial:image 15 | marv_tutorial:image_section 16 | 17 | detail_summary_widgets = 18 | bagmeta_table 19 | 20 | detail_sections = 21 | connections_section 22 | image_section 23 | -------------------------------------------------------------------------------- /tutorial/write-your-own0/sessionkey: -------------------------------------------------------------------------------- 1 | 8448745b-f301-4c68-8084-5b2728f0cc7e -------------------------------------------------------------------------------- /tutorial/write-your-own1/marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | 4 | [collection bags] 5 | scanner = marv_robotics.bag:scan 6 | scanroots = 7 | ./scanroot 8 | 9 | nodes = 10 | marv_nodes:dataset 11 | marv_robotics.bag:bagmeta 12 | marv_robotics.detail:bagmeta_table 13 | marv_robotics.detail:connections_section 14 | marv_tutorial:image 15 | marv_tutorial:image_section 16 | marv_tutorial:images 17 | marv_tutorial:gallery_section 18 | 19 | detail_summary_widgets = 20 | bagmeta_table 21 | 22 | detail_sections = 23 | connections_section 24 | image_section 25 | gallery_section 26 | -------------------------------------------------------------------------------- /tutorial/write-your-own1/sessionkey: -------------------------------------------------------------------------------- 1 | 8448745b-f301-4c68-8084-5b2728f0cc7e -------------------------------------------------------------------------------- /tutorial/write-your-own2/marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | 4 | [collection bags] 5 | scanner = marv_robotics.bag:scan 6 | scanroots = 7 | ./scanroot 8 | 9 | nodes = 10 | marv_nodes:dataset 11 | marv_robotics.bag:bagmeta 12 | marv_robotics.detail:bagmeta_table 13 | marv_robotics.detail:connections_section 14 | marv_tutorial:image 15 | marv_tutorial:image_section 16 | marv_tutorial:images 17 | marv_tutorial:gallery_section 18 | marv_tutorial:combined_section 19 | 20 | detail_summary_widgets = 21 | bagmeta_table 22 | 23 | detail_sections = 24 | connections_section 25 | image_section 26 | gallery_section 27 | combined_section 28 | -------------------------------------------------------------------------------- /tutorial/write-your-own2/sessionkey: -------------------------------------------------------------------------------- 1 | 8448745b-f301-4c68-8084-5b2728f0cc7e -------------------------------------------------------------------------------- /tutorial/write-your-own3/marv.conf: -------------------------------------------------------------------------------- 1 | [marv] 2 | collections = bags 3 | 4 | [collection bags] 5 | scanner = marv_robotics.bag:scan 6 | scanroots = 7 | ./scanroot 8 | 9 | nodes = 10 | marv_nodes:dataset 11 | marv_robotics.bag:bagmeta 12 | marv_robotics.detail:bagmeta_table 13 | marv_robotics.detail:connections_section 14 | marv_tutorial:image 15 | marv_tutorial:image_section 16 | marv_tutorial:images 17 | marv_tutorial:gallery_section 18 | marv_tutorial:filesize_plot 19 | marv_tutorial:combined_section 20 | 21 | detail_summary_widgets = 22 | bagmeta_table 23 | 24 | detail_sections = 25 | connections_section 26 | image_section 27 | gallery_section 28 | combined_section 29 | -------------------------------------------------------------------------------- /tutorial/write-your-own3/sessionkey: -------------------------------------------------------------------------------- 1 | 8448745b-f301-4c68-8084-5b2728f0cc7e --------------------------------------------------------------------------------