├── .dockerignore ├── .github └── workflows │ ├── gpu.yml │ ├── main.yml │ ├── pre-commit.yml │ └── release.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CODEOWNERS ├── MANIFEST.in ├── README.md ├── deepethogram ├── __init__.py ├── __main__.py ├── base.py ├── callbacks.py ├── conf │ ├── __init__.py │ ├── augs.yaml │ ├── config.yaml │ ├── debug.yaml │ ├── gui.yaml │ ├── inference.yaml │ ├── model │ │ ├── feature_extractor.yaml │ │ ├── flow_generator.yaml │ │ └── sequence.yaml │ ├── postprocessor.yaml │ ├── preset │ │ ├── deg_f.yaml │ │ ├── deg_m.yaml │ │ └── deg_s.yaml │ ├── project │ │ └── project_config_default.yaml │ ├── train.yaml │ ├── tune │ │ ├── feature_extractor.yaml │ │ ├── sequence.yaml │ │ └── tune.yaml │ └── zscore.yaml ├── configuration.py ├── data │ ├── __init__.py │ ├── augs.py │ ├── dali.py │ ├── dataloaders.py │ ├── datasets.py │ ├── keypoint_utils.py │ └── utils.py ├── debug.py ├── feature_extractor │ ├── __init__.py │ ├── __main__.py │ ├── inference.py │ ├── losses.py │ ├── models │ │ ├── CNN.py │ │ ├── __init__.py │ │ ├── classifiers │ │ │ ├── __init__.py │ │ │ ├── alexnet.py │ │ │ ├── densenet.py │ │ │ ├── inception.py │ │ │ ├── resnet.py │ │ │ ├── resnet3d.py │ │ │ ├── squeezenet.py │ │ │ └── vgg.py │ │ ├── hidden_two_stream.py │ │ └── utils.py │ └── train.py ├── file_io.py ├── flow_generator │ ├── __init__.py │ ├── __main__.py │ ├── inference.py │ ├── losses.py │ ├── models │ │ ├── FlowNetS.py │ │ ├── MotionNet.py │ │ ├── TinyMotionNet.py │ │ ├── TinyMotionNet3D.py │ │ ├── __init__.py │ │ └── components.py │ ├── train.py │ └── utils.py ├── gui │ ├── __init__.py │ ├── custom_widgets.py │ ├── icons │ │ ├── noun_Home_1158721.png │ │ ├── noun_Zoom In_744781.png │ │ ├── noun_pause_159135.png │ │ ├── noun_play_1713293.png │ │ └── noun_tap_145047.png │ ├── main.py │ ├── mainwindow.py │ ├── mainwindow.ui │ └── menus_and_popups.py ├── losses.py ├── metrics.py ├── postprocessing.py ├── projects.py ├── schedulers.py ├── sequence │ ├── __init__.py │ ├── __main__.py │ ├── inference.py │ ├── models │ │ ├── __init__.py │ │ ├── mlp.py │ │ ├── sequence.py │ │ └── tgm.py │ └── train.py ├── stoppers.py ├── tune │ ├── __init__.py │ ├── feature_extractor.py │ ├── sequence.py │ └── utils.py ├── utils.py ├── viz.py └── zscore.py ├── docker ├── Dockerfile-full ├── Dockerfile-gui └── Dockerfile-headless ├── docs ├── beta.md ├── code_examples.md ├── docker.md ├── file_structure.md ├── getting_started.md ├── images │ ├── deepethogram_schematic.png │ ├── ethogram_schematic.png │ ├── gui_annotated.png │ ├── label_format.png │ ├── project_config.png │ └── workflow.png ├── installation.md ├── model_performance.md ├── performance.md ├── testing.md ├── troubleshooting.md ├── using_CLI.md ├── using_code.md ├── using_config_files.md ├── using_gui.md └── using_tune.md ├── environment.yml ├── license.txt ├── pyproject.toml ├── pytest.ini ├── requirements.txt ├── reset_venv.sh ├── setup.py ├── setup_tests.py └── tests ├── setup_data.py ├── test_data.py ├── test_flow_generator.py ├── test_gui.py ├── test_integration.py ├── test_models.py ├── test_projects.py ├── test_softmax.py └── test_z_score.py /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.git 2 | **/.gitignore 3 | **/.vscode 4 | **/coverage 5 | **/.env 6 | **/.aws 7 | **/.ssh 8 | Dockerfile 9 | docker-compose.yml 10 | **/.DS_Store 11 | **/venv 12 | **/env 13 | build/* 14 | dist/* 15 | docs/* 16 | docker/ 17 | -------------------------------------------------------------------------------- /.github/workflows/gpu.yml: -------------------------------------------------------------------------------- 1 | # Temporarily disabled - requires GitHub Teams plan for GPU runners 2 | # name: GPU Tests 3 | # 4 | # on: 5 | # push: 6 | # branches: [ master ] 7 | # pull_request: 8 | # branches: [ master ] 9 | # 10 | # jobs: 11 | # gpu-test: 12 | # runs-on: ubuntu-20.04 13 | # 14 | # steps: 15 | # - uses: actions/checkout@v3 16 | # 17 | # - name: Set up Python 3.7 18 | # uses: actions/setup-python@v4 19 | # with: 20 | # python-version: '3.7' 21 | # 22 | # - name: Install FFMPEG 23 | # run: | 24 | # sudo apt-get update 25 | # sudo apt-get install -y ffmpeg 26 | # 27 | # - name: Install PySide2 28 | # run: | 29 | # python -m pip install --upgrade pip 30 | # pip install "pyside2==5.13.2" 31 | # 32 | # - name: Install PyTorch with CUDA 33 | # run: | 34 | # pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 -f https://download.pytorch.org/whl/torch_stable.html 35 | # 36 | # - name: Install package and test dependencies 37 | # run: | 38 | # python -m pip install --upgrade "pip<24.0" 39 | # pip install -r requirements.txt 40 | # pip install pytest pytest-cov 41 | # python setup.py develop 42 | # 43 | # - name: Setup test data 44 | # run: | 45 | # python setup_tests.py 46 | # 47 | # - name: GPU Tests 48 | # run: | 49 | # pytest -v -m "gpu" tests/ 50 | # env: 51 | # CUDA_VISIBLE_DEVICES: 0 52 | -------------------------------------------------------------------------------- /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: CPU Tests 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | test: 11 | name: Test on ${{ matrix.os }} 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | os: [ubuntu-20.04, windows-latest, macos-13] 17 | 18 | steps: 19 | - uses: actions/checkout@v3 20 | 21 | - name: Set up Python 3.7 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: '3.7' 25 | 26 | - name: Install FFMPEG (Ubuntu) 27 | if: matrix.os == 'ubuntu-20.04' 28 | run: | 29 | sudo apt-get update 30 | sudo apt-get install -y ffmpeg 31 | 32 | - name: Install FFMPEG (Windows) 33 | if: matrix.os == 'windows-latest' 34 | run: | 35 | choco install ffmpeg 36 | 37 | - name: Install FFMPEG (macOS) 38 | if: matrix.os == 'macos-13' 39 | run: | 40 | brew install ffmpeg 41 | 42 | - name: Install PySide2 43 | run: | 44 | python -m pip install --upgrade pip 45 | pip install "pyside2==5.13.2" 46 | 47 | - name: Install PyTorch CPU 48 | run: | 49 | pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu 50 | 51 | - name: Install package and test dependencies 52 | run: | 53 | python -m pip install --upgrade "pip<24.0" 54 | pip install -r requirements.txt 55 | pip install pytest pytest-cov 56 | python setup.py develop 57 | 58 | - name: Setup test data 59 | run: | 60 | python setup_tests.py 61 | 62 | - name: Run CPU tests 63 | run: | 64 | pytest -v -m "not gpu" tests/ 65 | -------------------------------------------------------------------------------- /.github/workflows/pre-commit.yml: -------------------------------------------------------------------------------- 1 | name: Pre-commit 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | pre-commit: 11 | runs-on: ubuntu-20.04 12 | steps: 13 | - uses: actions/checkout@v3 14 | 15 | - name: Set up Python 3.7 16 | uses: actions/setup-python@v4 17 | with: 18 | python-version: '3.7' 19 | 20 | - name: Install dependencies 21 | run: | 22 | python -m pip install --upgrade pip 23 | pip install pre-commit ruff 24 | 25 | - name: Run pre-commit 26 | run: | 27 | pre-commit install 28 | pre-commit run --all-files 29 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Publish Python 🐍 distribution 📦 to PyPI 2 | 3 | on: 4 | push: 5 | # Only run this workflow when a tag with the pattern 'v*' is pushed 6 | tags: 7 | - 'v*' 8 | 9 | jobs: 10 | # Step 1: Build the Python package 11 | build: 12 | name: Build distribution 📦 13 | runs-on: ubuntu-20.04 14 | steps: 15 | - uses: actions/checkout@v4 16 | with: 17 | persist-credentials: false 18 | 19 | - name: Set up Python 20 | uses: actions/setup-python@v4 21 | with: 22 | python-version: '3.7' 23 | 24 | - name: Install build dependencies 25 | run: | 26 | python -m pip install --upgrade pip 27 | pip install build pytest 28 | pip install -e . 29 | 30 | - name: Run tests 31 | run: pytest tests/ 32 | 33 | - name: Build package 34 | run: python -m build 35 | 36 | - name: Store the distribution packages 37 | uses: actions/upload-artifact@v4 38 | with: 39 | name: python-package-distributions 40 | path: dist/ 41 | 42 | # Step 2: Publish the distribution to PyPI 43 | publish-to-pypi: 44 | name: Publish to PyPI 45 | needs: build 46 | runs-on: ubuntu-latest 47 | 48 | steps: 49 | - name: Download distribution packages 50 | uses: actions/download-artifact@v4 51 | with: 52 | name: python-package-distributions 53 | path: dist/ 54 | 55 | - name: Publish to PyPI 56 | uses: pypa/gh-action-pypi-publish@v1.12.3 57 | with: 58 | # If using a secret-based token: 59 | username: '__token__' 60 | password: ${{ secrets.PYPI_API_TOKEN }} 61 | 62 | # Step 3: Sign the distribution and create a GitHub release 63 | github-release: 64 | name: Sign the distribution 📦 with Sigstore and upload to GitHub Release 65 | needs: publish-to-pypi 66 | runs-on: ubuntu-latest 67 | permissions: 68 | contents: write # Required to create GitHub Releases 69 | id-token: write # Required for sigstore 70 | 71 | steps: 72 | - name: Download distribution packages 73 | uses: actions/download-artifact@v4 74 | with: 75 | name: python-package-distributions 76 | path: dist/ 77 | 78 | - name: Sign the dists with Sigstore 79 | uses: sigstore/gh-action-sigstore-python@v3.0.0 80 | with: 81 | inputs: >- 82 | ./dist/*.tar.gz 83 | ./dist/*.whl 84 | 85 | - name: Create GitHub Release 86 | env: 87 | GITHUB_TOKEN: ${{ github.token }} 88 | run: | 89 | # $GITHUB_REF_NAME is the tag name, e.g. 'v1.0.0' 90 | gh release create "$GITHUB_REF_NAME" \ 91 | --repo "$GITHUB_REPOSITORY" \ 92 | --title "Release $GITHUB_REF_NAME" \ 93 | --notes "See CHANGELOG for details." 94 | 95 | - name: Upload artifact signatures to GitHub Release 96 | env: 97 | GITHUB_TOKEN: ${{ github.token }} 98 | run: | 99 | gh release upload "$GITHUB_REF_NAME" dist/** \ 100 | --repo "$GITHUB_REPOSITORY" 101 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # never add models 2 | *.h5 3 | *.pt 4 | deepethogram/models/* 5 | tests/DATA/testing_deepethogram 6 | tests/DATA/testing_deepethogram_archive 7 | tests/DATA/testing_deepethogram_archive.zip 8 | gui_logs/ 9 | .vscode/ 10 | # line profiling 11 | *.lprof 12 | 13 | *.user 14 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 15 | # User-specific stuff 16 | *.iml 17 | *.xml 18 | /.idea/* 19 | .idea/**/* 20 | /.idea/ 21 | .idea/ 22 | .idea/* 23 | .idea/workspace.xml 24 | /.idea/workspace.xml 25 | .idea/**/workspace.xml 26 | 27 | # Byte-compiled / optimized / DLL files 28 | __pycache__/ 29 | *.py[cod] 30 | *$py.class 31 | 32 | # C extensions 33 | *.so 34 | 35 | # Distribution / packaging 36 | .Python 37 | build/ 38 | develop-eggs/ 39 | dist/ 40 | downloads/ 41 | eggs/ 42 | .eggs/ 43 | lib/ 44 | lib64/ 45 | parts/ 46 | sdist/ 47 | var/ 48 | wheels/ 49 | pip-wheel-metadata/ 50 | share/python-wheels/ 51 | *.egg-info/ 52 | .installed.cfg 53 | *.egg 54 | MANIFEST 55 | 56 | # PyInstaller 57 | # Usually these files are written by a python script from a template 58 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 59 | *.manifest 60 | *.spec 61 | 62 | # Installer logs 63 | pip-log.txt 64 | pip-delete-this-directory.txt 65 | 66 | # Unit test / coverage reports 67 | htmlcov/ 68 | .tox/ 69 | .nox/ 70 | .coverage 71 | .coverage.* 72 | .cache 73 | nosetests.xml 74 | coverage.xml 75 | *.cover 76 | *.py,cover 77 | .hypothesis/ 78 | .pytest_cache/ 79 | 80 | # Translations 81 | *.mo 82 | *.pot 83 | 84 | # Django stuff: 85 | *.log 86 | local_settings.py 87 | db.sqlite3 88 | db.sqlite3-journal 89 | 90 | # Flask stuff: 91 | instance/ 92 | .webassets-cache 93 | 94 | # Scrapy stuff: 95 | .scrapy 96 | 97 | # Sphinx documentation 98 | docs/_build/ 99 | 100 | # PyBuilder 101 | target/ 102 | 103 | # Jupyter Notebook 104 | .ipynb_checkpoints 105 | 106 | # IPython 107 | profile_default/ 108 | ipython_config.py 109 | 110 | # pyenv 111 | .python-version 112 | 113 | # pipenv 114 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 115 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 116 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 117 | # install all needed dependencies. 118 | #Pipfile.lock 119 | 120 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 121 | __pypackages__/ 122 | 123 | # Celery stuff 124 | celerybeat-schedule 125 | celerybeat.pid 126 | 127 | # SageMath parsed files 128 | *.sage.py 129 | 130 | # Environments 131 | .env 132 | .venv 133 | env/ 134 | venv/ 135 | ENV/ 136 | env.bak/ 137 | venv.bak/ 138 | 139 | # Spyder project settings 140 | .spyderproject 141 | .spyproject 142 | 143 | # Rope project settings 144 | .ropeproject 145 | 146 | # mkdocs documentation 147 | /site 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v3.4.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | - id: check-ast 10 | - id: check-json 11 | - id: check-merge-conflict 12 | - id: detect-private-key 13 | 14 | - repo: https://github.com/astral-sh/ruff-pre-commit 15 | rev: v0.9.1 16 | hooks: 17 | - id: ruff 18 | args: [--fix] 19 | - id: ruff-format 20 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | @jbohnslav 2 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include deepethogram/gui/icons/*.png 3 | recursive-include deepethogram/conf * 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DeepEthogram 2 | - Written by Jim Bohnslav, except where as noted 3 | - JBohnslav@gmail.com 4 | 5 | DeepEthogram is an open-source package for automatically classifying each frame of a video into a set of pre-defined 6 | behaviors. Designed for neuroscience research, it could be used in any scenario where you need to detect actions from 7 | each frame of a video. 8 | 9 | Example use cases: 10 | * Measuring itching or scratching behaviors to assess the differences between wild-type and mutant animals 11 | * Measuring the amount of time animals spend courting, and comparing between experimental conditions 12 | * Counting licks from video for appetite measurement 13 | * Measuring reach onset times for alignment with neural activity 14 | 15 | DeepEthogram uses state-of-the-art algorithms for *temporal action detection*. We build on the following previous machine 16 | learning research into action detection: 17 | * [Hidden Two-Stream Convolutional Networks for Action Recognition](https://arxiv.org/abs/1704.00389) 18 | * [Temporal Gaussian Mixture Layer for Videos](https://arxiv.org/abs/1803.06316) 19 | 20 | ![deepethogram schematic](docs/images/deepethogram_schematic.png) 21 | 22 | ## Installation 23 | For full installation instructions, see [this readme file](docs/installation.md). 24 | 25 | In brief: 26 | * [Install PyTorch](https://pytorch.org/) 27 | * `pip install deepethogram` 28 | 29 | ## Data 30 | **NEW!** All datasets collected and annotated by the DeepEthogram authors are now available from this DropBox link: 31 | https://www.dropbox.com/sh/3lilfob0sz21och/AABv8o8KhhRQhYCMNu0ilR8wa?dl=0 32 | 33 | If you have issues downloading the data, please raise an issue on Github. 34 | 35 | ## COLAB 36 | I've written a Colab notebook that shows how to upload your data and train models. You can also use this if you don't 37 | have access to a decent GPU. 38 | 39 | To use it, please [click this link to the Colab notebook](https://colab.research.google.com/drive/1Nf9FU7FD77wgvbUFc608839v2jPYgDhd?usp=sharing). 40 | Then, click `copy to Drive` at the top. You won't be able to save your changes to the notebook as-is. 41 | 42 | 43 | ## News 44 | We now support docker! Docker is a way to run `deepethogram` in completely reproducible environments, without interacting 45 | with other system dependencies. [See docs/Docker for more information](docs/docker.md) 46 | 47 | ## Pretrained models 48 | Rather than start from scratch, we will start with model weights pretrained on the Kinetics700 dataset. Go to 49 | To download the pretrained weights, please use [this Google Drive link](https://drive.google.com/file/d/1ntIZVbOG1UAiFVlsAAuKEBEVCVevyets/view?usp=sharing). 50 | Unzip the files in your `project/models` directory. Make sure that you don't add an extra directory when unzipping! The path should be 51 | `your_project/models/pretrained_models/{models 1:6}`, not `your_project/models/pretrained_models/pretrained_models/{models1:6}`. 52 | 53 | ## Licensing 54 | Copyright (c) 2020 - President and Fellows of Harvard College. All rights reserved. 55 | 56 | This software is free for academic use. For commercial use, please contact the Harvard Office of Technology 57 | Development (hms_otd@harvard.edu) with cc to Dr. Chris Harvey. For details, see [license.txt](license.txt). 58 | 59 | ## Usage 60 | ### [To use the GUI, click](docs/using_gui.md) 61 | #### [To use the command line interface, click](docs/using_CLI.md) 62 | 63 | ## Dependencies 64 | The major dependencies for DeepEthogram are as follows: 65 | * pytorch, torchvision: all the neural networks, training, and inference pipelines were written in PyTorch 66 | * pytorch-lightning: for nice model training base classes 67 | * kornia: for GPU-based image augmentations 68 | * pyside2: for the GUI 69 | * opencv: for video and image reading and writing 70 | * opencv_transforms: for fast image augmentation 71 | * scikit-learn, scipy: for binary classification metrics 72 | * matplotlib: plotting metrics and neural network outputs 73 | * pandas: reading and writing CSVs 74 | * h5py: saving inference outputs as HDF5 files 75 | * omegaconf: for smoothly integrating configuration files and command line inputs 76 | * tqdm: for nice progress bars 77 | 78 | ## Hardware requirements 79 | For GUI usage, we expect that the users will be working on a local workstation with a good NVIDIA graphics card. For training via a cluster, you can use the command line interface. 80 | 81 | * CPU: 4 cores or more for parallel data loading 82 | * Hard Drive: SSD at minimum, NVMe drive is better. 83 | * GPU: DeepEthogram speed is directly related to GPU performance. An NVIDIA GPU is absolutely required, as PyTorch uses 84 | CUDA, while AMD does not. 85 | The more VRAM you have, the more data you can fit in one batch, which generally increases performance. a 86 | I'd recommend 6GB VRAM at absolute minimum. 8GB is better, with 10+ GB preferred. 87 | Recommended GPUs: `RTX 3090`, `RTX 3080`, `Titan RTX`, `2080 Ti`, `2080 super`, `2080`, `1080 Ti`, `2070 super`, `2070` 88 | Some older ones might also be fine, like a `1080` or even `1070 Ti`/ `1070`. 89 | 90 | ## testing 91 | Test coverage is still low, but in the future we will be expanding our unit tests. 92 | 93 | First, download a copy of [`testing_deepethogram_archive.zip`](https://drive.google.com/file/d/1IFz4ABXppVxyuhYik8j38k9-Fl9kYKHo/view?usp=sharing) 94 | Make a directory in tests called `DATA`. Unzip this and move it to the `deepethogram/tests/DATA` 95 | directory, so that the path is `deepethogram/tests/DATA/testing_deepethogram_archive/{DATA,models,project_config.yaml}`. 96 | 97 | To run tests: 98 | ```bash 99 | # Run all tests except GPU tests (default) 100 | pytest tests/ 101 | 102 | # Run only GPU tests (requires NVIDIA GPU) 103 | pytest -m gpu 104 | 105 | # Run all tests including GPU tests 106 | pytest -m "" 107 | ``` 108 | 109 | GPU tests are skipped by default as they require significant computational resources and time to complete. These tests perform end-to-end model training and inference. 110 | 111 | ## Developer Guide 112 | ### Code Style and Pre-commit Hooks 113 | We use pre-commit hooks to maintain code quality and consistency. The hooks include: 114 | - Ruff for Python linting and formatting 115 | - Various file checks (trailing whitespace, YAML validation, etc.) 116 | 117 | To set up the development environment: 118 | 119 | 1. Install the development dependencies: 120 | ```bash 121 | pip install -r requirements.txt 122 | ``` 123 | 124 | 2. Install pre-commit hooks: 125 | ```bash 126 | pre-commit install 127 | ``` 128 | 129 | The hooks will run automatically on every commit. You can also run them manually on all files: 130 | ```bash 131 | pre-commit run --all-files 132 | ``` 133 | 134 | ## Changelog 135 | * 0.1.4: bugfixes for dependencies; added docker 136 | * 0.1.2/3: fixes for multiclass (not multilabel) training 137 | * 0.1.1.post1/2: batch prediction 138 | * 0.1.1.post0: flow generator metric bug fix 139 | * 0.1.1: bug fixes 140 | * 0.1: deepethogram beta! See above for details. 141 | * 0.0.1.post1: bug fixes and video conversion scripts added 142 | * 0.0.1: initial version 143 | -------------------------------------------------------------------------------- /deepethogram/__init__.py: -------------------------------------------------------------------------------- 1 | # from deepethogram import feature_extractor, flow_generator, gui, sequence, dataloaders, metrics, utils, viz, zscore 2 | # from deepethogram import feature_extractor, flow_generator, gui, sequence, dataloaders, 3 | # from deepethogram.flow_generator.train import flow_generator_train 4 | # from deepethogram.feature_extractor.train import feature_extractor_train 5 | # from deepethogram.feature_extractor.inference import feature_extractor_inference 6 | # from deepethogram.sequence.train import sequence_train 7 | # from deepethogram.sequence.inference import sequence_inference 8 | import importlib.util 9 | 10 | spec = importlib.util.find_spec("hydra") 11 | if spec is not None: 12 | raise ValueError("Hydra installation found. Please run pip uninstall hydra-core: {}".format(spec)) 13 | # try: 14 | # import hydra 15 | # except Exception as e: 16 | # print(e) 17 | # # hydra is not found 18 | # pass 19 | # else: 20 | # 21 | -------------------------------------------------------------------------------- /deepethogram/__main__.py: -------------------------------------------------------------------------------- 1 | from deepethogram.gui.main import entry 2 | # import hydra 3 | # from omegaconf import DictConfig 4 | # 5 | # @hydra.main(config_path='conf/gui.yaml') 6 | # def main(cfg: DictConfig) -> None: 7 | # run(cfg) 8 | 9 | if __name__ == "__main__": 10 | entry() 11 | -------------------------------------------------------------------------------- /deepethogram/conf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/conf/__init__.py -------------------------------------------------------------------------------- /deepethogram/conf/augs.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | # data augmentation parameters 3 | augs: 4 | # randomly alter the brightness of the image. Larger values = more distortion 5 | brightness: 0.25 6 | # randomly alter the contrast of the image. Large values = more distortion 7 | contrast: 0.1 8 | # randomly alter hue of the image. large values = more distortion 9 | hue: 0.1 10 | # randomly alter saturation of the image. large values = more distortion 11 | saturation: 0.1 12 | # the probability to do brightness / contrast / hue / saturation 13 | color_p: 0.5 14 | # randomly change image to grayscale 15 | grayscale: 0.5 16 | # if not null, crop size will take random crops of the given shape during training, and center crop during inference. 17 | # Only use this if you know what you're doing: otherwise crop the videos before loading them into your project 18 | crop_size: null 19 | # IMPORTANT: either a single value (square output) or two values (height, width). All inputs to the models will be 20 | # resized to that shape. This should be as small as you can for a human to reasonably be able to tell the behavior. 21 | # decent defaults: 224, 256. Must be a multiple of 32 for resnet to work properly. Training and inference speed, 22 | # and VRAM used by the models are directly proportional to the input resolution. 23 | resize: null 24 | # use NVIDIA dali. experimental 25 | dali: false 26 | # deprecated 27 | random_resize: false 28 | # either a single value (pads all around) or four values (left, right, top, bottom). Use padding to increase both 29 | # height and width to be a multiple of 32 30 | pad: null 31 | # probability during training that the image will be flipped left-right. If this results in an image that could 32 | # actually appear in your training set, set it to 0.5. If it produces images that would never appear normally (for 33 | # example, if in your videos the animal is aligned to point to the left) set it to 0. 34 | LR: 0.5 35 | # probability during training that the image will be flipped up-down. If this results in an image that could 36 | # # actually appear in your training set, set it to 0.5. Example: mouse in an open field looks reasonable flipped 37 | # upside down. By contrast, an animal walking on a treadmill will never appear upside-down, so it should be set to 0. 38 | UD: 0.0 39 | # The image during training will be randomly rotated +/- the below degrees. 40 | degrees: 10 41 | # This will be overwritten automatically. It is the mean and std deviation of the RGB channels of your dataset 42 | normalization: 43 | N: 0 44 | mean: 45 | - 0.5 46 | - 0.5 47 | - 0.5 48 | std: 49 | - 0.5 50 | - 0.5 51 | - 0.5 52 | -------------------------------------------------------------------------------- /deepethogram/conf/config.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | split: 3 | # if true, attempt to reload a previously made data split. If false, will randomly assign to splits new 4 | reload: true 5 | # if file is specified, reload the data splits from this file. Useful for comparing performance with the same data 6 | file: null 7 | # Data will be assigned to splits with the below probabilities. By default, don't use a test set, but only validation 8 | train_val_test: 9 | - 0.8 10 | - 0.2 11 | - 0.0 12 | # parameters for computation, both training and testing 13 | compute: 14 | # currently unsupported: if true, use half-precision training and inference. Potentially faster 15 | fp16: false 16 | # number of CPU workers. Should be roughly the number of cores in your CPU 17 | num_workers: 8 18 | # batch size for training and inference. If you're getting out of VRAM errors, lower this. If you're less than say 8, 19 | # reduce model complexity or buy a better GPU instead 20 | # "auto" will use Pytorch Lightning's automatic batch sizing feature 21 | batch_size: auto 22 | min_batch_size: 8 # minimum batch size with auto 23 | max_batch_size: 512 # maximum batch size with auto 24 | # not implemented: for distributed training 25 | distributed: false 26 | # which GPU to use on a multi-GPU computer. use nvidia-smi to see which numbers correspond to which GPUs 27 | gpu_id: 0 28 | # not implemented: for nvidia dali dataloading 29 | dali: false 30 | # whether or not to use multiprocessing to run metrics computation 31 | metrics_workers: 0 32 | # for reloading model weights 33 | reload: 34 | # not used: overwrite specified config with the loaded one 35 | overwrite_cfg: false 36 | # path to weightfile. You can either use this or the model-specific .weights field, depending 37 | # weights: null 38 | # if True, will attempt to automatically load the most recent trained model 39 | latest: false 40 | # one of deg_f, deg_m, deg_s. see paper 41 | # use this field to set notes in both the saved configuration file and the run directory. e.g. 42 | # `train.dropout_p=0.5 notes=trying_lower_dropout` 43 | notes: null 44 | log: 45 | level: info 46 | # project: project_config 47 | # hyra configuration: specifies how to create the run directory 48 | # hydra: 49 | # run: 50 | # dir: ${project.path}/${project.model_path}/${now:%y%m%d_%H%M%S}_${run.model}_${run.type}_${notes} 51 | # output_subdir: '' 52 | -------------------------------------------------------------------------------- /deepethogram/conf/debug.yaml: -------------------------------------------------------------------------------- 1 | train: 2 | steps_per_epoch: 3 | train: 50 4 | val: 50 5 | test: 50 6 | num_epochs: 3 7 | tune: 8 | num_trials: 3 9 | debug: True 10 | -------------------------------------------------------------------------------- /deepethogram/conf/gui.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | type: gui 3 | # how many frames to show on the label viewer. Larger number = more zoomed out 4 | label_view_width: 31 5 | # how many frames to jump when using ctrl + the arrow keys 6 | control_arrow_jump: 31 7 | # how many frames the up and down arrows move forward and backward, respectively 8 | vertical_arrow_jump: 3 9 | # matplotlib colormap for the label and prediction viewer 10 | # https://matplotlib.org/3.1.1/gallery/color/colormap_reference.html 11 | # if deepethogram: each row is a different color. background is gray. probabilities are mapped to the saturation of the 12 | # color 13 | cmap: deepethogram 14 | # frames that are unlabeled will look transparent on the GUI, with the below alpha value 15 | # lower = more transparent 16 | unlabeled_alpha: 0.1 17 | # the prediction viewer shows both probablities and predictions (thresholded probabilities). the probabilities 18 | # will have the below alpha value (predictions are always 1) 19 | prediction_opacity: 0.2 20 | # notes to add to gui logs 21 | notes: null 22 | -------------------------------------------------------------------------------- /deepethogram/conf/inference.yaml: -------------------------------------------------------------------------------- 1 | inference: 2 | # list of DeepEthogram directories with video files and labels. See docs/file_structure.md 3 | # bracketed separated by commas. e.g. inference.directory_list=[path/to/vid1,path/to/vid2] 4 | # to extract features from all videos, use 'all' 5 | directory_list: all 6 | # if true, when an error is encountered, inference will attempt to continue to the next video without stopping 7 | ignore_error: true 8 | # if the group sequence.latent_name already exists in the HDF5 file, will overwrite it. 9 | # if false, that file would be skipped 10 | overwrite: false 11 | # if True, overwrite settings in the config file with the one loaded from disk. E.g. if you try to configure 12 | # inference dropout_p to be 0.9 but in the trained model it was 0.5, it will set dropout to 0.5 13 | use_loaded_model_cfg: true 14 | -------------------------------------------------------------------------------- /deepethogram/conf/model/feature_extractor.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | feature_extractor: 3 | # the CNN architecture used to extract features. Currently supported options: resnet18, resnet50, resnet3d_34. 4 | arch: resnet18 5 | # how to fuse the outputs of the spatial model and flow model. Default: averaging them together. Other option: 6 | # concatenate. This concatenates the two vectors, followed by a fully connected layer with outputs of shape (K), 7 | # aka number of behaviors. 8 | fusion: average 9 | # deprecated: used to over-sample rare classes 10 | sampler: null 11 | # use batchnorm in the final layer, just before sigmoid / softmax 12 | final_bn: false 13 | # deprecated: how much to over-sample 14 | sampling_ratio: null 15 | # The activation function on the final layer. options include sigmoid, softmax. Use softmax for mutually-exclusive 16 | # behaviors. Note: models have not been trained with this in mind. 17 | final_activation: sigmoid 18 | # the amount of dropout on the 512-d feature vector before the last fully connected layer. 0.9 is already very high. 19 | # if your model is under-fitting (highly unlikely), reduce this value. 20 | dropout_p: 0.25 21 | # how many optic flow frames to use as inputs to the flow CNN 22 | n_flows: 10 23 | # how many RGB frames to use as inputs to the spatial CNN 24 | n_rgb: 1 25 | # if true, trains feature extractors in a curriculum. first trains the spatial CNN, then the flow CNN, and finally 26 | # the two jointly end-to-end. If false, will be end-to-end from the start: faster, but with potentially lower 27 | # performance. 28 | curriculum: false 29 | # deprecated 30 | inputs: both 31 | # path to a checkpoint.pt file which will reload this model. Should have weights for both spatial and flow streams 32 | weights: pretrained 33 | train: 34 | # overwrite default steps per epoch: ideally we would not limit train steps at all. However, with very large datasets 35 | # this can be infeasible 36 | steps_per_epoch: 37 | train: 1000 38 | val: 1000 39 | test: null 40 | num_epochs: 20 41 | -------------------------------------------------------------------------------- /deepethogram/conf/model/flow_generator.yaml: -------------------------------------------------------------------------------- 1 | flow_generator: 2 | type: flow_generator 3 | flow_loss: MotionNet 4 | flow_max: 10 5 | input_images: 11 6 | # if true, add L1 penalty to the flow itself to encourage sparsity 7 | flow_sparsity: false 8 | smooth_weight_multiplier: 1.0 9 | sparsity_weight: 0.0 10 | # loss function to use for self-supervised training. Options: MotionNet (see Hidden two stream paper) or SelfSupervised 11 | # which is a blend of SSIM, L1, smoothness, and a sparsity loss. 12 | loss: MotionNet 13 | # the maximum flow generated by the model. currently used to set the color scale on visualizations and nothing else. 14 | max: 5 15 | # number of RGB frames to input to the flow generator. Number of output frames will be this value -1 16 | n_rgb: 11 17 | # the architecture for the flow generator. Currently supported models: TinyMotionNet, MotionNet, TinyMotionNet3d 18 | arch: TinyMotionNet 19 | # path to a checkpoint.pt weight file for reloading. 20 | weights: pretrained 21 | train: 22 | # overwrite default steps per epoch: because we don't care about rare classes for optic flow, don't need so much 23 | # validation 24 | steps_per_epoch: 25 | train: 1000 26 | val: 200 27 | test: 20 28 | num_epochs: 10 29 | -------------------------------------------------------------------------------- /deepethogram/conf/model/sequence.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | sequence: 3 | # the architecture for the sequence model. choices=['linear', 'conv_nonlinear', 'rnn', 'tgm', 'tgmj', 'mlp'] 4 | # if RNN, the actual architecture will be rnn_style below 5 | arch: tgmj 6 | # # the INPUTS to the sequence model will be in HDF5 groups with the below name. If None: will default to the 7 | # # feature extractor architecture that created it 8 | latent_name: null 9 | # # the OUTPUTS from the sequence model will be written to an HDF5 group with the below name. If None: will default 10 | # # to the architecture of the sequence model 11 | output_name: null 12 | # number of timepoints in the sequence. Performance is mostly invariant to this value 13 | sequence_length: 180 14 | # the dropout to be placed on the penultimate layer before the 1d convolution with K (# behaviors) output features 15 | dropout_p: 0.5 16 | # number of layers in RNN, 1d cnn, or TGM models. typically 1-3 17 | num_layers: 3 18 | # for RNNs or CNNs. For CNNs, only 1 hidden layer by default. 19 | hidden_size: 64 20 | # if true, only loads data in non-overlapping chunks of length `filter_length`. E.g., the first batch of your dataset 21 | # would be frames 0-180, then 181-360, etc. If false, the first batch would be 0-180, the second would be 1-181, etc. 22 | # Setting to True will dramatically speed up training 23 | nonoverlapping: True 24 | # tgm parameters 25 | # Length (in time) per TGM filter 26 | filter_length: 15 27 | # dropout the input features before concatentating again to the smoothed features. paper: 0.5 28 | input_dropout: 0.5 29 | # Number of filters to use per TGM layer 30 | n_filters: 8 31 | # How to reduce tgm stack N x C_out x D x T -> N x D x T. choices=['max', 'mean', 'conv1x1'] Paper: max 32 | tgm_reduction: max 33 | # What shape inputs are: N x C_in x D x T 34 | c_in: 1 35 | # How many representations of DxT per TGM layer. Paper: 8 36 | c_out: 8 37 | # whether to use soft attention or 1d conv to reduce C_in x D x T -> 1 x D x T 38 | soft_attn: true 39 | # how many features in final layer before logits. paper: 512 (too many parameters for our case) 40 | num_features: 128 41 | # rnn parameters 42 | # should RNNs run bidirectionally, adds a lot of time 43 | bidirectional: false 44 | # choices=['RNN', 'GRU', 'LSTM'] 45 | rnn_style: lstm 46 | # only for num_layers > 1. adds dropout between layers 47 | hidden_dropout: 0.0 48 | # path to checkpoint.pt weights file for reloading 49 | weights: null 50 | # Inputs to TGM model: N x D x T features. This is smoothed to a new N x D x T tensor. If true, use conv1d to reduce 51 | # to N x num_features x T, before another conv1d to N x num_classes x T. 52 | # if False, go straight from N x D x T to N x num_classes x T 53 | nonlinear_classification: True 54 | # use a batch normalization layer on the outputs 55 | final_bn: True 56 | # what kinds of inputs to use for the sequence model. choices: ['features', 'keypoints'] 57 | input_type: features 58 | train: 59 | regularization: 60 | # can't use l2_sp regularization because we are not pretraining sequence models 61 | style: l2 62 | alpha: 0.01 63 | # overwrite patience: because of Nonoverlapping, train epochs can be very low 64 | patience: 5 65 | # overwrite num epochs. due to nonoverlapping, one epoch takes only a minute or two 66 | num_epochs: 100 67 | compute: 68 | min_batch_size: 2 69 | max_batch_size: 64 # sequence can get weird when batch sizes are too high 70 | -------------------------------------------------------------------------------- /deepethogram/conf/postprocessor.yaml: -------------------------------------------------------------------------------- 1 | postprocessor: 2 | # options: null, min_bout 3 | # null: the postprocessor will simply threshold, and then compute the background 4 | # min_bout: the postprocessor will threshold, remove sets of 1s or 0s of length less than 5 | # postprocessor.min_bout_length, and then will compute background 6 | type: min_bout_per_behavior 7 | # if type is min_bout, postprocessor will remove any consecutive set of 1s or 0s of less than this length. 8 | # if type is min_bout_per_behavior, this will be the PERCENTILE of each behavior's bout length distribution 9 | # if value is 5, then all bouts less than the 5th percentile of the label distribution will be removed 10 | # see deepethogram/postprocessing.py for details 11 | min_bout_length: 1 12 | -------------------------------------------------------------------------------- /deepethogram/conf/preset/deg_f.yaml: -------------------------------------------------------------------------------- 1 | # this specifies parameters for the Fast version of the deepethogram models. 2 | # this file will be loaded if the flag `preset=deg_f` is used at the command line. 3 | # see paper for details 4 | feature_extractor: 5 | arch: resnet18 6 | n_flow: 10 7 | n_rgb: 1 8 | flow_generator: 9 | arch: TinyMotionNet 10 | n: 10 11 | -------------------------------------------------------------------------------- /deepethogram/conf/preset/deg_m.yaml: -------------------------------------------------------------------------------- 1 | # this specifies parameters for the Medium version of the deepethogram models. 2 | # this file will be loaded if the flag `preset=deg_m` is used at the command line. 3 | # see paper for details 4 | feature_extractor: 5 | arch: resnet50 6 | n_flow: 10 7 | n_rgb: 1 8 | flow_generator: 9 | arch: MotionNet 10 | n: 10 11 | -------------------------------------------------------------------------------- /deepethogram/conf/preset/deg_s.yaml: -------------------------------------------------------------------------------- 1 | # this specifies parameters for the Slow version of the deepethogram models. 2 | # this file will be loaded if the flag `preset=deg_s` is used at the command line. 3 | # see paper for details 4 | feature_extractor: 5 | arch: resnet3d_34 6 | n_flow: 10 7 | n_rgb: 11 8 | flow_generator: 9 | arch: TinyMotionNet3D 10 | n: 10 11 | flow_sparsity: true 12 | sparsity_weight : 0.05 13 | smooth_weight_multiplier: 0.25 14 | -------------------------------------------------------------------------------- /deepethogram/conf/project/project_config_default.yaml: -------------------------------------------------------------------------------- 1 | augs: 2 | LR: 0.5 3 | UD: 0.0 4 | brightness: 0.25 5 | contrast: 0.1 6 | crop_size: null 7 | degrees: 10 8 | hue: 0.1 9 | saturation: 0.1 10 | grayscale: 0.5 11 | normalization: 12 | N: 0 13 | mean: 14 | - 0.5 15 | - 0.5 16 | - 0.5 17 | std: 18 | - 0.5 19 | - 0.5 20 | - 0.5 21 | pad: null 22 | random_resize: false 23 | resize: 24 | - 224 25 | - 224 26 | compute: 27 | batch_size: 32 28 | distributed: false 29 | gpu_id: 0 30 | num_workers: 8 31 | project: 32 | class_names: null 33 | config_file: null 34 | data_path: DATA 35 | labeler: null 36 | model_path: models 37 | pretrained_path: pretrained_models 38 | name: null 39 | path: null 40 | sequence: 41 | filter_length: 15 42 | split: 43 | file: null 44 | reload: true 45 | train: 46 | loss_weight_exp: 1.0 47 | -------------------------------------------------------------------------------- /deepethogram/conf/train.yaml: -------------------------------------------------------------------------------- 1 | # @package _global_ 2 | train: 3 | # learning rate of your optimizer. default: 1e-4. Potentially set lower if you're retraining from a decent 4 | # checkpoint 5 | lr: 0.0001 6 | # learning rate scheduler. See deepethogram/schedulers.py. options: 7 | # plateau: when the validation metric plateaus, reduce the learning rate 8 | # cosine: using cosine annealing with warm restarts 9 | # multistep: decrease learning rate at fixed milestones (below) 10 | scheduler: plateau 11 | # number of epochs to train before automatically stopping. Use to decrease training time 12 | num_epochs: 100 13 | # number of gradient descent steps during each "epoch". for large datasets, it will take too long to go through 14 | # the entire dataset before validation (when model saving occurs). Reduce to speed up training, or increase to train 15 | # further before visualization, checkpointing, learning rate decrements, etc. 16 | steps_per_epoch: 17 | train: 1000 18 | val: 1000 19 | test: 20 20 | # used in the plateau scheduler. if the learning rate drops below this value, stop training altogether 21 | min_lr: 0.0000005 # 5e-7 22 | # see deepethogram/stoppers.py 23 | # options: 24 | # early: will stop after your validation loss saturates for train.patience epochs. Clashes with learning rate 25 | # scheduler 26 | # learning_rate: when the learning rate drops below train.min_lr, stop 27 | # null: train will go for num_epochs, and then stop 28 | stopping_type: learning_rate 29 | # for early_stopping, reduce learning rate by train.reduction_factor at the below epochs 30 | milestones: 31 | - 50 32 | - 100 33 | - 150 34 | - 200 35 | - 250 36 | - 300 37 | # whether or not to weight your loss function by the number of examples. Should essentially always keep to true 38 | weight_loss: true 39 | # patience used both in stopping and the learning rate scheduler 40 | patience: 3 41 | # if using early stopping, you can set this value and early stopping will not kick in until the below epoch is reached 42 | early_stopping_begins: 0 43 | # whether or not to visualize data. always keep true 44 | viz_metrics: true 45 | viz_examples: 10 46 | # the amount to reduce the learning rate by when the scheduler demands 47 | reduction_factor: 0.1 48 | # In the paper, see Methods / loss functions. This value is beta. 49 | # if 0, don't weight loss by number of examples. 50 | # if 1, weight loss in direct proportion to the rate of negative examples. For example, if only 1% of your data 51 | # for a given class is positive (==1), weight errors on these positive examples 100 X more highly. 52 | # in practice I find this is too strong an up-weighting. Instead, I will weight by 53 | # the proportion of pos examples ** loss_weight_exp 54 | # if only 1% of data ==1, and loss_weight_exp=0.5, the weight value will be 10 55 | loss_weight_exp: 0.25 56 | # focal loss gamma for feature extractor and sequence models 57 | # gamma == 0: BCELoss 58 | # the higher the gamma, the more the model "focuses" on mis-classified points near 0.5, and not trying to increase 59 | # the p(y=1) from 0.8 ->0.99, for example 60 | loss_gamma: 1.0 61 | label_smoothing: 0.05 62 | # how much to oversample rare classes. Formula: 1/(class_proportion ** oversampling_exp) 63 | # 0: no oversampling 64 | # 1: all classes will be sampled equally 65 | oversampling_exp: 0.0 66 | regularization: 67 | style: l2_sp 68 | alpha: 1e-5 69 | beta: 1e-3 70 | -------------------------------------------------------------------------------- /deepethogram/conf/tune/feature_extractor.yaml: -------------------------------------------------------------------------------- 1 | tune: 2 | name: tune_feature_extractor 3 | hparams: 4 | # feature_extractor.dropout_p: # each hparam key should be an attribute in a valid configuration 5 | # min: 0.0 # min: minimum of range 6 | # max: 0.9 # max: maximum of range 7 | # space: uniform # space: how to sample 8 | # short: dropout # a shortened version to view in Ray's command line interface 9 | # current_best: 0.25 # current best estimate. a moving target. used for initializing search space with hyperopt 10 | # train.regularization.alpha: 11 | # min: 1e-7 12 | # max: 1e-1 13 | # space: log 14 | # short: reg_alpha 15 | # current_best: 1e-5 16 | # train.regularization.beta: 17 | # min: 1e-4 18 | # max: 1e-1 19 | # space: log 20 | # short: reg_beta 21 | # current_best: 5e-4 22 | # train.loss_gamma: 23 | # # choices: [0, 0.5, 1, 2] 24 | # min: 0 25 | # max: 2 26 | # space: uniform 27 | # short: gamma 28 | # current_best: 0.5 29 | train.loss_weight_exp: 30 | min: 0.0 31 | max: 1.0 32 | space: uniform 33 | short: loss_weight_exp 34 | current_best: 0.25 35 | train.oversampling_exp: 36 | min: 0.0 37 | max: 1.0 38 | space: uniform 39 | short: oversampling_exp 40 | current_best: 0.0 41 | # train.label_smoothing: 42 | # min: 0.0 43 | # max: 0.2 44 | # space: uniform 45 | # short: label_smoothing 46 | # current_best: 0.0 47 | # train.lr: 48 | # min: 1e-5 49 | # max: 1e-3 50 | # space: log 51 | # short: lr 52 | # current_best: 1e-4 53 | # feature_extractor.final_bn: 54 | # choices: [True, False] 55 | # space: choice 56 | # short: final_bn 57 | # current_best: False 58 | # use these to overwrite default configuration parameters only when running tune jobs 59 | -------------------------------------------------------------------------------- /deepethogram/conf/tune/sequence.yaml: -------------------------------------------------------------------------------- 1 | tune: 2 | name: tune_sequence 3 | hparams: 4 | # feature_extractor.dropout_p: # each hparam key should be an attribute in a valid configuration 5 | # min: 0.0 # min: minimum of range 6 | # max: 0.9 # max: maximum of range 7 | # space: uniform # space: how to sample 8 | # short: dropout # a shortened version to view in Ray's command line interface 9 | # current_best: 0.25 # current best estimate. a moving target. used for initializing search space with hyperopt 10 | train.regularization.alpha: 11 | min: 1e-2 12 | max: 2 13 | space: log 14 | short: reg_alpha 15 | current_best: 0.00045170 16 | train.loss_gamma: 17 | # choices: [0, 0.5, 1, 2] 18 | min: 0 19 | max: 1 20 | space: uniform 21 | short: gamma 22 | current_best: 0.5 23 | train.loss_weight_exp: 24 | min: 0.0 25 | max: 1 26 | space: uniform 27 | short: loss_weight_exp 28 | current_best: 0.25 29 | train.lr: 30 | min: 1e-5 31 | max: 1e-3 32 | space: log 33 | short: lr 34 | current_best: 1e-4 35 | sequence.final_bn: 36 | choices: [True, False] 37 | space: choice 38 | short: final_bn 39 | current_best: True 40 | sequence.num_layers: 41 | choices: [1, 2, 3] 42 | space: choice 43 | current_best: 1 44 | short: seq_layers 45 | sequence.nonlinear_classification: 46 | choices: [False, True] 47 | space: choice 48 | current_best: False 49 | short: nonlinear_classification 50 | # use these to overwrite default configuration parameters only when running tune jobs 51 | compute: 52 | max_batch_size: 32 53 | -------------------------------------------------------------------------------- /deepethogram/conf/tune/tune.yaml: -------------------------------------------------------------------------------- 1 | tune: 2 | use: True 3 | metrics: 4 | - val/loss 5 | - val/data_loss 6 | - val/reg_loss 7 | - val/f1_class_mean_nobg 8 | key_metric: val/f1_class_mean_nobg 9 | num_trials: 128 # number of runs 10 | name: tune 11 | grace_period: 4 # number of epochs 12 | search: random # either random or hyperopt 13 | resources_per_trial: 14 | gpu: 0.5 15 | cpu: 3 16 | train: 17 | viz_examples: 0 # don't spend time and space making example images 18 | steps_per_epoch: 19 | train: 1000 20 | val: 1000 21 | num_epochs: 20 22 | compute: 23 | metrics_workers: 0 # sometimes has a bug in already multiprocessed jobs 24 | # batch_size: 64 # auto batch sizing takes a long time 25 | -------------------------------------------------------------------------------- /deepethogram/conf/zscore.yaml: -------------------------------------------------------------------------------- 1 | run: 2 | type: zscore 3 | model: null 4 | # path to the video file to z-score 5 | videofile: null 6 | # every STRIDE frames will be read and statistics computed 7 | stride: 100 8 | -------------------------------------------------------------------------------- /deepethogram/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/data/__init__.py -------------------------------------------------------------------------------- /deepethogram/debug.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import pprint 4 | from typing import Union 5 | 6 | import numpy as np 7 | from omegaconf import OmegaConf 8 | from tqdm import tqdm 9 | from vidio import VideoReader 10 | 11 | from deepethogram import file_io, projects 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | def print_models(model_path: Union[str, os.PathLike]) -> None: 17 | """Prints all models found in the model path 18 | 19 | Parameters 20 | ---------- 21 | model_path : Union[str, os.PathLike] 22 | Absolute path to models directory 23 | """ 24 | trained_models = projects.get_weights_from_model_path(model_path) 25 | log.info("Trained models: {}".format(pprint.pformat(trained_models))) 26 | 27 | 28 | def print_dataset_info(datadir: Union[str, os.PathLike]) -> None: 29 | """Prints information about your dataset. 30 | 31 | - video path 32 | - number of unlabeled rows in a video 33 | - number of examples of each behavior in each video 34 | 35 | Parameters 36 | ---------- 37 | datadir : Union[str, os.PathLike] 38 | [description] 39 | """ 40 | records = projects.get_records_from_datadir(datadir) 41 | 42 | for key, record in records.items(): 43 | log.info("Information about subdir {}".format(key)) 44 | if record["rgb"] is not None: 45 | log.info("Video: {}".format(record["rgb"])) 46 | 47 | if record["label"] is not None: 48 | label = file_io.read_labels(record["label"]) 49 | if np.sum(label == -1) > 0: 50 | unlabeled_rows = np.any(label == -1, axis=0) 51 | n_unlabeled = np.sum(unlabeled_rows) 52 | log.warning( 53 | "{} UNLABELED ROWS!".format(n_unlabeled) 54 | + "VIDEO WILL NOT BE USED FOR FEATURE_EXTRACTOR OR SEQUENCE TRAINING." 55 | ) 56 | else: 57 | class_counts = label.sum(axis=0) 58 | log.info("Labels with counts: {}".format(class_counts)) 59 | 60 | 61 | def try_load_all_frames(datadir: Union[str, os.PathLike]): 62 | """Attempts to read every image from every video. 63 | 64 | Useful for debugging corrupted videos, e.g. if saving to disk was aborted improperly during acquisition 65 | If there is an error reading a frame, it will print the video name and frame number 66 | 67 | Parameters 68 | ---------- 69 | datadir : Union[str, os.PathLike] 70 | absolute path to the project/DATA directory 71 | """ 72 | log.info("Iterating through all frames of all movies to test for frame reading bugs") 73 | records = projects.get_records_from_datadir(datadir) 74 | for key, record in tqdm(records.items()): 75 | with VideoReader(record["rgb"]) as reader: 76 | log.info("reading all frames from file {}".format(record["rgb"])) 77 | had_error = False 78 | for i in tqdm(range(len(reader)), leave=False): 79 | try: 80 | _ = reader[i] 81 | except Exception: 82 | had_error = True 83 | print("error reading frame {} from video {}".format(i, record["rgb"])) 84 | except KeyboardInterrupt: 85 | raise 86 | if had_error: 87 | log.warning("Error in file {}. Is this video corrupted?".format(record["rgb"])) 88 | else: 89 | log.info("No problems in {}".format(key)) 90 | 91 | 92 | if __name__ == "__main__": 93 | if os.path.isfile("debug.log"): 94 | os.remove("debug.log") 95 | logging.basicConfig( 96 | level=logging.INFO, 97 | format="%(asctime)s [%(levelname)s] %(message)s", 98 | handlers=[logging.FileHandler("debug.log"), logging.StreamHandler()], 99 | ) 100 | 101 | cfg = OmegaConf.from_cli() 102 | if cfg.project.path is None and cfg.project.config_file is None: 103 | raise ValueError("must input either a path or a config file") 104 | elif cfg.project.path is not None: 105 | cfg.project.config_file = os.path.join(cfg.project.path, "project_config.yaml") 106 | elif cfg.project.config_file is not None: 107 | cfg.project.path = os.path.dirname(cfg.project.config_file) 108 | else: 109 | raise ValueError("must input either a path or a config file, not {}".format(cfg)) 110 | 111 | assert os.path.isfile(cfg.project.config_file) and os.path.isdir(cfg.project.path) 112 | 113 | user_cfg = OmegaConf.load(cfg.project.config_file) 114 | cfg = OmegaConf.merge(cfg, user_cfg) 115 | cfg = projects.convert_config_paths_to_absolute(cfg) 116 | 117 | logging.info(OmegaConf.to_yaml(cfg)) 118 | 119 | print_models(cfg.project.model_path) 120 | 121 | print_dataset_info(cfg.project.data_path) 122 | 123 | try_load_all_frames(cfg.project.data_path) 124 | -------------------------------------------------------------------------------- /deepethogram/feature_extractor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/feature_extractor/__init__.py -------------------------------------------------------------------------------- /deepethogram/feature_extractor/__main__.py: -------------------------------------------------------------------------------- 1 | from .train import main 2 | 3 | main() 4 | -------------------------------------------------------------------------------- /deepethogram/feature_extractor/models/CNN.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from inspect import isfunction 3 | import warnings 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn as nn 8 | 9 | from deepethogram import utils 10 | from .classifiers import alexnet, densenet, inception, vgg, resnet, squeezenet, resnet3d 11 | from .utils import pop 12 | 13 | log = logging.getLogger(__name__) 14 | 15 | 16 | # from nvidia 17 | # https://github.com/NVIDIA/flownet2-pytorch/blob/master/utils/tools.py 18 | def module_to_dict(module, exclude=[]): 19 | return dict( 20 | [ 21 | (x, getattr(module, x)) 22 | for x in dir(module) 23 | if isfunction(getattr(module, x)) and x not in exclude and getattr(module, x) not in exclude 24 | ] 25 | ) 26 | 27 | 28 | # model definitions can be accessed by indexing into this dictionary 29 | # e.g. model = models['resnet50'] 30 | models = {} 31 | for model in [alexnet, densenet, inception, vgg, resnet, squeezenet, resnet3d]: 32 | model_dict = module_to_dict(model) 33 | for key, value in model_dict.items(): 34 | models[key] = value 35 | 36 | 37 | # https://pytorch.org/tutorials/beginner/finetuning_torchvision_models_tutorial.html 38 | def get_cnn( 39 | model_name: str, 40 | in_channels: int = 3, 41 | reload_imagenet: bool = True, 42 | num_classes: int = 1000, 43 | freeze: bool = False, 44 | pos: np.ndarray = None, 45 | neg: np.ndarray = None, 46 | final_bn: bool = False, 47 | **kwargs, 48 | ): 49 | """Initializes a pretrained CNN from Torchvision. 50 | 51 | Currently supported models: 52 | AlexNet, DenseNet, Inception, VGGXX, ResNets, SqueezeNets, and Resnet3Ds (not torchvision) 53 | 54 | Args: 55 | model_name (str): 56 | in_channels (int): number of input channels. If not 3, the per-channel weights will be averaged and replicated 57 | in_channels times 58 | reload_imagenet (bool): if True, reload imagenet weights from Torchvision 59 | num_classes (int): number of output classes (neurons in final FC layer) 60 | freeze (bool): if true, model weights will be freezed 61 | pos (np.ndarray): number of positive examples in training set. Used for custom bias initialization in 62 | final layer 63 | neg (np.ndarray): number of negative examples in training set. Used for custom bias initialization in 64 | final layer 65 | **kwargs (): passed to model initialization function 66 | 67 | Returns: 68 | model: a pytorch CNN 69 | 70 | """ 71 | model = models[model_name](pretrained=reload_imagenet, in_channels=in_channels, **kwargs) 72 | 73 | if freeze: 74 | log.info("Before freezing: {:,}".format(utils.get_num_parameters(model))) 75 | for param in model.parameters(): 76 | param.requires_grad = False 77 | log.info("After freezing: {:,}".format(utils.get_num_parameters(model))) 78 | 79 | # we have to use the pop function because the final layer in these models has different names 80 | model, num_features, final_layer = pop(model, model_name, 1) 81 | linear_layer = nn.Linear(num_features, num_classes, bias=not final_bn) 82 | modules = [model, linear_layer] 83 | if final_bn: 84 | bn_layer = nn.BatchNorm1d(num_classes) 85 | modules.append(bn_layer) 86 | # initialize bias to roughly approximate the probability of positive examples in the training set 87 | # https://www.tensorflow.org/tutorials/structured_data/imbalanced_data#optional_set_the_correct_initial_bias 88 | if pos is not None and neg is not None: 89 | with torch.no_grad(): 90 | with warnings.catch_warnings(): 91 | warnings.filterwarnings("ignore", category=RuntimeWarning) 92 | bias = np.nan_to_num(np.log(pos / neg), neginf=0.0, posinf=1.0) 93 | bias = torch.nn.Parameter(torch.from_numpy(bias).float()) 94 | if final_bn: 95 | bn_layer.bias = bias 96 | else: 97 | linear_layer.bias = bias 98 | log.info("Custom bias: {}".format(bias)) 99 | 100 | model = nn.Sequential(*modules) 101 | return model 102 | -------------------------------------------------------------------------------- /deepethogram/feature_extractor/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/feature_extractor/models/__init__.py -------------------------------------------------------------------------------- /deepethogram/feature_extractor/models/classifiers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/feature_extractor/models/classifiers/__init__.py -------------------------------------------------------------------------------- /deepethogram/feature_extractor/models/classifiers/alexnet.py: -------------------------------------------------------------------------------- 1 | # this entire page was edited from torchvision: 2 | # https://github.com/pytorch/vision/blob/master/torchvision/models/ 3 | # BSD 3-Clause License 4 | # 5 | # Copyright (c) Soumith Chintala 2016, 6 | # All rights reserved. 7 | # 8 | # Redistribution and use in source and binary forms, with or without 9 | # modification, are permitted provided that the following conditions are met: 10 | # 11 | # * Redistributions of source code must retain the above copyright notice, this 12 | # list of conditions and the following disclaimer. 13 | # 14 | # * Redistributions in binary form must reproduce the above copyright notice, 15 | # this list of conditions and the following disclaimer in the documentation 16 | # and/or other materials provided with the distribution. 17 | # 18 | # * Neither the name of the copyright holder nor the names of its 19 | # contributors may be used to endorse or promote products derived from 20 | # this software without specific prior written permission. 21 | # 22 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 23 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 25 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 26 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 28 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 29 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | import torch.nn as nn 33 | import torch.utils.model_zoo as model_zoo 34 | 35 | 36 | __all__ = ["AlexNet", "alexnet"] 37 | 38 | 39 | model_urls = { 40 | "alexnet": "https://download.pytorch.org/models/alexnet-owt-4df8aa71.pth", 41 | } 42 | 43 | 44 | class AlexNet(nn.Module): 45 | def __init__(self, in_channels=3, num_classes=1000, dropout_p=0.5): 46 | super(AlexNet, self).__init__() 47 | self.features = nn.Sequential( 48 | nn.Conv2d(in_channels, 64, kernel_size=11, stride=4, padding=2), 49 | nn.ReLU(inplace=True), 50 | nn.MaxPool2d(kernel_size=3, stride=2), 51 | nn.Conv2d(64, 192, kernel_size=5, padding=2), 52 | nn.ReLU(inplace=True), 53 | nn.MaxPool2d(kernel_size=3, stride=2), 54 | nn.Conv2d(192, 384, kernel_size=3, padding=1), 55 | nn.ReLU(inplace=True), 56 | nn.Conv2d(384, 256, kernel_size=3, padding=1), 57 | nn.ReLU(inplace=True), 58 | nn.Conv2d(256, 256, kernel_size=3, padding=1), 59 | nn.ReLU(inplace=True), 60 | # modified to be fully convolutional 61 | nn.AdaptiveMaxPool2d((6, 6)), 62 | ) 63 | self.classifier = nn.Sequential( 64 | nn.Dropout(p=dropout_p), 65 | # fc_6 66 | nn.Linear(256 * 6 * 6, 4096), 67 | nn.ReLU(inplace=True), 68 | nn.Dropout(p=dropout_p), 69 | # fc_7 70 | nn.Linear(4096, 4096), 71 | nn.ReLU(inplace=True), 72 | # fc_8 73 | nn.Linear(4096, num_classes), 74 | ) 75 | 76 | def forward(self, x): 77 | x = self.features(x) 78 | x = x.view(x.size(0), 256 * 6 * 6) 79 | x = self.classifier(x) 80 | return x 81 | 82 | 83 | def alexnet(pretrained=False, in_channels=3, **kwargs): 84 | r"""AlexNet model architecture from the 85 | `"One weird trick..." `_ paper. 86 | 87 | Args: 88 | pretrained (bool): If True, returns a model pre-trained on ImageNet 89 | """ 90 | model = AlexNet(in_channels=in_channels, **kwargs) 91 | if pretrained: 92 | print("Downloading pretrained weights...") 93 | # from Wang et al. 2015: Towards good practices for very deep two-stream convnets 94 | state_dict = model_zoo.load_url(model_urls["alexnet"]) 95 | if in_channels != 3: 96 | rgb_kernel_key = list(state_dict.keys())[0] 97 | rgb_kernel = state_dict[rgb_kernel_key] 98 | flow_kernel = rgb_kernel.mean(dim=1).unsqueeze(1).repeat(1, in_channels, 1, 1) 99 | state_dict[rgb_kernel_key] = flow_kernel 100 | state_dict.update(state_dict) 101 | model.load_state_dict(state_dict) 102 | return model 103 | -------------------------------------------------------------------------------- /deepethogram/feature_extractor/models/utils.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | import torch 3 | import logging 4 | 5 | log = logging.getLogger(__name__) 6 | 7 | 8 | def pop(model, model_name, n_layers): 9 | # Goal of the pop function: 10 | # for each model, remove the N final layers. 11 | # Whatever permutation you make to the model, it MUST STILL USE THE MODEL'S OWN .FORWARD FUNCTION 12 | # Also goal: make it an intelligent number of layers 13 | # so when you pop off, for example, 1 layer from AlexNet 14 | # you also want to pop off the previous ReLU so that you get the unscaled linear units from fc_7 15 | # just doing something like model = nn.Sequential(*list(model.children())[:-1]) would not get rid of 16 | # this ReLU, so that's an unintelligent version of this 17 | if model_name.startswith("resnet"): 18 | if n_layers == 1: 19 | # use empty sequential module as an identity function 20 | num_features = model.fc.in_features 21 | final_layer = model.fc 22 | model.fc = nn.Identity() 23 | else: 24 | raise NotImplementedError("Can only pop off the final layer of a resnet") 25 | elif model_name == "alexnet": 26 | final_layer = model.classifier 27 | if n_layers == 1: 28 | model.classifier = nn.Sequential( 29 | nn.Dropout(), 30 | # fc_6 31 | nn.Linear(256 * 6 * 6, 4096), 32 | nn.ReLU(inplace=True), 33 | nn.Dropout(), 34 | # fc_7 35 | nn.Linear(4096, 4096), 36 | ) 37 | num_features = 4096 38 | log.info("Final layer of encoder: AlexNet FC_7") 39 | elif n_layers == 2: 40 | model.classifier = nn.Sequential( 41 | nn.Dropout(), 42 | # fc_6 43 | nn.Linear(256 * 6 * 6, 4096), 44 | ) 45 | num_features = 4096 46 | log.info("Final layer of encoder: AlexNet FC_6") 47 | elif n_layers == 3: 48 | # do nothing 49 | model.classifier = nn.Sequential() 50 | num_features = 256 * 6 * 6 51 | log.info("Final layer of encoder: AlexNet Maxpool 3") 52 | else: 53 | raise ValueError("Invalid parameter %d to pop function for %s: " % (n_layers, model_name)) 54 | 55 | elif model_name.startswith("vgg"): 56 | final_layer = model.classifier 57 | if n_layers == 1: 58 | model.classifier = nn.Sequential( 59 | nn.Linear(512 * 7 * 7, 4096), 60 | nn.ReLU(True), 61 | nn.Dropout(), 62 | nn.Linear(4096, 4096), 63 | ) 64 | num_features = 4096 65 | log.info("Final layer of encoder: VGG fc2") 66 | elif n_layers == 2: 67 | model.classifier = nn.Sequential( 68 | nn.Linear(512 * 7 * 7, 4096), 69 | ) 70 | log.info("Final layer of encoder: VGG fc1") 71 | num_features = 4096 72 | elif n_layers == 3: 73 | model.classifier = nn.Sequential() 74 | log.info("Final layer of encoder: VGG pool5") 75 | num_features = 512 * 7 * 7 76 | else: 77 | raise ValueError("Invalid parameter %d to pop function for %s: " % (n_layers, model_name)) 78 | 79 | elif model_name.startswith("squeezenet"): 80 | raise NotImplementedError 81 | elif model_name.startswith("densenet"): 82 | raise NotImplementedError 83 | elif model_name.startswith("inception"): 84 | raise NotImplementedError 85 | else: 86 | raise ValueError("%s is not a valid model name" % (model_name)) 87 | return model, num_features, final_layer 88 | 89 | 90 | def remove_cnn_classifier_layer(cnn): 91 | """Removes the final layer of a torchvision classification model, and figures out dimensionality of final layer""" 92 | # cnn should be a nn.Sequential(custom_model, nn.Linear) 93 | module_list = list(cnn.children()) 94 | assert (len(module_list) == 2 or len(module_list) == 3) and isinstance(module_list[1], nn.Linear) 95 | in_features = module_list[1].in_features 96 | module_list[1] = nn.Identity() 97 | cnn = nn.Sequential(*module_list) 98 | return cnn, in_features 99 | 100 | 101 | class Fusion(nn.Module): 102 | """Module for fusing spatial and flow features and passing through Linear layer""" 103 | 104 | def __init__( 105 | self, 106 | style, 107 | num_spatial_features, 108 | num_flow_features, 109 | num_classes, 110 | flow_fusion_weight=1.5, 111 | activation=nn.Identity(), 112 | ): 113 | super().__init__() 114 | self.style = style 115 | self.num_classes = num_classes 116 | self.activation = activation 117 | self.flow_fusion_weight = flow_fusion_weight 118 | 119 | if self.style == "average": 120 | # self.spatial_fc = nn.Linear(num_spatial_features,num_classes) 121 | # self.flow_fc = nn.Linear(num_flow_features, num_classes) 122 | 123 | self.num_features_out = num_classes 124 | 125 | elif self.style == "concatenate": 126 | self.num_features_out = num_classes 127 | self.fc = nn.Linear(num_spatial_features + num_flow_features, num_classes) 128 | 129 | elif self.style == "weighted_average": 130 | self.flow_weight = nn.Parameter(torch.Tensor([0.5]).float(), requires_grad=True) 131 | else: 132 | raise NotImplementedError 133 | 134 | def forward(self, spatial_features, flow_features): 135 | if self.style == "average": 136 | # spatial_logits = self.spatial_fc(spatial_features) 137 | # flow_logits = self.flow_fc(flow_features) 138 | 139 | return (spatial_features + flow_features * self.flow_fusion_weight) / (1 + self.flow_fusion_weight) 140 | # return((spatial_logits+flow_logits*self.flow_fusion_weight)/(1+self.flow_fusion_weight)) 141 | elif self.style == "concatenate": 142 | # if we're concatenating, we want the model to learn nonlinear mappings from the spatial logits and flow 143 | # logits that means we should apply an activation function note: this won't work if you froze both 144 | # encoding models 145 | features = self.activation(torch.cat((spatial_features, flow_features), dim=1)) 146 | return self.fc(features) 147 | elif self.style == "weighted_average": 148 | return self.flow_weight * flow_features + (1 - self.flow_weight) * spatial_features 149 | 150 | 151 | # from TResNet: https://arxiv.org/abs/2003.13630 152 | # in my hands it was not appreciably faster 153 | class FastGlobalAvgPool2d(nn.Module): 154 | def __init__(self, flatten=False): 155 | super().__init__() 156 | self.flatten = flatten 157 | 158 | def forward(self, x): 159 | if self.flatten: 160 | in_size = x.size() 161 | return x.view((in_size[0], in_size[1], -1)).mean(dim=2) 162 | else: 163 | return x.view(x.size(0), x.size(1), -1).mean(-1).view(x.size(0), x.size(1), 1, 1) 164 | -------------------------------------------------------------------------------- /deepethogram/file_io.py: -------------------------------------------------------------------------------- 1 | import os 2 | import warnings 3 | from typing import Union 4 | 5 | import h5py 6 | import numpy as np 7 | import pandas as pd 8 | from vidio import VideoReader, VideoWriter 9 | 10 | 11 | def read_labels(labelfile: Union[str, os.PathLike]) -> np.ndarray: 12 | """convenience function for reading labels from a .csv or .h5 file""" 13 | labeltype = os.path.splitext(labelfile)[1][1:] 14 | if labeltype == "csv": 15 | label = read_label_csv(labelfile) 16 | elif labeltype == "h5": 17 | label = read_label_hdf5(labelfile) 18 | else: 19 | raise ValueError("Unknown labeltype: {}".format(labeltype)) 20 | H, W = label.shape 21 | # labels should be time x num_behaviors 22 | if W > H: 23 | label = label.T 24 | if label.shape[1] == 1: 25 | # add a background class 26 | warnings.warn("binary labels found, adding background class") 27 | label = np.hstack((np.logical_not(label), label)) 28 | return label 29 | 30 | 31 | def read_label_hdf5(labelfile: Union[str, os.PathLike]) -> np.ndarray: 32 | """read labels from an HDF5 file. Must end in .h5 33 | 34 | Assumes that labels are in a dataset with name 'scores' or 'labels' 35 | Parameters 36 | ---------- 37 | labelfile 38 | 39 | Returns 40 | ------- 41 | 42 | """ 43 | with h5py.File(labelfile, "r") as f: 44 | keys = list(f.keys()) 45 | if "scores" in keys: 46 | key = "scores" 47 | elif "labels" in keys: 48 | key = "labels" 49 | else: 50 | raise ValueError("not sure which dataset in hdf5 contains labels: {}".format(keys)) 51 | label = f[key][:].astype(np.int64) 52 | if label.ndim == 1: 53 | label = label[..., np.newaxis] 54 | return label 55 | 56 | 57 | def read_label_csv(labelfile: Union[str, os.PathLike]) -> np.ndarray: 58 | """Reads CSV of labels into a numpy array 59 | 60 | Parameters 61 | ---------- 62 | labelfile : Union[str, os.PathLike] 63 | Path to label .csv 64 | 65 | Returns 66 | ------- 67 | np.ndarray 68 | T x K binary array of labels 69 | """ 70 | 71 | df = pd.read_csv(labelfile, index_col=0) 72 | label = df.values.astype(np.int64) 73 | if label.ndim == 1: 74 | label = label[..., np.newaxis] 75 | return label 76 | 77 | 78 | def convert_video(videofile: Union[str, os.PathLike], movie_format: str, *args, **kwargs) -> None: 79 | """Converts videos from one file format to another using VidIO 80 | 81 | Parameters 82 | ---------- 83 | videofile : Union[str, os.PathLike] 84 | Path to a video file 85 | movie_format : str 86 | One of ['ffmpeg', 'opencv', 'hdf5', 'directory'] 87 | ffmpeg: converts to libx264 using ffmpeg 88 | OpenCV: converts to MJPG (by default) 89 | HDF5: converts to an HDF5 file or PNG bytestrings. Lossless compression. Compromise between fastest reading 90 | (PNG directory), and ease of transferring across filesystems (e.g. to a server) 91 | directory: explodes into directory of PNG files 92 | 93 | Raises 94 | ------ 95 | ValueError 96 | If movie format is not one of above, raise 97 | """ 98 | with VideoReader(videofile) as reader: 99 | basename = os.path.splitext(videofile)[0] 100 | if movie_format == "ffmpeg": 101 | out_filename = basename + ".mp4" 102 | elif movie_format == "opencv": 103 | out_filename = basename + ".avi" 104 | elif movie_format == "hdf5": 105 | out_filename = basename + ".h5" 106 | elif movie_format == "directory": 107 | out_filename = basename 108 | else: 109 | raise ValueError("unexpected value of movie format: {}".format(movie_format)) 110 | with VideoWriter(out_filename, movie_format=movie_format, *args, **kwargs) as writer: 111 | for frame in reader: 112 | writer.write(frame) 113 | -------------------------------------------------------------------------------- /deepethogram/flow_generator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/flow_generator/__init__.py -------------------------------------------------------------------------------- /deepethogram/flow_generator/__main__.py: -------------------------------------------------------------------------------- 1 | from .train import main 2 | 3 | main() 4 | -------------------------------------------------------------------------------- /deepethogram/flow_generator/models/FlowNetS.py: -------------------------------------------------------------------------------- 1 | """ 2 | modified from here 3 | # https://github.com/ClementPinard/FlowNetPytorch/blob/master/models/FlowNetS.py 4 | # MIT License 5 | # 6 | # Copyright (c) 2017 Clément Pinard 7 | # 8 | # Permission is hereby granted, free of charge, to any person obtaining a copy 9 | # of this software and associated documentation files (the "Software"), to deal 10 | # in the Software without restriction, including without limitation the rights 11 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | # copies of the Software, and to permit persons to whom the Software is 13 | # furnished to do so, subject to the following conditions: 14 | # 15 | # The above copyright notice and this permission notice shall be included in all 16 | # copies or substantial portions of the Software. 17 | # 18 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | # SOFTWARE. 25 | 26 | References 27 | ------- 28 | .. [1]: Fischer et al. FlowNet: Learning optical flow with convolutional networks. ICCV 2015 29 | https://arxiv.org/abs/1504.06852 30 | """ 31 | 32 | import torch 33 | import torch.nn as nn 34 | import torch.nn.functional as F 35 | from torch.nn import init 36 | 37 | from .components import conv, deconv, get_hw, predict_flow 38 | 39 | 40 | class FlowNetS(nn.Module): 41 | def __init__(self, num_images=2, batchNorm=True, flow_div=1): 42 | super(FlowNetS, self).__init__() 43 | self.flow_div = flow_div 44 | input_channels = num_images * 3 45 | self.batchNorm = batchNorm 46 | self.conv1 = conv(self.batchNorm, input_channels, 64, kernel_size=7, stride=2) 47 | self.conv2 = conv(self.batchNorm, 64, 128, kernel_size=5, stride=2) 48 | self.conv3 = conv(self.batchNorm, 128, 256, kernel_size=5, stride=2) 49 | self.conv3_1 = conv(self.batchNorm, 256, 256) 50 | self.conv4 = conv(self.batchNorm, 256, 512, stride=2) 51 | self.conv4_1 = conv(self.batchNorm, 512, 512) 52 | self.conv5 = conv(self.batchNorm, 512, 512, stride=2) 53 | self.conv5_1 = conv(self.batchNorm, 512, 512) 54 | self.conv6 = conv(self.batchNorm, 512, 1024, stride=2) 55 | self.conv6_1 = conv(self.batchNorm, 1024, 1024) 56 | 57 | self.deconv5 = deconv(1024, 512) 58 | self.deconv4 = deconv(1026, 256) 59 | self.deconv3 = deconv(770, 128) 60 | self.deconv2 = deconv(386, 64) 61 | 62 | self.predict_flow6 = predict_flow(1024) 63 | self.predict_flow5 = predict_flow(1026) 64 | self.predict_flow4 = predict_flow(770) 65 | self.predict_flow3 = predict_flow(386) 66 | self.predict_flow2 = predict_flow(194) 67 | 68 | self.upsampled_flow6_to_5 = nn.ConvTranspose2d(2, 2, 4, 2, 1, bias=False) 69 | self.upsampled_flow5_to_4 = nn.ConvTranspose2d(2, 2, 4, 2, 1, bias=False) 70 | self.upsampled_flow4_to_3 = nn.ConvTranspose2d(2, 2, 4, 2, 1, bias=False) 71 | self.upsampled_flow3_to_2 = nn.ConvTranspose2d(2, 2, 4, 2, 1, bias=False) 72 | 73 | # initialize weights 74 | for m in self.modules(): 75 | if isinstance(m, nn.Conv2d): 76 | if m.bias is not None: 77 | init.uniform_(m.bias) 78 | init.xavier_uniform_(m.weight) 79 | 80 | if isinstance(m, nn.ConvTranspose2d): 81 | if m.bias is not None: 82 | init.uniform_(m.bias) 83 | init.xavier_uniform_(m.weight) 84 | 85 | self.upsample1 = nn.Upsample(scale_factor=4, mode="bilinear") 86 | 87 | def forward(self, x): 88 | out_conv1 = self.conv1(x) 89 | 90 | out_conv2 = self.conv2(out_conv1) 91 | out_conv3 = self.conv3_1(self.conv3(out_conv2)) 92 | out_conv4 = self.conv4_1(self.conv4(out_conv3)) 93 | out_conv5 = self.conv5_1(self.conv5(out_conv4)) 94 | out_conv6 = self.conv6_1(self.conv6(out_conv5)) 95 | 96 | flow6 = self.predict_flow6(out_conv6) * self.flow_div 97 | 98 | # EXPLANATION FOR MULTIPLYING BY 2 99 | # when reconstructing t0_est from t1 and the flow, our reconstructor normalizes by the width of the flow. 100 | # a 256x256 image / 32 = 8x8 101 | # a value of 1 in flow6 will be divided by (WIDTH / 2), because the top-left corner is (-1, -1) and the 102 | # top-right is (1, -1). so a value of 2 is 100% of the image. dividing by (WIDTH / 2) ensures that a value of 103 | # So if flow6 has a value of 8, the calculation is: 8 / (8/2) = 2, so a flow that moves all the way across the 104 | # image will be mapped to a value of 2, as expected above. SO 105 | # a value of 1 in flow6: 1 / (8 / 2) = 0.25, which corresponds to 1/8 of the image, or one pixel. 106 | # in the next line of code, we will upsample flow6 by 2, to a size of 16x16 107 | # a value of 1 in flow6 will naively be mapped to a value of 1 in flow5. now, this movement of 1 pixel no 108 | # longer means 1/8 of the image, it will only move 1/16 of the image. So to correct for this, we multiply 109 | # the upsampled version by 2. 110 | flow6_up = self.upsampled_flow6_to_5(flow6) * 2 111 | out_deconv5 = self.deconv5(out_conv6) 112 | 113 | concat5 = torch.cat((out_conv5, out_deconv5, flow6_up), 1) 114 | flow5 = self.predict_flow5(concat5) * self.flow_div 115 | flow5_up = self.upsampled_flow5_to_4(flow5) * 2 116 | out_deconv4 = self.deconv4(concat5) 117 | 118 | concat4 = torch.cat((out_conv4, out_deconv4, flow5_up), 1) 119 | flow4 = self.predict_flow4(concat4) * self.flow_div 120 | flow4_up = self.upsampled_flow4_to_3(flow4) * 2 121 | out_deconv3 = self.deconv3(concat4) 122 | 123 | if get_hw(out_conv3) != get_hw(out_deconv3): 124 | out_conv3 = F.interpolate(out_conv3, size=get_hw(out_deconv3), mode="bilinear", align_corners=False) 125 | 126 | concat3 = torch.cat((out_conv3, out_deconv3, flow4_up), 1) 127 | flow3 = self.predict_flow3(concat3) * self.flow_div 128 | flow3_up = self.upsampled_flow3_to_2(flow3) * 2 129 | out_deconv2 = self.deconv2(concat3) 130 | 131 | concat2 = torch.cat((out_conv2, out_deconv2, flow3_up), 1) 132 | flow2 = self.predict_flow2(concat2) * self.flow_div 133 | 134 | if self.training: 135 | return flow2, flow3, flow4, flow5, flow6 136 | else: 137 | return (flow2,) 138 | -------------------------------------------------------------------------------- /deepethogram/flow_generator/models/MotionNet.py: -------------------------------------------------------------------------------- 1 | """Re-implementation of the MotionNet architecture 2 | 3 | References 4 | ------- 5 | .. [1]: Zhu, Lan, Newsam, and Hauptman. Hidden Two-stream convolutional networks for action recognition. 6 | https://arxiv.org/abs/1704.00389 7 | 8 | Based on code from Nvidia's FlowNet2 9 | Copyright 2017 NVIDIA CORPORATION 10 | 11 | Licensed under the Apache License, Version 2.0 (the "License"); 12 | you may not use this file except in compliance with the License. 13 | You may obtain a copy of the License at 14 | 15 | http://www.apache.org/licenses/LICENSE-2.0 16 | 17 | Unless required by applicable law or agreed to in writing, software 18 | distributed under the License is distributed on an "AS IS" BASIS, 19 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 20 | See the License for the specific language governing permissions and 21 | limitations under the License. 22 | 23 | Changes: changed filter sizes, number of input images, number of layers, added cropping or interpolation for 24 | non-power-of-two shaped images, and multiplication... only kept their naming convention and overall structure 25 | """ 26 | 27 | import logging 28 | 29 | import torch.nn as nn 30 | from torch.nn import init 31 | 32 | from .components import CropConcat, conv, deconv, i_conv, predict_flow 33 | 34 | log = logging.getLogger(__name__) 35 | 36 | 37 | class MotionNet(nn.Module): 38 | def __init__(self, num_images=11, batchNorm=True, flow_div=1): 39 | super(MotionNet, self).__init__() 40 | 41 | self.num_images = num_images 42 | self.out_channels = int((num_images - 1) * 2) 43 | self.batchNorm = batchNorm 44 | 45 | log.debug("ignoring flow div value of {}: setting to 1 instead".format(flow_div)) 46 | self.flow_div = 1 47 | 48 | self.conv1 = conv(self.batchNorm, self.num_images * 3, 64) 49 | self.conv1_1 = conv(self.batchNorm, 64, 64) 50 | 51 | self.conv2 = conv(self.batchNorm, 64, 128, stride=2) 52 | self.conv2_1 = conv(self.batchNorm, 128, 128) 53 | 54 | self.conv3 = conv(self.batchNorm, 128, 256, stride=2) 55 | self.conv3_1 = conv(self.batchNorm, 256, 256) 56 | 57 | self.conv4 = conv(self.batchNorm, 256, 512, stride=2) 58 | self.conv4_1 = conv(self.batchNorm, 512, 512) 59 | 60 | self.conv5 = conv(self.batchNorm, 512, 512, stride=2) 61 | self.conv5_1 = conv(self.batchNorm, 512, 512) 62 | 63 | self.conv6 = conv(self.batchNorm, 512, 1024, stride=2) 64 | self.conv6_1 = conv(self.batchNorm, 1024, 1024) 65 | 66 | self.deconv5 = deconv(1024, 512) 67 | self.deconv4 = deconv(1024 + self.out_channels, 256) 68 | self.deconv3 = deconv(768 + self.out_channels, 128) 69 | self.deconv2 = deconv(384 + self.out_channels, 64) 70 | 71 | self.xconv5 = i_conv(self.batchNorm, 1024 + self.out_channels, 512) 72 | self.xconv4 = i_conv(self.batchNorm, 768 + self.out_channels, 256) 73 | self.xconv3 = i_conv(self.batchNorm, 384 + self.out_channels, 128) 74 | self.xconv2 = i_conv(self.batchNorm, 192 + self.out_channels, 64) 75 | 76 | self.predict_flow6 = predict_flow(1024, out_planes=self.out_channels) 77 | self.predict_flow5 = predict_flow(512, out_planes=self.out_channels) 78 | self.predict_flow4 = predict_flow(256, out_planes=self.out_channels) 79 | self.predict_flow3 = predict_flow(128, out_planes=self.out_channels) 80 | self.predict_flow2 = predict_flow(64, out_planes=self.out_channels) 81 | 82 | self.upsampled_flow6_to_5 = nn.ConvTranspose2d(self.out_channels, self.out_channels, 4, 2, 1) 83 | self.upsampled_flow5_to_4 = nn.ConvTranspose2d(self.out_channels, self.out_channels, 4, 2, 1) 84 | self.upsampled_flow4_to_3 = nn.ConvTranspose2d(self.out_channels, self.out_channels, 4, 2, 1) 85 | self.upsampled_flow3_to_2 = nn.ConvTranspose2d(self.out_channels, self.out_channels, 4, 2, 1) 86 | self.concat = CropConcat(dim=1) 87 | 88 | for m in self.modules(): 89 | if isinstance(m, nn.Conv2d): 90 | if m.bias is not None: 91 | init.uniform_(m.bias) 92 | init.xavier_uniform_(m.weight) 93 | 94 | if isinstance(m, nn.ConvTranspose2d): 95 | if m.bias is not None: 96 | init.uniform_(m.bias) 97 | init.xavier_uniform_(m.weight) 98 | # init_deconv_bilinear(m.weight) 99 | self.upsample1 = nn.Upsample(scale_factor=4, mode="bilinear") 100 | 101 | def forward(self, x): 102 | N, C, H, W = x.shape 103 | # 1 -> 1 104 | out_conv1 = self.conv1_1(self.conv1(x)) 105 | # 1 -> 1/2 106 | out_conv2 = self.conv2_1(self.conv2(out_conv1)) 107 | # 1/2 -> 1/4 108 | out_conv3 = self.conv3_1(self.conv3(out_conv2)) 109 | # 1/4 -> 1/8 110 | out_conv4 = self.conv4_1(self.conv4(out_conv3)) 111 | # 1/8 -> 1/16 112 | out_conv5 = self.conv5_1(self.conv5(out_conv4)) 113 | # 1/16 -> 1/32 114 | out_conv6 = self.conv6_1(self.conv6(out_conv5)) 115 | 116 | flow6 = self.predict_flow6(out_conv6) * self.flow_div 117 | # EXPLANATION FOR MULTIPLYING BY 2 118 | # when reconstructing t0_est from t1 and the flow, our reconstructor normalizes by the width of the flow. 119 | # a 256x256 image / 32 = 8x8 120 | # a value of 1 in flow6 will be divided by (WIDTH / 2), because the top-left corner is (-1, -1) and the 121 | # top-right is (1, -1). so a value of 2 is 100% of the image. dividing by (WIDTH / 2) ensures that a raw pixel 122 | # value of IMAGE_WIDTH gets mapped to 2! 123 | # So if flow6 has a value of 8, the calculation is: 8 / (8/2) = 2, so a flow that moves all the way across the 124 | # image will be mapped to a value of 2, as expected above. SO 125 | # a value of 1 in flow6: 1 / (8 / 2) = 0.25, which corresponds to 1/8 of the image, or one pixel. 126 | # in the next line of code, we will upsample flow6 by 2, to a size of 16x16 127 | # a value of 1 in flow6 will naively be mapped to a value of 1 in flow5. now, this movement of 1 pixel no 128 | # longer means 1/8 of the image, it will only move 1/16 of the image. So to correct for this, we multiply 129 | # the upsampled version by 2. 130 | flow6_up = self.upsampled_flow6_to_5(flow6) * 2 131 | out_deconv5 = self.deconv5(out_conv6) 132 | 133 | # if the image sizes are not divisible by 8, there will be rounding errors in the size 134 | # between the downsampling and upsampling phases 135 | concat5 = self.concat((out_conv5, out_deconv5, flow6_up)) 136 | out_interconv5 = self.xconv5(concat5) 137 | flow5 = self.predict_flow5(out_interconv5) * self.flow_div 138 | 139 | flow5_up = self.upsampled_flow5_to_4(flow5) * 2 140 | out_deconv4 = self.deconv4(concat5) 141 | 142 | concat4 = self.concat((out_conv4, out_deconv4, flow5_up)) 143 | out_interconv4 = self.xconv4(concat4) 144 | flow4 = self.predict_flow4(out_interconv4) * self.flow_div 145 | flow4_up = self.upsampled_flow4_to_3(flow4) * 2 146 | out_deconv3 = self.deconv3(concat4) 147 | 148 | # if the image sizes are not divisible by 8, there will be rounding errors in the size 149 | # between the downsampling and upsampling phases 150 | concat3 = self.concat((out_conv3, out_deconv3, flow4_up)) 151 | out_interconv3 = self.xconv3(concat3) 152 | flow3 = self.predict_flow3(out_interconv3) * self.flow_div 153 | flow3_up = self.upsampled_flow3_to_2(flow3) * 2 154 | out_deconv2 = self.deconv2(concat3) 155 | 156 | concat2 = self.concat((out_conv2, out_deconv2, flow3_up)) 157 | out_interconv2 = self.xconv2(concat2) 158 | flow2 = self.predict_flow2(out_interconv2) * self.flow_div 159 | 160 | return flow2, flow3, flow4 161 | -------------------------------------------------------------------------------- /deepethogram/flow_generator/models/TinyMotionNet.py: -------------------------------------------------------------------------------- 1 | """Re-implementation of the TinyMotionNet architecture 2 | 3 | References 4 | ------- 5 | .. [1]: Zhu, Lan, Newsam, and Hauptman. Hidden Two-stream convolutional networks for action recognition. 6 | https://arxiv.org/abs/1704.00389 7 | 8 | Based on code from Nvidia's FlowNet2 9 | Copyright 2017 NVIDIA CORPORATION 10 | 11 | Licensed under the Apache License, Version 2.0 (the "License"); 12 | you may not use this file except in compliance with the License. 13 | You may obtain a copy of the License at 14 | 15 | http://www.apache.org/licenses/LICENSE-2.0 16 | 17 | Unless required by applicable law or agreed to in writing, software 18 | distributed under the License is distributed on an "AS IS" BASIS, 19 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 20 | See the License for the specific language governing permissions and 21 | limitations under the License. 22 | 23 | Changes: changed filter sizes, number of input images, number of layers, added cropping or interpolation for 24 | non-power-of-two shaped images, and multiplication... only kept their naming convention and overall structure 25 | """ 26 | 27 | import logging 28 | 29 | import torch.nn as nn 30 | 31 | from .components import CropConcat, Interpolate, conv, deconv, i_conv, predict_flow 32 | 33 | log = logging.getLogger(__name__) 34 | 35 | 36 | # modified from https://github.com/NVIDIA/flownet2-pytorch/blob/master/networks/FlowNetSD.py 37 | # https://github.com/NVIDIA/flownet2-pytorch/blob/master/networks/submodules.py 38 | class TinyMotionNet(nn.Module): 39 | def __init__(self, num_images=11, input_channels=None, batchNorm=True, output_channels=None, flow_div=1): 40 | super().__init__() 41 | self.num_images = num_images 42 | if input_channels is None: 43 | self.input_channels = self.num_images * 3 44 | else: 45 | self.input_channels = int(input_channels) 46 | if output_channels is None: 47 | self.output_channels = int((num_images - 1) * 2) 48 | else: 49 | self.output_channels = int(output_channels) 50 | 51 | self.batchNorm = batchNorm 52 | log.debug("ignoring flow div value of {}: setting to 1 instead".format(flow_div)) 53 | self.flow_div = 1 54 | 55 | self.conv1 = conv(self.batchNorm, self.input_channels, 64, kernel_size=7) 56 | self.conv2 = conv(self.batchNorm, 64, 128, stride=2, kernel_size=5) 57 | self.conv3 = conv(self.batchNorm, 128, 256, stride=2) 58 | self.conv4 = conv(self.batchNorm, 256, 128, stride=2) 59 | 60 | self.deconv3 = deconv(128, 128) 61 | self.deconv2 = deconv(128, 64) 62 | 63 | self.xconv3 = i_conv(self.batchNorm, 384 + self.output_channels, 128) 64 | self.xconv2 = i_conv(self.batchNorm, 192 + self.output_channels, 64) 65 | 66 | self.predict_flow4 = predict_flow(128, out_planes=self.output_channels) 67 | self.predict_flow3 = predict_flow(128, out_planes=self.output_channels) 68 | self.predict_flow2 = predict_flow(64, out_planes=self.output_channels) 69 | 70 | self.upsampled_flow4_to_3 = nn.ConvTranspose2d(self.output_channels, self.output_channels, 4, 2, 1) 71 | self.upsampled_flow3_to_2 = nn.ConvTranspose2d(self.output_channels, self.output_channels, 4, 2, 1) 72 | 73 | self.concat = CropConcat(dim=1) 74 | self.interpolate = Interpolate 75 | 76 | def forward(self, x): 77 | N, C, H, W = x.shape 78 | out_conv1 = self.conv1(x) # 1 -> 1 79 | out_conv2 = self.conv2(out_conv1) # 1 -> 1/2 80 | out_conv3 = self.conv3(out_conv2) # 1/2 -> 1/4 81 | out_conv4 = self.conv4(out_conv3) # 1/4 -> 1/8 82 | 83 | flow4 = self.predict_flow4(out_conv4) * self.flow_div 84 | # see motionnet.py for explanation of multiplying by 2 85 | flow4_up = self.upsampled_flow4_to_3(flow4) * 2 86 | out_deconv3 = self.deconv3(out_conv4) 87 | 88 | concat3 = self.concat((out_conv3, out_deconv3, flow4_up)) 89 | out_interconv3 = self.xconv3(concat3) 90 | flow3 = self.predict_flow3(out_interconv3) * self.flow_div 91 | flow3_up = self.upsampled_flow3_to_2(flow3) * 2 92 | out_deconv2 = self.deconv2(out_interconv3) 93 | 94 | concat2 = self.concat((out_conv2, out_deconv2, flow3_up)) 95 | out_interconv2 = self.xconv2(concat2) 96 | flow2 = self.predict_flow2(out_interconv2) * self.flow_div 97 | 98 | return flow2, flow3, flow4 99 | -------------------------------------------------------------------------------- /deepethogram/flow_generator/models/TinyMotionNet3D.py: -------------------------------------------------------------------------------- 1 | """ 2 | Based on code from Nvidia's FlowNet2 3 | Copyright 2017 NVIDIA CORPORATION 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | 17 | Changes: 2D -> 3D. changed filter sizes, number of input images, number of layers... only kept their naming 18 | convention and overall structure 19 | """ 20 | 21 | import logging 22 | 23 | import torch.nn as nn 24 | 25 | from .components import CropConcat, conv3d, deconv3d, predict_flow_3d 26 | 27 | 28 | class TinyMotionNet3D(nn.Module): 29 | def __init__(self, num_images=11, input_channels=3, batchnorm=True, flow_div=1, channel_base=16): 30 | super().__init__() 31 | self.num_images = num_images 32 | if input_channels is None: 33 | self.input_channels = self.num_images * 3 34 | else: 35 | self.input_channels = int(input_channels) 36 | 37 | self.batchnorm = batchnorm 38 | bias = not self.batchnorm 39 | logging.debug("ignoring flow div value of {}: setting to 1 instead".format(flow_div)) 40 | self.flow_div = 1 41 | 42 | self.channels = [channel_base * (2**i) for i in range(0, 3)] 43 | print(self.channels) 44 | 45 | self.conv1 = conv3d(self.input_channels, self.channels[0], kernel_size=7, batchnorm=batchnorm, bias=bias) 46 | self.conv2 = conv3d( 47 | self.channels[0], self.channels[1], stride=(1, 2, 2), kernel_size=5, batchnorm=batchnorm, bias=bias 48 | ) 49 | self.conv3 = conv3d(self.channels[1], self.channels[2], stride=(1, 2, 2), batchnorm=batchnorm, bias=bias) 50 | self.conv4 = conv3d(self.channels[2], self.channels[1], stride=(1, 2, 2), batchnorm=batchnorm, bias=bias) 51 | 52 | self.conv5 = conv3d(self.channels[1], self.channels[1], kernel_size=(2, 3, 3), batchnorm=batchnorm, bias=bias) 53 | 54 | self.deconv3 = deconv3d( 55 | self.channels[1], 56 | self.channels[1], 57 | kernel_size=(1, 4, 4), 58 | stride=(1, 2, 2), 59 | padding=(0, 1, 1), 60 | batchnorm=batchnorm, 61 | bias=bias, 62 | ) 63 | self.deconv2 = deconv3d( 64 | self.channels[1], 65 | self.channels[0], 66 | kernel_size=(1, 4, 4), 67 | stride=(1, 2, 2), 68 | padding=(0, 1, 1), 69 | batchnorm=batchnorm, 70 | bias=bias, 71 | ) 72 | 73 | self.iconv3 = conv3d(self.channels[2], self.channels[2], kernel_size=(2, 3, 3), batchnorm=batchnorm, bias=bias) 74 | self.iconv2 = conv3d(self.channels[1], self.channels[1], kernel_size=(2, 3, 3), batchnorm=batchnorm, bias=bias) 75 | 76 | self.xconv3 = conv3d(self.channels[1] + self.channels[2] + 2, self.channels[1], batchnorm=batchnorm, act=False) 77 | self.xconv2 = conv3d(self.channels[0] + self.channels[1] + 2, self.channels[0], batchnorm=batchnorm, act=False) 78 | 79 | self.predict_flow4 = predict_flow_3d(self.channels[1], 2) 80 | self.predict_flow3 = predict_flow_3d(self.channels[1], 2) 81 | self.predict_flow2 = predict_flow_3d(self.channels[0], 2) 82 | 83 | self.upsampled_flow4_to_3 = nn.ConvTranspose3d(2, 2, kernel_size=(1, 4, 4), stride=(1, 2, 2), padding=(0, 1, 1)) 84 | self.upsampled_flow3_to_2 = nn.ConvTranspose3d(2, 2, kernel_size=(1, 4, 4), stride=(1, 2, 2), padding=(0, 1, 1)) 85 | 86 | self.concat = CropConcat(dim=1) 87 | 88 | def forward(self, x): 89 | # N, C, T, H, W = x.shape 90 | out_conv1 = self.conv1(x) # 1 -> 1 91 | out_conv2 = self.conv2(out_conv1) # 1 -> 1/2 92 | out_conv3 = self.conv3(out_conv2) # 1/2 -> 1/4 93 | out_conv4 = self.conv4(out_conv3) # 1/4 -> 1/8 94 | out_conv5 = self.conv5(out_conv4) 95 | 96 | flow4 = self.predict_flow4(out_conv5) * self.flow_div 97 | # see motionnet.py for explanation of multiplying by 2 98 | flow4_up = self.upsampled_flow4_to_3(flow4) * 2 99 | out_deconv3 = self.deconv3(out_conv5) 100 | 101 | iconv3 = self.iconv3(out_conv3) 102 | concat3 = self.concat((iconv3, out_deconv3, flow4_up)) 103 | out_interconv3 = self.xconv3(concat3) 104 | flow3 = self.predict_flow3(out_interconv3) * self.flow_div 105 | flow3_up = self.upsampled_flow3_to_2(flow3) * 2 106 | out_deconv2 = self.deconv2(out_interconv3) 107 | 108 | iconv2 = self.iconv2(out_conv2) 109 | 110 | concat2 = self.concat((iconv2, out_deconv2, flow3_up)) 111 | out_interconv2 = self.xconv2(concat2) 112 | flow2 = self.predict_flow2(out_interconv2) * self.flow_div 113 | 114 | return flow2, flow3, flow4 115 | -------------------------------------------------------------------------------- /deepethogram/flow_generator/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/flow_generator/models/__init__.py -------------------------------------------------------------------------------- /deepethogram/flow_generator/models/components.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | 3 | import torch 4 | import torch.nn as nn 5 | 6 | 7 | def conv(batchNorm: bool, in_planes: int, out_planes: int, kernel_size: int = 3, stride: int = 1, bias: bool = True): 8 | """Convenience function for conv2d + optional BN + leakyRELU""" 9 | if batchNorm: 10 | return nn.Sequential( 11 | nn.Conv2d( 12 | in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=(kernel_size - 1) // 2, bias=bias 13 | ), 14 | nn.BatchNorm2d(out_planes), 15 | nn.LeakyReLU(0.1, inplace=True), 16 | ) 17 | else: 18 | return nn.Sequential( 19 | nn.Conv2d( 20 | in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=(kernel_size - 1) // 2, bias=bias 21 | ), 22 | nn.LeakyReLU(0.1, inplace=True), 23 | ) 24 | 25 | 26 | def crop_like(input: torch.Tensor, target: torch.Tensor) -> torch.Tensor: 27 | """Crops input to target's H,W""" 28 | if input.size()[2:] == target.size()[2:]: 29 | return input 30 | else: 31 | return input[:, :, : target.size(2), : target.size(3)] 32 | 33 | 34 | def deconv(in_planes: int, out_planes: int, bias: bool = True): 35 | """Convenience function for ConvTranspose2d + leakyRELU""" 36 | return nn.Sequential( 37 | nn.ConvTranspose2d(in_planes, out_planes, kernel_size=4, stride=2, padding=1, bias=bias), 38 | nn.LeakyReLU(0.1, inplace=True), 39 | ) 40 | 41 | 42 | class Interpolate(nn.Module): 43 | """Wrapper to be able to perform interpolation in a nn.Sequential 44 | 45 | Modified from the PyTorch Forums: 46 | https://discuss.pytorch.org/t/using-nn-function-interpolate-inside-nn-sequential/23588/2 47 | """ 48 | 49 | def __init__(self, size=None, scale_factor=None, mode: str = "bilinear"): 50 | super(Interpolate, self).__init__() 51 | self.interp = nn.functional.interpolate 52 | self.size = size 53 | self.scale_factor = scale_factor 54 | assert mode in ["nearest", "linear", "bilinear", "bicubic", "trilinear", "area"] 55 | self.mode = mode 56 | if self.mode == "nearest": 57 | self.align_corners = None 58 | else: 59 | self.align_corners = False 60 | 61 | def forward(self, x): 62 | x = self.interp( 63 | x, size=self.size, scale_factor=self.scale_factor, mode=self.mode, align_corners=self.align_corners 64 | ) 65 | return x 66 | 67 | 68 | def i_conv(batchNorm: bool, in_planes: int, out_planes: int, kernel_size: int = 3, stride: int = 1, bias: bool = True): 69 | """Convenience function for conv2d + optional BN + no activation""" 70 | if batchNorm: 71 | return nn.Sequential( 72 | nn.Conv2d( 73 | in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=(kernel_size - 1) // 2, bias=bias 74 | ), 75 | nn.BatchNorm2d(out_planes), 76 | ) 77 | else: 78 | return nn.Sequential( 79 | nn.Conv2d( 80 | in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=(kernel_size - 1) // 2, bias=bias 81 | ), 82 | ) 83 | 84 | 85 | def predict_flow(in_planes: int, out_planes: int = 2, bias: bool = False): 86 | """Convenience function for 3x3 conv2d with same padding""" 87 | return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=1, padding=1, bias=bias) 88 | 89 | 90 | def get_hw(tensor): 91 | """Convenience function for getting the size of the last two dimensions in a tensor""" 92 | return tensor.size(-2), tensor.size(-1) 93 | 94 | 95 | class CropConcat(nn.Module): 96 | """Module for concatenating 2 tensors of slightly different shape.""" 97 | 98 | def __init__(self, dim: int = 1): 99 | super().__init__() 100 | self.dim = dim 101 | 102 | def forward(self, tensors: tuple) -> torch.Tensor: 103 | assert isinstance(tensors, tuple) 104 | hs, ws = [tensor.size(-2) for tensor in tensors], [tensor.size(-1) for tensor in tensors] 105 | h, w = min(hs), min(ws) 106 | 107 | return torch.cat(tuple([tensor[..., :h, :w] for tensor in tensors]), dim=self.dim) 108 | 109 | 110 | def conv3d( 111 | in_planes: int, 112 | out_planes: int, 113 | kernel_size: Union[int, tuple] = 3, 114 | stride: Union[int, tuple] = 1, 115 | bias: bool = True, 116 | batchnorm: bool = True, 117 | act: bool = True, 118 | padding: Union[int, tuple, None] = None, 119 | ): 120 | """3D convolution 121 | 122 | Expects inputs of shape N, C, D/F/T, H, W. 123 | D/F/T is frames, depth, time-- the extra axis compared to 2D convolution. 124 | Returns output of shape N, C_out, D/F/T_out, H_out, W_out. 125 | Out shape will be determined by input parameters. for more information see PyTorch docs 126 | https://pytorch.org/docs/master/generated/torch.nn.Conv3d.html 127 | 128 | Args: 129 | in_planes: int 130 | Number of channels in input tensor. 131 | out_planes: int 132 | Number of channels in output tensor 133 | kernel_size: int, tuple 134 | Size of 3D convolutional kernel. in order of (D/F/T, H, W). If int, size is repeated 3X 135 | stride: int, tuple 136 | Stride of convolutional kernel in D/F/T, H, W order 137 | bias: bool 138 | if True, adds a bias parameter 139 | batchnorm: bool 140 | if True, adds batchnorm 3D 141 | act: bool 142 | if True, adds LeakyRelu after (optional) batchnorm 143 | padding: int, tuple 144 | padding in T, H, W. If int, repeats 3X. if none, "same" padding, so that the inputs are the same shape 145 | as the outputs (assuming stride 1) 146 | Returns: 147 | nn.Sequential with conv3d, (batchnorm), (activation function) 148 | """ 149 | modules = [] 150 | if padding is None and isinstance(kernel_size, int): 151 | padding = (kernel_size - 1) // 2 152 | elif padding is None and isinstance(kernel_size, tuple): 153 | padding = ((kernel_size[0] - 1) // 2, (kernel_size[1] - 1) // 2, (kernel_size[2] - 1) // 2) 154 | else: 155 | raise ValueError("Unknown padding type {} and kernel_size type: {}".format(padding, kernel_size)) 156 | 157 | modules.append(nn.Conv3d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias)) 158 | if batchnorm: 159 | modules.append(nn.BatchNorm3d(out_planes)) 160 | if act: 161 | modules.append(nn.LeakyReLU(0.1, inplace=True)) 162 | return nn.Sequential(*modules) 163 | 164 | 165 | def deconv3d( 166 | in_planes: int, 167 | out_planes: int, 168 | kernel_size: int = 4, 169 | stride: int = 2, 170 | bias: bool = True, 171 | batchnorm: bool = True, 172 | act: bool = True, 173 | padding: int = 1, 174 | ): 175 | """Convenience function for ConvTranspose3D. Optionally adds batchnorm3d, leakyrelu""" 176 | modules = [ 177 | nn.ConvTranspose3d(in_planes, out_planes, kernel_size=kernel_size, stride=stride, bias=bias, padding=padding) 178 | ] 179 | if batchnorm: 180 | modules.append(nn.BatchNorm3d(out_planes)) 181 | if act: 182 | modules.append(nn.LeakyReLU(0.1, inplace=True)) 183 | return nn.Sequential(*modules) 184 | 185 | 186 | def predict_flow_3d(in_planes: int, out_planes: int): 187 | """Convenience function for conv3d, 3x3, no activation or batchnorm""" 188 | return conv3d(in_planes, out_planes, kernel_size=3, stride=1, bias=True, act=False, batchnorm=False) 189 | -------------------------------------------------------------------------------- /deepethogram/gui/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/gui/__init__.py -------------------------------------------------------------------------------- /deepethogram/gui/icons/noun_Home_1158721.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/gui/icons/noun_Home_1158721.png -------------------------------------------------------------------------------- /deepethogram/gui/icons/noun_Zoom In_744781.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/gui/icons/noun_Zoom In_744781.png -------------------------------------------------------------------------------- /deepethogram/gui/icons/noun_pause_159135.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/gui/icons/noun_pause_159135.png -------------------------------------------------------------------------------- /deepethogram/gui/icons/noun_play_1713293.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/gui/icons/noun_play_1713293.png -------------------------------------------------------------------------------- /deepethogram/gui/icons/noun_tap_145047.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/gui/icons/noun_tap_145047.png -------------------------------------------------------------------------------- /deepethogram/gui/menus_and_popups.py: -------------------------------------------------------------------------------- 1 | import pathlib 2 | import warnings 3 | 4 | from PySide2 import QtCore, QtWidgets 5 | 6 | 7 | def simple_popup_question(parent, message: str): 8 | # message = 'You have unsaved changes. Are you sure you want to quit?' 9 | reply = QtWidgets.QMessageBox.question( 10 | parent, "Message", message, QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No 11 | ) 12 | return reply == QtWidgets.QMessageBox.Yes 13 | 14 | 15 | # https://stackoverflow.com/questions/15682665/how-to-add-custom-button-to-a-qmessagebox-in-pyqt4 16 | 17 | 18 | def overwrite_or_not(parent): 19 | msgBox = QtWidgets.QMessageBox(parent) 20 | msgBox.setIcon(QtWidgets.QMessageBox.Question) 21 | msgBox.setText( 22 | "Do you want to overwrite your labels with these predictions, or only import the predictions" 23 | " for frames you haven" 24 | "t labeled?" 25 | ) 26 | overwrite = msgBox.addButton("Overwrite", QtWidgets.QMessageBox.YesRole) 27 | unlabeled = msgBox.addButton("Only import unlabeled", QtWidgets.QMessageBox.NoRole) 28 | msgBox.exec_() 29 | if msgBox.clickedButton() is overwrite: 30 | return True 31 | elif msgBox.clickedButton() is unlabeled: 32 | return False 33 | else: 34 | return 35 | 36 | 37 | class OverwriteOrNot(QtWidgets.QDialog): 38 | def __init__(self, parent=None): 39 | super().__init__(parent) 40 | 41 | msgBox = QtWidgets.QMessageBox() 42 | msgBox.setText( 43 | "Do you want to overwrite your labels with these predictions, or only import the predictions" 44 | " for frames you haven" 45 | "t labeled?" 46 | ) 47 | msgBox.addButton(QtWidgets.QPushButton("Overwrite"), QtWidgets.QMessageBox.YesRole) 48 | msgBox.addButton(QtWidgets.QPushButton("Only import unlabeled"), QtWidgets.QMessageBox.NoRole) 49 | msgBox.exec_() 50 | 51 | 52 | class CreateProject(QtWidgets.QDialog): 53 | def __init__(self, parent=None): 54 | super().__init__(parent) 55 | 56 | string = "Pick directory for your project. SHOULD BE ON YOUR FASTEST HARD DRIVE. VIDEOS WILL BE COPIED HERE" 57 | project_directory = QtWidgets.QFileDialog.getExistingDirectory(self, string) 58 | if len(project_directory) == 0: 59 | warnings.warn("Please choose a directory") 60 | return 61 | project_directory = str(pathlib.Path(project_directory).resolve()) 62 | self.project_directory = project_directory 63 | 64 | self.project_name_default = "Project Name" 65 | self.project_box = QtWidgets.QLineEdit(self.project_name_default) 66 | self.label_default_string = "Name of person labeling" 67 | self.labeler_box = QtWidgets.QLineEdit(self.label_default_string) 68 | # self.labeler_box. 69 | self.behavior_default_string = ( 70 | 'List of behaviors, e.g. "walk,scratch,itch". Do not include none,other,background,etc ' 71 | ) 72 | self.behaviors_box = QtWidgets.QLineEdit(self.behavior_default_string) 73 | # self.finish_button = QPushButton('Ok') 74 | # self.cancel_button = QPushButton('Cancel') 75 | button_box = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel) 76 | button_box.accepted.connect(self.accept) 77 | button_box.rejected.connect(self.reject) 78 | 79 | layout = QtWidgets.QFormLayout() 80 | layout.addRow(self.project_box) 81 | layout.addRow(self.labeler_box) 82 | layout.addRow(self.behaviors_box) 83 | # layout.addRow(hbox) 84 | layout.addWidget(button_box) 85 | self.setLayout(layout) 86 | # win = QtWidgets.QWidget() 87 | self.setWindowTitle("Create project") 88 | self.resize(800, 400) 89 | self.show() 90 | 91 | 92 | # modified from https://pythonspot.com/pyqt5-form-layout/ 93 | class ShouldRunInference(QtWidgets.QDialog): 94 | def __init__(self, record_keys: list, should_start_checked: list): 95 | super(ShouldRunInference, self).__init__() 96 | 97 | self.createFormGroupBox(record_keys, should_start_checked) 98 | 99 | buttonBox = QtWidgets.QDialogButtonBox(QtWidgets.QDialogButtonBox.Ok | QtWidgets.QDialogButtonBox.Cancel) 100 | buttonBox.accepted.connect(self.accept) 101 | buttonBox.rejected.connect(self.reject) 102 | 103 | mainLayout = QtWidgets.QVBoxLayout() 104 | mainLayout.addWidget(self.scrollArea) 105 | mainLayout.addWidget(buttonBox, alignment=QtCore.Qt.AlignLeft) 106 | self.setLayout(mainLayout) 107 | 108 | self.setWindowTitle("Select videos to run inference on") 109 | sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Preferred) 110 | sizePolicy.setHorizontalStretch(0) 111 | sizePolicy.setVerticalStretch(0) 112 | 113 | self.scrollArea.setSizePolicy(sizePolicy) 114 | # self.scrollArea.setGeometry(QtCore.QSize(400, 25)) 115 | self.scrollArea.setMaximumSize(QtCore.QSize(400, 16777215)) 116 | 117 | def createFormGroupBox(self, record_keys: list, should_start_checked: list): 118 | self.scrollArea = QtWidgets.QScrollArea(self) 119 | self.scrollArea.setWidgetResizable(True) 120 | self.buttons = [] 121 | # make a container widget 122 | self.scrollWidget = QtWidgets.QWidget() 123 | layout = QtWidgets.QGridLayout() 124 | for row, (record, check) in enumerate(zip(record_keys, should_start_checked)): 125 | button = QtWidgets.QCheckBox(self) 126 | button.setChecked(check) 127 | text = QtWidgets.QLabel(record + ":") 128 | layout.addWidget(text, row, 0) 129 | layout.addWidget(button, row, 1) 130 | self.buttons.append(button) 131 | # put this layout in the container widget 132 | self.scrollWidget.setLayout(layout) 133 | # set the scroll area's widget to be the container 134 | self.scrollArea.setWidget(self.scrollWidget) 135 | 136 | def get_outputs(self): 137 | if not hasattr(self, "buttons"): 138 | return None 139 | answers = [] 140 | for button in self.buttons: 141 | answers.append(button.isChecked()) 142 | return answers 143 | 144 | 145 | if __name__ == "__main__": 146 | app = QtWidgets.QApplication([]) 147 | num = 50 148 | form = ShouldRunInference( 149 | ["M134_20141203_v001", "M134_20141203_v002", "M134_20141203_v004"] * num, [True, True, False] * num 150 | ) 151 | ret = form.exec_() 152 | if ret: 153 | print(form.get_outputs()) 154 | # ret = app.exec_() 155 | # print(ret) 156 | -------------------------------------------------------------------------------- /deepethogram/schedulers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import math 3 | 4 | import torch 5 | from omegaconf import DictConfig 6 | from torch.optim.optimizer import Optimizer 7 | 8 | log = logging.getLogger(__name__) 9 | 10 | 11 | class _LRScheduler: 12 | def __init__(self, optimizer, last_epoch=-1): 13 | if not isinstance(optimizer, Optimizer): 14 | raise TypeError("{} is not an Optimizer".format(type(optimizer).__name__)) 15 | self.optimizer = optimizer 16 | if last_epoch == -1: 17 | for group in optimizer.param_groups: 18 | group.setdefault("initial_lr", group["lr"]) 19 | else: 20 | for i, group in enumerate(optimizer.param_groups): 21 | if "initial_lr" not in group: 22 | raise KeyError( 23 | "param 'initial_lr' is not specified in param_groups[{}] when resuming an optimizer".format(i) 24 | ) 25 | self.base_lrs = list(map(lambda group: group["initial_lr"], optimizer.param_groups)) 26 | self.step(last_epoch + 1) 27 | self.last_epoch = last_epoch 28 | 29 | def state_dict(self): 30 | """Returns the state of the scheduler as a :class:`dict`. 31 | It contains an entry for every variable in self.__dict__ which 32 | is not the optimizer. 33 | """ 34 | return {key: value for key, value in self.__dict__.items() if key != "optimizer"} 35 | 36 | def load_state_dict(self, state_dict): 37 | """Loads the schedulers state. 38 | Arguments: 39 | state_dict (dict): scheduler state. Should be an object returned 40 | from a call to :meth:`state_dict`. 41 | """ 42 | self.__dict__.update(state_dict) 43 | 44 | def get_lr(self): 45 | raise NotImplementedError 46 | 47 | def step(self, epoch=None): 48 | if epoch is None: 49 | epoch = self.last_epoch + 1 50 | self.last_epoch = epoch 51 | for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()): 52 | param_group["lr"] = lr 53 | 54 | 55 | # UNMERGED PULL REQUEST! NOT WRITTEN BY ME BUT SUPER USEFUL! 56 | # https://github.com/pytorch/pytorch/pull/11104 57 | class CosineAnnealingRestartsLR(_LRScheduler): 58 | r"""Set the learning rate of each parameter group using a cosine annealing 59 | schedule with warm restarts, where :math:`\eta_{max}` is set to the 60 | initial learning rate, :math:`T_{cur}` is the number of epochs since the 61 | last restart and :math:`T_i` is the number of epochs in :math:`i`-th run 62 | (after performing :math:`i` restarts). If the learning rate is set 63 | solely by this scheduler, the learning rate at each step becomes: 64 | .. math:: 65 | \eta_t = \eta_{min} + \frac{1}{2} \eta_{mult}^i (\eta_{max}-\eta_{min}) 66 | (1 + \cos(\frac{T_{cur}}{T_i - 1}\pi)) 67 | T_i = T T_{mult}^i 68 | Notice that because the schedule is defined recursively, the learning rate 69 | can be simultaneously modified outside this scheduler by other operators. 70 | When last_epoch=-1, sets initial lr as lr. 71 | It has been proposed in 72 | `SGDR: Stochastic Gradient Descent with Warm Restarts`_. Note that in the 73 | paper the :math:`i`-th run takes :math:`T_i + 1` epochs, while in this 74 | implementation it takes :math:`T_i` epochs only. This implementation 75 | also enables updating the range of learning rates by multiplicative factor 76 | :math:`\eta_{mult}` after each restart. 77 | Args: 78 | optimizer (Optimizer): Wrapped optimizer. 79 | T (int): Length of the initial run (in number of epochs). 80 | eta_min (float): Minimum learning rate. Default: 0. 81 | T_mult (float): Multiplicative factor adjusting number of epochs in 82 | the next run that is applied after each restart. Default: 2. 83 | eta_mult (float): Multiplicative factor of decay in the range of 84 | learning rates that is applied after each restart. Default: 1. 85 | last_epoch (int): The index of last epoch. Default: -1. 86 | .. _SGDR\: Stochastic Gradient Descent with Warm Restarts: 87 | https://arxiv.org/abs/1608.03983 88 | """ 89 | 90 | def __init__(self, optimizer, T, eta_min=0, T_mult=2.0, eta_mult=1.0, last_epoch=-1): 91 | self.T = T 92 | self.eta_min = eta_min 93 | self.eta_mult = eta_mult 94 | 95 | if T_mult < 1: 96 | raise ValueError("T_mult should be >= 1.0.") 97 | self.T_mult = T_mult 98 | 99 | super(CosineAnnealingRestartsLR, self).__init__(optimizer, last_epoch) 100 | 101 | def get_lr(self): 102 | if self.last_epoch == 0: 103 | return self.base_lrs 104 | 105 | if self.T_mult == 1: 106 | i_restarts = self.last_epoch // self.T 107 | last_restart = i_restarts * self.T 108 | else: 109 | # computation of the last restarting epoch is based on sum of geometric series: 110 | # last_restart = T * (1 + T_mult + T_mult ** 2 + ... + T_mult ** i_restarts) 111 | i_restarts = int(math.log(1 - self.last_epoch * (1 - self.T_mult) / self.T, self.T_mult)) 112 | last_restart = int(self.T * (1 - self.T_mult**i_restarts) / (1 - self.T_mult)) 113 | 114 | if self.last_epoch == last_restart: 115 | T_i1 = self.T * self.T_mult ** (i_restarts - 1) # T_{i-1} 116 | lr_update = self.eta_mult / self._decay(T_i1 - 1, T_i1) 117 | else: 118 | T_i = self.T * self.T_mult**i_restarts 119 | t = self.last_epoch - last_restart 120 | lr_update = self._decay(t, T_i) / self._decay(t - 1, T_i) 121 | 122 | return [lr_update * (group["lr"] - self.eta_min) + self.eta_min for group in self.optimizer.param_groups] 123 | 124 | @staticmethod 125 | def _decay(t, T): 126 | """Cosine decay for step t in run of length T, where 0 <= t < T.""" 127 | return 0.5 * (1 + math.cos(math.pi * t / T)) 128 | 129 | 130 | def initialize_scheduler(optimizer, cfg: DictConfig, mode: str = "max", reduction_factor: float = 0.1): 131 | """Makes a learning rate scheduler from an OmegaConf DictConfig 132 | 133 | Parameters 134 | ---------- 135 | optimizer: torch.optim.Optimizer 136 | one of ADAM, SGDM, etc 137 | cfg: DictConfig 138 | configuration generated by Hydra 139 | mode: str 140 | min: lr will reduce when metric stops DECREASING. useful for ERRORS, e.g. loss, SSIM 141 | max: lr will reduce when metric stops INCREASING. useful for PERFORMANCE, e.g. accuracy, F1 142 | reduction_factor: float 143 | Factor to multiply learning rate by each time the scheduler steps 144 | Useful values 145 | 0.1: reduces by a factor of 10 146 | 0.31622: with this value, takes two decrements to reduce learning rate by 1/10 147 | Returns 148 | ------- 149 | scheduler 150 | Learning rate scheduler 151 | """ 152 | if cfg.train.scheduler == "multistep": 153 | scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, milestones=cfg.train.milestones, gamma=0.5) 154 | # for convenience 155 | scheduler.name = "multistep" 156 | elif cfg.train.scheduler == "cosine": 157 | # todo: reconfigure this to use pytorch's new built-in cosine annealing 158 | scheduler = CosineAnnealingRestartsLR(optimizer, T=25, T_mult=1, eta_mult=0.5, eta_min=1e-7) 159 | scheduler.name = "cosine" 160 | elif cfg.train.scheduler == "plateau": 161 | scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( 162 | optimizer, 163 | mode=mode, 164 | factor=reduction_factor, 165 | patience=cfg.train.patience, 166 | verbose=True, 167 | min_lr=cfg.train.min_lr, 168 | ) 169 | scheduler.name = "plateau" 170 | else: 171 | scheduler = None 172 | return scheduler 173 | -------------------------------------------------------------------------------- /deepethogram/sequence/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/sequence/__init__.py -------------------------------------------------------------------------------- /deepethogram/sequence/__main__.py: -------------------------------------------------------------------------------- 1 | from .train import main 2 | 3 | main() 4 | -------------------------------------------------------------------------------- /deepethogram/sequence/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/sequence/models/__init__.py -------------------------------------------------------------------------------- /deepethogram/sequence/models/mlp.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | from torch import nn 4 | 5 | 6 | class MLP(nn.Module): 7 | """Multi-layer perceptron model. Baseline for sequence modeling""" 8 | 9 | def __init__( 10 | self, 11 | D: int, 12 | classes: int, 13 | dropout_p: float = 0.4, 14 | hidden_layers=( 15 | 256, 16 | 128, 17 | ), 18 | pos=None, 19 | neg=None, 20 | ): 21 | """Constructor 22 | 23 | Parameters 24 | ---------- 25 | D : int 26 | Number of neurons in our input layer 27 | classes : int 28 | Number of behaviors / neurons in our output layer 29 | dropout_p : float, optional 30 | P(dropout) for layers after input, by default 0.4 31 | hidden_layers : tuple, optional 32 | Number of neurons in each hidden layer, by default (256, 128,) 33 | pos : np.ndarray, optional 34 | Number of positive examples for each class, by default None 35 | neg : np.ndarray, optional 36 | Number of negative examples for each class, by default None 37 | """ 38 | super().__init__() 39 | 40 | neurons = [D] 41 | for hidden_layer in hidden_layers: 42 | neurons.append(hidden_layer) 43 | neurons.append(classes) 44 | 45 | layers = [] 46 | for i in range(len(neurons) - 1): 47 | print(i, neurons[i]) 48 | layers.append(nn.Linear(neurons[i], neurons[i + 1], bias=True)) 49 | if i < len(neurons) - 2: 50 | layers.append(nn.ReLU()) 51 | layers.append(nn.Dropout(p=dropout_p)) 52 | 53 | # https://www.tensorflow.org/tutorials/structured_data/imbalanced_data 54 | if pos is not None and neg is not None: 55 | with torch.no_grad(): 56 | bias = np.nan_to_num(np.log(pos / neg), neginf=0.0, posinf=1.0) 57 | bias = torch.nn.Parameter(torch.from_numpy(bias).float()) 58 | layers[-1].bias = bias 59 | 60 | self.model = nn.Sequential(*layers) 61 | 62 | def forward(self, features): 63 | return self.model(features) 64 | -------------------------------------------------------------------------------- /deepethogram/sequence/models/sequence.py: -------------------------------------------------------------------------------- 1 | from torch import nn 2 | 3 | 4 | def conv1d_same(in_channels, out_channels, kernel_size, stride=1, dilation=1, groups=1, bias=True): 5 | # if stride is two, output should be exactly half the size of input 6 | padding = kernel_size // 2 * dilation 7 | 8 | return nn.Conv1d( 9 | in_channels, 10 | out_channels, 11 | kernel_size, 12 | stride=stride, 13 | padding=padding, 14 | dilation=dilation, 15 | groups=groups, 16 | bias=bias, 17 | ) 18 | 19 | 20 | class Linear(nn.Module): 21 | def __init__(self, num_features, num_classes, kernel_size=1): 22 | super().__init__() 23 | self.conv1 = conv1d_same(num_features, num_classes, kernel_size=kernel_size, stride=1, bias=True) 24 | 25 | def forward(self, x): 26 | return self.conv1(x) 27 | 28 | 29 | class Conv_Nonlinear(nn.Module): 30 | def __init__(self, num_features, num_classes, batchnorm=True, hidden_size=64, dropout_p=0.0): 31 | super().__init__() 32 | 33 | bias = not batchnorm 34 | self.conv1 = conv1d_same(num_features, hidden_size, kernel_size=7, stride=1, bias=bias) 35 | self.bn1 = nn.BatchNorm1d(hidden_size) 36 | self.conv2 = conv1d_same(hidden_size, num_classes, kernel_size=3, stride=1, bias=bias) 37 | 38 | self.activation = nn.ReLU() 39 | 40 | layers = [] 41 | layers.append(self.conv1) 42 | if batchnorm: 43 | layers.append(self.bn1) 44 | layers.append(self.activation) 45 | if dropout_p > 0: 46 | layers.append(nn.Dropout(p=dropout_p)) 47 | layers.append(self.conv2) 48 | 49 | self.net = nn.Sequential(*layers) 50 | 51 | def forward(self, x): 52 | return self.net(x) 53 | 54 | 55 | class RNN(nn.Module): 56 | def __init__( 57 | self, 58 | num_features, 59 | num_classes, 60 | style="lstm", 61 | hidden_size=64, 62 | num_layers=1, 63 | dropout=0.0, 64 | output_dropout=0.0, 65 | bidirectional=False, 66 | ): 67 | super().__init__() 68 | 69 | assert style in ["rnn", "lstm", "gru"] 70 | if style == "rnn": 71 | func = nn.RNN 72 | elif style == "lstm": 73 | func = nn.LSTM 74 | elif style == "gru": 75 | func = nn.GRU 76 | 77 | self.rnn = func( 78 | num_features, 79 | hidden_size, 80 | num_layers=num_layers, 81 | bias=True, 82 | batch_first=True, 83 | dropout=dropout, 84 | bidirectional=bidirectional, 85 | ) 86 | self.dropout = nn.Dropout(output_dropout) 87 | size = hidden_size * 2 if bidirectional else hidden_size 88 | self.hidden_to_output = nn.Linear(size, num_classes) 89 | 90 | def forward(self, x): 91 | # change from N, C, L (for 1d conv) to N, L, C 92 | x = x.permute(0, 2, 1).contiguous() 93 | # hidden state is always 0 at input 94 | # hiddens is hidden units at each T, shape: N,L,C 95 | hiddens, _ = self.rnn(x) 96 | hiddens = self.dropout(hiddens) 97 | # outputs is N, L, C 98 | outputs = self.hidden_to_output(hiddens) 99 | # return outputs in shape N, C, L to be the same as conv1d 100 | return outputs.permute(0, 2, 1).contiguous() 101 | -------------------------------------------------------------------------------- /deepethogram/stoppers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Type 3 | 4 | from omegaconf import DictConfig 5 | 6 | log = logging.getLogger(__name__) 7 | 8 | 9 | class Stopper: 10 | """Base class for stopping training""" 11 | 12 | def __init__(self, name: str, start_epoch: int = 0, num_epochs: int = 1000): 13 | """constructor for stopper 14 | 15 | Parameters 16 | ---------- 17 | name: str 18 | name of the stopper. could be used by a training routine. E.g. if stopper.name == 'early': # do something 19 | start_epoch: int 20 | initializes epoch number. useful in case you want to pick up training from an exact state 21 | num_epochs: int 22 | number of epochs before training will automatically stop. used by subclasses 23 | """ 24 | self.name = name 25 | self.epoch_counter = start_epoch 26 | self.num_epochs = num_epochs 27 | 28 | def step(self, *args, **kwargs): 29 | """increment internal counter""" 30 | self.epoch_counter += 1 31 | 32 | def __call__(self, *args, **kwargs): 33 | return self.step(*args, **kwargs) 34 | 35 | 36 | class NumEpochsStopper(Stopper): 37 | def __init__(self, name: str = "num_epochs", start_epoch: int = 0, num_epochs: int = 1000): 38 | super().__init__(name, start_epoch, num_epochs) 39 | 40 | def step(self, *args, **kwargs): 41 | super().step() 42 | should_stop = False 43 | if self.epoch_counter > self.num_epochs: 44 | should_stop = True 45 | return should_stop 46 | 47 | 48 | class EarlyStopping(Stopper): 49 | """EarlyStopping handler can be used to stop the training if no improvement after a given number of events 50 | Args: 51 | patience (int): 52 | Number of events to wait if no improvement and then stop the training 53 | modified from here 54 | https://github.com/pytorch/ignite/blob/master/ignite/handlers/early_stopping.py 55 | """ 56 | 57 | def __init__( 58 | self, name="early", start_epoch=0, num_epochs=1000, patience=5, is_error=False, early_stopping_begins: int = 0 59 | ): 60 | super().__init__(name, start_epoch, num_epochs) 61 | if patience < 1: 62 | raise ValueError("Argument patience should be positive integer") 63 | self.patience = patience 64 | self.best_score = None 65 | self.is_error = is_error 66 | self.early_stopping_begins = early_stopping_begins 67 | 68 | def step(self, score): 69 | super().step() 70 | best = False 71 | should_stop = False 72 | # if the metric is actually an error, then we want to stop when validation error 73 | # stops SHRINKING rather than growing. Make it negative 74 | if self.is_error: 75 | score = -score 76 | if self.best_score is None: 77 | self.best_score = score 78 | best = True 79 | 80 | elif score < self.best_score: 81 | self.counter += 1 82 | if self.counter >= self.patience and self.epoch_counter >= self.early_stopping_begins: 83 | print("EarlyStopping: Stop training") 84 | should_stop = True 85 | else: 86 | self.best_score = score 87 | self.counter = 0 88 | best = True 89 | if self.epoch_counter > self.num_epochs: 90 | should_stop = True 91 | 92 | return best, should_stop 93 | 94 | 95 | class LearningRateStopper(Stopper): 96 | """Simple early stopper that stops when the learning rate drops below some threshold. 97 | Example usage: you reduce your learning rate when your validation loss stops improving. If the learning rate drops 98 | below some minimum value, automatically stop training. 99 | 100 | Example (pseudo-python): 101 | stopper = LearningRateStopper(5e-7) 102 | for i in range(num_epochs): 103 | train(model) 104 | if is_saturated(validation_loss): 105 | reduce_learning_rate(optimizer) 106 | if stopper(optimizer.learning_rate): 107 | break 108 | """ 109 | 110 | def __init__( 111 | self, 112 | name="learning_rate", 113 | minimum_learning_rate: float = 5e-7, 114 | start_epoch=0, 115 | num_epochs=1000, 116 | eps: float = 1e-8, 117 | ): 118 | super().__init__(name, start_epoch, num_epochs) 119 | """Constructor for LearningRateStopper. 120 | Args: 121 | minimum_learning_rate: if learning rate drops below this value, automatically stop training 122 | """ 123 | self.minimum_learning_rate = minimum_learning_rate 124 | self.eps = eps 125 | 126 | def step(self, lr: float) -> bool: 127 | """Computes if learning rate is below the set value 128 | Args: 129 | lr (float): learning rate 130 | Returns: 131 | should_stop: whether or not to stop training 132 | """ 133 | super().step() 134 | should_stop = False 135 | if lr < self.minimum_learning_rate + self.eps or self.epoch_counter >= self.num_epochs: 136 | print("Reached learning rate {}, stopping...".format(lr)) 137 | should_stop = True 138 | return should_stop 139 | 140 | 141 | def get_stopper(cfg: DictConfig) -> Type[Stopper]: 142 | """ 143 | 144 | Parameters 145 | ---------- 146 | cfg 147 | 148 | Returns 149 | ------- 150 | stopper: subclass of stoppers.Stopper 151 | 152 | """ 153 | # ASSUME WE'RE USING LOSS AS THE KEY METRIC, WHICH IS AN ERROR 154 | stopping_type = cfg.train.stopping_type 155 | log.debug("Using stopper type {}".format(stopping_type)) 156 | if stopping_type == "early": 157 | return EarlyStopping( 158 | start_epoch=0, 159 | num_epochs=cfg.train.num_epochs, 160 | patience=cfg.train.patience, 161 | is_error=True, 162 | early_stopping_begins=cfg.train.early_stopping_begins, 163 | ) 164 | elif stopping_type == "learning_rate": 165 | return LearningRateStopper( 166 | start_epoch=0, num_epochs=cfg.train.num_epochs, minimum_learning_rate=cfg.train.min_lr 167 | ) 168 | elif stopping_type == "num_epochs": 169 | return NumEpochsStopper("num_epochs", start_epoch=0, num_epochs=cfg.train.num_epochs) 170 | else: 171 | raise ValueError("invalid stopping name detected! {}".format(stopping_type)) 172 | -------------------------------------------------------------------------------- /deepethogram/tune/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/deepethogram/tune/__init__.py -------------------------------------------------------------------------------- /deepethogram/tune/feature_extractor.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from omegaconf import OmegaConf, DictConfig 5 | 6 | try: 7 | import ray 8 | from ray import tune 9 | from ray.tune import CLIReporter 10 | from ray.tune.schedulers import ASHAScheduler 11 | from ray.tune.suggest.hyperopt import HyperOptSearch 12 | except ImportError: 13 | print("To use the deepethogram.tune module, you must `pip install 'ray[tune]`") 14 | raise 15 | 16 | from deepethogram.configuration import make_config 17 | from deepethogram.feature_extractor.train import feature_extractor_train 18 | from deepethogram import projects 19 | from deepethogram.tune.utils import dict_to_dotlist, generate_tune_cfg 20 | 21 | 22 | def tune_feature_extractor(cfg: DictConfig): 23 | """Tunes feature extractor hyperparameters. 24 | 25 | Parameters 26 | ---------- 27 | cfg : DictConfig 28 | Configuration, with a 'tune' key 29 | 30 | Raises 31 | ------ 32 | NotImplementedError 33 | Checks that search method is either 'random' or 'hyperopt' 34 | """ 35 | scheduler = ASHAScheduler( 36 | max_t=cfg.train.num_epochs, # epochs 37 | grace_period=cfg.tune.grace_period, 38 | reduction_factor=2, 39 | ) 40 | 41 | reporter_dict = {} 42 | for key in cfg.tune.hparams.keys(): 43 | reporter_dict[key] = cfg.tune.hparams[key].short 44 | # reporter_dict = {key: value for key, value in zip(cfg.tune.hparams.keys(), )} 45 | reporter = CLIReporter(parameter_columns=reporter_dict) 46 | 47 | # this converts what's in our cfg to a dictionary containing the search space of our hyperparameters 48 | tune_experiment_cfg = generate_tune_cfg(cfg) 49 | 50 | if cfg.tune.search == "hyperopt": 51 | # https://docs.ray.io/en/master/tune/api_docs/suggestion.html#tune-hyperopt 52 | current_best = {} 53 | for key, value in cfg.tune.hparams.items(): 54 | current_best[key] = value.current_best 55 | # hyperopt wants this to be a list of dicts 56 | current_best = [current_best] 57 | search = HyperOptSearch(metric=cfg.tune.key_metric, mode="max", points_to_evaluate=current_best) 58 | elif cfg.tune.search == "random": 59 | search = None 60 | else: 61 | raise NotImplementedError 62 | 63 | print("Running hyperparamter tuning with configuration: ") 64 | print(OmegaConf.to_yaml(cfg)) 65 | 66 | analysis = tune.run( 67 | tune.with_parameters( 68 | run_ray_experiment, 69 | cfg=cfg, 70 | ), 71 | resources_per_trial=OmegaConf.to_container(cfg.tune.resources_per_trial), 72 | metric=cfg.tune.key_metric, 73 | mode="max", 74 | config=tune_experiment_cfg, 75 | num_samples=cfg.tune.num_trials, # how many experiments to run 76 | scheduler=scheduler, 77 | progress_reporter=reporter, 78 | name=cfg.tune.name, 79 | local_dir=cfg.project.model_path, 80 | search_alg=search, 81 | ) 82 | print("Best hyperparameters found were: ", analysis.best_config) 83 | analysis.results_df.to_csv(os.path.join(cfg.project.model_path, "ray_results.csv")) 84 | 85 | 86 | def run_ray_experiment(ray_cfg, cfg): 87 | """trains a model based on the base config and the one generated for this experiment 88 | 89 | Parameters 90 | ---------- 91 | ray_cfg : DictConfig 92 | hparam study configuration 93 | cfg : DictConfig 94 | base configuration with all non-tuned hyperparameters and information 95 | """ 96 | ray_cfg = OmegaConf.from_dotlist(dict_to_dotlist(ray_cfg)) 97 | 98 | cfg = OmegaConf.merge(cfg, ray_cfg) 99 | 100 | if cfg.notes is None: 101 | cfg.notes = f"{cfg.tune.name}_{tune.get_trial_id()}" 102 | else: 103 | cfg.notes += f"{cfg.tune.name}_{tune.get_trial_id()}" 104 | feature_extractor_train(cfg) 105 | 106 | 107 | if __name__ == "__main__": 108 | # USAGE 109 | # to run locally, type `ray start --head --port 6385`, then run this script 110 | 111 | ray.init(address="auto") # num_gpus=1 112 | 113 | config_list = [ 114 | "config", 115 | "augs", 116 | "model/flow_generator", 117 | "train", 118 | "model/feature_extractor", 119 | "tune/tune", 120 | "tune/feature_extractor", 121 | ] 122 | run_type = "train" 123 | model = "feature_extractor" 124 | 125 | project_path = projects.get_project_path_from_cl(sys.argv) 126 | cfg = make_config( 127 | project_path=project_path, 128 | config_list=config_list, 129 | run_type=run_type, 130 | model=model, 131 | use_command_line=True, 132 | debug=True, 133 | ) 134 | cfg = projects.convert_config_paths_to_absolute(cfg) 135 | 136 | if "preset" in cfg.keys(): 137 | cfg.tune.name += "_{}".format(cfg.preset) 138 | if "debug" in cfg.keys(): 139 | cfg.tune.name += "_debug" 140 | 141 | tune_feature_extractor(cfg) 142 | -------------------------------------------------------------------------------- /deepethogram/tune/sequence.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from omegaconf import OmegaConf, DictConfig 5 | 6 | try: 7 | import ray 8 | from ray import tune 9 | from ray.tune import CLIReporter 10 | from ray.tune.schedulers import ASHAScheduler 11 | from ray.tune.suggest.hyperopt import HyperOptSearch 12 | except ImportError: 13 | print("To use the deepethogram.tune module, you must `pip install 'ray[tune]`") 14 | raise 15 | 16 | from deepethogram.configuration import make_config 17 | from deepethogram import sequence_train 18 | from deepethogram import projects 19 | from deepethogram.tune.utils import dict_to_dotlist, generate_tune_cfg 20 | 21 | 22 | def tune_sequence(cfg: DictConfig): 23 | """Tunes sequence model hyperparameters 24 | 25 | Parameters 26 | ---------- 27 | cfg : DictConfig 28 | configuration 29 | 30 | Raises 31 | ------ 32 | NotImplementedError 33 | Checks that search method is either 'random' or 'hyperopt' 34 | """ 35 | 36 | scheduler = ASHAScheduler( 37 | max_t=cfg.train.num_epochs, # epochs 38 | grace_period=cfg.tune.grace_period, 39 | reduction_factor=2, 40 | ) 41 | 42 | reporter_dict = {} 43 | for key in cfg.tune.hparams.keys(): 44 | reporter_dict[key] = cfg.tune.hparams[key].short 45 | # reporter_dict = {key: value for key, value in zip(cfg.tune.hparams.keys(), )} 46 | reporter = CLIReporter(parameter_columns=reporter_dict) 47 | 48 | # this converts what's in our cfg to a dictionary containing the search space of our hyperparameters 49 | tune_experiment_cfg = generate_tune_cfg(cfg) 50 | 51 | if cfg.tune.search == "hyperopt": 52 | # https://docs.ray.io/en/master/tune/api_docs/suggestion.html#tune-hyperopt 53 | current_best = {} 54 | for key, value in cfg.tune.hparams.items(): 55 | current_best[key] = value.current_best 56 | # hyperopt wants this to be a list of dicts 57 | current_best = [current_best] 58 | search = HyperOptSearch(metric=cfg.tune.key_metric, mode="max", points_to_evaluate=current_best) 59 | elif cfg.tune.search == "random": 60 | search = None 61 | else: 62 | raise NotImplementedError 63 | 64 | print("Running hyperparamter tuning with configuration: ") 65 | print(OmegaConf.to_yaml(cfg)) 66 | 67 | analysis = tune.run( 68 | tune.with_parameters( 69 | run_ray_experiment, 70 | cfg=cfg, 71 | ), 72 | resources_per_trial=OmegaConf.to_container(cfg.tune.resources_per_trial), 73 | metric=cfg.tune.key_metric, 74 | mode="max", 75 | config=tune_experiment_cfg, 76 | num_samples=cfg.tune.num_trials, # how many experiments to run 77 | scheduler=scheduler, 78 | progress_reporter=reporter, 79 | name=cfg.tune.name, 80 | local_dir=cfg.project.model_path, 81 | search_alg=search, 82 | ) 83 | print("Best hyperparameters found were: ", analysis.best_config) 84 | analysis.results_df.to_csv(os.path.join(cfg.project.model_path, "ray_results.csv")) 85 | 86 | 87 | def run_ray_experiment(ray_cfg, cfg): 88 | # cfg = make_feature_extractor_train_cfg(project_path, use_command_line=False, preset='deg_f') 89 | # tune_cfg = load_config_by_name('tune') 90 | 91 | ray_cfg = OmegaConf.from_dotlist(dict_to_dotlist(ray_cfg)) 92 | 93 | cfg = OmegaConf.merge(cfg, ray_cfg) 94 | # cfg.tune.use = True 95 | 96 | # cfg.flow_generator.weights = 'latest' 97 | # cfg.feature_extractor.weights = '/media/jim/DATA_SSD/niv_revision_deepethogram/models/pretrained_models/200415_125824_hidden_two_stream_kinetics_degf/checkpoint.pt' 98 | # cfg.compute.batch_size = 64 99 | # cfg.train.steps_per_epoch.train = 20 100 | # cfg.train.steps_per_epoch.val = 20 101 | if cfg.notes is None: 102 | cfg.notes = f"{cfg.tune.name}_{tune.get_trial_id()}" 103 | else: 104 | cfg.notes += f"{cfg.tune.name}_{tune.get_trial_id()}" 105 | sequence_train(cfg) 106 | 107 | 108 | if __name__ == "__main__": 109 | # USAGE 110 | # to run locally, type `ray start --head --port 6385`, then run this script 111 | 112 | ray.init(address="auto") # num_gpus=1 113 | 114 | config_list = ["config", "model/feature_extractor", "train", "model/sequence", "tune/tune", "tune/sequence"] 115 | run_type = "train" 116 | model = "sequence" 117 | 118 | project_path = projects.get_project_path_from_cl(sys.argv) 119 | cfg = make_config( 120 | project_path=project_path, 121 | config_list=config_list, 122 | run_type=run_type, 123 | model=model, 124 | use_command_line=True, 125 | debug=False, 126 | ) 127 | cfg = projects.convert_config_paths_to_absolute(cfg) 128 | 129 | cfg.tune.name = "tune_sequence_2" 130 | 131 | if "debug" in cfg.keys(): 132 | cfg.tune.name += "_debug" 133 | 134 | tune_sequence(cfg) 135 | -------------------------------------------------------------------------------- /deepethogram/tune/utils.py: -------------------------------------------------------------------------------- 1 | from omegaconf import OmegaConf 2 | 3 | try: 4 | import ray # noqa: F401 5 | from ray import tune # noqa: F401 6 | except ImportError: 7 | print("To use the deepethogram.tune module, you must `pip install 'ray[tune]`") 8 | raise 9 | 10 | 11 | # code modified from official ray docs: 12 | # https://docs.ray.io/en/master/tune/tutorials/tune-pytorch-lightning.html 13 | def dict_to_dotlist(cfg_dict): 14 | dotlist = [f"{key}={value}" for key, value in cfg_dict.items()] 15 | return dotlist 16 | 17 | 18 | def generate_tune_cfg(cfg): 19 | """from a configuration, e.g. conf/tune/feature_extractor.yaml, generate a search space for specific hyperparameters""" 20 | 21 | def get_space(hparam_dict): 22 | if hparam_dict.space == "uniform": 23 | return tune.uniform(hparam_dict.min, hparam_dict.max) 24 | elif hparam_dict.space == "log": 25 | return tune.loguniform(hparam_dict.min, hparam_dict.max) 26 | elif hparam_dict.space == "choice": 27 | return tune.choice(OmegaConf.to_container(hparam_dict.choices)) 28 | else: 29 | raise NotImplementedError 30 | 31 | tune_cfg = {} 32 | for key, value in cfg.tune.hparams.items(): 33 | tune_cfg[key] = get_space(value) 34 | 35 | return tune_cfg 36 | -------------------------------------------------------------------------------- /docker/Dockerfile-full: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 nvidia/cuda:11.5.2-cudnn8-devel-ubuntu20.04 2 | 3 | # modified from here 4 | # https://github.com/anibali/docker-pytorch/blob/master/dockerfiles/1.10.0-cuda11.3-ubuntu20.04/Dockerfile 5 | # Install some basic utilities 6 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ 7 | curl ca-certificates sudo git bzip2 libx11-6 \ 8 | ffmpeg libsm6 libxext6 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 libxcb-render-util0 libxcb-xinerama0 \ 9 | libxcb-xkb-dev libxkbcommon-x11-0 libpulse-mainloop-glib0 ubuntu-restricted-extras libqt5multimedia5-plugins vlc \ 10 | libkrb5-3 libgssapi-krb5-2 libkrb5support0 \ 11 | && rm -rf /var/lib/apt/lists/* 12 | 13 | # don't ask for location etc user input when building 14 | # this is for opencv, apparently 15 | RUN apt-get update && apt-get install -y 16 | 17 | # Create a working directory and data directory 18 | RUN mkdir /app 19 | WORKDIR /app 20 | 21 | # Set up the Conda environment 22 | ENV CONDA_AUTO_UPDATE_CONDA=false \ 23 | PATH=/opt/miniconda/bin:$PATH 24 | 25 | # install miniconda 26 | RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh \ 27 | && chmod +x ~/miniconda.sh \ 28 | && ~/miniconda.sh -b -p /opt/miniconda \ 29 | && rm ~/miniconda.sh \ 30 | && conda update conda 31 | 32 | # install 33 | RUN conda install python=3.7 -y 34 | RUN pip install setuptools --upgrade && pip install --upgrade "pip<24.0" 35 | RUN pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 -f https://download.pytorch.org/whl/torch_stable.html 36 | 37 | ADD . /app/deepethogram 38 | WORKDIR /app/deepethogram 39 | ENV DEG_VERSION='full' 40 | RUN pip install -e . 41 | -------------------------------------------------------------------------------- /docker/Dockerfile-gui: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 ubuntu:20.04 2 | 3 | # modified from here 4 | # https://github.com/anibali/docker-pytorch/blob/master/dockerfiles/1.10.0-cuda11.3-ubuntu20.04/Dockerfile 5 | # Install some basic utilities 6 | RUN apt-get update && apt-get install -y \ 7 | curl \ 8 | ca-certificates \ 9 | sudo \ 10 | git \ 11 | bzip2 \ 12 | libx11-6 \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | # don't ask for location etc user input when building 16 | # this is for opencv, apparently 17 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ffmpeg libsm6 libxext6 libxcb-icccm4 \ 18 | libxcb-image0 libxcb-keysyms1 libxcb-render-util0 libxcb-xinerama0 libxcb-xkb-dev libxkbcommon-x11-0 \ 19 | libpulse-mainloop-glib0 ubuntu-restricted-extras libqt5multimedia5-plugins vlc 20 | 21 | # Create a working directory and data directory 22 | RUN mkdir /app 23 | WORKDIR /app 24 | 25 | # Set up the Conda environment 26 | ENV CONDA_AUTO_UPDATE_CONDA=false \ 27 | PATH=/opt/miniconda/bin:$PATH 28 | 29 | # install miniconda 30 | RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh \ 31 | && chmod +x ~/miniconda.sh \ 32 | && ~/miniconda.sh -b -p /opt/miniconda \ 33 | && rm ~/miniconda.sh \ 34 | && conda update conda 35 | 36 | # install 37 | RUN conda install python=3.7 -y 38 | RUN pip install setuptools --upgrade && pip install --upgrade pip 39 | 40 | # TODO: REFACTOR CODE SO IT'S POSSIBLE TO RUN GUI WITHOUT TORCH 41 | RUN conda install pytorch cpuonly -c pytorch 42 | 43 | # # needed for pandas for some reason 44 | ADD . /app/deepethogram 45 | WORKDIR /app/deepethogram 46 | ENV DEG_VERSION='gui' 47 | RUN pip install -e . 48 | -------------------------------------------------------------------------------- /docker/Dockerfile-headless: -------------------------------------------------------------------------------- 1 | FROM --platform=linux/amd64 nvidia/cuda:11.5.2-cudnn8-devel-ubuntu20.04 2 | 3 | # modified from here 4 | # https://github.com/anibali/docker-pytorch/blob/master/dockerfiles/1.10.0-cuda11.3-ubuntu20.04/Dockerfile 5 | # Install some basic utilities 6 | RUN apt-get update && apt-get install -y \ 7 | curl \ 8 | ca-certificates \ 9 | sudo \ 10 | git \ 11 | bzip2 \ 12 | libx11-6 \ 13 | && rm -rf /var/lib/apt/lists/* 14 | 15 | # don't ask for location etc user input when building 16 | # this is for opencv, apparently 17 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ffmpeg 18 | 19 | # Create a working directory and data directory 20 | RUN mkdir /app 21 | WORKDIR /app 22 | 23 | # Set up the Conda environment 24 | ENV CONDA_AUTO_UPDATE_CONDA=false \ 25 | PATH=/opt/miniconda/bin:$PATH 26 | 27 | # install miniconda 28 | RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh \ 29 | && chmod +x ~/miniconda.sh \ 30 | && ~/miniconda.sh -b -p /opt/miniconda \ 31 | && rm ~/miniconda.sh \ 32 | && conda update conda 33 | 34 | # install 35 | RUN conda install python=3.7 -y 36 | RUN pip install setuptools --upgrade && pip install --upgrade pip 37 | RUN conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch 38 | 39 | # # needed for pandas for some reason 40 | ADD . /app/deepethogram 41 | WORKDIR /app/deepethogram 42 | ENV DEG_VERSION='headless' 43 | RUN pip install -e . 44 | -------------------------------------------------------------------------------- /docs/beta.md: -------------------------------------------------------------------------------- 1 | # DeepEthogram Beta 2 | 3 | DeepEthogram is now in Beta, version 0.1! There are major changes to the codebase and to model training and inference. 4 | Model performance, measured by F1, accuracy, etc. should be higher in version 0.1. Model training times and inference 5 | times should be dramatically reduced. 6 | 7 | **Important note: your old project files, models, and (most importantly) human labels will all still work!** However, 8 | I do recommend training new feature extractor and sequence models, as performance should improve somewhat. This will 9 | be the last major refactor of DeepEthogram (model improvements and new features will still come out), however I will 10 | not be majorly changing dependencies after this. Future upgrades will be easier (e.g. `pip install --upgrade deepethogram`). 11 | 12 | ## Summary of changes 13 | * Basic training pipeline re-implemented with PyTorch Lightning. This gives us some great features, such as tensorboard 14 | logging, automatic batch sizing, and Ray Tune integration. 15 | * Image augmentations moved to GPU with Kornia. [see Performance guide for details](performance.md) 16 | * New, parallelized inference 17 | * Hyperparameter tuning 18 | * New defaults for all models to improve performance 19 | * improved unit tests 20 | * new `configuration` module to make generation of configurations (e.g. `cfg`) more understandable and easy 21 | * Refactor of the whole data module 22 | * (alpha): support for importing DeepLabCut keypoints to train sequence models 23 | * new performance documentation, among others 24 | 25 | ## Migration guide 26 | 27 | There are some new dependency changes; making sure that install works correctly is the hardest part about migration. 28 | 29 | * activate your conda environment, e.g. `conda activate deg` 30 | * uninstall hydra: `pip uninstall hydra-core` 31 | * uninstall pytorch to upgrade: `conda uninstall pytorch` 32 | * uninstall pyside2 via conda: `conda uninstall pyside2` 33 | * upgrade pytorch. note that cudatoolkit version is not important `conda install pytorch torchvision torchaudio cudatoolkit=10.2 -c pytorch` 34 | * Uninstall DeepEthogram: `pip uninstall deepethogram` 35 | * Install the new version: `pip install deepethogram` 36 | 37 | ### upgrade issues 38 | * `AttributeError: type object 'OmegaConf' has no attribute 'to_yaml'` 39 | * this indicates that OmegaConf did not successfully upgrade to version 2.0+, and also likely that there was a problem 40 | with your upgrade. please follow the above steps. If you're sure that everything else installed correctly, you can run 41 | `pip install --upgrade omegaconf` 42 | * `error: torch 1.5.1 is installed but torch>=1.6.0 is required by {'kornia'}` 43 | * this indicates that your PyTorch version is too low. Please uninstall and reinstall PyTorch. 44 | * `ValueError: Hydra installation found. Please run pip uninstall hydra-core` 45 | * do as the error message says: run `pip uninstall hydra-core` 46 | * if you've already done this, you might have to manually delete hydra files. Mine were at 47 | `'C:\\ProgramData\\Anaconda3\\lib\\site-packages\\hydra_core-0.11.3-py3.7.egg\\hydra'`. Please delete the `hydra_core` folder. 48 | -------------------------------------------------------------------------------- /docs/code_examples.md: -------------------------------------------------------------------------------- 1 | # Code examples 2 | 3 | TODO 4 | -------------------------------------------------------------------------------- /docs/docker.md: -------------------------------------------------------------------------------- 1 | # Using deepethogram in Docker 2 | Install Docker: https://docs.docker.com/get-docker/ 3 | 4 | Install nvidia-docker: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker 5 | 6 | ## running the gui on Linux with training support 7 | In a terminal, run `xhost +local:docker`. You'll need to do this every time you restart. 8 | 9 | To run, type this command: `docker run --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw --shm-size 16G -v /media:/media -it jbohnslav/deepethogram:full python -m deepethogram` 10 | 11 | Explanation 12 | * `--gpus all`: required to have GPUs accessible in the container 13 | * `-e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw`: so that the container has access to your screen 14 | * `--shm-size 16G`: required for pytorch to be able to use multiprocessing. we might be able to lower this amount 15 | * `-v /media:/media`: use this to mount your data hard drive inside the container. Replace with whatever works for your system. For example, if your data lives on a drive called `/mnt/data/DATA`, replace this with `-v /mnt:/mnt` 16 | * `it deepethogram:dev python -m deepethogram`: run the deepethogram GUI in interactive mode 17 | 18 | ## Running the GUI without training support (no pytorch, etc.) 19 | Again, change `/media` to your hard drive with your training data 20 | 21 | `docker run -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw -v /media:/media -it jbohnslav/deepethogram:gui python -m deepethogram` 22 | 23 | ## Running the CLI without GUI support 24 | `docker run --gpus all -v /media:/media -it jbohnslav/deepethogram:headless pytest tests/` 25 | 26 | ## making sure all 3 images work 27 | #### full 28 | * GUI: `docker run --gpus all -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw -v /media:/media -it deepethogram:full python -m deepethogram` 29 | * tests: `docker run --gpus all -it deepethogram:full pytest tests/` 30 | 31 | #### gui only 32 | * GUI: `docker run -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw -v /media:/media -it deepethogram:gui python -m deepethogram` 33 | 34 | #### CLI only 35 | * tests: `docker run --gpus all -it deepethogram:full pytest tests/` 36 | 37 | # building it yourself 38 | To build the container with both GUI and model training support: 39 | * `cd` to your `deepethogram` directory 40 | * `docker build -t deepethogram:full -f docker/Dockerfile-full .` 41 | -------------------------------------------------------------------------------- /docs/file_structure.md: -------------------------------------------------------------------------------- 1 | # Expected filepaths 2 | 3 | To train the DeepEthogram models, we need to be able to find a bunch of files (below). If you use the GUI, this directory 4 | structure will be created for you. 5 | * models: a list of recent model runs of various types, along with their weights, and their performance 6 | * data 7 | * for each video, we need the video file itself 8 | * labels 9 | * labels are encoded very simply: each column is a behavior, and each row is frame number. Each element of this matrix 10 | is either {0: no behavior, 1: behavior is present: -1: this frame has not yet been labeled} 11 | * a file for model outputs 12 | * for the feature extractor, we save the 512-dimensional image features and 512-dimensional flow features to this file 13 | * we also save probabilities and predictions (thresholded probabilities) to this file, as well as the thresholds used 14 | * video statistics: following normal convention in machine learning, we z-score our input data. For images, this is done independently 15 | for the read, green, and blue channels. We z-score each video as they are added to a project, and save the channel 16 | means and std deviations to a file 17 | * project configuration file: holds project-specific information, like behavior names and variables to override. For defaults, see [the default configuration file](../deepethogram/conf/project/project_config.yaml) 18 | 19 | Therefore, the data loading scripts expect the following consistent folder structure. Note: if you write your own 20 | dataloaders, you can use whatever file structure you want. 21 | 22 | ```bash 23 | project_directory 24 | ├── project_config.yaml: See above 25 | ├── DATA 26 | | ├── experiment_0 27 | | | ├── experiment_0.avi (or .mp4, etc): the video file 28 | | | ├── experiment_0_labels.csv: a label file (see above for formatting) 29 | | | ├── experiment_0_outputs.h5: an HDF5 file with extracted features (see above) 30 | | | ├── stats.yaml: channel-wise mean and standard deviation 31 | | | ├── record.yaml: a yaml file containing the names of all the above files (so other scripts can easily find them, especially if you have multiple video formats in one directory) 32 | | ├── experiment_1 33 | | | ├── experiment_1.avi 34 | | | ├── etc... 35 | ├── models 36 | | ├── 200504_flow_generator_None 37 | | | ├── checkpoint.pt: the model weights for pytorch 38 | | | ├── hydra.yaml: the logs for how hydra built the configuration file 39 | | | ├── overrides.yaml: the overrides to the default configuration that the user specified from the command line 40 | | | ├── config.yaml: the configuration used to train this model 41 | | | ├── split.yaml: the train, validation, test split for this training run 42 | | | ├── model_name_definition.pt: the PyTorch model definition 43 | | | ├── train.log: log information for this training run 44 | | | ├── model_type_metrics.h5: saved metrics for this model. e.g. f1, accuracy, SSIM, depending 45 | | ├── 200504_feature_extractor_None 46 | | | ├── checkpoint.pt: etc... 47 | ``` 48 | -------------------------------------------------------------------------------- /docs/getting_started.md: -------------------------------------------------------------------------------- 1 | # Getting started 2 | 3 | The goal of DeepEthogram is as follows: 4 | * You have videos (inputs) 5 | * You also have a set of behaviors that you've defined based on your research project / interests 6 | * You want to know, for every frame, which behaviors are present on that frame 7 | * DeepEthogram will learn a mapping , where 8 | p(k,t) is the probability of behavior k on frame t. X is the video at time t. 9 | * These probabilities are thresholded to give a binary prediction. The binary matrix of each behavior at each timepoint is what we call an *ethogram*. 10 | * To train this model, you must label videos manually for each behavior at each timepoint. 11 | * After training, you can run inference on a new video. Most of the frames should be labeled correctly. You can then quickly edit the errors. 12 | 13 | This is schematized below: 14 | ![DeepEthogram figure 1](images/ethogram_schematic.png) 15 | 16 | In the above figure, the image sequence are our inputs, and the ethogram is depicted below. 17 | 18 | ## Installation 19 | See [the installation documentation](installation.md). 20 | 21 | ## Making a project 22 | The most important decision to make when starting a DeepEthogram project is which behaviors to include. Each frame must have a label 23 | for each behavior. While DeepEthogram contains code for adding and removing behaviors, all previous models must need to be 24 | retrained when a behavior has been added or removed. After all, if you used to have 5 behaviors and now you have 6, 25 | the final layer of the neural network models will all have the wrong shape. Furthermore, previously labeled videos must be 26 | updated with new behaviors before they can be used for training. 27 | 28 | Open your terminal window, activate your `conda` environment, and open the GUI by typing `deepethogram`. For more information, see 29 | [using the GUI](using_gui.md). 30 | 31 | Go to `file -> new project`. Select a location for the new project to be created. It is *essential* that the project 32 | be created on a Solid State Drive (or NVMe drive), because during training DeepEthogram will load hundreds of images per second. 33 | 34 | After selecting a location, a screen will appear with three fields: 35 | * `project name`: the name of your project. Examples might be: `mouse_reach`, `grooming`, `itch_mutation_screen`, etc. 36 | * `name of person labeling`: your name. Currently unused, in the future it could be used to compare labels between humans. 37 | * `list of behaviors`: the list of behaviors you want to label. Think carefully! (see above). Separate the behaviors with commas. 38 | Do not include `none`, `other`, `background`, `etc`, or `misc` or anything like that. 39 | 40 | Press OK. 41 | 42 | A directory will be created in the location you specified, with the name `projectname_deepethogram`. It will initialize the 43 | file structure needed to run deepethogram. See [the docs](file_structure.md) for details. 44 | 45 | ## Edit the default configuration file 46 | For more information see [the config file docs](using_config_files.md). 47 | 48 | ## Add videos 49 | When you add videos to DeepEthogram, we will **copy them to the deepethogram project directory** (not move or use a symlink). We highly 50 | recommend starting with at least 3 videos, so you have more than 1 to train, and 1 for validation (roughly, videos are assigned to splits 51 | probabilistically). When you add videos, DeepEthogram will automatically compute mean and standard deviation statistics 52 | and save them to disk. This might take a few moments (or minutes for very long videos). This is required for model training 53 | and inference. 54 | 55 | ## Download models 56 | Rather than start from scratch, we will start with model weights pretrained on the Kinetics700 dataset. Go to 57 | To download the pretrained weights, please use [this Google Drive link](https://drive.google.com/file/d/1ntIZVbOG1UAiFVlsAAuKEBEVCVevyets/view?usp=sharing). 58 | Unzip the files in your `project/models` directory. The path should be 59 | `your_project/models/pretrained/{models 1:6}`. 60 | 61 | ## Start training the flow generator 62 | These models (see paper) estimate local motion from video frames. They are trained in a self-supervised manner, so they 63 | require no labels. First, select the pretrained model architecture you chose with the drop down menu in the flow_generator box. 64 | Then simply click the `train` button in the `flow_generator` section on the left 65 | side of the GUI. The model will use the architecture you've put in your configuration file, or `TinyMotionNet` by default. 66 | You can continue to label while this model is training. 67 | 68 | ## Label a few videos 69 | To see how to import videos to DeepEthogram and label them, please see [using the GUI docs](using_gui.md). 70 | 71 | ## Train models! 72 | The workflow is depicted below. For more information, read the paper. 73 | 74 | ![DeepEthogram workflow](images/workflow.png) 75 | 76 | 1. Train the feature extractor. To do this, use the drop down menu to select the pretrained weights for the architecture 77 | you've specified in your configuration file. Then click `train`. This will take a few hours at least, perhaps overnight 78 | the first time. 79 | 2. Run inference using the pretrained feature extractor. The weights file from the model you've just trained should be 80 | pre-selected in the drop-down menu. Click `infer`. Select the videos you want to run inference on. This will go frame-by-frame 81 | through all your videos and save the 512-d spatial features and 512-d motion features to disk. These are the inputs to our 82 | sequence model. 83 | 3. Train the sequence model. In the sequence box, simply click `train`. 84 | 4. Run inference using the sequence model, as above. 85 | 86 | Now, you have a trained flow, feature extractor, and sequence model. 87 | 88 | ## Add more videos 89 | Using the GUI, add videos to your project. After you add the videos, as depicted in the workflow figure above, 90 | extract features to disk. This will take about 30-60 frames per second, depending on the model and your video resolution. 91 | Then run inference using the pretrained sequence model (should be instantaneous). 92 | 93 | Now, for your newly added videos, you have probabilities and predictions for every video frame. Use the `import predictions as labels` 94 | button on the GUI, and it will move the predictions to your labeling box. There, you can quickly edit them for errors. 95 | 96 | ## Re-train models 97 | Now that you have even more videos and labels, we want to incorporate these into our models. Use the "train models" workflow above 98 | to re-train your previous models with the new data. 99 | 100 | ## Continued usage 101 | Continue cyling through the above workflow 102 | * add videos 103 | * run inference 104 | * edit model errors 105 | * retrain models 106 | -------------------------------------------------------------------------------- /docs/images/deepethogram_schematic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/docs/images/deepethogram_schematic.png -------------------------------------------------------------------------------- /docs/images/ethogram_schematic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/docs/images/ethogram_schematic.png -------------------------------------------------------------------------------- /docs/images/gui_annotated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/docs/images/gui_annotated.png -------------------------------------------------------------------------------- /docs/images/label_format.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/docs/images/label_format.png -------------------------------------------------------------------------------- /docs/images/project_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/docs/images/project_config.png -------------------------------------------------------------------------------- /docs/images/workflow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/docs/images/workflow.png -------------------------------------------------------------------------------- /docs/installation.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | 3 | ## Brief version 4 | * Install Anaconda 5 | * Create a new anaconda environment: `conda create --name deg python=3.7` 6 | * Activate your environment: `conda activate deg` 7 | * Install PySide2: `conda install -c conda-forge pyside2==5.13.2` 8 | * Install PyTorch: [Use this link for official instructions.](https://pytorch.org/) 9 | * `pip install deepethogram`. 10 | 11 | ## Installing from source 12 | * `git clone https://github.com/jbohnslav/deepethogram.git` 13 | * `cd deepethogram` 14 | * `conda env create -f environment.yml` 15 | * Be prepared to wait a long time!! On mechanical hard drives, this may take 5-10 minutes (or more). Interrupting here will cause installation to fail. 16 | * `conda activate deg` 17 | * `python setup.py develop` 18 | 19 | ### Installing Anaconda 20 | For instructions on installing anaconda, 21 | please [use this link](https://www.anaconda.com/distribution/). This will install Python, some basic dependencies, and 22 | install the Anaconda package manager. This will ensure that if you use some other project that (say) requires Python 2, 23 | you can have both installed on your machine without interference. 24 | 25 | * First things first, download and install Anaconda for your operating system. You can find the downloads [here](https://www.anaconda.com/distribution/#download-section). Make sure you pick the Python 3.7 version. When you're installing, make sure you pick the option something along the lines of "add anaconda to path". That way, you can use `conda` on the command line. 26 | * Install git for your operating system (a good idea anyway!) [Downloads page here](https://git-scm.com/download) 27 | * Open up the command line, such as terminal on mac or cmd.exe. **VERY IMPORTANT: On Windows, make sure you run the command prompt as an administrator! To do this, right click the shortcut to the command prompt, click `run as administrator`, then say yes to whatever pops up.** 28 | 29 | ## Installing from pip 30 | First install the latest version of PyTorch for your system. [Use this link for official instructions.](https://pytorch.org/) 31 | It should be as simple as `conda install pytorch torchvision cudatoolkit=10.2 -c pytorch`. 32 | 33 | Note: if you have an RTX3000 series graphics card, such as a 3060 or 3090, please use `cudatoolkit=11.1` or higher. 34 | 35 | After installing PyTorch, simply use `pip install deepethogram`. 36 | 37 | ## Install FFMPEG 38 | We use FFMPEG for reading and writing `.mp4` files (with libx264 encoding). Please use [this link](https://www.ffmpeg.org/) 39 | to install on your system. 40 | 41 | ## Startup 42 | * `conda activate deg`. This activates the environment. 43 | * type `python -m deepethogram`, in the command line to open the GUI. 44 | 45 | ## Upgrading to Beta 46 | Please see [the beta docs for instructions](beta.md) 47 | 48 | ## Common installation problems 49 | * You might have dependency issues with other packages you've installed. Please make a new anaconda or miniconda 50 | environment with `conda create --name deg python=3.8` before installation. 51 | * `module not found: PySide2`. Some versions of PySide2 install poorly from pip. use `pip uninstall pyside2`, then 52 | `conda install -c conda-forge pyside2` 53 | * When opening the GUI, you might get `Segmentation fault (core dumped)`. In this case; please `pip uninstall pyside2`, 54 | `conda uninstall pyside2`. `pip install pyside2` 55 | * `ImportError: C:\Users\jbohn\.conda\envs\deg2\lib\site-packages\shiboken2\libshiboken does not exist` 56 | * something went wrong with your PySide2 installation, likely on Windows. 57 | * Make sure you have opened your command prompt as administrator 58 | * If it tells you to install a new version of Visual Studio C++, please do that. 59 | * Now you should be set up: let's reinstall PySide2 and libshiboken. 60 | * `pip install --force-reinstall pyside2` 61 | * `_init_pyside_extension is not defined` 62 | * This is an issue where Shiboken and PySide2 are not playing nicely together. Please `pip uninstall pyside2` and `conda remove pyside2`. Don't manually install these packages; instead, let DeepEthogram install it for you via pip. Therefore, `pip uninstall deepethogram` and `pip install deepethogram`. 63 | * `qt.qpa.plugin: Could not load the Qt platform plugin "xcb" in ".../python3.8/site-packages/cv2/qt/plugins" even though it was found. This application failed to start because no Qt platform plugin could be initialized. Reinstalling the application may fix this problem.` 64 | * This is an issue with a recent version of `opencv-python` not working well with Qt. Please do `pip install --force-reinstall opencv-python-headless==4.1.2.30` 65 | -------------------------------------------------------------------------------- /docs/model_performance.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/docs/model_performance.md -------------------------------------------------------------------------------- /docs/testing.md: -------------------------------------------------------------------------------- 1 | # Testing DeepEthogram 2 | 3 | This document describes how to run and contribute to DeepEthogram's test suite. 4 | 5 | ## Test Categories 6 | 7 | DeepEthogram's tests are divided into two main categories: 8 | 9 | 1. **Standard Tests**: These include unit tests and basic integration tests that don't require GPU resources. These tests run quickly and are executed by default. 10 | 11 | 2. **GPU Tests**: These are end-to-end integration tests that require an NVIDIA GPU and significant computational resources. They perform actual model training and inference to ensure the full pipeline works correctly. These tests are marked with the `@pytest.mark.gpu` decorator and are skipped by default. 12 | 13 | ## Running Tests 14 | 15 | ### Basic Usage 16 | 17 | ```bash 18 | # Run all tests except GPU tests (default) 19 | pytest tests/ 20 | 21 | # Run only GPU tests (requires NVIDIA GPU) 22 | pytest -m gpu 23 | 24 | # Run all tests including GPU tests 25 | pytest -m "" 26 | ``` 27 | 28 | ### Test Data Setup 29 | 30 | Before running tests: 31 | 32 | 1. Download [`testing_deepethogram_archive.zip`](https://drive.google.com/file/d/1IFz4ABXppVxyuhYik8j38k9-Fl9kYKHo/view?usp=sharing) 33 | 2. Create a directory called `DATA` in the tests directory 34 | 3. Unzip the archive and move its contents to `deepethogram/tests/DATA/testing_deepethogram_archive/` 35 | 4. Verify the path structure: `deepethogram/tests/DATA/testing_deepethogram_archive/{DATA,models,project_config.yaml}` 36 | 37 | ## Writing Tests 38 | 39 | ### Adding GPU Tests 40 | 41 | When writing tests that require GPU resources: 42 | 43 | 1. Mark the test with the `@pytest.mark.gpu` decorator 44 | 2. Place GPU-intensive tests in appropriate test modules 45 | 3. Keep GPU tests focused and efficient to minimize resource usage 46 | 47 | Example: 48 | ```python 49 | import pytest 50 | 51 | @pytest.mark.gpu 52 | def test_model_training(): 53 | # GPU-intensive test code here 54 | pass 55 | ``` 56 | 57 | ### Best Practices 58 | 59 | 1. Keep GPU tests separate from standard tests when possible 60 | 2. Document resource requirements in test docstrings 61 | 3. Use small datasets and minimal epochs for GPU tests 62 | 4. Add appropriate error handling for cases where GPU is not available 63 | 64 | ## Continuous Integration 65 | 66 | The CI pipeline runs standard tests by default. GPU tests are only run in specific environments or when explicitly requested to avoid unnecessary resource usage. 67 | -------------------------------------------------------------------------------- /docs/troubleshooting.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting 2 | 3 | Please use the `issues` button on GitHub for any bugs you think you've encountered. During GUI usage, model training, 4 | and model inference, hydra creates a `.log` file (e.g. `train.log`, `main.log`) with various log messages. Please 5 | copy this into your GitHub page. When using the command line, including starting the GUI, use the flag `hydra.verbose=true`. 6 | This will add debugging information to your logs (and also print them to the command line). 7 | 8 | 9 | # FAQ 10 | #### Model generates poor predictions 11 | The most important factors that determine model performance are as follows: 12 | 1. number of data points 13 | 2. frequency of behavior (better performance for more common ones) 14 | 15 | Please have at least a few hundred frames for each behavior before further inspection is needed. 16 | -------------------------------------------------------------------------------- /docs/using_CLI.md: -------------------------------------------------------------------------------- 1 | # Using the command line interface 2 | 3 | A common way to implement a simple command line interface is to use python's builtin [argparse module](https://docs.python.org/3/library/argparse.html). 4 | However, for this project, we have multiple model types, which share some hyperparameters (such as learning rate) while also 5 | having unique hyperparameters (such as the loss function used for the optic flow generator). Furthermore, I've put a lot of 6 | thought into default hyperparameters, so I want to be able to include defaults. Finally, each user must override specific hyperparameters 7 | for their own project, such as the names of the different behaviors. Therefore, for the CLI, we want to be able to 8 | * have nested arguments, such as train.learning_rate, train.scheduler, etc 9 | * be able to use configuration files to load many parameters at once 10 | * be able to override defaults with our own configuration files 11 | * be able to override everything from the command line 12 | 13 | Luckily, [OmegaConf package](https://omegaconf.readthedocs.io/en/2.0_branch/) does a lot of this for us! Therefore, we use OmegaConf configurations for everything. 14 | 15 | ## Common usage 16 | For all DeepEthogram projects, we [expect a consistent file structure](file_structure.md). Therefore, when using the CLI, always use the flag 17 | `project.config_file=path/to/config/file.yaml` or `project.path=path/to/deepethogram_project` 18 | 19 | ## Creating a project in code 20 | If you don't want to use the GUI, you still need to set up your project with the [consistent file structure](file_structure.md). 21 | 22 | 23 | ### Make a project directory 24 | You will need to open a python interpreter, or launch a Jupyter notebook. 25 | 26 | First, let's create a project directory with a properly formatted `project_config.yaml` 27 | ```python 28 | from deepethogram import projects 29 | 30 | # this is a path to a base directory on your hard drive 31 | # this is just an example! change this to whereever you want, e.g. 'C:\DATA\movies` 32 | data_path = '/mnt/DATA' 33 | 34 | # pick a project name 35 | project_name = 'open_field' 36 | 37 | # make a list of behaviors. try to choose your behaviors carefully! 38 | # you'll have to re-train all your models if you add or remove behaviors. 39 | behaviors = ['background', 40 | 'groom', 41 | 'locomote', 42 | 'etc1', 43 | 'etc2'] 44 | 45 | # this will create a folder called /mnt/DATA/open_field_deepethogram 46 | # there will be subdirectories called DATA and models 47 | # there will also be a project_config.yaml 48 | project_config = projects.initialize_project(data_path, project_name, behaviors) 49 | ``` 50 | 51 | ### add videos and labels 52 | 53 | This presumes you have a list of movies ready for training, and separately have labeled frames. 54 | 55 | The labels should be `.csv` files with this format: 56 | * there should be one row per frame. The label CSV should have the same number of rows as the video has frames. 57 | * there should be one column for each behavior. the name of the column should be the name of the behavior. The order 58 | should be the same as you specified in the `project_config` above. 59 | * the first column should be called "background", and it is the logical not of any of the other columns being one. 60 | * NOTE: if you don't have this, `projects.add_label_to_project` will do this for you! 61 | * there should be a 1 if the labeled behavior is present on this frame, a zero otherwise. 62 | 63 | ![label format screenshot](images/label_format.png) 64 | 65 | Here's an example in code: 66 | 67 | ```python 68 | # adding videos 69 | list_of_movies = ['/path/to/movie1.avi', 70 | '/path/to/movie2.avi'] 71 | mode = 'copy' # or 'symlink' or 'move' 72 | 73 | # depending on the mode, it will copy, symlink, or move each video file 74 | # it will also compute the mean and standard deviation of each RGB channel 75 | for movie_path in list_of_movies: 76 | projects.add_video_to_project(project_config, movie_path, mode=mode) 77 | 78 | 79 | # now, we have our new movie files properly in our deepethogram project 80 | new_list_of_movies = ['/mnt/DATA/open_field_deepethogram/DATA/movie1.avi', 81 | '/mnt/DATA/open_field_deepethogram/DATA/movie2.avi'] 82 | 83 | # we also have a list of label files, created by some other means 84 | list_of_labels = ['/mnt/DATA/my_other_project/movie1/labels.csv', 85 | '/mnt/DATA/my_other_project/movie2/labels.csv'] 86 | 87 | for movie_path, label_path in zip(new_list_of_movies, list_of_labels): 88 | projects.add_label_to_project(label_path, movie_path) 89 | ``` 90 | 91 | ### Add pretrained models to your project/models directory 92 | For detailed instructions, please go to [the project README's pretrained models section](../README.md) 93 | 94 | ## Training examples 95 | To train the flow generator with the larger MotionNet architecture and a batch size of 16: 96 | 97 | `deepethogram.flow_generator.train project.config_file=path/to/config/file.yaml flow_generator.arch=MotionNet compute.batch_size=16` 98 | 99 | To train the feature extractor with the ResNet18 base, without the curriculum training, with an initial learning rate of 1e-5: 100 | `deepethogram.feature_extractor.train project.config_file=path/to/config/file.yaml feature_extractor.arch=resnet18 train.lr=1e-5 feature_extractor.curriculum=false notes=no_curriculum` 101 | 102 | To train the flow generator with specific weights loaded from disk, with a specific train/test split, with the DEG_s preset (3D MotionNet): 103 | `python -m deepethogram.flow_generator.train project.config_file=path/to/config/file.yaml reload.weights=path/to/flow/weights.pt split.file=path/to/split.yaml preset=deg_s` 104 | 105 | To train the feature extractor on the secondary GPU with the latest optic flow weights, but a specific feature extractor weights: 106 | `python -m deepethogram.feature_extractor.train project.config_file=path/to/config/file.yaml compute.gpu_id=1 flow_generator.weights=latest feature_extractor.weights=path/to/kinetics_weights.pt` 107 | 108 | # Questions? 109 | For any questions on how to use the command line interface for your training, please raise an issue on GitHub. 110 | -------------------------------------------------------------------------------- /docs/using_code.md: -------------------------------------------------------------------------------- 1 | # Expected filepaths 2 | 3 | TODO 4 | -------------------------------------------------------------------------------- /docs/using_config_files.md: -------------------------------------------------------------------------------- 1 | # Using project configuration files 2 | 3 | DeepEthogram uses configuration files (.yaml) to save information and load hyperparameters. [For reasoning, see the 4 | CLI docs](using_CLI.md). In each project directory is a file called `project_config.yaml`. There, you can edit model 5 | architectures, change the batch size, specify the learning rate, etc. Both the GUI and the command line interface 6 | will overwrite the defaults with whatever you have in this configuration file. 7 | 8 | Hyperparameters can be specified in multiple places. Let's say you want to experiment with adding more regularization 9 | to your model. The weight decay is specified in multiple places: 10 | * the defaults in `deepethogram\conf\model\feature_extractor.yaml` 11 | * maybe in your project configuration file 12 | * the command line with `feature_extractor.weight_decay=0.01` 13 | 14 | Per [the hydra docs](hydra.cc), the loading order is as follows, with the last one actually being used: 15 | `default -> project config -> command line`. This means even if you normally use `weight_decay=0.001` in your project 16 | configuration, you can still run experiments at the command line. 17 | 18 | ## How to edit your project configuration file 19 | Navigate to your project dictionary with your `project_config.yaml` file. Importantly: **not every hyperparameter 20 | will be in the default project configuration! This means if you want to edit a less-used hyperparameter, you'll 21 | have to add lines to the configuration, not just edit them.** 22 | 23 | The left part of the below screenshot shows the default project configuration in `deepethogram/conf/project/project_config.yaml`. 24 | The right shows my configuration for the `mouse_reach` example [used in the GUI docs](using_gui.md). 25 | 26 | ![screenshot of config dictionary](images/project_config.png) 27 | 28 | Note the following: 29 | * `augs/normalization` has been overwritten for the statistics of my dataset 30 | * I've edited `augs/resize` to resize my data to be 128 tall and 256 wide. This speeds up training, and the images are 31 | not square due to the multiple views that I've concatenated side by side ([see the GUI docs](using_gui.md)). 32 | * the `project` dictionary has been totally changed 33 | * I've added lines to the train dictionary: 34 | * `patience: 10`: this means the learning rate will only be reduced if learning stalls for 10 epochs 35 | (see `deepethogram/conf/train.yaml` for explanation) 36 | * `reduction_factor: 0.3162277`: this means the learning rate will be reduced by this value (1 / sqrt(10), which means 37 | the learning rate will go down by a factor of 0.1 after two steps) 38 | 39 | This is how to edit a configuration file. You *add or edit fields in your project config in the nested structure shown in `deepethogram/conf`.* 40 | For example, the `train` dictionary in the config file takes values shown in `deepethogram/conf/train.yaml`. 41 | To find out what hyperparameters there are and what values they can take, 42 | read through the configuration files in `deepethogram/conf`. The most commonly edited ones are already in the 43 | default `project_config.yaml`, such as batch size and image augmentation. 44 | 45 | ## Creating configuration files in code 46 | If you want to use functions that require a configuration file, but want to use the codebase instead of the command 47 | line interface, you can create a nested dictionary with your configuration, then do as follows: 48 | ```python 49 | from omegaconf import OmegaConf 50 | nested_dictionary = {'project': {'class_names': ['background', 'etc', 'etc']}, 51 | 'etc': 'etc'} 52 | cfg = OmegaConf.create(nested_dictionary) 53 | print(cfg.pretty()) 54 | # etc: etc 55 | # project: 56 | # class_names: 57 | # - background 58 | # - etc 59 | # - etc 60 | ``` 61 | -------------------------------------------------------------------------------- /docs/using_tune.md: -------------------------------------------------------------------------------- 1 | # Hyperparameter tuning 2 | 3 | ## usage locally 4 | `ray start --head` 5 | `python -m deepethogram.tune.feature_extractor ARGS` 6 | 7 | ## Usage on slurm cluster 8 | ### usage on one node 9 | `ray start --head --num-cpus 16 --num-gpus 2` 10 | in code: `ray.init(address='auto')` 11 | 12 | ## possible errors 13 | asyncio error message: TypeError: __init__() got an unexpected keyword argument 'loop' 14 | install aiohttp 3.6.0 15 | https://github.com/ray-project/ray/issues/8749 16 | for bugs like "could not terminate" 17 | "/usr/bin/redis-server 127.0.0.1:6379" "" "" "" "" "" "" ""` due to psutil.AccessDenied (pid=56271, name='redis-server') 18 | sudo /etc/init.d/redis-server stop 19 | if you have a GPU you can't use for training (e.g. I have a tiny, old GPU just for my monitors) exclude that 20 | using command line arguments. e.g. CUDA_VISIBLE_DEVICES=0,1 ray start --head 21 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: deg 2 | channels: 3 | - pytorch 4 | - conda-forge 5 | - defaults 6 | dependencies: 7 | - pip 8 | - conda-forge::pyside2=5.13.2 9 | - python>3.7, <3.9 10 | - pytorch::pytorch 11 | - pip: 12 | - -r requirements.txt 13 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | Academic and Non-Commercial Research Use Software License and Terms of Use 2 | DeepEthogram: Automated behavior classification (the “Software”, tracked by Harvard Case No. 8336). The Software was developed by Christopher Harvey and Jim Bohnslav, Department of Neurobiology, Harvard Medical School. 3 | Using the Software indicates your agreement to be bound by the terms of this Software Use Agreement (“Agreement”). Absent your agreement to the terms below, you (the “End User”) have no rights to hold or use the Software whatsoever. 4 | President and Fellows of Harvard College (“Harvard”) agrees to grant hereunder the limited non-exclusive license to End User for the use of the Software in the performance of End User’s internal, non-commercial research and academic use at End User’s academic or not-for-profit research institution (“Institution”) on the following terms and conditions: 5 | 1. NO REDISTRIBUTION. The Software remains the property of Harvard, and End User shall not publish, distribute, or otherwise transfer or make available the Software to any other party. 6 | 7 | 2. NO COMMERCIAL USE. End User shall not use the Software for commercial purposes, including for any internal research purposes for or on behalf of any for-profit entity, and any such use of the Software is expressly prohibited. This prohibition includes, but is not limited to, use of the Software in fee-for-service arrangements, core facilities or laboratories or to provide research services to (or in collaboration with) third parties for a fee, and in industry-sponsored collaborative research projects where any commercial rights are granted to the sponsor. If End User wishes to use the Software for commercial purposes or for any other restricted purpose, End User must execute a separate license agreement with Harvard. 8 | 9 | Requests for use of the Software for or on behalf of for-profit entities or for any commercial purposes, please contact: 10 | 11 | Office of Technology Development 12 | Harvard University 13 | Smith Campus Center, Suite 727E 14 | 1350 Massachusetts Avenue 15 | Cambridge, MA 02138 USA 16 | Telephone: (617) 495-3067 17 | E-mail: otd@harvard.edu 18 | 19 | 3. OWNERSHIP AND COPYRIGHT NOTICE. Harvard owns all intellectual property in the Software. End User shall gain no ownership to the Software. End User shall not remove or delete and shall retain in the Software, in any modifications to Software and in any Derivative Works, the copyright, trademark, or other notices pertaining to Software as provided with the Software. 20 | 21 | 4. DERIVATIVE WORKS. End User may create and use Derivative Works, as such term is defined under U.S. copyright laws, provided that any such Derivative Works shall be restricted to non-commercial, internal research and academic use at End User’s Institution. End User shall not distribute Derivative Works to any third party without the express written consent of Harvard. 22 | 23 | 5. FEEDBACK. In order to improve the Software, comments from End Users may be useful. End User agrees to provide Harvard with feedback on the End User’s use of the Software (e.g., any bugs in the Software, the user experience, etc.). Harvard is permitted to use such information provided by End User in making changes and improvements to the Software without compensation or an accounting to End User. 24 | 25 | 6. NON ASSERT. End User acknowledges that Harvard may develop modifications to the Software that may be based on the feedback provided by End User under Section 5 above. Harvard shall not be restricted in any way by End User regarding the use of such information. End User acknowledges the right of Harvard to prepare, publish, display, reproduce, transmit and or use modifications to the Software that may be substantially similar or functionally equivalent to End User’s modifications and/or improvements if any. In the event that End User obtains patent protection for any modification or improvement to Software, End User agrees not to allege or enjoin infringement of End User’s patent against Harvard or any of its researchers, medical or research staff, officers, directors and employees. 26 | 27 | 7. PUBLICATION & ATTRIBUTION. End User has the right to publish, present, or share results from the use of the Software.  In accordance with customary academic practice, End User will acknowledge Harvard as the provider of the Software and may cite the relevant reference(s) from the following list of publications: N/A 28 | 29 | 8. NO WARRANTIES. THE SOFTWARE IS PROVIDED "AS IS." TO THE FULLEST EXTENT PERMITTED BY LAW, HARVARD HEREBY DISCLAIMS ALL WARRANTIES OF ANY KIND (EXPRESS, IMPLIED OR OTHERWISE) REGARDING THE SOFTWARE, INCLUDING BUT NOT LIMITED TO ANY IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, OWNERSHIP, AND NON-INFRINGEMENT. HARVARD MAKES NO WARRANTY ABOUT THE ACCURACY, RELIABILITY, COMPLETENESS, TIMELINESS, SUFFICIENCY OR QUALITY OF THE SOFTWARE. HARVARD DOES NOT WARRANT THAT THE SOFTWARE WILL OPERATE WITHOUT ERROR OR INTERRUPTION. 30 | 31 | 9. Limitations of Liability and Remedies. USE OF THE SOFTWARE IS AT END USER’S OWN RISK. IF END USER IS DISSATISFIED WITH THE SOFTWARE, ITS EXCLUSIVE REMEDY IS TO STOP USING IT. IN NO EVENT SHALL HARVARD BE LIABLE TO END USER OR ITS INSTITUTION, IN CONTRACT, TORT OR OTHERWISE, FOR ANY DIRECT, INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL, PUNITIVE OR OTHER DAMAGES OF ANY KIND WHATSOEVER ARISING OUT OF OR IN CONNECTION WITH THE SOFTWARE, EVEN IF HARVARD IS NEGLIGENT OR OTHERWISE AT FAULT, AND REGARDLESS OF WHETHER HARVARD IS ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. 32 | 33 | 10. INDEMNIFICATION. To the extent permitted by law, End User shall indemnify, defend and hold harmless Harvard and its current or future directors, trustees, officers, faculty, medical and professional staff, employees, students and agents and their respective successors, heirs and assigns (the "Indemnitees"), against any liability, damage, loss or expense (including reasonable attorney's fees and expenses of litigation) incurred by or imposed upon the Indemnitees or any one of them in connection with any claims, suits, actions, demands or judgments arising from End User’s breach of this Agreement or its Institution’s use of the Software except to the extent caused by the gross negligence or willful misconduct of Harvard. This indemnification provision shall survive expiration or termination of this Agreement. 34 | 35 | 11. GOVERNING LAW. This Agreement shall be construed and governed by the laws of the Commonwealth of Massachusetts regardless of otherwise applicable choice of law standards. 36 | 37 | 12. NON-USE OF NAME. Nothing in this License and Terms of Use shall be construed as granting End Users or their Institutions any rights or licenses to use any trademarks, service marks or logos associated with the Software. You may not use the terms “Harvard” (or a substantially similar term) in any way that is inconsistent with the permitted uses described herein. You agree not to use any name or emblem of Harvard or any of its subdivisions for any purpose, or to falsely suggest any relationship between End User (or its Institution) and Harvard, or in any manner that would infringe or violate any of Harvard’s rights. 38 | 39 | 13. End User represents and warrants that it has the legal authority to enter into this License and Terms of Use on behalf of itself and its Institution. 40 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.ruff] 2 | # Python version compatibility 3 | target-version = "py37" 4 | 5 | # Same as Black. 6 | line-length = 120 7 | 8 | # Exclude a variety of commonly ignored directories. 9 | exclude = [ 10 | ".bzr", 11 | ".direnv", 12 | ".eggs", 13 | ".git", 14 | ".git-rewrite", 15 | ".hg", 16 | ".mypy_cache", 17 | ".nox", 18 | ".pants.d", 19 | ".pytype", 20 | ".ruff_cache", 21 | ".svn", 22 | ".tox", 23 | ".venv", 24 | "__pypackages__", 25 | "_build", 26 | "buck-out", 27 | "build", 28 | "dist", 29 | "node_modules", 30 | "venv", 31 | "tests/", 32 | "docs/" 33 | ] 34 | 35 | [tool.ruff.lint] 36 | 37 | # Allow autofix for all enabled rules (when `--fix`) is provided. 38 | fixable = ["ALL"] 39 | unfixable = [] 40 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | filterwarnings = 3 | ignore::DeprecationWarning:pkg_resources.*: 4 | ignore::DeprecationWarning:distutils.*: 5 | ignore::DeprecationWarning:torch.utils.tensorboard.*: 6 | ignore::DeprecationWarning:pytorch_lightning.*: 7 | 8 | markers = 9 | gpu: marks tests that require GPU (deselect with '-m "not gpu"') 10 | 11 | # Skip GPU tests by default 12 | addopts = -m "not gpu" 13 | 14 | python_functions = test_* *_test gpu_test_* 15 | 16 | # Configure test ordering - GPU tests will run last 17 | python_classes = Test* *Test 18 | python_files = test_*.py *_test.py 19 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | chardet<4.0 2 | h5py 3 | kornia>=0.5 4 | matplotlib 5 | numpy 6 | omegaconf>=2 7 | opencv-python-headless 8 | opencv-transforms 9 | pandas<1.4 10 | PySide2==5.13.2 11 | pytest 12 | scikit-learn<1.1 13 | scipy<1.8 14 | tqdm 15 | vidio 16 | pytorch_lightning==1.6.5 17 | ruff>=0.1.0 18 | pre-commit>=2.20.0,<3.0.0 19 | setuptools 20 | gdown 21 | -------------------------------------------------------------------------------- /reset_venv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Remove existing venv if it exists 4 | if [ -d ".venv" ]; then 5 | echo "Removing existing .venv directory..." 6 | rm -rf .venv 7 | fi 8 | 9 | # Create new venv with Python 3.7 10 | echo "Creating new virtual environment with Python 3.7..." 11 | uv venv --python 3.7 12 | 13 | # Activate the virtual environment 14 | echo "Activating virtual environment..." 15 | source .venv/bin/activate 16 | 17 | # Install requirements first 18 | echo "Installing requirements..." 19 | uv pip install -r requirements.txt 20 | 21 | # Install pytest 22 | echo "Installing pytest..." 23 | uv pip install pytest pytest-cov 24 | 25 | # Install package in editable mode 26 | echo "Installing package in editable mode..." 27 | uv pip install -e . 28 | 29 | # Setup test data 30 | echo "Setting up test data..." 31 | python setup_tests.py 32 | 33 | # Run tests 34 | echo "Running tests..." 35 | pytest -v tests/ 36 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r") as f: 4 | long_description = f.read() 5 | 6 | 7 | def get_requirements(): 8 | with open("requirements.txt") as f: 9 | return f.read().splitlines() 10 | 11 | 12 | setuptools.setup( 13 | name="deepethogram", 14 | version="0.2.0", 15 | author="Jim Bohnslav", 16 | author_email="jbohnslav@gmail.com", 17 | description="Temporal action detection for biology", 18 | long_description=long_description, 19 | long_description_content_type="text/markdown", 20 | include_package_data=True, 21 | packages=setuptools.find_packages(), 22 | classifiers=[ 23 | "Programming Language :: Python :: 3", 24 | "Operating System :: OS Independent", 25 | ], 26 | entry_points={"console_scripts": ["deepethogram = deepethogram.gui.main:entry"]}, 27 | python_requires=">=3.7,<3.8", 28 | install_requires=get_requirements(), 29 | options={ 30 | "ruff": { 31 | "target-version": "py37", 32 | }, 33 | }, 34 | setup_requires=["setuptools"], 35 | ) 36 | -------------------------------------------------------------------------------- /setup_tests.py: -------------------------------------------------------------------------------- 1 | """This script downloads the test data archive and sets up the testing environment for DeepEthogram. 2 | 3 | For it to work, you need to `pip install gdown` 4 | """ 5 | 6 | import sys 7 | import zipfile 8 | from pathlib import Path 9 | 10 | import gdown 11 | import requests 12 | 13 | 14 | def download_file(url, destination): 15 | """Downloads a file from a URL to a destination with progress indication.""" 16 | response = requests.get(url, stream=True) 17 | total_size = int(response.headers.get("content-length", 0)) 18 | block_size = 8192 19 | 20 | if total_size == 0: 21 | print("Warning: Content length not provided by server") 22 | 23 | print(f"Downloading to: {destination}") 24 | 25 | with open(destination, "wb") as f: 26 | downloaded = 0 27 | for data in response.iter_content(block_size): 28 | downloaded += len(data) 29 | f.write(data) 30 | 31 | # Print progress 32 | if total_size > 0: 33 | progress = int(50 * downloaded / total_size) 34 | sys.stdout.write(f"\r[{'=' * progress}{' ' * (50 - progress)}] {downloaded}/{total_size} bytes") 35 | sys.stdout.flush() 36 | print("\nDownload completed!") 37 | 38 | 39 | def setup_tests(): 40 | """Sets up the testing environment for DeepEthogram.""" 41 | try: 42 | # Create tests/DATA directory if it doesn't exist 43 | tests_dir = Path("tests") 44 | data_dir = tests_dir / "DATA" 45 | data_dir.mkdir(parents=True, exist_ok=True) 46 | 47 | # Define paths and requirements 48 | archive_path = data_dir / "testing_deepethogram_archive" 49 | zip_path = data_dir / "testing_deepethogram_archive.zip" 50 | required_items = ["DATA", "models", "project_config.yaml"] 51 | 52 | # Check if test data already exists and is complete 53 | if archive_path.exists(): 54 | missing_items = [item for item in required_items if not (archive_path / item).exists()] 55 | if not missing_items: 56 | print("Test data already exists and appears complete. Skipping download.") 57 | return True 58 | print("Test data exists but is incomplete. Re-downloading...") 59 | 60 | # Download and extract if needed 61 | if not archive_path.exists() or not all((archive_path / item).exists() for item in required_items): 62 | # Download if zip doesn't exist 63 | if not zip_path.exists(): 64 | print("Downloading test data archive...") 65 | gdown.download(id="1IFz4ABXppVxyuhYik8j38k9-Fl9kYKHo", output=str(zip_path), quiet=False) 66 | 67 | # Extract archive 68 | print("Extracting archive...") 69 | with zipfile.ZipFile(zip_path, "r") as zip_ref: 70 | zip_ref.extractall(data_dir) 71 | 72 | # Clean up zip file after successful extraction 73 | zip_path.unlink() 74 | 75 | # Final verification 76 | missing_items = [item for item in required_items if not (archive_path / item).exists()] 77 | if missing_items: 78 | print(f"Warning: The following items are missing: {missing_items}") 79 | return False 80 | 81 | print("Setup completed successfully!") 82 | print("\nYou can now run the tests using: pytest tests/") 83 | print("Note: The gpu tests will take a few minutes to complete.") 84 | return True 85 | 86 | except Exception as e: 87 | print(f"Error during setup: {str(e)}") 88 | return False 89 | 90 | 91 | if __name__ == "__main__": 92 | setup_tests() 93 | -------------------------------------------------------------------------------- /tests/setup_data.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import time 4 | import platform 5 | 6 | # from projects import get_records_from_datadir, fix_config_paths 7 | from deepethogram import projects 8 | 9 | this_path = os.path.abspath(__file__) 10 | test_path = os.path.dirname(this_path) 11 | deg_path = os.path.dirname(test_path) 12 | 13 | test_data_path = os.path.join(test_path, "DATA") 14 | # the deepethogram test archive should only be read from, never written to 15 | archive_path = os.path.join(test_data_path, "testing_deepethogram_archive") 16 | assert os.path.isdir(archive_path), "{} does not exist!".format(archive_path) 17 | project_path = os.path.join(test_data_path, "testing_deepethogram") 18 | data_path = os.path.join(project_path, "DATA") 19 | 20 | config_path = os.path.join(project_path, "project_config.yaml") 21 | config_path_archive = os.path.join(archive_path, "project_config.yaml") 22 | # config_path = os.path.join(project_path, 'project_config.yaml') 23 | cfg_archive = projects.get_config_from_path(archive_path) 24 | 25 | 26 | def change_to_deepethogram_directory(): 27 | os.chdir(deg_path) 28 | 29 | 30 | def clean_test_data(): 31 | if not os.path.isdir(project_path): 32 | return 33 | 34 | # On Windows, we need to handle file permission errors 35 | if platform.system() == 'Windows': 36 | max_retries = 3 37 | for i in range(max_retries): 38 | try: 39 | shutil.rmtree(project_path) 40 | break 41 | except PermissionError: 42 | if i < max_retries - 1: 43 | time.sleep(1) # Wait a bit for file handles to be released 44 | continue 45 | else: 46 | # If we still can't delete after retries, try to ignore errors 47 | try: 48 | shutil.rmtree(project_path, ignore_errors=True) 49 | except: 50 | pass # If we still can't delete, just continue 51 | else: 52 | shutil.rmtree(project_path) 53 | 54 | 55 | def make_project_from_archive(): 56 | change_to_deepethogram_directory() 57 | clean_test_data() 58 | shutil.copytree(archive_path, project_path) 59 | # this also fixes paths 60 | cfg = projects.get_config_from_path(project_path) 61 | # projects.fix_config_paths(cfg) 62 | 63 | 64 | def get_records(origin="project"): 65 | if origin == "project": 66 | return projects.get_records_from_datadir(data_path) 67 | elif origin == "archive": 68 | return projects.get_records_from_datadir(os.path.join(archive_path, "DATA")) 69 | else: 70 | raise NotImplementedError 71 | -------------------------------------------------------------------------------- /tests/test_data.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from deepethogram.data import utils as data_utils 4 | 5 | 6 | def test_loss_weight(): 7 | class_counts = np.array([1, 2]) 8 | num_pos = np.array([1, 2]) 9 | num_neg = np.array([2, 1]) 10 | 11 | pos_weight_transformed, softmax_weight_transformed = data_utils.make_loss_weight( 12 | class_counts, num_pos, num_neg, weight_exp=1.0 13 | ) 14 | assert np.allclose(pos_weight_transformed, np.array([2, 0.5])) 15 | assert np.allclose(softmax_weight_transformed, np.array([2 / 3, 1 / 3])) 16 | 17 | class_counts = np.array([0, 300]) 18 | num_pos = np.array([0, 300]) 19 | num_neg = np.array([300, 0]) 20 | 21 | pos_weight_transformed, softmax_weight_transformed = data_utils.make_loss_weight( 22 | class_counts, num_pos, num_neg, weight_exp=1.0 23 | ) 24 | print(pos_weight_transformed, softmax_weight_transformed) 25 | assert np.allclose(pos_weight_transformed, np.array([0, 1])) 26 | assert np.allclose(softmax_weight_transformed, np.array([0, 1])) 27 | # assert np.allclose(pos_weight_transformed, np.array([2, 0.5])) 28 | # assert np.allclose(softmax_weight_transformed, np.array([2 / 3, 1 / 3])) 29 | -------------------------------------------------------------------------------- /tests/test_flow_generator.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from deepethogram import projects, utils, viz 4 | from deepethogram.configuration import make_flow_generator_train_cfg 5 | from deepethogram.flow_generator.train import ( 6 | get_datasets_from_cfg, 7 | build_model_from_cfg, 8 | get_metrics, 9 | OpticalFlowLightning, 10 | ) 11 | from setup_data import make_project_from_archive, project_path 12 | 13 | 14 | def test_metrics(): 15 | make_project_from_archive() 16 | cfg = make_flow_generator_train_cfg(project_path=project_path) 17 | cfg = projects.setup_run(cfg) 18 | 19 | datasets, data_info = get_datasets_from_cfg(cfg, "flow_generator", input_images=cfg.flow_generator.n_rgb) 20 | flow_generator = build_model_from_cfg(cfg) 21 | utils.save_dict_to_yaml(data_info["split"], os.path.join(os.getcwd(), "split.yaml")) 22 | flow_weights = projects.get_weightfile_from_cfg(cfg, "flow_generator") 23 | if flow_weights is not None: 24 | print("reloading weights...") 25 | flow_generator = utils.load_weights(flow_generator, flow_weights, device="cpu") 26 | 27 | # stopper = get_stopper(cfg) 28 | metrics = get_metrics(cfg, os.getcwd(), utils.get_num_parameters(flow_generator)) 29 | lightning_module = OpticalFlowLightning(flow_generator, cfg, datasets, metrics, viz.visualize_logger_optical_flow) 30 | assert lightning_module.scheduler_mode == "min" 31 | assert lightning_module.metrics.key_metric == "SSIM" 32 | -------------------------------------------------------------------------------- /tests/test_gui.py: -------------------------------------------------------------------------------- 1 | # from pytestqt import qtbot 2 | import os 3 | 4 | import pytest 5 | 6 | DEG_VERSION = os.environ.get("DEG_VERSION", "full") 7 | 8 | 9 | @pytest.mark.skipif( 10 | DEG_VERSION == "headless", 11 | reason="Dont run GUI tests for headless deepethogram", 12 | ) 13 | def test_setup(): 14 | # put imports here so that headless version does not import gui tools 15 | from deepethogram.gui.main import setup_gui_cfg 16 | 17 | cfg = setup_gui_cfg() 18 | 19 | assert cfg.run.type == "gui" 20 | 21 | 22 | # def test_new_project(): 23 | # cfg = setup_gui_cfg() 24 | 25 | # window = MainWindow(cfg) 26 | # window.resize(1024, 768) 27 | # window.show() 28 | 29 | # window._new_project() 30 | # def test_open(): 31 | # run() 32 | -------------------------------------------------------------------------------- /tests/test_integration.py: -------------------------------------------------------------------------------- 1 | # this is named test__zz_commandline so that it comes last, after all module-specific tests 2 | import subprocess 3 | import pytest 4 | 5 | from deepethogram import utils 6 | 7 | from setup_data import make_project_from_archive, change_to_deepethogram_directory, config_path, data_path 8 | # from setup_data import get_testing_directory 9 | 10 | # testing_directory = get_testing_directory() 11 | # config_path = os.path.join(testing_directory, 'project_config.yaml') 12 | BATCH_SIZE = 2 # small but not too small 13 | # if less than 10, might have bugs with visualization 14 | STEPS_PER_EPOCH = 20 15 | NUM_EPOCHS = 2 16 | VIZ_EXAMPLES = 2 17 | 18 | make_project_from_archive() 19 | 20 | change_to_deepethogram_directory() 21 | 22 | 23 | def command_from_string(string): 24 | command = string.split(" ") 25 | if command[-1] == "": 26 | command = command[:-1] 27 | print(command) 28 | return command 29 | 30 | 31 | def add_default_arguments(string, train=True): 32 | string += f"project.config_file={config_path} " 33 | string += f"compute.batch_size={BATCH_SIZE} " 34 | if train: 35 | string += f"train.steps_per_epoch.train={STEPS_PER_EPOCH} train.steps_per_epoch.val={STEPS_PER_EPOCH} " 36 | string += f"train.steps_per_epoch.test={STEPS_PER_EPOCH} " 37 | string += f"train.num_epochs={NUM_EPOCHS} " 38 | string += f"train.viz_examples={VIZ_EXAMPLES}" 39 | return string 40 | 41 | 42 | # def test_python(): 43 | # command = ['which', 'python'] 44 | # ret = subprocess.run(command) 45 | 46 | # command = ['which', 'pytest'] 47 | # ret = subprocess.run(command) 48 | # # assert ret.returncode == 0 49 | # # print(ret) 50 | 51 | # print(os.environ['PATH']) 52 | # print(os.getcwd()) 53 | 54 | 55 | @pytest.mark.gpu 56 | def test_flow(): 57 | make_project_from_archive() 58 | string = "python -m deepethogram.flow_generator.train preset=deg_f " 59 | string = add_default_arguments(string) 60 | command = command_from_string(string) 61 | ret = subprocess.run(command) 62 | assert ret.returncode == 0 63 | 64 | string = "python -m deepethogram.flow_generator.train preset=deg_m " 65 | string = add_default_arguments(string) 66 | command = command_from_string(string) 67 | ret = subprocess.run(command) 68 | assert ret.returncode == 0 69 | 70 | string = "python -m deepethogram.flow_generator.train preset=deg_s " 71 | string = add_default_arguments(string) 72 | command = command_from_string(string) 73 | ret = subprocess.run(command) 74 | assert ret.returncode == 0 75 | 76 | 77 | @pytest.mark.gpu 78 | def test_feature_extractor(): 79 | string = "python -m deepethogram.feature_extractor.train preset=deg_f flow_generator.weights=latest " 80 | string = add_default_arguments(string) 81 | command = command_from_string(string) 82 | ret = subprocess.run(command) 83 | assert ret.returncode == 0 84 | 85 | string = "python -m deepethogram.feature_extractor.train preset=deg_m flow_generator.weights=latest " 86 | string = add_default_arguments(string) 87 | command = command_from_string(string) 88 | ret = subprocess.run(command) 89 | assert ret.returncode == 0 90 | 91 | # for resnet3d, must specify weights, because we can't just download them from the torchvision repo 92 | string = ( 93 | "python -m deepethogram.feature_extractor.train preset=deg_s flow_generator.weights=latest " 94 | "feature_extractor.weights=latest " 95 | ) 96 | string = add_default_arguments(string) 97 | command = command_from_string(string) 98 | ret = subprocess.run(command) 99 | assert ret.returncode == 0 100 | 101 | # testing softmax 102 | string = ( 103 | "python -m deepethogram.feature_extractor.train preset=deg_m flow_generator.weights=latest " 104 | "feature_extractor.final_activation=softmax " 105 | ) 106 | string = add_default_arguments(string) 107 | command = command_from_string(string) 108 | ret = subprocess.run(command) 109 | assert ret.returncode == 0 110 | 111 | 112 | @pytest.mark.gpu 113 | def test_feature_extraction(softmax: bool = False): 114 | # the reason for this complexity is that I don't want to run inference on all directories 115 | string = ( 116 | "python -m deepethogram.feature_extractor.inference preset=deg_f feature_extractor.weights=latest " 117 | "flow_generator.weights=latest " 118 | ) 119 | if softmax: 120 | string += "feature_extractor.final_activation=softmax " 121 | # datadir = os.path.join(testing_directory, 'DATA') 122 | subdirs = utils.get_subfiles(data_path, "directory") 123 | # np.random.seed(42) 124 | # subdirs = np.random.choice(subdirs, size=100, replace=False) 125 | dir_string = ",".join([str(i) for i in subdirs]) 126 | dir_string = "[" + dir_string + "]" 127 | string += f"inference.directory_list={dir_string} inference.overwrite=True " 128 | string = add_default_arguments(string, train=False) 129 | command = command_from_string(string) 130 | ret = subprocess.run(command) 131 | assert ret.returncode == 0 132 | # string += 'inference.directory_list=[]' 133 | 134 | 135 | @pytest.mark.gpu 136 | def test_sequence_train(): 137 | string = "python -m deepethogram.sequence.train " 138 | string = add_default_arguments(string) 139 | command = command_from_string(string) 140 | print(command) 141 | ret = subprocess.run(command) 142 | assert ret.returncode == 0 143 | 144 | # mutually exclusive 145 | string = "python -m deepethogram.sequence.train feature_extractor.final_activation=softmax " 146 | string = add_default_arguments(string) 147 | command = command_from_string(string) 148 | print(command) 149 | ret = subprocess.run(command) 150 | assert ret.returncode == 0 151 | 152 | 153 | @pytest.mark.gpu 154 | def test_softmax(): 155 | make_project_from_archive() 156 | string = "python -m deepethogram.flow_generator.train preset=deg_f " 157 | string = add_default_arguments(string) 158 | command = command_from_string(string) 159 | ret = subprocess.run(command) 160 | assert ret.returncode == 0 161 | 162 | string = ( 163 | "python -m deepethogram.feature_extractor.train preset=deg_f flow_generator.weights=latest " 164 | "feature_extractor.final_activation=softmax " 165 | ) 166 | string = add_default_arguments(string) 167 | command = command_from_string(string) 168 | ret = subprocess.run(command) 169 | assert ret.returncode == 0 170 | 171 | test_feature_extraction(softmax=True) 172 | 173 | string = "python -m deepethogram.sequence.train feature_extractor.final_activation=softmax " 174 | string = add_default_arguments(string) 175 | command = command_from_string(string) 176 | print(command) 177 | ret = subprocess.run(command) 178 | assert ret.returncode == 0 179 | 180 | 181 | if __name__ == "__main__": 182 | test_softmax() 183 | -------------------------------------------------------------------------------- /tests/test_models.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | 4 | from deepethogram.feature_extractor.models.CNN import get_cnn 5 | 6 | 7 | def test_get_cnn(): 8 | model_name = "resnet18" 9 | num_classes = 2 10 | 11 | pos = np.array([0, 300]) 12 | neg = np.array([300, 0]) 13 | 14 | model = get_cnn(model_name=model_name, num_classes=num_classes, pos=pos, neg=neg) 15 | bias = list(model.children())[-1].bias 16 | 17 | assert torch.allclose(bias, torch.Tensor([0, 1]).float()) 18 | print() 19 | -------------------------------------------------------------------------------- /tests/test_projects.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import shutil 4 | 5 | import numpy as np 6 | import pandas as pd 7 | import pytest 8 | 9 | from deepethogram import projects 10 | from setup_data import make_project_from_archive, project_path, test_data_path, clean_test_data, get_records 11 | 12 | # make_project_from_archive() 13 | 14 | 15 | def test_initialization(): 16 | clean_test_data() 17 | 18 | with pytest.raises(AssertionError): 19 | project_dict = projects.initialize_project(test_data_path, "testing", ["scratch", "itch"]) 20 | 21 | project_dict = projects.initialize_project(test_data_path, "testing", ["background", "scratch", "itch"]) 22 | # print(project_dict) 23 | # print(project_dict['project']) 24 | assert os.path.isdir(project_dict["project"]["path"]) 25 | assert project_dict["project"]["path"] == project_path 26 | 27 | data_abs = os.path.join(project_dict["project"]["path"], project_dict["project"]["data_path"]) 28 | assert os.path.isdir(data_abs) 29 | 30 | model_abs = os.path.join(project_dict["project"]["path"], project_dict["project"]["model_path"]) 31 | assert os.path.isdir(model_abs) 32 | 33 | 34 | # mouse01 tests image directories 35 | @pytest.mark.parametrize("key", ["mouse00", "mouse01"]) 36 | def test_add_video(key): 37 | make_project_from_archive() 38 | 39 | project_dict = projects.load_config(os.path.join(project_path, "project_config.yaml")) 40 | # project_dict = utils.load_yaml() 41 | key_path = os.path.join(project_path, "DATA", key) 42 | assert os.path.isdir(key_path) 43 | shutil.rmtree(key_path) 44 | 45 | records = get_records("archive") 46 | # test image directory 47 | videofile = records[key]["rgb"] 48 | print(project_dict) 49 | # this also z-scores, which is pretty slow 50 | projects.add_video_to_project(project_dict, videofile) 51 | assert os.path.isdir(os.path.join(project_path, "DATA", key)) 52 | assert os.path.exists(os.path.join(project_path, "DATA", key, os.path.basename(videofile))) 53 | 54 | 55 | @pytest.mark.parametrize("key", ["mouse00", "mouse01"]) 56 | def test_is_deg_file(key): 57 | make_project_from_archive() 58 | records = get_records() 59 | 60 | rgb = records[key]["rgb"] 61 | assert projects.is_deg_file(rgb) 62 | 63 | record_yaml = os.path.join(os.path.dirname(rgb), "record.yaml") 64 | assert os.path.isfile(record_yaml) 65 | os.remove(record_yaml) 66 | 67 | assert not projects.is_deg_file(rgb) 68 | 69 | 70 | def test_add_behavior(): 71 | make_project_from_archive() 72 | cfg_path = os.path.join(project_path, "project_config.yaml") 73 | 74 | projects.add_behavior_to_project(cfg_path, "A") 75 | records = get_records() 76 | mice = list(records.keys()) 77 | labelfile = records[random.choice(mice)]["label"] 78 | assert os.path.isfile(labelfile) 79 | df = pd.read_csv(labelfile, index_col=0) 80 | assert df.shape[1] == 6 81 | assert np.all(df.iloc[:, -1].values == -1) 82 | assert df.columns[5] == "A" 83 | 84 | 85 | def test_remove_behavior(): 86 | make_project_from_archive() 87 | cfg_path = os.path.join(project_path, "project_config.yaml") 88 | # can't remove behaviors that don't exist 89 | with pytest.raises(AssertionError): 90 | projects.remove_behavior_from_project(cfg_path, "A") 91 | 92 | # can't remove background 93 | with pytest.raises(ValueError): 94 | projects.remove_behavior_from_project(cfg_path, "background") 95 | 96 | projects.remove_behavior_from_project(cfg_path, "face_groom") 97 | 98 | records = get_records() 99 | mice = list(records.keys()) 100 | labelfile = records[random.choice(mice)]["label"] 101 | assert os.path.isfile(labelfile) 102 | df = pd.read_csv(labelfile, index_col=0) 103 | assert df.shape[1] == 4 104 | assert "face_groom" not in df.columns 105 | 106 | 107 | @pytest.mark.filterwarnings("ignore::UserWarning") 108 | def test_add_external_label(): 109 | make_project_from_archive() 110 | mousedir = os.path.join(project_path, "DATA", "mouse06") 111 | assert os.path.isdir(mousedir), "{} does not exist!".format(mousedir) 112 | labelfile = os.path.join(mousedir, "labels.csv") 113 | videofile = os.path.join(mousedir, "mouse06.h5") 114 | 115 | projects.add_label_to_project(labelfile, videofile) 116 | 117 | 118 | if __name__ == "__main__": 119 | test_add_external_label() 120 | -------------------------------------------------------------------------------- /tests/test_softmax.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jbohnslav/deepethogram/a77c77c4249b5ed027cb897f3607cdfd10d25479/tests/test_softmax.py -------------------------------------------------------------------------------- /tests/test_z_score.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from deepethogram import projects 4 | from deepethogram.zscore import get_video_statistics 5 | from setup_data import make_project_from_archive, data_path 6 | 7 | make_project_from_archive() 8 | 9 | 10 | def test_single_video(): 11 | records = projects.get_records_from_datadir(data_path) 12 | videofile = records["mouse00"]["rgb"] 13 | stats = get_video_statistics(videofile, 10) 14 | print(stats) 15 | 16 | mean = np.array([0.010965, 0.02345, 0.0161]) 17 | std = np.array([0.02623, 0.04653, 0.0349]) 18 | 19 | assert np.allclose(stats["mean"], mean, rtol=0, atol=1e-4) 20 | assert np.allclose(stats["std"], std, rtol=0, atol=1e-4) 21 | # assert stats["N"] == 1875000 22 | --------------------------------------------------------------------------------