├── .dockerignore ├── .gitattributes ├── .github └── workflows │ ├── publish_to_dockerhub.yaml │ ├── push_to_dockerhub │ ├── unittests.yaml │ └── workflow.yaml ├── .gitignore ├── .gitlab-ci.yml ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── LICENSE ├── README.md ├── build_package.sh ├── description.md ├── docker-compose.yaml ├── docker ├── Dockerfile └── build.sh ├── docs ├── .nojekyll ├── Makefile ├── gen_apidoc.bat ├── gen_apidoc.sh ├── hosting_EncoderMap.md ├── make.bat ├── run_livereload.py ├── source │ ├── _static │ │ ├── Binder_Hub.png │ │ ├── Google_Colaboratory.png │ │ ├── complete_api_ico.svg │ │ ├── custom.css │ │ ├── customization_nb_01_scalars_thumbnail.png │ │ ├── customization_nb_02_loss_thumbnail.png │ │ ├── customization_nb_03_images_thumbnail.png │ │ ├── customization_nb_04_lr_thumbnail.png │ │ ├── featurization_ico.svg │ │ ├── layers_ico.svg │ │ ├── loss_functions_ico.svg │ │ ├── md_nb_01_traj_ensemble_thumbnail.png │ │ ├── models_ico.svg │ │ ├── neural_net_ico.svg │ │ ├── parameters_ico.svg │ │ ├── robots.txt │ │ ├── single_traj_ico.svg │ │ ├── starter_nb_01_basic_thumbnail.png │ │ ├── starter_nb_02_advanced_thumbnail.png │ │ ├── starter_nb_03_your_data_thumbnail.png │ │ └── traj_ensemble_ico.svg │ ├── _templates │ │ ├── module.rst_t │ │ ├── package.rst_t │ │ └── toc.rst_t │ ├── api │ │ ├── index.rst │ │ └── trajinfo.rst │ ├── changelog │ │ └── index.rst │ ├── conf.py │ ├── contributing │ │ ├── hosting_encodermap.rst │ │ ├── index.rst │ │ ├── scripts.rst │ │ └── testing.rst │ ├── coverage_report.rst │ ├── getting_started │ │ ├── em_1_compatibility.rst │ │ ├── index.rst │ │ ├── install.rst │ │ └── link_to_readme.rst │ ├── index.rst │ ├── notebooks │ │ ├── customization_nb │ │ │ ├── customization_nb_01_scalars.nblink │ │ │ ├── customization_nb_02_loss.nblink │ │ │ ├── customization_nb_03_images.nblink │ │ │ └── customization_nb_04_lr.nblink │ │ ├── index.rst │ │ ├── intermediate_nb │ │ │ └── intermediate_nb_02_sparse.nblink │ │ ├── md_nb │ │ │ └── md_nb_01_traj_ensemble.nblink │ │ ├── starter_nb │ │ │ ├── starter_nb_01_basic.nblink │ │ │ ├── starter_nb_02_advanced.nblink │ │ │ └── starter_nb_03_your_data.nblink │ │ └── static_code.rst │ ├── refs.bib │ ├── test_report.rst │ └── user_guide │ │ ├── autoencoder_classes.rst │ │ ├── featurization.rst │ │ ├── index.rst │ │ ├── layers.rst │ │ ├── loss_functions.rst │ │ ├── models.rst │ │ ├── parameter_classes.rst │ │ ├── singletraj.rst │ │ └── trajensemble.rst └── sphinx_requirements.txt ├── encodermap ├── __init__.py ├── _typing.py ├── _version.py ├── autoencoder │ ├── __init__.py │ └── autoencoder.py ├── callbacks │ ├── __init__.py │ ├── callbacks.py │ └── metrics.py ├── encodermap_tf1 │ ├── __init__.py │ ├── angle_dihedral_cartesian_encodermap.py │ ├── autoencoder.py │ ├── backmapping.py │ ├── encodermap.py │ ├── misc.py │ ├── moldata.py │ ├── parameters.py │ └── plot.py ├── examples │ ├── cube.py │ ├── cube_distance_analysis.py │ ├── dihedral_to_cartesian_diubi.py │ ├── dihedral_to_cartesian_diubi_analysis.py │ └── trp_cage.py ├── kondata.py ├── loading │ ├── __init__.py │ ├── delayed.py │ ├── features.py │ └── featurizer.py ├── loss_functions │ ├── __init__.py │ ├── loss_classes.py │ └── loss_functions.py ├── misc │ ├── __init__.py │ ├── backmapping.c │ ├── backmapping.py │ ├── clustering.py │ ├── distances.py │ ├── function_def.py │ ├── logo_cube_300.png │ ├── misc.py │ ├── rotate.py │ ├── saving_loading_models.py │ ├── summaries.py │ ├── xarray.py │ └── xarray_save_wrong_hdf5.py ├── models │ ├── __init__.py │ ├── layers.py │ └── models.py ├── moldata │ ├── __init__.py │ └── moldata.py ├── parameters │ ├── __init__.py │ └── parameters.py ├── plot │ ├── __init__.py │ ├── assets │ │ └── favicon.ico │ ├── dashboard.py │ ├── interactive_plotting.py │ ├── jinja_template.py │ ├── plotting.py │ └── utils.py └── trajinfo │ ├── __init__.py │ ├── info_all.py │ ├── info_single.py │ ├── load_traj.py │ └── trajinfo_utils.py ├── md_requirements.txt ├── pic ├── book_pictogram.svg ├── brackets_pictogram.svg ├── distance_histogram_cube_reasonable_sigma.png ├── distance_histogram_cube_too_high_sigma.png ├── encodermap_graph_abstract.svg ├── encodermap_new_logo.svg ├── favicon.ico ├── homepage_pictograms.svg ├── logo_cube_300.png ├── logo_cube_400.png ├── logo_cube_hd.png ├── logo_m.png ├── new_logo_m.png ├── rocket_pictogram.svg ├── shield.svg ├── thumbnail.png ├── tools_pictogram.svg └── traj_ensemble_pictograms.svg ├── pyproject.toml ├── questions.md ├── requirements.txt ├── setup.py ├── tests ├── .coveragerc ├── README.md ├── __init__.py ├── conftest.py ├── data │ ├── 1GHC.pdb │ ├── 1UBQ.pdb │ ├── 1YUF.pdb │ ├── 1YUF_numpy.npy │ ├── 1YUF_text.txt │ ├── 1YUG.pdb │ ├── 1YUG_and_1YUF_dataset.nc │ ├── 1YUG_numpy.npy │ ├── 1YUG_text.txt │ ├── 1am7_center_of_mass.npy │ ├── 1am7_center_of_mass.txt │ ├── 1am7_center_of_mass_x.nc │ ├── 1am7_center_of_mass_x.npy │ ├── 1am7_corrected.xtc │ ├── 1am7_corrected_part1.xtc │ ├── 1am7_corrected_part2.xtc │ ├── 1am7_corrected_with_CVs.h5 │ ├── 1am7_first_frame_first_atom_xyz.npy │ ├── 1am7_protein.pdb │ ├── 1am7_uncorrected.xtc │ ├── Ala10_helix.pdb │ ├── Ala10_helix.xtc │ ├── Ala10_straight.pdb │ ├── PFFP_MD_fin_protonly.gro │ ├── PFFP_MD_fin_protonly.tpr │ ├── PFFP_MD_fin_protonly_dt_100.xtc │ ├── PFFP_vac.top │ ├── README.md │ ├── alanine_dipeptide.pdb │ ├── alanine_dipeptide.xtc │ ├── asp7.h5 │ ├── asp7.pdb │ ├── asp7.xtc │ ├── glu7.pdb │ ├── glu7.xtc │ ├── known_angles.h5 │ ├── known_angles.pdb │ ├── known_angles_1.h5 │ ├── known_angles_1.pdb │ ├── known_angles_2.h5 │ ├── known_angles_2.pdb │ ├── known_angles_angles.npy │ ├── known_angles_dihedrals.npy │ ├── known_angles_dists.npy │ ├── known_distances.pdb │ ├── larger_dataset.nc │ ├── out.pdb │ ├── straightened.pdb │ ├── test_chain_in_plane_expected.png │ ├── test_guess_amide_H_expected.png │ ├── test_guess_amide_O_expected.png │ ├── test_straight_to_helix_array_expected.png │ ├── test_straight_to_helix_v2_expected.png │ ├── tip3p_300K_1ATM.pdb │ ├── tip3p_300K_1ATM.xtc │ └── traj.h5 ├── debug_tests.py ├── delete_run_directories.py ├── find_long_comments.py ├── find_undocumented_package_parts.py ├── import_with_beartype.py ├── long_training.py ├── run_docbuild_test_and_cover.py ├── run_doctests.py ├── run_two_tests.py ├── test_angles.py ├── test_autoencoder.py ├── test_backmapping_em1_em2.py ├── test_dihedral_to_cartesian.py ├── test_doctests.py ├── test_featurizer.py ├── test_interactive_plotting.py ├── test_losses.py ├── test_moldata.py ├── test_non_backbone_atoms.py ├── test_notebooks.py ├── test_optional_imports.py ├── test_pairwise_distances.py ├── test_project_structure.py ├── test_requirements.txt ├── test_tf1_tf2_deterministic.py ├── test_trajinfo.py └── test_xarray.py ├── tutorials ├── Dockerfile ├── README.md ├── build.sh ├── docker │ ├── Dockerfile │ ├── README.md │ ├── grafana │ │ └── provisioning │ │ │ └── datasources │ │ │ └── prometheus_ds.yml │ ├── layout.txt │ ├── ldap_client │ │ ├── Dockerfile │ │ ├── ldap_client_entrypoint.sh │ │ ├── nfs_server_entrypoint.sh │ │ ├── nslcd.conf │ │ ├── nsswitch.conf │ │ └── pam.d │ │ │ └── common-session │ ├── ldap_server │ │ ├── custom.ldif │ │ └── ldapmodify.sh │ ├── local_template.sh │ ├── minimal_job.sh │ ├── modules │ │ ├── Dockerfile │ │ ├── GMXRC.bash │ │ ├── README.md │ │ ├── build_and_test.sh │ │ ├── cmake_modulefile │ │ ├── gromacs_modulefile │ │ ├── liblog.sh │ │ ├── profile.sh │ │ ├── test.sh │ │ └── test_modules_entrypoint.sh │ ├── sh_libs │ │ ├── libfs.sh │ │ ├── libldapclient.sh │ │ ├── liblog.sh │ │ ├── libos.sh │ │ └── libvalidations.sh │ ├── short_test.sh │ ├── slurm-old │ │ ├── README.md │ │ ├── base │ │ │ └── Dockerfile │ │ ├── cgroup.conf │ │ ├── cgroup_allowed_devices_file.conf │ │ ├── munge.key │ │ ├── slurm.conf │ │ ├── slurm_acct_db.sql │ │ ├── slurm_db │ │ │ └── Dockerfile │ │ ├── slurm_db_entrypoint.sh │ │ ├── slurm_master │ │ │ └── Dockerfile │ │ ├── slurm_master_entrypoint.sh │ │ ├── slurm_node │ │ │ └── Dockerfile │ │ ├── slurm_node_entrypoint.sh │ │ ├── slurm_version.h.in │ │ ├── slurmdbd.conf │ │ ├── ssh_config │ │ └── teardown.sh │ ├── slurm_base │ │ ├── Dockerfile │ │ ├── cgroup.conf │ │ ├── cgroup_allowed_devices_file.conf │ │ └── slurm.conf │ ├── slurm_db │ │ ├── Dockerfile │ │ ├── slurm_acct_db.sql │ │ ├── slurm_db_entrypoint.sh │ │ └── slurmdbd.conf │ ├── slurm_master │ │ ├── Dockerfile │ │ └── slurm_master_entrypoint.sh │ ├── slurm_node │ │ ├── Dockerfile │ │ ├── run_dask_tests_on_slurm.sh │ │ ├── slurm_node_entrypoint.sh │ │ └── test_slurm_node.sh │ ├── slurm_node_w_prometheus │ │ ├── Dockerfile │ │ ├── nodes.json │ │ ├── prometheus │ │ │ └── prometheus.yml │ │ ├── queued.json │ │ ├── running.json │ │ └── slurm_node_w_prometheus_entrypoint.sh │ ├── slurm_web │ │ └── Dockerfile │ ├── ssh │ │ └── ssh_config │ ├── start_slurm.sh │ ├── teardown.sh │ ├── template.sh │ ├── test_in_docker.sh │ └── water_simulation │ │ ├── mdp │ │ ├── eql.mdp │ │ ├── eql2.mdp │ │ ├── min.mdp │ │ └── min2.mdp │ │ ├── production.mdp │ │ ├── production.tpr │ │ ├── production_fails.tpr │ │ ├── production_short.mdp │ │ ├── production_short.tpr │ │ ├── spc216.gro │ │ ├── spc216_box.gro │ │ ├── spc216_fails.gro │ │ ├── spc216_stacked.gro │ │ ├── topol.top │ │ └── topol_stacked.top ├── install_encodermap_google_colab.sh ├── notebooks_MD │ ├── 01_Introduction_to_Trajectory_Ensembles.ipynb │ ├── 01_Understanding_MD.ipynb │ ├── 01_Working_with_trajectory_ensembles.ipynb │ ├── 02_Ub_mutants.ipynb │ ├── CV_overview.png │ └── topology_examples.png ├── notebooks_customization │ ├── 01_custom_scalars_to_tensorboard.ipynb │ ├── 02_custom_loss_functions.ipynb │ ├── 03_custom_images.ipynb │ ├── 04_learning_rate_schedulers.ipynb │ ├── asp7.csv │ ├── custom_images_1.png │ ├── custom_images_2.png │ ├── custom_loss_functions_1.png │ ├── custom_scalars_1.png │ ├── custom_scalars_2.png │ ├── lr_scheduler_1.png │ └── lr_scheduler_2.png ├── notebooks_intermediate │ ├── 01_explanation_of_loss_functions.ipynb │ ├── 02_training_with_different_topologies.ipynb │ ├── 03_dask_featurization.ipynb │ ├── asp7.pdb │ ├── asp7.xtc │ ├── glu7.pdb │ ├── glu7.xtc │ └── path.npy ├── notebooks_starter │ ├── 01_Basic_Usage-Cube_Example.ipynb │ ├── 02_Advanced_Usage-Asp7_Example.ipynb │ ├── 03_Your_Data.ipynb │ ├── Tensorboard_Cost.png │ ├── Tensorboard_Histograms.png │ ├── Tensorboard_Images.png │ ├── Tensorboard_Parameters.png │ ├── asp7.csv │ ├── asp7.pdb │ └── asp7.xtc ├── notebooks_tensorflow1 │ ├── 01_Basic_Usage-Cube_Example.ipynb │ ├── 02_Advanced_Usage-Asp7_Example.ipynb │ ├── 03_Your_Data.ipynb │ ├── asp7.csv │ ├── asp7.pdb │ └── clear_outputs.sh └── run.sh ├── versioneer.py └── vmd └── sscache.tcl /.dockerignore: -------------------------------------------------------------------------------- 1 | * 2 | tests/ 3 | docs/ 4 | development/ 5 | !requirements.txt 6 | !md_requirements.txt 7 | !encodermap 8 | !setup.py 9 | !pyproject.toml 10 | !description.md 11 | !versioneer.py 12 | !tests/data/glu7.xtc 13 | !tests/data/glu7.pdb 14 | !tests/test_requirements.txt 15 | !tutorials/notebooks_starter/*.ipynb 16 | !tutorials/notebooks_intermediate/*.ipynb 17 | !tutorials/notebooks_MD/*.ipynb 18 | !tutorials/notebooks_customization/*.ipynb 19 | 20 | !tutorials/docker/slurm_node_w_prometheus 21 | !tutorials/docker/slurm_node 22 | !tutorials/docker/slurm_master 23 | !tutorials/docker/slurm_database 24 | !tutorials/docker/slurm_base 25 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | encodermap/_version.py export-subst 2 | -------------------------------------------------------------------------------- /.github/workflows/publish_to_dockerhub.yaml: -------------------------------------------------------------------------------- 1 | name: publish_to_dockerhub 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | 8 | jobs: 9 | push_to_dockerhub: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | name: Check out code 14 | 15 | - uses: mr-smithers-excellent/docker-build-push@v5 16 | name: Build & push Docker image 17 | with: 18 | image: agpeter/encodermap 19 | tags: latest 20 | registry: hub.docker.com 21 | dockerfile: Dockerfile 22 | username: ${{ secrets.DOCKERHUB_USERNAME }} 23 | password: ${{ secrets.DOCKERHUB_TOKEN }} 24 | -------------------------------------------------------------------------------- /.github/workflows/push_to_dockerhub: -------------------------------------------------------------------------------- 1 | # name: push_to_dockerhub 2 | # 3 | # on: 4 | # push: 5 | # branches: 6 | # - 'main' 7 | # 8 | # jobs: 9 | # docker: 10 | # runs-on: ubuntu-latest 11 | # steps: 12 | # - 13 | # name: Set up QEMU 14 | # uses: docker/setup-qemu-action@v2 15 | # - 16 | # name: Set up Docker Buildx 17 | # uses: docker/setup-buildx-action@v2 18 | # - 19 | # name: Login to Docker Hub 20 | # uses: docker/login-action@v2 21 | # with: 22 | # username: ${{ secrets.DOCKERHUB_USERNAME }} 23 | # password: ${{ secrets.DOCKERHUB_TOKEN }} 24 | # - 25 | # name: Build and push 26 | # uses: docker/build-push-action@v4 27 | # with: 28 | # push: true 29 | # tags: agpeter/encodermap:latest 30 | -------------------------------------------------------------------------------- /.github/workflows/unittests.yaml: -------------------------------------------------------------------------------- 1 | # This is a basic workflow to help you get started with Actions 2 | 3 | name: Unittests 4 | 5 | # Controls when the action will run. 6 | on: 7 | # Triggers the workflow on push or pull request events but only for the main branch 8 | push: 9 | branches: [ main ] 10 | pull_request: 11 | branches: [ main ] 12 | 13 | # Allows you to run this workflow manually from the Actions tab 14 | workflow_dispatch: 15 | 16 | # A workflow run is made up of one or more jobs that can run sequentially or in parallel 17 | jobs: 18 | # This workflow contains a single job called "build" 19 | tests: 20 | # The type of runner that the job will run on 21 | runs-on: ubuntu-latest 22 | strategy: 23 | matrix: 24 | python-version: [3.9] 25 | 26 | # Steps represent a sequence of tasks that will be executed as part of the job 27 | steps: 28 | # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it 29 | - name: Checkout 🛎️ 30 | uses: actions/checkout@v3 31 | 32 | - name: Set up Python ${{ matrix.python-version }} 33 | uses: actions/setup-python@v4 34 | with: 35 | python-version: ${{ matrix.python-version }} 36 | 37 | - name: Install dependencies 🔧 38 | run: | 39 | python -m pip install --upgrade pip 40 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 41 | pip install . 42 | 43 | - name: Install MD dependencies 🔧 44 | run: | 45 | if [ -f md_requirements.txt ]; then pip install -r md_requirements.txt ; fi 46 | if [ -f tests/test_requirements.txt ]; then pip install -r tests/test_requirements.txt ; fi 47 | 48 | - name: Run Unittests 🧪 and Coverage 🐍 49 | env: 50 | ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' 51 | run: | 52 | echo "Fixing HtmlTestRunner" 53 | HTML_TEST_RUNNER_FILE=$( pip show html-testRunner | sed -n '8p' | awk '{print $2}' ) 54 | HTML_TEST_RUNNER_FILE="$HTML_TEST_RUNNER_FILE/HtmlTestRunner/result.py" 55 | sed -i "s/length = self._count_relevant_tb_levels(tb)/# length = self._count_relevant_tb_levels(tb)/g" $HTML_TEST_RUNNER_FILE 56 | sed -i "s/msg_lines = traceback.format_exception(exctype, value, tb, length)/msg_lines = traceback.format_exception(exctype, value, tb)/g" $HTML_TEST_RUNNER_FILE 57 | else 58 | echo "No previous Test results to compare this one with" 59 | fi 60 | out=$( python tests/run_docbuild_test_and_cover.py -t) 61 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | stages: 2 | - test 3 | - deploy 4 | 5 | image: python:3.8.5 6 | 7 | before_script: 8 | - export PYTHONPATH=src 9 | - python3 --version 10 | - python3 -m pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.2.0-cp38-cp38-manylinux2010_x86_64.whl 11 | - python3 -m pip install --upgrade --no-binary numpy==1.19.0 numpy==1.19.0 12 | - pip install -r requirements.txt 13 | - pip uninstall -y numpy 14 | - pip install numpy==1.19.0 15 | 16 | test: 17 | stage: test 18 | script: 19 | - ls 20 | - python -m unittest discover -s tests -v 21 | only: 22 | - tf2 23 | 24 | pages: 25 | stage: deploy 26 | script: 27 | - mkdir .public 28 | - cp -r docs/build/html/* .public 29 | - mv .public public 30 | artifacts: 31 | paths: 32 | - public 33 | only: 34 | - tf2 35 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | # See https://pre-commit.com for more information 2 | # See https://pre-commit.com/hooks.html for more hooks 3 | repos: 4 | - repo: https://github.com/pre-commit/pre-commit-hooks 5 | rev: v4.5.0 6 | hooks: 7 | - id: trailing-whitespace 8 | - id: check-added-large-files 9 | exclude: 'tutorials/notebooks_starter/asp7.csv|tutorials/notebooks_starter/asp7.xtc|tutoials/notebooks_starter/Tensorboard_Images.png|tutorials/notebooks_intermediate/glu7.xtc|tests/data/1am7_corrected_with_CVs.h5|tests/data/PFFP_MD_fin_protonly.tpr|tests/data/1YUF.pdb|tests/data/tip3p_300K_1ATM.xtc|tests/data/1GHC.pdb|tests/data/1YUG.pdb|tests/data/messy_dataset.nc|weights_and_biases_m1_diUbq.npz|tutorials/docker/water_simulation/spc216_stacked.gro|tutorials/docker/water_simulation/spc216_fails.gro' 10 | - id: end-of-file-fixer 11 | - id: check-ast 12 | exclude: development/ 13 | - id: check-yaml 14 | exclude: .github/ 15 | - id: detect-private-key 16 | 17 | #- repo: https://github.com/pre-commit/mirrors-mypy 18 | # rev: v0.910-1 # Use the sha / tag you want to point at 19 | # hooks: 20 | # - id: mypy 21 | # args: [--config-file pyproject.toml] 22 | 23 | - repo: https://github.com/pycqa/isort 24 | rev: 5.13.2 25 | hooks: 26 | - id: isort 27 | name: isort (python) 28 | args: ["--profile", "black", "--project", "encodermap", "--filter-files"] 29 | 30 | - repo: https://github.com/psf/black 31 | rev: 24.2.0 32 | hooks: 33 | - id: black 34 | args: [--config=./pyproject.toml] 35 | 36 | # - repo: https://github.com/jendrikseipp/vulture 37 | # rev: v2.3 # or any later Vulture version 38 | # hooks: 39 | # - id: vulture 40 | 41 | - repo: https://github.com/kevinsawade/pre-commit-hooks 42 | rev: 0.0.6 # new pushed of my repos are available as @latest 43 | hooks: 44 | - id: clear-ipynb-cells 45 | additional_dependencies: ["notebook==6.4.12", "traitlets==5.9.0"] 46 | - id: assert-version-advance 47 | args: [ --branch=main, --remote=public ] 48 | # - id: run-pycodestyle 49 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, gender identity and expression, level of experience, 9 | nationality, personal appearance, race, religion, or sexual identity and 10 | orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at kevin.sawade@uni-konstanz.de. All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at [http://contributor-covenant.org/version/1/4][version] 72 | 73 | [homepage]: http://contributor-covenant.org 74 | [version]: http://contributor-covenant.org/version/1/4/ 75 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # inherit from python's base image 2 | FROM python:3.12 3 | LABEL maintainer="kevinsawade" version="0.0.1" 4 | 5 | # update pip 6 | RUN apt-get update 7 | RUN pip install --upgrade pip 8 | 9 | # create non-root user 10 | RUN adduser encodermap_user 11 | 12 | # according to the 13 | WORKDIR /app 14 | 15 | # install encodermap in /app 16 | COPY requirements.txt /app/requirements.txt 17 | COPY md_requirements.txt /app/md_requirements.txt 18 | RUN pip install --no-cache-dir -r requirements.txt 19 | RUN pip install --no-cache-dir -r md_requirements.txt 20 | RUN pip install --no-cache-dir notebook ipython 21 | 22 | # copy encodermap 23 | ADD encodermap /app/encodermap 24 | COPY setup.py /app/setup.py 25 | COPY pyproject.toml /app/pyproject.toml 26 | COPY versioneer.py /app/versioneer.py 27 | COPY description.md /app/description.md 28 | COPY requirements.txt /app/requirements.txt 29 | RUN pip install . 30 | 31 | # change into homne and copy tutorials 32 | ADD tutorials /home/encodermap_user/tutorials 33 | WORKDIR /home/encodermap_user/tutorials 34 | 35 | # expose tensorboard and jupyter notebook port port 36 | EXPOSE 6006 37 | EXPOSE 8888 38 | EXPOSE 8787 39 | 40 | # run 41 | # CMD ["which", "python"] 42 | CMD ["/usr/local/bin/python", "-m", "notebook", "--ip=0.0.0.0", "--port=8888", "--no-browser", "--NotebookApp.token=''", "--NotebookApp.password=''", "--allow-root"] 43 | -------------------------------------------------------------------------------- /build_package.sh: -------------------------------------------------------------------------------- 1 | python3 -m pip install --user --upgrade setuptools wheel 2 | python3 setup.py sdist bdist_wheel 3 | -------------------------------------------------------------------------------- /description.md: -------------------------------------------------------------------------------- 1 | # EncoderMap 2 | Dimensionality reduction and generation of high-dimensional data with autoencoders 3 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tensorflow/tensorflow:1.13.1-gpu-py3 2 | 3 | RUN pip install encodermap 4 | 5 | EXPOSE 6006 6 | -------------------------------------------------------------------------------- /docker/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | docker build --tag encodermap_gpu . 3 | -------------------------------------------------------------------------------- /docs/.nojekyll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/.nojekyll -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/gen_apidoc.bat: -------------------------------------------------------------------------------- 1 | sphinx-apidoc -f -P -o source ../encodermap/ ../encodermap/examples/ ../encodermap/misc/transormations.py -V 1.0.0 -H EncoderMap 2 | make html 3 | -------------------------------------------------------------------------------- /docs/gen_apidoc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | sphinx-apidoc -f -P -o source ../encodermap/ ../encodermap/examples/ -V 3.0.0 -H EncoderMap --templatedir _templates 3 | # make html 4 | # make pdf 5 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/run_livereload.py: -------------------------------------------------------------------------------- 1 | # Standard Library Imports 2 | from pathlib import Path 3 | 4 | # Third Party Imports 5 | from livereload import Server, shell 6 | 7 | 8 | if __name__ == "__main__": 9 | server = Server() 10 | files = list(Path(__file__).parent.rglob("*.rst")) 11 | files.extend(list(Path(__file__).parent.rglob("*.nblink"))) 12 | files.extend(list(Path(__file__).parent.rglob("*.ipynb"))) 13 | files.extend(list(Path(__file__).parent.parent.rglob("*.py"))) 14 | files.extend(list(Path(__file__).parent.rglob("*.md"))) 15 | files.extend(list((Path(__file__).parent / "_static").glob("*"))) 16 | for file in files: 17 | server.watch( 18 | str(file), 19 | shell("make html"), 20 | delay=1, 21 | ) 22 | server.serve(root="build/html") 23 | -------------------------------------------------------------------------------- /docs/source/_static/Binder_Hub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/Binder_Hub.png -------------------------------------------------------------------------------- /docs/source/_static/Google_Colaboratory.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/Google_Colaboratory.png -------------------------------------------------------------------------------- /docs/source/_static/customization_nb_01_scalars_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/customization_nb_01_scalars_thumbnail.png -------------------------------------------------------------------------------- /docs/source/_static/customization_nb_02_loss_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/customization_nb_02_loss_thumbnail.png -------------------------------------------------------------------------------- /docs/source/_static/customization_nb_03_images_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/customization_nb_03_images_thumbnail.png -------------------------------------------------------------------------------- /docs/source/_static/customization_nb_04_lr_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/customization_nb_04_lr_thumbnail.png -------------------------------------------------------------------------------- /docs/source/_static/layers_ico.svg: -------------------------------------------------------------------------------- 1 | 2 | 65 | -------------------------------------------------------------------------------- /docs/source/_static/loss_functions_ico.svg: -------------------------------------------------------------------------------- 1 | 2 | 56 | -------------------------------------------------------------------------------- /docs/source/_static/md_nb_01_traj_ensemble_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/md_nb_01_traj_ensemble_thumbnail.png -------------------------------------------------------------------------------- /docs/source/_static/robots.txt: -------------------------------------------------------------------------------- 1 | User-agent: * 2 | Disallow: / 3 | -------------------------------------------------------------------------------- /docs/source/_static/starter_nb_01_basic_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/starter_nb_01_basic_thumbnail.png -------------------------------------------------------------------------------- /docs/source/_static/starter_nb_02_advanced_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/starter_nb_02_advanced_thumbnail.png -------------------------------------------------------------------------------- /docs/source/_static/starter_nb_03_your_data_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/docs/source/_static/starter_nb_03_your_data_thumbnail.png -------------------------------------------------------------------------------- /docs/source/_templates/module.rst_t: -------------------------------------------------------------------------------- 1 | {%- if show_headings %} 2 | {{- [basename, "module"] | join(' ') | e | heading }} 3 | 4 | {% endif -%} 5 | .. automodule:: {{ qualname }} 6 | {%- for option in automodule_options %} 7 | :{{ option }}: 8 | {%- endfor %} 9 | -------------------------------------------------------------------------------- /docs/source/_templates/package.rst_t: -------------------------------------------------------------------------------- 1 | {%- macro automodule(modname, options) -%} 2 | .. automodule:: {{ modname }} 3 | {%- for option in options %} 4 | :{{ option }}: 5 | {%- endfor %} 6 | {%- endmacro %} 7 | 8 | {%- macro toctree(docnames) -%} 9 | .. toctree:: 10 | :maxdepth: {{ maxdepth }} 11 | {% for docname in docnames %} 12 | {{ docname }} 13 | {%- endfor %} 14 | {%- endmacro %} 15 | 16 | {%- if is_namespace %} 17 | {{- [pkgname, "namespace"] | join(" ") | e | heading }} 18 | {% else %} 19 | {{- [pkgname, "package"] | join(" ") | e | heading }} 20 | {% endif %} 21 | 22 | {%- if modulefirst and not is_namespace %} 23 | {{ automodule(pkgname, automodule_options) }} 24 | {% endif %} 25 | 26 | {%- if subpackages %} 27 | Subpackages 28 | ----------- 29 | 30 | {{ toctree(subpackages) }} 31 | {% endif %} 32 | 33 | {%- if submodules %} 34 | Submodules 35 | ---------- 36 | {% if separatemodules %} 37 | {{ toctree(submodules) }} 38 | {% else %} 39 | {%- for submodule in submodules %} 40 | {% if show_headings %} 41 | {{- [submodule, "module"] | join(" ") | e | heading(2) }} 42 | {% endif %} 43 | {{ automodule(submodule, automodule_options) }} 44 | {% endfor %} 45 | {%- endif %} 46 | {%- endif %} 47 | 48 | {%- if not modulefirst and not is_namespace %} 49 | Module contents 50 | --------------- 51 | 52 | {{ automodule(pkgname, automodule_options) }} 53 | {% endif %} 54 | -------------------------------------------------------------------------------- /docs/source/_templates/toc.rst_t: -------------------------------------------------------------------------------- 1 | {{ header | heading }} 2 | 3 | .. toctree:: 4 | :maxdepth: {{ maxdepth }} 5 | {% for docname in docnames %} 6 | {{ docname }} 7 | {%- endfor %} 8 | -------------------------------------------------------------------------------- /docs/source/api/index.rst: -------------------------------------------------------------------------------- 1 | .. _api/index: 2 | 3 | API 4 | === 5 | 6 | Here follows a complete collection of EncoderMap's low level functions and classes. These are mostly interlinked with the documentation in :ref:`user_guide/index`. 7 | 8 | .. toctree:: 9 | 10 | trajinfo 11 | -------------------------------------------------------------------------------- /docs/source/api/trajinfo.rst: -------------------------------------------------------------------------------- 1 | .. _api/trajinfo: 2 | 3 | trajinfo.info_single 4 | -------------------- 5 | 6 | .. automodule:: encodermap.trajinfo.info_single 7 | :members: 8 | :exclude-members: SingleTraj 9 | :ignore-module-all: 10 | :undoc-members: 11 | 12 | trajinfo.trajinfo_utils 13 | ----------------------- 14 | 15 | .. automodule:: encodermap.trajinfo.trajinfo_utils 16 | :members: 17 | :ignore-module-all: 18 | :undoc-members: 19 | -------------------------------------------------------------------------------- /docs/source/changelog/index.rst: -------------------------------------------------------------------------------- 1 | .. _changelog/index: 2 | 3 | .. include:: ../../../CHANGELOG.md 4 | :parser: myst_parser.sphinx_ 5 | -------------------------------------------------------------------------------- /docs/source/contributing/hosting_encodermap.rst: -------------------------------------------------------------------------------- 1 | .. _contributing/hosting_encodermap: 2 | 3 | .. include:: ../../hosting_EncoderMap.md 4 | :parser: myst_parser.sphinx_ 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 3 9 | -------------------------------------------------------------------------------- /docs/source/contributing/index.rst: -------------------------------------------------------------------------------- 1 | .. _contributing/index: 2 | 3 | .. include:: ../../../CONTRIBUTING.md 4 | :parser: myst_parser.sphinx_ 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 3 9 | :titlesonly: 10 | :hidden: 11 | 12 | testing 13 | hosting_encodermap 14 | scripts 15 | -------------------------------------------------------------------------------- /docs/source/contributing/scripts.rst: -------------------------------------------------------------------------------- 1 | .. _contributing/scripts: 2 | 3 | Scripts 4 | ======= 5 | 6 | 7 | EncoderMap has a few scripts, that can help with development. 8 | 9 | 10 | .. click:: tests.run_docbuild_test_and_cover:main 11 | :prog: run_docbuild_test_and_cover 12 | :nested: full 13 | 14 | 15 | .. click:: tests.run_doctests:main 16 | :prog: run_doctests 17 | :nested: full 18 | 19 | 20 | .. click:: tests:run_two_tests 21 | :prog: main 22 | :nested: full 23 | 24 | 25 | .. click:: tests:find_long_comments 26 | :prog: main 27 | :nested: full 28 | 29 | 30 | .. click:: tests:find_undocumented_package_parts 31 | :prog: main 32 | :nested: full 33 | 34 | 35 | .. click:: tests:long_training 36 | :prog: main 37 | :nested: full 38 | 39 | 40 | .. click:: tests:run_livereload 41 | :prog: main 42 | :nested: full 43 | -------------------------------------------------------------------------------- /docs/source/contributing/testing.rst: -------------------------------------------------------------------------------- 1 | .. _contributing/testing: 2 | 3 | .. include:: ../../../tests/README.md 4 | :parser: myst_parser.sphinx_ 5 | 6 | 7 | .. toctree:: 8 | :maxdepth: 3 9 | -------------------------------------------------------------------------------- /docs/source/coverage_report.rst: -------------------------------------------------------------------------------- 1 | .. _coverage_report: 2 | 3 | .. raw:: html 4 | :file: _static/coverage/index.html 5 | -------------------------------------------------------------------------------- /docs/source/getting_started/em_1_compatibility.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started/em_1_compatibility: 2 | 3 | EncoderMap 1.0 and 2.0 Compatibility 4 | ------------------------------------ 5 | 6 | Version 2 of the TensorFlow package brought some changes with it. While TensorFlow 1 generally has higher performance, TensorFlow 2 allows for more customization. EncoderMap 3.0 leverages this new feature to allow you to tune the model and the training to your liking. With the new EncoderMap it is even easier to understand what your MD data contains and how it is trained in EncoderMap's Neural Network. 7 | 8 | Here's how TensorFlow 2 code looks like with EncoderMap 3.0 looks like. 9 | 10 | Tensorflow 2 11 | ~~~~~~~~~~~~ 12 | 13 | .. code-block:: python 14 | 15 | >>> import tensorflow as tf 16 | >>> tf.__version__ 17 | 2.16 18 | >>> tf.executing_eagerly() 19 | True 20 | >>> import encodermap as em 21 | >>> encodermap.__version__ 22 | 3.0 23 | 24 | If you come from EncoderMap 1.0/2.0 and want to run your old EncoderMap code and gradually shift to the new TensorFlow version you can use EncoderMap's compatibility mode like so: 25 | 26 | Tensorflow 2 in Tensorflow 1 compatibility mode 27 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 28 | 29 | .. code-block:: python 30 | 31 | >>> import tensorflow as tf2 32 | >>> import tensorflow.compat.v1 as tf 33 | >>> tf.disable_eager_execution() 34 | >>> tf2.executing_eagerly() 35 | False 36 | >>> import encodermap.encodermap_tf1 as em 37 | >>> encodermap.__version__ 38 | 2.0 39 | -------------------------------------------------------------------------------- /docs/source/getting_started/index.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started/index: 2 | 3 | Getting started 4 | =============== 5 | 6 | These pages help you with setting up EncoderMap on your computer and provide a minimal example, that you can run to verify your installation. 7 | 8 | .. toctree:: 9 | 10 | install 11 | em_1_compatibility 12 | link_to_readme 13 | -------------------------------------------------------------------------------- /docs/source/getting_started/link_to_readme.rst: -------------------------------------------------------------------------------- 1 | .. _getting_started/link_to_readme: 2 | 3 | Review the README 4 | ================= 5 | 6 | .. include:: ../../../README.md 7 | :parser: myst_parser.sphinx_ 8 | -------------------------------------------------------------------------------- /docs/source/notebooks/customization_nb/customization_nb_01_scalars.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../../tutorials/notebooks_customization/01_custom_scalars_to_tensorboard.ipynb", 3 | "extra-media": [ 4 | "../../../../tutorials/notebooks_customization/custom_scalars_1.png", 5 | "../../../../tutorials/notebooks_customization/custom_scalars_2.png", 6 | "../../../../tutorials/notebooks_starter/asp7.csv" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /docs/source/notebooks/customization_nb/customization_nb_02_loss.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../../tutorials/notebooks_customization/02_custom_loss_functions.ipynb", 3 | "extra-media": [ 4 | "../../../../tutorials/notebooks_customization/custom_loss_functions_1.png", 5 | "../../../../tutorials/notebooks_starter/asp7.csv" 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /docs/source/notebooks/customization_nb/customization_nb_03_images.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../../tutorials/notebooks_customization/03_custom_images.ipynb", 3 | "extra-media": [ 4 | "../../../../tutorials/notebooks_customization/custom_images_1.png", 5 | "../../../../tutorials/notebooks_customization/custom_images_2.png", 6 | "../../../../tutorials/notebooks_starter/asp7.csv" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /docs/source/notebooks/customization_nb/customization_nb_04_lr.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../../tutorials/notebooks_customization/04_learning_rate_schedulers.ipynb", 3 | "extra-media": [ 4 | "../../../../tutorials/notebooks_customization/lr_scheduler_1.png", 5 | "../../../../tutorials/notebooks_customization/lr_scheduler_2.png", 6 | "../../../../tutorials/notebooks_starter/asp7.csv" 7 | ] 8 | } 9 | -------------------------------------------------------------------------------- /docs/source/notebooks/intermediate_nb/intermediate_nb_02_sparse.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../tutorials/notebooks_intermediate/02_training_with_different_topologies.ipynb", 3 | "extra-media": [ 4 | "../../../tutorials/notebooks_starter/asp7.pdb", 5 | "../../../tutorials/notebooks_starter/asp7.xtc", 6 | "../../../tutorials/notebooks_intermediate/glu7.xtc", 7 | "../../../tutorials/notebooks_intermediate/glu7.pdb", 8 | "../../../tutorials/notebooks_intermediate/path.npy" 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /docs/source/notebooks/md_nb/md_nb_01_traj_ensemble.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../../tutorials/notebooks_MD/01_Introduction_to_Trajectory_Ensembles.ipynb" 3 | } 4 | -------------------------------------------------------------------------------- /docs/source/notebooks/starter_nb/starter_nb_01_basic.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../../tutorials/notebooks_starter/01_Basic_Usage-Cube_Example.ipynb" 3 | } 4 | -------------------------------------------------------------------------------- /docs/source/notebooks/starter_nb/starter_nb_02_advanced.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../../tutorials/notebooks_starter/02_Advanced_Usage-Asp7_Example.ipynb", 3 | "extra-media": [ 4 | "../../../../tutorials/notebooks_starter/asp7.csv", 5 | "../../../../tutorials/notebooks_starter/asp7.pdb", 6 | "../../../../tutorials/notebooks_starter/asp7.xtc", 7 | "../../../../tutorials/notebooks_starter/Tensorboard_Cost.png", 8 | "../../../../tutorials/notebooks_starter/Tensorboard_Histograms.png", 9 | "../../../../tutorials/notebooks_starter/Tensorboard_Parameters.png", 10 | "../../../../tutorials/notebooks_starter/Tensorboard_Images.png" 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /docs/source/notebooks/starter_nb/starter_nb_03_your_data.nblink: -------------------------------------------------------------------------------- 1 | { 2 | "path" : "../../../../tutorials/notebooks_starter/03_Your_Data.ipynb" 3 | } 4 | -------------------------------------------------------------------------------- /docs/source/notebooks/static_code.rst: -------------------------------------------------------------------------------- 1 | .. _notebooks/static_code: 2 | 3 | Static Code Examples 4 | ==================== 5 | 6 | These static code examples can also help you with your EncoderMap endeavors. 7 | 8 | Cube Example 9 | ------------ 10 | 11 | This is the classic EncoderMap example. In this example, EncoderMap is used to project a 3D cube into 2D and back. 12 | 13 | .. dropdown:: Cube Example 14 | 15 | .. literalinclude:: ../../../encodermap/examples/cube.py 16 | :language: python 17 | 18 | 19 | Cube Distance Analysis 20 | ---------------------- 21 | 22 | In this example a way of visualizing the mapping of high-dimensional differences to low-dimensional distances is introduced. 23 | 24 | 25 | .. dropdown:: Cube Distance Analysis 26 | 27 | .. literalinclude:: ../../../encodermap/examples/cube_distance_analysis.py 28 | :language: python 29 | 30 | 31 | Trp Cage 32 | -------- 33 | 34 | In this example, you can work with actual MD data from simulations of the Trp-cage protein. 35 | 36 | 37 | .. dropdown:: Trp Cage 38 | 39 | .. literalinclude:: ../../../encodermap/examples/trp_cage.py 40 | :language: python 41 | 42 | 43 | Dihedral to Cartesian diUbi 44 | --------------------------- 45 | 46 | In this example, you can work with MD data from simulations of the M1-linked diUbi protein. 47 | 48 | .. dropdown:: Data Generation 49 | 50 | .. literalinclude:: ../../../encodermap/examples/dihedral_to_cartesian_diubi.py 51 | :language: python 52 | 53 | 54 | .. dropdown:: Data Analysis 55 | 56 | .. literalinclude:: ../../../encodermap/examples/dihedral_to_cartesian_diubi_analysis.py 57 | :language: python 58 | -------------------------------------------------------------------------------- /docs/source/refs.bib: -------------------------------------------------------------------------------- 1 | @article{lemke2019encodermap, 2 | title={Encodermap: Dimensionality reduction and generation of molecule conformations}, 3 | author={Lemke, Tobias and Peter, Christine}, 4 | journal={Journal of chemical theory and computation}, 5 | volume={15}, 6 | number={2}, 7 | pages={1209--1215}, 8 | year={2019}, 9 | publisher={ACS Publications} 10 | } 11 | 12 | @article{lemke2019encodermap2, 13 | title={EncoderMap (II): Visualizing important molecular motions with improved generation of protein conformations}, 14 | author={Lemke, Tobias and Berg, Andrej and Jain, Alok and Peter, Christine}, 15 | journal={Journal of Chemical Information and Modeling}, 16 | volume={59}, 17 | number={11}, 18 | pages={4550--4560}, 19 | year={2019}, 20 | publisher={ACS Publications} 21 | } 22 | -------------------------------------------------------------------------------- /docs/source/test_report.rst: -------------------------------------------------------------------------------- 1 | .. _test_report: 2 | 3 | 4 | Test Report 5 | =========== 6 | 7 | .. raw:: html 8 | :file: _static/html_test_runner_report.html 9 | -------------------------------------------------------------------------------- /docs/source/user_guide/autoencoder_classes.rst: -------------------------------------------------------------------------------- 1 | .. _user_guide/autoencoder_classes: 2 | 3 | Autoencoder Classes 4 | =================== 5 | 6 | Autoencoder 7 | ----------- 8 | 9 | .. autoclass:: encodermap.autoencoder.autoencoder.Autoencoder 10 | :class-doc-from: both 11 | :private-members: 12 | :special-members: __init__ 13 | :member-order: groupwise 14 | 15 | EncoderMap 16 | ---------- 17 | 18 | .. autoclass:: encodermap.autoencoder.autoencoder.EncoderMap 19 | :class-doc-from: both 20 | :private-members: 21 | :special-members: __init__ 22 | :member-order: groupwise 23 | 24 | AngleDihedralCartesianEncoderMap 25 | -------------------------------- 26 | 27 | .. autoclass:: encodermap.autoencoder.autoencoder.AngleDihedralCartesianEncoderMap 28 | :class-doc-from: both 29 | :private-members: 30 | :special-members: __init__ 31 | :member-order: groupwise 32 | -------------------------------------------------------------------------------- /docs/source/user_guide/featurization.rst: -------------------------------------------------------------------------------- 1 | .. _user_guide/featurization: 2 | 3 | Featurization 4 | ============= 5 | 6 | .. autoclass:: encodermap.loading.featurizer.Featurizer 7 | :class-doc-from: both 8 | :private-members: 9 | :special-members: __init__ 10 | :member-order: groupwise 11 | 12 | 13 | Features 14 | -------- 15 | 16 | .. automodule:: encodermap.loading.features 17 | :members: 18 | :undoc-members: 19 | :show-inheritance: 20 | :private-members: 21 | :ignore-module-all: 22 | -------------------------------------------------------------------------------- /docs/source/user_guide/layers.rst: -------------------------------------------------------------------------------- 1 | .. _user_guide/layers: 2 | 3 | Layers 4 | ====== 5 | 6 | .. automodule:: encodermap.models.layers 7 | :members: 8 | :undoc-members: 9 | :show-inheritance: 10 | :private-members: 11 | :ignore-module-all: 12 | -------------------------------------------------------------------------------- /docs/source/user_guide/loss_functions.rst: -------------------------------------------------------------------------------- 1 | .. _user_guide/loss_functions: 2 | 3 | Loss functions 4 | ============== 5 | 6 | .. automodule:: encodermap.loss_functions.loss_functions 7 | :members: 8 | :undoc-members: 9 | :show-inheritance: 10 | :private-members: 11 | :ignore-module-all: 12 | -------------------------------------------------------------------------------- /docs/source/user_guide/models.rst: -------------------------------------------------------------------------------- 1 | .. _user_guide/models: 2 | 3 | Models 4 | ====== 5 | 6 | .. automodule:: encodermap.models.models 7 | :members: 8 | :undoc-members: 9 | :show-inheritance: 10 | :private-members: 11 | :ignore-module-all: 12 | -------------------------------------------------------------------------------- /docs/source/user_guide/parameter_classes.rst: -------------------------------------------------------------------------------- 1 | .. _user_guide/parameter_classes: 2 | 3 | Parameter Classes 4 | ================= 5 | 6 | Parameters 7 | ---------- 8 | 9 | .. autoclass:: encodermap.parameters.parameters.Parameters 10 | :class-doc-from: both 11 | :private-members: 12 | :special-members: __init__ 13 | :member-order: groupwise 14 | 15 | ADCParameters 16 | ------------- 17 | 18 | .. autoclass:: encodermap.parameters.parameters.ADCParameters 19 | :class-doc-from: both 20 | :private-members: 21 | :special-members: __init__ 22 | :member-order: groupwise 23 | -------------------------------------------------------------------------------- /docs/source/user_guide/singletraj.rst: -------------------------------------------------------------------------------- 1 | .. _user_guide/singletraj: 2 | 3 | SingleTraj 4 | ========== 5 | 6 | .. autoclass:: encodermap.trajinfo.info_single.SingleTraj 7 | :class-doc-from: both 8 | :private-members: 9 | :special-members: __init__ 10 | :member-order: groupwise 11 | -------------------------------------------------------------------------------- /docs/source/user_guide/trajensemble.rst: -------------------------------------------------------------------------------- 1 | .. _user_guide/trajensmeble: 2 | 3 | TrajEnsemble 4 | ============ 5 | 6 | .. autoclass:: encodermap.trajinfo.info_all.TrajEnsemble 7 | :class-doc-from: both 8 | :private-members: 9 | :special-members: __init__ 10 | :member-order: groupwise 11 | -------------------------------------------------------------------------------- /docs/sphinx_requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx 2 | sphinxcontrib-youtube 3 | sphinx-copybutton 4 | sphinxcontrib-bibtex 5 | nbsphinx-link 6 | sphinx-design 7 | pydata-sphinx-theme 8 | mistune>=2.0 9 | livereload 10 | ipykernel 11 | myst-parser 12 | livereload 13 | sphinx-gallery 14 | imohash 15 | myst-nb 16 | -------------------------------------------------------------------------------- /encodermap/_typing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/_typing.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | """Typing for the encodermap package""" 23 | 24 | 25 | # Standard Library Imports 26 | import typing 27 | from collections.abc import Sequence 28 | from typing import Literal, Union 29 | 30 | # Third Party Imports 31 | import numpy as np 32 | 33 | 34 | ################################################################################ 35 | # Type Defs 36 | ################################################################################ 37 | 38 | 39 | CanBeIndex = Union[int, Sequence[int], Sequence[np.ndarray], slice] 40 | 41 | 42 | DihedralOrBondDict = dict[ 43 | Literal[ 44 | "bonds", 45 | "optional_bonds", 46 | "delete_bonds", 47 | "optional_delete_bonds", 48 | "PHI", 49 | "PSI", 50 | "OMEGA", 51 | "not_PHI", 52 | "not_PSI", 53 | "not_OMEGA", 54 | "CHI1", 55 | "CHI2", 56 | "CHI3", 57 | "CHI4", 58 | "CHI5", 59 | ], 60 | Union[list[str], list[tuple[Union[str, int], Union[str, int]]]], 61 | ] 62 | 63 | 64 | CustomAAsDict = dict[ 65 | Union[str, tuple[str, str]], 66 | Union[ 67 | None, 68 | tuple[str, None], 69 | tuple[ 70 | str, 71 | DihedralOrBondDict, 72 | ], 73 | ], 74 | ] 75 | -------------------------------------------------------------------------------- /encodermap/autoencoder/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/autoencoder/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | """Front-facing autoencoder classes.""" 23 | # Local Folder Imports 24 | from .autoencoder import * 25 | -------------------------------------------------------------------------------- /encodermap/callbacks/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/callbacks/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | # Future Imports at the top 23 | from __future__ import annotations 24 | 25 | # Standard Library Imports 26 | from typing import TYPE_CHECKING 27 | 28 | # Local Folder Imports 29 | from .callbacks import * 30 | from .metrics import ( 31 | ADCClashMetric, 32 | ADCRMSDMetric, 33 | AngleDihedralCartesianEncoderMapBaseMetric, 34 | EncoderMapBaseMetric, 35 | ) 36 | -------------------------------------------------------------------------------- /encodermap/encodermap_tf1/__init__.py: -------------------------------------------------------------------------------- 1 | # Local Folder Imports 2 | from . import misc, plot 3 | from .angle_dihedral_cartesian_encodermap import ( 4 | AngleDihedralCartesianEncoderMap, 5 | AngleDihedralCartesianEncoderMapDummy, 6 | ) 7 | from .autoencoder import Autoencoder 8 | from .backmapping import * 9 | from .encodermap import EncoderMap 10 | from .moldata import MolData 11 | from .parameters import ADCParameters, Parameters 12 | 13 | 14 | try: 15 | # Third Party Imports 16 | import tensorflow as tf 17 | 18 | tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR) 19 | except: 20 | pass 21 | -------------------------------------------------------------------------------- /encodermap/encodermap_tf1/encodermap.py: -------------------------------------------------------------------------------- 1 | # Third Party Imports 2 | import tensorflow.compat.v1 as tf 3 | 4 | # Local Folder Imports 5 | from .autoencoder import Autoencoder 6 | from .misc import distance_cost 7 | 8 | 9 | class EncoderMap(Autoencoder): 10 | def _setup_cost(self): 11 | self._auto_cost() 12 | self._center_cost() 13 | self._l2_reg_cost() 14 | self._distance_cost() 15 | 16 | def _distance_cost(self): 17 | if self.p.distance_cost_scale is not None: 18 | dist_cost = distance_cost( 19 | self.main_inputs, 20 | self.latent, 21 | *self.p.dist_sig_parameters, 22 | self.p.periodicity, 23 | ) 24 | tf.summary.scalar("sigmoid_loss", dist_cost) 25 | if self.p.distance_cost_scale != 0: 26 | self.cost += self.p.distance_cost_scale * dist_cost 27 | -------------------------------------------------------------------------------- /encodermap/examples/cube.py: -------------------------------------------------------------------------------- 1 | # Encodermap imports 2 | import encodermap as em 3 | 4 | 5 | # generating data: 6 | high_d_data, ids = em.misc.random_on_cube_edges(10000, sigma=0.05) 7 | 8 | # setting parameters: 9 | parameters = em.Parameters() 10 | parameters.main_path = em.misc.run_path("runs/cube/") 11 | parameters.n_steps = 10000 12 | parameters.dist_sig_parameters = (0.2, 3, 6, 1, 2, 6) 13 | parameters.periodicity = float("inf") 14 | 15 | # training: 16 | e_map = em.EncoderMap(parameters, high_d_data) 17 | e_map.train() 18 | 19 | # projecting: 20 | low_d_projection = e_map.encode(high_d_data) 21 | generated = e_map.generate(low_d_projection) 22 | 23 | 24 | ######################################################################### 25 | # Plotting: 26 | ######################################################################### 27 | 28 | # Third Party Imports 29 | import matplotlib.pyplot as plt 30 | from mpl_toolkits.mplot3d import ( # somehow this conflicts with tensorflow if imported earlier 31 | Axes3D, 32 | ) 33 | 34 | 35 | fig = plt.figure() 36 | axe = fig.add_subplot(111, projection="3d") 37 | axe.scatter( 38 | high_d_data[:, 0], 39 | high_d_data[:, 1], 40 | high_d_data[:, 2], 41 | c=ids, 42 | marker="o", 43 | linewidths=0, 44 | cmap="tab10", 45 | ) 46 | 47 | fig, axe = plt.subplots() 48 | axe.scatter( 49 | low_d_projection[:, 0], 50 | low_d_projection[:, 1], 51 | c=ids, 52 | s=5, 53 | marker="o", 54 | linewidths=0, 55 | cmap="tab10", 56 | ) 57 | 58 | fig = plt.figure() 59 | axe = fig.add_subplot(111, projection="3d") 60 | axe.scatter( 61 | generated[:, 0], 62 | generated[:, 1], 63 | generated[:, 2], 64 | c=ids, 65 | marker="o", 66 | linewidths=0, 67 | cmap="tab10", 68 | ) 69 | 70 | plt.show() 71 | -------------------------------------------------------------------------------- /encodermap/examples/cube_distance_analysis.py: -------------------------------------------------------------------------------- 1 | # Third Party Imports 2 | import matplotlib.pyplot as plt 3 | 4 | # Encodermap imports 5 | import encodermap as em 6 | 7 | 8 | data, ids = em.misc.random_on_cube_edges(1000, sigma=0.05) 9 | 10 | dist_sig_parameters = (0.2, 3, 6, 1, 2, 6) 11 | periodicity = float("inf") 12 | 13 | axe = em.plot.distance_histogram(data, periodicity, dist_sig_parameters, bins=50) 14 | plt.show() 15 | -------------------------------------------------------------------------------- /encodermap/examples/dihedral_to_cartesian_diubi_analysis.py: -------------------------------------------------------------------------------- 1 | # Standard Library Imports 2 | import os 3 | 4 | # Third Party Imports 5 | import matplotlib.pyplot as plt 6 | import MDAnalysis as md 7 | import numpy as np 8 | 9 | # Encodermap imports 10 | import encodermap as em 11 | 12 | 13 | molname = "diubi" 14 | run_id = 3 15 | step = 50000 16 | selection_for_alignment = "resid 0:77" 17 | 18 | main_path = "runs/{}/run{}".format(molname, run_id) 19 | 20 | 21 | # ######################### Load data ######################### 22 | 23 | structure_path = "data/{}/01.pdb".format(molname) 24 | trajectory_paths = ["data/{}/{:02d}.xtc".format(molname, i + 1) for i in range(12)] 25 | 26 | uni = md.Universe(structure_path, trajectory_paths) 27 | selected_atoms = uni.select_atoms( 28 | "backbone or name H or name O1 or (name CD and resname PRO)" 29 | ) 30 | moldata = em.MolData(selected_atoms, cache_path="data/{}/cache".format(molname)) 31 | 32 | 33 | # ######################### Load parameters and checkpoint ######################### 34 | 35 | parameters = em.ADCParameters.load(os.path.join(main_path, "parameters.json")) 36 | e_map = em.AngleDihedralCartesianEncoderMap( 37 | parameters, 38 | moldata, 39 | checkpoint_path=os.path.join(main_path, "checkpoints", "step{}.ckpt".format(step)), 40 | read_only=True, 41 | ) 42 | 43 | 44 | # ######################### Project Data to map ######################### 45 | 46 | projected = e_map.encode(moldata.dihedrals) 47 | 48 | 49 | # ######################### Plot histogram with path generator and lasso Select ######################### 50 | 51 | hist, xedges, yedges = np.histogram2d(projected[:, 0], projected[:, 1], bins=500) 52 | 53 | fig1, axe1 = plt.subplots() 54 | 55 | caxe = axe1.imshow( 56 | -np.log(hist.T), 57 | origin="low", 58 | extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]], 59 | aspect="auto", 60 | ) 61 | cbar = fig1.colorbar(caxe) 62 | cbar.set_label("-ln(p)", labelpad=0) 63 | axe1.set_title("Path Generator") 64 | generator = em.plot.PathGenerateCartesians( 65 | axe1, 66 | e_map, 67 | moldata, 68 | vmd_path="/home/soft/bin/vmd", 69 | align_reference=moldata.sorted_atoms, 70 | align_select=selection_for_alignment, 71 | ) 72 | 73 | fig2, axe2 = plt.subplots() 74 | caxe = axe2.imshow( 75 | -np.log(hist.T), 76 | origin="low", 77 | extent=[xedges[0], xedges[-1], yedges[0], yedges[-1]], 78 | aspect="auto", 79 | ) 80 | cbar = fig2.colorbar(caxe) 81 | cbar.set_label("-ln(p)", labelpad=0) 82 | axe2.set_title("Selector") 83 | selector = em.plot.PathSelect( 84 | axe2, 85 | projected, 86 | moldata, 87 | e_map.p.main_path, 88 | vmd_path="/home/soft/bin/vmd", 89 | align_reference=moldata.sorted_atoms, 90 | align_select=selection_for_alignment, 91 | ) 92 | 93 | plt.show() 94 | -------------------------------------------------------------------------------- /encodermap/examples/trp_cage.py: -------------------------------------------------------------------------------- 1 | # Standard Library Imports 2 | import os 3 | 4 | # Third Party Imports 5 | import matplotlib.pyplot as plt 6 | import numpy as np 7 | 8 | # Encodermap imports 9 | import encodermap as em 10 | 11 | 12 | # setting parameters 13 | data_path = "./data" 14 | run_path = em.misc.run_path("./runs") 15 | csv_path = os.path.join(data_path, "trp_cage.csv") # can be downloaded from: 16 | # https://www.kaggle.com/tobiasle/trp-cage-dihedrals 17 | parameters = em.Parameters() 18 | parameters.main_path = run_path 19 | parameters.n_steps = 50000 20 | 21 | 22 | print("loading data ...") 23 | data = np.loadtxt(csv_path, skiprows=1, delimiter=",") 24 | dihedrals = data[:, 3:41] 25 | 26 | 27 | print("training autoencoder ...") 28 | e_map = em.EncoderMap(parameters, dihedrals) 29 | e_map.train() 30 | 31 | 32 | print("projecting data ...") 33 | low_d_projection = e_map.encode(dihedrals) 34 | 35 | 36 | print("plotting result ...") 37 | fig, axe = plt.subplots() 38 | caxe = axe.scatter( 39 | low_d_projection[:, 0], 40 | low_d_projection[:, 1], 41 | c=data[:, -1], 42 | s=0.1, 43 | cmap="nipy_spectral", 44 | marker="o", 45 | linewidths=0, 46 | ) 47 | cbar = fig.colorbar(caxe) 48 | cbar.set_label("helix rmsd") 49 | 50 | 51 | # generate structures along path 52 | pdb_path = os.path.join(data_path, "trp_cage_extended.pdb") 53 | generator = em.plot.PathGenerateDihedrals(axe, e_map, pdb_path) 54 | 55 | plt.show() 56 | -------------------------------------------------------------------------------- /encodermap/loading/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/loading/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | # Local Folder Imports 23 | from .features import CustomFeature 24 | from .featurizer import * 25 | -------------------------------------------------------------------------------- /encodermap/loss_functions/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/loss_functions/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | """EncoderMap's loss functions.""" 23 | # Local Folder Imports 24 | from .loss_functions import * 25 | -------------------------------------------------------------------------------- /encodermap/misc/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/misc/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | # Encodermap imports 23 | import encodermap.misc.clustering 24 | 25 | # Local Folder Imports 26 | from .backmapping import * 27 | from .distances import * 28 | from .misc import * 29 | from .summaries import * 30 | from .xarray import * 31 | from .xarray_save_wrong_hdf5 import * 32 | -------------------------------------------------------------------------------- /encodermap/misc/backmapping.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | int square(int i) { 5 | return i * i; 6 | } 7 | -------------------------------------------------------------------------------- /encodermap/misc/function_def.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/misc/function_def.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | """Wraps tensorflow's `tf.function` again to accept a debug=True or debug=False argument. 23 | 24 | With debug=True, the function will not be compiled. With debug=False (which is 25 | teh default), it will be compiled. 26 | 27 | """ 28 | # Future Imports at the top 29 | from __future__ import annotations 30 | 31 | # Standard Library Imports 32 | from typing import Any 33 | 34 | # Third Party Imports 35 | import tensorflow as tf 36 | 37 | 38 | def function(debug: bool = False) -> Any: 39 | """Encodermap's implementation of `tf.function`. 40 | 41 | Args: 42 | debug (bool): If True, the decorated function will not be compiled. 43 | Defaults to False. 44 | 45 | """ 46 | 47 | def decorator(f: Any) -> Any: 48 | """The decorator, that takes the function.""" 49 | 50 | def wrapper(*args: Any, **kwargs: Any) -> Any: 51 | """The wrapper, that calls the function based on the debug argument.""" 52 | if debug: 53 | result = f(*args, **kwargs) 54 | else: 55 | compiled = tf.function(f) 56 | result = compiled(*args, **kwargs) 57 | return result 58 | 59 | return wrapper 60 | 61 | return decorator 62 | -------------------------------------------------------------------------------- /encodermap/misc/logo_cube_300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/encodermap/misc/logo_cube_300.png -------------------------------------------------------------------------------- /encodermap/models/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/models/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | """EncoderMap's tensorflow models. 23 | 24 | In tensorflow a model is a grouping of layers with training/inference features. 25 | 26 | """ 27 | # Local Folder Imports 28 | from .models import * 29 | -------------------------------------------------------------------------------- /encodermap/moldata/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/moldata/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | # Local Folder Imports 23 | from .moldata import * 24 | -------------------------------------------------------------------------------- /encodermap/parameters/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/parameters/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | # Local Folder Imports 23 | from .parameters import ADCParameters, Parameters 24 | -------------------------------------------------------------------------------- /encodermap/plot/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/plot/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | # Local Folder Imports 23 | from .plotting import * 24 | -------------------------------------------------------------------------------- /encodermap/plot/assets/favicon.ico: -------------------------------------------------------------------------------- 1 | ../../../pic/favicon.ico -------------------------------------------------------------------------------- /encodermap/trajinfo/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # encodermap/trajinfo/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | # Local Folder Imports 23 | from .info_all import * 24 | from .info_single import * 25 | from .trajinfo_utils import CustomTopology 26 | -------------------------------------------------------------------------------- /md_requirements.txt: -------------------------------------------------------------------------------- 1 | h5py 2 | h5netcdf 3 | mdtraj 4 | scipy 5 | networkx 6 | seaborn 7 | pandas 8 | nglview 9 | PyYAML 10 | xarray 11 | mdshare 12 | dask 13 | dask-jobqueue 14 | numba 15 | distributed 16 | dash 17 | plotly 18 | dash_bio 19 | dash-bootstrap-components 20 | dash-mantine-components 21 | dash-bootstrap-templates 22 | pydot 23 | graphviz 24 | dash_auth 25 | rich 26 | kaleido 27 | ipycanvas 28 | orjson 29 | bokeh 30 | anywidget 31 | -------------------------------------------------------------------------------- /pic/book_pictogram.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 36 | 38 | 44 | 45 | 50 | 54 | 59 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /pic/brackets_pictogram.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 36 | 38 | 44 | 50 | 51 | 56 | [...] 66 | 67 | 68 | -------------------------------------------------------------------------------- /pic/distance_histogram_cube_reasonable_sigma.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/distance_histogram_cube_reasonable_sigma.png -------------------------------------------------------------------------------- /pic/distance_histogram_cube_too_high_sigma.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/distance_histogram_cube_too_high_sigma.png -------------------------------------------------------------------------------- /pic/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/favicon.ico -------------------------------------------------------------------------------- /pic/logo_cube_300.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/logo_cube_300.png -------------------------------------------------------------------------------- /pic/logo_cube_400.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/logo_cube_400.png -------------------------------------------------------------------------------- /pic/logo_cube_hd.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/logo_cube_hd.png -------------------------------------------------------------------------------- /pic/logo_m.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/logo_m.png -------------------------------------------------------------------------------- /pic/new_logo_m.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/new_logo_m.png -------------------------------------------------------------------------------- /pic/rocket_pictogram.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 36 | 38 | 43 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /pic/thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/pic/thumbnail.png -------------------------------------------------------------------------------- /pic/tools_pictogram.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 16 | 36 | 38 | 44 | 45 | 50 | 55 | 60 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /questions.md: -------------------------------------------------------------------------------- 1 | ## Frequently Asked Questions 2 | 3 | ### All high-dimensional points are projected to a single point in the map. What is wrong? 4 | Probably the used sigmoid parameters are not suitable for your data set. 5 | Check the sigmoid parameters using the distance_histogram function as shown in the [cube_distance_analysis example](encodermap/examples/cube_distance_analysis.py). 6 | This will create a plot like this: 7 |

8 | In the above example the dist_sig_parameters are set to (10, 3, 6, 1, 2, 6). 9 | The high-d sigma of 10 is much to high for the example data. 10 | The black lines connecting the plots of the high-d sigmoid (top) and the low-d sigmoid (bottom) indicate to which low-dimensional distances high-dimensional distences are ideally mapped with your choice of sigmoid parameters. 11 | With such a choice of a too large high-d sigma value, all high-d distances are maped to verry small low-d distances which could explain why all points in the map are on top of eachother. 12 | With a more reasonable choice of the high-d sigma the plot would look somewhat like this: 13 |

14 | The ideal sigmoid parameters depend on the question what you want to have separated or not separated in the map which can not universally be answered. 15 | 16 | ### Why does my map exhibit a weird grid-like pattern? 17 | Grid patterns tend to appear when the auto cost is strongly weighted compared to the the distance cost(Sketchmap cost). 18 | Have a look at Figure S2 in the [supplement of the first EncoderMap paper](https://pubs.acs.org/doi/suppl/10.1021/acs.jctc.8b00975/suppl_file/ct8b00975_si_001.pdf) where we discuss this matter. 19 | If you are only interested in the map and not in the decoder, you can also try to turn off the auto cost completely (set auto_cost_scale = 0). 20 | Then you will obtain a map purely optimized according to the distance cost. 21 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | tqdm 2 | tensorboard 3 | traitlets 4 | numpy 5 | matplotlib 6 | tensorflow 7 | tables 8 | MDAnalysis 9 | optional-imports >= 1.0.4 10 | tensorflow_probability 11 | tensorflow_graphics 12 | rich 13 | versioneer 14 | requests 15 | packaging 16 | pillow 17 | transformations 18 | scipy 19 | ipywidgets 20 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Third Party Imports 2 | import versioneer 3 | from setuptools import setup 4 | 5 | 6 | with open("description.md", "r") as fh: 7 | long_description = fh.read() 8 | 9 | # read _version.py to have a single source file for version. 10 | # deprecated on 2023-08-25 and replaced with versioneer 11 | # exec(open("encodermap/_version.py").read()) 12 | 13 | # setup 14 | setup( 15 | name="encodermap", 16 | version=versioneer.get_version(), 17 | cmdclass=versioneer.get_cmdclass(), 18 | python_requires=">=3.9", 19 | include_package_data=True, 20 | description="python library for dimensionality reduction", 21 | long_description=long_description, 22 | long_description_content_type="text/markdown", 23 | author="Tobias Lemke, Kevin Sawade", 24 | url="https://github.com/AG-Peter/encodermap", 25 | packages=[ 26 | "encodermap", 27 | "encodermap.plot", 28 | "encodermap.callbacks", 29 | "encodermap.autoencoder", 30 | "encodermap.encodermap_tf1", 31 | "encodermap.loading", 32 | "encodermap.loss_functions", 33 | "encodermap.misc", 34 | "encodermap.models", 35 | "encodermap.moldata", 36 | "encodermap.parameters", 37 | "encodermap.trajinfo", 38 | ], 39 | install_requires=[ 40 | "transformations", 41 | "tomli", 42 | "versioneer", 43 | "numpy", 44 | "matplotlib", 45 | "scipy", 46 | "MDAnalysis", 47 | "tqdm>=4.4.0", 48 | "tensorflow==2.15.0", 49 | "nglview>=3.0.1", 50 | "seaborn>=0.11.1", 51 | "pillow>=10.0.1", 52 | "ipywidgets>=8.0", 53 | "optional_imports>=1.0.4", 54 | "tensorflow_probability", 55 | "rich", 56 | ], 57 | classifiers=[ 58 | "Programming Language :: Python :: 3", 59 | "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", 60 | "Operating System :: OS Independent", 61 | ], 62 | ) 63 | -------------------------------------------------------------------------------- /tests/.coveragerc: -------------------------------------------------------------------------------- 1 | [report] 2 | exclude_lines = 3 | pragma: no cover 4 | def __repr__ 5 | def __str__ 6 | __all__ 7 | if self.debug: 8 | if settings.DEBUG 9 | raise AssertionError 10 | raise NotImplementedError 11 | if __name__ == .__main__.: 12 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # tests/__init__.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | """ 23 | 24 | """ 25 | -------------------------------------------------------------------------------- /tests/data/1YUF_numpy.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1YUF_numpy.npy -------------------------------------------------------------------------------- /tests/data/1YUG_and_1YUF_dataset.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1YUG_and_1YUF_dataset.nc -------------------------------------------------------------------------------- /tests/data/1YUG_numpy.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1YUG_numpy.npy -------------------------------------------------------------------------------- /tests/data/1am7_center_of_mass.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_center_of_mass.npy -------------------------------------------------------------------------------- /tests/data/1am7_center_of_mass_x.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_center_of_mass_x.nc -------------------------------------------------------------------------------- /tests/data/1am7_center_of_mass_x.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_center_of_mass_x.npy -------------------------------------------------------------------------------- /tests/data/1am7_corrected.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_corrected.xtc -------------------------------------------------------------------------------- /tests/data/1am7_corrected_part1.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_corrected_part1.xtc -------------------------------------------------------------------------------- /tests/data/1am7_corrected_part2.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_corrected_part2.xtc -------------------------------------------------------------------------------- /tests/data/1am7_corrected_with_CVs.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_corrected_with_CVs.h5 -------------------------------------------------------------------------------- /tests/data/1am7_first_frame_first_atom_xyz.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_first_frame_first_atom_xyz.npy -------------------------------------------------------------------------------- /tests/data/1am7_uncorrected.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/1am7_uncorrected.xtc -------------------------------------------------------------------------------- /tests/data/Ala10_helix.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/Ala10_helix.xtc -------------------------------------------------------------------------------- /tests/data/PFFP_MD_fin_protonly.tpr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/PFFP_MD_fin_protonly.tpr -------------------------------------------------------------------------------- /tests/data/PFFP_MD_fin_protonly_dt_100.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/PFFP_MD_fin_protonly_dt_100.xtc -------------------------------------------------------------------------------- /tests/data/README.md: -------------------------------------------------------------------------------- 1 | Example data taken from mdtraj. https://github.com/mdtraj/mdtraj/tree/master/tests/data 2 | 3 | 1am7_protein.pdb, 1am7_corrected.xtc, and 1am7_uncorrected.xtc are Gromacs trajectories of lambda phage lysozyme (PDB: 1AM7). The uncorrected xtc file has the protein drifting across the periodic box. The corrected xtc file has been corrected with trjconv -pbc mol. This are useful as a sanity check for PBC issues. 4 | 5 | The trajectory PFFP_MD_fin_protonly_dt_100.xtc and the topologies (PFFP_MD_fin_protonly.grp, PFFP_MD_fin_protonly.tpr, and PFFP_vac.top) was created by Kevin Sawade during his Bachelor thesis in 2015. The creation employed GROMACS4.6.1. The protein was solvated in a 50/50 MeOH/Water mixture. Here is the mdp-file: 6 | 7 | ``` 8 | title = Full MD 9 | cpp = /lib/cpp 10 | ; 11 | ; Run control: 12 | ; 13 | constraints = H-bonds 14 | integrator = md 15 | dt = 0.002 ; ps 16 | nsteps = 2500 ; total 5 ps 17 | ; 18 | ; Neighbor searching: 19 | ; 20 | nstlist = 5 21 | ns_type = grid 22 | rlist = 1.4 ; nm 23 | ; 24 | ; Output control: 25 | ; 26 | nstcomm = 1 27 | nstxout = 250000 28 | nstvout = 500000 29 | nstfout = 0 30 | nstlog = 1000 31 | nstenergy = 500 32 | nstxtcout = 500 33 | xtc-precision = 1000 34 | xtc_grps = Protein Non-Protein 35 | ; 36 | ; Electrostatics 37 | ; 38 | coulombtype = PME 39 | pme_order = 4 40 | fourierspacing = 0.14 ; nm 41 | fourier_nx = 0 42 | fourier_ny = 0 43 | fourier_nz = 0 44 | rcoulomb = 1.4 ; nm 45 | ewald_rtol = 1e-5 46 | optimize_fft = yes 47 | ; 48 | ; VdW 49 | ; 50 | vdwtype = cut-off 51 | rvdw = 1.4 ; nm 52 | ; 53 | ; Temperature & Pressure coupling: 54 | ; 55 | tcoupl = v-rescale 56 | tc-grps = Protein Non-Protein 57 | tau_t = 0.1 0.1 ; ps 58 | ref_t = 300 300 ; K 59 | 60 | pcoupl = Berendsen 61 | pcoupltype = isotropic 62 | tau_p = 1 63 | compressibility = 4.5e-5 64 | ref_p = 1 65 | ; 66 | ; Velocity generation: 67 | ; 68 | gen_vel = no 69 | ;gen_temp = 300 ; K 70 | ;gen_seed = -1 71 | 72 | 73 | ``` 74 | -------------------------------------------------------------------------------- /tests/data/alanine_dipeptide.pdb: -------------------------------------------------------------------------------- 1 | MODEL 0 2 | ATOM 1 N ALA A 1 -1.176 -2.547 -0.799 1.00 0.00 N 3 | ATOM 2 CA ALA A 1 -0.133 -2.398 0.196 1.00 0.00 C 4 | ATOM 3 C ALA A 1 0.390 -0.969 0.196 1.00 0.00 C 5 | ATOM 4 O ALA A 1 1.283 -0.633 0.972 1.00 0.00 O 6 | ATOM 5 CB ALA A 1 1.045 -3.325 -0.082 1.00 0.00 C 7 | ATOM 6 N ALA A 2 -0.169 -0.128 -0.677 1.00 0.00 N 8 | ATOM 7 CA ALA A 2 0.241 1.259 -0.774 1.00 0.00 C 9 | ATOM 8 C ALA A 2 -0.568 1.969 -1.851 1.00 0.00 C 10 | ATOM 9 O ALA A 2 -1.044 3.064 -1.559 1.00 0.00 O 11 | ATOM 10 OXT ALA A 2 -0.685 1.394 -2.931 1.00 0.00 O 12 | ATOM 11 CB ALA A 2 1.716 1.383 -1.142 1.00 0.00 C 13 | TER 12 ALA A 2 14 | ENDMDL 15 | END 16 | -------------------------------------------------------------------------------- /tests/data/alanine_dipeptide.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/alanine_dipeptide.xtc -------------------------------------------------------------------------------- /tests/data/asp7.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/asp7.h5 -------------------------------------------------------------------------------- /tests/data/asp7.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/asp7.xtc -------------------------------------------------------------------------------- /tests/data/glu7.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/glu7.xtc -------------------------------------------------------------------------------- /tests/data/known_angles.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/known_angles.h5 -------------------------------------------------------------------------------- /tests/data/known_angles.pdb: -------------------------------------------------------------------------------- 1 | REMARK 1 CREATED WITH MDTraj 1.9.6, 2022-08-19 2 | CRYST1 64.441 64.441 64.441 60.00 60.00 90.00 P 1 1 3 | MODEL 0 4 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 5 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 6 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 7 | ATOM 4 B4 RES A 4 20.000 10.000 0.000 1.00 0.00 B 8 | TER 5 RES A 4 9 | ENDMDL 10 | MODEL 1 11 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 12 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 13 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 14 | ATOM 4 B4 RES A 4 0.000 10.000 0.000 1.00 0.00 B 15 | TER 5 RES A 4 16 | ENDMDL 17 | MODEL 2 18 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 19 | ATOM 2 B2 RES A 2 20.000 0.000 0.000 1.00 0.00 B 20 | ATOM 3 B3 RES A 3 20.000 10.000 0.000 1.00 0.00 B 21 | ATOM 4 B4 RES A 4 30.000 5.000 5.000 1.00 0.00 B 22 | TER 5 RES A 4 23 | ENDMDL 24 | MODEL 3 25 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 26 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 27 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 28 | ATOM 4 B4 RES A 4 15.000 10.000 0.000 1.00 0.00 B 29 | TER 5 RES A 4 30 | ENDMDL 31 | MODEL 4 32 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 33 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 34 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 35 | ATOM 4 B4 RES A 4 15.000 0.000 10.000 1.00 0.00 B 36 | TER 5 RES A 4 37 | ENDMDL 38 | MODEL 5 39 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 40 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 41 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 42 | ATOM 4 B4 RES A 4 10.000 15.000 10.000 1.00 0.00 B 43 | TER 5 RES A 4 44 | ENDMDL 45 | END 46 | -------------------------------------------------------------------------------- /tests/data/known_angles_1.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/known_angles_1.h5 -------------------------------------------------------------------------------- /tests/data/known_angles_1.pdb: -------------------------------------------------------------------------------- 1 | REMARK 1 CREATED WITH MDTraj 1.9.6, 2022-09-01 2 | CRYST1 64.441 64.441 64.441 60.00 60.00 90.00 P 1 1 3 | MODEL 0 4 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 5 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 6 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 7 | ATOM 4 B4 RES A 4 20.000 10.000 0.000 1.00 0.00 B 8 | TER 5 RES A 4 9 | ENDMDL 10 | MODEL 1 11 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 12 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 13 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 14 | ATOM 4 B4 RES A 4 0.000 10.000 0.000 1.00 0.00 B 15 | TER 5 RES A 4 16 | ENDMDL 17 | MODEL 2 18 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 19 | ATOM 2 B2 RES A 2 20.000 0.000 0.000 1.00 0.00 B 20 | ATOM 3 B3 RES A 3 20.000 10.000 0.000 1.00 0.00 B 21 | ATOM 4 B4 RES A 4 30.000 5.000 5.000 1.00 0.00 B 22 | TER 5 RES A 4 23 | ENDMDL 24 | END 25 | -------------------------------------------------------------------------------- /tests/data/known_angles_2.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/known_angles_2.h5 -------------------------------------------------------------------------------- /tests/data/known_angles_2.pdb: -------------------------------------------------------------------------------- 1 | REMARK 1 CREATED WITH MDTraj 1.9.6, 2022-09-01 2 | CRYST1 64.441 64.441 64.441 60.00 60.00 90.00 P 1 1 3 | MODEL 0 4 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 5 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 6 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 7 | ATOM 4 B4 RES A 4 15.000 10.000 0.000 1.00 0.00 B 8 | TER 5 RES A 4 9 | ENDMDL 10 | MODEL 1 11 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 12 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 13 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 14 | ATOM 4 B4 RES A 4 15.000 0.000 10.000 1.00 0.00 B 15 | TER 5 RES A 4 16 | ENDMDL 17 | MODEL 2 18 | ATOM 1 B1 RES A 1 0.000 0.000 0.000 1.00 0.00 B 19 | ATOM 2 B2 RES A 2 10.000 0.000 0.000 1.00 0.00 B 20 | ATOM 3 B3 RES A 3 10.000 10.000 0.000 1.00 0.00 B 21 | ATOM 4 B4 RES A 4 10.000 15.000 10.000 1.00 0.00 B 22 | TER 5 RES A 4 23 | ENDMDL 24 | END 25 | -------------------------------------------------------------------------------- /tests/data/known_angles_angles.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/known_angles_angles.npy -------------------------------------------------------------------------------- /tests/data/known_angles_dihedrals.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/known_angles_dihedrals.npy -------------------------------------------------------------------------------- /tests/data/known_angles_dists.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/known_angles_dists.npy -------------------------------------------------------------------------------- /tests/data/known_distances.pdb: -------------------------------------------------------------------------------- 1 | CRYST1 0.000 0.000 0.000 90.00 90.00 90.00 P 1 1 2 | ATOM 1 C X 1 0.000 0.000 0.000 0.00 0.00 3 | ATOM 2 C X 1 1.000 0.000 0.000 0.00 0.00 B 4 | END 5 | ATOM 1 C X 1 0.000 0.000 0.000 0.00 0.00 6 | ATOM 2 C X 1 0.000 1.000 0.000 0.00 0.00 B 7 | END 8 | ATOM 1 C X 1 0.000 0.000 0.000 0.00 0.00 9 | ATOM 2 C X 1 0.000 0.000 1.000 0.00 0.00 B 10 | END 11 | -------------------------------------------------------------------------------- /tests/data/larger_dataset.nc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/larger_dataset.nc -------------------------------------------------------------------------------- /tests/data/test_chain_in_plane_expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/test_chain_in_plane_expected.png -------------------------------------------------------------------------------- /tests/data/test_guess_amide_H_expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/test_guess_amide_H_expected.png -------------------------------------------------------------------------------- /tests/data/test_guess_amide_O_expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/test_guess_amide_O_expected.png -------------------------------------------------------------------------------- /tests/data/test_straight_to_helix_array_expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/test_straight_to_helix_array_expected.png -------------------------------------------------------------------------------- /tests/data/test_straight_to_helix_v2_expected.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/test_straight_to_helix_v2_expected.png -------------------------------------------------------------------------------- /tests/data/tip3p_300K_1ATM.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/tip3p_300K_1ATM.xtc -------------------------------------------------------------------------------- /tests/data/traj.h5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tests/data/traj.h5 -------------------------------------------------------------------------------- /tests/find_long_comments.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # encodermap/tests/find_long_comments.py 4 | ################################################################################ 5 | # EncoderMap: A python library for dimensionality reduction. 6 | # 7 | # Copyright 2019-2024 University of Konstanz and the Authors 8 | # 9 | # Authors: 10 | # Kevin Sawade 11 | # 12 | # Encodermap is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU Lesser General Public License as 14 | # published by the Free Software Foundation, either version 2.1 15 | # of the License, or (at your option) any later version. 16 | # This package is distributed in the hope that it will be useful to other 17 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 18 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 19 | # See the GNU Lesser General Public License for more details. 20 | # 21 | # See . 22 | ################################################################################ 23 | 24 | 25 | # Future Imports at the top 26 | from __future__ import annotations 27 | 28 | # Standard Library Imports 29 | import random 30 | from pathlib import Path 31 | from typing import Optional 32 | 33 | # Third Party Imports 34 | import click 35 | 36 | # Encodermap imports 37 | import encodermap as em 38 | from test_project_structure import find_long_comments 39 | 40 | 41 | @click.command() 42 | @click.option("--start-dir", default=None) 43 | @click.option( 44 | "--n-print", 45 | default=5, 46 | help=( 47 | f"How many missing docstrings should be printed. Set to -1 to print all " 48 | f"missing docstrings. Defaults to 5." 49 | ), 50 | ) 51 | def main( 52 | start_dir: Optional[str] = None, 53 | n_print: int = 5, 54 | ) -> int: 55 | if start_dir is None: 56 | start_dir = em.__file__ 57 | start_dir = Path(start_dir).resolve() 58 | py_files = start_dir.parent.rglob("*.py") 59 | py_files = list(filter(lambda x: "tf1" not in str(x), py_files)) 60 | exclude_files = ["utils.py"] 61 | long_comments = find_long_comments(py_files, exclude_files=exclude_files) 62 | print(f"Found {len(long_comments)} missing documentations.") 63 | if n_print == -1: 64 | for m in range(n_print): 65 | print(long_comments[m]) 66 | else: 67 | if len(long_comments) < n_print: 68 | for m in long_comments: 69 | print(m) 70 | else: 71 | for m in random.sample(long_comments, n_print): 72 | print(m) 73 | return 0 74 | 75 | 76 | if __name__ == "__main__": 77 | raise SystemExit(main()) 78 | -------------------------------------------------------------------------------- /tests/find_undocumented_package_parts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # encodermap/tests/find_undocumented_package_parts.py 4 | ################################################################################ 5 | # EncoderMap: A python library for dimensionality reduction. 6 | # 7 | # Copyright 2019-2024 University of Konstanz and the Authors 8 | # 9 | # Authors: 10 | # Kevin Sawade 11 | # 12 | # Encodermap is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU Lesser General Public License as 14 | # published by the Free Software Foundation, either version 2.1 15 | # of the License, or (at your option) any later version. 16 | # This package is distributed in the hope that it will be useful to other 17 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 18 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 19 | # See the GNU Lesser General Public License for more details. 20 | # 21 | # See . 22 | ################################################################################ 23 | 24 | 25 | # Future Imports at the top 26 | from __future__ import annotations 27 | 28 | # Standard Library Imports 29 | import random 30 | from pathlib import Path 31 | from typing import Optional 32 | 33 | # Third Party Imports 34 | import click 35 | 36 | # Encodermap imports 37 | import encodermap as em 38 | from test_project_structure import check_missing_docstrings 39 | 40 | 41 | @click.command() 42 | @click.option("--start-dir", default=None) 43 | @click.option( 44 | "--n-print", 45 | default=5, 46 | help=( 47 | f"How many missing docstrings should be printed. Set to -1 to print all " 48 | f"missing docstrings. Defaults to 5." 49 | ), 50 | ) 51 | def main( 52 | start_dir: Optional[str] = None, 53 | n_print: int = 5, 54 | ) -> int: 55 | if start_dir is None: 56 | start_dir = em.__file__ 57 | start_dir = Path(start_dir).resolve() 58 | py_files = start_dir.parent.rglob("*.py") 59 | py_files = list(filter(lambda x: "tf1" not in str(x), py_files)) 60 | exclude_files = ["utils.py"] 61 | missing = check_missing_docstrings(py_files, exclude_files=exclude_files) 62 | print(f"Found {len(missing)} missing documentations.") 63 | if n_print == -1: 64 | for m in range(n_print): 65 | print(missing[m]) 66 | else: 67 | if len(missing) < n_print: 68 | for m in missing: 69 | print(m) 70 | else: 71 | for m in random.sample(missing, n_print): 72 | print(m) 73 | return 0 74 | 75 | 76 | if __name__ == "__main__": 77 | raise SystemExit(main()) 78 | -------------------------------------------------------------------------------- /tests/import_with_beartype.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # encodermap/tests/import_with_beartype.py 4 | ################################################################################ 5 | # EncoderMap: A python library for dimensionality reduction. 6 | # 7 | # Copyright 2019-2024 University of Konstanz and the Authors 8 | # 9 | # Authors: 10 | # Kevin Sawade 11 | # 12 | # Encodermap is free software: you can redistribute it and/or modify 13 | # it under the terms of the GNU Lesser General Public License as 14 | # published by the Free Software Foundation, either version 2.1 15 | # of the License, or (at your option) any later version. 16 | # This package is distributed in the hope that it will be useful to other 17 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 18 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 19 | # See the GNU Lesser General Public License for more details. 20 | # 21 | # See . 22 | ################################################################################ 23 | 24 | # Standard Library Imports 25 | import os 26 | 27 | 28 | os.environ["ENCODERMAP_BEARTYPE"] = "True" 29 | 30 | if __name__ == "__main__": 31 | # Encodermap imports 32 | import encodermap as em 33 | 34 | print(em.__version__) 35 | -------------------------------------------------------------------------------- /tests/test_optional_imports.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # tests/test_optional_imports.py 3 | ################################################################################ 4 | # EncoderMap: A python library for dimensionality reduction. 5 | # 6 | # Copyright 2019-2024 University of Konstanz and the Authors 7 | # 8 | # Authors: 9 | # Kevin Sawade, Tobias Lemke 10 | # 11 | # Encodermap is free software: you can redistribute it and/or modify 12 | # it under the terms of the GNU Lesser General Public License as 13 | # published by the Free Software Foundation, either version 2.1 14 | # of the License, or (at your option) any later version. 15 | # This package is distributed in the hope that it will be useful to other 16 | # researches. IT DOES NOT COME WITH ANY WARRANTY WHATSOEVER; without even the 17 | # implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 18 | # See the GNU Lesser General Public License for more details. 19 | # 20 | # See . 21 | ################################################################################ 22 | # Standard Library Imports 23 | import unittest 24 | 25 | 26 | class TestOptionalImport(unittest.TestCase): 27 | def test_working_import(self): 28 | # Third Party Imports 29 | from optional_imports import _optional_import 30 | 31 | np = _optional_import("numpy") 32 | try: 33 | a = np.array([[1, 2], [3, 4]]) 34 | except ValueError: 35 | self.fail("np = _optional_import('numpy') raised an unexpected Error") 36 | rando = _optional_import("numpy", "random.random") 37 | try: 38 | a = rando((2, 2)) 39 | except ValueError: 40 | self.fail( 41 | "rando = _optional_import('numpy', 'random.random') raised an unexpected Error" 42 | ) 43 | 44 | def test_non_working_import(self): 45 | # Third Party Imports 46 | from optional_imports import _optional_import 47 | 48 | non_existent_package = _optional_import("non_existent_package") 49 | with self.assertRaises(ValueError): 50 | return_vale = non_existent_package() 51 | with self.assertRaises(ValueError): 52 | return_var = non_existent_package.non_existent_var 53 | 54 | 55 | test_cases = (TestOptionalImport,) 56 | 57 | 58 | def load_tests(loader, tests, pattern): 59 | suite = unittest.TestSuite() 60 | for test_class in test_cases: 61 | tests = loader.loadTestsFromTestCase(test_class) 62 | filtered_tests = [t for t in tests if not t.id().endswith(".test_session")] 63 | suite.addTests(filtered_tests) 64 | return suite 65 | 66 | 67 | if __name__ == "__main__": 68 | unittest.main() 69 | -------------------------------------------------------------------------------- /tests/test_requirements.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | html-testRunner 3 | black 4 | isort 5 | vulture 6 | pycodestyle 7 | tensor_annotations_tensorflow_stubs 8 | tensor_annotations 9 | data-science-types 10 | progress-reporter 11 | beartype 12 | pydoctest 13 | kaggle 14 | transformations 15 | docstring_parser 16 | pre-commit 17 | GitPython 18 | flake8 19 | pyproject-flake8 20 | nbconvert 21 | docker 22 | nbval 23 | -------------------------------------------------------------------------------- /tutorials/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tensorflow/tensorflow:1.15.0-py3-jupyter 2 | 3 | RUN rm -rf tensorflow-tutorials &&\ 4 | pip install encodermap==2.0.1 5 | 6 | RUN pip install jupyter_contrib_nbextensions &&\ 7 | jupyter contrib nbextension install &&\ 8 | jupyter nbextension enable spellchecker/main --system 9 | 10 | RUN pip install nglview &&\ 11 | jupyter-nbextension enable nglview --py --system 12 | 13 | EXPOSE 8888 14 | EXPOSE 6006 15 | -------------------------------------------------------------------------------- /tutorials/README.md: -------------------------------------------------------------------------------- 1 | # Tutorials 2 | 3 | There are two options how to do the tutorials. 4 | You can either install all requirements directly on your system or you can run the tutorials in an isolated Docker environment where all requirements for the tutorials are set up for you. 5 | 6 | ## Inside Docker 7 | 8 | If Docker is not installed on your system already, first [install Docker](https://docs.docker.com/install/). 9 | If you are on a linux system you should also [add your user to the docker group](https://docs.docker.com/install/linux/linux-postinstall/) to get permissions to start docker containers. 10 | 11 | Clone this repository to some location of your choice: 12 | ```bash 13 | git clone https://github.com/AG-Peter/encodermap.git 14 | ``` 15 | 16 | Go into the tutorial directory: 17 | ```bash 18 | cd encodermap/tutorials 19 | ``` 20 | 21 | Build the EncoderMap Docker container: 22 | ```bash 23 | docker build --tag encodermap . 24 | ``` 25 | 26 | and run the container: 27 | ```bash 28 | docker run --rm -u $(id -u):$(id -g) -it -p 6006:6006 -p 8888:8888 -v $(pwd)/notebooks:/tf/notebooks --name emap encodermap 29 | ``` 30 | You might need to adapt the above run command if you are not on a Linux system. Try to leave away the ```-u $(id -u):$(id -g)``` and specify the notebooks path in the notation of your operating system with: ```-v path-to-notebooks:/tf/notebooks``` 31 | 32 | This should start a jupyter-notebook server and you should be able to open the tutorial notebooks in your browser. 33 | 34 | ## Outside Docker 35 | To run the tutorials outside the provided Docker container the following software needs to be installed on your system: 36 | * [TensorFlow](https://www.tensorflow.org/install) 37 | * TensorBoard (usually comes along with TensorFlow) 38 | * [EncoderMap](https://github.com/AG-Peter/encodermap) 39 | * [Jupyter-Notebook](https://jupyter.org/) (with python3) 40 | 41 | Once everything is installed, clone this repository to some location of your choice: 42 | ```bash 43 | git clone https://github.com/AG-Peter/encodermap.git 44 | ``` 45 | 46 | Go into the tutorial directory: 47 | ```bash 48 | cd encodermap/tutorials 49 | ``` 50 | 51 | Start jupyter-notebook: 52 | ```bash 53 | jupyter-notbook 54 | ``` 55 | This should start a jupyter-notebook server and you should be able to open the tutorial notebooks in your browser. 56 | -------------------------------------------------------------------------------- /tutorials/build.sh: -------------------------------------------------------------------------------- 1 | docker build --tag encodermap . 2 | -------------------------------------------------------------------------------- /tutorials/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/slurm-node 2 | 3 | ##################### 4 | # PIPENV 5 | ##################### 6 | RUN mkdir -p /app 7 | WORKDIR /app 8 | COPY requirements.txt /app/base_requirements.txt 9 | COPY md_requirements.txt /app/md_requirements.txt 10 | COPY sphinx_requirements.txt /app/docs_requirements.txt 11 | COPY test_requirements.txt /app/test_requirements.txt 12 | RUN cat /app/*requirements.txt > /app/requirements.txt 13 | RUN python -m pip install -r /app/requirements.txt 14 | RUN #set -ex \ 15 | # && mkdir -p /usr/local/anaconda \ 16 | # && wget https://repo.anaconda.com/archive/Anaconda3-2023.03-1-Linux-x86_64.sh -O /usr/local/anaconda/Anaconda3-2023.03-1-Linux-x86_64.sh \ 17 | # && bash /usr/local/anaconda/Anaconda3-2023.03-1-Linux-x86_64.sh -b -p /usr/local/anaconda3 \ 18 | # && /usr/local/anaconda3/bin/python -m pip install -r requirements.txt 19 | 20 | ##################### 21 | # TESTS 22 | ##################### 23 | RUN mkdir -p /sh_libs 24 | COPY sh_libs/liblog.sh /sh_libs/liblog.sh 25 | COPY water_simulation/production.tpr /app/production.tpr 26 | COPY water_simulation/production_fails.tpr /app/production_fails.tpr 27 | COPY water_simulation/production_short.tpr /app/production_short.tpr 28 | -------------------------------------------------------------------------------- /tutorials/docker/README.md: -------------------------------------------------------------------------------- 1 | # Simulation Attender Tests 2 | 3 | These tests build a complete HPC network with SLURM/LDAP/SSH/ENVIRONMENT MODULES 4 | 5 | ## Important stuff at the start 6 | 7 | The nfs docker also needs the nfs kernel module. Do 8 | 9 | ```bash 10 | sudo modprobe {nfs,nfsd,rpcsec_gss_krb5} 11 | ``` 12 | 13 | to check, whether nfs is enabled in your kernel. If no error is raised, nfs is enabled in your kernel. 14 | 15 | ## Quickstart 16 | 17 | Start the container network with 18 | 19 | ```bash 20 | $ bash start_slurm.sh 21 | ``` 22 | 23 | ssh into the client machine 24 | 25 | ```bash 26 | $ ssh -p 222 localadmin@localhost 27 | ``` 28 | 29 | with the password `password` 30 | 31 | from there, you can log in into an ssh-machine which has gromacs installed: 32 | 33 | ```bash 34 | $ ssh gromacs@gromacs 35 | ``` 36 | 37 | with password `gromacs` 38 | 39 | or you can ssh into the SLURM cluster with 40 | 41 | ```bash 42 | $ ssh user01@cluster 43 | ``` 44 | 45 | with password `password1`. 46 | 47 | ## LDAP 48 | 49 | The users on the docker-composed SLURM cluster are centrally managed by an LDAP server. This server running openldap is provided by the `bitnami/openldap` docker image. 50 | 51 | ### Graphical interface for LDAP 52 | 53 | A graphical interface for the LDAP server is available under https://127.0.0.1:10443, use 54 | 55 | ````` 56 | user: cn=admin,dc=example,dc=org 57 | password: adminpassword 58 | ````` 59 | 60 | for login credentials. The graphical interface is provided by the `osixia/phpldapadmin` image. 61 | 62 | ### Log into LDAP server 63 | 64 | The LDAP server itself can be accessed via `docker exec -it openldap /bin/bash`. There, you can probe the LDAP configuration with some of the these commands: 65 | 66 | List everything: 67 | 68 | ```bash 69 | slapcat 70 | ldapsearch -H ldapi:/// -Y EXTERNAL -b "cn=config" -LLL -Q 71 | ``` 72 | 73 | List organizational units: 74 | 75 | ```bash 76 | ldapsearch -H ldapi:/// -Y EXTERNAL -b "dc=example,dc=org" -LLL -Q 77 | ``` 78 | 79 | List users: 80 | 81 | ```bash 82 | ldapsearch -H ldapi:/// -Y EXTERNAL -b "ou=users,dc=example,dc=org" -LLL -Q 83 | ``` 84 | 85 | ## MariaDB 86 | 87 | SLURM needs access to a sql database. Inside the docker compose environment, the database is provided by the `mariadb` image. 88 | 89 | ### Graphical interface for MariaDB 90 | 91 | You can inspect the database with a graphical user interface using these credentials on http://localhost:8080 92 | 93 | ``` 94 | Username: mysql_user 95 | Password: sql_passw0rd 96 | ``` 97 | 98 | ## Grafana 99 | 100 | ## Permissions 101 | 102 | Sometimes permissions of the persistent directories can make docker containers fail. I fixed it with 103 | 104 | ```bash 105 | chmod -R ugo+rwx openldap_data 106 | ``` 107 | 108 | But I have to admit, that this is not the most elegant solution. There needs to be a better way in the future. 109 | -------------------------------------------------------------------------------- /tutorials/docker/grafana/provisioning/datasources/prometheus_ds.yml: -------------------------------------------------------------------------------- 1 | datasources: 2 | - name: Prometheus 3 | access: proxy 4 | type: prometheus 5 | url: http://prometheus:9090 6 | isDefault: true 7 | -------------------------------------------------------------------------------- /tutorials/docker/ldap_client/ldap_client_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | . /opt/bitnami/scripts/liblog.sh 7 | 8 | # some debug for startup 9 | info "EncoderMap's LDAP client spooling up." 10 | info "Checking whether port 636 is open." 11 | if nc -z openldap 636 ; then 12 | info "Port is open." 13 | else 14 | error "Port is not open." 15 | exit 16 | fi 17 | 18 | # write the password 19 | info "Writing passwords to .secret files." 20 | echo $LDAP_ADMIN_PASSWORD > /etc/ldap.secret 21 | echo $LDAP_ADMIN_PASSWORD > /etc/pam_ldap.secret 22 | echo $LDAP_ADMIN_PASSWORD > /etc/libnss-ldap.secret 23 | unset LDAP_ADMIN_PASSWORD 24 | 25 | # wait for the nodes to spin up and create passwordless ssh 26 | /wait-for-it.sh openldap:636 --strict -- echo "openldap 636 is up" && /etc/init.d/ssh start && /etc/init.d/nscd restart && /etc/init.d/nslcd restart 27 | 28 | # use this, if the docker container automatically terminates, but you want to keep it running 29 | tail -f /dev/null 30 | -------------------------------------------------------------------------------- /tutorials/docker/ldap_client/nfs_server_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | # print uid 7 | id 8 | 9 | 10 | # wait for the nodes to spin up and create passwordless ssh 11 | # /wait-for-it.sh openldap:636 --strict -- echo "openldap.example.org 636 is up" 12 | 13 | # use this, if the docker container automatically terminates, but you want to keep it running 14 | tail -f /dev/null 15 | -------------------------------------------------------------------------------- /tutorials/docker/ldap_client/nslcd.conf: -------------------------------------------------------------------------------- 1 | # /etc/nslcd.conf 2 | # nslcd configuration file. See nslcd.conf(5) 3 | # for details. 4 | 5 | # The user and group nslcd should run as. 6 | uid nslcd 7 | gid nslcd 8 | 9 | # The location at which the LDAP server(s) should be reachable. 10 | uri ldap://openldap.example.org/ 11 | 12 | # The search base that will be used for all queries. 13 | base dc=example,dc=org 14 | 15 | # The LDAP protocol version to use. 16 | #ldap_version 3 17 | 18 | # The DN to bind with for normal lookups. 19 | #binddn cn=annonymous,dc=example,dc=net 20 | #bindpw secret 21 | 22 | # The DN used for password modifications by root. 23 | #rootpwmoddn cn=admin,dc=example,dc=com 24 | 25 | # SSL options 26 | #ssl off 27 | #tls_reqcert never 28 | tls_cacertfile /etc/ssl/certs/ca-certificates.crt 29 | 30 | # The search scope. 31 | #scope sub 32 | -------------------------------------------------------------------------------- /tutorials/docker/ldap_client/nsswitch.conf: -------------------------------------------------------------------------------- 1 | # /etc/nsswitch.conf 2 | # 3 | # Example configuration of GNU Name Service Switch functionality. 4 | # If you have the `glibc-doc-reference' and `info' packages installed, try: 5 | # `info libc "Name Service Switch"' for information about this file. 6 | 7 | passwd: files ldap 8 | group: files ldap 9 | shadow: files ldap 10 | gshadow: files 11 | 12 | hosts: files dns 13 | networks: files 14 | 15 | protocols: db files 16 | services: db files 17 | ethers: db files 18 | rpc: db files 19 | 20 | netgroup: nis 21 | -------------------------------------------------------------------------------- /tutorials/docker/ldap_client/pam.d/common-session: -------------------------------------------------------------------------------- 1 | # 2 | # /etc/pam.d/common-session - session-related modules common to all services 3 | # 4 | # This file is included from other service-specific PAM config files, 5 | # and should contain a list of modules that define tasks to be performed 6 | # at the start and end of interactive sessions. 7 | # 8 | # As of pam 1.0.1-6, this file is managed by pam-auth-update by default. 9 | # To take advantage of this, it is recommended that you configure any 10 | # local modules either before or after the default block, and use 11 | # pam-auth-update to manage selection of other modules. See 12 | # pam-auth-update(8) for details. 13 | 14 | # here are the per-package modules (the "Primary" block) 15 | session [default=1] pam_permit.so 16 | # here's the fallback if no module succeeds 17 | session requisite pam_deny.so 18 | # prime the stack with a positive return value if there isn't one already; 19 | # this avoids us returning an error just because nothing sets a success code 20 | # since the modules above will each just jump around 21 | session required pam_permit.so 22 | # The pam_umask module will set the umask according to the system default in 23 | # /etc/login.defs and user settings, solving the problem of different 24 | # umask settings with different shells, display managers, remote sessions etc. 25 | # See "man pam_umask". 26 | session optional pam_umask.so 27 | # and here are more per-package modules (the "Additional" block) 28 | session required pam_unix.so 29 | session [success=ok default=ignore] pam_ldap.so minimum_uid=1000 30 | # end of pam-auth-update config 31 | session optional pam_mkhomedir.so skel=/etc/skel umask=077 32 | -------------------------------------------------------------------------------- /tutorials/docker/ldap_server/custom.ldif: -------------------------------------------------------------------------------- 1 | # LDIF Export for dc=example,dc=org 2 | # Server: openldap.example.org (openldap.example.org) 3 | # Search Scope: sub 4 | # Search Filter: (objectClass=*) 5 | # Total Entries: 5 6 | # 7 | # Generated by phpLDAPadmin (http://phpldapadmin.sourceforge.net) on April 23, 2024 7:06 am 8 | # Version: 1.2.5 9 | 10 | version: 1 11 | 12 | # Entry 1: dc=example,dc=org 13 | dn: dc=example,dc=org 14 | dc: example 15 | o: example 16 | objectclass: dcObject 17 | objectclass: organization 18 | 19 | # Entry 2: ou=users,dc=example,dc=org 20 | dn: ou=users,dc=example,dc=org 21 | objectclass: organizationalUnit 22 | ou: users 23 | 24 | # Entry 3: cn=readers,ou=users,dc=example,dc=org 25 | dn: cn=readers,ou=users,dc=example,dc=org 26 | cn: readers 27 | member: cn=user01,ou=users,dc=example,dc=org 28 | member: cn=user02,ou=users,dc=example,dc=org 29 | objectclass: groupOfNames 30 | 31 | # Entry 4: cn=user01,ou=users,dc=example,dc=org 32 | dn: cn=user01,ou=users,dc=example,dc=org 33 | cn: User1 34 | cn: user01 35 | gidnumber: 1010 36 | homedirectory: /home/user01 37 | loginshell: /bin/bash 38 | objectclass: inetOrgPerson 39 | objectclass: posixAccount 40 | objectclass: shadowAccount 41 | sn: Bar1 42 | uid: user01 43 | uidnumber: 1010 44 | userpassword: password1 45 | 46 | # Entry 5: cn=user02,ou=users,dc=example,dc=org 47 | dn: cn=user02,ou=users,dc=example,dc=org 48 | cn: User2 49 | cn: user02 50 | gidnumber: 1011 51 | homedirectory: /home/user02 52 | loginshell: /bin/bash 53 | objectclass: inetOrgPerson 54 | objectclass: posixAccount 55 | objectclass: shadowAccount 56 | sn: Bar2 57 | uid: user02 58 | uidnumber: 1011 59 | userpassword: password2 60 | -------------------------------------------------------------------------------- /tutorials/docker/ldap_server/ldapmodify.sh: -------------------------------------------------------------------------------- 1 | ldapmodify -Y EXTERNAL < proc.err 1> proc.out & 6 | CMD_PID=$! 7 | echo $CMD_PID 8 | -------------------------------------------------------------------------------- /tutorials/docker/minimal_job.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --job-name=test 3 | #SBATCH --output=res.txt 4 | #SBATCH --partition=debug 5 | #SBATCH --time=10:00 6 | #SBATCH --ntasks=1 7 | #SBATCH --cpus-per-task=1 8 | #SBATCH --mem-per-cpu=100 9 | 10 | srun hostname 11 | srun pwd 12 | srun sleep 2 13 | -------------------------------------------------------------------------------- /tutorials/docker/modules/README.md: -------------------------------------------------------------------------------- 1 | # Environment modules in docker 2 | 3 | This docker-container is used as a base for a container with environment-modules installed in it. The `modules` command needs to be sourced first with 4 | 5 | ```bash 6 | $ source /usr/share/Modules/init/profile.sh 7 | ``` 8 | 9 | Gromacs is then available as a module with: 10 | 11 | ```bash 12 | module load gromacs/2023.1 13 | ``` 14 | 15 | ## Build args 16 | 17 | This container supports various build args: 18 | 19 | - GOSU_VERSION (standard 1.11) 20 | - ENVIRONMENT_MODULES_VERSION (standard 5.2.0) 21 | - CMAKE_VERSION (standard 3.26.3) 22 | - GMX_VERSION (standard 2023.1) 23 | 24 | Set them when building with: 25 | 26 | ```bash 27 | $ docker build --build-arg ENVIRONMENT_MODULES_VERSION="5.2.0" . 28 | ``` 29 | -------------------------------------------------------------------------------- /tutorials/docker/modules/build_and_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -m 4 | 5 | docker build -t modules-test --build-arg ENVIRONMENT_MODULES_VERSION="5.2.0" . 6 | docker run --rm modules-test 7 | -------------------------------------------------------------------------------- /tutorials/docker/modules/cmake_modulefile: -------------------------------------------------------------------------------- 1 | #%Module1.0 2 | # 3 | # cmake compilers and headers modulefile 4 | # Date: 5 | # Author: 6 | # 7 | set name "cmake Toolkit" 8 | set version "3.18.2" 9 | set arch "x86_64" 10 | set desc "cmake compilers and headers" 11 | set url "http://www.cmake.org/" 12 | set CMAKE_ROOT /home/$USER/bin/cmake-3.18.2 13 | 14 | proc ModulesHelp { } { 15 | global name 16 | global desc 17 | global url 18 | puts stderr "This module provides $desc\n" 19 | puts stderr "More information about $name can be found at:" 20 | puts stderr " $url\n" 21 | } 22 | 23 | module-whatis "Sets the environment for $desc" 24 | 25 | conflict cmake 26 | 27 | module load gcc/5.5.0 28 | module load mvapich2/gnu/ib/2.2 29 | 30 | prepend-path PATH $CMAKE_ROOT 31 | prepend-path PATH $CMAKE_ROOT/bin 32 | prepend-path PATH /usr/lib64 33 | setenv CMAKE_ROOT $CMAKE_ROOT 34 | -------------------------------------------------------------------------------- /tutorials/docker/modules/gromacs_modulefile: -------------------------------------------------------------------------------- 1 | #%Module 2 | set name "GROMACS" 3 | set version "GMX_VERSION" 4 | set arch "x86_64" 5 | set comp "gcc/5.5.0" 6 | set desc "$name ($version, $arch, $comp)" 7 | set url "http://www.gromacs.org/" 8 | set GMXPREFIX "/usr/local/gromacs" 9 | 10 | proc ModulesHelp { } { 11 | global name 12 | global desc 13 | global url 14 | puts stderr "This module provides $desc\n" 15 | puts stderr "More information about $name can be found at:" 16 | puts stderr " $url\n" 17 | } 18 | 19 | module-whatis "Sets the environment for $desc" 20 | 21 | set GROMACS_DIR ${GMXPREFIX} 22 | setenv GMXBIN ${GROMACS_DIR}/bin 23 | prepend-path PATH ${GROMACS_DIR}/bin 24 | setenv GMXLDLIB ${GROMACS_DIR}/lib 25 | prepend-path LD_LIBRARY_PATH ${GROMACS_DIR}/lib64 26 | prepend-path PKG_CONFIG_PATH ${GROMACS_DIR}/lib64/pkgconfig 27 | setenv GMXMAN ${GROMACS_DIR}/share/man 28 | prepend-path MANPATH ${GROMACS_DIR}/share/man 29 | setenv GMXDATA ${GROMACS_DIR}/share/gromacs 30 | setenv GMX_FONT 10x20 31 | -------------------------------------------------------------------------------- /tutorials/docker/modules/liblog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Library for logging functions 4 | 5 | # Constants 6 | RESET='\033[0m' 7 | RED='\033[38;5;1m' 8 | GREEN='\033[38;5;2m' 9 | YELLOW='\033[38;5;3m' 10 | MAGENTA='\033[38;5;5m' 11 | CYAN='\033[38;5;6m' 12 | 13 | # Functions 14 | 15 | ######################## 16 | # Print to STDERR 17 | # Arguments: 18 | # Message to print 19 | # Returns: 20 | # None 21 | ######################### 22 | stderr_print() { 23 | # 'is_boolean_yes' is defined in libvalidations.sh, but depends on this file so we cannot source it 24 | local bool="${BITNAMI_QUIET:-false}" 25 | # comparison is performed without regard to the case of alphabetic characters 26 | shopt -s nocasematch 27 | if ! [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then 28 | printf "%b\\n" "${*}" >&2 29 | fi 30 | } 31 | 32 | ######################## 33 | # Log message 34 | # Arguments: 35 | # Message to log 36 | # Returns: 37 | # None 38 | ######################### 39 | log() { 40 | stderr_print "${CYAN}${MODULE:-} ${MAGENTA}$(date "+%T.%2N ")${RESET}${*}" 41 | } 42 | ######################## 43 | # Log an 'info' message 44 | # Arguments: 45 | # Message to log 46 | # Returns: 47 | # None 48 | ######################### 49 | info() { 50 | log "${GREEN}INFO ${RESET} ==> ${*}" 51 | } 52 | ######################## 53 | # Log message 54 | # Arguments: 55 | # Message to log 56 | # Returns: 57 | # None 58 | ######################### 59 | warn() { 60 | log "${YELLOW}WARN ${RESET} ==> ${*}" 61 | } 62 | ######################## 63 | # Log an 'error' message 64 | # Arguments: 65 | # Message to log 66 | # Returns: 67 | # None 68 | ######################### 69 | error() { 70 | log "${RED}ERROR${RESET} ==> ${*}" 71 | } 72 | ######################## 73 | # Log a 'debug' message 74 | # Globals: 75 | # BITNAMI_DEBUG 76 | # Arguments: 77 | # None 78 | # Returns: 79 | # None 80 | ######################### 81 | debug() { 82 | # 'is_boolean_yes' is defined in libvalidations.sh, but depends on this file so we cannot source it 83 | local bool="${BITNAMI_DEBUG:-false}" 84 | # comparison is performed without regard to the case of alphabetic characters 85 | shopt -s nocasematch 86 | if [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then 87 | log "${MAGENTA}DEBUG${RESET} ==> ${*}" 88 | fi 89 | } 90 | 91 | ######################## 92 | # Indent a string 93 | # Arguments: 94 | # $1 - string 95 | # $2 - number of indentation characters (default: 4) 96 | # $3 - indentation character (default: " ") 97 | # Returns: 98 | # None 99 | ######################### 100 | indent() { 101 | local string="${1:-}" 102 | local num="${2:?missing num}" 103 | local char="${3:-" "}" 104 | # Build the indentation unit string 105 | local indent_unit="" 106 | for ((i = 0; i < num; i++)); do 107 | indent_unit="${indent_unit}${char}" 108 | done 109 | # shellcheck disable=SC2001 110 | # Complex regex, see https://github.com/koalaman/shellcheck/wiki/SC2001#exceptions 111 | echo "$string" | sed "s/^/${indent_unit}/" 112 | } 113 | -------------------------------------------------------------------------------- /tutorials/docker/modules/profile.sh: -------------------------------------------------------------------------------- 1 | # /etc/profile: system-wide .profile file for the Bourne shell (sh(1)) 2 | # and Bourne compatible shells (bash(1), ksh(1), ash(1), ...). 3 | 4 | source /etc/profile.d/modules.sh 5 | export GMX_VERSION="SED_GMX_VERSION" 6 | 7 | if [ "${PS1-}" ]; then 8 | if [ "${BASH-}" ] && [ "$BASH" != "/bin/sh" ]; then 9 | # The file bash.bashrc already sets the default PS1. 10 | # PS1='\h:\w\$ ' 11 | if [ -f /etc/bash.bashrc ]; then 12 | . /etc/bash.bashrc 13 | fi 14 | else 15 | if [ "$(id -u)" -eq 0 ]; then 16 | PS1='# ' 17 | else 18 | PS1='$ ' 19 | fi 20 | fi 21 | fi 22 | 23 | if [ -d /etc/profile.d ]; then 24 | for i in /etc/profile.d/*.sh; do 25 | if [ -r $i ]; then 26 | . $i 27 | fi 28 | done 29 | unset i 30 | fi 31 | -------------------------------------------------------------------------------- /tutorials/docker/modules/test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -m 4 | . /sh_libs/liblog.sh 5 | 6 | info "Testing the base image." 7 | info "Testing current user." 8 | current_user=$(whoami) 9 | if [[ $current_user == "encodermap" ]] ; then 10 | info "Correct user selected." 11 | else 12 | error "Tests not running with the encodermap user. Current user is ${current_user}. Make sure to call docker with -u=1000." 13 | error "Here's a cat of /etc/passwd. Check, whether the encodermap user is even listed here." 14 | cat /etc/passwd 15 | fi 16 | 17 | info "Testing bash_profile" 18 | if [ -f ~/.profile ] ; then 19 | info "User $current_user has a ~/.profile file." 20 | else 21 | error "The user doesn't have a ~/.profile file. Environment modules can't load without it." 22 | exit 23 | fi 24 | 25 | info "Testing user's login-shell" 26 | login_shell=$(awk -F: -v user=$current_user '$1 == user {print $NF}' /etc/passwd) 27 | if [[ $login_shell == "/usr/bin/bash" ]] ; then 28 | info "User has correct login shell ${login_shell}." 29 | else 30 | error "User has wrong login shell ${login_shell}." 31 | exit 32 | fi 33 | 34 | info "Testing whether environment modules is available" 35 | if command -v module &> /dev/null ; then 36 | info "Environment modules is available. Sourcing gromacs now." 37 | else 38 | error "Environment modules is either not available or not sourced." 39 | exit 40 | fi 41 | 42 | module load gromacs/$GMX_VERSION 43 | if command -v gmx &> /dev/null ; then 44 | info "Gromacs sourced via environment modules." 45 | else 46 | error "Gromacs was not sourced." 47 | exit 48 | fi 49 | info "All tests passed." 50 | -------------------------------------------------------------------------------- /tutorials/docker/modules/test_modules_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | # load libs 7 | . /sh_libs/liblog.sh 8 | 9 | # print uid 10 | info "Running a test docker container with environment-modules installed." 11 | info "Sourcing /usr/share/Modules/init/profile.sh" 12 | source /usr/share/Modules/init/profile.sh 13 | info "Finished. Happy Testing." 14 | -------------------------------------------------------------------------------- /tutorials/docker/sh_libs/liblog.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Library for logging functions 4 | 5 | # Constants 6 | RESET='\033[0m' 7 | RED='\033[38;5;1m' 8 | GREEN='\033[38;5;2m' 9 | YELLOW='\033[38;5;3m' 10 | MAGENTA='\033[38;5;5m' 11 | CYAN='\033[38;5;6m' 12 | 13 | # Functions 14 | 15 | ######################## 16 | # Print to STDERR 17 | # Arguments: 18 | # Message to print 19 | # Returns: 20 | # None 21 | ######################### 22 | stderr_print() { 23 | # 'is_boolean_yes' is defined in libvalidations.sh, but depends on this file so we cannot source it 24 | local bool="${BITNAMI_QUIET:-false}" 25 | # comparison is performed without regard to the case of alphabetic characters 26 | shopt -s nocasematch 27 | if ! [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then 28 | printf "%b\\n" "${*}" >&2 29 | fi 30 | } 31 | 32 | ######################## 33 | # Log message 34 | # Arguments: 35 | # Message to log 36 | # Returns: 37 | # None 38 | ######################### 39 | log() { 40 | stderr_print "${CYAN}${MODULE:-} ${MAGENTA}$(date "+%T.%2N ")${RESET}${*}" 41 | } 42 | ######################## 43 | # Log an 'info' message 44 | # Arguments: 45 | # Message to log 46 | # Returns: 47 | # None 48 | ######################### 49 | info() { 50 | log "${GREEN}INFO ${RESET} ==> ${*}" 51 | } 52 | ######################## 53 | # Log message 54 | # Arguments: 55 | # Message to log 56 | # Returns: 57 | # None 58 | ######################### 59 | warn() { 60 | log "${YELLOW}WARN ${RESET} ==> ${*}" 61 | } 62 | ######################## 63 | # Log an 'error' message 64 | # Arguments: 65 | # Message to log 66 | # Returns: 67 | # None 68 | ######################### 69 | error() { 70 | log "${RED}ERROR${RESET} ==> ${*}" 71 | } 72 | ######################## 73 | # Log a 'debug' message 74 | # Globals: 75 | # BITNAMI_DEBUG 76 | # Arguments: 77 | # None 78 | # Returns: 79 | # None 80 | ######################### 81 | debug() { 82 | # 'is_boolean_yes' is defined in libvalidations.sh, but depends on this file so we cannot source it 83 | local bool="${BITNAMI_DEBUG:-false}" 84 | # comparison is performed without regard to the case of alphabetic characters 85 | shopt -s nocasematch 86 | if [[ "$bool" = 1 || "$bool" =~ ^(yes|true)$ ]]; then 87 | log "${MAGENTA}DEBUG${RESET} ==> ${*}" 88 | fi 89 | } 90 | 91 | ######################## 92 | # Indent a string 93 | # Arguments: 94 | # $1 - string 95 | # $2 - number of indentation characters (default: 4) 96 | # $3 - indentation character (default: " ") 97 | # Returns: 98 | # None 99 | ######################### 100 | indent() { 101 | local string="${1:-}" 102 | local num="${2:?missing num}" 103 | local char="${3:-" "}" 104 | # Build the indentation unit string 105 | local indent_unit="" 106 | for ((i = 0; i < num; i++)); do 107 | indent_unit="${indent_unit}${char}" 108 | done 109 | # shellcheck disable=SC2001 110 | # Complex regex, see https://github.com/koalaman/shellcheck/wiki/SC2001#exceptions 111 | echo "$string" | sed "s/^/${indent_unit}/" 112 | } 113 | -------------------------------------------------------------------------------- /tutorials/docker/short_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | . sh_libs/liblog.sh 4 | 5 | info "Killing gmx commands" 6 | pkill gmx 7 | info "Deleting test_sims/ and sims.h5 and .sims*" 8 | rm -rf test_sims/ 9 | rm -f sims.h5 10 | rm -f .sims* 11 | info "Creating dirs" 12 | mkdir -p test_sims/sim1 13 | mkdir -p test_sims/sim2 14 | mkdir -p test_sims/sim3 15 | info "Running gmx" 16 | gmx grompp -f water_simulation/production.mdp -c water_simulation/spc216_stacked.gro -p water_simulation/topol_stacked.top -o test_sims/sim1/production.tpr -po test_sims/sim1/mdout.mdp -maxwarn 1 &> /dev/null 17 | gmx grompp -f water_simulation/production.mdp -c water_simulation/spc216_fails.gro -p water_simulation/topol_stacked.top -o test_sims/sim2/producton_fails.tpr -po test_sims/sim2/mdout.mdp -maxwarn 1 &> /dev/null 18 | gmx grompp -f water_simulation/production_short.mdp -c water_simulation/spc216_stacked.gro -p water_simulation/topol_stacked.top -o test_sims/sim3/production_short.tpr -po test_sims/sim3/mdout.mdp -maxwarn 1 &> /dev/null 19 | cp ../simulation_attender/simulation_attender.py . 20 | exit 21 | info "Running sim_attender commands" 22 | python simulation_attender.py --help 23 | python simulation_attender.py collect test_sims/ 24 | python simulation_attender.py template --command "gmx mdrun -deffnm {{ stem }}" 25 | python simulation_attender.py submit -cm local 26 | python simulation_attender.py run -cm local 27 | info "Checking again for completion or crash" 28 | python simulation_attender.py run -cm local 29 | info "Sleeping for 15 seconds" 30 | sleep 15 31 | info "Checking again" 32 | python simulation_attender.py run -cm local 33 | sleep 6 34 | info "Checking short simulation." 35 | python simulation_attender.py run -cm local 36 | sleep 6 37 | info "Checking failed simulation." 38 | python simulation_attender.py run -cm local 39 | info "Clearing workspace" 40 | rm -r test_sims/ 41 | rm sims.h5 42 | rm .sims* 43 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/README.md: -------------------------------------------------------------------------------- 1 | # slurm-master 2 | 3 | access the mariadb server running in another container via: 4 | 5 | ```bash 6 | mysql -u slurm -p some_pass -h slurm-db.local.dev 7 | ``` 8 | 9 | Compose with: 10 | 11 | docker-compose up -d --build 12 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/base/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | MAINTAINER kevinsawade 3 | 4 | # install munge packages 5 | RUN apt-get update \ 6 | && apt-get install -y --no-install-recommends \ 7 | sudo 8 | 9 | # set up munge for slurm following this tutorial: 10 | # https://southgreenplatform.github.io/trainings/hpc/slurminstallation/ 11 | ENV MUNGEUSER=1001 12 | RUN groupadd -g $MUNGEUSER munge 13 | RUN useradd -m -c "Munge Uid 'N' Gid Emporium" -d /var/lib/munge -u $MUNGEUSER -g munge -s /sbin/nologin munge 14 | ENV SLURMUSER=1002 15 | RUN groupadd -g $SLURMUSER slurm 16 | RUN useradd -m -c "SLURM workload manager" -d /var/lib/slurm -u $SLURMUSER -g slurm -s /bin/bash slurm 17 | 18 | # create admin user which has sudo priviliges 19 | RUN useradd -ms /bin/bash admin 20 | RUN usermod -aG sudo admin 21 | RUN echo "admin:adminpassword" | chpasswd 22 | RUN echo 'admin ALL=(ALL:ALL) ALL' >> /etc/sudoers 23 | RUN chown -R admin /home/admin 24 | 25 | # install munge packages 26 | RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata 27 | RUN apt-get update && apt-get install -y --no-install-recommends \ 28 | vim \ 29 | munge \ 30 | git \ 31 | wget \ 32 | mysql-client \ 33 | libmunge-dev \ 34 | libpam0g-dev \ 35 | perl \ 36 | python3 \ 37 | python3-dev \ 38 | libreadline-dev \ 39 | openssh-server \ 40 | sshpass \ 41 | netcat \ 42 | libslurm-dev 43 | 44 | # make changes to openssh 45 | RUN mkdir -p /var/run/sshd 46 | RUN echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config 47 | RUN echo "PermitRootLogin no" >> /etc/ssh/sshd_config 48 | RUN if [ ! -d /var/run/sshd ] ; then mkdir /var/run/sshd ; ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key -N '' ; fi 49 | RUN chown admin /etc/ssh/ssh_host_rsa_key 50 | RUN chown admin /etc/ssh/ssh_host_rsa_key.pub 51 | COPY ssh_config /home/admin/.ssh/config 52 | RUN mkdir -p /home/admin/.ssh \ 53 | && chmod 0700 /home/admin/.ssh \ 54 | && ssh-keygen -b 2048 -t rsa -f /home/admin/.ssh/id_rsa -q -N "" -C "admin@$(hostname)-$(date -I)" \ 55 | && touch /home/admin/.ssh/authorized_keys \ 56 | && cat /home/admin/.ssh/id_rsa.pub > /home/admin/.ssh/authorized_keys \ 57 | && chmod 0640 home/admin/.ssh/authorized_keys \ 58 | && chown -R admin /home/admin/.ssh \ 59 | && chmod 0644 home/admin/.ssh/config 60 | 61 | 62 | # copy the munge-key 63 | COPY munge.key /etc/munge/munge.key 64 | 65 | # fix permissions on munge stuff 66 | RUN mkdir -p /run/munge 67 | RUN chown -R munge: /etc/munge/ /var/log/munge/ /var/lib/munge/ /run/munge/ 68 | RUN chmod 0700 /etc/munge/ /var/log/munge/ 69 | RUN chmod 0711 /var/lib/munge/ 70 | RUN chmod 0755 /var/run/munge/ 71 | RUN chmod a+x /run/munge 72 | 73 | # cgroup stuff 74 | RUN mkdir -p /etc/slurm/cgroup 75 | RUN chown -R slurm: /etc/slurm/cgroup 76 | 77 | # expose some ports 78 | EXPOSE 22 6817 6818 6819 79 | 80 | # set up tini and wait-for-it.sh 81 | ADD https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh /wait-for-it.sh 82 | RUN chmod a+rx /wait-for-it.sh 83 | 84 | ENV TINI_VERSION v0.19.0 85 | ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini 86 | RUN chmod a+rx /tini 87 | ENTRYPOINT ["/tini", "--"] 88 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/cgroup.conf: -------------------------------------------------------------------------------- 1 | CgroupAutomount=yes 2 | ConstrainCores=no 3 | ConstrainRAMSpace=no 4 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/cgroup_allowed_devices_file.conf: -------------------------------------------------------------------------------- 1 | /dev/null 2 | /dev/urandom 3 | /dev/zero 4 | /dev/sda* 5 | /dev/cpu/*/* 6 | /dev/pts/* 7 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/munge.key: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/docker/slurm-old/munge.key -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurm_acct_db.sql: -------------------------------------------------------------------------------- 1 | create database slurm_acct_db; 2 | create user 'slurm'@'%'; 3 | set password for 'slurm'@'%' = password('some_pass'); 4 | grant usage on *.* to 'slurm'@'%'; 5 | grant all privileges on slurm_acct_db.* to 'slurm'@'%'; 6 | flush privileges; 7 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurm_db/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/slurm_base 2 | MAINTAINER kevinsawade 3 | 4 | # install slurm-required packages 5 | RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata 6 | RUN apt-get install -y \ 7 | openssl \ 8 | libssl-dev \ 9 | rpm \ 10 | numactl \ 11 | hwloc \ 12 | lua5.3 \ 13 | lua5.3-dev \ 14 | lua5.3-filesystem \ 15 | lua5.3-posix \ 16 | man2html \ 17 | libibmad-dev \ 18 | libibumad-dev \ 19 | slurmdbd \ 20 | make \ 21 | mysql-client 22 | 23 | # fix permissions on slurm stuff 24 | RUN mkdir -p /etc/slurm-llnl 25 | RUN chmod 777 /etc/slurm-llnl 26 | 27 | # ENV DBD_ADDR=database \ 28 | # DBD_HOST=database \ 29 | # DBD_PORT=6819 \ 30 | # STORAGE_HOST=database.local.dev \ 31 | # STORAGE_PORT=3306 \ 32 | # STORAGE_PASS=password \ 33 | # STORAGE_USER=slurm 34 | 35 | # set up slurmdbd 36 | RUN mkdir -p /var/spool/slurm/d 37 | RUN mkdir -p /var/log/slurm 38 | RUN chown slurm: /var/spool/slurm/d 39 | RUN chown slurm: /var/log/slurm 40 | RUN mkdir -p /var/spool/slurmd 41 | COPY slurmdbd.conf /etc/slurm-llnl/slurmdbd.conf 42 | 43 | # set up slurmctld 44 | RUN mkdir /var/spool/slurmctld 45 | RUN chown slurm:slurm /var/spool/slurmctld 46 | RUN chmod 755 /var/spool/slurmctld 47 | RUN mkdir -p /var/log/slurm 48 | RUN touch /var/log/slurm/slurmctld.log 49 | RUN touch /var/log/slurm/slurm_jobacct.log /var/log/slurm/slurm_jobcomp.log 50 | RUN chown -R slurm: /var/log/slurm/ 51 | COPY slurm.conf /etc/slurm-llnl/slurm.conf 52 | COPY cgroup.conf /etc/slurm-llnl/cgroup.conf 53 | 54 | 55 | # ENV USE_SLURMDBD=true \ 56 | # CLUSTER_NAME=snowflake \ 57 | # CONTROL_MACHINE=controller \ 58 | # SLURMCTLD_PORT=6817 \ 59 | # SLURMD_PORT=6818 \ 60 | # ACCOUNTING_STORAGE_HOST=database \ 61 | # ACCOUNTING_STORAGE_PORT=6819 \ 62 | # PARTITION_NAME=docker 63 | 64 | # run the entrypoint 65 | COPY slurm_db_entrypoint.sh slurm_db_entrypoint.sh 66 | RUN chmod a+wrx slurm_db_entrypoint.sh 67 | COPY slurm_acct_db.sql slurm_acct_db.sql 68 | RUN chmod a+wrx slurm_acct_db.sql 69 | CMD ["/slurm_db_entrypoint.sh"] 70 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurm_db_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | # bring up sshd 7 | /usr/sbin/sshd 8 | 9 | # print uid 10 | id 11 | 12 | # wait for the sql-server to be available and add slurm to the database 13 | MYSQL_ROOT_PASSWORD=sql_root_passw0rd 14 | /wait-for-it.sh slurm-db.local.dev:3306 --strict -- echo "slurm-db.local.dev db(3306) is up" ; mysql -h slurm-db.local.dev -u root -p$MYSQL_ROOT_PASSWORD < /slurm_acct_db.sql 15 | 16 | # start the munge daeomon 17 | sudo -u munge service munge start 18 | # su -u munge /sbin/munged 19 | munge -n 20 | munge -n | unmunge 21 | remunge 22 | 23 | # start the slurmdb daemon 24 | sudo -u slurm service slurmdbd start 25 | 26 | # now we bring the primary process back into the foreground 27 | # and leave it there 28 | # use the forgrounding of a process, if the process docker is running doesn't automatically terminate (i.e. webserver) 29 | # fg %1 30 | 31 | # use this, if the docker container automatically terminates, but you want to keep it running 32 | tail -f /dev/null 33 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurm_master/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/slurm_base 2 | MAINTAINER kevinsawade 3 | 4 | # install slurm-required packages 5 | RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata 6 | RUN apt-get update && apt-get install -y \ 7 | openssl \ 8 | libssl-dev \ 9 | rpm \ 10 | numactl \ 11 | hwloc \ 12 | lua5.3 \ 13 | lua5.3-dev \ 14 | lua5.3-filesystem \ 15 | lua5.3-posix \ 16 | man2html \ 17 | libibmad-dev \ 18 | libibumad-dev \ 19 | slurmd \ 20 | slurmctld \ 21 | slurmdbd \ 22 | make \ 23 | mysql-client \ 24 | python3 \ 25 | python3-dev \ 26 | python3-pip 27 | 28 | # fix permissions on slurm stuff 29 | RUN mkdir -p /etc/slurm-llnl 30 | RUN chmod 777 /etc/slurm-llnl 31 | 32 | # set up slurmdbd 33 | RUN mkdir -p /var/spool/slurm/d 34 | RUN mkdir -p /var/log/slurm 35 | RUN chown slurm: /var/spool/slurm/d 36 | RUN chown slurm: /var/log/slurm 37 | RUN mkdir -p /var/spool/slurmd 38 | COPY slurmdbd.conf /etc/slurm-llnl/slurmdbd.conf 39 | 40 | # # set up slurmctld 41 | RUN mkdir /var/spool/slurmctld 42 | RUN chown slurm:slurm /var/spool/slurmctld 43 | RUN chmod 755 /var/spool/slurmctld 44 | RUN mkdir -p /var/log/slurm 45 | RUN touch /var/log/slurmctld.log 46 | RUN chown slurm: /var/log/slurmctld.log 47 | RUN touch /var/log/slurm_jobacct.log /var/log/slurm_jobcomp.log 48 | RUN chown slurm: /var/log/slurm* 49 | RUN chown slurm: /var/log/slurm/ 50 | COPY slurm.conf /etc/slurm-llnl/slurm.conf 51 | COPY cgroup.conf /etc/slurm-llnl/cgroup.conf 52 | COPY cgroup_allowed_devices_file.conf /etc/slurm/cgroup_allowed_devices_file.conf 53 | 54 | # install slurm-web 55 | RUN pip install flask 56 | RUN pip install cython 57 | RUN git clone https://github.com/PySlurm/pyslurm.git 58 | RUN ln -sf /usr/include/slurm-wlm/ /usr/include/slurm 59 | RUN mkdir /usr/lib/slurm 60 | RUN ln -sf /usr/lib/x86_64-linux-gnu /usr/lib/slurm/lib 61 | # RUN python3 pyslurm/setup.py build 62 | # RUN python3 pyslurm/setup.py build --slurm-inc=/usr/include --slurm-lib=/usr/lib/slurm 63 | # RUN python3 pyslurm/setup.py build --slurm-inc=/usr/include/slurm-wlm --slurm-lib=/usr/lib/x86_64-linux-gnu/ 64 | # RUN python3 pyslurm/setup.py install 65 | # RUN git clone https://github.com/edf-hpc/slurm-web.git 66 | 67 | 68 | # ENV USE_SLURMDBD=true \ 69 | # CLUSTER_NAME=snowflake \ 70 | # CONTROL_MACHINE=controller \ 71 | # SLURMCTLD_PORT=6817 \ 72 | # SLURMD_PORT=6818 \ 73 | # ACCOUNTING_STORAGE_HOST=database \ 74 | # ACCOUNTING_STORAGE_PORT=6819 \ 75 | # PARTITION_NAME=docker 76 | 77 | COPY slurm_master_entrypoint.sh slurm_master_entrypoint.sh 78 | RUN chmod a+wrx slurm_master_entrypoint.sh 79 | CMD ["/slurm_master_entrypoint.sh"] 80 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurm_master_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | # bring up sshd 7 | /usr/sbin/sshd 8 | 9 | # print uid 10 | id 11 | 12 | # wait for the nodes to spin up and create passwordless ssh 13 | /wait-for-it.sh slurm-node1.local.dev:22 --strict -- echo "slurm-node1.local.dev ssh(22) is up" ; sshpass -p adminpassword ssh-copy-id -i /etc/ssh/ssh_host_rsa_key.pub slurm-node1.local.dev 14 | /wait-for-it.sh slurm-node2.local.dev:22 --strict -- echo "slurm-node2.local.dev ssh(22) is up" ; sshpass -p adminpassword ssh-copy-id -i /etc/ssh/ssh_host_rsa_key.pub slurm-node2.local.dev 15 | /wait-for-it.sh slurm-login-node.local.dev:22 --strict -- echo "slurm-login-node.local.dev ssh(22) is up" ; sshpass -p adminpassword ssh-copy-id -i /etc/ssh/ssh_host_rsa_key.pub slurm-login-node.local.dev 16 | 17 | # wait for the slurmdbd to spin up 18 | /wait-for-it.sh slurm-database.local.dev:6819 --strict -- echo "slurm-database.local.dev db(6819) is up" 19 | 20 | # start the munge daeomon 21 | service munge start 22 | su -u munge /sbin/munged 23 | munge -n 24 | munge -n | unmunge 25 | remunge 26 | 27 | # fix stuff in the slurm configuration 28 | sed -i "s/REPLACE_IT/CPUs=$(nproc)/g" /etc/slurm-llnl/slurm.conf 29 | 30 | 31 | # start the slurm control daemons 32 | sacctmgr -i add_cluster "cluster" 33 | sleep 2s 34 | /wait-for-it.sh slurm-node1.local.dev:6818 --strict -- service slurmctld start 35 | 36 | # now we bring the primary process back into the foreground 37 | # and leave it there 38 | # use the forgrounding of a process, if the process docker is running doesn't automatically terminate (i.e. webserver) 39 | # fg %1 40 | 41 | # use this, if the docker container automatically terminates, but you want to keep it running 42 | tail -f /dev/null 43 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurm_node/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/slurm_base 2 | MAINTAINER kevinsawade 3 | 4 | # install slurm-required packages 5 | RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata 6 | RUN apt-get install -y \ 7 | openssl \ 8 | libssl-dev \ 9 | rpm \ 10 | numactl \ 11 | hwloc \ 12 | lua5.3 \ 13 | lua5.3-dev \ 14 | lua5.3-filesystem \ 15 | lua5.3-posix \ 16 | man2html \ 17 | libibmad-dev \ 18 | libibumad-dev \ 19 | slurmd \ 20 | slurmctld \ 21 | slurmdbd \ 22 | make \ 23 | mysql-client 24 | 25 | # fix permissions on slurm stuff 26 | RUN mkdir -p /etc/slurm-llnl 27 | RUN chmod 777 /etc/slurm-llnl 28 | 29 | # set up slurmdbd 30 | RUN mkdir -p /var/spool/slurm/d 31 | RUN mkdir -p /var/log/slurm 32 | RUN chown slurm: /var/spool/slurm/d 33 | RUN chown slurm: /var/log/slurm 34 | RUN mkdir -p /var/spool/slurmd 35 | COPY slurmdbd.conf /etc/slurm-llnl/slurmdbd.conf 36 | 37 | # # set up slurmctld 38 | RUN mkdir /var/spool/slurmctld 39 | RUN chown slurm:slurm /var/spool/slurmctld 40 | RUN chmod 755 /var/spool/slurmctld 41 | RUN mkdir -p /var/log/slurm 42 | RUN touch /var/log/slurm/slurmctld.log 43 | RUN touch /var/log/slurm/slurm_jobacct.log /var/log/slurm/slurm_jobcomp.log 44 | RUN chown -R slurm: /var/log/slurm/ 45 | COPY slurm.conf /etc/slurm-llnl/slurm.conf 46 | COPY cgroup.conf /etc/slurm-llnl/cgroup.conf 47 | COPY cgroup_allowed_devices_file.conf /etc/slurm/cgroup_allowed_devices_file.conf 48 | 49 | # set up slurmd 50 | RUN mkdir -p /var/spool/slurmd 51 | RUN chown slurm:slurm /var/spool/slurmd 52 | RUN chmod 755 /var/spool/slurmd 53 | RUN touch /var/log/slurmd.log 54 | RUN chown -R slurm: /var/log/slurmd.log 55 | 56 | # run the entrypoint 57 | COPY slurm_node_entrypoint.sh slurm_node_entrypoint.sh 58 | RUN chmod a+wrx slurm_node_w_prometheus_entrypoint.sh 59 | CMD ["/slurm_node_entrypoint.sh"] 60 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurm_node_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | # bring up sshd 7 | /usr/sbin/sshd 8 | 9 | # print uid 10 | id 11 | 12 | # start the munge daeomon 13 | service munge start 14 | su -u munge /sbin/munged 15 | munge -n 16 | munge -n | unmunge 17 | remunge 18 | 19 | # replace nproc 20 | sed -i "s/REPLACE_IT/CPUs=$(nproc)/g" /etc/slurm-llnl/slurm.conf 21 | 22 | # start the slurm daemon 23 | service slurmd start 24 | 25 | # now we bring the primary process back into the foreground 26 | # and leave it there 27 | # use the forgrounding of a process, if the process docker is running doesn't automatically terminate (i.e. webserver) 28 | # fg %1 29 | 30 | # use this, if the docker container automatically terminates, but you want to keep it running 31 | tail -f /dev/null 32 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurm_version.h.in: -------------------------------------------------------------------------------- 1 | /*****************************************************************************\ 2 | * slurm_version.h - Slurm version info 3 | ***************************************************************************** 4 | * Copyright (C) 2021 SchedMD LLC. 5 | * Written by Tim Wickberg 6 | * 7 | * This file is part of Slurm, a resource management program. 8 | * For details, see . 9 | * Please also read the included file: DISCLAIMER. 10 | * 11 | * Slurm is free software; you can redistribute it and/or modify it under 12 | * the terms of the GNU General Public License as published by the Free 13 | * Software Foundation; either version 2 of the License, or (at your option) 14 | * any later version. 15 | * 16 | * In addition, as a special exception, the copyright holders give permission 17 | * to link the code of portions of this program with the OpenSSL library under 18 | * certain conditions as described in each individual source file, and 19 | * distribute linked combinations including the two. You must obey the GNU 20 | * General Public License in all respects for all of the code used other than 21 | * OpenSSL. If you modify file(s) with this exception, you may extend this 22 | * exception to your version of the file(s), but you are not obligated to do 23 | * so. If you do not wish to do so, delete this exception statement from your 24 | * version. If you delete this exception statement from all source files in 25 | * the program, then also delete it here. 26 | * 27 | * Slurm is distributed in the hope that it will be useful, but WITHOUT ANY 28 | * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 29 | * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 30 | * details. 31 | * 32 | * You should have received a copy of the GNU General Public License along 33 | * with Slurm; if not, write to the Free Software Foundation, Inc., 34 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 35 | \*****************************************************************************/ 36 | 37 | #ifndef _SLURM_VERSION_H_ 38 | #define _SLURM_VERSION_H_ 39 | 40 | #undef SLURM_VERSION_NUMBER 41 | 42 | /* 43 | * Define Slurm version number. 44 | * High-order byte is major version. 45 | * Middle byte is minor version. 46 | * Low-order byte is micro version 47 | * (Excludes "-pre#" component of micro version used in pre-releases.) 48 | * 49 | * Use SLURM_VERSION_NUM macro to compare versions, for example 50 | * #if SLURM_VERSION_NUMBER > SLURM_VERSION_NUM(2,1,0) 51 | */ 52 | #define SLURM_VERSION_NUM(a,b,c) (((a) << 16) + ((b) << 8) + (c)) 53 | #define SLURM_VERSION_MAJOR(a) (((a) >> 16) & 0xff) 54 | #define SLURM_VERSION_MINOR(a) (((a) >> 8) & 0xff) 55 | #define SLURM_VERSION_MICRO(a) ((a) & 0xff) 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/slurmdbd.conf: -------------------------------------------------------------------------------- 1 | AuthType=auth/munge 2 | AuthInfo=/var/run/munge/munge.socket.2 3 | DbdAddr=slurm-database 4 | DbdHost=slurm-database 5 | SlurmUser=slurm 6 | DebugLevel=4 7 | DbdPort=6819 8 | LogFile=/var/log/slurm/slurmdbd.log 9 | PidFile=/var/run/slurmdbd.pid 10 | StorageType=accounting_storage/mysql 11 | StorageHost=slurm-db.local.dev 12 | StoragePort=3306 13 | StoragePass=some_pass 14 | StorageUser=slurm 15 | StorageLoc=slurm_acct_db 16 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/ssh_config: -------------------------------------------------------------------------------- 1 | Host * 2 | StrictHostKeyChecking no 3 | UserKnownHostsFile /dev/null 4 | LogLevel QUIET 5 | -------------------------------------------------------------------------------- /tutorials/docker/slurm-old/teardown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker-compose stop 4 | docker-compose down 5 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_base/cgroup.conf: -------------------------------------------------------------------------------- 1 | CgroupAutomount=yes 2 | # CgroupPlugin=cgroup/v2 3 | # CgroupMountpoint=/etc/cgroup_devel 4 | # IgnoreSystemd=yes 5 | # IgnoreSystemdOnFailure=yes 6 | # CgroupReleaseAgentDir="/etc/slurm/cgroup" 7 | 8 | ConstrainCores=yes 9 | ConstrainDevices=yes 10 | # TaskAffinity=yes 11 | ConstrainRAMSpace=yes 12 | # ConstrainSwapSpace=yes 13 | MaxRAMPercent=98 14 | AllowedSwapSpace=0 15 | AllowedRAMSpace=100 16 | MemorySwappiness=0 17 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_base/cgroup_allowed_devices_file.conf: -------------------------------------------------------------------------------- 1 | /dev/null 2 | /dev/urandom 3 | /dev/zero 4 | /dev/sda* 5 | /dev/cpu/*/* 6 | /dev/pts/* 7 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_base/slurm.conf: -------------------------------------------------------------------------------- 1 | # slurm.conf 2 | # 3 | # See the slurm.conf man page for more information. 4 | # 5 | ClusterName=linux 6 | ControlMachine=slurm-master 7 | ControlAddr=slurm-master 8 | #BackupController= 9 | #BackupAddr= 10 | # 11 | SlurmUser=slurm 12 | #SlurmdUser=root 13 | SlurmctldPort=6817 14 | SlurmdPort=6818 15 | AuthType=auth/munge 16 | #JobCredentialPrivateKey= 17 | #JobCredentialPublicCertificate= 18 | StateSaveLocation=/var/lib/slurmd 19 | SlurmdSpoolDir=/var/spool/slurmd 20 | SwitchType=switch/none 21 | MpiDefault=none 22 | SlurmctldPidFile=/var/run/slurmd/slurmctld.pid 23 | SlurmdPidFile=/var/run/slurmd/slurmd.pid 24 | ProctrackType=proctrack/linuxproc 25 | #PluginDir= 26 | #CacheGroups=0 27 | #FirstJobId= 28 | ReturnToService=0 29 | #MaxJobCount= 30 | #PlugStackConfig= 31 | #PropagatePrioProcess= 32 | #PropagateResourceLimits= 33 | #PropagateResourceLimitsExcept= 34 | #Prolog= 35 | #Epilog= 36 | #SrunProlog= 37 | #SrunEpilog= 38 | #TaskProlog= 39 | #TaskEpilog= 40 | #TaskPlugin= 41 | #TrackWCKey=no 42 | #TreeWidth=50 43 | #TmpFS= 44 | #UsePAM= 45 | # 46 | # TIMERS 47 | SlurmctldTimeout=300 48 | SlurmdTimeout=300 49 | InactiveLimit=0 50 | MinJobAge=300 51 | KillWait=30 52 | Waittime=0 53 | # 54 | # SCHEDULING 55 | SchedulerType=sched/backfill 56 | #SchedulerAuth= 57 | #SchedulerPort= 58 | #SchedulerRootFilter= 59 | SelectType=select/cons_res 60 | SelectTypeParameters=CR_CPU_Memory 61 | FastSchedule=1 62 | #PriorityType=priority/multifactor 63 | #PriorityDecayHalfLife=14-0 64 | #PriorityUsageResetPeriod=14-0 65 | #PriorityWeightFairshare=100000 66 | #PriorityWeightAge=1000 67 | #PriorityWeightPartition=10000 68 | #PriorityWeightJobSize=1000 69 | #PriorityMaxAge=1-0 70 | # 71 | # LOGGING 72 | SlurmctldDebug=3 73 | SlurmctldLogFile=/var/log/slurm/slurmctld.log 74 | SlurmdDebug=3 75 | SlurmdLogFile=/var/log/slurm/slurmd.log 76 | JobCompType=jobcomp/filetxt 77 | JobCompLoc=/var/log/slurm/jobcomp.log 78 | # 79 | # ACCOUNTING 80 | JobAcctGatherType=jobacct_gather/linux 81 | JobAcctGatherFrequency=30 82 | # 83 | AccountingStorageType=accounting_storage/slurmdbd 84 | AccountingStorageHost=slurm-database.example.org 85 | AccountingStoragePort=6819 86 | #AccountingStorageLoc=slurm_acct_db 87 | #AccountingStoragePass= 88 | #AccountingStorageUser= 89 | # 90 | # COMPUTE NODES 91 | NodeName=c[1-2] REPLACE_IT RealMemory=1000 State=UNKNOWN 92 | PartitionName=normal Default=yes Nodes=c[1-2] Priority=50 DefMemPerCPU=500 Shared=NO MaxNodes=2 MaxTime=12:00:00 DefaultTime=00:30:00 Default=YES State=UP 93 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_db/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/slurm-base 2 | MAINTAINER kevinsawade 3 | 4 | # fix permissions on slurm stuff 5 | RUN mkdir -p /etc/slurm-llnl 6 | RUN chmod 777 /etc/slurm-llnl 7 | 8 | # ENV DBD_ADDR=database \ 9 | # DBD_HOST=database \ 10 | # DBD_PORT=6819 \ 11 | # STORAGE_HOST=database.local.dev \ 12 | # STORAGE_PORT=3306 \ 13 | # STORAGE_PASS=password \ 14 | # STORAGE_USER=slurm 15 | 16 | # set up slurmdbd 17 | RUN mkdir -p /var/spool/slurm/d 18 | RUN mkdir -p /var/log/slurm 19 | RUN chown slurm: /var/spool/slurm/d 20 | RUN chown slurm: /var/log/slurm 21 | RUN mkdir -p /var/spool/slurmd 22 | 23 | # set up slurmctld 24 | RUN mkdir /var/spool/slurmctld 25 | RUN chown slurm:slurm /var/spool/slurmctld 26 | RUN chmod 755 /var/spool/slurmctld 27 | RUN mkdir -p /var/log/slurm 28 | RUN touch /var/log/slurm/slurmctld.log 29 | RUN touch /var/log/slurm/slurm_jobacct.log /var/log/slurm/slurm_jobcomp.log 30 | RUN chown -R slurm: /var/log/slurm/ 31 | RUN mkdir -p /var/run/slurm 32 | RUN chown slurm: /var/run/slurm 33 | RUN mkdir -p /usr//etc/slurm 34 | RUN chown slurm:slurm /etc/slurm 35 | COPY slurm_base/slurm.conf /usr/etc/slurm.conf 36 | COPY slurm_base/cgroup.conf /usr/etc/cgroup.conf 37 | COPY slurm_base/cgroup_allowed_devices_file.conf /usr/etc/cgroup_allowed_devices_file.conf 38 | 39 | 40 | # ENV USE_SLURMDBD=true \ 41 | # CLUSTER_NAME=snowflake \ 42 | # CONTROL_MACHINE=controller \ 43 | # SLURMCTLD_PORT=6817 \ 44 | # SLURMD_PORT=6818 \ 45 | # ACCOUNTING_STORAGE_HOST=database \ 46 | # ACCOUNTING_STORAGE_PORT=6819 \ 47 | # PARTITION_NAME=docker 48 | 49 | # copy the database conf 50 | COPY slurm_db/slurmdbd.conf /usr/etc/slurmdbd.conf 51 | RUN chmod 600 /usr/etc/slurmdbd.conf 52 | RUN chown -R slurm: /usr/etc/ 53 | 54 | # run the entrypoint 55 | COPY slurm_db/slurm_db_entrypoint.sh slurm_db_entrypoint.sh 56 | RUN chmod a+wrx slurm_db_entrypoint.sh 57 | COPY slurm_db/slurm_acct_db.sql slurm_acct_db.sql 58 | RUN chmod a+wrx slurm_acct_db.sql 59 | EXPOSE 6817 6818 6819 60 | CMD ["/slurm_db_entrypoint.sh"] 61 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_db/slurm_acct_db.sql: -------------------------------------------------------------------------------- 1 | drop database if exists slurm_acct_db; 2 | create database slurm_acct_db; 3 | drop user if exists 'slurm'@'%'; 4 | create user 'slurm'@'%'; 5 | set password for 'slurm'@'%' = password('some_pass'); 6 | grant usage on *.* to 'slurm'@'%'; 7 | grant all privileges on slurm_acct_db.* to 'slurm'@'%'; 8 | flush privileges; 9 | exit 10 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_db/slurm_db_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | . /opt/bitnami/scripts/liblog.sh 7 | 8 | # some debug for startup 9 | info "EncoderMap's SLURM DB spooling up." 10 | 11 | # write the password 12 | info "Writing passwords to .secret files." 13 | echo $LDAP_ADMIN_PASSWORD > /etc/ldap.secret 14 | echo $LDAP_ADMIN_PASSWORD > /etc/pam_ldap.secret 15 | echo $LDAP_ADMIN_PASSWORD > /etc/libnss-ldap.secret 16 | unset LDAP_ADMIN_PASSWORD 17 | 18 | # wait for the nodes to spin up and create passwordless ssh 19 | info "Waiting for OPENLDAP server to be up." 20 | /wait-for-it.sh openldap:636 --strict -- echo "openldap.example.org 636 is up" && /etc/init.d/ssh start && /etc/init.d/nscd restart && /etc/init.d/nslcd restart 21 | 22 | # bring up sshd 23 | /usr/sbin/sshd 24 | 25 | # print uid 26 | id 27 | 28 | # fix the slurm config 29 | sed -i "s/REPLACE_IT/CPUs=$(nproc)/g" /etc/slurm/slurm.conf 30 | 31 | # start the munge daemon 32 | gosu root service munge start 33 | 34 | 35 | # wait for the sql-server to be available and add slurm to the database 36 | MYSQL_ROOT_PASSWORD=sql_root_passw0rd 37 | /wait-for-it.sh db.example.org:3306 --timeout=30 --strict -- echo "db.example.org db(3306) is up" 38 | echo "SELECT 1" | mysql -h db.example.org -u root -p$MYSQL_ROOT_PASSWORD 39 | mysql -h db.example.org -u root -p$MYSQL_ROOT_PASSWORD < /slurm_acct_db.sql 40 | unset MYSQL_ROOT_PASSWORD 41 | 42 | # start the slurm database 43 | exec gosu root /usr/sbin/slurmdbd -Dvvv 44 | # tail -f /dev/null 45 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_db/slurmdbd.conf: -------------------------------------------------------------------------------- 1 | AuthType=auth/munge 2 | AuthInfo=/var/run/munge/munge.socket.2 3 | DbdAddr=slurm-database 4 | DbdHost=slurm-database 5 | SlurmUser=slurm 6 | DebugLevel=4 7 | DbdPort=6819 8 | LogFile=/var/log/slurm/slurmdbd.log 9 | PidFile=/var/run/slurm/slurmdbd.pid 10 | StorageType=accounting_storage/mysql 11 | StorageHost=db.example.org 12 | StoragePort=3306 13 | StoragePass=some_pass 14 | StorageUser=slurm 15 | StorageLoc=slurm_acct_db 16 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_master/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/slurm-base 2 | 3 | #RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata 4 | #RUN apt-get update && apt-get install -y \ 5 | # wget \ 6 | 7 | ##################### 8 | # SLURM 9 | ##################### 10 | # set up slurmctld 11 | # RUN mkdir /var/spool/slurmctld 12 | # RUN chown slurm:slurm /var/spool/slurmctld 13 | # RUN chmod 755 /var/spool/slurmctld 14 | # RUN mkdir -p /var/log/slurm 15 | # RUN touch /var/log/slurmctld.log 16 | # RUN chown slurm: /var/log/slurmctld.log 17 | # RUN touch /var/log/slurm_jobacct.log /var/log/slurm_jobcomp.log 18 | # RUN chown slurm: /var/log/slurm* 19 | # RUN chown slurm: /var/log/slurm/ 20 | # RUN chown slurm:slurm /etc/slurm 21 | COPY slurm_base/slurm.conf /usr/etc/slurm.conf 22 | COPY slurm_base/cgroup.conf /usr/etc/cgroup.conf 23 | COPY slurm_base/cgroup_allowed_devices_file.conf /usr/etc/cgroup_allowed_devices_file.conf 24 | 25 | # 6819 26 | # run the entrypoint 27 | COPY slurm_master/slurm_master_entrypoint.sh slurm_master_entrypoint.sh 28 | RUN chmod a+wrx slurm_master_entrypoint.sh 29 | EXPOSE 6817 6818 6819 30 | CMD ["/slurm_master_entrypoint.sh"] 31 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_master/slurm_master_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | # write the password 7 | echo $LDAP_ADMIN_PASSWORD > /etc/ldap.secret 8 | echo $LDAP_ADMIN_PASSWORD > /etc/pam_ldap.secret 9 | echo $LDAP_ADMIN_PASSWORD > /etc/libnss-ldap.secret 10 | unset LDAP_ADMIN_PASSWORD 11 | 12 | # wait for the nodes to spin up and create passwordless ssh 13 | /wait-for-it.sh openldap:636 --strict -- echo "openldap.example.org 636 is up" && /etc/init.d/ssh start && /etc/init.d/nscd restart && /etc/init.d/nslcd restart 14 | 15 | # bring up sshd 16 | /usr/sbin/sshd 17 | 18 | # start the munge daemon 19 | gosu root service munge start 20 | 21 | # fix stuff in the slurm configuration 22 | # for Hardware with weird architecture (newer intel CPUs with high power and low power cores, this won't work 23 | sed -i "s/REPLACE_IT/CPUs=$(nproc)/g" /usr/etc/slurm.conf 24 | 25 | # wait for the slurmdb to become active 26 | /wait-for-it.sh slurm-database.example.org:6819 --timeout=60 --strict -- echo "slurm-database.example.org 6819 is up" 27 | 28 | # start the slurm control daemons 29 | gosu root /usr/bin/sacctmgr --immediate add cluster name=linux 30 | sleep 2s 31 | gosu root /usr/sbin/slurmctld -Dvvv 32 | 33 | # use this, if the docker container automatically terminates, but you want to keep it running 34 | tail -f /dev/null 35 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_node/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/slurm-base 2 | 3 | ##################### 4 | # SOME ADDITIONAL STUFF 5 | ##################### 6 | 7 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ 8 | sshpass 9 | 10 | ##################### 11 | # SLURM 12 | ##################### 13 | # set up slurmctld 14 | # RUN mkdir /var/spool/slurmctld 15 | # RUN chown slurm:slurm /var/spool/slurmctld 16 | # RUN chmod 755 /var/spool/slurmctld 17 | # RUN mkdir -p /var/log/slurm 18 | # RUN touch /var/log/slurmctld.log 19 | # RUN chown slurm: /var/log/slurmctld.log 20 | # RUN touch /var/log/slurm_jobacct.log /var/log/slurm_jobcomp.log 21 | # RUN chown slurm: /var/log/slurm* 22 | # RUN chown slurm: /var/log/slurm/ 23 | # RUN chown slurm:slurm /etc/slurm 24 | COPY slurm_base/slurm.conf /usr/etc/slurm.conf 25 | COPY slurm_base/cgroup.conf /usr/etc/cgroup.conf 26 | COPY slurm_base/cgroup_allowed_devices_file.conf /usr/etc/cgroup_allowed_devices_file.conf 27 | 28 | # run the entrypoint 29 | WORKDIR / 30 | RUN mkdir /work 31 | COPY slurm_node/slurm_node_entrypoint.sh slurm_node_entrypoint.sh 32 | COPY slurm_node/test_slurm_node.sh test_slurm_node.sh 33 | COPY slurm_node/run_dask_tests_on_slurm.sh run_dask_tests_on_slurm.sh 34 | RUN chmod a+wrx slurm_node_entrypoint.sh 35 | RUN chmod a+wrx test_slurm_node.sh 36 | RUN chmod a+wrx run_dask_tests_on_slurm.sh 37 | EXPOSE 6817 6818 6819 38 | CMD ["/slurm_node_entrypoint.sh"] 39 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_node/run_dask_tests_on_slurm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -e 5 | 6 | # load libs 7 | . /sh_libs/liblog.sh 8 | info "Starting dask tests on EncoderMap SLURM." 9 | 10 | echo $LDAP_ADMIN_PASSWORD > /etc/ldap.secret 11 | echo $LDAP_ADMIN_PASSWORD > /etc/pam_ldap.secret 12 | echo $LDAP_ADMIN_PASSWORD > /etc/libnss-ldap.secret 13 | unset LDAP_ADMIN_PASSWORD 14 | 15 | # wait for the nodes to spin up and create passwordless ssh 16 | /wait-for-it.sh openldap:636 --strict -- echo "openldap.example.org 636 is up" && /etc/init.d/ssh start && /etc/init.d/nscd restart && /etc/init.d/nslcd restart 17 | 18 | # bring up sshd 19 | /usr/sbin/sshd 20 | 21 | # start the munge daemon 22 | gosu root service munge start 23 | 24 | # fix stuff in the slurm configuration 25 | sed -i "s/REPLACE_IT/CPUs=$(nproc)/g" /usr/etc/slurm.conf 26 | 27 | # wait for the slurm master to become active 28 | /wait-for-it.sh slurm-master.example.org:6817 --timeout=100 --strict -- echo "slurm-master.example.org 6817 is up" 29 | exec gosu root /usr/sbin/slurmd & 30 | 31 | info "Started LDAP, Munge, and SLURMD." 32 | 33 | cd /app 34 | /usr/venv/emap_env/bin/python -m pip freeze | grep tensor 35 | /usr/venv/emap_env/bin/python tests/test_featurizer.py TestSLURMFeatures 36 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_node/slurm_node_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | # write the password 7 | echo $LDAP_ADMIN_PASSWORD > /etc/ldap.secret 8 | echo $LDAP_ADMIN_PASSWORD > /etc/pam_ldap.secret 9 | echo $LDAP_ADMIN_PASSWORD > /etc/libnss-ldap.secret 10 | unset LDAP_ADMIN_PASSWORD 11 | 12 | # wait for the nodes to spin up and create passwordless ssh 13 | /wait-for-it.sh openldap:636 --strict -- echo "openldap.example.org 636 is up" && /etc/init.d/ssh start && /etc/init.d/nscd restart && /etc/init.d/nslcd restart 14 | 15 | # bring up sshd 16 | /usr/sbin/sshd 17 | 18 | # print uid 19 | id 20 | 21 | # start the munge daemon 22 | gosu root service munge start 23 | 24 | # fix permissions on the /work space 25 | gosu root mkdir -p /work/user01 26 | gosu root mkdir -p /work/user02 27 | gosu root chown -R user01 /work/user01 28 | gosu root chown -R user02 /work/user02 29 | 30 | # fix stuff in the slurm configuration 31 | sed -i "s/REPLACE_IT/CPUs=$(nproc)/g" /usr/etc/slurm.conf 32 | 33 | # wait for the slurm master to become active 34 | /wait-for-it.sh slurm-master.example.org:6817 --timeout=100 --strict -- echo "slurm-master.example.org 6817 is up" 35 | exec gosu root /usr/sbin/slurmd -Dvvv 36 | 37 | # use this, if the docker container automatically terminates, but you want to keep it running 38 | tail -f /dev/null 39 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_node/test_slurm_node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -e 5 | 6 | # load libs 7 | . /sh_libs/liblog.sh 8 | 9 | # write the password 10 | info "Running tests in EncoderMap's SLURM node." 11 | echo $LDAP_ADMIN_PASSWORD > /etc/ldap.secret 12 | echo $LDAP_ADMIN_PASSWORD > /etc/pam_ldap.secret 13 | echo $LDAP_ADMIN_PASSWORD > /etc/libnss-ldap.secret 14 | unset LDAP_ADMIN_PASSWORD 15 | 16 | # wait for the nodes to spin up and create passwordless ssh 17 | info "Using wait-for-it to wait for openldap to spool up." 18 | /wait-for-it.sh openldap:636 --strict -- echo "openldap.example.org 636 is up" && /etc/init.d/ssh start && /etc/init.d/nscd restart && /etc/init.d/nslcd restart 19 | 20 | # bring up sshd 21 | info "Bringing up sshd." 22 | /usr/sbin/sshd 23 | 24 | info "Testing ssh to master. First with localadmin." 25 | if sshpass -p password ssh -oStrictHostKeyChecking=no -q localadmin@slurm-master.example.org exit ; then 26 | info "Can ssh into slurm-master using localadmin." 27 | else 28 | error "Can't ssh into slurm-master using localadmin." 29 | fi 30 | 31 | info "Testing ssh to master with user01." 32 | if sshpass -p password1 ssh -oStrictHostKeyChecking=no -q user01@slurm-master.example.org exit ; then 33 | info "Can ssh into slurm-master using user01." 34 | else 35 | error "Can't ssh into slurm-master using user01." 36 | fi 37 | 38 | # start the munge daemon 39 | info "Starting munge daemon." 40 | gosu root service munge start 41 | 42 | # fix stuff in the slurm configuration 43 | info "Fixing the SLURM configuration by replacing REPLACE_IT with $(nproc)}" 44 | sed -i "s/REPLACE_IT/CPUs=$(nproc)/g" /usr/etc/slurm.conf 45 | 46 | # wait for the slurm master to become active 47 | info "Waiting for slurm-master 6817 to open." 48 | /wait-for-it.sh slurm-master.example.org:6817 --timeout=100 --strict -- echo "slurm-master.example.org 6817 is up" 49 | info "Starting slurmd." 50 | exec gosu root /usr/sbin/slurmd & 51 | sleep 3 52 | 53 | info "Testing sinfo command." 54 | mapfile -t arr < <(sinfo) 55 | if [[ $? -eq 0 ]] && [[ ${arr[@]} =~ "normal* up 1:00 1 idle c1" ]]; then 56 | info "The sinfo command succeeded. Shutting down." 57 | exit 0 58 | else 59 | sinfo 60 | error "The sinfo command errored with code $? .Shutting down." 61 | exit 62 | fi 63 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_node_w_prometheus/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/slurm-node 2 | 3 | #RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \ 4 | # wget \ 5 | # bzip2 \ 6 | # perl \ 7 | 8 | ##################### 9 | # GO 10 | ##################### 11 | ARG GO_VERSION=1.15 12 | RUN mkdir -p /golang 13 | RUN set -x \ 14 | && wget https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz -O /golang/go$GO_VERSION.linux-amd64.tar.gz \ 15 | && cd /golang \ 16 | && tar -xzvf go$GO_VERSION.linux-amd64.tar.gz 17 | ENV PATH="${PATH}:/golang/go/bin" 18 | 19 | ##################### 20 | # HASKELL_STACK 21 | ##################### 22 | RUN curl -sSL https://get.haskellstack.org/ | sh 23 | 24 | ##################### 25 | # Prometheus 26 | ##################### 27 | RUN set -x \ 28 | && mkdir -p /slurm_prometheus_exporter \ 29 | && cd /slurm_prometheus_exporter \ 30 | && git clone https://github.com/flatironinstitute/slurm-prometheus-exporter.git . \ 31 | && stack install --flag=slurm-prometheus-exporter:-pkgconfig --extra-lib-dirs=/usr/lib/slurm 32 | RUN cp /root/.local/bin/slurm-exporter /usr/bin/slurm-exporter 33 | RUN chmod a+wrx /usr/bin/slurm-exporter 34 | RUN chown localadmin /usr/bin/slurm-exporter 35 | 36 | ##################### 37 | # Encodermap 38 | ##################### 39 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ 40 | python3.10 \ 41 | python3.10-venv 42 | 43 | RUN mkdir -p /usr/venv 44 | RUN python3.10 -m venv /usr/venv/emap_env 45 | RUN mkdir /app 46 | COPY requirements.txt /app/requirements.txt 47 | COPY md_requirements.txt /app/md_requirements.txt 48 | COPY tests/test_requirements.txt /app/test_requirements.txt 49 | WORKDIR /app 50 | RUN /usr/venv/emap_env/bin/python -m pip install Cython==3.0.9 51 | RUN /usr/venv/emap_env/bin/python -m pip install -r requirements.txt 52 | RUN /usr/venv/emap_env/bin/python -m pip install -r md_requirements.txt 53 | RUN /usr/venv/emap_env/bin/python -m pip install -r test_requirements.txt 54 | COPY encodermap /app/encodermap 55 | COPY description.md /app 56 | COPY setup.py /app 57 | COPY pyproject.toml /app 58 | COPY versioneer.py /app 59 | RUN /usr/venv/emap_env/bin/python -m pip install . 60 | 61 | #################### 62 | # Jupyter 63 | #################### 64 | 65 | RUN mkdir -p /home/encodermap 66 | RUN chown -R 1000:1000 /home/encodermap 67 | COPY tutorials/docker/slurm_node_w_prometheus/slurm_node_w_prometheus_entrypoint.sh /slurm_node_w_prometheus_entrypoint.sh 68 | RUN chmod a+wrx /slurm_node_w_prometheus_entrypoint.sh 69 | COPY tutorials/notebooks_starter /home/encodermap 70 | COPY tutorials/notebooks_intermediate /home/encodermap 71 | COPY tutorials/notebooks_MD /home/encodermap 72 | COPY tutorials/notebooks_customization /home/encodermap 73 | WORKDIR /home/encodermap 74 | 75 | # run the entrypoint 76 | EXPOSE 8090 8888 77 | CMD ["/slurm_node_w_prometheus_entrypoint.sh"] 78 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_node_w_prometheus/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 60s 3 | 4 | scrape_configs: 5 | - job_name: 'my_slurm_exporter' 6 | scrape_interval: 30s 7 | scrape_timeout: 30s 8 | static_configs: 9 | - targets: ['c1.example.org:8090'] 10 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_node_w_prometheus/slurm_node_w_prometheus_entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # load libs 4 | . /sh_libs/liblog.sh 5 | 6 | # turn on bash's job control 7 | set -m 8 | 9 | # write the password 10 | info "Running tests in EncoderMap's SLURM node with PROMETHEUS." 11 | echo $LDAP_ADMIN_PASSWORD > /etc/ldap.secret 12 | echo $LDAP_ADMIN_PASSWORD > /etc/pam_ldap.secret 13 | echo $LDAP_ADMIN_PASSWORD > /etc/libnss-ldap.secret 14 | unset LDAP_ADMIN_PASSWORD 15 | 16 | # wait for the nodes to spin up and create passwordless ssh 17 | info "Waiting for OPENLDAP to spool up." 18 | /wait-for-it.sh openldap:636 --strict -- echo "openldap.example.org 636 is up" && /etc/init.d/ssh start && /etc/init.d/nscd restart && /etc/init.d/nslcd restart 19 | 20 | # bring up sshd 21 | info "Bringing up sshd." 22 | /usr/sbin/sshd 23 | 24 | # start the munge daemon 25 | info "Starting munge daemon." 26 | gosu root service munge start 27 | 28 | # fix stuff in the slurm configuration 29 | info "Fixing the SLURM configuration by replacing REPLACE_IT with $(nproc)." 30 | sed -i "s/REPLACE_IT/CPUs=$(nproc)/g" /usr/etc/slurm.conf 31 | 32 | # wait for the slurm master to become active 33 | info "Waiting for slurm-master 6817 to open." 34 | /wait-for-it.sh slurm-master.example.org:6817 --timeout=100 --strict -- echo "slurm-master.example.org 6817 is up." 35 | exec gosu root /usr/sbin/slurmd & 36 | 37 | # start the prometheus exporter 38 | info "Starting the slurm-exporter." 39 | exec gosu localadmin /usr/bin/slurm-exporter & 40 | 41 | # Starting jupyter notebook 42 | info "Starting jupyter notebook." 43 | exec gosu encodermap /usr/venv/emap_env/bin/python -m jupyter notebook --ip='*' --NotebookApp.token='' --NotebookApp.password='' --port="8888" 44 | 45 | # use this, if the docker container automatically terminates, but you want to keep it running 46 | tail -f /dev/null 47 | -------------------------------------------------------------------------------- /tutorials/docker/slurm_web/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM kevinsawade/ldap-client 2 | 3 | ##################### 4 | # NEEDED PACKAGES 5 | ##################### 6 | # install munge packages 7 | RUN #apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y \ 8 | # wget \ 9 | # bzip2 \ 10 | # perl \ 11 | # gcc \ 12 | # build-essential\ 13 | # git \ 14 | # gnupg \ 15 | # make \ 16 | # munge \ 17 | # libmunge-dev \ 18 | # libpython3-dev \ 19 | # python3-pip \ 20 | # python3 \ 21 | # psmisc \ 22 | # mysql-client \ 23 | # libmysqlclient-dev \ 24 | # cgroup-tools 25 | 26 | ##################### 27 | # INSTALL SLURM WEB 28 | ##################### 29 | 30 | ARG SLURM_WEB_VERSION=2.4.0 31 | RUN mkdir -p /slurm_web_install 32 | RUN set -x \ 33 | && wget https://github.com/rackslab/slurm-web/archive/refs/tags/$SLURM_WEB_VERSION.zip -O /slurm_web_install \ 34 | && cd /slurm_web_install \ 35 | -------------------------------------------------------------------------------- /tutorials/docker/ssh/ssh_config: -------------------------------------------------------------------------------- 1 | Host * 2 | StrictHostKeyChecking no 3 | UserKnownHostsFile /dev/null 4 | LogLevel QUIET 5 | -------------------------------------------------------------------------------- /tutorials/docker/teardown.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | docker-compose stop 4 | docker-compose down --remove-orphans 5 | -------------------------------------------------------------------------------- /tutorials/docker/template.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #SBATCH --chdir={{ directory }} 3 | #SBATCH --partition=single 4 | #SBATCH --gres=gpu:4 5 | #SBATCH --nodes=1 6 | #SBATCH --time=120:00:00 7 | #SBATCH --mem=50gb 8 | #SBATCH --ntasks-per-node=16 9 | #SBATCH --export=NONE 10 | #SBATCH --mail-user={{ email }} 11 | #SBATCH --mail-type=BEGIN,END 12 | 13 | {{ module_loads }} 14 | 15 | cd {{ directory }} 16 | 17 | cmd="{{ command }}" 18 | 19 | if compgen -G "*{{ gro_out_file }}*gro*" > /dev/null ; then 20 | echo "JOB FINISHED" 21 | exit 22 | fi 23 | 24 | # count the number of slurm out files 25 | num=$( ls -1 *out* | wc -l ) 26 | if [ $num -gt {{ max_out_files }} ] ; then 27 | echo "Too many out files exiting." 28 | exit 29 | fi 30 | 31 | 32 | if [! -e {{ cpt_out_file }}.cpt ]; then 33 | echo " ### Initial submit. Starting job. ### " 34 | $cmd 35 | else 36 | echo " ### Continuation ### " 37 | cp {{ cpt_out_file }}.cpt {{ cpt_out_file }}-$( date --iso-8601=seconds ).cpt.back 38 | $cmd -cpi {{ cpt_out_file }}.cpt 39 | fi 40 | 41 | if ! compgen -G "*{{ gro_out_file }}*gro*" > /dev/null ; then 42 | echo "Job not finished." 43 | echo "Submiting..." 44 | sbatch job.sh 45 | fi 46 | -------------------------------------------------------------------------------- /tutorials/docker/test_in_docker.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # turn on bash's job control 4 | set -m 5 | 6 | # load libs 7 | . /sh_libs/liblog.sh 8 | 9 | # loading gromacs 10 | info "Starting tests for simulation_attender.py" 11 | info "Staring general tests to ensure the environment is working." 12 | info "Sourcing environment modules" 13 | source /usr/share/Modules/init/profile.sh 14 | info "Loading gromacs module" 15 | module load gromacs/2023.1 16 | info "Checking whether a gromacs command exists" 17 | if ! command -v gmx &> /dev/null ; then 18 | error "The command gmx did not succeed." 19 | gmx 20 | exit 21 | else 22 | info "Gromacs installation present." 23 | fi 24 | 25 | # anaconda 26 | info "Sourcing anaconda" 27 | source /usr/local/anaconda3/bin/activate &> /dev/null 28 | info "Testing the python installation." 29 | if ! command -v pytest &> /dev/null ; then 30 | error "The command pytest did not succeed." 31 | gmx 32 | exit 33 | else 34 | info "Pytest is working. Starting the tests." 35 | fi 36 | 37 | # testing 38 | pytest test_simulation_attender.py 39 | -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/mdp/eql.mdp: -------------------------------------------------------------------------------- 1 | 2 | integrator = md 3 | dt = 0.002 ; 2 fs 4 | nsteps = 50000 ; 100 ps 5 | 6 | nstenergy = 200 7 | nstlog = 2000 8 | nstxout-compressed = 10000 9 | 10 | gen-vel = yes 11 | gen-temp = 298.15 12 | 13 | constraint-algorithm = lincs 14 | constraints = h-bonds 15 | 16 | cutoff-scheme = Verlet 17 | 18 | coulombtype = PME 19 | rcoulomb = 1.0 20 | 21 | vdwtype = Cut-off 22 | rvdw = 1.0 23 | DispCorr = EnerPres 24 | 25 | tcoupl = Nose-Hoover 26 | tc-grps = System 27 | tau-t = 2.0 28 | ref-t = 298.15 29 | nhchainlength = 1 30 | -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/mdp/eql2.mdp: -------------------------------------------------------------------------------- 1 | 2 | integrator = md 3 | dt = 0.002 ; 2 fs 4 | nsteps = 500000 ; 1.0 ns 5 | 6 | nstenergy = 200 7 | nstlog = 2000 8 | nstxout-compressed = 10000 9 | 10 | continuation = yes 11 | constraint-algorithm = lincs 12 | constraints = h-bonds 13 | 14 | cutoff-scheme = Verlet 15 | 16 | coulombtype = PME 17 | rcoulomb = 1.0 18 | 19 | vdwtype = Cut-off 20 | rvdw = 1.0 21 | DispCorr = EnerPres 22 | 23 | tcoupl = Nose-Hoover 24 | tc-grps = System 25 | tau-t = 2.0 26 | ref-t = 298.15 27 | nhchainlength = 1 28 | 29 | pcoupl = Parrinello-Rahman 30 | tau_p = 2.0 31 | compressibility = 4.46e-5 32 | ref_p = 1.0 33 | -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/mdp/min.mdp: -------------------------------------------------------------------------------- 1 | 2 | define = -DFLEXIBLE 3 | integrator = steep 4 | nsteps = 1000 5 | 6 | nstenergy = 500 7 | nstlog = 500 8 | nstxout-compressed = 1000 9 | 10 | constraint-algorithm = lincs 11 | constraints = h-bonds 12 | 13 | cutoff-scheme = Verlet 14 | 15 | coulombtype = PME 16 | rcoulomb = 1.0 17 | 18 | vdwtype = Cut-off 19 | rvdw = 1.0 20 | DispCorr = EnerPres 21 | -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/mdp/min2.mdp: -------------------------------------------------------------------------------- 1 | 2 | integrator = steep 3 | nsteps = 50000 4 | 5 | nstenergy = 500 6 | nstlog = 500 7 | nstxout-compressed = 1000 8 | 9 | cutoff-scheme = Verlet 10 | 11 | coulombtype = PME 12 | rcoulomb = 1.0 13 | 14 | vdwtype = Cut-off 15 | rvdw = 1.0 16 | DispCorr = EnerPres 17 | -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/production.mdp: -------------------------------------------------------------------------------- 1 | 2 | integrator = md 3 | dt = 0.002 ; 2 fs 4 | nsteps = 500 ; 10.0 ns 5 | 6 | nstenergy = 5000 7 | nstlog = 5000 8 | nstxout-compressed = 2000 9 | 10 | continuation = yes 11 | constraint-algorithm = lincs 12 | constraints = h-bonds 13 | 14 | cutoff-scheme = Verlet 15 | 16 | coulombtype = PME 17 | rcoulomb = 1.0 18 | 19 | vdwtype = Cut-off 20 | rvdw = 1.0 21 | DispCorr = EnerPres 22 | 23 | tcoupl = Nose-Hoover 24 | tc-grps = System 25 | tau-t = 2.0 26 | ref-t = 298.15 27 | nhchainlength = 1 28 | 29 | pcoupl = Parrinello-Rahman 30 | tau_p = 4.0 31 | compressibility = 4.46e-5 32 | ref_p = 1.0 33 | -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/production.tpr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/docker/water_simulation/production.tpr -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/production_fails.tpr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/docker/water_simulation/production_fails.tpr -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/production_short.mdp: -------------------------------------------------------------------------------- 1 | 2 | integrator = md 3 | dt = 0.002 ; 2 fs 4 | nsteps = 5 ; 10.0 ns 5 | 6 | nstenergy = 5000 7 | nstlog = 5000 8 | nstxout-compressed = 2000 9 | 10 | continuation = yes 11 | constraint-algorithm = lincs 12 | constraints = h-bonds 13 | 14 | cutoff-scheme = Verlet 15 | 16 | coulombtype = PME 17 | rcoulomb = 1.0 18 | 19 | vdwtype = Cut-off 20 | rvdw = 1.0 21 | DispCorr = EnerPres 22 | 23 | tcoupl = Nose-Hoover 24 | tc-grps = System 25 | tau-t = 2.0 26 | ref-t = 298.15 27 | nhchainlength = 1 28 | 29 | pcoupl = Parrinello-Rahman 30 | tau_p = 4.0 31 | compressibility = 4.46e-5 32 | ref_p = 1.0 33 | -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/production_short.tpr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/docker/water_simulation/production_short.tpr -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/topol.top: -------------------------------------------------------------------------------- 1 | ; 2 | ; File 'topol.top' was generated 3 | ; By user: kevin (1000028) 4 | ; On host: pccp15 5 | ; At date: Mon May 8 16:18:06 2023 6 | ; 7 | ; This is a standalone topology file 8 | ; 9 | ; Created by: 10 | ; :-) GROMACS - gmx pdb2gmx, 2022.5 (-: 11 | ; 12 | ; Executable: /home/soft/gromacs/gromacs-2022.5/inst/cuda_shared_18.04/bin/gmx 13 | ; Data prefix: /home/soft/gromacs/gromacs-2022.5/inst/cuda_shared_18.04 14 | ; Working dir: /home/kevin/git/simulation_attender/tests/water_simulation 15 | ; Command line: 16 | ; gmx pdb2gmx -f spc216.gro -o spc216_box.gro 17 | ; Force field was read from the standard GROMACS share directory. 18 | ; 19 | 20 | ; Include forcefield parameters 21 | #include "gromos54a7.ff/forcefield.itp" 22 | 23 | ; Include water topology 24 | #include "gromos54a7.ff/spc.itp" 25 | 26 | #ifdef POSRES_WATER 27 | ; Position restraint for each water oxygen 28 | [ position_restraints ] 29 | ; i funct fcx fcy fcz 30 | 1 1 1000 1000 1000 31 | #endif 32 | 33 | ; Include topology for ions 34 | #include "gromos54a7.ff/ions.itp" 35 | 36 | [ system ] 37 | ; Name 38 | 216H2O,WATJP01,SPC216,SPC-MODEL,300K,BOX(M)=1.86206NM,WFVG,MAR. 1984 39 | 40 | [ molecules ] 41 | ; Compound #mols 42 | SOL 216 43 | -------------------------------------------------------------------------------- /tutorials/docker/water_simulation/topol_stacked.top: -------------------------------------------------------------------------------- 1 | ; 2 | ; File 'topol.top' was generated 3 | ; By user: kevin (1000028) 4 | ; On host: pccp15 5 | ; At date: Mon May 8 16:18:06 2023 6 | ; 7 | ; This is a standalone topology file 8 | ; 9 | ; Created by: 10 | ; :-) GROMACS - gmx pdb2gmx, 2022.5 (-: 11 | ; 12 | ; Executable: /home/soft/gromacs/gromacs-2022.5/inst/cuda_shared_18.04/bin/gmx 13 | ; Data prefix: /home/soft/gromacs/gromacs-2022.5/inst/cuda_shared_18.04 14 | ; Working dir: /home/kevin/git/simulation_attender/tests/water_simulation 15 | ; Command line: 16 | ; gmx pdb2gmx -f spc216.gro -o spc216_box.gro 17 | ; Force field was read from the standard GROMACS share directory. 18 | ; 19 | 20 | ; Include forcefield parameters 21 | #include "gromos54a7.ff/forcefield.itp" 22 | 23 | ; Include water topology 24 | #include "gromos54a7.ff/spc.itp" 25 | 26 | #ifdef POSRES_WATER 27 | ; Position restraint for each water oxygen 28 | [ position_restraints ] 29 | ; i funct fcx fcy fcz 30 | 1 1 1000 1000 1000 31 | #endif 32 | 33 | ; Include topology for ions 34 | #include "gromos54a7.ff/ions.itp" 35 | 36 | [ system ] 37 | ; Name 38 | 216H2O,WATJP01,SPC216,SPC-MODEL,300K,BOX(M)=1.86206NM,WFVG,MAR. 1984 39 | 40 | [ molecules ] 41 | ; Compound #mols 42 | SOL 5832 43 | -------------------------------------------------------------------------------- /tutorials/install_encodermap_google_colab.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | vercomp () { 4 | if [[ $1 == $2 ]] 5 | then 6 | echo 0 7 | exit 8 | fi 9 | local IFS=. 10 | local i ver1=($1) ver2=($2) 11 | # fill empty fields in ver1 with zeros 12 | for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)) 13 | do 14 | ver1[i]=0 15 | done 16 | for ((i=0; i<${#ver1[@]}; i++)) 17 | do 18 | if [[ -z ${ver2[i]} ]] 19 | then 20 | # fill empty fields in ver2 with zeros 21 | ver2[i]=0 22 | fi 23 | if ((10#${ver1[i]} > 10#${ver2[i]})) 24 | then 25 | echo 1 26 | exit 27 | fi 28 | if ((10#${ver1[i]} < 10#${ver2[i]})) 29 | then 30 | echo 2 31 | exit 32 | fi 33 | done 34 | echo 0 35 | exit 36 | } 37 | 38 | # Get the current python version 39 | echo "I will install EncoderMap in your Google colab notebook. Please stand by..." 40 | current_py_ver=$(python -V) 41 | current_py_ver=${current_py_ver#"Python "} 42 | echo "Your Google colab notebook runs python $current_py_ver" 43 | 44 | comp=$(vercomp ${current_py_ver} 3.9.0) 45 | if [ "$comp" = "2" ] ; then 46 | echo "EncoderMap needs at least python 3.9. I will install that version now." 47 | 48 | # update apt and install python 49 | apt-get update -y > /dev/notebook 50 | apt-get install python3.9 > /dev/null 51 | update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1 > /dev/null 52 | update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 2 > /dev/null 53 | 54 | # make the Colab Kernel run with python 55 | apt-get install python3.9-distutil 56 | wget https://bootstrap.pypa.io/get-pip.py 57 | python get-pip.py 58 | python -m pip install ipython ipython_genutils ipykernel jupyter_console prompt_toolkit httplib2 astor 59 | ln -s /usr/local/lib/python3.8/dist-packages/google /usr/local/lib/python3.9/dist-packages/google 60 | 61 | # check version 62 | new_py_ver=$(python -V) 63 | new_py_ver=${new_py_ver#"Python "} 64 | echo -e "Succesfully installed python version $new_py_ver$. You can now also change the current python version by calling\n\n!sudo update-alternatives --config python3" 65 | else 66 | echo "This python version is sufficient for EncoderMap. I will install EncoderMap now" 67 | fi 68 | 69 | echo "Installing pip packages for EncoderMap" 70 | pip install -r https://raw.githubusercontent.com/AG-Peter/encodermap/main/requirements.txt 71 | pip install -r https://raw.githubusercontent.com/AG-Peter/encodermap/main/md_requirements.txt 72 | git clone https://github.com/AG-Peter/encodermap.git 73 | cd encodermap 74 | pip install . 75 | -------------------------------------------------------------------------------- /tutorials/notebooks_MD/CV_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_MD/CV_overview.png -------------------------------------------------------------------------------- /tutorials/notebooks_MD/topology_examples.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_MD/topology_examples.png -------------------------------------------------------------------------------- /tutorials/notebooks_customization/asp7.csv: -------------------------------------------------------------------------------- 1 | ../notebooks_starter/asp7.csv -------------------------------------------------------------------------------- /tutorials/notebooks_customization/custom_images_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_customization/custom_images_1.png -------------------------------------------------------------------------------- /tutorials/notebooks_customization/custom_images_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_customization/custom_images_2.png -------------------------------------------------------------------------------- /tutorials/notebooks_customization/custom_loss_functions_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_customization/custom_loss_functions_1.png -------------------------------------------------------------------------------- /tutorials/notebooks_customization/custom_scalars_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_customization/custom_scalars_1.png -------------------------------------------------------------------------------- /tutorials/notebooks_customization/custom_scalars_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_customization/custom_scalars_2.png -------------------------------------------------------------------------------- /tutorials/notebooks_customization/lr_scheduler_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_customization/lr_scheduler_1.png -------------------------------------------------------------------------------- /tutorials/notebooks_customization/lr_scheduler_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_customization/lr_scheduler_2.png -------------------------------------------------------------------------------- /tutorials/notebooks_intermediate/03_dask_featurization.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "id": "273571b7", 7 | "metadata": {}, 8 | "outputs": [], 9 | "source": [] 10 | } 11 | ], 12 | "metadata": { 13 | "language_info": { 14 | "name": "python" 15 | } 16 | }, 17 | "nbformat": 4, 18 | "nbformat_minor": 5 19 | } 20 | -------------------------------------------------------------------------------- /tutorials/notebooks_intermediate/asp7.pdb: -------------------------------------------------------------------------------- 1 | ../notebooks_starter/asp7.pdb -------------------------------------------------------------------------------- /tutorials/notebooks_intermediate/asp7.xtc: -------------------------------------------------------------------------------- 1 | ../notebooks_starter/asp7.xtc -------------------------------------------------------------------------------- /tutorials/notebooks_intermediate/glu7.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_intermediate/glu7.xtc -------------------------------------------------------------------------------- /tutorials/notebooks_intermediate/path.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_intermediate/path.npy -------------------------------------------------------------------------------- /tutorials/notebooks_starter/Tensorboard_Cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_starter/Tensorboard_Cost.png -------------------------------------------------------------------------------- /tutorials/notebooks_starter/Tensorboard_Histograms.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_starter/Tensorboard_Histograms.png -------------------------------------------------------------------------------- /tutorials/notebooks_starter/Tensorboard_Images.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_starter/Tensorboard_Images.png -------------------------------------------------------------------------------- /tutorials/notebooks_starter/Tensorboard_Parameters.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_starter/Tensorboard_Parameters.png -------------------------------------------------------------------------------- /tutorials/notebooks_starter/asp7.xtc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AG-Peter/encodermap/a51a15607bdaf71dc7b4785c5262361743e6e62c/tutorials/notebooks_starter/asp7.xtc -------------------------------------------------------------------------------- /tutorials/notebooks_tensorflow1/03_Your_Data.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# Your Data\n", 8 | "\n", 9 | "Now it's time to take advantage of your new knowledge about dimensionality reduction with EncoderMap. Load your own data and get started! The data set you use should be a table where each line contains one sample and the number of columns is the dimensionality of the data-set." 10 | ] 11 | }, 12 | { 13 | "cell_type": "markdown", 14 | "metadata": {}, 15 | "source": [ 16 | "### Load Libraries" 17 | ] 18 | }, 19 | { 20 | "cell_type": "code", 21 | "execution_count": null, 22 | "metadata": {}, 23 | "outputs": [], 24 | "source": [ 25 | "import encodermap as em\n", 26 | "import matplotlib.pyplot as plt\n", 27 | "import numpy as np\n", 28 | "from math import pi\n", 29 | "%config Completer.use_jedi=False" 30 | ] 31 | }, 32 | { 33 | "cell_type": "markdown", 34 | "metadata": {}, 35 | "source": [ 36 | "### Load Your Data" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [ 45 | "csv_path = \"path/to/your/data.csv\"\n", 46 | "high_d_data = np.loadtxt(csv_path, delimiter=\",\")" 47 | ] 48 | }, 49 | { 50 | "cell_type": "markdown", 51 | "metadata": {}, 52 | "source": [ 53 | "### Set Parameters" 54 | ] 55 | }, 56 | { 57 | "cell_type": "code", 58 | "execution_count": null, 59 | "metadata": {}, 60 | "outputs": [], 61 | "source": [ 62 | "parameters = em.Parameters()\n", 63 | "parameters.main_path = em.misc.run_path(\"runs/my_data\")\n", 64 | "parameters.n_steps = 1000\n", 65 | "parameters.dist_sig_parameters = (4.5, 12, 6, 1, 2, 6)\n", 66 | "parameters.periodicity = 2*pi\n", 67 | "\n", 68 | "# if your data set is large you should not try to calculate \n", 69 | "# the pairwise distance histogram with the complete data. \n", 70 | "em.plot.distance_histogram(high_d_data, # e.g. use high_d_data[::10] to use evrey 10th point\n", 71 | " parameters.periodicity, \n", 72 | " parameters.dist_sig_parameters)" 73 | ] 74 | }, 75 | { 76 | "cell_type": "markdown", 77 | "metadata": {}, 78 | "source": [ 79 | "### Run the Dimensionality Reduction" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": null, 85 | "metadata": {}, 86 | "outputs": [], 87 | "source": [ 88 | "e_map = em.EncoderMap(parameters, high_d_data)\n", 89 | "history = e_map.train()\n", 90 | "\n", 91 | "low_d_projection = e_map.encode(dihedrals)" 92 | ] 93 | }, 94 | { 95 | "cell_type": "markdown", 96 | "metadata": {}, 97 | "source": [ 98 | "### Plot the Results" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": {}, 105 | "outputs": [], 106 | "source": [ 107 | "%matplotlib notebook\n", 108 | "fig, axe = plt.subplots()\n", 109 | "axe.scatter(low_d_projection[:, 0], low_d_projection[:, 1], s=5, marker=\"o\", linewidths=0)" 110 | ] 111 | } 112 | ], 113 | "metadata": { 114 | "language_info": { 115 | "name": "python" 116 | } 117 | }, 118 | "nbformat": 4, 119 | "nbformat_minor": 2 120 | } 121 | -------------------------------------------------------------------------------- /tutorials/notebooks_tensorflow1/clear_outputs.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | rm -rf runs/ 3 | # form some reason --clear-ouput is broken 4 | # It will be fixed in v6.0 of jupyter_contrib_extensions 5 | for file in *.ipynb ; do 6 | nbstripout $file 7 | # jupyter nbconvert --clear-output --inplace $file 8 | done 9 | -------------------------------------------------------------------------------- /tutorials/run.sh: -------------------------------------------------------------------------------- 1 | docker run --rm -u $(id -u):$(id -g) -it -p 6006:6006 -p 8888:8888 -v $(pwd)/notebooks:/tf/notebooks --name emap encodermap 2 | # docker exec -it emap bash 3 | --------------------------------------------------------------------------------