├── .gitignore ├── HPC_scripts ├── create_ipcluster_environment.sh ├── script_engines.sh ├── steps_to_submit_IGM.txt └── submit_igm.sh ├── IGM_documentation.pdf ├── LICENSE ├── README.md ├── bin ├── igm-info-dir ├── igm-register-dir ├── igm-report ├── igm-run ├── igm-server ├── igm-stop └── igm-unregister-dir ├── demo ├── .workers.restart ├── WTC11_HiC_2Mb.hcs ├── config_file.json ├── demo_sample_outputs │ ├── igm-model.hss.T │ ├── igm-model.hss.randomInit │ └── stepdb.sqlite └── deprecated │ ├── ipworkernode.pbs │ ├── startController.sh │ └── startWorker.sh ├── html_history.py ├── igm-config_all.json ├── igm-run_scheme.pdf ├── igm ├── __init__.py ├── _preprocess.py ├── core │ ├── __init__.py │ ├── config.py │ ├── defaults │ │ ├── __init__.py │ │ ├── config_default.json │ │ └── config_schema.json │ ├── job_tracking.py │ └── step.py ├── cython_compiled │ ├── __init.py__ │ ├── cpp_sprite_assignment.cpp │ ├── cpp_sprite_assignment.h │ ├── setup.py │ ├── sprite.pyx │ └── tests.py ├── model │ ├── __init__.py │ ├── forces.py │ ├── kernel │ │ ├── __init__.py │ │ ├── lammps.py │ │ ├── lammps_io.py │ │ ├── lammps_model.py │ │ └── util.py │ ├── model.py │ └── particle.py ├── parallel │ ├── __init__.py │ ├── async_file_operations.py │ ├── dask_controller.py │ ├── ipyparallel_controller.py │ ├── parallel_controller.py │ ├── slurm_controller.py │ └── utils.py ├── report │ ├── __init__.py │ ├── damid.py │ ├── distance_map.py │ ├── hic.py │ ├── html_report.py │ ├── images.py │ ├── plots.py │ ├── radials.py │ ├── radius_of_gyration.py │ ├── shells.py │ ├── utils.py │ └── violations.py ├── restraints │ ├── __init__.py │ ├── damid.py │ ├── envelope.py │ ├── fish.py │ ├── genenvelope.py │ ├── hic.py │ ├── inter_hic.py │ ├── intra_hic.py │ ├── nucleolus.py │ ├── polymer.py │ ├── restraint.py │ ├── sprite.py │ └── steric.py ├── steps │ ├── ActivationDistanceStep.py │ ├── DamidActivationDistanceStep.py │ ├── FishAssignmentStep.py │ ├── HicEvaluationStep.py │ ├── ModelingStep.py │ ├── RandomInit.py │ ├── RelaxInit.py │ ├── SpriteAssignmentStep.py │ └── __init__.py ├── tasks │ ├── __init__.py │ └── modeling_step.py ├── ui │ ├── __init__.py │ ├── communication.py │ ├── config_parse.py │ ├── folders_database.py │ ├── navigation.py │ ├── static │ │ ├── css │ │ │ ├── filebrowser.css │ │ │ ├── igm-edit-config.css │ │ │ └── main.css │ │ ├── favicon.ico │ │ └── js │ │ │ ├── filebrowser.js │ │ │ ├── genome_visualizer │ │ │ ├── filebrowser.js │ │ │ ├── genomeapp.js │ │ │ ├── interface.js │ │ │ ├── main.js │ │ │ ├── util.js │ │ │ ├── viewer.js │ │ │ └── visualizer.js │ │ │ ├── igm-edit-config.js │ │ │ └── main.js │ ├── static_pages.py │ ├── templates │ │ ├── base.html │ │ ├── cfg_form.html │ │ ├── history.html │ │ └── main.html │ ├── utils.py │ └── views.py └── utils │ ├── __init__.py │ ├── actdist.py │ ├── emails.py │ ├── files.py │ ├── kernel_testing.py │ ├── log.py │ └── sprite │ └── sprite.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- 1 | #user added 2 | # 3 | *.hss 4 | *.pdf 5 | ipython-log* 6 | 7 | # Byte-compiled / optimized / DLL files 8 | __pycache__/ 9 | *.py[cod] 10 | *$py.class 11 | 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | tmp/ 35 | *.db 36 | 37 | # PyInstaller 38 | # Usually these files are written by a python script from a template 39 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 40 | *.manifest 41 | *.spec 42 | 43 | # Installer logs 44 | pip-log.txt 45 | pip-delete-this-directory.txt 46 | 47 | # Unit test / coverage reports 48 | htmlcov/ 49 | .tox/ 50 | .coverage 51 | .coverage.* 52 | .cache 53 | nosetests.xml 54 | coverage.xml 55 | *.cover 56 | .hypothesis/ 57 | 58 | # Translations 59 | *.mo 60 | *.pot 61 | 62 | # Django stuff: 63 | *.log 64 | local_settings.py 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # pyenv 83 | .python-version 84 | 85 | # celery beat schedule file 86 | celerybeat-schedule 87 | 88 | # SageMath parsed files 89 | *.sage.py 90 | 91 | # dotenv 92 | .env 93 | 94 | # virtualenv 95 | .venv 96 | venv/ 97 | ENV/ 98 | 99 | # Spyder project settings 100 | .spyderproject 101 | .spyproject 102 | 103 | # Rope project settings 104 | .ropeproject 105 | 106 | # mkdocs documentation 107 | /site 108 | 109 | # mypy 110 | .mypy_cache/ 111 | 112 | # mac temporary 113 | ._* 114 | 115 | # cython auto generated 116 | igm/cython_compiled/sprite.cpp 117 | 118 | # pycharm 119 | 120 | # pycharm 121 | .idea/ 122 | 123 | -------------------------------------------------------------------------------- /HPC_scripts/create_ipcluster_environment.sh: -------------------------------------------------------------------------------- 1 | # This script prepares the controller/engines environment, using the MONITOR option 2 | # which allows to keep track of how the tasks are split among the different engines 3 | # On the top of this, the serial IGM job will be submitted 4 | 5 | # number of workers 6 | NTASKS=100 7 | # memory per worker 8 | MEM=$2 9 | # walltime 10 | WTIME=$3 11 | # conda env to use 12 | CONDA=$4 13 | # scheduler memory, usual a little larger than workers 14 | SMEM=$5 15 | 16 | let NTOT=${NTASKS}+1 17 | if [[ -z "$2" ]]; then 18 | MEM=2G 19 | fi 20 | if [[ -z "$3" ]]; then 21 | WTIME=200:59:59 22 | fi 23 | 24 | if [[ -z "$4" ]]; then 25 | CONDA=py3 26 | fi 27 | 28 | if [[ -z "$5" ]]; then 29 | SMEM=10G 30 | fi 31 | 32 | echo "requesting ${NTASKS} processes, ${MEM} per cpu, walltime: ${WTIME}" 33 | 34 | CURRDIR=`pwd` 35 | 36 | TMPFILE=`mktemp` || exit 1 37 | # write the slurm script 38 | cat > $TMPFILE <<- EOF 39 | #!/bin/bash 40 | #$ -M bonimba@g.ucla.edu 41 | #$ -m ea 42 | #$ -N ipycontroller 43 | #$ -l h_data=${SMEM} 44 | #$ -l h_rt=${WTIME} 45 | #$ -l highp 46 | #$ -cwd 47 | #$ -o out_controller 48 | #$ -e err_controller 49 | #$ -V 50 | 51 | export PATH="$PATH" 52 | ulimit -s 8192 53 | 54 | 55 | cd $SGE_O_WORKDIR 56 | 57 | myip=\$(getent hosts \$(hostname) | awk '{print \$1}') 58 | MONITOR=$(command -v monitor_process) 59 | if [[ ! -z "$MONITOR" ]]; then 60 | monitor_process --wtype S ipcontroller --nodb --ip=\$myip 61 | else 62 | ipcontroller --nodb --ip=\$myip 63 | fi 64 | EOF 65 | 66 | #cat $TMPFILE >> 'script1.txt' 67 | 68 | SCHEDJOB=$(qsub $TMPFILE | awk '{print $4}') 69 | echo 'scheduler job submitted:' $SCHEDJOB 70 | 71 | TMPFILE=`mktemp` || exit 1 72 | # write the slurm script 73 | cat > $TMPFILE <<- EOF 74 | #!/bin/bash 75 | #$ -M bonimba@g.ucla.edu 76 | #$ -m ea 77 | #$ -N ipycluster 78 | #$ -l h_data=${MEM} 79 | #$ -l h_rt=${WTIME} 80 | #$ -l highp 81 | #$ -cwd 82 | #$ -o out_engines 83 | #$ -e err_engines 84 | #$ -V 85 | #$ -pe dc* ${NTASKS} 86 | 87 | 88 | export PATH="$PATH" 89 | ulimit -s 8192 90 | 91 | cd $SGE_O_WORKDIR 92 | 93 | # let the scheduler setup finish 94 | sleep 10 95 | MONITOR=$(command -v monitor_process) 96 | if [[ ! -z "$MONITOR" ]]; then 97 | mpirun --n=${NTASKS} monitor_process --wtype W ipengine 98 | else 99 | mpirun --n=${NTASKS} ipengine 100 | fi 101 | 102 | EOF 103 | 104 | #cat $TMPFILE 105 | sleep 1 106 | cat $TMPFILE >> 'script_engines.sh' 107 | 108 | echo "The engines will start only after the controller job $SCHEDJOB starts..." 109 | 110 | echo "Need to manually submit $NTASKS engines on the top of the controller, by executing the script_engines.sh script" 111 | -------------------------------------------------------------------------------- /HPC_scripts/script_engines.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | #$ -M bonimba@g.ucla.edu 3 | #$ -m ea 4 | #$ -N ipycluster 5 | #$ -l h_data=2G 6 | #$ -l h_rt=200:59:59 7 | #$ -l highp 8 | #$ -cwd 9 | #$ -o out_engines 10 | #$ -e err_engines 11 | #$ -V 12 | #$ -pe dc* 100 13 | 14 | 15 | export PATH="/u/home/b/bonimba/miniconda3/bin:/u/home/b/bonimba/miniconda3/condabin:/u/local/compilers/intel/18.0.4/parallel_studio_xe_2018/bin:/u/local/compilers/intel/18.0.4/compilers_and_libraries_2018.5.274/linux/mpi/intel64/bin:/u/local/compilers/intel/18.0.4/compilers_and_libraries_2018.5.274/linux/bin/intel64:/u/local/compilers/intel/18.0.4/clck/2018.3/bin/intel64:/u/local/compilers/intel/18.0.4/itac/2018.4.025/intel64/bin:/u/local/compilers/intel/18.0.4/inspector/bin64:/u/local/compilers/intel/18.0.4/vtune_amplifier_2018/bin64:/u/local/compilers/intel/18.0.4/advisor_2018/bin64:/u/systems/UGE8.6.4/bin/lx-amd64:/u/local/bin:/u/local/sbin:/usr/lib64/qt-3.3/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/u/home/b/bonimba/bin" 16 | ulimit -s 8192 17 | 18 | cd 19 | 20 | # let the scheduler setup finish 21 | sleep 10 22 | MONITOR= 23 | if [[ ! -z "" ]]; then 24 | mpirun --n=100 monitor_process --wtype W ipengine 25 | else 26 | mpirun --n=100 ipengine 27 | fi 28 | 29 | 30 | -------------------------------------------------------------------------------- /HPC_scripts/steps_to_submit_IGM.txt: -------------------------------------------------------------------------------- 1 | How to create the parallel environment (step 1) and submit a successful parallel IGM calculation (step 2) on top of it. 2 | 3 | - remove the ```submit_engines.sh``` file (if there). 4 | - ```bash create_ipcluster_environment.sh```: this will submit the controller and create the ```submit_engines``` script, ready for you to submit (step 1a). 5 | - Wait for the controller job to start (and generate its own ```out`` and `err` files), and then submit the engines with ```qsub submit_engines``` (step 1b). 6 | - When the engines are also running, you can submit the actual IGM calculation on top of the (controller + engines) scaffold, using ```qsub submit_igm``` (step 2). 7 | -------------------------------------------------------------------------------- /HPC_scripts/submit_igm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #----------------------------------------# 4 | #- IGM SUBMISSION SCRIPT, multi-threaded JOB 5 | #----------------------------------------# 6 | 7 | #$ -M bonimba@g.ucla.edu 8 | #$ -m ea 9 | #$ -N jobname 10 | #$ -l h_data=40G 11 | #$ -l h_rt=30:00:00 12 | #$ -l highp 13 | #$ -cwd 14 | #$ -o out_igm 15 | #$ -e err_igm 16 | #$ -V 17 | #$ -pe shared 2 18 | 19 | export PATH="$PATH" 20 | ulimit -s 8192 21 | 22 | # ----------------------- 23 | # print JOB ID, can be useful for keeping track of status 24 | echo $JOB_ID 25 | 26 | # print PATH pointing to directory the job is run from 27 | echo $SGE_O_WORKDIR 28 | 29 | 30 | # shared memory parallelization: same node, more cores, export number of threads 31 | export OMP_NUM_THREADS=2 32 | echo "submitting IGM optimization..." 33 | 34 | # execute job 35 | igm-run igm_config.json >> igm_output.txt 36 | 37 | -------------------------------------------------------------------------------- /IGM_documentation.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/IGM_documentation.pdf -------------------------------------------------------------------------------- /bin/igm-info-dir: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import argparse 4 | import sys 5 | import json 6 | from igm.ui.folders_database import folder_info 7 | 8 | 9 | if __name__ == "__main__": 10 | 11 | parser = argparse.ArgumentParser(description='Get a record from the IGM database in json format') 12 | parser.add_argument('folder', nargs='?', default=os.getcwd(), help='Optional. Folder to retrieve. If not specified, it defaults the current directory. Ignored if --all is specified.') 13 | parser.add_argument('--all', action='store_true', help='if specified, print info for all the folders') 14 | 15 | args = parser.parse_args() 16 | 17 | try: 18 | if args.all: 19 | sys.stdout.write(json.dumps(folder_info(), indent=4) + '\n') 20 | else: 21 | sys.stdout.write(json.dumps(folder_info(args.folder), indent=4) + '\n') 22 | 23 | except RuntimeError as e: 24 | sys.stderr.write(str(e)) 25 | exit(1) 26 | 27 | -------------------------------------------------------------------------------- /bin/igm-register-dir: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os.path 3 | import argparse 4 | import sys 5 | from igm.ui.folders_database import register_folder 6 | 7 | if __name__ == "__main__": 8 | 9 | parser = argparse.ArgumentParser(description='Register a directory in the IGM database') 10 | parser.add_argument('folder', nargs='?', default=os.getcwd(), help='Optional. Directory to register. If not specified, it register the current directory.') 11 | parser.add_argument('--cell-line', default='', help='(Optional) Cell line') 12 | parser.add_argument('--resolution', default='', help='(Optional) Model resolution') 13 | parser.add_argument('--notes', default='', help='(Optional) Additional notes') 14 | parser.add_argument('--tags', default='', help='(Optional) Comma separated list of tags') 15 | 16 | #ssh -f -N -T -M -S -R :: 17 | 18 | args = parser.parse_args() 19 | 20 | try: 21 | register_folder(**vars(args)) 22 | except RuntimeError as e: 23 | sys.stderr.write(str(e)) 24 | exit(1) 25 | 26 | sys.stdout.write('ok: ' + args.folder + '\n') 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /bin/igm-stop: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import socket, argparse, os, stat, sys 4 | 5 | 6 | def is_socket(path): 7 | mode = os.stat(path).st_mode 8 | return stat.S_ISSOCK(mode) 9 | 10 | 11 | if __name__ == "__main__": 12 | 13 | parser = argparse.ArgumentParser(description='Stop a running igm instance') 14 | parser.add_argument('dir', nargs='?', default=os.getcwd(), help='Optional. Directory where the IGM instance is running.') 15 | 16 | args = parser.parse_args() 17 | 18 | if not os.path.isdir(args.dir): 19 | sys.stderr.write('Cannot find directory {}\n'.format(args.dir)) 20 | exit(1) 21 | 22 | socket_file = os.path.join(args.dir, '.igm-socket') 23 | 24 | try: 25 | if not is_socket(socket_file): 26 | sys.stderr.write('File "{}" does not appear to be a valid socket.\n'.format(socket_file)) 27 | exit(1) 28 | except FileNotFoundError: 29 | sys.stderr.write('Cannot find socket file "{}"\nIs IGM running in {}?\n'.format(socket_file, args.dir)) 30 | exit(1) 31 | 32 | s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 33 | s.connect(socket_file) 34 | s.send(b'{"q": "kill"}') 35 | s.close() 36 | -------------------------------------------------------------------------------- /bin/igm-unregister-dir: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os.path 3 | import argparse 4 | import sys 5 | 6 | from igm.ui.folders_database import unregister_folder 7 | 8 | if __name__ == "__main__": 9 | 10 | parser = argparse.ArgumentParser(description='Unregister a directory in the IGM database') 11 | parser.add_argument('folder', nargs='?', default=os.getcwd(), help='Optional. Directory to delete. If not specified, it deletes the current directory.') 12 | 13 | args = parser.parse_args() 14 | 15 | try: 16 | unregister_folder(args.folder) 17 | sys.stdout.write('removed: ' + args.folder + '\n') 18 | except RuntimeError as e: 19 | sys.stderr.write(str(e)) 20 | exit(1) 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /demo/.workers.restart: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/demo/.workers.restart -------------------------------------------------------------------------------- /demo/WTC11_HiC_2Mb.hcs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/demo/WTC11_HiC_2Mb.hcs -------------------------------------------------------------------------------- /demo/config_file.json: -------------------------------------------------------------------------------- 1 | { 2 | "genome": { 3 | "assembly": "hg38", 4 | "ploidy": "male", 5 | "usechr": [ 6 | "#", 7 | "X", 8 | "Y" 9 | ], 10 | "segmentation": 2000000 11 | }, 12 | "model": { 13 | "population_size": 100, 14 | "occupancy": 0.4, 15 | "init_radius": 7000, 16 | "restraints": { 17 | "excluded": { 18 | "evfactor": 1.0 19 | }, 20 | "envelope": { 21 | "nucleus_shape": "sphere", 22 | "nucleus_radius": 5500, 23 | "nucleus_kspring": 1.0 24 | }, 25 | "polymer": { 26 | "contact_range": 2.0, 27 | "polymer_bonds_style": "simple", 28 | "polymer_kspring": 1.0 29 | } 30 | } 31 | }, 32 | "restraints": { 33 | "Hi-C": { 34 | "input_matrix": "WTC11_HiC_2Mb.hcs", 35 | "intra_sigma_list": [ 36 | 1.0, 37 | 0.2, 38 | 0.1, 39 | 0.05, 40 | 0.02, 41 | 0.01 42 | ], 43 | "inter_sigma_list": [ 44 | 1.0, 45 | 0.2, 46 | 0.1, 47 | 0.05, 48 | 0.02, 49 | 0.01 50 | ], 51 | "contact_range": 2.0, 52 | "contact_kspring": 1.0, 53 | "actdist_file": "actdist.h5", 54 | "tmp_dir": "actdist", 55 | "keep_temporary_files": false, 56 | "run_evaluation_step": false 57 | } 58 | }, 59 | "optimization": { 60 | "min_iterations": 5, 61 | "force_last_iteration": true, 62 | "force_minimum_iterations_hic_cutoff": 0.0, 63 | "max_iterations": 10, 64 | "violation_tolerance": 0.05, 65 | "max_violations": 0.01, 66 | "structure_output": "igm-model.hss", 67 | "keep_intermediate_structures": true, 68 | "kernel": "lammps", 69 | "tmp_dir": "opt", 70 | "clean_restart": false, 71 | "keep_temporary_files": false, 72 | "kernel_opts": { 73 | "lammps": { 74 | "lammps_executable": "path to lammpgen/src/lmp_serial", 75 | "seed": 6535, 76 | "max_neigh": 6000, 77 | "use_gpu": false 78 | } 79 | }, 80 | "optimizer_options": { 81 | "mdsteps": 45000, 82 | "timestep": 0.25, 83 | "tstart": 500.0, 84 | "tstop": 0.01, 85 | "custom_annealing_protocol": { 86 | "num_steps": 4, 87 | "mdsteps": [ 88 | 5000, 89 | 15000, 90 | 15000, 91 | 10000 92 | ], 93 | "tstarts": [ 94 | 5000.0, 95 | 500.0, 96 | 50.0, 97 | 1.0 98 | ], 99 | "tstops": [ 100 | 500.0, 101 | 50.0, 102 | 1.0, 103 | 0.0 104 | ], 105 | "evfactors": [ 106 | 0.5, 107 | 1.0, 108 | 1.0, 109 | 1.0 110 | ], 111 | "envelope_factors": [ 112 | 1.2, 113 | 1.0, 114 | 1.0, 115 | 1.0 116 | ], 117 | "relax": { 118 | "mdsteps": 500, 119 | "temperature": 1.0, 120 | "max_velocity": 10.0 121 | } 122 | }, 123 | "damp": 50.0, 124 | "max_velocity": 1000.0, 125 | "etol": 0.0001, 126 | "ftol": 1e-06, 127 | "max_cg_iter": 500, 128 | "max_cg_eval": 500, 129 | "thermo": 1000, 130 | "write": -1 131 | } 132 | }, 133 | "parameters": { 134 | "step_db": "stepdb.sqlite", 135 | "tmp_dir": "tmp", 136 | "log": "igm.log", 137 | "log_level": "debug" 138 | }, 139 | "parallel": { 140 | "controller": "ipyparallel", 141 | "controller_options": { 142 | "ipyparallel": { 143 | "max_tasks": 5000 144 | } 145 | } 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /demo/demo_sample_outputs/igm-model.hss.T: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/demo/demo_sample_outputs/igm-model.hss.T -------------------------------------------------------------------------------- /demo/demo_sample_outputs/igm-model.hss.randomInit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/demo/demo_sample_outputs/igm-model.hss.randomInit -------------------------------------------------------------------------------- /demo/demo_sample_outputs/stepdb.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/demo/demo_sample_outputs/stepdb.sqlite -------------------------------------------------------------------------------- /demo/deprecated/ipworkernode.pbs: -------------------------------------------------------------------------------- 1 | 2 | #!/bin/bash 3 | #PBS -N workernode 4 | #PBS -l nodes=1:Xeon:ppn=1 5 | #PBS -l walltime=100:00:00 6 | #PBS -q cmb 7 | #PBS -l pmem=3gb 8 | #PBS -l pvmem=3gb 9 | #PBS -l mem=3gb 10 | #PBS -l vmem=3gb 11 | #PBS -j oe 12 | #PBS -o /dev/null 13 | #PBS -m p 14 | 15 | cd ${PBS_O_WORKDIR} 16 | 17 | source /home/cmb-08/fa/local/setup.sh 18 | ulimit -s 8192 19 | #IPEID=$(uuidgen) 20 | #touch ".${IPEID}.restart" 21 | ipengine --quiet 2> logs/workernode_${PBS_JOBID}.log 22 | 23 | while [ -e ".workers.restart" ] 24 | do 25 | sleep 1 26 | ipengine --quiet 2> logs/workernode_${PBS_JOBID}.log 27 | done -------------------------------------------------------------------------------- /demo/deprecated/startController.sh: -------------------------------------------------------------------------------- 1 | export MYIP=$(ifconfig eno1 | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1' | head -1) 2 | 3 | ipcontroller --ip=$MYIP --nodb 4 | -------------------------------------------------------------------------------- /demo/deprecated/startWorker.sh: -------------------------------------------------------------------------------- 1 | mkdir logs 2 | mkdir tmp 3 | for i in `seq 280` 4 | do 5 | qsub ipworkernode.pbs >> worker_job_ids.log 6 | done 7 | touch .workers.restart 8 | 9 | -------------------------------------------------------------------------------- /html_history.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import igm.ui.views as vw 3 | 4 | with open('history.html', 'w') as f: 5 | f.write( vw.history( sys.argv[1] ) ) -------------------------------------------------------------------------------- /igm-config_all.json: -------------------------------------------------------------------------------- 1 | { 2 | "genome": { 3 | "assembly": "hg38", 4 | "ploidy": "male", 5 | "usechr": [ 6 | "#", 7 | "X", 8 | "Y" 9 | ], 10 | "segmentation": 200000 11 | }, 12 | "model": { 13 | "population_size": 1000, 14 | "occupancy": 0.4, 15 | "restraints": { 16 | "excluded": { 17 | "evfactor": 1.0 18 | }, 19 | "envelope": { 20 | "nucleus_shape": "ellipsoid", 21 | "nucleus_semiaxes": [7840.0, 6470.0, 2450.0], 22 | "nucleus_kspring": 1.0 23 | }, 24 | "polymer": { 25 | "contact_range": 2.0, 26 | "polymer_bonds_style": "simple", 27 | "polymer_kspring": 1.0 28 | } 29 | } 30 | }, 31 | "restraints": { 32 | "Hi-C": { 33 | "input_matrix": ".hcs file, Hi-C matrix input of contact probabilities", 34 | "intra_sigma_list": [ 35 | 1.0, 36 | 0.5, 37 | 0.2, 38 | 0.1, 39 | 0.05, 40 | 0.01, 41 | 0.008 42 | ], 43 | "inter_sigma_list": [ 44 | 1.0, 45 | 0.5, 46 | 0.2, 47 | 0.1, 48 | 0.05, 49 | 0.01, 50 | 0.008 51 | ], 52 | "contact_range": 2.0, 53 | "contact_kspring": 1.0, 54 | "actdist_file": "actdist.h5", 55 | "tmp_dir": "actdist", 56 | "keep_temporary_files": false, 57 | "run_evaluation_step": false 58 | }, 59 | "FISH" : { 60 | "input_fish" : "h5py file, FISH input, list of distance distributions for radial or pairwise 3D HIPMap FISH data", 61 | "rtype" : "p", 62 | "kspring" : 1.0, 63 | "tmp_dir" : "fish_actdist", 64 | "batch_size": 200, 65 | "keep_temporary_files" : "true", 66 | "tol_list" : [200.0, 100.0, 100.0, 100.0, 50.0, 50.0, 50.0, 25.0], 67 | "fish_assignment_file": "fish_assignment.h5" 68 | }, 69 | "sprite" : { 70 | "clusters": "h5py file, SPRITE input, list of loci colocalizing = list of clusters of loci", 71 | "volume_fraction_list": [0.001, 0.001, 0.005, 0.005, 0.005, 0.01, 0.01, 0.05], 72 | "radius_kt": 50.0, 73 | "assignment_file": "sprite_assignment.h5", 74 | "tmp_dir": "sprite_assign", 75 | "keep_temporary_files": false, 76 | "batch_size": 10, 77 | "kspring": 1.0 78 | }, 79 | 80 | "DamID": { 81 | "input_profile": ".txt file, lamina DamID input, contact probabilities with the lamina", 82 | "sigma_list": [ 83 | 0.8, 0.8, 0.6, 0.6, 0.45, 0.45, 0.3, 0.3 84 | ], 85 | "contact_range": 0.05, 86 | "contact_kspring": 1.0, 87 | "damid_actdist_file": "actdist.h5", 88 | "tmp_dir": "damid", 89 | "keep_temporary_files": false 90 | } 91 | 92 | }, 93 | "optimization": { 94 | "iter_corr_knob": 1, 95 | "min_iterations": 5, 96 | "force_last_iteration": true, 97 | "force_minimum_iterations_hic_cutoff": 0.0, 98 | "max_iterations": 12, 99 | "violation_tolerance": 0.05, 100 | "max_violations": 0.01, 101 | "structure_output": "igm-model.hss", 102 | "keep_intermediate_structures": true, 103 | "kernel": "lammps", 104 | "tmp_dir": "opt", 105 | "clean_restart": false, 106 | "keep_temporary_files": false, 107 | "kernel_opts": { 108 | "lammps": { 109 | "lammps_executable": "/u/home/b/bonimba/lammpgen/src/lmp_serial", 110 | "seed": 6535, 111 | "max_neigh": 6000, 112 | "use_gpu": false 113 | } 114 | }, 115 | "optimizer_options": { 116 | "mdsteps": 40000, 117 | "timestep": 0.25, 118 | "tstart": 500.0, 119 | "tstop": 0.01, 120 | "custom_annealing_protocol": { 121 | "num_steps": 4, 122 | "mdsteps": [ 123 | 5000, 124 | 15000, 125 | 15000, 126 | 10000 127 | ], 128 | "tstarts": [ 129 | 5000.0, 130 | 500.0, 131 | 50.0, 132 | 1.0 133 | ], 134 | "tstops": [ 135 | 500.0, 136 | 50.0, 137 | 1.0, 138 | 0.0 139 | ], 140 | "evfactors": [ 141 | 0.5, 142 | 1.0, 143 | 1.0, 144 | 1.0 145 | ], 146 | "envelope_factors": [ 147 | 1.2, 148 | 1.0, 149 | 1.0, 150 | 1.0 151 | ], 152 | "relax": { 153 | "mdsteps": 200, 154 | "temperature": 1.0, 155 | "max_velocity": 10.0 156 | } 157 | }, 158 | "damp": 50.0, 159 | "max_velocity": 1000.0, 160 | "etol": 0.0001, 161 | "ftol": 1e-06, 162 | "max_cg_iter": 500, 163 | "max_cg_eval": 500, 164 | "thermo": 1000, 165 | "write": -1 166 | } 167 | }, 168 | "parameters": { 169 | "step_db": "stepdb.sqlite", 170 | "tmp_dir": "/u/scratch/b/bonimba/HFF_all_longer/tmp", 171 | "log": "igm.log", 172 | "log_level": "debug" 173 | }, 174 | "parallel": { 175 | "controller": "ipyparallel", 176 | "controller_options": { 177 | "ipyparallel": { 178 | "max_tasks": 5000 179 | } 180 | } 181 | } 182 | } 183 | 184 | -------------------------------------------------------------------------------- /igm-run_scheme.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/igm-run_scheme.pdf -------------------------------------------------------------------------------- /igm/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | import warnings 3 | warnings.simplefilter(action='ignore', category=FutureWarning) 4 | 5 | from . import model 6 | from . import restraints 7 | from . import parallel 8 | 9 | 10 | from .core import Config 11 | 12 | 13 | from .steps import * 14 | from ._preprocess import Preprocess 15 | 16 | from .utils import SetupLogging, logger 17 | -------------------------------------------------------------------------------- /igm/core/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from __future__ import division, print_function 3 | 4 | 5 | from .config import Config 6 | from .step import Step, StructGenStep 7 | 8 | -------------------------------------------------------------------------------- /igm/core/config.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | import json 4 | import hashlib 5 | from configparser import ConfigParser 6 | import os, os.path 7 | from copy import deepcopy 8 | from six import string_types, raise_from 9 | from ..utils.files import make_absolute_path 10 | 11 | from . import defaults 12 | schema_file = os.path.join( 13 | os.path.dirname(os.path.abspath(defaults.__file__)), 14 | 'config_schema.json' 15 | ) 16 | schema = json.load(open(schema_file, 'r')) 17 | 18 | ##TODO 19 | #use walk_schema to validate user args 20 | def walk_schema(n, w, group_callback, post_group_callback, 21 | item_callback): 22 | if isinstance(w, dict): 23 | if w.get("role") is not None and "group" in w["role"]: 24 | group_callback(n, w) 25 | for x in w: 26 | walk_tree(n + '/' + x, w[x], 27 | group_callback, post_group_callback, 28 | item_callback) 29 | exit_group_callback(n, w) 30 | return 31 | if w.get("dtype") is not None: 32 | item_callback(n, w) 33 | 34 | def make_default_dict(group): 35 | out = dict() 36 | for k, item in group.items(): 37 | if not isinstance(item, dict): 38 | continue 39 | if ('role' in item) and (item['role'] == 'group'): 40 | out[k] = make_default_dict(group[k]) 41 | elif ('role' in item) and (item['role'] in ['optional-group', 'optional_input']): 42 | pass 43 | else: 44 | try: 45 | out[k] = item['default'] 46 | except KeyError: 47 | pass 48 | return out 49 | 50 | RAISE = object() 51 | 52 | class Config(dict): 53 | """ 54 | Config is the class which holds all static parameters 55 | e.g. # structures, hic file, SPRITE file etc. 56 | and dynamic parameters e.g. activation distance and cluster assignment. 57 | """ 58 | RAISE = RAISE 59 | 60 | def __init__(self, cfg=None): 61 | 62 | super(Config, self).__init__() 63 | 64 | # create default dictionary 65 | self.update(make_default_dict(schema)) 66 | 67 | # runtime is a required key, and is home to all the generated parameters 68 | self['runtime'] = dict() 69 | 70 | # update with user files in ${HOME}/.igm 71 | user_defaults_file = os.environ['HOME'] + '/.igm/user_defaults.cfg' 72 | if os.path.isfile(user_defaults_file): 73 | user_defaults = ConfigParser() 74 | user_defaults.read(user_defaults_file) 75 | for s in user_defaults.sections() + ['DEFAULT']: 76 | for k in user_defaults[s]: 77 | self.set(k, user_defaults[s][k]) 78 | 79 | # update the default dictionary with the provided file 80 | if cfg is not None: 81 | if isinstance(cfg, string_types): 82 | with open(cfg) as f: 83 | self.update(json.load(f)) 84 | elif isinstance(cfg, dict): 85 | self.update(deepcopy(cfg)) 86 | else: 87 | raise TypeError('Config argument needs to be a path or a dictionary.') 88 | 89 | self.igm_parameter_abspath() 90 | 91 | # fix optimizer arguments 92 | self.preprocess_optimization_arguments() 93 | 94 | for k in self['restraints']: 95 | if k not in self['runtime']: 96 | self['runtime'][k] = dict() 97 | 98 | def get(self, keypath, default=RAISE): 99 | split_path = keypath.split("/") 100 | 101 | try: 102 | d = self 103 | for p in split_path: 104 | d = d[p] 105 | except KeyError: 106 | if default is not RAISE: 107 | return default 108 | d = schema 109 | for p in split_path: 110 | if p in d: 111 | d = d[p] 112 | else: 113 | raise_from(KeyError("{} does not exist".format(keypath)), None) 114 | d = d.get("default") 115 | return d 116 | 117 | def set(self, keypath, val): 118 | split_path = keypath.split("/") 119 | 120 | d = self 121 | for p in split_path[:-1]: 122 | if p not in d: 123 | d[p] = dict() 124 | d = d[p] 125 | d[split_path[-1]] = val 126 | 127 | return val 128 | 129 | def igm_parameter_abspath(self): 130 | # if a working directory is not specified, we set it to the 131 | # current directory. 132 | self['parameters']['workdir'] = make_absolute_path( self.get("parameters/workdir", os.getcwd()) ) 133 | 134 | # We use absolute paths because workers may be running on different 135 | # directories. 136 | self['parameters']['tmp_dir'] = make_absolute_path( self.get("parameters/tmp_dir", 'tmp'), self['parameters']['workdir'] ) 137 | self['parameters']['log'] = make_absolute_path( self.get("parameters/log", 'igm.log'), self['parameters']['workdir'] ) 138 | self['optimization']['structure_output'] = make_absolute_path( self['optimization']['structure_output'], self['parameters']['workdir']) 139 | 140 | def preprocess_optimization_arguments(self): 141 | 142 | opt = self['optimization']['optimizer_options'] 143 | opt['ev_step'] = 0 144 | try: 145 | if opt['write'] == -1: 146 | opt['write'] = opt['mdsteps'] # write only final step 147 | except: 148 | pass 149 | 150 | def preprocess_sprite_arguments(self): 151 | if 'sprite' not in self['restraints']: 152 | return 153 | self['restraints']['sprite'] = validate_user_args(self['restraints']['sprite'], SPRITE_DEFAULT) 154 | opt = self['restraints']['sprite'] 155 | opt['tmp_dir'] = make_absolute_path(opt['tmp_dir'], self['tmp_dir']) 156 | opt['assignment_file'] = make_absolute_path(opt['assignment_file'], opt['tmp_dir']) 157 | opt['clusters'] = make_absolute_path(opt['clusters'], self['workdir']) 158 | 159 | def static_hash(self): 160 | ''' 161 | Returns a hash for the static run options 162 | ''' 163 | return hashlib.md5( 164 | json.dumps({ 165 | self[k] for k in self if k != 'runtime' 166 | }).encode() 167 | ).hexdigest() 168 | 169 | def runtime_hash(self): 170 | ''' 171 | Returns a hash for the current configuration, 172 | including runtime status. 173 | ''' 174 | return hashlib.md5( 175 | json.dumps(self).encode() 176 | ).hexdigest() 177 | 178 | def save(self, fname): 179 | with open(fname, 'w') as f: 180 | json.dump(self, f, indent=4) 181 | 182 | #== 183 | 184 | def validate_user_args(inargs, defaults, strict=True): 185 | args = {k: v for k, v, _, _ in defaults} 186 | atypes = {k: t for k, _, t, _ in defaults} 187 | for k, v in inargs.items(): 188 | if k not in args and strict is True: 189 | raise ValueError('Keywords argument \'%s\' not recognized.' % k) 190 | if v is not None: 191 | args[k] = atypes[k](v) 192 | 193 | return args 194 | 195 | 196 | -------------------------------------------------------------------------------- /igm/core/defaults/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/igm/core/defaults/__init__.py -------------------------------------------------------------------------------- /igm/core/defaults/config_default.json: -------------------------------------------------------------------------------- 1 | { 2 | 3 | "genome" : { 4 | "genome" : "hg38", 5 | "segmentation" : "tad.bed", 6 | "ploidy" : "diploid" 7 | }, 8 | 9 | "model" : { 10 | "nucleus_shape" : "sphere", 11 | "nucleus_radius" : 5000.0, 12 | "occupancy" : 0.2, 13 | "contact_range" : 2.0, 14 | "evfactor" : 0.05 15 | }, 16 | 17 | "restraints" : { 18 | 19 | }, 20 | 21 | "optimization" : { 22 | "kernel" : "lammps", 23 | "tmp_dir" : "opt", 24 | "run_name" : "test", 25 | "keep_temporary_files" : true, 26 | "lammps_executable" : "lmp_serial_mod", 27 | "optimizer_options" : { 28 | "mdsteps" : 20000, 29 | "timestep" : 0.25, 30 | "tstart" : 250.0, 31 | "tstop" : 0.5, 32 | "damp" : 50.0, 33 | "max_velocity": 100, 34 | "ev_factor": 0.05 35 | } 36 | 37 | }, 38 | 39 | "step_db" : "stepdb.sqlite", 40 | "max_iterations" : 10, 41 | "log" : "log.txt", 42 | "log_level" : "debug", 43 | "keep_intermediate_structures" : true, 44 | "violation_tolerance" : 0.01 45 | 46 | } 47 | 48 | -------------------------------------------------------------------------------- /igm/core/job_tracking.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sqlite3 3 | import time 4 | import json 5 | 6 | from ..utils.log import logger 7 | from .config import Config 8 | 9 | class StepDB(object): 10 | 11 | SCHEMA = [ 12 | ('uid', 'TEXT'), 13 | ('name', 'TEXT'), 14 | ('cfg', 'TEXT'), 15 | ('time', 'INT'), 16 | ('status', 'TEXT'), 17 | ('data', 'TEXT') 18 | ] 19 | 20 | JSONCOLS = ['cfg', 'data'] 21 | 22 | COLUMNS = [x[0] for x in SCHEMA] 23 | 24 | def __init__(self, cfg, mode='a'): 25 | self.db = None 26 | if isinstance(cfg, str): 27 | try: 28 | cfg = Config(cfg) 29 | except: 30 | pass 31 | elif isinstance(cfg, Config): 32 | pass 33 | elif isinstance(cfg, dict): 34 | cfg = Config(cfg) 35 | else: 36 | raise ValueError('Invalid argument (needs to be Config, dict or string, got %s)' % type(cfg) ) 37 | self.prepare_db(cfg, mode=mode) 38 | 39 | def prepare_db(self, cfg, mode='a'): 40 | if isinstance(cfg, str): 41 | self.db = cfg 42 | else: 43 | self.db = cfg.get('parameters/step_db', None) 44 | if self.db: 45 | if os.path.isfile(self.db): 46 | logger.debug('db file found') 47 | # check db follows the schema 48 | try: 49 | with sqlite3.connect(self.db) as conn: 50 | s = conn.execute('PRAGMA table_info(steps)').fetchall() 51 | assert s is not None 52 | for i, (n, t) in enumerate(StepDB.SCHEMA): 53 | if n != s[i][1] or t != s[i][2]: 54 | msg = 'invalid column %s %s ' % (s[i][1], s[i][2]) 55 | msg += '(expected: %s %s)' % (n, t) 56 | raise AssertionError(msg) 57 | 58 | except AssertionError as e: 59 | msg = 'Invalid database file %s.' % self.db 60 | raise type(e)(e.message + '\n' + msg) 61 | 62 | else: 63 | if mode == 'r': 64 | raise OSError('File not found') 65 | with sqlite3.connect(self.db) as conn: 66 | conn.execute( 67 | 'CREATE TABLE steps (' + 68 | ','.join([ 69 | ' '.join(x) for x in StepDB.SCHEMA 70 | ]) + 71 | ')' 72 | ) 73 | 74 | def record(self, **kwargs): 75 | 76 | if self.db is None: 77 | return 78 | 79 | data = {} 80 | 81 | # prepare columns 82 | for c, t in StepDB.SCHEMA: 83 | if c == 'time': 84 | data['time'] = kwargs.get('time', time.time()) 85 | elif c in StepDB.JSONCOLS: 86 | data[c] = json.dumps( kwargs.get(c, None) ) 87 | else: 88 | data[c] = kwargs.get(c, '') 89 | 90 | with sqlite3.connect(self.db) as conn: 91 | conn.execute( 92 | 'INSERT INTO steps (' + 93 | ','.join(StepDB.COLUMNS) + 94 | ') VALUES (' + 95 | ','.join( ['?'] * len(StepDB.COLUMNS) ) + 96 | ')', 97 | tuple(data[c] for c in StepDB.COLUMNS) 98 | ) 99 | 100 | def unpack(self, result): 101 | out = {} 102 | for i, c in enumerate(StepDB.COLUMNS): 103 | if c in StepDB.JSONCOLS: 104 | out[c] = json.loads(result[i]) 105 | else: 106 | out[c] = result[i] 107 | return out 108 | 109 | def get_history(self, uid=None): 110 | 111 | with sqlite3.connect(self.db) as conn: 112 | if uid is None: 113 | query = 'SELECT * FROM steps ORDER BY time' 114 | r = conn.execute( 115 | query 116 | ).fetchall() 117 | else: 118 | query = 'SELECT * FROM steps WHERE uid=? ORDER BY time' 119 | r = conn.execute( 120 | query, 121 | (uid,) 122 | ).fetchall() 123 | return [ self.unpack(x) for x in r ] 124 | -------------------------------------------------------------------------------- /igm/cython_compiled/__init.py__: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/igm/cython_compiled/__init.py__ -------------------------------------------------------------------------------- /igm/cython_compiled/cpp_sprite_assignment.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #define INF (100000000); 5 | 6 | using namespace std; 7 | 8 | class Vec3{ 9 | public: 10 | float X[3]; 11 | 12 | Vec3() {X[0] = 0; X[1] = 0; X[2] = 0;} 13 | explicit Vec3(float* a) {X[0] = a[0]; X[1] = a[1]; X[2] = a[2];} 14 | Vec3(float x, float y, float z) {X[0] = x; X[1] = y; X[2] = z;} 15 | 16 | float operator[](int i){return X[i];} 17 | const float operator[](int i) const{return X[i];} 18 | 19 | Vec3& operator=(float* a) {X[0] = a[0]; X[1] = a[1]; X[2] = a[2]; return *this;} 20 | Vec3& operator=(Vec3& a) {X[0] = a[0]; X[1] = a[1]; X[2] = a[2]; return *this;} 21 | 22 | Vec3 operator+(const Vec3& w) const {return Vec3(X[0] + w.X[0], X[1] + w.X[1], X[2] + w.X[2]); } 23 | Vec3 operator-(const Vec3& w) const {return Vec3(X[0] - w.X[0], X[1] - w.X[1], X[2] - w.X[2]); } 24 | 25 | Vec3 operator-() const {return Vec3(-X[0], -X[1], -X[2]); } 26 | 27 | Vec3 operator*(const float w) const {return Vec3(X[0] * w, X[1] * w, X[2] * w); } 28 | Vec3 operator/(const float w) const {return Vec3(X[0] / w, X[1] / w, X[2] / w); } 29 | 30 | Vec3& operator+=(const Vec3& w){X[0] += w[0]; X[1] += w[1]; X[2] += w[2]; return *this;} 31 | Vec3& operator-=(const Vec3& w){X[0] -= w[0]; X[1] -= w[1]; X[2] -= w[2]; return *this;} 32 | 33 | Vec3& operator+=(float w){X[0] += w; X[1] += w; X[2] += w; return *this;} 34 | Vec3& operator-=(float w){X[0] -= w; X[1] -= w; X[2] -= w; return *this;} 35 | 36 | Vec3& operator*=(const float w) {X[0] *= w; X[1] *= w; X[2] *= w; return *this;} 37 | Vec3& operator/=(const float w) {X[0] /= w; X[1] /= w; X[2] /= w; return *this;} 38 | 39 | float dot(const Vec3& w) const { return X[0]*w[0] + X[1]*w[1] + X[2]*w[2]; } 40 | float normsq() const { return dot(*this); } 41 | float norm() const { return sqrt(normsq()); } 42 | }; 43 | 44 | float dot(const Vec3& a, const Vec3& b) { return a.dot(b); } 45 | float normsq(const Vec3& a) { return a.normsq(); } 46 | float norm(const Vec3& a) { return a.norm(); } 47 | 48 | 49 | float gyration_radius_sq(const vector& crd) 50 | { 51 | Vec3 r_mean(0.0f, 0.0f, 0.0f); 52 | int n = crd.size(); 53 | for (int i = 0; i < n; ++i) 54 | r_mean += crd[i]; 55 | r_mean /= n; 56 | 57 | float rg = 0; 58 | for (int i = 0; i < n; ++i) 59 | rg += normsq(crd[i] - r_mean); 60 | return (rg / n); 61 | } 62 | 63 | void get_combination(const vector >& all_copies, 64 | int* copies_num, 65 | int n_regions, 66 | int k, 67 | vector& comb, 68 | vector& copy_idx) 69 | { 70 | for (int i = 0; i < n_regions; ++i) 71 | { 72 | int si = k % copies_num[i]; 73 | k /= copies_num[i]; 74 | comb[i] = all_copies[i][si]; 75 | copy_idx[i] = si; 76 | } 77 | } 78 | 79 | void get_rg2s_cpp(float* crds, 80 | int n_struct, 81 | int n_bead, 82 | int n_regions, 83 | int* copies_num, 84 | float* rg2s, 85 | int* copy_idxs, 86 | int* min_struct) 87 | { 88 | 89 | vector > all_copies(n_bead); // all coordinates 90 | vector current(n_regions); // coords of only one combination 91 | vector copy_idx(n_regions); // copies selection for current combination 92 | 93 | float min_rg2 = INF; // absolute minimum 94 | 95 | // prepare all_copies and compute the total 96 | // number of possible combinations 97 | int n_combinations = 1; 98 | for (int i = 0; i < n_regions; ++i) 99 | { 100 | all_copies[i].resize(copies_num[i]); 101 | n_combinations *= copies_num[i]; 102 | } 103 | 104 | // loop through structures 105 | for (int s = 0; s < n_struct; ++s) 106 | { 107 | // set all_copies coordinates 108 | int k = 0; 109 | for (int i = 0; i < n_regions; ++i) 110 | { 111 | for (int j = 0; j < copies_num[i]; ++j) 112 | { 113 | all_copies[i][j] = crds + k*n_struct*3 + s*3; 114 | k += 1; 115 | } 116 | } 117 | 118 | // explore all the combinations 119 | float min_struct_rg2 = INF; 120 | vector min_copy_idx(n_regions, -1); 121 | for (int i = 0; i < n_combinations; ++i) 122 | { 123 | get_combination(all_copies, copies_num, n_regions, i, current, copy_idx); 124 | float rg2 = gyration_radius_sq(current); 125 | if (rg2 < min_struct_rg2){ 126 | min_struct_rg2 = rg2; 127 | min_copy_idx.swap(copy_idx); 128 | } 129 | } 130 | 131 | // save results and check minimum 132 | rg2s[s] = min_struct_rg2; 133 | for (int i = 0; i < n_regions; ++i) 134 | { 135 | copy_idxs[n_regions*s + i] = min_copy_idx[i]; 136 | } 137 | if (min_struct_rg2 < min_rg2) 138 | { 139 | min_rg2 = min_struct_rg2; 140 | *min_struct = s; 141 | } 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /igm/cython_compiled/cpp_sprite_assignment.h: -------------------------------------------------------------------------------- 1 | void get_rg2s_cpp(float* crds, 2 | int n_struct, 3 | int n_bead, 4 | int n_regions, 5 | int* copies_num, 6 | float* rg2s, 7 | int* copy_idxs, 8 | int* min_struct); -------------------------------------------------------------------------------- /igm/cython_compiled/setup.py: -------------------------------------------------------------------------------- 1 | from distutils.core import setup 2 | from Cython.Build import cythonize 3 | import numpy 4 | 5 | setup( 6 | name = 'sprite', 7 | ext_modules = cythonize( 8 | ['sprite.pyx', 'cpp_sprite_assignment.cpp'] 9 | ), 10 | include_dirs=[numpy.get_include()], 11 | ) 12 | -------------------------------------------------------------------------------- /igm/cython_compiled/tests.py: -------------------------------------------------------------------------------- 1 | from sprite import * 2 | 3 | import numpy as np 4 | 5 | crd = np.array([ [ [1,0,0] ] , [ [-1, 0, 0] ], [ [0,-1,0] ], [ [0,1,0] ]], dtype=np.float32) 6 | copies_num = np.array([1,1,1,1], dtype=np.int32) 7 | 8 | get_rgs2(crd, copies_num) 9 | # (array([1.], dtype=float32), 0, array([[0, 0, 0, 0]], dtype=int32)) 10 | 11 | 12 | crd = np.array([ [ [1,0,0] ] , [ [0.5, 0, 0] ], [ [0,-0.5,0] ], [ [0,0.5,0] ]], dtype=np.float32) 13 | copies_num = np.array([2,2], dtype=np.int32) 14 | get_rgs2(crd, copies_num) 15 | 16 | # (array([1.], dtype=float32), 0, array([[0, 0, 0, 0]], dtype=int32)) 17 | 18 | crd = np.array([ [ [1,0,0], [0.1,0,0] ] , [ [0.5, 0, 0], [1,0,0] ], [ [0,-0.5,0], [-1,0,0] ], [ [0,0.5,0], [-0.1,0,0] ] ], dtype=np.float32) 19 | copies_num = np.array([2,2], dtype=np.int32) 20 | get_rgs2(crd, copies_num) 21 | # (array([0.125, 0.01 ], dtype=float32), 1, array([[1, 0], [0, 1]], dtype=int32)) -------------------------------------------------------------------------------- /igm/model/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | from .model import Model 4 | from .particle import Particle 5 | 6 | -------------------------------------------------------------------------------- /igm/model/kernel/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /igm/model/kernel/lammps_io.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, absolute_import, print_function 2 | import numpy as np 3 | 4 | from .util import reverse_readline 5 | 6 | def get_info_from_log(output): 7 | ''' gets final energy, excluded volume energy and bond energy. 8 | TODO: get more info? ''' 9 | info = {} 10 | generator = reverse_readline(output) 11 | 12 | for l in generator: 13 | if l[:9] == ' Force t': 14 | ll = next(generator) 15 | info['final-energy'] = float(ll.split()[2]) 16 | break 17 | 18 | for l in generator: 19 | if l[:4] == 'Loop': 20 | ll = next(generator) 21 | vals = [float(s) for s in ll.split()] 22 | info['pair-energy'] = vals[2] 23 | info['bond-energy'] = vals[3] 24 | while not ll.startswith('Step'): 25 | ll = next(generator) 26 | keys = ll.split() 27 | info['thermo'] = {k: v for k, v in zip(keys[1:], vals[1:])} 28 | break 29 | 30 | for l in generator: 31 | if l[:4] == 'Loop': 32 | # MD minimization 33 | info['md-time'] = float(l.split()[3]) 34 | 35 | # EN=`grep -A 1 "Energy initial, next-to-last, final =" $LAMMPSLOGTMP \ 36 | # | tail -1 | awk '{print $3}'` 37 | return info 38 | 39 | def get_last_frame(fh): 40 | 41 | """ Quite self-explanatory: extract coordinates from last frame produced by simulation """ 42 | atomlines = [] 43 | for l in reverse_readline(fh): 44 | if 'ITEM: ATOMS' in l: 45 | v = l.split() 46 | ii = v.index('id') - 2 47 | ix = v.index('x') - 2 48 | iy = v.index('y') - 2 49 | iz = v.index('z') - 2 50 | break 51 | atomlines.append(l) 52 | 53 | crds = np.empty((len(atomlines), 3)) 54 | for l in atomlines: 55 | v = l.split() 56 | i = int(v[ii]) - 1 # ids are in range 1-N 57 | x = float(v[ix]) 58 | y = float(v[iy]) 59 | z = float(v[iz]) 60 | crds[i][0] = x 61 | crds[i][1] = y 62 | crds[i][2] = z 63 | 64 | return crds 65 | -------------------------------------------------------------------------------- /igm/model/kernel/util.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | def reverse_readline(fh, buf_size=8192): 4 | """a generator that returns the lines of a file in reverse order""" 5 | segment = None 6 | offset = 0 7 | fh.seek(0, os.SEEK_END) 8 | file_size = remaining_size = fh.tell() 9 | while remaining_size > 0: 10 | offset = min(file_size, offset + buf_size) 11 | fh.seek(file_size - offset) 12 | buffer = fh.read(min(remaining_size, buf_size)) 13 | remaining_size -= buf_size 14 | lines = buffer.split('\n') 15 | # the first line of the buffer is probably not a complete line so 16 | # we'll save it and append it to the last line of the next buffer 17 | # we read 18 | if segment is not None: 19 | # if the previous chunk starts right from the beginning of line 20 | # do not concact the segment to the last line of new chunk 21 | # instead, yield the segment first 22 | if buffer[-1] is not '\n': 23 | lines[-1] += segment 24 | else: 25 | yield segment 26 | segment = lines[0] 27 | for index in range(len(lines) - 1, 0, -1): 28 | if len(lines[index]): 29 | yield lines[index] 30 | # Don't yield None if the file was empty 31 | if segment is not None: 32 | yield segment 33 | 34 | def resolve_templates(templates, args): 35 | rt = {} 36 | for key, val in templates.items(): 37 | rt[key] = val.format(*args) 38 | return rt 39 | -------------------------------------------------------------------------------- /igm/model/model.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, absolute_import, print_function 2 | import numpy as np 3 | from .particle import Particle 4 | try: 5 | from itertools import izip as zip 6 | except ImportError: 7 | pass 8 | 9 | class Model(object): 10 | """ 11 | 12 | Modeling target system 13 | 14 | This is an abstract mid-layer between data-restraints (ASSIGNMENT) and minization kernel (LAMMPS). 15 | 16 | The Model holds all information about the target system by particles and 17 | harmonic forces. 18 | 19 | """ 20 | def __init__(self, uid=0): 21 | self.particles = [] 22 | self.forces = [] 23 | self.id = uid 24 | 25 | 26 | #====Particle methods 27 | def addParticle(self, pos, r, t, **kwargs): 28 | """ 29 | Add particle to system 30 | """ 31 | 32 | self.particles.append(Particle(pos, r, t, **kwargs)) 33 | 34 | return len(self.particles)-1 35 | 36 | 37 | def getParticle(self,i): 38 | """ 39 | Get particle in the system 40 | """ 41 | return self.particles[i] 42 | 43 | def setParticlePos(self, i, pos): 44 | """ 45 | set particle coordinates 46 | """ 47 | 48 | self.particles[i].setCoordinates(pos) 49 | #==== 50 | 51 | 52 | #===bulk particle methods 53 | def initParticles(self, crd, rad): 54 | """ 55 | initialize particles using numpy array 56 | 57 | Parameters 58 | ---------- 59 | crd : 2D numpy array (float), N*3 60 | Numpy array of coordinates for each particle, N is the number of particles 61 | rad : 2D numpy array (float), N*1 62 | particle radius 63 | """ 64 | assert crd.shape[0] == rad.shape[0] 65 | 66 | for pos, r in zip(crd, rad): 67 | self.addParticle(pos, r, Particle.NORMAL) 68 | #==== 69 | 70 | 71 | #====bulk get methods 72 | def getCoordinates(self): 73 | """ 74 | Get all particles' Coordinates in numpy array form 75 | """ 76 | return np.array([p.pos for p in self.particles if p.ptype == Particle.NORMAL]) 77 | 78 | 79 | def getRadii(self): 80 | """ 81 | Get all particles' radii in numpy array vector form 82 | """ 83 | return np.array([[p.r for p in self.particles if p.ptype == Particle.NORMAL]]).T 84 | #==== 85 | 86 | 87 | #====force methods 88 | def addForce(self, f): 89 | """ 90 | Add a basic force 91 | """ 92 | 93 | self.forces.append(f) 94 | 95 | return len(self.forces)-1 96 | 97 | def getForce(self, i): 98 | """ 99 | get a force 100 | """ 101 | 102 | return self.forces[i] 103 | 104 | def evalForceScore(self, i): 105 | """ 106 | get force score 107 | """ 108 | 109 | return self.forces[i].getScore(self.particles) 110 | 111 | def evalForceViolationRatio(self, i): 112 | """ 113 | get force violation ratio 114 | """ 115 | 116 | return self.forces[i].getViolationRatio(self.particles) 117 | #==== 118 | 119 | 120 | #====restraint methods 121 | def addRestraint(self, res, override=False): 122 | """ 123 | Add a type of restraint to model 124 | """ 125 | res._apply_model(self, override) 126 | 127 | res._apply(self) 128 | 129 | 130 | def optimize(self, cfg): 131 | """ 132 | optimize the model by selected kernel 133 | """ 134 | 135 | if cfg['optimization']["kernel"] == "lammps": 136 | from .kernel import lammps 137 | return lammps.optimize(self, cfg) 138 | #- 139 | #- 140 | 141 | def saveCoordinates(self, filename): 142 | """ 143 | save xyz coordinates into numpy npy file 144 | """ 145 | 146 | np.save(filename, self.getCoordinates()) 147 | #- 148 | 149 | def saveXYZR(self, filename): 150 | """ 151 | save xyzr into numpy npz file 152 | """ 153 | 154 | np.savez(filename, xyz=self.getCoordinates(), r=self.getRadii()) 155 | #- 156 | 157 | #= 158 | 159 | 160 | -------------------------------------------------------------------------------- /igm/model/particle.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, absolute_import, print_function 2 | import numpy as np 3 | 4 | POSDTYPE = np.float32 5 | 6 | # define "Particle" class 7 | 8 | class Particle(object): 9 | 10 | NORMAL = 0 11 | DUMMY_STATIC = 1 12 | DUMMY_DYNAMIC = 2 13 | 14 | PTYPES = ["NORMAL","DUMMY_STATIC","DUMMY_DYNAMIC"] 15 | 16 | def __init__(self, pos, r, t, **kwargs): 17 | self.pos = np.array(pos).astype(POSDTYPE) # particle coordinates 18 | self.r = POSDTYPE(r) # particle radius (if 0, then no excluded voluem) 19 | self.ptype = t # particle type (see PTYPES list) 20 | for k in kwargs: 21 | setattr(self, k, kwargs[k]) 22 | 23 | def __str__(self): 24 | return "({} {} {}, {}):{}".format(self.pos[0], self.pos[1], self.pos[2], 25 | self.r, 26 | Particle.PTYPES[self.ptype]) 27 | __repr__ = __str__ 28 | 29 | def getCoordinates(self): 30 | return self.pos 31 | 32 | def setCoordinates(self, pos): 33 | self.pos = pos 34 | 35 | def __sub__(self, other): 36 | return np.linalg.norm(self.pos - other.pos) 37 | 38 | 39 | -------------------------------------------------------------------------------- /igm/parallel/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | from .parallel_controller import SerialController 4 | 5 | from .ipyparallel_controller import BasicIppController 6 | 7 | from .slurm_controller import SlurmController 8 | 9 | controller_class = { 10 | "serial" : SerialController, 11 | "slurm" : SlurmController, 12 | "ipyparallel" : BasicIppController, 13 | "ipyparallel_basic" : BasicIppController, 14 | } 15 | 16 | def Controller(cfg): 17 | parallel_cfg = cfg.get("parallel", dict()) 18 | pctype = parallel_cfg.get("controller", "ipyparallel") 19 | pcopts = parallel_cfg.get("controller_options", dict()).get(pctype, dict()) 20 | return controller_class[pctype](**pcopts) 21 | -------------------------------------------------------------------------------- /igm/parallel/async_file_operations.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | import time 4 | import traceback 5 | import multiprocessing 6 | 7 | POLL_INTERVAL = 30 # check 30 seconds 8 | 9 | 10 | class GeneratorLen(object): 11 | def __init__(self, gen, length): 12 | self.gen = gen 13 | self.length = length 14 | 15 | def __len__(self): 16 | return self.length 17 | 18 | def __iter__(self): 19 | return self.gen 20 | 21 | 22 | class FilePoller(object): 23 | 24 | """ Define polling function """ 25 | 26 | def __init__(self, files, callback, args=None, kwargs=None, remove_after_callback=False, 27 | setup=None, setup_args=tuple(), setup_kwargs=dict(), 28 | teardown=None, teardown_args=tuple(), teardown_kwargs=dict()): 29 | self._manager = multiprocessing.Manager() 30 | 31 | self.files = files 32 | self.args = args 33 | if args is None: 34 | self.args = [list()] * len(files) 35 | 36 | self.kwargs = kwargs 37 | if kwargs is None: 38 | self.kwargs = [dict()] * len(files) 39 | 40 | self.callback = callback 41 | 42 | self.th = None 43 | 44 | self.remove_flag = remove_after_callback 45 | self.completed = self._manager.list() 46 | self.setup = setup 47 | self.setup_args = setup_args 48 | self.setup_kwargs = setup_kwargs 49 | self.teardown = teardown 50 | self.teardown_args = teardown_args 51 | self.teardown_kwargs = teardown_kwargs 52 | self._traceback = self._manager.list() 53 | 54 | # watch function, which scans over the 'to_poll' list 55 | def watch(self, completed, tb, timeout=None, interval=POLL_INTERVAL): 56 | try: 57 | if self.setup: 58 | self.setup(*self.setup_args, **self.setup_kwargs) 59 | to_poll = set(range(len(self.files))) 60 | start = time.time() 61 | while True: 62 | last_poll = time.time() 63 | for i in list(to_poll): 64 | if os.path.isfile(self.files[i]): 65 | self.callback(*self.args[i], **self.kwargs[i]) 66 | if self.remove_flag: 67 | os.remove(self.files[i]) 68 | to_poll.remove(i) 69 | completed.append(i) 70 | 71 | if len(to_poll) == 0: 72 | break 73 | 74 | now = time.time() 75 | if timeout is not None: 76 | if now - start > timeout: 77 | raise RuntimeError('Timeout expired (%f seconds)' % (timeout,)) 78 | 79 | delta = now - last_poll 80 | if delta < interval: 81 | time.sleep(interval - delta) 82 | 83 | except KeyboardInterrupt: 84 | pass 85 | 86 | except: 87 | tb.append(traceback.format_exc()) 88 | 89 | finally: 90 | if self.teardown: 91 | try: 92 | self.teardown(*self.teardown_args, **self.teardown_kwargs) 93 | except: 94 | stb = traceback.format_exc() 95 | try: 96 | # the tb manager could already be down 97 | tb.append(stb) 98 | except: 99 | print(stb) 100 | 101 | def watch_async(self, timeout=None, interval=POLL_INTERVAL): 102 | self.th = multiprocessing.Process(target=self.watch, args=(self.completed, self._traceback, timeout, interval)) 103 | self.th.start() 104 | 105 | def wait(self, timeout=None): 106 | self.th.join(timeout) 107 | if len(self._traceback): 108 | raise RuntimeError('\n'.join(self._traceback)) 109 | 110 | def _enumerate(self): 111 | lastc = 0 112 | while True: 113 | #print('lastdc = ' + str(lastc)) # LB 114 | if len(self._traceback): 115 | self.th.join() 116 | raise RuntimeError('\n'.join(self._traceback)) 117 | 118 | if lastc == len(self.files): 119 | if self.th: 120 | self.th.join() 121 | break 122 | 123 | if len(self.completed) > lastc: 124 | lastc += 1 125 | yield self.completed[lastc - 1] 126 | 127 | else: 128 | time.sleep(POLL_INTERVAL) 129 | 130 | def enumerate(self): 131 | return GeneratorLen(self._enumerate(), len(self.files)) 132 | -------------------------------------------------------------------------------- /igm/parallel/dask_controller.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division 2 | 3 | from dask.distributed import Client 4 | 5 | import traceback 6 | 7 | from .parallel_controller import ParallelController 8 | 9 | class TimedFunction(object): 10 | ''' 11 | Utility wrapper for a remote task. Allows to avoid 12 | ''' 13 | def __init__(self, func, timeout=None): 14 | self.func = func 15 | self.timeout = timeout 16 | 17 | def run(self, *args, **kwargs): 18 | try: 19 | from time import time 20 | tstart = time() 21 | res = self.func(*args, **kwargs) 22 | self._q.put( (0, res, time()-tstart) ) 23 | except: 24 | self._q.put( (-1, traceback.format_exc()) ) 25 | 26 | def __call__(self, *args, **kwargs): 27 | # runs on a child process to terminate execution if timeout exceedes 28 | try: 29 | import multiprocessing 30 | try: 31 | from Queue import Empty 32 | except: 33 | from queue import Empty 34 | 35 | self._q = multiprocessing.Queue() 36 | p = multiprocessing.Process(target=self.run, args=args, kwargs=kwargs) 37 | p.start() 38 | rval = self._q.get(block=True, timeout=self.timeout) 39 | p.join() 40 | if rval[0] == -1: 41 | raise RuntimeError(rval[1]) 42 | 43 | except Empty: 44 | p.terminate() 45 | raise RuntimeError('Processing time exceeded (%f)' % self.timeout) 46 | 47 | return rval[1] 48 | 49 | 50 | class DaskController(ParallelController): 51 | def __init__(self): 52 | self.client = None 53 | super(DaskController, self).__init__() 54 | 55 | def setup(self, cfg): 56 | if 'ip' not in cfg: 57 | pass 58 | # TODO: check some kind of default file 59 | self.client = Client(cfg['ip']) 60 | 61 | def map(self, parallel_task, args): 62 | futures = self.client.map(parallel_task, args) 63 | results = [] 64 | for f in futures: 65 | results.append(f.result()) 66 | return results 67 | 68 | def reduce(self, reduce_task, outs): 69 | return reduce_task(outs) 70 | -------------------------------------------------------------------------------- /igm/parallel/parallel_controller.py: -------------------------------------------------------------------------------- 1 | ''' 2 | The reduce function should always accept an iterable 3 | The map function should always return an iterable 4 | ''' 5 | from tqdm import tqdm 6 | 7 | class ParallelController(object): 8 | 9 | def __init__(self): 10 | """ 11 | A parallel controller that map parallel jobs into workers 12 | """ 13 | 14 | def setup(self): 15 | pass 16 | 17 | def map(self, parallel_task, args): 18 | raise NotImplementedError() 19 | 20 | def reduce(self, reduce_task, outs): 21 | return reduce_task(outs) 22 | 23 | def map_reduce(self, parallel_task, reduce_task, args): 24 | return self.reduce(reduce_task, self.map(parallel_task, args)) 25 | 26 | def teardown(self): 27 | pass 28 | 29 | class SerialController(ParallelController): 30 | def map(self, parallel_task, args): 31 | return [parallel_task(a) for a in tqdm(args, desc="(SERIAL)")] 32 | 33 | 34 | 35 | def map_reduce(parallel_task, reduce_function, args, controller): 36 | controller.setup() 37 | result = controller.map_reduce(parallel_task, reduce_function, args) 38 | controller.teardown() 39 | return result 40 | -------------------------------------------------------------------------------- /igm/parallel/slurm_controller.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division 2 | 3 | from six import raise_from 4 | 5 | import os 6 | import time 7 | from .parallel_controller import ParallelController 8 | from tqdm import tqdm 9 | from .utils import split_evenly 10 | import subprocess 11 | import shutil 12 | 13 | 14 | import cloudpickle 15 | from uuid import uuid4 16 | 17 | base_template = '''#!/bin/bash 18 | #SBATCH --ntasks=1 19 | #SBATCH --mem-per-cpu={{mem}} 20 | #SBATCH --time={{walltime}} 21 | #SBATCH --partition=cmb 22 | #SBATCH --job-name={{jname}} 23 | #SBATCH --output={{out}} 24 | #SBATCH --error={{err}} 25 | 26 | echo $(which python) 27 | #source /auto/cmb-08/fa/local/setup.sh 28 | umask 002 29 | 30 | {{interpreter}} -c 'from igm.parallel.slurm_controller import SlurmController; SlurmController.execute("{{sfile}}", {{i}})' 31 | 32 | ''' 33 | 34 | default_args = { 35 | 'mem' : '2GB', 36 | 'walltime' : '6:00:00', 37 | 'interpreter' : 'python' 38 | } 39 | 40 | def parse_template(template, **kwargs): 41 | for k, v in kwargs.items(): 42 | template = template.replace('{{' + k + '}}', v) 43 | return template 44 | 45 | class SlurmController(ParallelController): 46 | def __init__(self, template=None, max_tasks=4000, tmp_dir='tmp', simultaneous_tasks=430, **kwargs): 47 | 48 | sargs = {} 49 | sargs.update(default_args) 50 | sargs.update(kwargs) 51 | 52 | self.max_tasks = max_tasks 53 | self.sim_tasks = simultaneous_tasks 54 | if template is not None: 55 | self.template = open(template).read() 56 | else: 57 | self.template = base_template 58 | self.template = parse_template(self.template, **sargs) 59 | if not os.path.isdir(tmp_dir): 60 | os.makedirs(tmp_dir) 61 | self.tmp_dir = os.path.abspath(tmp_dir) 62 | 63 | 64 | def send_job(self, outd, i): 65 | slurmscript = os.path.join(outd, '%d.slurm' % i) 66 | with open(slurmscript, 'w') as f: 67 | f.write( 68 | parse_template( 69 | self.template, 70 | i=str(i), 71 | jname=str(i), 72 | out=os.path.join(outd, '%d.out' % i), 73 | err=os.path.join(outd, '%d.err' % i) 74 | ) 75 | ) 76 | 77 | # writes "Submitted batch job XXXXXXX" 78 | jid = subprocess.check_output(['sbatch', slurmscript]).split()[3] 79 | return jid 80 | 81 | def job_was_successful(self, outd, i): 82 | if os.path.isfile(os.path.join(outd, '%d.complete' % i )): 83 | return True 84 | return False 85 | 86 | def job_is_completed(self, outd, i, jid): 87 | try: 88 | out = subprocess.check_output(['squeue', '-j', jid]).decode('utf-8').split('\n') 89 | keys, vals, _ = out 90 | kv = {k: v for k, v in zip(keys.split(), vals.split())} 91 | if kv['ST'] == 'CD': 92 | return True 93 | else: 94 | return False 95 | except (subprocess.CalledProcessError, ValueError): 96 | if not self.job_was_successful(outd, i): 97 | raise_from(RuntimeError('(SLURM): Remote error. Error file:' + os.path.join(outd, '%d.err' % i)), None) 98 | return True 99 | return False 100 | 101 | 102 | def poll_loop(self, n_tasks, outd, timeout=1): 103 | to_send = set(range(n_tasks)) 104 | processing = dict() 105 | 106 | while len(to_send) > 0 or len(processing) > 0 : 107 | just_completed = set() 108 | for i, jid in processing.items(): 109 | if self.job_is_completed(outd, i, jid): 110 | just_completed.add(i) 111 | yield i 112 | 113 | while len(processing) < self.sim_tasks and len(to_send) > 0: 114 | i = to_send.pop() 115 | processing[i] = self.send_job(outd, i) 116 | 117 | for i in just_completed: 118 | del processing[i] 119 | 120 | time.sleep(timeout) 121 | 122 | raise StopIteration 123 | 124 | 125 | def map(self, parallel_task, args): 126 | uid = 'slurmc.' + str(uuid4()) 127 | outd = os.path.join(self.tmp_dir, uid) 128 | os.makedirs(outd) 129 | 130 | batches = list(split_evenly(args, self.max_tasks)) 131 | 132 | sfile = os.path.join(outd, 'exdata.cloudpickle') 133 | with open(sfile, 'wb') as f: 134 | cloudpickle.dump({'f': parallel_task, 'args': batches, 'outd': outd}, f) 135 | 136 | self.template = parse_template(self.template, sfile=sfile) 137 | 138 | n_tasks = len(batches) 139 | 140 | ar = self.poll_loop(n_tasks, outd) 141 | for i in tqdm(ar, desc="(SLURM)", total=n_tasks): 142 | pass 143 | shutil.rmtree(outd) 144 | 145 | @staticmethod 146 | def execute(sfile, i): 147 | v = cloudpickle.load(open(sfile, 'rb')) 148 | for x in v['args'][i]: 149 | v['f'](x) 150 | open(os.path.join(v['outd'], '%d.complete' % i), 'w').close() 151 | -------------------------------------------------------------------------------- /igm/parallel/utils.py: -------------------------------------------------------------------------------- 1 | def batch(sequence, n=1): 2 | l = len(sequence) 3 | for ndx in range(0, l, n): 4 | yield sequence[ndx:min(ndx + n, l)] 5 | 6 | def split_evenly(sequence, n=1): 7 | l = len(sequence) 8 | if l == 0: 9 | return [] 10 | n = min(l, n) 11 | 12 | k = l // n 13 | if l % n != 0: 14 | k += 1 15 | 16 | for ndx in range(0, l, k): 17 | yield sequence[ndx:min(ndx + k, l)] -------------------------------------------------------------------------------- /igm/report/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/igm/report/__init__.py -------------------------------------------------------------------------------- /igm/report/damid.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import logging 4 | import traceback 5 | from alabtools import HssFile 6 | from scipy.stats import pearsonr 7 | from .utils import create_folder, snormsq_ellipse 8 | 9 | 10 | # noinspection PyTypeChecker 11 | def report_damid(hssfname, damid_file, contact_range, semiaxes=None, run_label=''): 12 | logger = logging.getLogger("DamID") 13 | logger.info('Executing DamID report') 14 | 15 | if run_label: 16 | run_label = '-' + run_label 17 | try: 18 | create_folder("damid") 19 | 20 | with HssFile(hssfname, 'r') as hss: 21 | index = hss.index 22 | radii = hss.radii 23 | if semiaxes is None: 24 | # see if we have information about semiaxes in the file 25 | try: 26 | semiaxes = hss['envelope']['params'][()] 27 | if len(semiaxes.shape) == 0: # is scalar 28 | semiaxes = np.array([semiaxes, semiaxes, semiaxes]) 29 | except KeyError: 30 | semiaxes = np.array([5000., 5000., 5000.]) 31 | 32 | out_damid_prob = np.zeros(len(index.copy_index)) 33 | for locid in index.copy_index.keys(): 34 | ii = index.copy_index[locid] 35 | n_copies = len(ii) 36 | 37 | r = radii[ii[0]] 38 | 39 | # rescale pwish considering the number of copies 40 | # pwish = np.clip(pwish/n_copies, 0, 1) 41 | 42 | d_sq = np.empty(n_copies * hss.nstruct) 43 | 44 | for i in range(n_copies): 45 | x = hss.get_bead_crd(ii[i]) 46 | nuc_rad = np.array(semiaxes) * (1 - contact_range) 47 | d_sq[i * hss.nstruct:(i + 1) * hss.nstruct] = snormsq_ellipse(x, nuc_rad, r) 48 | 49 | contact_count = np.count_nonzero(d_sq >= 1) 50 | out_damid_prob[locid] = float(contact_count) / hss.nstruct / n_copies 51 | np.savetxt('damid/output.txt', out_damid_prob) 52 | 53 | if damid_file: 54 | damid_profile = np.loadtxt(damid_file, dtype='float32') 55 | np.savetxt(f'damid/input{run_label}.txt', damid_profile) 56 | fig = plt.figure(figsize=(4, 4)) 57 | plt.title(f'DamID{run_label}') 58 | vmax = max(np.nanmax(damid_profile), np.nanmax(out_damid_prob)) 59 | vmin = min(np.nanmin(damid_profile), np.nanmin(out_damid_prob)) 60 | corr = pearsonr(damid_profile, out_damid_prob) 61 | np.savetxt(f'damid/pearsonr{run_label}.txt', corr) 62 | plt.scatter(damid_profile, out_damid_prob, s=6) 63 | plt.xlim(vmin, vmax) 64 | plt.ylim(vmin, vmax) 65 | plt.text(vmin * 1.01, vmax * 0.95, f'pearson correlation: {corr[0]:.5f}') 66 | plt.plot([vmin, vmax], [vmin, vmax], 'k--') 67 | plt.xlabel('input') 68 | plt.ylabel('output') 69 | fig.savefig(f'damid/scatter{run_label}.pdf') 70 | fig.savefig(f'damid/scatter{run_label}.png') 71 | plt.close(fig) 72 | logger.info('Done.') 73 | 74 | except KeyboardInterrupt: 75 | logger.error('User interrupt. Exiting.') 76 | exit(1) 77 | 78 | except Exception: 79 | traceback.print_exc() 80 | logger.error('Error in DamID step\n==============================') 81 | -------------------------------------------------------------------------------- /igm/report/distance_map.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from alabtools import HssFile 3 | 4 | 5 | def create_average_distance_map(hssfname): 6 | with HssFile(hssfname, 'r') as h: 7 | crd = h.coordinates 8 | index = h.index 9 | n = len(index.copy_index) 10 | fmap = np.empty((n, n)) 11 | for i in range(n): 12 | for j in range(i + 1, n): 13 | aves = [] 14 | if index.chrom[i] == index.chrom[j]: 15 | for k, m in zip(index.copy_index[i], index.copy_index[j]): 16 | aves.append(np.linalg.norm(crd[k] - crd[m], axis=1).mean()) 17 | else: 18 | for k in index.copy_index[i]: 19 | for m in index.copy_index[j]: 20 | aves.append(np.linalg.norm(crd[k] - crd[m], axis=1).mean()) 21 | fmap[i, j] = np.mean(aves) 22 | return fmap 23 | -------------------------------------------------------------------------------- /igm/report/html_report.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path as op 3 | import base64 4 | from alabtools.utils import natural_sort 5 | 6 | 7 | css = ''' 8 | table { 9 | border: 1px solid grey; 10 | } 11 | 12 | td, th { 13 | text-align: center; 14 | padding: .3em 1em .3em 1em; 15 | } 16 | 17 | .folder { 18 | padding-left:1em; 19 | border-left:1px solid grey; 20 | } 21 | 22 | .lvl-1 { 23 | padding-left:0; 24 | border-left:0; 25 | } 26 | 27 | pre { 28 | border: 1px dashed grey; 29 | background-color: #eeeeee; 30 | } 31 | 32 | th { 33 | font-weight:bold; 34 | } 35 | 36 | .pfile { 37 | padding: 1em; 38 | border: 1px dotted #dddddd; 39 | } 40 | 41 | img { 42 | max-width: 70vw; 43 | max-height: 80vh; 44 | width: auto; 45 | height: auto; 46 | } 47 | ''' 48 | 49 | 50 | def looks_like_a_table(filename): 51 | lines = open(filename).readlines() 52 | if len(lines) < 2: 53 | return False 54 | if not lines[0].startswith('#'): 55 | return False 56 | n = len(lines[1].split()) 57 | for l in lines[2:]: 58 | if l.strip() == '': 59 | continue 60 | if len(l.split()) != n: 61 | return False 62 | return True 63 | 64 | 65 | def to_html_table(textfile, attrs=''): 66 | lines = open(textfile).readlines() 67 | headers = lines[0].replace('#', '').strip().split() 68 | thead = '' + ''.join(headers) + '' 69 | tbody = '' 70 | for l in lines[1:]: 71 | values = l.strip().split() 72 | tbody += '' + ''.join(values) + '' 73 | tbody += '' 74 | return '' + thead + tbody + '
' 75 | 76 | 77 | def handle_file(filename, lines): 78 | lines.append(f'' 79 | f'{op.basename(filename)}

') 80 | _, ext = op.splitext(filename) 81 | if ext in ['.png', '.jpg', '.bmp', '.gif']: 82 | data_uri = base64.b64encode( 83 | open(filename, 'rb').read()).decode('utf-8').replace('\n', '') 84 | lines.append(f'') 85 | elif ext == '.txt': 86 | # check if it is a table 87 | if looks_like_a_table(filename): 88 | lines.append(to_html_table(filename, 'border="1"')) 89 | else: 90 | content = open(filename).read() 91 | clines = content.split('\n') 92 | nlines = len(clines) 93 | if nlines > 40: 94 | preview_text = '\n'.join(clines[:8]) + f'\n... (+ {nlines - 8} other lines)' 95 | lines.append(f'
{preview_text}
') 96 | else: 97 | lines.append(f'
{content}
') 98 | 99 | else: 100 | lines.append('

Preview not available.

') 101 | 102 | 103 | def sort_items(base, items): 104 | files = [] 105 | folders = [] 106 | for it in items: 107 | if op.isfile(op.join(base, it)): 108 | files.append(it) 109 | elif op.isdir(op.join(base, it)): 110 | folders.append(it) 111 | folders.sort() 112 | files = natural_sort(files) 113 | return folders + files 114 | 115 | 116 | def process_dir(folder, lvl, lines): 117 | items = sort_items(folder, os.listdir(folder)) 118 | lines.append(f'
') 119 | lines.append(f'{op.basename(folder)}') 120 | 121 | for it in items: 122 | if op.isfile(op.join(folder, it)): 123 | lines.append('
') 124 | handle_file(op.join(folder, it), lines) 125 | lines.append(f'
') 126 | elif op.isdir(op.join(folder, it)): 127 | process_dir(op.join(folder, it), lvl+1, lines) 128 | 129 | lines.append(f'
') 130 | 131 | 132 | def generate_html_report(folder): 133 | """ 134 | Scrapes the output folder and generates a single, navigable file. 135 | Parameters 136 | ---------- 137 | folder : str 138 | """ 139 | 140 | while folder.endswith('/'): 141 | folder = folder[:-1] 142 | 143 | # get the label 144 | try: 145 | with open('label.txt', 'r') as f: 146 | run_label = f.read() 147 | except IOError: 148 | run_label = '' 149 | 150 | if run_label == '': 151 | run_label = op.basename(folder) 152 | 153 | lines = [] 154 | process_dir(folder, 1, lines) 155 | head = f'IGM Report: {op.basename(folder)}' 156 | 157 | header = f'

{run_label}

{op.abspath(folder)}

' 158 | 159 | body = '' + header + '\n'.join(lines) + '' 160 | return head + body 161 | -------------------------------------------------------------------------------- /igm/report/images.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import logging 3 | import traceback 4 | from alabtools import HssFile 5 | 6 | from .utils import create_folder 7 | 8 | 9 | def render_structures(hssfname, n=4, random=True): 10 | logger = logging.getLogger('Render') 11 | logger.info('Starting render... (it may take a while)') 12 | 13 | try: 14 | create_folder('images') 15 | with HssFile(hssfname, 'r') as h: 16 | 17 | if random: 18 | ii = np.random.choice(range(h.nstruct), size=n, replace=False) 19 | else: 20 | ii = list(range(min(n, h.nstruct))) 21 | 22 | h.dump_pdb(ii, fname='images/structure_%d.pdb', render=True, high_quality=True) 23 | logger.info('Done.') 24 | 25 | except KeyboardInterrupt: 26 | logger.error('User interrupt. Exiting.') 27 | exit(1) 28 | 29 | except Exception: 30 | traceback.print_exc() 31 | logger.error('Error in rendering step\n==============================') 32 | 33 | -------------------------------------------------------------------------------- /igm/report/plots.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from matplotlib.colors import LogNorm 4 | from scipy.ndimage import gaussian_filter 5 | 6 | 7 | def logloghist2d(d1, d2, bins=(100, 100), ranges=((1e-3, 1), (1e-3, 1)), outfile=None, vmin=1e2, vmax=1e5, nlevels=5, 8 | sigma=None, xlabel='in', ylabel='out', smooth=None, **kwargs): 9 | 10 | if ranges[0] is None: 11 | x0, x1 = np.min(d1), np.max(d1) 12 | else: 13 | x0, x1 = ranges[0] 14 | 15 | if ranges[1] is None: 16 | y0, y1 = np.min(d2), np.max(d2) 17 | else: 18 | y0, y1 = ranges[1] 19 | 20 | xx = np.logspace(np.log10(x0), np.log10(x1), bins[0] + 1, base=10) 21 | yy = np.logspace(np.log10(y0), np.log10(y1), bins[1] + 1, base=10) 22 | bottom_left = max(xx[0], yy[0]) 23 | top_right = min(xx[-1], yy[-1]) 24 | 25 | # Note: we invert axes, because in a plot, the x axis actually correspond to 26 | # columns, while the y to rows. np.histogram2d puts the value from the first dataset on 27 | # rows, and the ones from the second in columns. 28 | h, e1, e2 = np.histogram2d(d2, d1, bins=(yy, xx)) 29 | 30 | # Smooth histogram with a gaussian kernel for visualization purposes 31 | if smooth: 32 | h = gaussian_filter(h, **smooth) 33 | 34 | # pad the histogram, so that we can extend the image to fit the whole space, and set the x and 35 | # y axes point to the midpoints in the plot (in this case we need to take into account the log 36 | # scaling of the axis) 37 | lex = np.log(e1) 38 | ley = np.log(e2) 39 | dx = lex[1] - lex[0] 40 | dy = ley[1] - ley[0] 41 | midx = np.exp(np.arange(lex[0] - dx / 2, lex[-1] + dx, dx)) 42 | midy = np.exp(np.arange(ley[0] - dy / 2, ley[-1] + dy, dy)) 43 | 44 | h = np.pad(h, 1, 'edge') 45 | 46 | f = plt.figure(figsize=(5, 5)) 47 | # p = plt.pcolormesh(xx, yy, h, **kwargs) 48 | grid_x, grid_y = np.meshgrid(midx, midy) 49 | q = h.copy() 50 | 51 | # set the maximum histogram value to the cutoff, or we will have blank areas in the contour plots 52 | h[h > vmax] = vmax 53 | 54 | levels = np.logspace(np.log10(vmin), np.log10(vmax), nlevels, base=10) 55 | p = plt.contourf(grid_x, grid_y, h, norm=LogNorm(vmin, vmax), cmap='Reds', levels=levels, **kwargs) 56 | if sigma is not None: 57 | plt.axvline(x=sigma, ls='--', c='#dddddd') 58 | 59 | plt.plot([bottom_left, top_right], [bottom_left, top_right], 'k--') 60 | plt.xscale('log') 61 | plt.yscale('log') 62 | plt.xlim(xx[0], xx[-1]) 63 | plt.ylim(yy[0], yy[-1]) 64 | plt.xlabel(xlabel) 65 | plt.ylabel(ylabel) 66 | plt.tight_layout() 67 | if outfile is not None: 68 | plt.savefig(outfile) 69 | return f, p, q, e1, e2 70 | 71 | 72 | def density_histogram_2d(d1, d2, bins=(100, 100), ranges=((1e-3, 1), (1e-3, 1)), 73 | outfile=None, vmin=1e1, vmax=1e5, nlevels=5, 74 | xlabel='in', ylabel='out', smooth=None, **kwargs): 75 | 76 | if ranges is None: 77 | ranges = (None, None) 78 | 79 | if ranges[0] is None: 80 | x0, x1 = np.min(d1), np.max(d1) 81 | else: 82 | x0, x1 = ranges[0] 83 | 84 | if ranges[1] is None: 85 | y0, y1 = np.min(d2), np.max(d2) 86 | else: 87 | y0, y1 = ranges[1] 88 | 89 | xx = np.linspace(x0, x1, bins[0]) 90 | yy = np.linspace(y0, y1, bins[1]) 91 | 92 | bottom_left = max(xx[0], yy[0]) 93 | top_right = min(xx[-1], yy[-1]) 94 | 95 | # Note: we invert axes, because in a plot, the x axis actually correspond to 96 | # columns, while the y to rows. np.histogram2d puts the value from the first dataset on 97 | # rows, and the ones from the second in columns. 98 | h, e1, e2 = np.histogram2d(d2, d1, bins=(yy, xx)) 99 | 100 | # Smooth histogram with a gaussian kernel for visualization purposes 101 | if smooth: 102 | h = gaussian_filter(h, **smooth) 103 | 104 | # pad the histogram, so that we can extend the image to fit the whole space, and set the x and 105 | # y axes point to the midpoints in the plot 106 | lex = e1 107 | ley = e2 108 | dx = lex[1] - lex[0] 109 | dy = ley[1] - ley[0] 110 | midx = np.arange(lex[0] - dx / 2, lex[-1] + dx, dx) 111 | midy = np.arange(ley[0] - dy / 2, ley[-1] + dy, dy) 112 | 113 | h = np.pad(h, 1, 'edge') 114 | 115 | f = plt.figure(figsize=(5, 5)) 116 | # p = plt.pcolormesh(xx, yy, h, **kwargs) 117 | grid_x, grid_y = np.meshgrid(midx, midy) 118 | q = h.copy() 119 | 120 | h[h > vmax] = vmax 121 | levels = np.logspace(np.log10(vmin), np.log10(vmax), nlevels, base=10) 122 | p = plt.contourf(grid_x, grid_y, h, norm=LogNorm(vmin, vmax), 123 | cmap='Blues', levels=levels, **kwargs) 124 | plt.plot([bottom_left, top_right], [bottom_left, top_right], 'k--') 125 | plt.xlim(xx[0], xx[-1]) 126 | plt.ylim(yy[0], yy[-1]) 127 | plt.xlabel(xlabel) 128 | plt.ylabel(ylabel) 129 | plt.tight_layout() 130 | if outfile is not None: 131 | plt.savefig(outfile) 132 | return f, p, q, e1, e2 133 | -------------------------------------------------------------------------------- /igm/report/radials.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | from matplotlib.cm import get_cmap 4 | from matplotlib.colors import Colormap 5 | from matplotlib.patches import Circle 6 | import logging 7 | import traceback 8 | import os.path 9 | from alabtools import HssFile 10 | from alabtools.plots import plot_by_chromosome 11 | 12 | from .utils import create_folder, average_copies 13 | 14 | 15 | def get_radial_level(crd, index, semiaxes): 16 | ''' 17 | Use coordinates in bead-major format to extract average radial levels. 18 | Levels are defined for a point (x, y, z) as the square root of 19 | (x/a)^2 + (y/b)^2 + (z/c)^2 20 | where a, b, c are the three semiaxes. 21 | That generalizes to the fraction of the nucleus radius for spheres 22 | 23 | Parameters 24 | ---------- 25 | crd: np.ndarray 26 | coordinates in bead-major format (n_beads x n_structures x 3) 27 | index: alabtools.Index 28 | index of genomic locations 29 | semiaxes: np.ndarray 30 | the 3 semiaxes of the envelope 31 | ''' 32 | 33 | semiaxes = np.array(semiaxes) 34 | 35 | radials = np.array([ 36 | np.sqrt(np.sum(np.square(crd[i] / semiaxes), axis=1)).mean() for i in range(len(index)) 37 | ]) 38 | 39 | return average_copies(radials, index) 40 | 41 | 42 | def radial_plot_p(edges, val, cmap='Greys', **kwargs): 43 | ''' 44 | Plots radial densities on a sphere, colorcoded 45 | ''' 46 | fig = plt.figure() 47 | ax = fig.gca() 48 | vmax = kwargs.get('vmax', max(val)) 49 | vmin = kwargs.get('vmin', min(val)) 50 | maxe = edges[-1] 51 | plt.axis('equal') 52 | plt.xlim(-maxe, maxe) 53 | plt.ylim(-maxe, maxe) 54 | 55 | if not isinstance(cmap, Colormap): 56 | cmap = get_cmap(cmap) 57 | 58 | def get_color(v): 59 | rng = vmax - vmin 60 | d = np.clip((v - vmin) / rng, 0, 0.999) 61 | idx = int(d * cmap.N) 62 | return cmap.colors[idx] 63 | 64 | for i in reversed(range(len(val))): 65 | c = Circle((0, 0), edges[i + 1], facecolor=get_color(val[i])) 66 | ax.add_patch(c) 67 | 68 | 69 | def plot_radial_density(hssfname, semiaxes, n=11, vmax=1.1, run_label=''): 70 | 71 | with HssFile(hssfname, 'r') as hss: 72 | 73 | crd = hss.coordinates.reshape((hss.nstruct * hss.nbead, 3)) 74 | radials = np.sqrt(np.sum(np.square(crd / semiaxes), axis=1)) 75 | 76 | counts, edges = np.histogram(radials, bins=n, range=(0, vmax)) 77 | volumes = np.array([edges[i + 1]**3 - edges[i]**3 for i in range(n)]) 78 | fig = plt.figure() 79 | plt.title(f'Radial density distribution {run_label}') 80 | plt.bar(np.arange(n) + 0.5, height=counts / volumes, width=1) 81 | plt.xticks(range(n + 1), ['{:.2f}'.format(x) for x in edges], rotation=60) 82 | plt.tight_layout() 83 | fig.savefig(f'radials/density_histo{run_label}.pdf') 84 | fig.savefig(f'radials/density_histo{run_label}.png') 85 | plt.close(fig) 86 | 87 | np.savetxt(f'radials/density_histo{run_label}.txt', counts / volumes) 88 | 89 | 90 | def report_radials(hssfname, semiaxes=None, run_label=''): 91 | if run_label: 92 | run_label = '-' + run_label 93 | logger = logging.getLogger('Radials') 94 | logger.info('Executing Radials report...') 95 | try: 96 | create_folder("radials") 97 | with HssFile(hssfname, 'r') as hss: 98 | index = hss.index 99 | if semiaxes is None: 100 | # see if we have information about semiaxes in the file 101 | try: 102 | semiaxes = hss['envelope']['params'][()] 103 | if len(semiaxes.shape) == 0: # is scalar 104 | semiaxes = np.array([semiaxes, semiaxes, semiaxes]) 105 | except KeyError: 106 | semiaxes = np.array([5000., 5000., 5000.]) 107 | radials = get_radial_level(hss.coordinates, index, semiaxes) 108 | np.savetxt(f'radials/radials{run_label}.txt', radials) 109 | fig, _ = plot_by_chromosome(radials, index.get_haploid(), vmin=.4, vmax=1.0, 110 | suptitle=f'Radial position per bead {run_label}') 111 | 112 | fig.savefig(f'radials/radials{run_label}.pdf') 113 | fig.savefig(f'radials/radials{run_label}.png') 114 | 115 | plt.close(fig) 116 | 117 | plot_radial_density(hssfname, semiaxes, run_label=run_label) 118 | 119 | if os.path.isfile(f'shells/ave_radial{run_label}.txt'): 120 | logger.info('Note: normalizing with respect to last shell') 121 | n = np.loadtxt(f'shells/ave_radial{run_label}.txt')[-1] 122 | np.savetxt(f'radials/radials_norm{run_label}.txt', radials / n) 123 | fig, _ = plot_by_chromosome(radials / n, index.get_haploid(), vmin=.4, vmax=1.0) 124 | fig.savefig(f'radials/radials_norm{run_label}.pdf') 125 | fig.savefig(f'radials/radials_norm{run_label}.png') 126 | plt.close(fig) 127 | logger.info('Done.') 128 | 129 | except KeyboardInterrupt: 130 | logger.error('User interrupt. Exiting.') 131 | exit(1) 132 | 133 | except Exception: 134 | traceback.print_exc() 135 | logger.error('Error in radials step\n==============================') 136 | -------------------------------------------------------------------------------- /igm/report/radius_of_gyration.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import logging 4 | import traceback 5 | from alabtools import HssFile 6 | from .utils import create_folder 7 | 8 | 9 | def rg(points): 10 | ''' 11 | Get the radius of gyration for a set of points 12 | :param points: np.ndarray 13 | set of points (n_points x 3) 14 | :return: float 15 | Rg(points) 16 | ''' 17 | ave = np.average(points, axis=0) 18 | v = points - ave 19 | return np.sqrt(np.sum(np.square(v)) / len(v)) 20 | 21 | 22 | def get_chroms_rgs(crds, index): 23 | ''' 24 | Get the average radius of gyration for each chromosome chain in the 25 | index. Returns a list of size n_struct*n_copies for each chromosome. 26 | :param crds: np.ndarray 27 | coordinates in bead-major format 28 | :param index: alabtools.Index 29 | index of genomic locations 30 | :return: list[np.ndarray] 31 | ''' 32 | rgs = [] 33 | for chrom in index.get_chromosomes(): 34 | copies = index.get_chrom_copies(chrom) 35 | data = list() 36 | for copy in copies: 37 | ii = index.get_chrom_pos(chrom, copy) 38 | for crd in crds[ii, :].swapaxes(0, 1): 39 | data.append(rg(crd)) 40 | rgs.append(np.array(data)) 41 | return rgs 42 | 43 | 44 | def set_box_colors(bp, clr): 45 | if not isinstance(clr, (list, tuple, np.ndarray)): 46 | clr = [clr] * len(bp['boxes']) 47 | for i in range(len(clr)): 48 | plt.setp(bp['boxes'][i], facecolor=clr[i]) 49 | plt.setp(bp['medians'][i], color='black') 50 | 51 | 52 | def boxplots_group(data, group_labels, n_per_row=6, subplot_width=10, 53 | subplot_height=2.5, vmin=800, vmax=3500, outfile=None, 54 | title='', color='#fb7b04'): 55 | ''' 56 | Splits a large number of boxex across multiple rows 57 | ''' 58 | n_groups = len(group_labels) 59 | n_rows = n_groups // n_per_row if n_groups % n_per_row == 0 else n_groups // n_per_row + 1 60 | f, plots = plt.subplots(n_rows, 1, figsize=(subplot_width, subplot_height * n_rows), sharey=True) 61 | f.suptitle(title) 62 | for ip, i in enumerate(range(0, n_groups, n_per_row)): 63 | 64 | # select data subset 65 | boxdata = data[i:i + n_per_row] 66 | plots[ip].set_ylim(vmin, vmax) 67 | bp = plots[ip].boxplot(boxdata, labels=group_labels[i:i + n_per_row], patch_artist=True, showfliers=False) 68 | set_box_colors(bp, color) 69 | 70 | plt.tight_layout(rect=[0, 0.03, 1, 0.95]) 71 | if outfile is not None: 72 | plt.savefig(outfile) 73 | return f, plots 74 | 75 | 76 | def report_radius_of_gyration(hssfname, run_label=''): 77 | 78 | if run_label: 79 | run_label = '-' + run_label 80 | 81 | logger = logging.getLogger('GyrRadius') 82 | try: 83 | 84 | logger.info('Executing Radius of Gyration report...') 85 | create_folder("radius_of_gyration") 86 | with HssFile(hssfname, 'r') as hss: 87 | chroms = hss.genome.chroms 88 | rgs = get_chroms_rgs(hss.coordinates, hss.index) 89 | np.savez(f'radius_of_gyration/chromosomes{run_label}.npz', 90 | **{c: arr for c, arr in zip(hss.genome.chroms, rgs)}) 91 | fig, _ = boxplots_group(rgs, chroms, 92 | title=f'Chromosomes Radii of Gyration {run_label}', 93 | outfile=f'radius_of_gyration/rgs{run_label}.pdf') 94 | fig.savefig(f'radius_of_gyration/rgs{run_label}.png') 95 | plt.close(fig) 96 | logger.info('Done.') 97 | 98 | except KeyboardInterrupt: 99 | logger.error('User interrupt. Exiting.') 100 | exit(1) 101 | 102 | except Exception: 103 | traceback.print_exc() 104 | logger.error('Error in radius of gyration step\n==============================') 105 | 106 | -------------------------------------------------------------------------------- /igm/report/shells.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import matplotlib.pyplot as plt 3 | import logging 4 | import traceback 5 | from alabtools import HssFile 6 | from .utils import create_folder 7 | 8 | 9 | def report_shells(hssfname, semiaxes=None, nshell=5, hvmax=1.5, hnbins=150, run_label=''): 10 | if run_label: 11 | run_label = '-' + run_label 12 | logger = logging.getLogger('Shells') 13 | 14 | try: 15 | logger.info('Executiong Shells report') 16 | create_folder("shells") 17 | with HssFile(hssfname, 'r') as hss: 18 | crd = np.swapaxes(hss.coordinates, 0, 1) 19 | if semiaxes is None: 20 | # see if we have information about semiaxes in the file 21 | try: 22 | semiaxes = hss['envelope']['params'][()] 23 | if len(semiaxes.shape) == 0: # is scalar 24 | semiaxes = np.array([semiaxes, semiaxes, semiaxes]) 25 | except KeyError: 26 | semiaxes = np.array([5000., 5000., 5000.]) 27 | 28 | n = hss.nbead 29 | kth = [int(k * n / nshell) for k in range(0, nshell)] 30 | bds = kth + [n] 31 | ave_shell_rad = np.empty((hss.nstruct, nshell)) 32 | pos_histos = np.zeros((nshell, hnbins)) 33 | edges = None 34 | for i in range(hss.nstruct): 35 | radials = np.sqrt(np.sum(np.square(crd[i] / semiaxes), axis=1)) 36 | radials = np.partition(radials, kth) 37 | for j in range(nshell): 38 | h, edges = np.histogram(radials[bds[j]:bds[j + 1]], bins=hnbins, 39 | range=(0, hvmax)) 40 | pos_histos[j] += h 41 | ave_shell_rad[i][j] = np.average(radials[bds[j]:bds[j + 1]]) 42 | 43 | np.savetxt(f'shells/ave_radial{run_label}.txt', np.average(ave_shell_rad, axis=0)) 44 | midpoints = (edges[:-1] + edges[1:]) / 2 45 | fig = plt.figure() 46 | plt.title('Radial position distributions per shell') 47 | for j in range(nshell): 48 | plt.bar(midpoints, height=pos_histos[j], alpha=.6, width=hvmax/hnbins, label='shell {:d}'.format(j+1)) 49 | plt.legend() 50 | fig.savefig(f'shells/positions_histograms_by_shell{run_label}.pdf') 51 | fig.savefig(f'shells/positions_histograms_by_shell{run_label}.png') 52 | plt.close(fig) 53 | logger.info('Done.') 54 | 55 | except KeyboardInterrupt: 56 | logger.error('User interrupt. Exiting.') 57 | exit(1) 58 | 59 | except Exception: 60 | traceback.print_exc() 61 | logger.error('Error in shells step\n==============================') 62 | -------------------------------------------------------------------------------- /igm/report/utils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os.path 3 | 4 | 5 | def aggregate_copies(v, index, fun=np.nanmean, *args, **kwargs): 6 | ''' 7 | Aggregate genomic regions if more copies are present 8 | :param v: np.ndarray 9 | the values for all the regions/beads 10 | :param index: alabtools.Index 11 | the index 12 | :param fun: callable 13 | the function to apply (default: np.nanmean). It should take one vector 14 | and return a scalar 15 | :return: np.ndarray 16 | ''' 17 | v = np.array(v) 18 | assert len(v) == len(index) 19 | ci = index.copy_index 20 | x = np.zeros(len(ci)) 21 | for i in range(len(ci)): 22 | x[i] = fun(v[ci[i]], *args, **kwargs) 23 | return x 24 | 25 | 26 | def average_copies(v, index): 27 | return aggregate_copies(v, index) 28 | 29 | 30 | def sum_copies(v, index): 31 | return aggregate_copies(v, index, np.sum) 32 | 33 | 34 | def create_folder(folder): 35 | if not os.path.isdir(folder): 36 | os.makedirs(folder) 37 | 38 | 39 | def snormsq_ellipse(x, semiaxes, r=0): 40 | ''' 41 | Returns the level of a point inside an ellipsoid 42 | (0 -> center, ]0, 1[ -> inside, 1 -> surface, >1 outside). 43 | :param x: np.ndarray 44 | the coordinates of the point 45 | :param semiaxes: np.ndarray 46 | the 3 semiaxes of the ellipsoid 47 | :param r: float 48 | if specified, the point is considered a sphere and level=1 means 49 | the surface is touching the ellipsoid. 50 | :return: 51 | ''' 52 | a, b, c = np.array(semiaxes) - r 53 | sq = np.square(x) 54 | return sq[:, 0] / (a**2) + sq[:, 1] / (b**2) + sq[:, 2] / (c**2) 55 | -------------------------------------------------------------------------------- /igm/report/violations.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from matplotlib.patches import Rectangle 3 | from matplotlib.ticker import PercentFormatter 4 | import matplotlib.pyplot as plt 5 | import logging 6 | import traceback 7 | import json 8 | from alabtools import HssFile 9 | 10 | from .utils import create_folder 11 | 12 | 13 | def plot_violation_histogram(h, edges, tol=0.05, nticks=20, title='', figsize=(10, 4), outfile=None, log=False): 14 | fig = plt.figure(figsize=figsize) 15 | step = edges[1] - edges[0] 16 | tick_step = int(len(edges) / nticks) 17 | 18 | if log: 19 | z = h.copy() 20 | for i in range(len(h)): 21 | if h[i] > 0: 22 | z[i] = np.log(h[i]) 23 | h = z 24 | else: 25 | # transform to percentage 26 | totsum = np.sum(h) 27 | h = h / totsum 28 | 29 | xx = np.arange(len(h) - 1) + 0.5 30 | xx = np.concatenate([xx, [len(h) + tick_step + 0.5]]) 31 | 32 | tick_pos = list(range(len(edges))[::tick_step]) 33 | tick_labels = ['{:.2f}'.format(edges[i]) for i in tick_pos] 34 | tick_pos.append(len(h) + tick_step + 0.5) 35 | tick_labels.append('>{:.2f}'.format(edges[-2])) 36 | 37 | # ignore the first bin to determine height 38 | # vmax = np.max(h[1:]) * 1.1 39 | vmax = max(np.max(h) * 1.05, 1) 40 | plt.title(title) 41 | 42 | plt.xlabel('Relative restraint violation') 43 | if log: 44 | plt.ylabel('Restraints count (Log)') 45 | else: 46 | plt.ylabel('Percentage of restraints') 47 | plt.gca().yaxis.set_major_formatter(PercentFormatter(1.0)) 48 | 49 | plt.axvline(x=tol / step, ls='--', c='green') 50 | plt.gca().add_patch(Rectangle((tol / step, 0), width=tick_pos[-1] - tol / step + tick_step, 51 | height=vmax, fill=True, facecolor='darkred', alpha=.3)) 52 | plt.ylim(0, vmax) 53 | 54 | plt.bar(xx, height=h, width=1, color='grey') 55 | plt.xticks(tick_pos, tick_labels, rotation=60) 56 | plt.xlim(0, tick_pos[-1] + tick_step) 57 | plt.tight_layout() 58 | if outfile is not None: 59 | plt.savefig(outfile) 60 | return fig 61 | 62 | 63 | def report_violations(hssfname, violation_tolerance, run_label=''): 64 | logger = logging.getLogger('Violations') 65 | logger.info('Executing violation report...') 66 | if run_label: 67 | run_label = '-' + run_label 68 | try: 69 | 70 | create_folder('violations') 71 | 72 | with HssFile(hssfname, 'r') as hss: 73 | stats = json.loads(hss['summary'][()]) 74 | 75 | # save a copy of the data 76 | with open(f'violations/stats{run_label}.json', 'w') as f: 77 | json.dump(stats, f, indent=4) 78 | 79 | with open(f'violations/restraints_summary{run_label}.txt', 'w') as f: 80 | f.write('# type imposed violated\n') 81 | f.write('"all" {} {}\n'.format( 82 | stats['n_imposed'], 83 | stats['n_violations'], 84 | )) 85 | for k, ss in stats['byrestraint'].items(): 86 | f.write('"{}" {} {}\n'.format( 87 | k, 88 | ss['n_imposed'], 89 | ss['n_violations'], 90 | )) 91 | 92 | create_folder('violations/histograms') 93 | 94 | h = stats['histogram']['counts'] 95 | edges = stats['histogram']['edges'] 96 | fig = plot_violation_histogram(h, edges, violation_tolerance, nticks=10, 97 | title="Histogram of all Violations", 98 | outfile=f"violations/histograms/summary{run_label}.pdf") 99 | fig.savefig(f"violations/histograms/summary{run_label}.png") 100 | plt.close(fig) 101 | 102 | fig = plot_violation_histogram(h, edges, violation_tolerance, nticks=10, log=True, 103 | title="Histogram of all Violations (Log)", 104 | outfile=f"violations/histograms/summary_log-{run_label}.pdf") 105 | fig.savefig(f"violations/histograms/summary_log{run_label}.png") 106 | plt.close(fig) 107 | 108 | for k, v in stats['byrestraint'].items(): 109 | h = v['histogram']['counts'] 110 | fig = plot_violation_histogram(h, edges, violation_tolerance, nticks=10, 111 | title='Histogram of Violations for ' + k, 112 | outfile=f"violations/histograms/{k}{run_label}.pdf") 113 | fig.savefig(f"violations/histograms/{k}{run_label}.png") 114 | plt.close(fig) 115 | 116 | fig = plot_violation_histogram(h, edges, violation_tolerance, nticks=10, log=True, 117 | title='Histogram of Violations (Log) for ' + k, 118 | outfile=f"violations/histograms/{k}_log{run_label}.pdf") 119 | fig.savefig(f"violations/histograms/{k}_log{run_label}.png") 120 | plt.close(fig) 121 | 122 | # TODO: energies and stuff 123 | logger.info('Done.') 124 | 125 | except KeyboardInterrupt: 126 | logger.error('User interrupt. Exiting.') 127 | exit(1) 128 | 129 | except Exception: 130 | traceback.print_exc() 131 | logger.error('Error trying to compute violation statistics\n==============================') 132 | -------------------------------------------------------------------------------- /igm/restraints/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | from .envelope import Envelope 4 | from .polymer import Polymer 5 | from .steric import Steric 6 | from .hic import HiC 7 | from .sprite import Sprite 8 | from .damid import Damid 9 | from .nucleolus import Nucleolus 10 | from .genenvelope import GenEnvelope 11 | from .fish import Fish 12 | from .inter_hic import interHiC 13 | from .intra_hic import intraHiC 14 | -------------------------------------------------------------------------------- /igm/restraints/damid.py: -------------------------------------------------------------------------------- 1 | ## --------------------------- 2 | # Allocate DamID restraints (using the "DamIDActdist.hdf5" file from the Assignment Step) as "EllipticEnvelope" forces between particles to the model object in IGM 3 | ## --------------------------- 4 | 5 | 6 | 7 | from __future__ import division, print_function 8 | 9 | import numpy as np 10 | import h5py 11 | 12 | from .restraint import Restraint 13 | from ..model.forces import HarmonicLowerBound, EllipticEnvelope # restraint forces associated with damid 14 | from ..model import Particle 15 | 16 | try: 17 | UNICODE_EXISTS = bool(type(unicode)) 18 | except NameError: 19 | unicode = lambda s: str(s) 20 | 21 | 22 | # ---- AUXILIARY FUNCTIONS FOR DAMID CALCULATIONS ----# 23 | 24 | def snormsq_sphere(x, R, r): 25 | 26 | """ 27 | Compute radial distance of a bead to spherical nuclear envelope 28 | 29 | INPUT 30 | x (float), bead/locus coordinates 31 | R (float), radius of nuclear envelope, when spherical 32 | r (float), radius of bead 33 | 34 | OUTPUT (normalized) distance between bead surface and nuclear envelope 35 | d = 1 if bead surface touches the envelope 36 | d < 1 otherwise 37 | """ 38 | 39 | # return np.square( 40 | # (np.linalg.norm(x, axis=1) + r) / R**2 41 | # ) 42 | return np.sum(np.square(x), axis=1) / (R-r)**2 43 | 44 | 45 | 46 | 47 | 48 | def snormsq_ellipsoid(x, semiaxes, r): 49 | 50 | """ 51 | Compute radial distance of a bead to ellipsoidal nuclear envelope 52 | 53 | INPUT 54 | x (float), bead/locus coordinates 55 | r (float), radius of bead 56 | semiaxes (float, float, float), semiaxes of nuclear envelope, if ellipsoidal 57 | 58 | OUTPUT (normalized) distance between bead surface and nuclear envelope: x**2/(a-r)**2 + y**2/(b-r)**2 + z**2/(c-r)**2 59 | d = 1 if bead surface touches the envelope (this means the bead center is laying on a concentric ellipsoid 60 | of semiaxes (a - r, b - r, c - r)) 61 | d < 1 otherwise (the bead center is laying on a concentric ellipsoid with even shorter semiaxes) 62 | """ 63 | 64 | a, b, c = np.array(semiaxes) - r 65 | sq = np.square(x) 66 | return sq[0]/(a**2) + sq[1]/(b**2) + sq[2]/(c**2) 67 | 68 | 69 | def snorm(x, shape=u"sphere", **kwargs): 70 | if shape == u"sphere": 71 | return np.linalg.norm(x) / kwargs['a'] 72 | elif shape == u"ellipsoid": 73 | a, b, c = kwargs['a'], kwargs['b'], kwargs['c'] 74 | return np.sqrt( (x[0]/a)**2 + (x[1]/b)**2 + (x[2]/c)**2 ) 75 | 76 | 77 | class Damid(Restraint): 78 | """ 79 | Object handles Hi-C restraint 80 | 81 | Parameters 82 | ---------- 83 | damid_file : activation distance file for damid 84 | 85 | contact_range : int 86 | defining contact range between 2 particles as contactRange*(r1+r2) 87 | 88 | 89 | """ 90 | 91 | def __init__(self, damid_file, contact_range=0.05, nuclear_radius=5000.0, shape="sphere", 92 | semiaxes=(5000, 5000, 5000), k=1.0): 93 | 94 | self.shape = unicode(shape) 95 | 96 | if self.shape == u"sphere": 97 | self.a = self.b = self.c = nuclear_radius 98 | elif self.shape == u"ellipsoid": 99 | self.a, self.b, self.c = semiaxes 100 | 101 | # recapitulate parameters 102 | self.contact_range = contact_range 103 | self.nuclear_radius = nuclear_radius 104 | self.k = k 105 | self.forceID = [] 106 | self._load_actdist(damid_file) 107 | #- 108 | 109 | 110 | def _load_actdist(self,damid_actdist): 111 | 112 | """ Read in file containing current DAMID activation distances """ 113 | self.damid_actdist = DamidActivationDistanceDB(damid_actdist) 114 | 115 | 116 | def _apply_envelope(self, model): 117 | 118 | """ Effectively apply damid restraints to the different beads, if distance is smaller than activation distance """ 119 | 120 | center = model.addParticle([0., 0., 0.], 0., Particle.DUMMY_STATIC) 121 | cutoff = 1 - self.contact_range 122 | 123 | affected_particles = [ 124 | i for i, d in self.damid_actdist 125 | if snormsq_ellipsoid( 126 | model.particles[i].pos, 127 | np.array([self.a, self.b, self.c]) * cutoff, 128 | model.particles[i].r 129 | ) >= d**2 130 | ] 131 | 132 | f = model.addForce( 133 | EllipticEnvelope( 134 | affected_particles, 135 | center, 136 | np.array([self.a, self.b, self.c])*cutoff, 137 | -self.k, 138 | scale=cutoff*np.mean([self.a, self.b, self.c]) 139 | ) 140 | ) 141 | 142 | self.forceID.append(f) 143 | 144 | 145 | def _apply(self, model): 146 | return self._apply_envelope(model) 147 | 148 | #== 149 | 150 | class DamidActivationDistanceDB(object): 151 | """ 152 | HDF5 activation distance iterator: read in damid activation distance file in chunks 153 | """ 154 | 155 | def __init__(self, damid_file, chunk_size = 10000): 156 | self.h5f = h5py.File(damid_file,'r') 157 | self._n = len(self.h5f['loc']) 158 | 159 | self.chunk_size = min(chunk_size, self._n) 160 | 161 | self._i = 0 162 | self._chk_i = chunk_size 163 | self._chk_end = 0 164 | 165 | def __len__(self): 166 | return self._n 167 | 168 | def __iter__(self): 169 | return self 170 | 171 | def __next__(self): 172 | if self._i < self._n: 173 | self._i += 1 174 | self._chk_i += 1 175 | 176 | if self._chk_i >= self.chunk_size: 177 | self._load_next_chunk() 178 | 179 | return (int(self._chk_row[self._chk_i]), 180 | self._chk_data[self._chk_i]) 181 | else: 182 | self._i = 0 183 | self._chk_i = self.chunk_size 184 | self._chk_end = 0 185 | raise StopIteration() 186 | 187 | def next(self): 188 | return self.__next__() 189 | 190 | def _load_next_chunk(self): 191 | self._chk_row = self.h5f['loc'][self._chk_end : self._chk_end + self.chunk_size] 192 | self._chk_data = self.h5f['dist'][self._chk_end : self._chk_end + self.chunk_size] 193 | self._chk_end += self.chunk_size 194 | self._chk_i = 0 195 | 196 | def __del__(self): 197 | try: 198 | self.h5f.close() 199 | except: 200 | pass 201 | 202 | -------------------------------------------------------------------------------- /igm/restraints/envelope.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | import numpy as np 4 | 5 | from .restraint import Restraint 6 | from ..model.particle import Particle 7 | from ..model.forces import HarmonicUpperBound, EllipticEnvelope 8 | try: 9 | UNICODE_EXISTS = bool(type(unicode)) 10 | except NameError: 11 | unicode = lambda s: str(s) 12 | 13 | class Envelope(Restraint): 14 | """ 15 | A object handles nuclear envelope restraints 16 | 17 | Parameters 18 | ---------- 19 | nucRadius : float or list 20 | nuclear radius or the length of the three semiaxes, in nm 21 | k : float 22 | spring constant 23 | """ 24 | def __init__(self, shape="sphere", nuclear_radius=5000.0, k=1.0): 25 | self.shape = unicode(shape) 26 | 27 | if self.shape == u"sphere": 28 | self.a = self.b = self.c = nuclear_radius 29 | elif self.shape == u"ellipsoid": 30 | self.a, self.b, self.c = nuclear_radius 31 | 32 | self.k = k 33 | self.forceID = [] 34 | 35 | 36 | def _apply_sphere_envelop(self, model): 37 | 38 | """ add a dummy dimensionless particle in the geometric center, to be used in force modeling """ 39 | center = model.addParticle([0., 0., 0.], 0., Particle.DUMMY_STATIC) 40 | 41 | normal_particles = [ 42 | i for i, p in enumerate(model.particles) 43 | if p.ptype == Particle.NORMAL 44 | ] 45 | f = model.addForce( 46 | EllipticEnvelope( 47 | normal_particles, 48 | center, 49 | (self.a, self.b, self.c), 50 | self.k, 51 | scale=0.1*np.mean([self.a, self.b, self.c]) 52 | # set arbitrary scale: 100% violation ratio if extend inside by 53 | # 1/10 of the nucleus radius. With usual parameters, it means 54 | # that will be noticed as violated when the bond is stretched 55 | # by 25nm. 56 | ) 57 | ) 58 | self.forceID.append(f) 59 | 60 | 61 | def _apply(self, model): 62 | self._apply_sphere_envelop(model) 63 | 64 | 65 | def __repr__(self): 66 | return 'Envelope[shape={},k={},a={},b={},c={}]'.format(self.shape, self.k, self.a, self.b, self.c) 67 | #= 68 | -------------------------------------------------------------------------------- /igm/restraints/genenvelope.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | import numpy as np 4 | 5 | from .restraint import Restraint 6 | from ..model.particle import Particle 7 | from ..model.forces import ExpEnvelope 8 | 9 | try: 10 | UNICODE_EXISTS = bool(type(unicode)) 11 | except NameError: 12 | unicode = lambda s: str(s) 13 | 14 | class GenEnvelope(Restraint): 15 | """ 16 | This class handles a (non ideal) nuclear envelope restraint from a density map 17 | 18 | Parameters 19 | ---------- 20 | volume_file: string 21 | name of file containing the binary density map we would like to fit the genome into 22 | k : float 23 | spring constant 24 | """ 25 | 26 | def __init__(self, shape, volume_file="", k=1.0): 27 | 28 | self.shape = unicode(shape) 29 | self.volume_file = volume_file # file containing the binary density map information 30 | self.k = k # elastic restraining constant (>0 attractive, <0 repulsive) 31 | 32 | self.forceID = [] 33 | 34 | 35 | def _apply_sphere_envelop(self, model): 36 | 37 | normal_particles = [ 38 | i for i, p in enumerate(model.particles) 39 | if p.ptype == Particle.NORMAL 40 | ] 41 | 42 | f = model.addForce( 43 | ExpEnvelope( 44 | normal_particles, 45 | self.volume_file, 46 | self.k, 47 | scale= 0.5 48 | # set arbitrary scale: 100% violation ratio if extend inside by 49 | # 1/10 of the nucleus radius. With usual parameters, it means 50 | # that will be noticed as violated when the bond is stretched 51 | # by 25nm. 52 | ) 53 | ) 54 | self.forceID.append(f) 55 | 56 | 57 | def _apply(self, model): 58 | self._apply_sphere_envelop(model) 59 | 60 | 61 | def __repr__(self): 62 | return 'ExpEnvelope[shape={},k={}]'.format(self.shape, self.k) 63 | #= 64 | -------------------------------------------------------------------------------- /igm/restraints/hic.py: -------------------------------------------------------------------------------- 1 | ## --------------------------- 2 | # Allocate Hi-C restraints (using the "Actdist.hdf5" file from the Assignment Step) as forces between particles to the model object in IGM 3 | ## --------------------------- 4 | 5 | from __future__ import division, print_function 6 | 7 | import numpy as np 8 | 9 | from .restraint import Restraint 10 | from ..model.forces import HarmonicUpperBound 11 | 12 | import h5py 13 | 14 | class HiC(Restraint): 15 | 16 | """ 17 | Object handles Hi-C restraint 18 | 19 | Parameters 20 | ---------- 21 | actdist_file : activation distance file (generated from ActivationDistanceStep.py) 22 | 23 | contactRange : int 24 | defining contact range between 2 particles as contactRange*(r1+r2) 25 | 26 | k: float 27 | elastic constant for harmonic restraint 28 | """ 29 | 30 | def __init__(self, actdist_file, contactRange=2, k=1.0): 31 | 32 | self.contactRange = contactRange 33 | self.k = k 34 | self.forceID = [] 35 | self._load_actdist(actdist_file) 36 | #- 37 | 38 | def _load_actdist(self,actdist): 39 | self.actdist = ActivationDistanceDB(actdist) 40 | 41 | def _apply(self, model): 42 | 43 | """ Apply harmonic restraint to those distances smaller than d_{act}""" 44 | 45 | for (i, j, d) in self.actdist: 46 | 47 | # calculate particle distances for i, j 48 | # if ||i-j|| <= d then assign a bond for i, j 49 | if model.particles[i] - model.particles[j] <= d : 50 | 51 | # harmonic mean distance 52 | dij = self.contactRange*(model.particles[i].r + 53 | model.particles[j].r) 54 | 55 | # add harmonic bond between i-th and j-th beads 56 | f = model.addForce(HarmonicUpperBound((i, j), dij, self.k, 57 | note=Restraint.HIC)) 58 | self.forceID.append(f) 59 | #- 60 | #- 61 | #= 62 | 63 | 64 | #== 65 | 66 | class ActivationDistanceDB(object): 67 | 68 | """ HDF5 activation distance iterator: read in file of activation distances, in chunks """ 69 | 70 | def __init__(self, actdist_file, chunk_size = 10000): 71 | self.h5f = h5py.File(actdist_file,'r') 72 | self._n = len(self.h5f['row']) 73 | 74 | self.chunk_size = min(chunk_size, self._n) 75 | 76 | self._i = 0 77 | self._chk_i = chunk_size 78 | self._chk_end = 0 79 | 80 | def __len__(self): 81 | return self._n 82 | 83 | def __iter__(self): 84 | return self 85 | 86 | def __next__(self): 87 | if self._i < self._n: 88 | self._i += 1 89 | self._chk_i += 1 90 | 91 | if self._chk_i >= self.chunk_size: 92 | self._load_next_chunk() 93 | 94 | return (self._chk_row[self._chk_i], 95 | self._chk_col[self._chk_i], 96 | self._chk_data[self._chk_i]) 97 | else: 98 | self._i = 0 99 | self._chk_i = self.chunk_size 100 | self._chk_end = 0 101 | raise StopIteration() 102 | def next(self): 103 | return self.__next__() 104 | 105 | # probabilities are not needed now, since they are already encoded in the list of activation distances 106 | def _load_next_chunk(self): 107 | self._chk_row = self.h5f['row'][self._chk_end : self._chk_end + self.chunk_size] 108 | self._chk_col = self.h5f['col'][self._chk_end : self._chk_end + self.chunk_size] 109 | self._chk_data = self.h5f['dist'][self._chk_end : self._chk_end + self.chunk_size] 110 | self._chk_end += self.chunk_size 111 | self._chk_i = 0 112 | 113 | def __del__(self): 114 | self.h5f.close() 115 | 116 | -------------------------------------------------------------------------------- /igm/restraints/inter_hic.py: -------------------------------------------------------------------------------- 1 | ## --------------------------- 2 | # Allocate INTER Hi-C restraints (using the "Actdist.hdf5" file from the Assignment Step) as forces between particles to the model object in IGM 3 | ## --------------------------- 4 | 5 | 6 | 7 | from __future__ import division, print_function 8 | 9 | import numpy as np 10 | 11 | from .restraint import Restraint 12 | from ..model.forces import HarmonicUpperBound 13 | from ..utils.log import logger 14 | 15 | import h5py 16 | 17 | class interHiC(Restraint): 18 | 19 | """ 20 | Object handles Hi-C restraint 21 | 22 | Parameters 23 | ---------- 24 | actdist_file : activation distance file (generated from ActivationDistanceStep.py) 25 | 26 | contactRange : int 27 | defining contact range between 2 particles as contactRange*(r1+r2) 28 | 29 | k: float 30 | elastic constant for harmonic restraint 31 | """ 32 | 33 | def __init__(self, actdist_file, chrom, contactRange=2, k=1.0): 34 | 35 | self.contactRange = contactRange 36 | self.k = k 37 | self.chrom = chrom 38 | self.forceID = [] 39 | self._load_actdist(actdist_file) 40 | #- 41 | 42 | def _load_actdist(self,actdist): 43 | self.actdist = ActivationDistanceDB(actdist) 44 | 45 | def _apply(self, model): 46 | 47 | """ Apply harmonic restraint to those distances smaller than d_{act}""" 48 | 49 | for (i, j, d) in self.actdist: 50 | 51 | # calculate particle distances for i, j 52 | # if ||i-j|| <= d then assign a bond for i, j 53 | if ((model.particles[i] - model.particles[j] <= d) and (self.chrom[i] != self.chrom[j])) : 54 | 55 | # harmonic mean distance 56 | dij = self.contactRange*(model.particles[i].r + 57 | model.particles[j].r) 58 | 59 | # add harmonic bond between i-th and j-th beads 60 | f = model.addForce(HarmonicUpperBound((i, j), dij, self.k, 61 | note=Restraint.INTER_HIC)) 62 | self.forceID.append(f) 63 | #- 64 | #- 65 | #= 66 | 67 | 68 | #== 69 | 70 | class ActivationDistanceDB(object): 71 | 72 | """ HDF5 activation distance iterator: read in file of activation distances, in chunks """ 73 | 74 | def __init__(self, actdist_file, chunk_size = 10000): 75 | self.h5f = h5py.File(actdist_file,'r') 76 | self._n = len(self.h5f['row']) 77 | 78 | self.chunk_size = min(chunk_size, self._n) 79 | 80 | self._i = 0 81 | self._chk_i = chunk_size 82 | self._chk_end = 0 83 | 84 | def __len__(self): 85 | return self._n 86 | 87 | def __iter__(self): 88 | return self 89 | 90 | def __next__(self): 91 | if self._i < self._n: 92 | self._i += 1 93 | self._chk_i += 1 94 | 95 | if self._chk_i >= self.chunk_size: 96 | self._load_next_chunk() 97 | 98 | return (self._chk_row[self._chk_i], 99 | self._chk_col[self._chk_i], 100 | self._chk_data[self._chk_i]) 101 | else: 102 | self._i = 0 103 | self._chk_i = self.chunk_size 104 | self._chk_end = 0 105 | raise StopIteration() 106 | def next(self): 107 | return self.__next__() 108 | 109 | # probabilities are not needed now, since they are already encoded in the list of activation distances 110 | def _load_next_chunk(self): 111 | self._chk_row = self.h5f['row'][self._chk_end : self._chk_end + self.chunk_size] 112 | self._chk_col = self.h5f['col'][self._chk_end : self._chk_end + self.chunk_size] 113 | self._chk_data = self.h5f['dist'][self._chk_end : self._chk_end + self.chunk_size] 114 | self._chk_end += self.chunk_size 115 | self._chk_i = 0 116 | 117 | def __del__(self): 118 | self.h5f.close() 119 | 120 | -------------------------------------------------------------------------------- /igm/restraints/intra_hic.py: -------------------------------------------------------------------------------- 1 | ## --------------------------- 2 | # Allocate INTRA Hi-C restraints (using the "Actdist.hdf5" file from the Assignment Step) as "HarmonicUpperBound" forces between particles to the model object in IGM 3 | ## --------------------------- 4 | 5 | 6 | 7 | 8 | from __future__ import division, print_function 9 | 10 | import numpy as np 11 | 12 | from .restraint import Restraint 13 | from ..model.forces import HarmonicUpperBound 14 | from ..utils.log import logger 15 | 16 | import h5py 17 | 18 | class intraHiC(Restraint): 19 | 20 | """ 21 | Object handles Hi-C restraint 22 | 23 | Parameters 24 | ---------- 25 | actdist_file : activation distance file (generated from ActivationDistanceStep.py) 26 | 27 | contactRange : int 28 | defining contact range between 2 particles as contactRange*(r1+r2) 29 | 30 | k: float 31 | elastic constant for harmonic restraint 32 | """ 33 | 34 | def __init__(self, actdist_file, chrom, contactRange=2, k=1.0): 35 | 36 | self.contactRange = contactRange 37 | self.k = k 38 | self.chrom = chrom 39 | self.forceID = [] 40 | self._load_actdist(actdist_file) 41 | #- 42 | 43 | def _load_actdist(self,actdist): 44 | self.actdist = ActivationDistanceDB(actdist) 45 | 46 | def _apply(self, model): 47 | 48 | """ Apply harmonic restraint to those distances smaller than d_{act}""" 49 | 50 | for (i, j, d) in self.actdist: 51 | 52 | # calculate particle distances for i, j 53 | # if ||i-j|| <= d then assign a bond for i, j 54 | if ((model.particles[i] - model.particles[j] <= d) and (self.chrom[i] == self.chrom[j])) : 55 | 56 | # harmonic mean distance 57 | dij = self.contactRange*(model.particles[i].r + 58 | model.particles[j].r) 59 | 60 | # add harmonic bond between i-th and j-th beads 61 | f = model.addForce(HarmonicUpperBound((i, j), dij, self.k, 62 | note=Restraint.INTRA_HIC)) 63 | self.forceID.append(f) 64 | #- 65 | #- 66 | #= 67 | 68 | 69 | #== 70 | 71 | class ActivationDistanceDB(object): 72 | 73 | """ HDF5 activation distance iterator: read in file of activation distances, in chunks """ 74 | 75 | def __init__(self, actdist_file, chunk_size = 10000): 76 | self.h5f = h5py.File(actdist_file,'r') 77 | self._n = len(self.h5f['row']) 78 | 79 | self.chunk_size = min(chunk_size, self._n) 80 | 81 | self._i = 0 82 | self._chk_i = chunk_size 83 | self._chk_end = 0 84 | 85 | def __len__(self): 86 | return self._n 87 | 88 | def __iter__(self): 89 | return self 90 | 91 | def __next__(self): 92 | if self._i < self._n: 93 | self._i += 1 94 | self._chk_i += 1 95 | 96 | if self._chk_i >= self.chunk_size: 97 | self._load_next_chunk() 98 | 99 | return (self._chk_row[self._chk_i], 100 | self._chk_col[self._chk_i], 101 | self._chk_data[self._chk_i]) 102 | else: 103 | self._i = 0 104 | self._chk_i = self.chunk_size 105 | self._chk_end = 0 106 | raise StopIteration() 107 | def next(self): 108 | return self.__next__() 109 | 110 | # probabilities are not needed now, since they are already encoded in the list of activation distances 111 | def _load_next_chunk(self): 112 | self._chk_row = self.h5f['row'][self._chk_end : self._chk_end + self.chunk_size] 113 | self._chk_col = self.h5f['col'][self._chk_end : self._chk_end + self.chunk_size] 114 | self._chk_data = self.h5f['dist'][self._chk_end : self._chk_end + self.chunk_size] 115 | self._chk_end += self.chunk_size 116 | self._chk_i = 0 117 | 118 | def __del__(self): 119 | self.h5f.close() 120 | 121 | -------------------------------------------------------------------------------- /igm/restraints/nucleolus.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | from .restraint import Restraint 4 | from ..model.particle import Particle 5 | from ..model.forces import NuclExcludedVolume 6 | 7 | class Nucleolus(Restraint): 8 | """ 9 | A object handles nucleolus excluded volume restraints 10 | 11 | Parameters 12 | ---------- 13 | body_pos, body_r: center coordinates and radius of nucleolus 14 | k : float 15 | spring constant 16 | """ 17 | 18 | def __init__(self, body_pos, body_r, k=1.0): 19 | 20 | """ Class initialization""" 21 | 22 | self.body_pos = body_pos # coordinates of nucleolus center 23 | self.body_r = body_r # radius of spherical nucleulus 24 | self.k = k 25 | self.forceID = [] 26 | 27 | def _apply(self, model): 28 | 29 | """ Apply force """ 30 | 31 | plist = [i for i, p in enumerate(model.particles) if p.ptype == Particle.NORMAL] 32 | 33 | f = model.addForce(NuclExcludedVolume(plist, self.body_pos, self.body_r, self.k)) 34 | self.forceID.append(f) 35 | #- 36 | #= 37 | -------------------------------------------------------------------------------- /igm/restraints/polymer.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | import numpy as np 4 | from .restraint import Restraint 5 | from ..model.forces import HarmonicUpperBound 6 | 7 | MIN_CONSECUTIVE = 0.5 8 | 9 | class Polymer(Restraint): 10 | """ 11 | Object handles consecutive bead restraint 12 | 13 | Parameters 14 | ---------- 15 | index : alabtools.index object 16 | chromosome chain index 17 | contactRange : int 18 | defining contact range between 2 particles as contactRange*(r1+r2) 19 | k : float 20 | spring constant 21 | 22 | """ 23 | 24 | def __init__(self, index, contactRange=2, k=1.0, contact_probabilities=None): 25 | self.index = index 26 | self.contactRange = contactRange 27 | self.k = k 28 | self.forceID = [] 29 | self.cp = np.load(contact_probabilities) if (contact_probabilities is not None) else None 30 | 31 | def _apply(self, model): 32 | 33 | for i in range(len(self.index) - 1): 34 | 35 | # if i and i+1 belong to the same chromosome (and copy) 36 | if (self.index.chrom[i] == self.index.chrom[i+1] and 37 | self.index.copy[i] == self.index.copy[i+1]): 38 | 39 | # do we have contact probabiities? 40 | if self.cp is None: 41 | dij = self.contactRange*(model.particles[i].r + 42 | model.particles[i+1].r) 43 | else: 44 | d0 = (model.particles[i].r + model.particles[i+1].r) 45 | d1 = self.contactRange * d0 46 | f = self.cp[i] 47 | # if we have inconsistent or no data, just assume MIN_CONSECUTIVE contact 48 | if f < MIN_CONSECUTIVE or f > 1: 49 | f = MIN_CONSECUTIVE 50 | x3 = ( d1**3 + ( f - 1 )*d0**3 ) / f 51 | dij = x3**(1./3) 52 | 53 | f = model.addForce(HarmonicUpperBound((i, i+1), dij, self.k, 54 | note=Restraint.CONSECUTIVE)) 55 | self.forceID.append(f) 56 | #- 57 | #-- 58 | #= 59 | 60 | 61 | -------------------------------------------------------------------------------- /igm/restraints/restraint.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | import numpy as np 3 | 4 | class Restraint(object): 5 | """ 6 | 7 | Restraint object, takes care of data and translate to forces in model. 8 | 9 | Also keep track of forces added and can evaluate 10 | 11 | """ 12 | 13 | OTHER = -1 14 | CONSECUTIVE = 0 15 | HIC = 1 16 | INTRA_HIC = 9 17 | INTER_HIC = 10 18 | DAMID = 2 19 | FISH_RADIAL = 3 20 | FISH_PAIR = 4 21 | SPRITE = 5 22 | ENVELOPE = 6 23 | EXCLUDED_VOLUME = 7 24 | NUCL_EXCLUDED_VOLUME = 8 25 | 26 | def __init__(self, data, args): 27 | self.forceID = [] 28 | 29 | def _apply_model(self, model, override=False): 30 | """ 31 | Attach restraint to model 32 | """ 33 | try: 34 | self.model 35 | if override: 36 | self.model = model 37 | else: 38 | raise(RuntimeError("Restraint alreay applyed to model! Set override to true to proceed.")) 39 | except AttributeError: 40 | self.model = model 41 | 42 | def _apply(self, model, override=False): 43 | self._apply_model(model, override) 44 | 45 | 46 | def __len__(self): 47 | return len(self.forceID) 48 | 49 | def __getitem__(self, key): 50 | return self.forceID[key] 51 | 52 | def evaluate(self): 53 | """ 54 | evaluate the restraint violations 55 | """ 56 | score = 0 57 | violations = 0 58 | 59 | for fid in self.forceID: 60 | s = self.model.evalForceScore(fid) 61 | if s > 0: 62 | violations += 1 63 | #- 64 | score += s 65 | 66 | return (violations, score) 67 | 68 | def get_violations(self, tolerance): 69 | violations = [] 70 | ratios = [] 71 | 72 | for fid in self.forceID: 73 | s = self.model.evalForceViolationRatio(fid) 74 | if s > tolerance: 75 | violations.append(repr(self.model.forces[fid])) 76 | ratios.append(s) 77 | 78 | return (violations, ratios) 79 | 80 | def get_violation_histogram(self, nbins=100, vmax=1, epsilon=1e-4): 81 | v = self.get_violations(tolerance=epsilon) 82 | over = len(v>vmax) 83 | inner = v[v<=vmax] 84 | H, edges = np.histogram(inner, bins=nbins) 85 | H = np.concatenate([H, [over]]) 86 | edges.append(np.array([vmax, float('inf')])) 87 | return H, edges 88 | 89 | def __repr__(self): 90 | return type(self).__name__ 91 | 92 | -------------------------------------------------------------------------------- /igm/restraints/sprite.py: -------------------------------------------------------------------------------- 1 | ## --------------------------- 2 | # Allocate single cell SPRITE restraints (using the "SpriteActdist.hdf5" file from the Assignment Step) as "HarmonicUpperBound" forces between particles to the model object in IGM 3 | ## --------------------------- 4 | 5 | 6 | from __future__ import division, print_function 7 | 8 | import numpy as np 9 | import h5py 10 | 11 | from ..model.particle import Particle 12 | from .restraint import Restraint 13 | from ..model.forces import HarmonicUpperBound 14 | 15 | class Sprite(Restraint): 16 | """ 17 | Add SPRITE restraint as a DUMMY DYNAMIC particle is introduced and all beads assigned to a given 18 | cluster are connected to it via harmonic springs 19 | 20 | Parameters 21 | ---------- 22 | assignment_file : SPRITE activation distance file 23 | 24 | volume_occupancy : float 25 | defining contact range between 2 particles as contactRange*(r1+r2) 26 | 27 | k (float): elastic constant for restraining 28 | """ 29 | 30 | def __init__(self, assignment_file, volume_occupancy, struct_id, k=1.0): 31 | 32 | """ Initialize SPRITE parameters and input file """ 33 | 34 | self.volume_occupancy = volume_occupancy 35 | self.struct_id = struct_id 36 | self.k = k 37 | self.forceID = [] 38 | self.assignment_file = h5py.File(assignment_file, 'r') 39 | #- 40 | 41 | def _apply(self, model): 42 | 43 | """ Apply SPRITE restraints """ 44 | 45 | assignment = self.assignment_file['assignment'][()] 46 | indptr = self.assignment_file['indptr'][()] 47 | selected_beads = self.assignment_file['selected'] 48 | clusters_ids = np.where(assignment == self.struct_id)[0] 49 | radii = model.getRadii() 50 | coord = model.getCoordinates() 51 | 52 | # loop over indexes in 'assignment' (see SpriteAcrivationStep.py) referring to structure 'struct_id' 53 | for i in clusters_ids: 54 | beads = selected_beads[ indptr[i] : indptr[i+1] ] 55 | crad = radii[beads] 56 | ccrd = coord[beads] 57 | csize = len(beads) 58 | csize = get_cluster_size(crad, self.volume_occupancy) 59 | 60 | # add a centroid for the cluster, in the geometric center 61 | centroid_pos = np.mean(ccrd, axis=0) 62 | centroid = model.addParticle(centroid_pos, 0, Particle.DUMMY_DYNAMIC) # no excluded volume 63 | 64 | for b in beads: 65 | # apply harmonic restraint(s) to all beads making up the cluster, using the position of the centroid as "origin" (one-body terms) 66 | f = model.addForce(HarmonicUpperBound( 67 | (b, centroid), float(csize-radii[b]), self.k, 68 | note=Restraint.SPRITE)) 69 | self.forceID.append(f) 70 | 71 | 72 | def cbrt(x): 73 | return (x)**(1./3.) 74 | 75 | def get_cluster_size(radii, volume_occupancy): 76 | return cbrt(np.sum(radii**3)/volume_occupancy) 77 | -------------------------------------------------------------------------------- /igm/restraints/steric.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | from .restraint import Restraint 4 | from ..model.particle import Particle 5 | from ..model.forces import ExcludedVolume 6 | 7 | class Steric(Restraint): 8 | """ 9 | A object handles nuclear envelope restraints 10 | 11 | Parameters 12 | ---------- 13 | nucRadius : float 14 | nuclear radius in unit of nm. 15 | k : float 16 | spring constant 17 | """ 18 | 19 | def __init__(self, k=1.0): 20 | self.k = k 21 | self.forceID = [] 22 | 23 | def _apply(self, model): 24 | 25 | plist = [i for i, p in enumerate(model.particles) if p.ptype == Particle.NORMAL] 26 | 27 | f = model.addForce(ExcludedVolume(plist, self.k)) 28 | 29 | self.forceID.append(f) 30 | #- 31 | #= 32 | -------------------------------------------------------------------------------- /igm/steps/RandomInit.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | import numpy as np 3 | import os, os.path 4 | 5 | from math import acos, sin, cos, pi 6 | 7 | from ..core import StructGenStep 8 | from ..utils import HmsFile 9 | from alabtools.analysis import HssFile 10 | 11 | class RandomInit(StructGenStep): 12 | 13 | def setup(self): 14 | self.tmp_file_prefix = "random" 15 | self.argument_list = range(self.cfg["model"]["population_size"]) 16 | 17 | @staticmethod 18 | def task(struct_id, cfg, tmp_dir): 19 | """ 20 | generate one random structure with territories 21 | """ 22 | k = np.random.randint(0, 2**32) 23 | np.random.seed( (k*struct_id) % (2**32) ) 24 | hssfilename = cfg["optimization"]["structure_output"] 25 | nucleus_radius = cfg.get("model/init_radius") 26 | 27 | with HssFile(hssfilename,'r') as hss: 28 | index = hss.index 29 | 30 | crd = generate_territories(index, nucleus_radius) 31 | 32 | ofname = os.path.join(tmp_dir, 'random_%d.hms' % struct_id) 33 | with HmsFile(ofname, 'w') as hms: 34 | hms.saveCoordinates(struct_id, crd) 35 | 36 | def intermediate_name(self): 37 | return '.'.join([ 38 | self.cfg["optimization"]["structure_output"], 39 | 'randomInit' 40 | ]) 41 | #- 42 | 43 | 44 | def uniform_sphere(R): 45 | """ 46 | Generates uniformly distributed points in a sphere 47 | 48 | Arguments: 49 | R (float): radius of the sphere 50 | Returns: 51 | np.array: 52 | triplet of coordinates x, y, z 53 | """ 54 | phi = np.random.uniform(0, 2 * pi) 55 | costheta = np.random.uniform(-1, 1) 56 | u = np.random.uniform(0, 1) 57 | 58 | theta = acos( costheta ) 59 | r = R * ( u**(1./3.) ) 60 | 61 | x = r * sin( theta) * cos( phi ) 62 | y = r * sin( theta) * sin( phi ) 63 | z = r * cos( theta ) 64 | 65 | return np.array([x,y,z]) 66 | 67 | 68 | 69 | def generate_territories(index, R=5000.0): 70 | ''' 71 | Creates a single random structure with chromosome territories. 72 | Each "territory" is a sphere with radius 0.75 times the average 73 | expected radius of a chromosome. 74 | Arguments: 75 | chrom : alabtools.utils.Index 76 | the bead index for the system. 77 | R : float 78 | radius of the cell 79 | 80 | Returns: 81 | np.array : structure coordinates 82 | ''' 83 | 84 | # chromosome ends are detected when 85 | # the name is changed 86 | n_tot = len(index) 87 | n_chrom = len(index.chrom_sizes) 88 | 89 | crds = np.empty((n_tot, 3)) 90 | # the radius of the chromosome is set as 75% of its 91 | # "volumetric sphere" one. This is totally arbitrary. 92 | # Note: using float division of py3 93 | chr_radii = [0.75 * R * (float(nb)/n_tot)**(1./3) for nb in index.chrom_sizes] 94 | crad = np.average(chr_radii) 95 | k = 0 96 | for i in range(n_chrom): 97 | center = uniform_sphere(R - crad) 98 | for j in range(index.chrom_sizes[i]): 99 | crds[k] = uniform_sphere(crad) + center 100 | k += 1 101 | 102 | return crds 103 | 104 | 105 | def generate_random_in_sphere(radii, R=5000.0): 106 | ''' 107 | Returns: 108 | np.array : structure coordinates 109 | ''' 110 | return np.array([uniform_sphere(R-r) for r in radii]) 111 | -------------------------------------------------------------------------------- /igm/steps/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | from .RandomInit import RandomInit 3 | from .RelaxInit import RelaxInit 4 | from .ActivationDistanceStep import ActivationDistanceStep 5 | from .DamidActivationDistanceStep import DamidActivationDistanceStep 6 | from .ModelingStep import ModelingStep 7 | from .SpriteAssignmentStep import SpriteAssignmentStep 8 | from .HicEvaluationStep import HicEvaluationStep 9 | from .FishAssignmentStep import FishAssignmentStep 10 | 11 | 12 | -------------------------------------------------------------------------------- /igm/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/igm/tasks/__init__.py -------------------------------------------------------------------------------- /igm/tasks/modeling_step.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | import numpy as np 4 | import os 5 | 6 | from functools import partial 7 | from alabtools.analysis import HssFile, COORD_DTYPE 8 | 9 | from ..model import Model, Particle 10 | from ..parallel.ipyparallel_controller import BasicIppController, AdvancedIppController 11 | from ..parallel.parallel_controller import SerialController 12 | from ..core.config import Config 13 | from ..restraints import Polymer, Envelope, Steric 14 | from ..kernel import lammps 15 | from ..util import resolve_templates 16 | 17 | controller_class = { 18 | u'serial' : SerialController, 19 | u'ipyparallel' : AdvancedIppController, 20 | u'ipyparallel_basic' : BasicIppController, 21 | } 22 | 23 | kernel_class = { 24 | u'lammps' : lammps 25 | } 26 | 27 | 28 | def modeling_task(struct_id, cfg_file): 29 | ''' 30 | Serial function to be mapped in parallel. // 31 | It is a wrapper intended to be used only internally by the parallel map 32 | function. Will be called as a partial with all the constant variables 33 | set, except i. 34 | Resolve the templates, obtains input data, 35 | runs the minimization routines and finally communicates back results. 36 | 37 | Parameters 38 | ---------- 39 | i : int 40 | number of the structure 41 | cfg_file : str 42 | configuration filename for the task 43 | 44 | Returns 45 | ------- 46 | None 47 | ''' 48 | cfg = Config(cfg_file) 49 | 50 | # importing here so it will be called on the parallel workers 51 | local_vars = resolve_templates(cfg['mstep']['templates'], [struct_id]) 52 | 53 | model = Model() 54 | with HssFile(cfg['mstep']['input_hss'], 'r') as f: 55 | radii = f.radii 56 | index = f.index 57 | crd = f['coordinates'][:, struct_id, :][()] 58 | 59 | n_particles = len(crd) 60 | for i in range(n_particles): 61 | model.addParticle(crd[i], radii[i], Particle.NORMAL) 62 | 63 | ee = Envelope(cfg['model']['nucleus_geometry']) 64 | model.addRestraint(ee) 65 | 66 | ex = Steric(cfg['model']['evfactor']) 67 | model.addRestraint(ex) 68 | 69 | pp = Polymer(index, 70 | cfg['model']['contact_range'], 71 | cfg['model']['contact_kspring']) 72 | model.addRestraint(pp) 73 | 74 | kernel = kernel_class[cfg['mstep']['kernel']] 75 | info = kernel.optimize(model, cfg['optimization']) 76 | 77 | new_crd = np.array([p.pos for p in model.particles], dtype=COORD_DTYPE) 78 | np.save(local_vars['crd_out'], new_crd) 79 | 80 | # make sure that is readable 81 | np.load(local_vars['crd_out']) 82 | 83 | with open(local_vars['info_out'], 'w') as f: 84 | for k in kernel.INFO_KEYS: 85 | if isinstance(info[k], float): 86 | out_str = '{:9.2f}'.format(info[k]) 87 | elif isinstance(info[k], int): 88 | out_str = '{:7d}'.format(info[k]) 89 | else: 90 | out_str = str(info[k]) 91 | f.write(out_str + '\t') 92 | 93 | def modeling_step(model, cfg): 94 | 95 | with HssFile(cfg['mstep']['input_hss'], 'r') as f: 96 | radii = f.radii 97 | index = f.index 98 | genome = f.genome 99 | 100 | n_struct = cfg['mstep']['n_struct'] 101 | n_beads = cfg['mstep']['n_beads'] 102 | 103 | basepath = os.path.join(cfg['mstep']['workdir'], cfg['mstep']['run_name']) 104 | cfg['mstep']['templates'] = { 105 | 'crd_out' : basepath + '.outcrd.{}.npy', 106 | 'info_out' : basepath + '.info.{}.txt' 107 | } 108 | cfg.save(basepath + '.config') 109 | 110 | serial_function = partial(modeling_task, cfg_file=basepath + '.config') 111 | pctype = cfg['parallel_controller'] 112 | pcopts = cfg['parallel_controller_options'] 113 | controller = controller_class[pctype](**pcopts) 114 | argument_list = list(range(n_struct)) 115 | 116 | controller.map(serial_function, argument_list) 117 | 118 | # write coordinates 119 | crd_shape = (n_beads, n_struct, 3) 120 | with HssFile(cfg['mstep']['output_hss'], 'w') as hss: 121 | hss.index = index 122 | hss.genome = genome 123 | hss.radii = radii 124 | all_crd = hss.create_dataset('coordinates', shape=crd_shape, dtype=COORD_DTYPE) 125 | for i in range(n_struct): 126 | local_vars = resolve_templates(cfg['mstep']['templates'], [i]) 127 | crd = np.load(local_vars['crd_out']) 128 | all_crd[:, i, :] = crd # note that we discard all the positions of added atoms 129 | 130 | # write info 131 | kernel = kernel_class[cfg['mstep']['kernel']] 132 | with open(cfg['mstep']['info_out'], 'w') as outf: 133 | outf.write('#') 134 | for k in kernel.INFO_KEYS: 135 | outf.write(k + '\t') 136 | outf.write('\n') 137 | for i in range(n_struct): 138 | local_vars = resolve_templates(cfg['mstep']['templates'], [i]) 139 | with open(local_vars['info_out']) as inf: 140 | outf.write(inf.read() + '\n') 141 | 142 | # cleanup 143 | for i in range(n_struct): 144 | local_vars = resolve_templates(cfg['mstep']['templates'], [i]) 145 | os.remove(local_vars['crd_out']) 146 | os.remove(local_vars['info_out']) 147 | 148 | 149 | 150 | 151 | 152 | 153 | -------------------------------------------------------------------------------- /igm/ui/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/igm/ui/__init__.py -------------------------------------------------------------------------------- /igm/ui/communication.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os, os.path 3 | from alabtools.analysis import HssFile 4 | from igm.core.job_tracking import StepDB 5 | from igm import Config 6 | from .folders_database import register_folder 7 | import glob 8 | 9 | 10 | def history(folder='.'): 11 | cfg = json.load(open(os.path.join(folder, 'igm-config.json'), 'r')) 12 | try: 13 | db = StepDB(cfg, mode='r') 14 | h = db.get_history() 15 | except OSError: 16 | h = [] 17 | # to avoid excessive data exchange 18 | for i in range(len(h)): 19 | del h[i]['cfg'] 20 | return h 21 | 22 | def readlog(folder='.'): 23 | 24 | # select the file 25 | logf = os.path.join(folder, 'igm-log.txt') 26 | cfgf = os.path.join(folder, 'igm-config.json') 27 | if os.path.isfile(logf): 28 | f = logf 29 | elif os.path.isfile(cfgf): 30 | current_cfg = json.load(open(cfgf)) 31 | f = current_cfg['log'] 32 | else: 33 | return None 34 | 35 | # read in binary mode, so will keep carriage returns 36 | lines = open(f, 'rb').readlines() 37 | # remove all the unnecessary carriage returns and reassemble 38 | log = '\n'.join([ l.decode('utf-8').split('\r')[-1].strip('\n') for l in lines ]) 39 | return log 40 | 41 | def igm_is_running(folder='.'): 42 | pidf = os.path.join(folder, '.igm-pid.txt') 43 | if os.path.isfile(pidf): 44 | pid = int(open(pidf).read()) 45 | try: 46 | # sending a 0 signal fails if the process does not exist 47 | # does nothing otherwise 48 | os.kill(pid, 0) 49 | status = 'yes' 50 | except OSError: 51 | # In this case the machine running the server 52 | # may be different from the machine running the igm script, 53 | # or something exploded before calling the atexit functions 54 | status = 'maybe' 55 | else: 56 | status = 'no' 57 | return status 58 | 59 | def kill_igm(folder='.'): 60 | ''' 61 | Try to kindly ask to terminate, then kill the process 62 | if it is not done in 5 seconds. 63 | ''' 64 | pidf = os.path.join(folder, '.igm-pid.txt') 65 | import time, multiprocessing 66 | if os.path.isfile(pidf): 67 | pid = int(open(pidf).read()) 68 | os.kill(pid, 2) 69 | def real_kill(pid): 70 | time.sleep(5) 71 | if os.path.isfile(pidf): 72 | os.kill(pid, 9) 73 | os.remove(pidf) 74 | p = multiprocessing.Process(target=real_kill, args=(pid,), daemon=True) 75 | p.start() 76 | 77 | 78 | def clear_previous_runs(folder='.'): 79 | cfgf = os.path.join(folder, 'igm-config.json') 80 | # this is ultra dangerous, one could change the config and 81 | # remove arbitrary files, should check they are files 82 | # inside the current directory 83 | cfg = Config(cfgf) 84 | os.remove(cfg['step_db']) 85 | os.remove(cfg['structure_output']) 86 | os.remove(cfg['structure_output'] + '.tmp') 87 | os.remove(cfg['log']) 88 | 89 | def get_structure(path, n, folder='.'): 90 | path = os.path.join(folder, path) 91 | 92 | with HssFile(path, 'r') as f: 93 | crd = f.get_struct_crd(n).tolist() 94 | chrom = f.genome.chroms[f.index.chrom].tolist() 95 | radius = f.radii.tolist() 96 | nstruct = f.nstruct 97 | cstarts = f.index.offset.tolist() 98 | 99 | return { 100 | 'crd' : [crd[cstarts[i]:cstarts[i+1]] for i in range(len(cstarts)-1)], 101 | 'idx' : chrom, 102 | 'rad' : [radius[cstarts[i]:cstarts[i+1]] for i in range(len(cstarts)-1)], 103 | 'n' : int(nstruct), 104 | 'cstarts': cstarts, 105 | 'chroms': [str(v) for i, v in enumerate(chrom) if i == 0 or v != chrom[i-1]], 106 | } 107 | 108 | def save_metadata(data, cwd): 109 | folder = data.pop('folder') 110 | if os.path.realpath(folder) != os.path.realpath(cwd): 111 | raise ValueError('Invalid folder %s' % folder) 112 | if 'notes' in data: 113 | if data['notes']: 114 | with open(os.path.realpath(folder) + '/IGM_NOTES.TXT', 'w') as f: 115 | f.write(data['notes']) 116 | register_folder(cwd, **data) 117 | -------------------------------------------------------------------------------- /igm/ui/config_parse.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os, os.path 3 | 4 | import igm.core.defaults 5 | 6 | 7 | schema_file = os.path.join( 8 | os.path.dirname( os.path.abspath(igm.core.defaults.__file__) ), 9 | 'config_schema.json' 10 | ) 11 | schema = json.load(open(schema_file, 'r')) 12 | 13 | def set_item(d, path, val): 14 | if len(path) == 1: 15 | d[path[0]] = val 16 | else: 17 | if path[0] not in d: 18 | d[path[0]] = dict() 19 | set_item(d[path[0]], path[1:], val) 20 | 21 | def get_item(d, path): 22 | if len(path) == 1: 23 | return d[ path[0] ] 24 | return get_item(d[ path[0] ], path[1:]) 25 | 26 | def split_path(path, sep='__'): 27 | return path.split(sep) 28 | 29 | def type_or_json(vtype, val): 30 | from six import raise_from 31 | if isinstance(val, vtype): 32 | return val 33 | if not isinstance(val, str): 34 | raise ValueError() 35 | try: 36 | v = json.loads(val) 37 | return v 38 | except json.JSONDecodeError: 39 | raise_from(ValueError(), None) 40 | if isinstance(val, vtype): 41 | return val 42 | raise ValueError() 43 | 44 | def validate_value(value, dtype, subdtype=None, alen=None): 45 | if dtype == 'int': 46 | return int(value) 47 | elif dtype == 'float': 48 | return float(value) 49 | elif dtype == 'bool': 50 | return bool(value) 51 | elif dtype == 'list': 52 | sval = type_or_json(list, value) 53 | for i in range(len(sval)): 54 | if subdtype == 'list': 55 | if not isinstance(sval[i], list): 56 | raise ValueError() 57 | else: 58 | sval[i] = validate_value(sval[i], subdtype) 59 | return sval 60 | elif dtype == 'array': 61 | sval = type_or_json(list, value) 62 | if len(sval) != alen: 63 | raise ValueError('array length does not match') 64 | for i in range(len(sval)): 65 | sval[i] = validate_value(sval[i], subdtype) 66 | return sval 67 | elif dtype == 'dict': 68 | return type_or_json(dict, value) 69 | elif dtype == 'str' or dtype == 'path' or dtype == 'path-dir': 70 | return(str(value)) 71 | else: 72 | raise ValueError() 73 | 74 | def save_cfg(data, folder='.'): 75 | warnings = [] 76 | errors = [] 77 | cfg = {} 78 | for path, value in data.items(): 79 | 80 | try: 81 | sitem = get_item(schema, split_path(path)) 82 | except KeyError: 83 | set_item(cfg, split_path(path), value) 84 | warnings.append('key "%s" not in schema' % path.replace('__', ' > ')) 85 | continue 86 | 87 | if sitem.get('blank', False): 88 | if value is None or (isinstance(value, str) and value.strip() == ''): 89 | continue 90 | 91 | dtypes = sitem['dtype'] 92 | 93 | if not isinstance(dtypes, list): 94 | dtypes = [dtypes] 95 | 96 | sval = None 97 | for dtype in dtypes: 98 | try: 99 | sval = validate_value(value, dtype, 100 | subdtype=sitem.get('subdtype'), 101 | alen=sitem.get('length')) 102 | break 103 | except ValueError: 104 | pass 105 | if sval is None: 106 | errors.append('%s: invalid data "%s". Valid data type are "%s" ' % ( 107 | path.replace('__', ' > '), value, dtypes)) 108 | continue 109 | 110 | if dtype in ['path', 'path-dir', 'str'] and not sitem.get('required', False): 111 | if sval.strip() == "": 112 | continue 113 | 114 | if dtype == 'path' and sitem.get('role', '') == 'input': 115 | if not os.path.isfile(sval): 116 | warnings.append('%s: cannot find input file "%s"' % (path.replace('__', ' > '), sval)) 117 | 118 | if sitem.get('allowed_values') is not None: 119 | if sval not in sitem['allowed_values']: 120 | errors.append('%s: invalid value "%s". Valid values are "%s" ' % ( 121 | path.replace('__', ' > '), value, sitem['allowed_values'])) 122 | continue 123 | 124 | set_item(cfg, split_path(path), sval) 125 | 126 | r = {'errors': errors, 'warnings': warnings} 127 | if len(errors): 128 | r['status'] = 'failed' 129 | r['cfg'] = None 130 | return r 131 | else: 132 | cfgf = os.path.join(folder, 'igm-config.json') 133 | with open(cfgf, 'w') as f: 134 | json.dump(cfg, f, indent=4) 135 | r['status'] = 'ok' 136 | r['cfg'] = cfg 137 | return r 138 | -------------------------------------------------------------------------------- /igm/ui/folders_database.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | from os.path import normpath, abspath, isfile, isdir 4 | import sqlite3 5 | import time 6 | 7 | def create_db(db_file): 8 | with sqlite3.connect(db_file) as db: 9 | q = 'CREATE TABLE paths (folder TEXT, name TEXT, cell_line TEXT, resolution TEXT, notes TEXT, tags TEXT, created INT, last_modified INT)' 10 | db.execute(q) 11 | 12 | def register_folder(folder, name="", cell_line="", resolution="", notes="", tags=""): 13 | 14 | folder = normpath( abspath(folder) ) 15 | 16 | if not isdir(folder): 17 | raise RuntimeError('Error: the directory `%s` does not exist or it is not readable\n' % folder) 18 | 19 | db_file = os.environ['HOME'] + '/.igm/paths.db' 20 | 21 | if not isfile(db_file): 22 | create_db(db_file) 23 | 24 | with sqlite3.connect(db_file) as db: 25 | 26 | c = db.execute('SELECT count(folder) FROM paths WHERE folder=?', (folder, )).fetchall()[0][0] 27 | if c > 0: 28 | db.execute('UPDATE paths SET name=?, cell_line=?, resolution=?, notes=?, tags=?, last_modified=? WHERE folder=?', (name, cell_line, resolution, notes, tags, int(time.time()), folder) ) 29 | 30 | else: 31 | db.execute('INSERT into paths (folder, name, cell_line, resolution, notes, tags, created, last_modified) VALUES (?, ?, ?, ?, ?, ?, ?, ?)', 32 | (folder, name, cell_line, resolution, notes, tags, int(time.time()), int(time.time())) ) 33 | 34 | return True 35 | 36 | def unregister_folder(folder): 37 | folder = os.path.normpath( os.path.abspath(folder) ) 38 | 39 | db_file = os.environ['HOME'] + '/.igm/paths.db' 40 | 41 | if not os.path.isfile(db_file): 42 | create_db(db_file) 43 | 44 | with sqlite3.connect(db_file) as db: 45 | x = db.execute('DELETE FROM paths WHERE folder=?', (folder,) ) 46 | if x.rowcount == 0: 47 | raise RuntimeError('The folder %s is not in the database\n' % folder) 48 | 49 | return True 50 | 51 | def folder_info(folder=None): 52 | if folder is not None: 53 | folder = os.path.normpath( os.path.abspath(folder) ) 54 | 55 | db_file = os.environ['HOME'] + '/.igm/paths.db' 56 | 57 | if not os.path.isfile(db_file): 58 | create_db(db_file) 59 | 60 | with sqlite3.connect(db_file) as db: 61 | if folder is not None: 62 | # return only specified folder 63 | c = db.execute('SELECT folder, name, cell_line, resolution, notes, tags, created, last_modified FROM paths WHERE folder=?', (folder, )).fetchall() 64 | if len(c) == 0: 65 | raise RuntimeError('Error: folder `%s` not in the database\n' % folder) 66 | else: 67 | folder, name, cell_line, resolution, notes, tags, ct, mt = c[0] 68 | return { 69 | 'folder' : folder, 70 | 'name': name, 71 | 'cell_line' : cell_line, 72 | 'resolution' : resolution, 73 | 'notes' : notes, 74 | 'tags' : tags, 75 | 'created' : ct, 76 | 'last_modified' : mt 77 | } 78 | else: 79 | # return all the folders 80 | c = db.execute('SELECT folder, name, cell_line, resolution, notes, tags, created, last_modified FROM paths').fetchall() 81 | res = [] 82 | for folder, name, cell_line, resolution, notes, tags, ct, mt in c: 83 | res.append({ 84 | 'folder' : folder, 85 | 'name': name, 86 | 'cell_line' : cell_line, 87 | 'resolution' : resolution, 88 | 'notes' : notes, 89 | 'tags' : tags, 90 | 'created' : ct, 91 | 'last_modified' : mt 92 | }) 93 | return res 94 | -------------------------------------------------------------------------------- /igm/ui/navigation.py: -------------------------------------------------------------------------------- 1 | import glob 2 | from os.path import basename, normpath, abspath, isdir, isfile 3 | 4 | def list_directory(path, root='/', ext=None): 5 | 6 | root = normpath(root) 7 | dname = normpath( abspath(root + '/' + path) ) 8 | 9 | if not dname.startswith(root) or not isdir(dname): 10 | return {} 11 | 12 | content = glob.glob(dname + '/*') 13 | dirs = [] 14 | 15 | if abspath(dname) != abspath(root): 16 | dirs += [ '..' ] 17 | 18 | rootlen = len(root)+1 19 | dirs += [basename( normpath(c) ) for c in content if isdir(c)] 20 | files = [basename( normpath(c) ) for c in content if isfile(c)] 21 | 22 | if ext is not None: 23 | if not isinstance(ext, list): 24 | ext = [ext] 25 | nfiles = [] 26 | for e in ext: 27 | nfiles += [c for c in files if c.endswith(e)] 28 | files = nfiles 29 | 30 | return { 31 | 'path': dname[rootlen:], 32 | 'dirs': dirs, 33 | 'files': files, 34 | } 35 | -------------------------------------------------------------------------------- /igm/ui/static/css/filebrowser.css: -------------------------------------------------------------------------------- 1 | #fb_u_loading { 2 | background: white; 3 | z-index:85; 4 | position:absolute; 5 | top:0px; 6 | left:0px; 7 | width:100%; 8 | height:100%; 9 | opacity:.3; 10 | } 11 | 12 | .fb_c_dir, .fb_c_file { 13 | line-height: 2.5em; 14 | position:relative; 15 | display: block; 16 | } 17 | 18 | .fb_l_dir, .fb_l_file { 19 | display: block-inline; 20 | width: 100%; 21 | height: 100%; 22 | word-break: break-all; 23 | } 24 | 25 | .fb_icon { 26 | width: 1.6em; 27 | display: inline-block; 28 | text-align: center; 29 | margin-right:1em; 30 | } 31 | 32 | .fb_c_dir:hover, .fb_c_file:hover { 33 | background-color: #eeeeee; 34 | } 35 | 36 | /*.fb_c_dir:before { 37 | content: "\F07B"; 38 | font-family: FontAwesome; 39 | left:-25px; 40 | position:absolute; 41 | top:0; 42 | } 43 | 44 | .fb_c_file:before { 45 | content: "\F15B"; 46 | font-family: FontAwesome; 47 | left:-25px; 48 | position:absolute; 49 | top:0; 50 | color: 'darkgreen'; 51 | }*/ 52 | 53 | 54 | .fb_l_dir { 55 | color: black; 56 | text-decoration: none; 57 | } 58 | 59 | .fb_l_dir:hover{ 60 | color: darkblue; 61 | text-decoration: none; 62 | } 63 | 64 | .fb_l_file { 65 | color: 'darkgreen'; 66 | } 67 | 68 | #fb_u_dirs, #fb_u_files { 69 | list-style: none; 70 | } 71 | -------------------------------------------------------------------------------- /igm/ui/static/css/igm-edit-config.css: -------------------------------------------------------------------------------- 1 | .igm-group { 2 | padding: 1em; 3 | border-radius:5px; 4 | margin: 1em; 5 | } 6 | 7 | .igm-group-1 { 8 | background-color: #ffffff; 9 | } 10 | 11 | 12 | .igm-group-2 { 13 | background-color: #e1e8d9; 14 | } 15 | 16 | 17 | .igm-group-3 { 18 | background-color: #eeeeee; 19 | 20 | } 21 | 22 | 23 | .igm-group-4 { 24 | background-color: #dddfec; 25 | } -------------------------------------------------------------------------------- /igm/ui/static/css/main.css: -------------------------------------------------------------------------------- 1 | html,body { height:100%; } 2 | 3 | #help-wrapper { 4 | background-color: white; 5 | position: fixed; 6 | width: 100%; 7 | height: 30%; 8 | bottom: 0px; 9 | left:0px; 10 | } 11 | 12 | #bottom-spacer { 13 | width: 100%; 14 | } 15 | 16 | 17 | #help-content { 18 | padding:2em; 19 | } 20 | 21 | #dragbar{ 22 | background-color: #cccccc; 23 | width: :100%; 24 | height: 4px; 25 | cursor: row-resize; 26 | } 27 | 28 | #log-ta { 29 | font-family: Inconsolata, Consolas, Monaco, monospace; 30 | color: #eeeeee; 31 | background-color: #282923; 32 | -webkit-box-sizing: border-box; 33 | -moz-box-sizing: border-box; 34 | box-sizing: border-box; 35 | width: 100%; 36 | height: 80vh; 37 | overflow-y: scroll; 38 | white-space: pre; 39 | } 40 | 41 | #load-screen { 42 | position: absolute; 43 | top: 0; 44 | left: 0; 45 | width: 100%; 46 | height: 100%; 47 | z-index: 90; 48 | } 49 | 50 | #load-screen > .transparent-background { 51 | position: relative; 52 | top: 0; 53 | left: 0; 54 | width: 100%; 55 | height: 100%; 56 | background:white; 57 | opacity:.3; 58 | } 59 | 60 | 61 | #load-screen > span { 62 | position: fixed; 63 | top: 50%; 64 | left: 50%; 65 | -webkit-transform: translate(-50%, -50%); 66 | transform: translate(-50%, -50%); 67 | color: darkgrey; 68 | } 69 | /* 70 | .vertical-left-text { 71 | -ms-transform: rotate(-90deg); 72 | -webkit-transform: rotate(-90deg); 73 | transform: rotate(-90deg); 74 | } 75 | */ 76 | #left-menu { 77 | height: 100%; 78 | float: left; 79 | padding-left: 0.5em; 80 | } 81 | 82 | #left-menu > button { 83 | clear:both; 84 | } 85 | 86 | #folder-navigator { 87 | border-right: solid 1px #cccccc; 88 | font-size: 13px; 89 | height: 100%; 90 | } 91 | 92 | .rotation-wrapper-outer { 93 | display: table; 94 | } 95 | .rotation-wrapper-inner { 96 | padding: 50% 0; 97 | height: 0; 98 | } 99 | .left-rotated { 100 | display: block; 101 | transform-origin: top left; 102 | /* Note: for a CLOCKWISE rotation, use the commented-out 103 | transform instead of this one. */ 104 | transform: rotate(-90deg) translate(-100%); 105 | /* transform: rotate(90deg) translate(0, -100%); */ 106 | margin-top: -50%; 107 | 108 | /* Not vital, but possibly a good idea if the element you're rotating contains 109 | text and you want a single long vertical line of text and the pre-rotation 110 | width of your element is small enough that the text wraps: */ 111 | white-space: nowrap; 112 | } 113 | 114 | #folder-notes-edt { 115 | font-family: Inconsolata, Consolas, Monaco, monospace; 116 | } 117 | 118 | #igm-content { 119 | } 120 | -------------------------------------------------------------------------------- /igm/ui/static/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alberlab/igm/edbb33135260443b8b41eb09d7ca4c37b19e330d/igm/ui/static/favicon.ico -------------------------------------------------------------------------------- /igm/ui/static/js/filebrowser.js: -------------------------------------------------------------------------------- 1 | 2 | function baseName(str) 3 | { 4 | var base = new String(str).substring(str.lastIndexOf('/') + 1); 5 | // if(base.lastIndexOf(".") != -1) 6 | // base = base.substring(0, base.lastIndexOf(".")); 7 | return base; 8 | } 9 | 10 | var FileBrowser = function(dom_element, onFileSelect) { 11 | 12 | var self = this; 13 | var root_path = '.'; 14 | self.current_path = root_path; 15 | self.dom_element = $(dom_element); 16 | 17 | self.loading_div = $('
'); 18 | self.dom_element.append(self.loading_div); 19 | self.loading_div.hide(); 20 | 21 | self.path_div = $('
'); 22 | self.dom_element.append(self.path_div); 23 | 24 | self.content_div = $('
'); 25 | self.dom_element.append(self.content_div); 26 | 27 | self.updateView = function(path, dirs, files) { 28 | 29 | self.current_path = path; 30 | 31 | self.loading_div.hide(); 32 | self.path_div.html('Path' + path + ''); 33 | 34 | var dirs_html = ''; 39 | 40 | var files_html = ''; 45 | 46 | self.content_div.html(dirs_html + files_html); 47 | 48 | $.each( self.content_div.find('.fb_l_dir'), function(index, item) { 49 | $(item).on( 'click', function() { 50 | self.navigate( $(item).attr('fb-target') ); 51 | }); 52 | }); 53 | 54 | $.each( self.content_div.find('.fb_l_file'), function(index, item) { 55 | $(item).on( 'click', function() { 56 | onFileSelect( $(item).attr('fb-target') ); 57 | }); 58 | }); 59 | 60 | } 61 | 62 | self.navigate = function(path) { 63 | self.loading_div.show(); 64 | req = { 65 | request: 'listdir', 66 | path: path, 67 | }; 68 | console.log(req); 69 | $.ajax({ 70 | type: "POST", 71 | url: '/ajax/', 72 | dataType: 'json', 73 | data: JSON.stringify(req), 74 | contentType: 'application/json', 75 | success: function(data){ 76 | console.log(data); 77 | self.updateView(data.path, data.dirs, data.hss); 78 | }, 79 | error: function(x, e, t){ 80 | console.log(x,e,t); 81 | alert('error!'); 82 | } 83 | }); 84 | } 85 | 86 | self.navigate(root_path); 87 | 88 | } 89 | -------------------------------------------------------------------------------- /igm/ui/static/js/genome_visualizer/filebrowser.js: -------------------------------------------------------------------------------- 1 | 2 | function baseName(str) 3 | { 4 | var base = new String(str).substring(str.lastIndexOf('/') + 1); 5 | // if(base.lastIndexOf(".") != -1) 6 | // base = base.substring(0, base.lastIndexOf(".")); 7 | return base; 8 | } 9 | 10 | var FileBrowser = function(dom_element, onFileSelect, root_path='') { 11 | 12 | var self = this; 13 | self.current_path = root_path; 14 | self.dom_element = $(dom_element); 15 | 16 | self.loading_div = $('
'); 17 | self.dom_element.append(self.loading_div); 18 | self.loading_div.hide(); 19 | 20 | self.path_div = $('
'); 21 | self.dom_element.append(self.path_div); 22 | 23 | self.content_div = $('
'); 24 | self.dom_element.append(self.content_div); 25 | 26 | self.dom_element.append($('')); 27 | 28 | self.updateView = function(path, dirs, files) { 29 | 30 | self.current_path = path; 31 | 32 | self.loading_div.hide(); 33 | self.path_div.html('' + path + ''); 34 | 35 | var dirs_html = ''; 40 | 41 | var files_html = ''; 46 | 47 | self.content_div.html(dirs_html + files_html); 48 | 49 | $.each( self.content_div.find('.fb_l_dir'), function(index, item) { 50 | $(item).on( 'click', function() { 51 | self.navigate( $(item).attr('fb-target') ); 52 | }); 53 | }); 54 | 55 | $.each( self.content_div.find('.fb_l_file'), function(index, item) { 56 | $(item).on( 'click', function() { 57 | onFileSelect( $(item).attr('fb-target') ); 58 | }); 59 | }); 60 | 61 | } 62 | 63 | self.navigate = function(path) { 64 | self.loading_div.show(); 65 | req = { 66 | request: 'listhss', 67 | path: self.current_path + '/' + path, 68 | }; 69 | $.ajax({ 70 | type: "POST", 71 | url: '/ajax/', 72 | data : { 73 | 'data' : JSON.stringify(req) 74 | }, 75 | dataType: 'json', 76 | success: function(data){ 77 | self.updateView(data.path, data.dirs, data.files); 78 | self.current_path = data.path; 79 | }, 80 | error: function(x, e, t){ 81 | console.log(x,e,t); 82 | alert('error!'); 83 | } 84 | }); 85 | } 86 | 87 | self.navigate(root_path); 88 | 89 | $("#fb_u_reload_btn").click( function() { 90 | self.navigate(self.current_path); 91 | }); 92 | 93 | } 94 | -------------------------------------------------------------------------------- /igm/ui/static/js/genome_visualizer/genomeapp.js: -------------------------------------------------------------------------------- 1 | 2 | var GenomeApp = function(){ 3 | 4 | var self = this; 5 | 6 | // set some constants 7 | 8 | // set the size for the viewer 9 | var el = $('#viewer'); //record the elem so you don't crawl the DOM everytime 10 | var bottom = el.offset().top + el.outerHeight(true); 11 | var height = $(window).height() - bottom; 12 | var container = document.getElementById('viewer') 13 | $(container).height(Math.round($(window).height()*0.8)); 14 | 15 | // create the viewer with default options 16 | var viewer = new Viewer(container); 17 | 18 | // create the app status 19 | var status = {}; 20 | 21 | var interface = new InterfaceControl(self); 22 | 23 | this.viewer = viewer; 24 | this.status = status; 25 | this.interface = interface; 26 | this.fileBrowser = null; 27 | 28 | var newFileFlag = false; 29 | 30 | 31 | // sets up the file loader 32 | 33 | var fileField = document.getElementById('file-field'); 34 | 35 | function getCopies( chroms ) { 36 | 37 | var copies = {}; 38 | copies.ordered_keys = []; 39 | 40 | for (var i = 0; i < chroms.length; i++) { 41 | 42 | var cname = chroms[i]; 43 | 44 | if ( copies[cname] === undefined ) { 45 | 46 | copies[cname] = [ i ]; 47 | copies.ordered_keys.push(cname); 48 | 49 | } else { 50 | 51 | copies[cname].push( i ); 52 | 53 | } 54 | 55 | } 56 | 57 | return copies; 58 | 59 | } 60 | 61 | self.gotFileData = function(data) { 62 | 63 | if (data.status === 'failed') { 64 | alert('ERROR:\n' + data.reason); 65 | interface.activate(); 66 | return false; 67 | } 68 | status.coordinates = data.crd; 69 | 70 | if (newFileFlag) { 71 | 72 | status.chroms = data.chroms; 73 | status.cstarts = data.cstarts; 74 | status.idx = data.idx; 75 | status.rad = data.rad; 76 | status.num_frames = data.n; 77 | status.num_beads = data.crd[0].length; 78 | status.num_chains = data.crd.length; 79 | status.copies = getCopies( status.chroms ); 80 | status.num_chrom = status.copies.ordered_keys.length; 81 | status.current_chains = []; 82 | 83 | for (var i = 0; i < status.num_chains; i++) { 84 | status.current_chains.push(i); 85 | } 86 | 87 | var original_palette = viewer.generatePalette( status.num_chrom ); 88 | status.palette = new Array(status.num_chains); 89 | 90 | for (var i = 0; i < status.num_chrom; i++) { 91 | var cname = status.copies.ordered_keys[i]; 92 | var copies = status.copies[ cname ]; 93 | var hsl = new THREE.Color(); 94 | original_palette[ i ].getHSL(hsl); 95 | var offset = 1.0; 96 | for (var j = 0; j < copies.length; j++) { 97 | 98 | var chain = copies[ j ]; 99 | var newhsl = { h: parseInt(hsl.h*360), s: parseInt(hsl.s*100.0), l: parseInt(hsl.l*offset*100.0) }; 100 | status.palette[ chain ] = new THREE.Color(`hsl(${newhsl.h},${newhsl.s}%,${newhsl.l}%)`); 101 | 102 | offset *= 0.66; 103 | 104 | } 105 | 106 | } 107 | 108 | interface.setChains( status ); 109 | 110 | 111 | } 112 | 113 | viewer.onWindowResize(); 114 | viewer.redraw(status); 115 | interface.activate(); 116 | interface.updateControls( status, 'traj' ); 117 | 118 | 119 | 120 | } 121 | 122 | self.requestData = function(fname, i) { 123 | 124 | interface.deactivate(); 125 | var req = { 126 | request: 'get_structure', 127 | path: fname, 128 | n : i 129 | }; 130 | console.log(req) 131 | $.ajax({ 132 | type: "POST", 133 | url: '/ajax/', 134 | data : { 135 | 'data' : JSON.stringify(req) 136 | }, 137 | success: self.gotFileData, 138 | dataType: 'json', 139 | error: function(x, err, et){ 140 | 141 | console.log(x, err, et); 142 | interface.showLoad(false); 143 | interface.activate() 144 | interface.errorMessage('failed'); 145 | 146 | }, 147 | }); 148 | 149 | } 150 | 151 | self.setFile = function(fname) { 152 | 153 | viewer.clear(); 154 | status.current_frame = 0; 155 | status.current_chain = -1; 156 | newFileFlag = true; 157 | status.fname = fname; 158 | 159 | interface.showLoad(false); 160 | 161 | self.requestData(fname, 0); 162 | interface.updateControls(status, 'traj'); 163 | 164 | } 165 | 166 | self.changeFrame = function( i ) { 167 | 168 | if ( i < 0 || i >= status.num_frames ) 169 | return false; 170 | 171 | 172 | newFileFlag = false; 173 | 174 | viewer.clear(); 175 | 176 | self.requestData(status.fname, i) 177 | 178 | status.current_frame = i; 179 | 180 | interface.updateControls(status, 'traj'); 181 | 182 | //viewer.redraw(status, true); 183 | 184 | } 185 | 186 | self.changeChain = function( i ) { 187 | 188 | if ( i < 0 || i >= status.num_chains ) 189 | return false; 190 | 191 | status.current_chain = i; 192 | 193 | interface.updateControls(status, 'all'); 194 | 195 | viewer.redraw(status, true); 196 | 197 | } 198 | 199 | self.updateView = function(request) { 200 | 201 | interface.deactivate(); 202 | 203 | if ( request.tube !== undefined ) 204 | viewer.tubeFlag = request.tube; 205 | 206 | if ( request.sphere !== undefined ) 207 | viewer.sphereFlag = request.sphere; 208 | 209 | if ( request.chain !== undefined ) { 210 | 211 | if ( request.chain === -1 ) { 212 | 213 | request.chain = []; 214 | for (var i = 0; i < status.num_chains; i++) { 215 | request.chain.push([i, true]); 216 | } 217 | 218 | } 219 | 220 | for (var i = 0; i < request.chain.length; i++) { 221 | 222 | var chain = request.chain[ i ][ 0 ]; 223 | var visible = request.chain[ i ][ 1 ]; 224 | var idx = status.current_chains.indexOf( chain ); 225 | 226 | if ( visible && idx === -1 ) { 227 | status.current_chains.push( chain ); 228 | } 229 | if ( !visible && idx !== -1 ) { 230 | status.current_chains.splice( idx, 1 ); 231 | } 232 | 233 | } 234 | 235 | } 236 | 237 | viewer.redraw(status); 238 | interface.activate(); 239 | 240 | } 241 | 242 | } 243 | 244 | -------------------------------------------------------------------------------- /igm/ui/static/js/genome_visualizer/interface.js: -------------------------------------------------------------------------------- 1 | 2 | var InterfaceControl = function(app) { 3 | 4 | var self = this; 5 | self.app = app; 6 | 7 | 8 | self.loadScreen = $('
'); 9 | self.loadScreen.append( $('
') ); 10 | self.loadScreen.append( $('') ); 11 | 12 | $('#structure-visualizer-tab').append(self.loadScreen); 13 | self.loadScreen.hide(); 14 | 15 | 16 | self.deactivate = function() { 17 | self.loadScreen.show() 18 | // disable all action while computing 19 | $("button#select-file-btn").prop('disabled', true); 20 | $("#download-link").off("click"); 21 | $(".toolbar-link").prop('disabled', true); 22 | $(".toolbar-link:not(.disabled)").addClass('disabled'); 23 | $("input, select").prop('disabled', true); 24 | $("#traj-next, #traj-prev").prop('disabled', true); 25 | 26 | } 27 | 28 | self.activate = function() { 29 | 30 | self.loadScreen.hide(); 31 | $("button#select-file-btn").prop('disabled', false); 32 | $(".toolbar-link").prop('disabled', false); 33 | $('.nav-link.disabled').removeClass("disabled"); 34 | $("input, select").prop('disabled', false); 35 | $("#traj-next, #traj-prev").prop('disabled', false); 36 | 37 | } 38 | 39 | self.updateControls = function(status, selector){ 40 | 41 | 42 | // trajectory controls 43 | if ( selector === 'all' || selector === 'traj' ) { 44 | 45 | if ( status.num_frames > 1 ) 46 | $('#trajectory-controls').show(); 47 | else 48 | $('#trajectory-controls').hide(); 49 | $('#traj-frame').html( 'Structure ' + ( status.current_frame + 1 ) + ' of ' + status.num_frames ); 50 | 51 | $('#structure-select-ctrl').val( status.current_frame ); 52 | $('#chromosome-ctrl').val( status.current_chain ); 53 | 54 | } 55 | 56 | } 57 | 58 | 59 | self.clearMessageBoard = function() { 60 | $("#result-info").html(""); 61 | } 62 | 63 | self.errorMessage = function(err, preformatted=false) { 64 | 65 | var ediv = ''; 66 | var html = preformatted ? '
'+ ediv + '
' : ediv; 67 | $("#result-info").append(html); 68 | 69 | } 70 | 71 | self.infoMessage = function(msg, preformatted=false) { 72 | 73 | var ediv = ''; 74 | var html = preformatted ? '
'+ ediv + '
' : ediv; 75 | $("#result-info").append(html); 76 | 77 | } 78 | 79 | 80 | self.setDownloadLink = function( data ) { 81 | 82 | var zip = new JSZip(); 83 | zip.file("output.txt", data.result.out); 84 | zip.file("err.txt", data.result.err); 85 | zip.file("log.txt", data.result.log); 86 | 87 | $("#download-link").on("click", function () { 88 | 89 | zip.generateAsync( {type: "blob"} ).then( function ( blob ) { // 1) generate the zip file 90 | 91 | saveAs(blob, "results.zip"); // 2) trigger the download 92 | 93 | }, function (err) { 94 | 95 | self.errorMessage(err, true); 96 | 97 | }); 98 | 99 | }); 100 | 101 | } 102 | 103 | 104 | self.setLogs = function( data ) { 105 | 106 | var parmtxt = JSON.stringify( data.param, null, 2 ); 107 | $('#log-prm').html( parmtxt ); 108 | $('#log-cmd').html( data.cmd ); 109 | $('#log-out').html( data.result.out ); 110 | $('#log-err').html( data.result.err ); 111 | 112 | } 113 | 114 | 115 | self.showLoad = function( yes=true ) { 116 | 117 | if (yes) 118 | $('#upload-dialog').modal('show'); 119 | else 120 | $('#upload-dialog').modal('hide') ; 121 | 122 | } 123 | 124 | self.setChains = function( status ) { 125 | 126 | var chains = status.chroms; 127 | var palette = status.palette; 128 | var chrctrl = $('#chromosome-ctrl-div'); 129 | var chromosomes = status.copies.ordered_keys; 130 | 131 | chrctrl.empty(); // remove old options 132 | 133 | var select = $('#chromosome-ctrl'); 134 | select.empty(); // remove old options 135 | select.append($("").attr("value", "-1") 136 | .text('-- all --')); 137 | 138 | for (var i = 0; i < chromosomes.length; i++) { 139 | 140 | select.append($("").attr("value", status.copies[ chromosomes[ i ] ]) 141 | .text(chromosomes[ i ])); 142 | 143 | } 144 | 145 | for (var i = 0; i < status.copies.ordered_keys.length; i++) { 146 | 147 | var cname = status.copies.ordered_keys[ i ]; 148 | var copies = status.copies[ cname ]; 149 | var inner_html = `
${cname}
`; 150 | 151 | for (var j = 0; j < copies.length; j++) { 152 | 153 | var chain = copies[ j ]; 154 | var color = '#' + palette[ chain ].getHexString(); 155 | 156 | inner_html += ` 157 |
158 |
159 | 163 |
164 |
165 | `; 166 | 167 | } 168 | 169 | chrctrl.append($("
").addClass("form-row").html(inner_html)); 170 | 171 | } 172 | 173 | $('input[name=chain_select]').on('change', function(event) { 174 | 175 | var caller = $(event.target) 176 | 177 | var selected = caller.prop('checked'); 178 | var chain_id = parseInt(caller.attr('chain')); 179 | 180 | self.app.updateView({ 181 | 182 | chain: [ [chain_id, selected] ] 183 | 184 | }); 185 | 186 | }); 187 | 188 | } 189 | 190 | 191 | 192 | } 193 | -------------------------------------------------------------------------------- /igm/ui/static/js/genome_visualizer/main.js: -------------------------------------------------------------------------------- 1 | var app = null; 2 | 3 | $(document).ready(function() { 4 | 5 | // setup the app 6 | app = new GenomeApp(); 7 | 8 | // actually disable links 9 | $(".toolbar-link.disabled").prop('disabled', true); 10 | 11 | // 12 | $("#bead-interval-invert").on("change", function() { 13 | 14 | app.changeBeadInterval(); 15 | 16 | }); 17 | 18 | $("#traj-next").on('click', function() { 19 | 20 | app.changeFrame( app.status.current_frame + 1 ); 21 | 22 | }); 23 | 24 | $("#traj-prev").on('click', function() { 25 | 26 | app.changeFrame( app.status.current_frame - 1 ); 27 | 28 | }); 29 | 30 | $("button#select-file-btn").click(function(){ 31 | 32 | app.setFile($('#file-field').val()); 33 | return false; 34 | 35 | }); 36 | 37 | 38 | $('#chromosome-ctrl').on('change', function(){ 39 | 40 | if ( $('#chromosome-ctrl').val() == "-1" ) { 41 | 42 | $('input[name=chain_select]').prop('checked', true); 43 | app.updateView({chain: -1}); 44 | 45 | } 46 | 47 | else if ( $('#chromosome-ctrl').val() == "-2" ) { 48 | 49 | app.status.current_chains = []; 50 | $('input[name=chain_select]').prop('checked', false); 51 | 52 | } 53 | 54 | else { 55 | 56 | var ids = $('#chromosome-ctrl').val().split(','); 57 | for (var i = 0; i < ids.length; i++) { 58 | 59 | ids[ i ] = parseInt(ids[ i ]); 60 | 61 | } 62 | 63 | var cc = []; 64 | for (var i = 0; i < app.status.num_chains; i++) { 65 | if ( ids.indexOf(i) !== -1 ) { 66 | cc.push([i, true]); 67 | $('input#chain_select_' + i).prop('checked', true); 68 | } else { 69 | cc.push([i, false]); 70 | $('input#chain_select_' + i).prop('checked', false); 71 | } 72 | } 73 | 74 | app.updateView({chain: cc}); 75 | 76 | } 77 | 78 | }); 79 | 80 | $('#structure-select-ctrl').on('change', function(){ 81 | 82 | app.changeFrame( parseInt( $('#structure-select-ctrl').val() ) - 1 ); 83 | 84 | }); 85 | 86 | $('#jobform').submit( function() { 87 | 88 | return false; 89 | 90 | }); 91 | 92 | $('#tube-box').on('change', function(){ 93 | 94 | app.updateView({ 95 | 96 | tube: $('#tube-box').prop('checked') 97 | 98 | }); 99 | 100 | }); 101 | 102 | $('#sphere-box').on('change', function(){ 103 | 104 | app.updateView({ 105 | 106 | sphere: $('#sphere-box').prop('checked') 107 | 108 | }); 109 | 110 | }); 111 | 112 | app.fileBrowser = new FileBrowser('#file-browser', function(path) { 113 | 114 | app.setFile(path); 115 | return false; 116 | 117 | }); 118 | 119 | }); 120 | -------------------------------------------------------------------------------- /igm/ui/static/js/genome_visualizer/util.js: -------------------------------------------------------------------------------- 1 | // deepcopy an object 2 | function deepcopy(obj){ 3 | return JSON.parse(JSON.stringify(obj)); 4 | } 5 | 6 | // python style range 7 | function range(start, end, increment) { 8 | var array = []; 9 | var current = start; 10 | 11 | increment = increment || 1; 12 | if (increment > 0) { 13 | while (current < end) { 14 | array.push(current); 15 | current += increment; 16 | } 17 | } else { 18 | while (current > end) { 19 | array.push(current); 20 | current += increment; 21 | } 22 | } 23 | return array; 24 | } 25 | -------------------------------------------------------------------------------- /igm/ui/static/js/genome_visualizer/visualizer.js: -------------------------------------------------------------------------------- 1 | var app = null; 2 | 3 | $(document).ready(function() { 4 | 5 | // setup the app 6 | app = new GenomeApp(); 7 | 8 | // actually disable links 9 | $(".toolbar-link.disabled").prop('disabled', true); 10 | 11 | // 12 | $("#bead-interval-invert").on("change", function() { 13 | 14 | app.changeBeadInterval(); 15 | 16 | }); 17 | 18 | $("#traj-next").on('click', function() { 19 | 20 | app.changeFrame( app.status.current_frame + 1 ); 21 | 22 | }); 23 | 24 | $("#traj-prev").on('click', function() { 25 | 26 | app.changeFrame( app.status.current_frame - 1 ); 27 | 28 | }); 29 | 30 | $("button#select-file-btn").click(function(){ 31 | 32 | app.setFile($('#file-field').val()); 33 | return false; 34 | 35 | }); 36 | 37 | 38 | window.scrollTo(0, 0); 39 | 40 | $('#chromosome-ctrl').on('change', function(){ 41 | 42 | if ( $('#chromosome-ctrl').val() == "-1" ) { 43 | 44 | $('input[name=chain_select]').prop('checked', true); 45 | app.updateView({chain: -1}); 46 | 47 | } 48 | 49 | else if ( $('#chromosome-ctrl').val() == "-2" ) { 50 | 51 | app.status.current_chains = []; 52 | $('input[name=chain_select]').prop('checked', false); 53 | 54 | } 55 | 56 | else { 57 | 58 | var ids = $('#chromosome-ctrl').val().split(','); 59 | for (var i = 0; i < ids.length; i++) { 60 | 61 | ids[ i ] = parseInt(ids[ i ]); 62 | 63 | } 64 | 65 | var cc = []; 66 | for (var i = 0; i < app.status.num_chains; i++) { 67 | if ( ids.indexOf(i) !== -1 ) { 68 | cc.push([i, true]); 69 | $('input#chain_select_' + i).prop('checked', true); 70 | } else { 71 | cc.push([i, false]); 72 | $('input#chain_select_' + i).prop('checked', false); 73 | } 74 | } 75 | 76 | app.updateView({chain: cc}); 77 | 78 | } 79 | 80 | }); 81 | 82 | // show the file dialog as soon as the page is ready. 83 | $('#upload-dialog').modal("show"); 84 | 85 | $('#structure-select-ctrl').on('change', function(){ 86 | 87 | app.changeFrame( parseInt( $('#structure-select-ctrl').val() ) - 1 ); 88 | 89 | }); 90 | 91 | $('#jobform').submit( function() { 92 | 93 | return false; 94 | 95 | }); 96 | 97 | $('#tube-box').on('change', function(){ 98 | 99 | app.updateView({ 100 | 101 | tube: $('#tube-box').prop('checked') 102 | 103 | }); 104 | 105 | }); 106 | 107 | $('#sphere-box').on('change', function(){ 108 | 109 | app.updateView({ 110 | 111 | sphere: $('#sphere-box').prop('checked') 112 | 113 | }); 114 | 115 | }); 116 | 117 | app.fileBrowser = new FileBrowser('#file-browser', function(path) { 118 | 119 | app.setFile(path); 120 | return false; 121 | }); 122 | 123 | }); 124 | 125 | -------------------------------------------------------------------------------- /igm/ui/static_pages.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /igm/ui/templates/base.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | {% block title %}Default title{% end %} 5 | 6 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | {% block head %} {% end %} 17 | 18 | 19 | 20 | {% block body %}Nothing to show{% end %} 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /igm/ui/templates/cfg_form.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block head %} 4 | 10 | 11 | 57 | 58 | {% end %} 59 | 60 | {% block title %}Generation status{% end %} 61 | 62 | {% block body %} 63 |

64 | IGM run configuration 65 |

66 | 67 |
68 | {% raw form_data %} 69 | 70 |
71 | {% end %} -------------------------------------------------------------------------------- /igm/ui/templates/history.html: -------------------------------------------------------------------------------- 1 | {% extends "base.html" %} 2 | 3 | {% block head %}{% end %} 4 | 5 | {% block title %}Generation status{% end %} 6 | 7 | {% block body %} 8 |

9 | Job history 10 |

11 |

12 | Directory: {{ directory }}
13 | Configuration file: {{ cfg_fname }} 14 |

15 |

16 | Cluster status: 17 |

18 |

19 | workers: {{ cstatus[0] }}, ({{ cstatus[2] }} active, {{ cstatus[3] }} idle). queued jobs: {{ cstatus[1] }} 20 |

21 | 22 | 23 | 26 | 29 | 32 | 35 | 38 | 41 | 42 | 43 | 44 | {% for h in history %} 45 | 47 | 50 | 53 | 56 | 59 | 62 | 65 | 66 | {% end %} 67 | 68 | 69 |
24 | Step 25 | 27 | Name 28 | 30 | Time started 31 | 33 | Time consumed 34 | 36 | Total time 37 | 39 | Status 40 |
48 | {{ h['step_no'] }} 49 | 51 | {{ h['name'] }} 52 | 54 | {{ h['strtime'] }} 55 | 57 | {{ h['consumed'] }} 58 | 60 | +{{ h['elapsed'] }} 61 | 63 | {{ h['status'] }} 64 |
70 | 71 | {% end %} -------------------------------------------------------------------------------- /igm/ui/views.py: -------------------------------------------------------------------------------- 1 | from ipyparallel import Client 2 | import os, os.path 3 | from tornado import template 4 | from ..core.config import Config 5 | from ..core.defaults.defaults import igm_options 6 | from ..core.job_tracking import StepDB 7 | from ..utils.log import pretty_time 8 | from .utils import generate_form, parse_post_config 9 | import time 10 | import igm 11 | 12 | template_dir = os.path.join( os.path.dirname( os.path.abspath(__file__) ), 'templates' ) 13 | loader = template.Loader(template_dir) 14 | 15 | def render(template, data): 16 | return loader.load(template).generate(**data) 17 | 18 | 19 | 20 | 21 | def cluster_status(): 22 | try: 23 | rcl = Client() 24 | nworkers = len(rcl[:]) 25 | qstat = rcl.queue_status() 26 | queued = qstat[u'unassigned'] 27 | working = sum([ qstat[w][u'tasks'] for w in rcl.ids ]) 28 | idle = nworkers - working 29 | rcl.close() 30 | except: 31 | nworkers, queued, working, idle = 0, 0, 0, 0 32 | return nworkers, queued, working, idle 33 | 34 | def history(cfgf): 35 | 36 | cfgf = os.path.abspath(cfgf) 37 | cfg = Config(cfgf) 38 | try: 39 | db = StepDB(cfg, mode='r') 40 | h = db.get_history() 41 | except OSError: 42 | h = [] 43 | for i in range(len(h)): 44 | h[i]['elapsed'] = pretty_time(h[i]['time'] - h[0]['time']) 45 | h[i]['step_no'] = i 46 | h[i]['consumed'] = pretty_time(h[i]['time'] - h[max(i-1, 0)]['time']) 47 | h[i]['strtime'] = time.strftime('%c', time.localtime(h[i]['time'])) 48 | 49 | return render( 'history.html', { 50 | 'history': h, 51 | 'directory': os.path.dirname(cfgf), 52 | 'cfg_fname': os.path.basename(cfgf), 53 | 'cstatus': cluster_status(), 54 | }) 55 | 56 | def config_form(): 57 | form_data = generate_form(igm_options) 58 | return render( 'cfg_form.html', { 59 | 'form_data': form_data 60 | }) 61 | 62 | import traceback 63 | def config_form_process(post): 64 | try: 65 | out = parse_post_config(igm_options, post, 'igm') 66 | except: 67 | out = '
' + traceback.format_exc()
68 |         out += str(post)
69 |         out += '
' 70 | return out 71 | 72 | def index(): 73 | schema_file = os.path.join( 74 | os.path.dirname( os.path.abspath(igm.__file__) ), 75 | 'config_schema.json' 76 | ) 77 | return render( 'index.html', { 78 | 'schema': open(schema_file).read() 79 | }) 80 | -------------------------------------------------------------------------------- /igm/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | from .actdist import get_actdist 4 | from .files import HmsFile 5 | 6 | from .log import SetupLogging, logger 7 | 8 | -------------------------------------------------------------------------------- /igm/utils/actdist.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | 3 | try: 4 | from itertools import izip as zip 5 | except ImportError: 6 | pass 7 | 8 | def cleanProbability(pij, pexist): 9 | if pexist < 1: 10 | pclean = (pij - pexist) / (1.0 - pexist) 11 | else: 12 | pclean = pij 13 | return max(0, pclean) 14 | 15 | def get_actdist(i, j, pwish, plast, hss, contactRange=2, option=0): 16 | ''' 17 | Serial function to compute the activation distances for a pair of loci 18 | It expects some variables to be defined in its scope: 19 | 20 | Parameters 21 | ---------- 22 | i, j : int 23 | index of the first, second locus 24 | pwish : float 25 | target contact probability 26 | plast : float 27 | the last refined probability 28 | hss : alabtools.analysis.HssFile 29 | file containing coordinates 30 | contactRange : int 31 | contact range of sum of radius of beads 32 | option : int 33 | calculation option: 34 | (0) intra chromosome contacts are considered intra 35 | (1) intra chromosome contacts are assigned intra/inter equally 36 | Returns 37 | ------- 38 | i (int) 39 | j (int) 40 | ad (float): the activation distance 41 | p (float): the corrected probability 42 | ''' 43 | 44 | # import here in case is executed on a remote machine 45 | import numpy as np 46 | 47 | if (i==j): 48 | return [] 49 | 50 | n_struct = hss.get_nstruct() 51 | copy_index = hss.get_index().copy_index 52 | chrom = hss.get_index().chrom 53 | 54 | ii = copy_index[i] 55 | jj = copy_index[j] 56 | 57 | n_combinations = len(ii) * len(jj) 58 | n_possible_contacts = min(len(ii), len(jj)) 59 | #for diploid cell n_combinations = 2*2 =4 60 | #n_possible_contacts = 2 61 | 62 | radii = hss.get_radii() 63 | ri, rj = radii[ii[0]], radii[jj[0]] 64 | 65 | d_sq = np.empty((n_combinations, n_struct)) 66 | 67 | it = 0 68 | for k in ii: 69 | for m in jj: 70 | x = hss.get_bead_crd(k) 71 | y = hss.get_bead_crd(m) 72 | d_sq[it] = np.sum(np.square(x - y), axis=1) 73 | it += 1 74 | #= 75 | 76 | rcutsq = np.square(contactRange * (ri + rj)) 77 | d_sq.sort(axis=0) 78 | 79 | contact_count = np.count_nonzero(d_sq[0:n_possible_contacts, :] <= rcutsq) 80 | pnow = float(contact_count) / (n_possible_contacts * n_struct) 81 | sortdist_sq = np.sort(d_sq[0:n_possible_contacts, :].ravel()) 82 | 83 | t = cleanProbability(pnow, plast) 84 | p = cleanProbability(pwish, t) 85 | 86 | res = [] 87 | if p>0: 88 | o = min(n_possible_contacts * n_struct - 1, 89 | int(round(n_possible_contacts * p * n_struct))) 90 | activation_distance = np.sqrt(sortdist_sq[o]) 91 | 92 | if (chrom[i] == chrom[j]) and (option == 0): 93 | res = [(i0, i1, activation_distance, p) for i0,i1 in zip(ii,jj)] 94 | else: 95 | res = [(i0, i1, activation_distance, p) for i0 in ii for i1 in jj] 96 | return res 97 | -------------------------------------------------------------------------------- /igm/utils/emails.py: -------------------------------------------------------------------------------- 1 | from .log import logger 2 | from email.message import EmailMessage 3 | import smtplib 4 | import traceback 5 | from ..core.config import Config 6 | 7 | ''' 8 | Works only with localhost for now. Using other smtp will require to setup the connection, with its own security issues 9 | "smtp" : { 10 | "label" : "SMTP server", 11 | "dtype" : "str", 12 | "role" : "optional-input", 13 | "blank": true, 14 | "description" : "SMTP server to use. If not specified, localhost is used" 15 | } 16 | ''' 17 | 18 | def send_email(to, sender=None, smtpserver='localhost', subject="", content=""): 19 | try: 20 | if isinstance(to, (Config, dict)): 21 | cfg = to 22 | to = cfg['address'] 23 | sender = cfg.get('from', sender) 24 | smtpserver = cfg.get('smtp', smtpserver) 25 | msg = EmailMessage() 26 | msg.set_content(content) 27 | msg['Subject'] = subject 28 | if sender is None: 29 | sender = to 30 | msg['From'] = sender 31 | msg['To'] = to 32 | with smtplib.SMTP(smtpserver) as s: 33 | s.send_message(msg) 34 | 35 | except: 36 | logger.error('Cannot send email') 37 | logger.error(traceback.format_exc()) 38 | -------------------------------------------------------------------------------- /igm/utils/files.py: -------------------------------------------------------------------------------- 1 | from __future__ import division, print_function 2 | import numpy as np 3 | import h5py 4 | from ..model import Model 5 | from ..restraints.restraint import Restraint 6 | __hms_version__ = 1 7 | 8 | class HmsFile(h5py.File): 9 | """ 10 | h5py.File like object for hms model structure files. 11 | """ 12 | 13 | def __init__(self, *args, **kwargs): 14 | h5py.File.__init__(self, *args, **kwargs) 15 | try: 16 | self._version = self.attrs['version'] 17 | except(KeyError): 18 | self._version = __hms_version__ 19 | self.attrs.create('version', __hms_version__, dtype='int32') 20 | try: 21 | self._nbead = self.attrs['nbead'] 22 | except(KeyError): 23 | self._nbead = 0 24 | self.attrs.create('nbead', 0, dtype='int32') 25 | try: 26 | self._struct_id = self.attrs['struct_id'] 27 | except(KeyError): 28 | self._struct_id = 0 29 | self.attrs.create('struct_id', 0, dtype='int32') 30 | 31 | try: 32 | self._restraints_grp = self['/restraints/'] 33 | except(KeyError): 34 | self._restraints_grp = self.create_group("restraints") 35 | #=== 36 | def get_version(self): 37 | return self._version 38 | 39 | def get_nbead(self): 40 | return self._nbead 41 | 42 | def get_struct_id(self): 43 | return self._struct_id 44 | 45 | def get_coordinates(self, read_to_memory=True): 46 | 47 | ''' 48 | Parameters 49 | ---------- 50 | read_to_memory (bool) : 51 | If True (default), the coordinates will be read and returned 52 | as a numpy.ndarray. If False, a h5py dataset object will be 53 | returned. In the latter case, note that the datased is valid 54 | only while the file is open. 55 | ''' 56 | 57 | if read_to_memory: 58 | return self['coordinates'][:] 59 | return self['coordinates'] 60 | 61 | def get_total_restraints(self): 62 | total = 0 63 | for r in self._restraints_grp.keys(): 64 | total += self._restraints_grp[r].attrs["total"] 65 | return total 66 | 67 | def get_total_violations(self): 68 | total = 0 69 | for r in self._restraints_grp.keys(): 70 | total += self._restraints_grp[r].attrs["nvios"] 71 | return total 72 | 73 | def set_nbead(self, n): 74 | self.attrs['nbead'] = self._nbead = n 75 | 76 | def set_struct_id(self, n): 77 | self.attrs['struct_id'] = self._struct_id = n 78 | 79 | def saveModel(self, struct_id, model): 80 | assert isinstance(model, Model), "Parameter has to be a Model instance" 81 | 82 | crd = model.getCoordinates() 83 | self.saveCoordinates(struct_id, crd) 84 | 85 | def saveCoordinates(self, struct_id, crd): 86 | if "coordinates" in self: 87 | self["coordinates"][...] = crd 88 | else: 89 | self.create_dataset("coordinates", data = crd, 90 | chunks=True, compression="gzip") 91 | 92 | self.set_nbead(len(crd)) 93 | self.set_struct_id(struct_id) 94 | 95 | def saveViolations(self, restraint, tolerance=0.05): 96 | assert isinstance(restraint, Restraint), "Parameter has to be a Restraint instance" 97 | 98 | violations, ratios = restraint.get_violations(tolerance) 99 | name = type(restraint).__name__ 100 | 101 | if name in self._restraints_grp: 102 | self._restraints_grp[name]["violations"] = np.array(violations).astype("S50") 103 | self._restraints_grp[name]["ratios"] = np.array(ratios) 104 | self._restraints_grp[name].attrs["total"] = len(restraint.forceID) 105 | self._restraints_grp[name].attrs["nvios"] = len(ratios) 106 | else: 107 | subgrp = self._restraints_grp.create_group(name) 108 | subgrp.create_dataset("violations", data = np.array(violations).astype("S50"), 109 | chunks = True, compression = "gzip") 110 | subgrp.create_dataset("ratios", data = np.array(ratios), 111 | chunks = True, compression = "gzip") 112 | subgrp.attrs.create("total", len(restraint.forceID), dtype='int32') 113 | subgrp.attrs.create("nvios", len(ratios), dtype='int32') 114 | #- 115 | return len(ratios) 116 | 117 | def make_absolute_path(path, basedir='.'): 118 | import os.path 119 | if os.path.isabs(path): 120 | return path 121 | return os.path.abspath( os.path.join(basedir, path) ) 122 | 123 | def h5_create_group_if_not_exist(root, groupname): 124 | if groupname in root: 125 | return root[groupname] 126 | else: 127 | return root.create_group(groupname) 128 | 129 | def h5_create_or_replace_dataset(root, dataname, data, **kwargs): 130 | if dataname in root: 131 | root[dataname][...] = data 132 | else: 133 | root.create_dataset(dataname, data=data, **kwargs) 134 | return root[dataname] 135 | -------------------------------------------------------------------------------- /igm/utils/kernel_testing.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | 4 | from igm.model import * 5 | from igm.model.kernel.lammps_model import LammpsModel 6 | from igm.steps.ModelingStep import ModelingStep 7 | from igm.core import Config 8 | from igm.core.job_tracking import StepDB 9 | from alabtools.analysis import HssFile 10 | from igm.restraints import Polymer, Envelope, Steric, HiC, Sprite 11 | from igm.utils import HmsFile 12 | 13 | def debug_minimization(cfg, struct_id, rname, **kwargs): 14 | if not isinstance(cfg, dict): 15 | cfg = Config(cfg) 16 | if os.path.isfile(cfg['step_db']): 17 | db = StepDB(cfg) 18 | h = db.get_history() 19 | cfg.update(h[-1]) 20 | 21 | cfg['optimization']['optimizer_options'].update(kwargs) 22 | cfg['optimization']['keep_temporary_files'] = True 23 | 24 | step_id = rname 25 | 26 | hssfilename = cfg['structure_output'] 27 | 28 | #read index, radii, coordinates 29 | with HssFile(hssfilename,'r') as hss: 30 | index = hss.index 31 | radii = hss.radii 32 | if cfg.get('random_shuffling', False): 33 | crd = generate_random_in_sphere(radii, cfg['model']['nucleus_radius']) 34 | else: 35 | crd = hss.get_struct_crd(struct_id) 36 | 37 | #init Model 38 | model = Model(uid=struct_id) 39 | 40 | # get the chain ids 41 | chain_ids = np.concatenate( [ [i]*s for i, s in enumerate(index.chrom_sizes) ] ) 42 | 43 | #add particles into model 44 | n_particles = len(crd) 45 | for i in range(n_particles): 46 | model.addParticle(crd[i], radii[i], Particle.NORMAL, chainID=chain_ids[i]) 47 | 48 | #========Add restraint 49 | monitored_restraints = [] 50 | 51 | #add excluded volume restraint 52 | ex = Steric(cfg['model']['evfactor']) 53 | model.addRestraint(ex) 54 | 55 | #add nucleus envelop restraint 56 | if cfg['model']['nucleus_shape'] == 'sphere': 57 | ev = Envelope(cfg['model']['nucleus_shape'], 58 | cfg['model']['nucleus_radius'], 59 | cfg['model']['contact_kspring']) 60 | elif cfg['model']['nucleus_shape'] == 'ellipsoid': 61 | ev = Envelope(cfg['model']['nucleus_shape'], 62 | cfg['model']['nucleus_semiaxes'], 63 | cfg['model']['contact_kspring']) 64 | else: 65 | raise NotImplementedError('Invalid nucleus shape') 66 | model.addRestraint(ev) 67 | 68 | #add consecutive polymer restraint 69 | pp = Polymer(index, 70 | cfg['model']['contact_range'], 71 | cfg['model']['contact_kspring']) 72 | model.addRestraint(pp) 73 | monitored_restraints.append(pp) 74 | 75 | #add Hi-C restraint 76 | # if "Hi-C" in cfg['restraints']: 77 | # dictHiC = cfg['restraints']['Hi-C'] 78 | # actdist_file = cfg['runtime']['Hi-C']['actdist_file'] 79 | # contact_range = dictHiC.get( 'contact_range', 2.0 ) 80 | # k = dictHiC.get( 'contact_kspring', 1.0 ) 81 | 82 | # hic = HiC(actdist_file, contact_range, k) 83 | # model.addRestraint(hic) 84 | # monitored_restraints.append(hic) 85 | 86 | # if "sprite" in cfg['restraints']: 87 | # sprite_opt = cfg['restraints']['sprite'] 88 | # sprite = Sprite( 89 | # sprite_opt['assignment_file'], 90 | # sprite_opt['volume_fraction'], 91 | # struct_id, 92 | # sprite_opt['kspring'] 93 | # ) 94 | # model.addRestraint(sprite) 95 | # monitored_restraints.append(sprite) 96 | 97 | #========Optimization 98 | #optimize model 99 | cfg['runtime']['run_name'] = rname 100 | model.optimize(cfg) 101 | 102 | tol = cfg.get('violation_tolerance', 0.01) 103 | lockfile = os.path.join('.', '%s.%d.ready' % (step_id, struct_id) ) 104 | with FileLock(lockfile): 105 | open(lockfile, 'w').close() # touch the ready-file 106 | ofname = os.path.join('.', 'mstep_%d.hms' % struct_id) 107 | with HmsFile(ofname, 'w') as hms: 108 | hms.saveModel(struct_id, model) 109 | 110 | for r in monitored_restraints: 111 | hms.saveViolations(r, tolerance=tol) 112 | 113 | # double check it has been written correctly 114 | with HmsFile(ofname, 'r') as hms: 115 | if np.all( hms.get_coordinates() == model.getCoordinates() ): 116 | raise RuntimeError('error writing the file %s' % ofname) 117 | -------------------------------------------------------------------------------- /igm/utils/log.py: -------------------------------------------------------------------------------- 1 | from time import time, strftime, localtime 2 | import sys 3 | import logging 4 | import os.path 5 | 6 | # clear root logger handler 7 | FORMAT = '(%(name)s) %(asctime)-15s [%(levelname)s] %(message)s' 8 | logging.basicConfig(format=FORMAT, level=logging.INFO) 9 | logger = logging.getLogger('IGM') 10 | 11 | class bcolors: 12 | HEADER = '\033[95m' 13 | OKBLUE = '\033[94m' 14 | OKGREEN = '\033[92m' 15 | WARNING = '\033[93m' 16 | FAIL = '\033[91m' 17 | ENDC = '\033[0m' 18 | BOLD = '\033[1m' 19 | UNDERLINE = '\033[4m' 20 | 21 | def SetupLogging(cfg): 22 | if 'log' in cfg['parameters']: 23 | loglevel=logging.INFO 24 | set_log(cfg['parameters']['log'], loglevel=loglevel) 25 | 26 | 27 | def set_log(fname, loglevel=logging.INFO): 28 | 29 | fname = os.path.abspath(fname) 30 | 31 | # ensure the file is not double added 32 | hs = [h.baseFilename for h in logger.handlers if isinstance(h, logging.FileHandler)] 33 | if fname in hs: 34 | return 35 | 36 | fh = logging.FileHandler(fname) 37 | fh.setFormatter(logging.Formatter(FORMAT)) 38 | logger.addHandler(fh) 39 | 40 | 41 | def pretty_time(seconds): 42 | ''' 43 | Prints the *seconds* in the format h mm ss 44 | ''' 45 | seconds = int(seconds) 46 | m, s = divmod(seconds, 60) 47 | h, m = divmod(m, 60) 48 | return "%dh %02dm %02ds" % (h, m, s) 49 | 50 | def print_progress(iterable, 51 | length=None, 52 | every=1, 53 | timeout=None, 54 | size=12, 55 | fmt='(IGM) {bar} {percent:6.2f}% ({completed}/{total}) | {elapsed:12s} | ETA: {remaining:12s}', 56 | timefmt='%c', 57 | fd=sys.stdout ): 58 | 59 | if hasattr(iterable, '__len__') and length is None: 60 | length = len(iterable) 61 | 62 | last = 0 63 | fill = 0 64 | if hasattr(fd, 'isatty') and (not fd.isatty()): 65 | fd.write('0 |' + ' ' * size + '| 100\n |') 66 | fd.flush() 67 | lastfill = fill 68 | start = time() 69 | for i, v in enumerate(iterable): 70 | if fd.isatty(): 71 | print_flag = ( i == 0 ) 72 | if timeout is not None and time() - last > timeout: 73 | last = time() 74 | print_flag = True 75 | elif every is not None and (i+1) % every == 0: 76 | print_flag = True 77 | elif length is not None and i == length - 1: 78 | print_flag = True 79 | 80 | if print_flag: 81 | vals = {} 82 | vals['completed'] = i + 1 83 | vals['total'] = length 84 | if length is not None: 85 | vals['percent'] = (i + 1) * 100.0 / length 86 | fill = int(size * float(i+1) / length) 87 | else: 88 | vals['percent'] = 0.0 89 | fill += 1 90 | 91 | now = time() 92 | elapsed = now - start 93 | vals['elapsed'] = pretty_time(elapsed) 94 | if i == 0 or length is None: 95 | vals['remaining'] = 'N/A' 96 | vals['eta'] = 'N/A' 97 | else: 98 | remaining = elapsed / i * ( length - i ) 99 | vals['remaining'] = pretty_time(remaining) 100 | eta = now + remaining 101 | vals['eta'] = strftime(timefmt, localtime(eta)) 102 | 103 | if length: 104 | pb = '[' + '=' * fill + ' ' * (size-fill) + '] ' 105 | else: 106 | pos = fill % (2*(size-1)) 107 | if pos >= size: 108 | pos = 2*size - pos - 2 109 | pb = '[' + ' ' * (pos) + '=' + ' ' * (size-pos-1) + '] ' 110 | vals['bar'] = pb 111 | fd.write( '\r' + fmt.format(**vals) ) 112 | fd.flush() 113 | else: 114 | if length: 115 | fill = ( (i + 1) * size ) // length 116 | if fill > lastfill: 117 | lastfill = fill 118 | fd.write('=') 119 | 120 | yield v 121 | 122 | if not fd.isatty(): 123 | fd.write('|') 124 | fd.write('\n') 125 | fd.flush() 126 | -------------------------------------------------------------------------------- /igm/utils/sprite/sprite.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import h5py 3 | 4 | SPRITE_H5_CHUNKSIZE = 1000000 5 | 6 | def sprite_clusters_txt_to_h5(textfile, outfile): 7 | with h5py.File(outfile, 'w') as h5f: 8 | indptr = [ 0 ] 9 | lastwrite = 0 10 | h5f.create_dataset('data', shape=(0,), maxshape=(None,), chunks=(SPRITE_H5_CHUNKSIZE,) , dtype=np.int32) 11 | data = np.empty(shape=(0,), dtype=np.int32) 12 | for line in open(textfile, 'r'): 13 | cluster = np.array([ int(x) for x in line.split()[1:] ], dtype=np.int32) 14 | indptr.append(indptr[-1] + len(cluster)) 15 | data = np.concatenate([data, cluster]) 16 | if indptr[-1] - lastwrite > SPRITE_H5_CHUNKSIZE: 17 | h5f['data'].resize( (indptr[-1], ) ) 18 | h5f['data'][lastwrite:] = data 19 | lastwrite = indptr[-1] 20 | data = np.empty(shape=(0,), dtype=np.int32) 21 | 22 | h5f['data'].resize( (indptr[-1], ) ) 23 | h5f['data'][lastwrite:] = data 24 | h5f.create_dataset('indptr', data=indptr, dtype=np.int32) 25 | 26 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from distutils.core import setup, Extension 3 | from Cython.Build import cythonize 4 | import numpy 5 | #from setuptools import setup, find_packages 6 | import re, os 7 | def find_packages(path='.'): 8 | ret = [] 9 | for root, dirs, files in os.walk(path): 10 | if '__init__.py' in files: 11 | ret.append(re.sub('^[^A-z0-9_]+', '', root.replace('/', '.'))) 12 | return ret 13 | 14 | install_requires = [ 15 | 'numpy>=1.9', 16 | 'scipy>=0.16', 17 | 'h5py>=2.5', 18 | 'alabtools>=0.0.1', 19 | 'tqdm', 'tornado', 20 | 'cloudpickle', 21 | 'cython' 22 | ] 23 | 24 | tests_require = [ 25 | 'mock' 26 | ] 27 | 28 | 29 | extras_require = { 30 | 'docs': [ 31 | 'Sphinx>=1.1', 32 | ] 33 | } 34 | 35 | extensions = [ 36 | Extension("igm.cython_compiled.sprite", 37 | ["igm/cython_compiled/sprite.pyx", "igm/cython_compiled/cpp_sprite_assignment.cpp"]), 38 | ] 39 | 40 | extensions = cythonize(extensions) 41 | 42 | setup( 43 | name = 'igm', 44 | version = '1.0.0', 45 | author = 'Guido Polles, Nan Hua', 46 | author_email = 'polles@usc.edu nhua@usc.edu', 47 | url = 'https://github.com/alberlab/igm', 48 | description = 'Integrated Genome Modeling', 49 | 50 | packages=find_packages('./igm/'), 51 | package_data={'igm.core' : ['defaults/*.json'], 52 | 'igm.ui': ['static/css/*', 'static/js/*', 'static/js/genome_visualizer/*', 'templates/*']}, 53 | #package_data={'' : ['core/defaults/*', 'ui/static/css/*', 'ui/static/js/*', 'ui/static/js/genome_visualizer/*', 'ui/templates/*']}, 54 | include_package_data=True, 55 | install_requires=install_requires, 56 | tests_require=tests_require, 57 | extras_require=extras_require, 58 | 59 | ext_modules=extensions, 60 | include_dirs=[numpy.get_include()], 61 | scripts=['bin/igm-run', 'bin/igm-server', 'bin/igm-register-dir', 'bin/igm-info-dir', 62 | 'bin/igm-unregister-dir', 'bin/igm-stop', 'bin/igm-report'], 63 | ) 64 | --------------------------------------------------------------------------------