├── .bumpversion.cfg ├── .editorconfig ├── .github ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── test.yml ├── .gitignore ├── .travis.yml ├── API.md ├── CHANGELOG.md ├── LICENSE ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── config.json.sample ├── docker-context ├── Dockerfile ├── default-viewconf-fixture.xml ├── hgserver_nginx.conf ├── httpfs.sh ├── nginx.conf ├── supervisord.conf ├── uwsgi.ini └── uwsgi_params ├── environment.yml ├── fragments ├── __init__.py ├── app.py ├── drf_disable_csrf.py ├── exceptions.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_auto_20170406_2322.py │ └── __init__.py ├── tests.py ├── urls.py ├── utils.py └── views.py ├── higlass_server ├── __init__.py ├── settings.py ├── settings_test.py ├── tests.py ├── urls.py ├── utils.py └── wsgi.py ├── log └── .keep ├── manage.py ├── managedb.sh ├── nginx ├── nginx.conf └── sites-enabled │ └── hgserver_nginx.conf ├── notebooks ├── .ipynb_checkpoints │ └── benchmarks-checkpoint.ipynb ├── Link unfurling.ipynb ├── Register url test.ipynb ├── Rename resolutions.ipynb └── stuff.ipynb ├── package.json ├── requirements-dev.txt ├── requirements.txt ├── scripts ├── __init__.py ├── add_attr_to_hdf5.py ├── benchmark_server.py ├── format_upload_command.py └── test_aws_bigWig_fetch.py ├── start.sh ├── test.sh ├── tilesets ├── __init__.py ├── admin.py ├── apps.py ├── chromsizes.py ├── exceptions.py ├── generate_tiles.py ├── json_schemas.py ├── management │ ├── __init__.py │ └── commands │ │ ├── __init__.py │ │ ├── delete_tileset.py │ │ ├── ingest_tileset.py │ │ ├── list_tilesets.py │ │ └── modify_tileset.py ├── migrations │ ├── 0001_initial.py │ ├── 0002_auto_20170223_1629.py │ ├── 0003_viewconf_higlassversion.py │ ├── 0004_auto_20181115_1744.py │ ├── 0005_auto_20181127_0239.py │ ├── 0006_tileset_description.py │ ├── 0007_auto_20181127_0254.py │ ├── 0008_auto_20181129_1304.py │ ├── 0009_tileset_temporary.py │ ├── 0010_auto_20181228_2250.py │ ├── 0011_auto_20181228_2252.py │ ├── 0012_auto_20190923_0257.py │ ├── 0013_auto_20211119_1935.py │ ├── 0014_auto_20211119_1939.py │ └── __init__.py ├── models.py ├── permissions.py ├── serializers.py ├── storage.py ├── suggestions.py ├── test_data │ └── hi.txt ├── tests.py ├── urls.py └── views.py ├── unit_tests.sh ├── update.sh ├── uwsgi_params └── website ├── tests.py ├── urls.py └── views.py /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 0.1.0 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | indent_style = space 11 | indent_size = 2 12 | 13 | # Matches multiple files with brace expansion notation 14 | # Set default charset 15 | [*.{js,py}] 16 | charset = utf-8 17 | 18 | # 4 space indentation 19 | [*.py] 20 | indent_style = space 21 | indent_size = 4 22 | 23 | # Tab indentation (no size specified) 24 | [Makefile] 25 | indent_style = tab 26 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | What was changed in this pull request? 4 | 5 | Why is it necessary? 6 | 7 | Fixes #\_\_\_ 8 | 9 | ## Checklist 10 | 11 | - [ ] Unit tests added or updated 12 | - [ ] Updated CHANGELOG.md 13 | - [ ] Run `black .` 14 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install Python dependencies, run tests and lint with a variety of Python versions 2 | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions 3 | 4 | name: Python package 5 | 6 | on: 7 | push: 8 | branches: [ develop, master ] 9 | pull_request: 10 | branches: [ develop, master ] 11 | 12 | jobs: 13 | build: 14 | 15 | runs-on: ubuntu-latest 16 | strategy: 17 | matrix: 18 | python-version: ['3.7', '3.8'] 19 | 20 | steps: 21 | - uses: actions/checkout@v2 22 | - name: Set up Python ${{ matrix.python-version }} 23 | uses: actions/setup-python@v2 24 | with: 25 | python-version: ${{ matrix.python-version }} 26 | - name: Install dependencies 27 | run: | 28 | python -m pip install --upgrade pip 29 | python -m pip install flake8 pytest 30 | if [ -f requirements.txt ]; then pip install -r requirements.txt; fi 31 | if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi 32 | - name: Lint with flake8 33 | run: | 34 | # stop the build if there are Python syntax errors or undefined names 35 | flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics 36 | # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide 37 | flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics 38 | - name: Test with pytest 39 | run: | 40 | ./test.sh 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | COMMANDS 2 | 3 | tmp 4 | notebooks/.ipynb_checkpoints 5 | *.sqllite 6 | api/coolers/migrations/* 7 | *.pyc 8 | *~ 9 | data/ 10 | real-data/ 11 | log/ 12 | *.sqlite3 13 | *.swp 14 | *.swo 15 | *.swn 16 | *.swm 17 | media/ 18 | .DS_Store 19 | .idea/ 20 | dump.rdb 21 | src 22 | # TODO: It looks like the build creates this directory? Not what I'd expect. 23 | 24 | npm-debug.log 25 | 26 | uploads/ 27 | static/ 28 | hgs-static/ 29 | 30 | config.json 31 | .envrc -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # Based on http://lmjohns3.com/2015/06/using-travis-ci-with-miniconda-scipy-and-nose.html 2 | # Tweaked to specify versions on everything for stability. 3 | services: 4 | - docker 5 | 6 | before_install: 7 | - docker build -t higlass-server -f docker-context/Dockerfile . 8 | 9 | install: 10 | - docker run -d --cap-add SYS_ADMIN --device /dev/fuse --security-opt apparmor:unconfined --name higlass-server higlass-server 11 | 12 | script: 13 | - docker exec -it higlass-server ./test.sh 14 | 15 | after_failure: 16 | - docker exec -it higlass-server cat /data/log/hgs.log 17 | -------------------------------------------------------------------------------- /API.md: -------------------------------------------------------------------------------- 1 | # Public API 2 | 3 | ## `/api/v1/fragments_by_loci/` 4 | 5 | **Type**: `POST` 6 | 7 | **Body**: `JSON` 8 | 9 | The body needs to contain a [BEDPE](https://bedtools.readthedocs.io/en/latest/content/general-usage.html#bedpe-format)-like array with the loci determening the fragments to be retrieved. Each locus is represented as an array of the following form: 10 | 11 | ```javascript 12 | [ chrom1, start1, end1, chrom2, start2, end2, dataUuid, zoomOutLevel ] 13 | ``` 14 | 15 | The columns need to be of the following form: 16 | 17 | - chrom1 _(str)_: 18 | 19 | First chromosome. E.g., `chr1` or `1`. 20 | 21 | - start1 _(int)_: 22 | 23 | First start position in base pairs relative to `chrom1`. E.g., `0` or `1`. 24 | 25 | - end1 _(int)_: 26 | 27 | First end position in base pairs relative to `chrom1`. E.g., `chr1` or `1`. 28 | 29 | - chrom2 _(str)_: 30 | 31 | Second chromosome. E.g., `chr1` or `1`. 32 | 33 | - start2 _(int)_: 34 | 35 | Second start position in base pairs relative to `chrom2`. E.g., `0` or `1`. 36 | 37 | - end2 _(int)_: 38 | 39 | Second end position in base pairs relative to `chrom2`. E.g., `chr1` or `1`. 40 | 41 | - dataUuid _(str)_: 42 | 43 | UUID of HiGlass server of the tileset representing a Hi-C map. E.g., `OHJakQICQD6gTD7skx4EWA`. 44 | 45 | - zoomOutLevel _(int)_: 46 | 47 | Inverted zoom level at which the fragment should be cut out. E.g., For GM12878 of Rao et al. (2014) at 1KB resolution, a _zoom out level_ of `0` corresponds to `1KB`, `1` corresponds to `2KB`, `2` corresponds to `4KB`, etc. 48 | 49 | 50 | For example: 51 | 52 | ```javascript 53 | [ 54 | [ 55 | "chr1", 56 | 0, 57 | 500000000, 58 | "chr1", 59 | 0, 60 | 500000000, 61 | "uuid-of-my-fancy-hi-c-map", 62 | 0 63 | ] 64 | ] 65 | ``` 66 | 67 | **Parameters**: 68 | 69 | - dims _(int)_: 70 | 71 | Width and height of the fragment in pixels. Defaults to `22`. 72 | 73 | - padding _(int)_: 74 | 75 | Percental padding related to the dimension of the fragment. E.g., 10 = 10% padding (5% per side). Defaults to `10`. 76 | 77 | - percentile _(float)_: 78 | 79 | Percentile clip. E.g., For 99 the maximum will be capped at the 99-percentile. Defaults to `100.0`. 80 | 81 | - no-balance _(bool)_: 82 | 83 | If `True` the fragment will **not** be balanced using Cooler. Defaults to `False`. 84 | 85 | - no-normalize _(bool)_: 86 | 87 | If `True` the fragment will **not** be normalized to [0, 1]. Defaults to `False`. 88 | 89 | - ignore-diags _(int)_: 90 | 91 | Number of diagonals to be ignored, i.e., set to 0. Defaults to `0`. 92 | 93 | - no-cache _(bool)_: 94 | 95 | If `True` the fragment will not be retrieved from cache. This is useful for development. Defaults to `False`. 96 | 97 | - precision _(int)_: 98 | 99 | Determines the float precision of the returned fragment. Defaults to `2`. 100 | 101 | **Return** _(obj)_: 102 | 103 | ``` 104 | { 105 | "fragments": [ 106 | [ 107 | [0, 0.48, 0, 0.04], 108 | [0.48, 0, 1, 0.07], 109 | [0, 1, 0, 0.47], 110 | [0.04, 0.07, 0.47, 0] 111 | ], 112 | ... 113 | ] 114 | } 115 | ``` 116 | 117 | _(This example comes from a request of `/api/v1/fragments_by_loci/?precision=2&dims=4&ignore-diags=1&percentile=99`)_ 118 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | v1.14.8 2 | 3 | - Fixed issue where 'error' was being set in chromsizes-tsv tileset_info 4 | 5 | v1.14.7 6 | 7 | - Bump Pillow 8 | 9 | v1.14.6 10 | 11 | - Bump urllib, clodius, pandas, requests, higlass-python and simple-httpfs 12 | 13 | v1.14.5 14 | 15 | - Don't require indexfile field in Tileset model 16 | - Don't require higlassVersion field in Viewconf model 17 | 18 | v1.14.4 19 | 20 | - Bump h5py requirement version 21 | 22 | v1.14.3 23 | 24 | - Bump clodius version 25 | 26 | v1.14.2 27 | 28 | - Turn off thumbnail generation 29 | 30 | v1.14.\* 31 | 32 | - Added support for bigBed files 33 | - Update readme installation instructions and troubleshooting instructions for macOS 10.15 34 | - Always consider proxy headers (X-Forwarded-Host, X-Forwarded-Proto) for redirect URL construction 35 | - Added support for server-side aggregation of multivec tiles by sending a `POST` request to the `/tiles` endpoint, where the body contains a JSON object mapping tileset UIDs to objects with properties `agg_groups` (a 2D array where each subarray is a group of rows to aggregate) and `agg_func` (the name of an aggregation function). 36 | - Added support for FASTA tileset from Clodius 37 | 38 | v1.13.0 39 | 40 | - Add support from cooler v2 files for the fragments API 41 | 42 | v1.12.0 43 | 44 | - Added support for BAM files 45 | 46 | v1.11.2 47 | 48 | - Check for the existence of a viewconf before creating the link 49 | 50 | v1.11.1 51 | 52 | - Switch to using networkidle0 53 | 54 | v1.11.0 55 | 56 | - Link unfurling endpoints /link and /thumbnail 57 | - **BREAKING CHANGE:** You need at least Python `v3.6` or higher 58 | 59 | v1.10.2 60 | 61 | - Added project to the admin interface 62 | - coordSystem2 is no longer a required field 63 | 64 | v1.10.1 65 | 66 | - Check to make sure project's owner is not None before returning username 67 | 68 | v1.10.0 69 | 70 | - Added support for mrmatrix files 71 | - Small bug fix for 500 available-chrom-sizes 72 | 73 | v1.9.2 74 | 75 | - Fixed STATIC_URL settings must end with a slash bug 76 | 77 | v1.9.1 78 | 79 | - Added support for the APP_BASEPATH setting 80 | 81 | v1.7.? (????-??-??) 82 | 83 | - Snippets API now allows limiting the size of snippets via `config.json` 84 | 85 | v1.7.3 (2018-07-12) 86 | 87 | - Return datatype along with tileset info 88 | 89 | v1.7.0 90 | 91 | - Merged all of Fritz's changes 92 | 93 | v1.6.0 (2018-05-07) 94 | 95 | - Start factoring out hgtiles code 96 | 97 | v1.5.3 (2018-01- 98 | 99 | - Refactored the chromsizes code to be more modular 100 | 101 | v1.5.2 (2017-12-15) 102 | 103 | - Catch error in fetching cached tiles and continue working 104 | 105 | v1.5.1 (2017-12-14) 106 | 107 | - Decode slugid in ingest command 108 | - Resolve datapath in chromsizes 109 | 110 | v1.5.0 (2017-12-05) 111 | 112 | - Added support for cooler-based chrom-sizes retrieval 113 | - Added support for beddb headers 114 | - Upgraded do django 2.0 115 | 116 | v1.4.2 (2017-11-13) 117 | 118 | - Fixed issue where bigWig files weren't being found 119 | 120 | v1.4.1 (2017-11-11) 121 | 122 | - Built a fixed build 123 | 124 | v1.4.0 (2017-11-08) 125 | 126 | - Added support for bigWig files 127 | 128 | v1.3.1 (2017-10-??) 129 | 130 | - Fixed a bug with ignore-diags in the fragments API (again) 131 | 132 | v1.3.1 (2017-11-02 133 | 134 | - Serve static files from `hgs-static` 135 | 136 | v1.3.0 (2017-10-21) 137 | 138 | - Support arbitrary resolution cooler files 139 | - Combine tile requests for beddb and bed2ddb files 140 | - Increase width of higlassVersion field in the ViewConfModel to 16 141 | 142 | v1.2.3 (2017-10-03) 143 | 144 | - Same changes as last time. They didn't actually make it into v1.2.2 145 | v1.2.2 (2017-10-03) 146 | 147 | - Fixed a bug with ignore-diags in the fragments API 148 | 149 | v1.2.1 (2017-08-30) 150 | 151 | - Fixed an out-of-bounds error 152 | 153 | v1.2.0 (2017-08-29) 154 | 155 | - Group cooler tile requests so they can be retrieved more quickly 156 | 157 | v1.1.3 (2017-08-14) 158 | 159 | - Fix retrieval of snippets starting at negative positions 160 | - Return 400 error for unsupported request bodies 161 | 162 | v1.1.2 (2017-08-08) 163 | 164 | - Return the created field as part of the serializer 165 | 166 | v1.1.1 (2017-08-08) 167 | 168 | - Introduced case insensitive ordering 169 | 170 | v1.1.0 (2017-07-26) 171 | 172 | - Extend endpoint for retrieval of normalized domains 173 | - Retrieve complete snippets (and not just the upper triangle) 174 | - Add option to balance the fragment endpoint 175 | - Add percentage-based padding to the fragment endpoint 176 | - Add diagonal ignoring to the fragment endpoint 177 | - Add percentile clipping to the fragment endpoint 178 | - Add [0,1]-normalization ignoring to the fragment endpoint 179 | 180 | v1.0.4 (2017-07-14) 181 | 182 | - Fixed cumulative JSON sizes error 183 | - Fixed fragment loading error (due to py3) 184 | 185 | v1.0.3 (2017-07-14) 186 | 187 | - Fixed viewconf export (needed to decode slugid.nice()) 188 | 189 | v1.0.2 (2017-07-13) 190 | 191 | - Removed some print statements 192 | - Fixed issues with testing in py3 193 | 194 | v1.0.1 (2017-07-13) 195 | 196 | - Removed Python 2 support 197 | 198 | v1.0.0 (2017-07-13) 199 | 200 | - Python 3 support 201 | - API for getting records by name 202 | 203 | v0.7.6 (2017-07-08) 204 | 205 | - Use cooler transforms 206 | - Always pass NaN values as Float32 arrays 207 | 208 | v0.7.5 (2017-07-07) 209 | 210 | - Use the binsizes for the individual zoom levels 211 | 212 | v0.7.4 (2017-06-20) 213 | 214 | - Added ordering to tileset list API 215 | 216 | v0.7.3 (2017-06-19) 217 | 218 | - Fixed reversion preventing the ingestion of large files 219 | 220 | v0.7.2 (2017-06-16) 221 | 222 | - Fixed tile data bug 223 | 224 | v0.7.1 225 | 226 | - Fixed merge conflicts (doh!) 227 | 228 | v0.7.0 229 | 230 | - Add setting to disable (public) uploads. 231 | - Add settings overloading with `config.json`; see `config.json.sample`. 232 | - Added `higlassVersion` to `viewconf` and extend the endpoint accordingly. 233 | - Code cleanup 234 | - Bug fixes and better error handling 235 | 236 | v0.6.2 237 | 238 | - Add missing `csv` import 239 | 240 | v0.6.1 - 2017-06-06 241 | 242 | - Fixed empty tiles bug 243 | 244 | v0.6.0 245 | 246 | - Removed chromosome table but API remains the same 247 | 248 | v0.5.3 249 | 250 | - Return coordSystem as part of tileset_info 251 | 252 | v0.5.2 253 | 254 | - Added test.higlass.io to allowed hosts 255 | - Turned off HashedFilenameStorage 256 | 257 | v0.5.1 258 | 259 | - Updated requirements to use mirnylab develop cooler 260 | 261 | v0.5.0 262 | 263 | - Add management command for adding chrom-sizes 264 | - Chrom-sizes endpoint parameter `coords` changes to `id` to avoid confusion. I.e., for one coordinate system there might exist multiple orderings, which means multiple IDs could reference the same coordinate system. 265 | 266 | v0.4.4 267 | 268 | - Set proper HTTP status codes for errors of the chrom-sizes endpoint 269 | - Robustify internal magic (a.k.a. bug fixes) 270 | 271 | v0.4.3 272 | 273 | - Fixed an error when the zoom-out levels for fragmentds was negative 274 | - Fixed wrong ordering for multi dataset and/or multi resolution fragment extraction 275 | 276 | v0.4.2 277 | 278 | - Fixed caching issue in loci extraction 279 | 280 | v0.4.1 281 | 282 | - Added test server IP to ALLOWED_HOSTS 283 | 284 | v0.4.0 285 | 286 | - Add endpoints for pulling out fragments aka snippets aka patches of the interaction map 287 | - Add endpoints for chrom-sizes in TSV and JSON format 288 | 289 | v0.3.5 290 | 291 | - Send min_pos with the tileset info 292 | 293 | v0.3.4 294 | 295 | - Bug fix for serving unbalanced data 296 | 297 | v0.3.3 298 | 299 | - Added **str** to Tileset models so that they're visible in the django 300 | interface 301 | 302 | v0.3.2 303 | 304 | - Added support for passing SITE_URL as an environment variable 305 | 306 | v0.3.0 307 | 308 | - Send back float16 data for heatmaps and possibly 1d tracks 309 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 the President and Fellows of Harvard College 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | What was changed in this pull request? 4 | 5 | Why is it necessary? 6 | 7 | Fixes #___ 8 | 9 | ## Checklist 10 | 11 | - [ ] Unit tests added or updated 12 | - [ ] Documentation added or updated 13 | - [ ] Updated CHANGELOG.md 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # HiGlass Server 2 | 3 | The HiGlass Server supports [HiGlass](https://github.com/higlass/higlass) and [HiPiler](https://github.com/flekschas/hipiler) 4 | by providing APIs for accessing and uploading tiles generated by 5 | [Clodius](https://github.com/higlass/clodius). 6 | 7 | [![demo](https://img.shields.io/badge/higlass-👍-red.svg?colorB=0f5d92)](http://higlass.io) 8 | [![api](https://img.shields.io/badge/api-documentation-red.svg?colorB=0f5d92)](API.md) 9 | [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1308945.svg)](https://doi.org/10.5281/zenodo.1308945) 10 | 11 | 12 | _Note: that the HiGlass Server itself only provides an API, and does not serve any HTML._ 13 | 14 | ## Installation 15 | 16 | **Prerequirements**: 17 | 18 | - Python `>=v3.6` 19 | 20 | ### Docker 21 | 22 | The easiest way to run HiGlass with HiGlass Server is with Docker. More information is available at [higlass-docker](https://github.com/higlass/higlass-docker#readme) or check out the [Dockerfile](docker-context/Dockerfile). 23 | 24 | This project also includes a Dockerfile in the docker-context directory that can be used to run a locally checked out copy of higlass-server as follows: 25 | ```bash 26 | docker build -t higlass-server -f docker-context/Dockerfile . 27 | docker run -d --cap-add SYS_ADMIN --device /dev/fuse --security-opt apparmor:unconfined --name higlass-server higlass-server 28 | ``` 29 | 30 | ### Manually 31 | 32 | To install HiGlass Server manually follow the steps below. Note we strongly recommend to create a virtual environment using [Virtualenvwrapper](https://pypi.python.org/pypi/virtualenvwrapper) for example. Skip step 2 if you don't work with virtual environments. 33 | 34 | ```bash 35 | git clone https://github.com/higlass/higlass-server && cd higlass-server 36 | ``` 37 | 38 | #### Manually with virtualenvwrapper 39 | 40 | ```bash 41 | mkvirtualenv -a $(pwd) -p $(which python3) higlass-server && workon higlass-server 42 | pip install --upgrade -r ./requirements.txt 43 | python manage.py migrate 44 | python manage.py runserver 45 | ``` 46 | 47 | #### Manually with conda 48 | 49 | ```bash 50 | conda env create -f environment.yml 51 | conda activate higlass-server 52 | python manage.py migrate 53 | python manage.py runserver 54 | ``` 55 | 56 | To enable the register_url api endpoint, HiGlass depends on a project called httpfs to cache external url files. Tests depend on this process running. Set it up as follows: 57 | ```bash 58 | pip install simple-httpfs 59 | 60 | mkdir -p media/http 61 | mkdir -p media/https 62 | simple-httpfs media/http 63 | simple-httpfs media/https 64 | ``` 65 | 66 | Or simply use `./unit_tests.sh`. 67 | 68 | --- 69 | 70 | ## Uploading Files 71 | 72 | Although there is an API endpoint for uploading files, but it is more direct to use a `manage.py` script: 73 | ``` 74 | COOLER=dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool 75 | HITILE=wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile 76 | 77 | wget -P data/ https://s3.amazonaws.com/pkerp/public/$COOLER 78 | wget -P data/ https://s3.amazonaws.com/pkerp/public/$HITILE 79 | 80 | python manage.py ingest_tileset --filename data/$COOLER --filetype cooler --datatype matrix --uid cooler-demo 81 | python manage.py ingest_tileset --filename data/$HITILE --filetype hitile --datatype vector --uid hitile-demo 82 | ``` 83 | 84 | We can now use the API to get information about a tileset, or to get the tile data itself: 85 | ``` 86 | curl http://localhost:8000/api/v1/tileset_info/?d=hitile-demo 87 | curl http://localhost:8000/api/v1/tiles/?d=hitile-demo.0.0.0 88 | ``` 89 | 90 | --- 91 | 92 | ## Development 93 | 94 | **Start** the server: 95 | 96 | ``` 97 | python manage.py runserver localhost:8001 98 | // or 99 | npm start 100 | ``` 101 | 102 | **Test** the server: 103 | 104 | ``` 105 | ./test.sh 106 | // or 107 | npm test 108 | ``` 109 | 110 | **Bump version** of server: 111 | 112 | ``` 113 | bumpversion patch 114 | ``` 115 | 116 | **Update** source code: 117 | 118 | ``` 119 | ./update.sh 120 | ``` 121 | 122 | ## Troubleshooting 123 | 124 | **pybbi installation fails on macOS**: Check out https://github.com/nvictus/pybbi/issues/2 125 | 126 | ### macOS 10.15 dependencies 127 | 128 | - `brew install hdf5` 129 | - `brew install libpng` 130 | - `brew install jpeg` 131 | - [FUSE for Mac](https://osxfuse.github.io/) 132 | 133 | 134 | ## License 135 | 136 | The code in this repository is provided under the MIT License. 137 | -------------------------------------------------------------------------------- /config.json.sample: -------------------------------------------------------------------------------- 1 | { 2 | "DEBUG": true, 3 | "LOG_LEVEL_CONSOLE": "DEBUG", 4 | "LOG_LEVEL_FILE": "DEBUG", 5 | "LOG_LEVEL_DJANGO": "DEBUG", 6 | "LOG_LEVEL_CHROMS": "DEBUG", 7 | "LOG_LEVEL_FRAGMENTS": "DEBUG", 8 | "LOG_LEVEL_TILESETS": "DEBUG", 9 | "UPLOAD_ENABLED": true, 10 | "PUBLIC_UPLOAD_ENABLED": false 11 | } 12 | -------------------------------------------------------------------------------- /docker-context/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM continuumio/miniconda3:4.6.14 2 | 3 | RUN apt-get update && apt-get install -y \ 4 | gcc \ 5 | nginx-full \ 6 | supervisor \ 7 | unzip \ 8 | uwsgi-plugin-python3 \ 9 | zlib1g-dev \ 10 | libcurl4-openssl-dev \ 11 | g++ \ 12 | vim \ 13 | build-essential \ 14 | libssl-dev \ 15 | libpng-dev \ 16 | procps \ 17 | git \ 18 | fuse \ 19 | && rm -rf /var/lib/apt/lists/* 20 | 21 | RUN conda install --yes cython numpy==1.14.0 22 | RUN conda install --yes --channel bioconda pysam htslib=1.3.2 23 | RUN conda install --yes -c conda-forge uwsgi 24 | 25 | RUN pip install simple-httpfs>=0.1.3 26 | 27 | ENV HTTPFS_HTTP_DIR /data/media/http 28 | ENV HTTPFS_HTTPS_DIR /data/media/https 29 | ENV HTTPFS_FTP_DIR /data/media/ftp 30 | 31 | WORKDIR /higlass-server 32 | COPY requirements.txt ./ 33 | COPY requirements-dev.txt ./ 34 | RUN pip install -r requirements.txt 35 | RUN pip install -r requirements-dev.txt 36 | 37 | COPY docker-context/nginx.conf /etc/nginx/ 38 | COPY docker-context/hgserver_nginx.conf /etc/nginx/sites-enabled/ 39 | RUN rm /etc/nginx/sites-*/default && grep 'listen' /etc/nginx/sites-*/* 40 | 41 | COPY docker-context/uwsgi_params ./ 42 | COPY docker-context/default-viewconf-fixture.xml ./ 43 | 44 | COPY docker-context/supervisord.conf ./ 45 | COPY docker-context/uwsgi.ini ./ 46 | 47 | COPY docker-context/httpfs.sh ./ 48 | 49 | EXPOSE 80 50 | 51 | ENV HIGLASS_SERVER_BASE_DIR /data 52 | VOLUME /data 53 | VOLUME /tmp 54 | 55 | ARG WORKERS=2 56 | ENV WORKERS ${WORKERS} 57 | RUN echo "WORKERS: $WORKERS" 58 | 59 | COPY . . 60 | 61 | CMD ["supervisord", "-n", "-c", "/higlass-server/supervisord.conf"] 62 | -------------------------------------------------------------------------------- /docker-context/default-viewconf-fixture.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 2017-03-01 18:00 5 | default 6 | { 7 | "editable": true, 8 | "zoomFixed": false, 9 | "trackSourceServers": [ 10 | "/api/v1", 11 | "http://higlass.io/api/v1" 12 | ], 13 | "exportViewUrl": "/api/v1/viewconfs/", 14 | "views": [ 15 | { 16 | 22 | "tracks": { 23 | "top": [], 24 | "left": [], 25 | "center": [], 26 | "right": [], 27 | "bottom": [] 28 | }, 29 | "initialXDomain": [ 0, 3200000000 ], 30 | "initialYDomain": [ 0, 3200000000 ], 31 | "layout": { 32 | "w": 12, 33 | "h": 12, 34 | "x": 0, 35 | "y": 0, 36 | "moved": false, 37 | "static": false 38 | } 39 | } 40 | ], 41 | "zoomLocks": { 42 | "locksByViewUid": {}, 43 | "locksDict": {} 44 | }, 45 | "locationLocks": { 46 | "locksByViewUid": {}, 47 | "locksDict": {} 48 | } 49 | } 50 | 51 | 52 | -------------------------------------------------------------------------------- /docker-context/hgserver_nginx.conf: -------------------------------------------------------------------------------- 1 | # the upstream component nginx needs to connect to 2 | upstream django { 3 | server 127.0.0.1:8001; # for a web port socket 4 | # server unix:///path/to/your/mysite/mysite.sock; # TODO: May be faster 5 | } 6 | 7 | # configuration of the server 8 | server { 9 | listen 80; 10 | charset utf-8; 11 | 12 | # max upload size 13 | client_max_body_size 10000M; # adjust to taste 14 | 15 | location /api/v1/ { 16 | uwsgi_pass django; 17 | uwsgi_read_timeout 600; 18 | include /higlass-server/uwsgi_params; 19 | } 20 | 21 | location /admin/ { 22 | uwsgi_pass django; 23 | uwsgi_read_timeout 600; 24 | include /higlass-server/uwsgi_params; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /docker-context/httpfs.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | ECHO "Mounting httpfs" 3 | mkdir -p $HTTPFS_HTTP_DIR 4 | mkdir -p $HTTPFS_HTTPS_DIR 5 | mkdir -p $HTTPFS_FTP_DIR 6 | 7 | simple-httpfs $HTTPFS_HTTPS_DIR --lru-capacity 1000 --disk-cache-dir /data/disk-cache --disk-cache-size 4294967296 8 | simple-httpfs $HTTPFS_HTTP_DIR --lru-capacity 1000 --disk-cache-dir /data/disk-cache --disk-cache-size 4294967296 9 | simple-httpfs $HTTPFS_FTP_DIR --lru-capacity 1000 --disk-cache-dir /data/disk-cache --disk-cache-size 4294967296 10 | -------------------------------------------------------------------------------- /docker-context/nginx.conf: -------------------------------------------------------------------------------- 1 | user www-data; 2 | worker_processes auto; 3 | pid /run/nginx.pid; 4 | daemon off; # For compatibility with supervisord 5 | 6 | events { 7 | worker_connections 768; 8 | # multi_accept on; 9 | } 10 | 11 | http { 12 | 13 | ## 14 | # Basic Settings 15 | ## 16 | 17 | sendfile on; 18 | tcp_nopush on; 19 | tcp_nodelay on; 20 | keepalive_timeout 65; 21 | types_hash_max_size 2048; 22 | # server_tokens off; 23 | 24 | log_format main '$remote_addr - $remote_user [$time_local] $status ' 25 | '"$request" $body_bytes_sent "$http_referer" ' 26 | '"$http_user_agent" "$http_x_forwarded_for" ' 27 | '$gzip_ratio $request_time' ; 28 | 29 | # server_names_hash_bucket_size 64; 30 | # server_name_in_redirect off; 31 | 32 | include /etc/nginx/mime.types; 33 | default_type application/octet-stream; 34 | 35 | ## 36 | # SSL Settings 37 | ## 38 | 39 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE 40 | ssl_prefer_server_ciphers on; 41 | 42 | ## 43 | # Logging Settings 44 | ## 45 | 46 | access_log /data/log/access.log main; 47 | error_log /data/log/error.log; 48 | 49 | ## 50 | # Gzip Settings 51 | ## 52 | 53 | gzip on; 54 | gzip_disable "msie6"; 55 | 56 | gzip_vary on; 57 | gzip_proxied any; 58 | gzip_comp_level 6; 59 | gzip_buffers 16 8k; 60 | gzip_http_version 1.1; 61 | gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; 62 | 63 | ## 64 | # Virtual Host Configs 65 | ## 66 | 67 | include /etc/nginx/conf.d/*.conf; 68 | include /etc/nginx/sites-enabled/*; 69 | } 70 | -------------------------------------------------------------------------------- /docker-context/supervisord.conf: -------------------------------------------------------------------------------- 1 | [program:uwsgi] 2 | directory = /higlass-server 3 | # /data is a mounted volume, so the Dockerfile can not create subdirectories. 4 | # If this is re-run, the loaddata will fail, which right now is a feature. 5 | command = bash -c "mkdir -p /data/log && ./httpfs.sh && python manage.py migrate && python manage.py loaddata default-viewconf-fixture.xml; uwsgi --ini /higlass-server/uwsgi.ini --socket :8001 --module higlass_server.wsgi --workers $WORKERS" 6 | # TODO: workers should be configured at runtime 7 | 8 | [program:nginx] 9 | command = /usr/sbin/nginx 10 | 11 | [supervisord] 12 | logfile = /var/log/supervisor/supervisord.log 13 | -------------------------------------------------------------------------------- /docker-context/uwsgi.ini: -------------------------------------------------------------------------------- 1 | [uwsgi] 2 | # this config will be loaded if nothing specific is specified 3 | # load base config from below 4 | ini = :base 5 | 6 | # %d is the dir this configuration file is in 7 | socket = %dapp.sock 8 | master = true 9 | processes = 4 10 | 11 | [local] 12 | ini = :base 13 | http = :8000 14 | # TODO: hgserver_nginx.conf says 8001: Is this one ignored? 15 | 16 | # set the virtual env to use 17 | # home=/Users/you/envs/env 18 | 19 | 20 | [base] 21 | # chdir to the folder of this config file, plus app/website 22 | # TODO: another config for website? and client? 23 | chdir = /higlass-server/ 24 | # load the module from wsgi.py, it is a python path from 25 | # the directory above. 26 | module=website.wsgi:application 27 | # allow anyone to connect to the socket. This is very permissive 28 | chmod-socket=666 29 | -------------------------------------------------------------------------------- /docker-context/uwsgi_params: -------------------------------------------------------------------------------- 1 | uwsgi_param QUERY_STRING $query_string; 2 | uwsgi_param REQUEST_METHOD $request_method; 3 | uwsgi_param CONTENT_TYPE $content_type; 4 | uwsgi_param CONTENT_LENGTH $content_length; 5 | 6 | uwsgi_param REQUEST_URI $request_uri; 7 | uwsgi_param PATH_INFO $document_uri; 8 | uwsgi_param DOCUMENT_ROOT $document_root; 9 | uwsgi_param SERVER_PROTOCOL $server_protocol; 10 | uwsgi_param REQUEST_SCHEME $scheme; 11 | uwsgi_param HTTPS $https if_not_empty; 12 | 13 | uwsgi_param REMOTE_ADDR $remote_addr; 14 | uwsgi_param REMOTE_PORT $remote_port; 15 | uwsgi_param SERVER_PORT $server_port; 16 | uwsgi_param SERVER_NAME $server_name; -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: higlass-server 2 | channels: 3 | - conda-forge 4 | - bioconda 5 | - defaults 6 | dependencies: 7 | - python>=3.6 8 | - pip 9 | - pip: 10 | - pybbi==0.2.2 11 | - bumpversion==0.5.3 12 | - CacheControl==0.12.4 13 | - cooler==0.8.6 14 | - django-cors-headers==3.0.2 15 | - django-guardian==1.5.1 16 | - django-rest-swagger==2.2.0 17 | - django==2.1.11 18 | - djangorestframework==3.9.1 19 | - h5py==2.6.0 20 | - higlass-python==0.2.1 21 | - jsonschema==3.2.0 22 | - numba==0.46.0 23 | - numpy==1.17.3 24 | - pandas==0.23.4 25 | - Pillow==5.0.0 26 | - pybase64==0.2.1 27 | - redis==2.10.5 28 | - requests==2.20.0 29 | - scikit-learn==0.19.2 30 | - slugid==2.0.0 31 | - redis==2.10.5 32 | - clodius==0.12.0 33 | - simple-httpfs==0.2.0 34 | - pyppeteer==0.0.25 35 | -------------------------------------------------------------------------------- /fragments/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/higlass/higlass-server/cbfe79fe3ae0e844b4c0c78142a83733c8cc66a2/fragments/__init__.py -------------------------------------------------------------------------------- /fragments/app.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.apps import AppConfig 4 | 5 | 6 | class FragmentsConfig(AppConfig): 7 | name = 'fragments' 8 | -------------------------------------------------------------------------------- /fragments/drf_disable_csrf.py: -------------------------------------------------------------------------------- 1 | from rest_framework.authentication import SessionAuthentication 2 | 3 | 4 | class CsrfExemptSessionAuthentication(SessionAuthentication): 5 | 6 | def enforce_csrf(self, request): 7 | return # To not perform the csrf check previously happening 8 | -------------------------------------------------------------------------------- /fragments/exceptions.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from rest_framework.exceptions import APIException 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | class CoolerFileBroken(APIException): 9 | status_code = 500 10 | default_detail = 'The cooler file is broken.' 11 | default_code = 'cooler_file_broken' 12 | 13 | class SnippetTooLarge(APIException): 14 | status_code = 400 15 | default_detail = 'The requested snippet is too large' 16 | default_code = 'snippet_too_large' 17 | -------------------------------------------------------------------------------- /fragments/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2017-04-06 18:40 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | import slugid.slugid 7 | 8 | 9 | class Migration(migrations.Migration): 10 | 11 | initial = True 12 | 13 | dependencies = [ 14 | ] 15 | 16 | operations = [ 17 | migrations.CreateModel( 18 | name='ChromInfo', 19 | fields=[ 20 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 21 | ('created', models.DateTimeField(auto_now_add=True)), 22 | ('uuid', models.CharField(default=slugid.slugid.nice, max_length=100, unique=True)), 23 | ('datafile', models.TextField()), 24 | ], 25 | options={ 26 | 'ordering': ('created',), 27 | }, 28 | ), 29 | migrations.CreateModel( 30 | name='ChromSizes', 31 | fields=[ 32 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 33 | ('created', models.DateTimeField(auto_now_add=True)), 34 | ('uuid', models.CharField(default=slugid.slugid.nice, max_length=100, unique=True)), 35 | ('datafile', models.TextField()), 36 | ], 37 | options={ 38 | 'ordering': ('created',), 39 | }, 40 | ), 41 | ] 42 | -------------------------------------------------------------------------------- /fragments/migrations/0002_auto_20170406_2322.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2017-04-06 23:22 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('fragments', '0001_initial'), 12 | ] 13 | 14 | operations = [ 15 | migrations.DeleteModel( 16 | name='ChromInfo', 17 | ), 18 | migrations.DeleteModel( 19 | name='ChromSizes', 20 | ), 21 | ] 22 | -------------------------------------------------------------------------------- /fragments/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/higlass/higlass-server/cbfe79fe3ae0e844b4c0c78142a83733c8cc66a2/fragments/migrations/__init__.py -------------------------------------------------------------------------------- /fragments/tests.py: -------------------------------------------------------------------------------- 1 | import django.core.files.uploadedfile as dcfu 2 | import django.test as dt 3 | import django.contrib.auth.models as dcam 4 | import tilesets.models as tm 5 | import json 6 | import numpy as np 7 | 8 | from urllib.parse import urlencode 9 | 10 | 11 | class FragmentsTest(dt.TestCase): 12 | def setUp(self): 13 | self.user1 = dcam.User.objects.create_user( 14 | username='user1', password='pass' 15 | ) 16 | upload_file = open( 17 | ( 18 | 'data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb' 19 | '.multires.cool' 20 | ), 21 | 'rb' 22 | ) 23 | tm.Tileset.objects.create( 24 | datafile=dcfu.SimpleUploadedFile( 25 | upload_file.name, upload_file.read() 26 | ), 27 | filetype='cooler', 28 | uuid='cool-v1', 29 | owner=self.user1 30 | ) 31 | upload_file = open( 32 | 'data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.mcoolv2', 33 | 'rb' 34 | ) 35 | tm.Tileset.objects.create( 36 | datafile=dcfu.SimpleUploadedFile( 37 | upload_file.name, upload_file.read() 38 | ), 39 | filetype='cooler', 40 | uuid='cool-v2', 41 | owner=self.user1 42 | ) 43 | 44 | def test_get_fragments(self): 45 | for hurz in [(1, 0), (2, 0), (2, 1000000)]: 46 | version, zoom_res = hurz 47 | data = { 48 | "loci": [ 49 | [ 50 | "chr1", 1000000000, 2000000000, 51 | "1", 1000000000, 2000000000, f"cool-v{version}", 52 | zoom_res 53 | ] 54 | ] 55 | } 56 | 57 | response = self.client.post( 58 | '/api/v1/fragments_by_loci/?precision=2&dims=22', 59 | json.dumps(data), 60 | content_type="application/json" 61 | ) 62 | 63 | ret = json.loads(str(response.content, encoding='utf8')) 64 | 65 | self.assertEqual(response.status_code, 200) 66 | 67 | self.assertTrue('fragments' in ret) 68 | 69 | self.assertEqual(len(ret['fragments']), 1) 70 | 71 | self.assertEqual(len(ret['fragments'][0]), 22) 72 | 73 | self.assertEqual(len(ret['fragments'][0][0]), 22) 74 | 75 | def test_string_request_body(self): 76 | data = ( 77 | '{loci: [["chr1", 1000000000, 2000000000, "1",' 78 | ' 1000000000, 2000000000, "cool-v1", 0]]}' 79 | ) 80 | 81 | response = self.client.post( 82 | '/api/v1/fragments_by_loci/?precision=2&dims=22', 83 | json.dumps(data), 84 | content_type="application/json" 85 | ) 86 | 87 | ret = json.loads(str(response.content, encoding='utf8')) 88 | 89 | self.assertEqual(response.status_code, 400) 90 | self.assertTrue('error' in ret) 91 | self.assertTrue('error_message' in ret) 92 | 93 | def test_too_large_request(self): 94 | for version in [1, 2]: 95 | data = [ 96 | [ 97 | "1", 1000000000, 2000000000, 98 | "1", 1000000000, 2000000000, 99 | f"cool-v{version}", 0 100 | ] 101 | ] 102 | 103 | response = self.client.post( 104 | '/api/v1/fragments_by_loci/?dims=1025', 105 | json.dumps(data), 106 | content_type="application/json" 107 | ) 108 | 109 | ret = json.loads(str(response.content, encoding='utf8')) 110 | 111 | self.assertEqual(response.status_code, 400) 112 | self.assertTrue('error' in ret) 113 | self.assertTrue('error_message' in ret) 114 | 115 | def test_both_body_data_types(self): 116 | for version in [1, 2]: 117 | loci = [ 118 | [ 119 | "chr1", 1000000000, 2000000000, 120 | "1", 1000000000, 2000000000, 121 | f"cool-v{version}", 0 122 | ] 123 | ] 124 | 125 | obj = { 126 | "loci": loci 127 | } 128 | 129 | response = self.client.post( 130 | '/api/v1/fragments_by_loci/?precision=2&dims=22', 131 | json.dumps(obj), 132 | content_type="application/json" 133 | ) 134 | ret = json.loads(str(response.content, encoding='utf8')) 135 | 136 | mat1 = np.array(ret['fragments'][0], float) 137 | 138 | response = self.client.post( 139 | '/api/v1/fragments_by_loci/?precision=2&dims=22', 140 | json.dumps(loci), 141 | content_type="application/json" 142 | ) 143 | ret = json.loads(str(response.content, encoding='utf8')) 144 | 145 | mat2 = np.array(ret['fragments'][0], float) 146 | 147 | self.assertTrue(np.array_equal(mat1, mat2)) 148 | 149 | def test_negative_start_fragments(self): 150 | for version in [1, 2]: 151 | data = [ 152 | [ 153 | "1", 154 | 0, 155 | 1, 156 | "2", 157 | 0, 158 | 1, 159 | f"cool-v{version}", 160 | 20 161 | ] 162 | ] 163 | 164 | dims = 60 165 | dims_h = (dims // 2) - 1 166 | 167 | response = self.client.post( 168 | '/api/v1/fragments_by_loci/' 169 | '?precision=2&dims={}&no-balance=1'.format(dims), 170 | json.dumps(data), 171 | content_type="application/json" 172 | ) 173 | 174 | self.assertEqual(response.status_code, 200) 175 | 176 | ret = json.loads(str(response.content, encoding='utf8')) 177 | 178 | self.assertTrue('fragments' in ret) 179 | 180 | mat = np.array(ret['fragments'][0], float) 181 | 182 | # Upper half should be empty 183 | self.assertTrue(np.sum(mat[0:dims_h]) == 0) 184 | 185 | # Lower half should not be empty 186 | self.assertTrue(np.sum(mat[dims_h:dims]) > 0) 187 | 188 | def test_domains_by_loci(self): 189 | for version in [1, 2]: 190 | data = { 191 | "loci": [ 192 | [ 193 | "chr1", 194 | 0, 195 | 2000000000, 196 | "1", 197 | 0, 198 | 2000000000, 199 | f"cool-v{version}", 200 | 0 201 | ] 202 | ] 203 | } 204 | 205 | response = self.client.post( 206 | '/api/v1/fragments_by_loci/?precision=2&dims=44', 207 | json.dumps(data), 208 | content_type="application/json" 209 | ) 210 | 211 | self.assertEqual(response.status_code, 200) 212 | 213 | ret = json.loads(str(response.content, encoding='utf8')) 214 | 215 | self.assertTrue('fragments' in ret) 216 | 217 | self.assertEqual(len(ret['fragments']), 1) 218 | 219 | self.assertEqual(len(ret['fragments'][0]), 44) 220 | 221 | self.assertEqual(len(ret['fragments'][0][0]), 44) 222 | 223 | def test_domains_normalizing(self): 224 | for version in [1, 2]: 225 | data = [ 226 | [ 227 | "chr2", 228 | 0, 229 | 500000000, 230 | "2", 231 | 0, 232 | 500000000, 233 | f"cool-v{version}", 234 | 0 235 | ] 236 | ] 237 | 238 | params = { 239 | 'dims': 60, 240 | 'precision': 3, 241 | 'padding': 2, 242 | 'ignore-diags': 2, 243 | 'percentile': 50 244 | } 245 | 246 | response = self.client.post( 247 | '/api/v1/fragments_by_loci/?{}'.format(urlencode(params)), 248 | json.dumps(data), 249 | content_type='application/json' 250 | ) 251 | 252 | self.assertEqual(response.status_code, 200) 253 | 254 | ret = json.loads(str(response.content, encoding='utf8')) 255 | 256 | self.assertTrue('fragments' in ret) 257 | 258 | mat = np.array(ret['fragments'][0], float) 259 | 260 | # Make sure matrix is not empty 261 | self.assertTrue(np.sum(mat) > 0) 262 | 263 | # Check that the diagonal is 1 (it's being ignored for normalizing 264 | # the data but set to 1 to visually make more sense) 265 | diag = np.diag_indices(params['dims']) 266 | self.assertEqual(np.sum(mat[diag]), params['dims']) 267 | self.assertEqual( 268 | np.sum( 269 | mat[((diag[0] - 1)[1:], diag[1][1:])] 270 | ), 271 | params['dims'] - 1 272 | ) 273 | self.assertEqual( 274 | np.sum( 275 | mat[((diag[0] + 1)[:-1], diag[1][:-1])] 276 | ), 277 | params['dims'] - 1 278 | ) 279 | 280 | # Check precision of matrix 281 | self.assertTrue(np.array_equal(mat, np.rint(mat * 1000) / 1000)) 282 | self.assertTrue(not np.array_equal(mat, np.rint(mat * 100) / 100)) 283 | 284 | # Check max 285 | self.assertEqual(np.max(mat), 1.0) 286 | 287 | # Get two more un-normalized matrices 288 | params1 = { 289 | 'dims': 60, 290 | 'precision': 3, 291 | 'padding': 2, 292 | 'ignore-diags': 2, 293 | 'percentile': 50.0, 294 | 'no-normalize': True 295 | } 296 | 297 | response = self.client.post( 298 | '/api/v1/fragments_by_loci/?{}'.format(urlencode(params1)), 299 | json.dumps(data), 300 | content_type='application/json' 301 | ) 302 | 303 | self.assertEqual(response.status_code, 200) 304 | 305 | ret = json.loads(str(response.content, encoding='utf8')) 306 | 307 | self.assertTrue('fragments' in ret) 308 | 309 | mat1 = np.array(ret['fragments'][0], float) 310 | 311 | params2 = { 312 | 'dims': 60, 313 | 'precision': 3, 314 | 'padding': 2, 315 | 'ignore-diags': 2, 316 | 'percentile': 100.0, 317 | 'no-normalize': True 318 | } 319 | 320 | response = self.client.post( 321 | '/api/v1/fragments_by_loci/?{}'.format(urlencode(params2)), 322 | json.dumps(data), 323 | content_type='application/json' 324 | ) 325 | 326 | self.assertEqual(response.status_code, 200) 327 | 328 | ret = json.loads(str(response.content, encoding='utf8')) 329 | 330 | self.assertTrue('fragments' in ret) 331 | 332 | mat2 = np.array(ret['fragments'][0], float) 333 | 334 | # Make sure matrix is not empty 335 | self.assertTrue(np.sum(mat2) > 0) 336 | max1 = np.max(mat1) 337 | max2 = np.max(mat2) 338 | 339 | self.assertTrue(max2 > max1) 340 | 341 | percentile = np.percentile(mat2, params['percentile']) 342 | 343 | self.assertEqual( 344 | np.rint(max1 * 10000000) / 10000000, 345 | np.rint(percentile * 10000000) / 10000000 346 | ) 347 | -------------------------------------------------------------------------------- /fragments/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url 2 | from fragments import views 3 | 4 | 5 | # The API URLs are now determined automatically by the router. 6 | # Additionally, we include the login URLs for the browsable API. 7 | urlpatterns = [ 8 | url(r'^fragments_by_loci/$', views.fragments_by_loci), 9 | url(r'^fragments_by_chr/$', views.fragments_by_chr), 10 | url(r'^loci/$', views.loci), 11 | ] 12 | -------------------------------------------------------------------------------- /fragments/views.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import hashlib 4 | import json 5 | import logging 6 | import numpy as np 7 | import pybase64 8 | from PIL import Image 9 | try: 10 | import cPickle as pickle 11 | except: 12 | import pickle 13 | 14 | import higlass_server.settings as hss 15 | 16 | from rest_framework.authentication import BasicAuthentication 17 | from .drf_disable_csrf import CsrfExemptSessionAuthentication 18 | from io import BytesIO 19 | from os import path 20 | from django.http import HttpResponse, JsonResponse 21 | from rest_framework.decorators import api_view, authentication_classes 22 | from tilesets.models import Tileset 23 | from fragments.utils import ( 24 | calc_measure_dtd, 25 | calc_measure_size, 26 | calc_measure_noise, 27 | calc_measure_sharpness, 28 | aggregate_frags, 29 | get_frag_by_loc_from_cool, 30 | get_frag_by_loc_from_imtiles, 31 | get_frag_by_loc_from_osm, 32 | get_intra_chr_loops_from_looplist, 33 | get_params, 34 | get_rep_frags, 35 | rel_loci_2_obj, 36 | np_to_png, 37 | write_png, 38 | grey_to_rgb, 39 | blob_to_zip 40 | ) 41 | from higlass_server.utils import getRdb 42 | from fragments.exceptions import SnippetTooLarge 43 | 44 | import h5py 45 | 46 | from math import floor, log 47 | 48 | rdb = getRdb() 49 | 50 | logger = logging.getLogger(__name__) 51 | 52 | SUPPORTED_MEASURES = ['distance-to-diagonal', 'noise', 'size', 'sharpness'] 53 | 54 | SUPPORTED_FILETYPES = ['matrix', 'im-tiles', 'osm-tiles'] 55 | 56 | GET_FRAG_PARAMS = { 57 | 'dims': { 58 | 'short': 'di', 59 | 'dtype': 'int', 60 | 'default': 22, 61 | 'help': 'Global number of dimensions. (Only used for cooler tilesets.)' 62 | }, 63 | 'padding': { 64 | 'short': 'pd', 65 | 'dtype': 'float', 66 | 'default': 0, 67 | 'help': 'Add given percent of the fragment size as padding.' 68 | }, 69 | 'no-balance': { 70 | 'short': 'nb', 71 | 'dtype': 'bool', 72 | 'default': False, 73 | 'help': ( 74 | 'Do not balance fragmens if true. (Only used for cooler tilesets.)' 75 | ) 76 | }, 77 | 'percentile': { 78 | 'short': 'pe', 79 | 'dtype': 'float', 80 | 'default': 100.0, 81 | 'help': ( 82 | 'Cap values at given percentile. (Only used for cooler tilesets.)' 83 | ) 84 | }, 85 | 'precision': { 86 | 'short': 'pr', 87 | 'dtype': 'int', 88 | 'default': 0, 89 | 'help': ( 90 | 'Number of decimals of the numerical values. ' 91 | '(Only used for cooler tilesets.)' 92 | ) 93 | }, 94 | 'no-cache': { 95 | 'short': 'nc', 96 | 'dtype': 'bool', 97 | 'default': 0, 98 | 'help': 'Do not cache fragments if true. Useful for debugging.' 99 | }, 100 | 'ignore-diags': { 101 | 'short': 'nd', 102 | 'dtype': 'int', 103 | 'default': 0, 104 | 'help': ( 105 | 'Ignore N diagonals, i.e., set them to zero. ' 106 | '(Only used for cooler tilesets.)' 107 | ) 108 | }, 109 | 'no-normalize': { 110 | 'short': 'nn', 111 | 'dtype': 'bool', 112 | 'default': False, 113 | 'help': ( 114 | 'Do not normalize fragments if true. ' 115 | '(Only used for cooler tilesets.)' 116 | ) 117 | }, 118 | 'aggregate': { 119 | 'short': 'ag', 120 | 'dtype': 'bool', 121 | 'default': False, 122 | 'help': 'Aggregate fragments if true.' 123 | }, 124 | 'aggregation-method': { 125 | 'short': 'am', 126 | 'dtype': 'str', 127 | 'default': 'mean', 128 | 'help': 'Aggregation method: mean, median, std, var.' 129 | }, 130 | 'max-previews': { 131 | 'short': 'mp', 132 | 'dtype': 'int', 133 | 'default': 0, 134 | 'help': ( 135 | 'Max. number of 1D previews to return. When the number of ' 136 | 'fragments s higher than the previews we cluster the frags by ' 137 | 'k-means.' 138 | ) 139 | }, 140 | 'encoding': { 141 | 'short': 'en', 142 | 'dtype': 'str', 143 | 'default': 'matrix', 144 | 'help': ( 145 | 'Data encoding: matrix, b64, or image. (Image encoding only ' 146 | 'supported when one fragment is to be returned)' 147 | ) 148 | }, 149 | 'representatives': { 150 | 'short': 'rp', 151 | 'dtype': 'int', 152 | 'default': 0, 153 | 'help': ( 154 | 'Number of representative fragments when requesting multiple ' 155 | 'fragments.' 156 | ) 157 | }, 158 | } 159 | 160 | 161 | @api_view(['GET', 'POST']) 162 | @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) 163 | def fragments_by_loci(request): 164 | if request.method == 'GET': 165 | return get_fragments_by_loci_info(request) 166 | 167 | return get_fragments_by_loci(request) 168 | 169 | 170 | def get_fragments_by_loci_info(request): 171 | return JsonResponse(GET_FRAG_PARAMS) 172 | 173 | 174 | def get_fragments_by_loci(request): 175 | ''' 176 | Retrieve a list of locations and return the corresponding matrix fragments 177 | 178 | Args: 179 | 180 | request (django.http.HTTPRequest): The request object containing the 181 | list of loci. 182 | 183 | Return: 184 | 185 | ''' 186 | 187 | if type(request.data) is str: 188 | return JsonResponse({ 189 | 'error': 'Request body needs to be an array or object.', 190 | 'error_message': 'Request body needs to be an array or object.' 191 | }, status=400) 192 | 193 | try: 194 | loci = request.data.get('loci', []) 195 | except AttributeError: 196 | loci = request.data 197 | except Exception as e: 198 | return JsonResponse({ 199 | 'error': 'Could not read request body.', 200 | 'error_message': str(e) 201 | }, status=400) 202 | 203 | try: 204 | forced_rep_idx = request.data.get('representativeIndices', None) 205 | except Exception as e: 206 | forced_rep_idx = None 207 | pass 208 | 209 | ''' 210 | Loci list must be of type: 211 | [cooler] [imtiles] 212 | 0: chrom1 start1 213 | 1: start1 end1 214 | 2: end1 start2 215 | 3: chrom2 end2 216 | 4: start2 dataset 217 | 5: end2 zoomLevel 218 | 6: dataset dim* 219 | 7: zoomOutLevel 220 | 8: dim* 221 | 222 | *) Optional 223 | ''' 224 | 225 | params = get_params(request, GET_FRAG_PARAMS) 226 | 227 | dims = params['dims'] 228 | padding = params['padding'] 229 | no_balance = params['no-balance'] 230 | percentile = params['percentile'] 231 | precision = params['precision'] 232 | no_cache = params['no-cache'] 233 | ignore_diags = params['ignore-diags'] 234 | no_normalize = params['no-normalize'] 235 | aggregate = params['aggregate'] 236 | aggregation_method = params['aggregation-method'] 237 | max_previews = params['max-previews'] 238 | encoding = params['encoding'] 239 | representatives = params['representatives'] 240 | 241 | # Check if requesting a snippet from a `.cool` cooler file 242 | is_cool = len(loci) and len(loci[0]) > 7 243 | tileset_idx = 6 if is_cool else 4 244 | zoom_level_idx = tileset_idx + 1 245 | 246 | filetype = None 247 | new_filetype = None 248 | previews = [] 249 | previews_2d = [] 250 | ts_cache = {} 251 | mat_idx = None 252 | 253 | total_valid_loci = 0 254 | loci_lists = {} 255 | loci_ids = [] 256 | try: 257 | for locus in loci: 258 | tileset_file = '' 259 | 260 | if locus[tileset_idx]: 261 | if locus[tileset_idx] in ts_cache: 262 | tileset = ts_cache[locus[tileset_idx]]['obj'] 263 | tileset_file = ts_cache[locus[tileset_idx]]['path'] 264 | elif locus[tileset_idx].endswith('.cool'): 265 | tileset_file = path.join('data', locus[tileset_idx]) 266 | else: 267 | try: 268 | tileset = Tileset.objects.get( 269 | uuid=locus[tileset_idx] 270 | ) 271 | tileset_file = tileset.datafile.path 272 | ts_cache[locus[tileset_idx]] = { 273 | "obj": tileset, 274 | "path": tileset_file 275 | } 276 | 277 | except AttributeError: 278 | return JsonResponse({ 279 | 'error': 'Tileset ({}) does not exist'.format( 280 | locus[tileset_idx] 281 | ), 282 | }, status=400) 283 | except Tileset.DoesNotExist: 284 | if locus[tileset_idx].startswith('osm'): 285 | new_filetype = locus[tileset_idx] 286 | else: 287 | return JsonResponse({ 288 | 'error': 'Tileset ({}) does not exist'.format( 289 | locus[tileset_idx] 290 | ), 291 | }, status=400) 292 | else: 293 | return JsonResponse({ 294 | 'error': 'Tileset not specified', 295 | }, status=400) 296 | 297 | # Get the dimensions of the snippets (i.e., width and height in px) 298 | inset_dim = ( 299 | locus[zoom_level_idx + 1] 300 | if ( 301 | len(locus) >= zoom_level_idx + 2 and 302 | locus[zoom_level_idx + 1] 303 | ) 304 | else None 305 | ) 306 | out_dim = dims if inset_dim is None else inset_dim 307 | 308 | # Make sure out dim (in pixel) is not too large 309 | if ( 310 | (is_cool and out_dim > hss.SNIPPET_MAT_MAX_OUT_DIM) or 311 | (not is_cool and out_dim > hss.SNIPPET_IMG_MAX_OUT_DIM) 312 | ): 313 | return JsonResponse({ 314 | 'error': 'Snippet too large', 315 | 'error_message': str(SnippetTooLarge()) 316 | }, status=400) 317 | 318 | if tileset_file not in loci_lists: 319 | loci_lists[tileset_file] = {} 320 | 321 | if is_cool: 322 | # Get max abs dim in base pairs 323 | max_abs_dim = max(locus[2] - locus[1], locus[5] - locus[4]) 324 | 325 | with h5py.File(tileset_file, 'r') as f: 326 | # get base resolution (bin size) of cooler file 327 | if 'resolutions' in f: 328 | # v2 329 | resolutions = sorted( 330 | [int(key) for key in f['resolutions'].keys()] 331 | ) 332 | closest_res = 0 333 | for i, res in enumerate(resolutions): 334 | if (max_abs_dim / out_dim) - res < 0: 335 | closest_res = resolutions[max(0, i - 1)] 336 | break 337 | zoomout_level = ( 338 | locus[zoom_level_idx] 339 | if locus[zoom_level_idx] >= 0 340 | else closest_res 341 | ) 342 | else: 343 | # v1 344 | max_zoom = f.attrs['max-zoom'] 345 | bin_size = int(f[str(max_zoom)].attrs['bin-size']) 346 | 347 | # Find closest zoom level if `zoomout_level < 0` 348 | # Assuming resolutions of powers of 2 349 | zoomout_level = ( 350 | locus[zoom_level_idx] 351 | if locus[zoom_level_idx] >= 0 352 | else floor(log((max_abs_dim / bin_size) / out_dim, 2)) 353 | ) 354 | 355 | else: 356 | # Get max abs dim in base pairs 357 | max_abs_dim = max(locus[1] - locus[0], locus[3] - locus[2]) 358 | 359 | bin_size = 1 360 | 361 | # Find closest zoom level if `zoomout_level < 0` 362 | # Assuming resolutions of powers of 2 363 | zoomout_level = ( 364 | locus[zoom_level_idx] 365 | if locus[zoom_level_idx] >= 0 366 | else floor(log((max_abs_dim / bin_size) / out_dim, 2)) 367 | ) 368 | 369 | if zoomout_level not in loci_lists[tileset_file]: 370 | loci_lists[tileset_file][zoomout_level] = [] 371 | 372 | locus_id = '.'.join(map(str, locus)) 373 | 374 | loci_lists[tileset_file][zoomout_level].append( 375 | locus[0:tileset_idx] + [total_valid_loci, inset_dim, locus_id] 376 | ) 377 | loci_ids.append(locus_id) 378 | 379 | if new_filetype is None: 380 | new_filetype = ( 381 | tileset.filetype 382 | if tileset 383 | else tileset_file[tileset_file.rfind('.') + 1:] 384 | ) 385 | 386 | if filetype is None: 387 | filetype = new_filetype 388 | 389 | if filetype != new_filetype: 390 | return JsonResponse({ 391 | 'error': ( 392 | 'Multiple file types per query are not supported yet.' 393 | ) 394 | }, status=400) 395 | 396 | total_valid_loci += 1 397 | 398 | except Exception as e: 399 | return JsonResponse({ 400 | 'error': 'Could not convert loci.', 401 | 'error_message': str(e) 402 | }, status=500) 403 | 404 | mat_idx = list(range(len(loci_ids))) 405 | 406 | # Get a unique string for caching 407 | dump = ( 408 | json.dumps(loci, sort_keys=True) + 409 | str(forced_rep_idx) + 410 | str(dims) + 411 | str(padding) + 412 | str(no_balance) + 413 | str(percentile) + 414 | str(precision) + 415 | str(ignore_diags) + 416 | str(no_normalize) + 417 | str(aggregate) + 418 | str(aggregation_method) + 419 | str(max_previews) + 420 | str(encoding) + 421 | str(representatives) 422 | ) 423 | uuid = hashlib.md5(dump.encode('utf-8')).hexdigest() 424 | 425 | # Check if something is cached 426 | if not no_cache: 427 | try: 428 | results = rdb.get('frag_by_loci_%s' % uuid) 429 | if results: 430 | return JsonResponse(pickle.loads(results)) 431 | except: 432 | pass 433 | 434 | matrices = [None] * total_valid_loci 435 | data_types = [None] * total_valid_loci 436 | try: 437 | for dataset in loci_lists: 438 | for zoomout_level in loci_lists[dataset]: 439 | if filetype == 'cooler' or filetype == 'cool': 440 | raw_matrices = get_frag_by_loc_from_cool( 441 | dataset, 442 | loci_lists[dataset][zoomout_level], 443 | dims, 444 | zoomout_level=zoomout_level, 445 | balanced=not no_balance, 446 | padding=int(padding), 447 | percentile=percentile, 448 | ignore_diags=ignore_diags, 449 | no_normalize=no_normalize, 450 | aggregate=aggregate, 451 | ) 452 | 453 | for i, matrix in enumerate(raw_matrices): 454 | idx = loci_lists[dataset][zoomout_level][i][6] 455 | matrices[idx] = matrix 456 | data_types[idx] = 'matrix' 457 | 458 | if filetype == 'imtiles' or filetype == 'osm-image': 459 | extractor = ( 460 | get_frag_by_loc_from_imtiles 461 | if filetype == 'imtiles' 462 | else get_frag_by_loc_from_osm 463 | ) 464 | 465 | sub_ims = extractor( 466 | imtiles_file=dataset, 467 | loci=loci_lists[dataset][zoomout_level], 468 | zoom_level=zoomout_level, 469 | padding=float(padding), 470 | no_cache=no_cache, 471 | ) 472 | 473 | for i, im in enumerate(sub_ims): 474 | idx = loci_lists[dataset][zoomout_level][i][4] 475 | 476 | matrices[idx] = im 477 | 478 | data_types[idx] = 'matrix' 479 | 480 | except Exception as ex: 481 | raise 482 | return JsonResponse({ 483 | 'error': 'Could not retrieve fragments.', 484 | 'error_message': str(ex) 485 | }, status=500) 486 | 487 | if aggregate and len(matrices) > 1: 488 | try: 489 | cover, previews_1d, previews_2d = aggregate_frags( 490 | matrices, 491 | loci_ids, 492 | aggregation_method, 493 | max_previews, 494 | ) 495 | matrices = [cover] 496 | mat_idx = [] 497 | if previews_1d is not None: 498 | previews = np.split( 499 | previews_1d, range(1, previews_1d.shape[0]) 500 | ) 501 | data_types = [data_types[0]] 502 | except Exception as ex: 503 | raise 504 | return JsonResponse({ 505 | 'error': 'Could not aggregate fragments.', 506 | 'error_message': str(ex) 507 | }, status=500) 508 | 509 | if representatives and len(matrices) > 1: 510 | if forced_rep_idx and len(forced_rep_idx) <= len(matrices): 511 | matrices = [matrices[i] for i in forced_rep_idx] 512 | mat_idx = forced_rep_idx 513 | data_types = [data_types[0]] * len(forced_rep_idx) 514 | else: 515 | try: 516 | rep_frags, rep_idx = get_rep_frags( 517 | matrices, loci, loci_ids, representatives, no_cache 518 | ) 519 | matrices = rep_frags 520 | mat_idx = rep_idx 521 | data_types = [data_types[0]] * len(rep_frags) 522 | except Exception as ex: 523 | raise 524 | return JsonResponse({ 525 | 'error': 'Could get representative fragments.', 526 | 'error_message': str(ex) 527 | }, status=500) 528 | 529 | if encoding != 'b64' and encoding != 'image': 530 | # Adjust precision and convert to list 531 | for i, matrix in enumerate(matrices): 532 | if precision > 0: 533 | matrix = np.round(matrix, decimals=precision) 534 | matrices[i] = matrix.tolist() 535 | 536 | if max_previews > 0: 537 | for i, preview in enumerate(previews): 538 | previews[i] = preview.tolist() 539 | for i, preview_2d in enumerate(previews_2d): 540 | previews_2d[i] = preview_2d.tolist() 541 | 542 | # Encode matrix if required 543 | if encoding == 'b64': 544 | for i, matrix in enumerate(matrices): 545 | id = loci_ids[mat_idx[i]] 546 | data_types[i] = 'dataUrl' 547 | if not no_cache and id: 548 | mat_b64 = None 549 | try: 550 | mat_b64 = rdb.get('im_b64_%s' % id) 551 | if mat_b64 is not None: 552 | matrices[i] = mat_b64.decode('ascii') 553 | continue 554 | except: 555 | pass 556 | 557 | mat_b64 = pybase64.b64encode(np_to_png(matrix)).decode('ascii') 558 | 559 | if not no_cache: 560 | try: 561 | rdb.set('im_b64_%s' % id, mat_b64, 60 * 30) 562 | except Exception as ex: 563 | # error caching a tile 564 | # log the error and carry forward, this isn't critical 565 | logger.warn(ex) 566 | 567 | matrices[i] = mat_b64 568 | 569 | if max_previews > 0: 570 | for i, preview in enumerate(previews): 571 | previews[i] = pybase64.b64encode( 572 | np_to_png(preview) 573 | ).decode('ascii') 574 | for i, preview_2d in enumerate(previews_2d): 575 | previews_2d[i] = pybase64.b64encode( 576 | np_to_png(preview_2d) 577 | ).decode('ascii') 578 | 579 | # Create results 580 | results = { 581 | 'fragments': matrices, 582 | 'indices': [int(i) for i in mat_idx], 583 | 'dataTypes': data_types, 584 | } 585 | 586 | # Return Y aggregates as 1D previews on demand 587 | if max_previews > 0: 588 | results['previews'] = previews 589 | results['previews2d'] = previews_2d 590 | 591 | # Cache results for 30 minutes 592 | try: 593 | rdb.set('frag_by_loci_%s' % uuid, pickle.dumps(results), 60 * 30) 594 | except Exception as ex: 595 | # error caching a tile 596 | # log the error and carry forward, this isn't critical 597 | logger.warn(ex) 598 | 599 | if encoding == 'image': 600 | if len(matrices) == 1: 601 | return HttpResponse( 602 | np_to_png(grey_to_rgb(matrices[0], to_rgba=True)), 603 | content_type='image/png' 604 | ) 605 | else: 606 | ims = [] 607 | for i, matrix in enumerate(matrices): 608 | ims.append({ 609 | 'name': '{}.png'.format(i), 610 | 'bytes': np_to_png(grey_to_rgb(matrix, to_rgba=True)) 611 | }) 612 | return blob_to_zip(ims, to_resp=True) 613 | 614 | return JsonResponse(results) 615 | 616 | 617 | @api_view(['GET']) 618 | @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) 619 | def fragments_by_chr(request): 620 | chrom = request.GET.get('chrom', False) 621 | cooler_file = request.GET.get('cooler', False) 622 | loop_list = request.GET.get('loop-list', False) 623 | 624 | if cooler_file: 625 | if cooler_file.endswith('.cool'): 626 | cooler_file = path.join('data', cooler_file) 627 | else: 628 | try: 629 | cooler_file = Tileset.objects.get(uuid=cooler_file).datafile.path 630 | except AttributeError: 631 | return JsonResponse({ 632 | 'error': 'Cooler file not in database', 633 | }, status=500) 634 | else: 635 | return JsonResponse({ 636 | 'error': 'Cooler file not specified', 637 | }, status=500) 638 | 639 | try: 640 | measures = request.GET.getlist('measures', []) 641 | except ValueError: 642 | measures = [] 643 | 644 | try: 645 | zoomout_level = int(request.GET.get('zoomout-level', -1)) 646 | except ValueError: 647 | zoomout_level = -1 648 | 649 | try: 650 | limit = int(request.GET.get('limit', -1)) 651 | except ValueError: 652 | limit = -1 653 | 654 | try: 655 | precision = int(request.GET.get('precision', False)) 656 | except ValueError: 657 | precision = False 658 | 659 | try: 660 | no_cache = bool(request.GET.get('no-cache', False)) 661 | except ValueError: 662 | no_cache = False 663 | 664 | try: 665 | for_config = bool(request.GET.get('for-config', False)) 666 | except ValueError: 667 | for_config = False 668 | 669 | # Get a unique string for the URL query string 670 | uuid = hashlib.md5( 671 | '-'.join([ 672 | cooler_file, 673 | chrom, 674 | loop_list, 675 | str(limit), 676 | str(precision), 677 | str(zoomout_level) 678 | ]) 679 | ).hexdigest() 680 | 681 | # Check if something is cached 682 | if not no_cache: 683 | try: 684 | results = rdb.get('frag_by_chrom_%s' % uuid) 685 | 686 | if results: 687 | return JsonResponse(pickle.loads(results)) 688 | except: 689 | pass 690 | 691 | # Get relative loci 692 | try: 693 | (loci_rel, chroms) = get_intra_chr_loops_from_looplist( 694 | path.join('data', loop_list), chrom 695 | ) 696 | except Exception as e: 697 | return JsonResponse({ 698 | 'error': 'Could not retrieve loci.', 699 | 'error_message': str(e) 700 | }, status=500) 701 | 702 | # Convert to chromosome-relative loci list 703 | loci_rel_chroms = np.column_stack( 704 | (chroms[:, 0], loci_rel[:, 0:2], chroms[:, 1], loci_rel[:, 2:4]) 705 | ) 706 | 707 | if limit > 0: 708 | loci_rel_chroms = loci_rel_chroms[:limit] 709 | 710 | # Get fragments 711 | try: 712 | matrices = get_frag_by_loc_from_cool( 713 | cooler_file, 714 | loci_rel_chroms, 715 | zoomout_level=zoomout_level 716 | ) 717 | except Exception as e: 718 | return JsonResponse({ 719 | 'error': 'Could not retrieve fragments.', 720 | 'error_message': str(e) 721 | }, status=500) 722 | 723 | if precision > 0: 724 | matrices = np.around(matrices, decimals=precision) 725 | 726 | fragments = [] 727 | 728 | loci_struct = rel_loci_2_obj(loci_rel_chroms) 729 | 730 | # Check supported measures 731 | measures_applied = [] 732 | for measure in measures: 733 | if measure in SUPPORTED_MEASURES: 734 | measures_applied.append(measure) 735 | 736 | i = 0 737 | for matrix in matrices: 738 | measures_values = [] 739 | 740 | for measure in measures: 741 | if measure == 'distance-to-diagonal': 742 | measures_values.append( 743 | calc_measure_dtd(matrix, loci_struct[i]) 744 | ) 745 | 746 | if measure == 'size': 747 | measures_values.append( 748 | calc_measure_size(matrix, loci_struct[i]) 749 | ) 750 | 751 | if measure == 'noise': 752 | measures_values.append(calc_measure_noise(matrix)) 753 | 754 | if measure == 'sharpness': 755 | measures_values.append(calc_measure_sharpness(matrix)) 756 | 757 | frag_obj = { 758 | # 'matrix': matrix.tolist() 759 | } 760 | 761 | frag_obj.update(loci_struct[i]) 762 | frag_obj.update({ 763 | "measures": measures_values 764 | }) 765 | fragments.append(frag_obj) 766 | i += 1 767 | 768 | # Create results 769 | results = { 770 | 'count': matrices.shape[0], 771 | 'dims': matrices.shape[1], 772 | 'fragments': fragments, 773 | 'measures': measures_applied, 774 | 'relativeLoci': True, 775 | 'zoomoutLevel': zoomout_level 776 | } 777 | 778 | if for_config: 779 | results['fragmentsHeader'] = [ 780 | 'chrom1', 781 | 'start1', 782 | 'end1', 783 | 'strand1', 784 | 'chrom2', 785 | 'start2', 786 | 'end2', 787 | 'strand2' 788 | ] + measures_applied 789 | 790 | fragments_arr = [] 791 | for fragment in fragments: 792 | tmp = [ 793 | fragment['chrom1'], 794 | fragment['start1'], 795 | fragment['end1'], 796 | fragment['strand1'], 797 | fragment['chrom2'], 798 | fragment['start2'], 799 | fragment['end2'], 800 | fragment['strand2'], 801 | ] + fragment['measures'] 802 | 803 | fragments_arr.append(tmp) 804 | 805 | results['fragments'] = fragments_arr 806 | 807 | # Cache results for 30 mins 808 | try: 809 | rdb.set('frag_by_chrom_%s' % uuid, pickle.dumps(results), 60 * 30) 810 | except Exception as ex: 811 | # error caching a tile 812 | # log the error and carry forward, this isn't critical 813 | logger.warn(ex) 814 | 815 | return JsonResponse(results) 816 | 817 | 818 | @api_view(['GET']) 819 | @authentication_classes((CsrfExemptSessionAuthentication, BasicAuthentication)) 820 | def loci(request): 821 | chrom = request.GET.get('chrom', False) 822 | loop_list = request.GET.get('loop-list', False) 823 | 824 | # Get relative loci 825 | (loci_rel, chroms) = get_intra_chr_loops_from_looplist( 826 | path.join('data', loop_list), chrom 827 | ) 828 | 829 | loci_rel_chroms = np.column_stack( 830 | (chroms[:, 0], loci_rel[:, 0:2], chroms[:, 1], loci_rel[:, 2:4]) 831 | ) 832 | 833 | # Create results 834 | results = { 835 | 'loci': rel_loci_2_obj(loci_rel_chroms) 836 | } 837 | 838 | return JsonResponse(results) 839 | -------------------------------------------------------------------------------- /higlass_server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/higlass/higlass-server/cbfe79fe3ae0e844b4c0c78142a83733c8cc66a2/higlass_server/__init__.py -------------------------------------------------------------------------------- /higlass_server/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for tutorial project. 3 | 4 | Generated by 'django-admin startproject' using Django 1.10.2. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/1.10/ref/settings/ 11 | """ 12 | 13 | import json 14 | import os 15 | import os.path as op 16 | import slugid 17 | import math 18 | 19 | from django.core.exceptions import ImproperlyConfigured 20 | 21 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 22 | if 'HIGLASS_SERVER_BASE_DIR' in os.environ: 23 | base_dir = os.environ['HIGLASS_SERVER_BASE_DIR'] 24 | 25 | if op.exists(base_dir): 26 | BASE_DIR = base_dir 27 | else: 28 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 29 | else: 30 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 31 | 32 | if 'HIGLASS_CACHE_DIR' in os.environ: 33 | # cache uploaded files 34 | # useful when using a mounted media directory 35 | CACHE_DIR = os.environ['HIGLASS_CACHE_DIR'] 36 | else: 37 | CACHE_DIR = None 38 | 39 | 40 | if 'MAX_BAM_TILE_WIDTH' in os.environ: 41 | MAX_BAM_TILE_WIDTH = int(os.environ['MAX_BAM_TILE_WIDTH']) 42 | else: 43 | MAX_BAM_TILE_WIDTH = int(1e5) 44 | 45 | if 'MAX_FASTA_TILE_WIDTH' in os.environ: 46 | MAX_FASTA_TILE_WIDTH = int(os.environ['MAX_FASTA_TILE_WIDTH']) 47 | else: 48 | MAX_FASTA_TILE_WIDTH = int(1e5) 49 | 50 | local_settings_file_path = os.path.join( 51 | BASE_DIR, 'config.json' 52 | ) 53 | 54 | # load config.json 55 | try: 56 | with open(local_settings_file_path, 'r') as f: 57 | local_settings = json.load(f) 58 | except IOError: 59 | local_settings = {} 60 | except ValueError as e: 61 | error_msg = "Invalid config '{}': {}".format(local_settings_file_path, e) 62 | raise ImproperlyConfigured(error_msg) 63 | 64 | 65 | def get_setting(name, default=None, settings=local_settings): 66 | """Get the local settings variable or return explicit exception""" 67 | if default is None: 68 | raise ImproperlyConfigured( 69 | "Missing default value for '{0}'".format(name) 70 | ) 71 | 72 | # Try looking up setting in `config.json` first 73 | try: 74 | return settings[name] 75 | except KeyError: 76 | pass 77 | 78 | # If setting is not found try looking for an env var 79 | try: 80 | return os.environ[name] 81 | 82 | # If nothing is found return the default setting 83 | except KeyError: 84 | if default is not None: 85 | return default 86 | else: 87 | raise ImproperlyConfigured( 88 | "Missing setting for '{0}' setting".format(name) 89 | ) 90 | 91 | 92 | # SECURITY WARNING: keep the secret key used in production secret! 93 | SECRET_KEY = get_setting('SECRET_KEY', slugid.nice()) 94 | 95 | # SECURITY WARNING: don't run with debug turned on in production! 96 | DEBUG = get_setting('DEBUG', False) 97 | 98 | ALLOWED_HOSTS = [ 99 | '*', 100 | ] 101 | 102 | if 'SITE_URL' in os.environ: 103 | ALLOWED_HOSTS += [os.environ['SITE_URL']] 104 | 105 | # this specifies where uploaded files will be place 106 | # (e.g. BASE_DIR/media/uplaods/file.x) 107 | MEDIA_URL = 'media/' 108 | 109 | if 'HIGLASS_MEDIA_ROOT' in os.environ: 110 | MEDIA_ROOT = os.environ['HIGLASS_MEDIA_ROOT'] 111 | else: 112 | MEDIA_ROOT = os.path.join(BASE_DIR, 'media') 113 | 114 | if 'HTTPFS_HTTP_DIR' in os.environ: 115 | HTTPFS_HTTP_DIR = os.environ['HTTPFS_HTTP_DIR'] 116 | else: 117 | HTTPFS_HTTP_DIR = os.path.join(MEDIA_ROOT, 'http') 118 | 119 | if 'HTTPFS_HTTPS_DIR' in os.environ: 120 | HTTPFS_HTTPS_DIR = os.environ['HTTPFS_HTTPS_DIR'] 121 | else: 122 | HTTPFS_HTTPS_DIR = os.path.join(MEDIA_ROOT, 'https') 123 | 124 | if 'HTTPFS_FTP_DIR' in os.environ: 125 | HTTPFS_FTP_DIR = os.environ['HTTPFS_FTP_DIR'] 126 | else: 127 | HTTPFS_FTP_DIR = os.path.join(MEDIA_ROOT, 'ftp') 128 | 129 | THUMBNAILS_ROOT = os.path.join(MEDIA_ROOT, 'thumbnails') 130 | AWS_BUCKET_MOUNT_POINT = os.path.join(MEDIA_ROOT, 'aws') 131 | THUMBNAIL_RENDER_URL_BASE = '/app/' 132 | 133 | LOGGING = { 134 | 'version': 1, 135 | 'disable_existing_loggers': False, 136 | 'formatters': { 137 | 'verbose': { 138 | 'format': 139 | "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", 140 | 'datefmt': "%d/%b/%Y %H:%M:%S" 141 | }, 142 | 'simple': { 143 | 'format': '%(levelname)s %(message)s' 144 | }, 145 | }, 146 | 'handlers': { 147 | 'console': { 148 | 'level': get_setting('LOG_LEVEL_CONSOLE', 'WARNING'), 149 | 'class': 'logging.StreamHandler', 150 | 'formatter': 'simple' 151 | }, 152 | 'file': { 153 | 'level': get_setting('LOG_LEVEL_FILE', 'WARNING'), 154 | 'class': 'logging.FileHandler', 155 | 'filename': os.path.join(BASE_DIR, 'log/hgs.log'), 156 | 'formatter': 'verbose' 157 | }, 158 | }, 159 | 'loggers': { 160 | 'django': { 161 | 'handlers': ['file'], 162 | 'propagate': True, 163 | 'level': get_setting('LOG_LEVEL_DJANGO', 'WARNING'), 164 | }, 165 | 'fragments': { 166 | 'handlers': ['file'], 167 | 'level': get_setting('LOG_LEVEL_FRAGMENTS', 'WARNING'), 168 | }, 169 | 'tilesets': { 170 | 'handlers': ['file'], 171 | 'level': get_setting('LOG_LEVEL_TILESETS', 'WARNING'), 172 | }, 173 | } 174 | } 175 | 176 | if DEBUG: 177 | # make all loggers use the console. 178 | for logger in LOGGING['loggers']: 179 | LOGGING['loggers'][logger]['handlers'] = ['console'] 180 | 181 | if 'REDIS_HOST' in os.environ and 'REDIS_PORT' in os.environ: 182 | REDIS_HOST = os.environ['REDIS_HOST'] 183 | REDIS_PORT = os.environ['REDIS_PORT'] 184 | else: 185 | REDIS_HOST = None 186 | REDIS_PORT = None 187 | 188 | # DEFAULT_FILE_STORAGE = 'tilesets.storage.HashedFilenameFileSystemStorage' 189 | 190 | # Application definition 191 | 192 | INSTALLED_APPS = [ 193 | 'django.contrib.admin', 194 | 'django.contrib.auth', 195 | 'django.contrib.contenttypes', 196 | 'django.contrib.sessions', 197 | 'django.contrib.messages', 198 | 'django.contrib.staticfiles', 199 | 'rest_framework', 200 | 'tilesets.apps.TilesetsConfig', 201 | 'fragments.app.FragmentsConfig', 202 | 'rest_framework_swagger', 203 | 'corsheaders', 204 | 'guardian' 205 | ] 206 | 207 | # We want to avoid loading into memory 208 | FILE_UPLOAD_HANDLERS = [ 209 | 'django.core.files.uploadhandler.TemporaryFileUploadHandler' 210 | ] 211 | 212 | MIDDLEWARE = [ 213 | 'django.middleware.security.SecurityMiddleware', 214 | 'django.contrib.sessions.middleware.SessionMiddleware', 215 | 'corsheaders.middleware.CorsMiddleware', 216 | 'django.middleware.common.CommonMiddleware', 217 | # 'django.middleware.csrf.CsrfViewMiddleware', 218 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 219 | 'django.contrib.messages.middleware.MessageMiddleware', 220 | # 'django.middleware.clickjacking.XFrameOptionsMiddleware', 221 | ] 222 | 223 | AUTHENTICATION_BACKENDS = ( 224 | 'django.contrib.auth.backends.ModelBackend', # this is default 225 | 'guardian.backends.ObjectPermissionBackend', 226 | ) 227 | 228 | CORS_ORIGIN_ALLOW_ALL = True 229 | # CORS_ALLOW_CREDENTIALS = False 230 | 231 | CORS_ORIGIN_WHITELIST = [ 232 | 'http://134.174.140.208:9000' 233 | ] 234 | 235 | # CORS_ALLOW_HEADERS = default_headers 236 | 237 | ROOT_URLCONF = 'higlass_server.urls' 238 | 239 | TEMPLATES = [ 240 | { 241 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 242 | 'DIRS': [], 243 | 'APP_DIRS': True, 244 | 'OPTIONS': { 245 | 'context_processors': [ 246 | 'django.template.context_processors.debug', 247 | 'django.template.context_processors.request', 248 | 'django.contrib.auth.context_processors.auth', 249 | 'django.contrib.messages.context_processors.messages', 250 | ], 251 | }, 252 | }, 253 | ] 254 | 255 | WSGI_APPLICATION = 'higlass_server.wsgi.application' 256 | 257 | 258 | # Database 259 | # https://docs.djangoproject.com/en/1.10/ref/settings/#databases 260 | 261 | DATABASES = { 262 | 'default': { 263 | 'ENGINE': 'django.db.backends.sqlite3', 264 | 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 265 | } 266 | } 267 | 268 | 269 | # Password validation 270 | # https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators 271 | 272 | AUTH_PASSWORD_VALIDATORS = [{ 273 | 'NAME': 274 | 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 275 | }, { 276 | 'NAME': 277 | 'django.contrib.auth.password_validation.MinimumLengthValidator', 278 | }, { 279 | 'NAME': 280 | 'django.contrib.auth.password_validation.CommonPasswordValidator', 281 | }, { 282 | 'NAME': 283 | 'django.contrib.auth.password_validation.NumericPasswordValidator', 284 | }] 285 | 286 | REST_FRAMEWORK = { 287 | 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', 288 | 'PAGE_SIZE': 10, 289 | 'DEFAULT_RENDERER_CLASSES': ( 290 | 'rest_framework.renderers.JSONRenderer', 291 | ) 292 | } 293 | # Internationalization 294 | # https://docs.djangoproject.com/en/1.10/topics/i18n/ 295 | 296 | LANGUAGE_CODE = 'en-us' 297 | 298 | TIME_ZONE = 'UTC' 299 | 300 | USE_I18N = True 301 | 302 | USE_L10N = True 303 | 304 | USE_TZ = True 305 | 306 | UPLOAD_ENABLED = get_setting('UPLOAD_ENABLED', True) 307 | PUBLIC_UPLOAD_ENABLED = get_setting('PUBLIC_UPLOAD_ENABLED', True) 308 | 309 | SNIPPET_MAT_MAX_OUT_DIM = get_setting('SNIPPET_MAT_MAX_OUT_DIM', 512) 310 | SNIPPET_MAT_MAX_DATA_DIM = get_setting('SNIPPET_MAT_MAX_DATA_DIM', 4096) 311 | SNIPPET_IMG_MAX_OUT_DIM = get_setting('SNIPPET_IMG_MAX_OUT_DIM', 1024) 312 | SNIPPET_OSM_MAX_DATA_DIM = get_setting('SNIPPET_OSM_MAX_DATA_DIM', 2048) 313 | SNIPPET_IMT_MAX_DATA_DIM = get_setting('SNIPPET_IMT_MAX_DATA_DIM', 2048) 314 | 315 | 316 | # Static files (CSS, JavaScript, Images) 317 | # https://docs.djangoproject.com/en/1.10/howto/static-files/ 318 | 319 | STATIC_URL = '/hgs-static/' 320 | STATIC_ROOT = 'hgs-static/' 321 | 322 | # allow multiple proxies, for i.e. tls termination 323 | # see https://docs.djangoproject.com/en/2.2/_modules/django/http/request/ 324 | USE_X_FORWARDED_HOST = True 325 | SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') 326 | 327 | if 'APP_BASEPATH' in os.environ: 328 | # https://stackoverflow.com/questions/44987110/django-in-subdirectory-admin-site-is-not-working 329 | FORCE_SCRIPT_NAME = os.environ['APP_BASEPATH'] 330 | SESSION_COOKIE_PATH = os.environ['APP_BASEPATH'] 331 | LOGIN_REDIRECT_URL = os.environ['APP_BASEPATH'] 332 | LOGOUT_REDIRECT_URL = os.environ['APP_BASEPATH'] 333 | 334 | STATIC_URL = op.join(os.environ['APP_BASEPATH'], 'hgs-static') + "/" 335 | 336 | ADMIN_URL = r'^admin/' 337 | 338 | # STATICFILES_DIRS = ( 339 | # os.path.join(BASE_DIR, 'static'), 340 | # ) 341 | 342 | # TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' 343 | 344 | NOSE_ARGS = ['--nocapture', '--nologcapture'] 345 | -------------------------------------------------------------------------------- /higlass_server/settings_test.py: -------------------------------------------------------------------------------- 1 | from .settings import * 2 | 3 | DATABASES = { 4 | 'default': { 5 | 'ENGINE': 'django.db.backends.sqlite3', 6 | 'NAME': os.path.join(BASE_DIR, 'db_test.sqlite3'), 7 | } 8 | } 9 | 10 | DEBUG = False 11 | -------------------------------------------------------------------------------- /higlass_server/tests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import slugid 3 | import subprocess 4 | 5 | import tilesets.models as tm 6 | 7 | class CommandlineTest(unittest.TestCase): 8 | def setUp(self): 9 | # TODO: There is probably a better way to clear data from previous test runs. Is it even necessary? 10 | # self.assertRun('python manage.py flush --noinput --settings=higlass_server.settings_test') 11 | pass 12 | 13 | def assertRun(self, command, output_res=[]): 14 | output = subprocess.check_output(command , shell=True).decode('utf-8').strip() 15 | for output_re in output_res: 16 | self.assertRegexpMatches(output, output_re) 17 | return output 18 | 19 | def test_hello(self): 20 | self.assertRun('echo "hello?"', [r'hello']) 21 | 22 | def test_bamfile_upload_with_index(self): 23 | settings = 'higlass_server.settings_test' 24 | uid = slugid.nice() 25 | 26 | self.assertRun('python manage.py ingest_tileset' + 27 | ' --filename data/SRR1770413.mismatched_bai.bam' + 28 | ' --indexfile data/SRR1770413.different_index_filename.bai' + 29 | ' --datatype reads' + 30 | ' --filetype bam' + 31 | ' --uid '+uid+' --settings='+settings) 32 | 33 | self.assertRun('python manage.py shell ' + 34 | '--settings ' + settings + 35 | ' --command="' + 36 | 'import tilesets.models as tm; '+ 37 | f'o = tm.Tileset.objects.get(uuid=\'{uid}\');' 38 | 'print(o.indexfile)"', '.bai$') 39 | 40 | def test_bamfile_upload_without_index(self): 41 | settings = 'higlass_server.settings_test' 42 | uid = slugid.nice() 43 | 44 | self.assertRun('python manage.py ingest_tileset' + 45 | ' --filename data/SRR1770413.sorted.short.bam' + 46 | ' --datatype reads' + 47 | ' --filetype bam' + 48 | ' --uid '+uid+' --settings='+settings) 49 | 50 | self.assertRun('python manage.py shell ' + 51 | '--settings ' + settings + 52 | ' --command="' + 53 | 'import tilesets.models as tm; '+ 54 | f'o = tm.Tileset.objects.get(uuid=\'{uid}\');' 55 | 'print(o.indexfile)"', '.bai$') 56 | 57 | def test_cli_upload(self): 58 | cooler = 'dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool' 59 | settings = 'higlass_server.settings_test' 60 | id = 'cli-test' 61 | self.assertRun('python manage.py ingest_tileset --filename data/'+cooler+' --datatype matrix --filetype cooler --uid '+id+' --settings='+settings) 62 | self.assertRun('curl -s http://localhost:6000/api/v1/tileset_info/?d='+id, 63 | [r'"name": "'+cooler+'"']) 64 | self.assertRun('curl -s http://localhost:6000/api/v1/tiles/?d='+id+'.1.1.1', 65 | [r'"'+id+'.1.1.1":', 66 | r'"max_value": 2.0264008045196533', 67 | r'"min_value": 0.0', 68 | r'"dense": "JTInPwAA']) 69 | 70 | def test_cli_huge_upload(self): 71 | cooler = 'huge.fake.cool' 72 | with open('data/'+cooler, 'w') as file: 73 | file.truncate(1024 ** 3) 74 | settings = 'higlass_server.settings_test' 75 | id = 'cli-huge-test' 76 | self.assertRun('python manage.py ingest_tileset --filename data/'+cooler+' --datatype foo --filetype bar --uid '+id+' --settings='+settings) 77 | self.assertRun('curl -s http://localhost:6000/api/v1/tileset_info/?d='+id, 78 | [r'"name": "'+cooler+'"']) 79 | self.assertRun('curl -s http://localhost:6000/api/v1/tiles/?d='+id+'.1.1.1', 80 | [r'"'+id+'.1.1.1"']) 81 | 82 | ''' 83 | id = 'cli-coord-system-test' 84 | self.assertRun('python manage.py ingest_tileset --filename data/'+cooler+' --datatype foo --filetype bar --uid '+id+' --settings='+settings, 'coordSystem') 85 | self.assertRun('curl -s http://localhost:6000/api/v1/tileset_info/?d='+id, 86 | [r'"coordSystem": "'+cooler+'"']) 87 | ''' 88 | # TODO: check the coordSystem parameters for ingest_tileset.py 89 | 90 | def test_get_from_foreign_host_file(self): 91 | # manage.py should have been started with 92 | # export SITE_URL=somesite.com 93 | #self.assertRun('curl -s -H "Host: someothersite.com" http://localhost:6000/api/v1/tilesets/', [r'400']) 94 | #self.assertRun('curl -s -H "Host: somesite.com" http://localhost:6000/api/v1/tilesets/', [r'count']) 95 | pass 96 | 97 | -------------------------------------------------------------------------------- /higlass_server/urls.py: -------------------------------------------------------------------------------- 1 | """tutorial URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/1.10/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.conf.urls import url, include 14 | 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) 15 | """ 16 | from django.conf.urls import url, include 17 | from django.contrib import admin 18 | from django.conf import settings 19 | 20 | urlpatterns = [ 21 | url(settings.ADMIN_URL, admin.site.urls), 22 | url(r'^api/v1/', include('tilesets.urls')), 23 | url(r'^api/v1/', include('fragments.urls')), 24 | url(r'^', include('website.urls')), 25 | ] 26 | -------------------------------------------------------------------------------- /higlass_server/utils.py: -------------------------------------------------------------------------------- 1 | import redis 2 | import higlass_server.settings as hss 3 | 4 | from redis.exceptions import ConnectionError 5 | 6 | 7 | class EmptyRDB: 8 | def __init__(self): 9 | pass 10 | 11 | def exists(self, name): 12 | return False 13 | 14 | def get(self, name): 15 | return None 16 | 17 | def set(self, name, value, ex=None, px=None, nx=False, xx=False): 18 | pass 19 | 20 | 21 | def getRdb(): 22 | if hss.REDIS_HOST is not None: 23 | try: 24 | rdb = redis.Redis( 25 | host=hss.REDIS_HOST, 26 | port=hss.REDIS_PORT) 27 | 28 | # Test server connection 29 | rdb.ping() 30 | 31 | return rdb 32 | except ConnectionError: 33 | return EmptyRDB() 34 | else: 35 | return EmptyRDB() 36 | -------------------------------------------------------------------------------- /higlass_server/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for tutorial project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | import sys 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | 15 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "higlass_server.settings") 16 | 17 | application = get_wsgi_application() 18 | -------------------------------------------------------------------------------- /log/.keep: -------------------------------------------------------------------------------- 1 | Keep this directory in the git repository 2 | -------------------------------------------------------------------------------- /manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ['DJANGO_SETTINGS_MODULE'] = "higlass_server.settings" 7 | try: 8 | from django.core.management import execute_from_command_line 9 | except ImportError: 10 | # The above import may fail for some other reason. Ensure that the 11 | # issue is really that Django is missing to avoid masking other 12 | # exceptions on Python 2. 13 | try: 14 | import django 15 | except ImportError: 16 | raise ImportError( 17 | "Couldn't import Django. Are you sure it's installed and " 18 | "available on your PYTHONPATH environment variable? Did you " 19 | "forget to activate a virtual environment?" 20 | ) 21 | raise 22 | execute_from_command_line(sys.argv) 23 | -------------------------------------------------------------------------------- /managedb.sh: -------------------------------------------------------------------------------- 1 | python manage.py makemigrations 2 | python manage.py migrate 3 | -------------------------------------------------------------------------------- /nginx/nginx.conf: -------------------------------------------------------------------------------- 1 | user www-data; 2 | worker_processes auto; 3 | pid /run/nginx.pid; 4 | 5 | events { 6 | worker_connections 768; 7 | # multi_accept on; 8 | } 9 | 10 | http { 11 | 12 | ## 13 | # Basic Settings 14 | ## 15 | 16 | sendfile on; 17 | tcp_nopush on; 18 | tcp_nodelay on; 19 | keepalive_timeout 65; 20 | types_hash_max_size 2048; 21 | # server_tokens off; 22 | 23 | log_format main '$remote_addr - $remote_user [$time_local] $status ' 24 | '"$request" $body_bytes_sent "$http_referer" ' 25 | '"$http_user_agent" "$http_x_forwarded_for" ' 26 | '$gzip_ratio $request_time' ; 27 | 28 | # server_names_hash_bucket_size 64; 29 | # server_name_in_redirect off; 30 | 31 | include /etc/nginx/mime.types; 32 | default_type application/octet-stream; 33 | 34 | ## 35 | # SSL Settings 36 | ## 37 | 38 | ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE 39 | ssl_prefer_server_ciphers on; 40 | 41 | ## 42 | # Logging Settings 43 | ## 44 | 45 | access_log /home/ubuntu/data/hg-local/log/access.log main; 46 | error_log /home/ubuntu/data/hg-local/log/error.log; 47 | 48 | ## 49 | # Gzip Settings 50 | ## 51 | 52 | gzip on; 53 | gzip_disable "msie6"; 54 | 55 | gzip_vary on; 56 | gzip_proxied any; 57 | gzip_comp_level 6; 58 | gzip_buffers 16 8k; 59 | gzip_http_version 1.1; 60 | gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; 61 | 62 | ## 63 | # Virtual Host Configs 64 | ## 65 | 66 | include /etc/nginx/conf.d/*.conf; 67 | include /etc/nginx/sites-enabled/*; 68 | } 69 | -------------------------------------------------------------------------------- /nginx/sites-enabled/hgserver_nginx.conf: -------------------------------------------------------------------------------- 1 | # the upstream component nginx needs to connect to 2 | upstream django { 3 | server 127.0.0.1:8001; # for a web port socket 4 | # server unix:///path/to/your/mysite/mysite.sock; # TODO: May be faster 5 | } 6 | 7 | # configuration of the server 8 | server { 9 | listen 80; 10 | charset utf-8; 11 | 12 | # max upload size 13 | client_max_body_size 10000M; # adjust to taste 14 | 15 | location /api/v1/ { 16 | uwsgi_pass django; 17 | uwsgi_read_timeout 600; 18 | include /home/ubuntu/projects/higlass-server/uwsgi_params; 19 | } 20 | 21 | location /admin/ { 22 | uwsgi_pass django; 23 | uwsgi_read_timeout 600; 24 | include /home/ubuntu/projects/higlass-server/uwsgi_params; 25 | } 26 | 27 | location /static { 28 | alias /home/ubuntu/projects/higlass-server/static/; 29 | } 30 | 31 | location / { 32 | alias /home/ubuntu/projects/higlass-website/; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /notebooks/.ipynb_checkpoints/benchmarks-checkpoint.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:6e4dcae24efa782554cea0cfb5d96dc48b18d6281cb34f6622d4b2b7d1bb798c" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "%load_ext autoreload\n", 16 | "%autoreload 2" 17 | ], 18 | "language": "python", 19 | "metadata": {}, 20 | "outputs": [], 21 | "prompt_number": 1 22 | }, 23 | { 24 | "cell_type": "code", 25 | "collapsed": false, 26 | "input": [ 27 | "import sys\n", 28 | "sys.path.append('../tilesets')\n", 29 | "import getter" 30 | ], 31 | "language": "python", 32 | "metadata": {}, 33 | "outputs": [], 34 | "prompt_number": 2 35 | }, 36 | { 37 | "cell_type": "code", 38 | "collapsed": false, 39 | "input": [ 40 | "import h5py" 41 | ], 42 | "language": "python", 43 | "metadata": {}, 44 | "outputs": [], 45 | "prompt_number": 3 46 | }, 47 | { 48 | "cell_type": "code", 49 | "collapsed": false, 50 | "input": [ 51 | "f1000 = h5py.File('../data/dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool', 'r')\n", 52 | "f10 = h5py.File('../data/dixon2012-h1hesc-hindiii-allreps-filtered.10kb.multires.cool', 'r')" 53 | ], 54 | "language": "python", 55 | "metadata": {}, 56 | "outputs": [], 57 | "prompt_number": 4 58 | }, 59 | { 60 | "cell_type": "code", 61 | "collapsed": false, 62 | "input": [], 63 | "language": "python", 64 | "metadata": {}, 65 | "outputs": [], 66 | "prompt_number": 4 67 | }, 68 | { 69 | "cell_type": "code", 70 | "collapsed": false, 71 | "input": [ 72 | "import cooler, numpy as np\n" 73 | ], 74 | "language": "python", 75 | "metadata": {}, 76 | "outputs": [], 77 | "prompt_number": 5 78 | }, 79 | { 80 | "cell_type": "code", 81 | "collapsed": false, 82 | "input": [ 83 | "\n", 84 | "def getData3(f, zoomLevel, startPos1, endPos1, startPos2, endPos2):\n", 85 | " #t1 = time.time()\n", 86 | " #f = h5py.File(fpath,'r')\n", 87 | " #print(zoomLevel, startPos1, endPos1, startPos2, endPos2)\n", 88 | "\n", 89 | " c = cooler.Cooler(f[str(zoomLevel)])\n", 90 | " #matrix = c.matrix(balance=True, as_pixels=True, join=True)\n", 91 | " #cooler_matrix = {'cooler': c, 'matrix': matrix}\n", 92 | " #c = cooler_matrix['cooler']\n", 93 | "\n", 94 | " i0 = getter.absCoord2bin(c, startPos1)\n", 95 | " i1 = getter.absCoord2bin(c, endPos1)\n", 96 | " j0 = getter.absCoord2bin(c, startPos2)\n", 97 | " j1 = getter.absCoord2bin(c, endPos2)\n", 98 | "\n", 99 | "\n", 100 | " if (i1-i0) == 0 or (j1-j0) == 0:\n", 101 | " return pd.DataFrame(columns=['genome_start', 'genome_end', 'balanced'])\n", 102 | "\n", 103 | " pixels = c.matrix(as_pixels=True, max_chunk=np.inf)[i0:i1, j0:j1]\n", 104 | "\n", 105 | " if not len(pixels):\n", 106 | " return pd.DataFrame(columns=['genome_start', 'genome_end', 'balanced'])\n", 107 | "\n", 108 | " lo = min(i0, j0)\n", 109 | " hi = max(i1, j1)\n", 110 | " bins = c.bins()[['chrom', 'start', 'end', 'weight']][lo:hi]\n", 111 | " bins['chrom'] = bins['chrom'].cat.codes\n", 112 | " pixels = cooler.annotate(pixels, bins)\n", 113 | " pixels['genome_start'] = getter.cumul_lengths[pixels['chrom1']] + pixels['start1']\n", 114 | " pixels['genome_end'] = getter.cumul_lengths[pixels['chrom2']] + pixels['end2']\n", 115 | " pixels['balanced'] = pixels['count'] * pixels['weight1'] * pixels['weight2']\n", 116 | "\n", 117 | " return pixels[['genome_start', 'genome_end', 'balanced']]" 118 | ], 119 | "language": "python", 120 | "metadata": {}, 121 | "outputs": [], 122 | "prompt_number": 6 123 | }, 124 | { 125 | "cell_type": "code", 126 | "collapsed": false, 127 | "input": [ 128 | "max_width = 4194303999\n", 129 | "\n", 130 | "def get_tile(f, zoom, x, y):\n", 131 | " tile_width = max_width / 2 ** zoom\n", 132 | " \n", 133 | " return getData3(f, zoom, x*tile_width, (x+1) * tile_width, y * tile_width, (y+1) * tile_width )\n" 134 | ], 135 | "language": "python", 136 | "metadata": {}, 137 | "outputs": [], 138 | "prompt_number": 7 139 | }, 140 | { 141 | "cell_type": "code", 142 | "collapsed": false, 143 | "input": [ 144 | "for i in range(5):\n", 145 | " %timeit get_tile(f1000, i,0,0)" 146 | ], 147 | "language": "python", 148 | "metadata": {}, 149 | "outputs": [ 150 | { 151 | "output_type": "stream", 152 | "stream": "stdout", 153 | "text": [ 154 | "10 loops, best of 3: 32.4 ms per loop\n", 155 | "10 loops, best of 3: 43.6 ms per loop" 156 | ] 157 | }, 158 | { 159 | "output_type": "stream", 160 | "stream": "stdout", 161 | "text": [ 162 | "\n", 163 | "10 loops, best of 3: 42.3 ms per loop" 164 | ] 165 | }, 166 | { 167 | "output_type": "stream", 168 | "stream": "stdout", 169 | "text": [ 170 | "\n", 171 | "10 loops, best of 3: 51.3 ms per loop" 172 | ] 173 | }, 174 | { 175 | "output_type": "stream", 176 | "stream": "stdout", 177 | "text": [ 178 | "\n", 179 | "10 loops, best of 3: 93.7 ms per loop" 180 | ] 181 | }, 182 | { 183 | "output_type": "stream", 184 | "stream": "stdout", 185 | "text": [ 186 | "\n" 187 | ] 188 | } 189 | ], 190 | "prompt_number": 10 191 | }, 192 | { 193 | "cell_type": "code", 194 | "collapsed": false, 195 | "input": [ 196 | "%timeit get_tile(f10, 0,0,0)" 197 | ], 198 | "language": "python", 199 | "metadata": {}, 200 | "outputs": [ 201 | { 202 | "ename": "KeyError", 203 | "evalue": "\"Unable to open object (Object '0' doesn't exist)\"", 204 | "output_type": "pyerr", 205 | "traceback": [ 206 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)", 207 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmagic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mu'timeit get_tile(f10, 0,0,0)'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 208 | "\u001b[0;32m/Library/Python/2.7/site-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mmagic\u001b[0;34m(self, arg_s)\u001b[0m\n\u001b[1;32m 2203\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marg_s\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpartition\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2204\u001b[0m \u001b[0mmagic_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmagic_name\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprefilter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mESC_MAGIC\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2205\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_line_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_name\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2207\u001b[0m \u001b[0;31m#-------------------------------------------------------------------------\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 209 | "\u001b[0;32m/Library/Python/2.7/site-packages/IPython/core/interactiveshell.pyc\u001b[0m in \u001b[0;36mrun_line_magic\u001b[0;34m(self, magic_name, line)\u001b[0m\n\u001b[1;32m 2124\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'local_ns'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msys\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf_locals\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2125\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2126\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 2127\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 2128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", 210 | "\u001b[0;32m/Library/Python/2.7/site-packages/IPython/core/magics/execution.pyc\u001b[0m in \u001b[0;36mtimeit\u001b[0;34m(self, line, cell)\u001b[0m\n", 211 | "\u001b[0;32m/Library/Python/2.7/site-packages/IPython/core/magic.pyc\u001b[0m in \u001b[0;36m\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m 191\u001b[0m \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 192\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 212 | "\u001b[0;32m/Library/Python/2.7/site-packages/IPython/core/magics/execution.pyc\u001b[0m in \u001b[0;36mtimeit\u001b[0;34m(self, line, cell)\u001b[0m\n\u001b[1;32m 1011\u001b[0m \u001b[0mnumber\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1012\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1013\u001b[0;31m \u001b[0;32mif\u001b[0m \u001b[0mtimer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimeit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnumber\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m0.2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1014\u001b[0m \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1015\u001b[0m \u001b[0mnumber\u001b[0m \u001b[0;34m*=\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 213 | "\u001b[0;32m/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/timeit.pyc\u001b[0m in \u001b[0;36mtimeit\u001b[0;34m(self, number)\u001b[0m\n\u001b[1;32m 199\u001b[0m \u001b[0mgc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 200\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m \u001b[0mtiming\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minner\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtimer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 202\u001b[0m \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 203\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mgcold\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 214 | "\u001b[0;32m\u001b[0m in \u001b[0;36minner\u001b[0;34m(_it, _timer)\u001b[0m\n", 215 | "\u001b[0;32m\u001b[0m in \u001b[0;36mget_tile\u001b[0;34m(f, zoom, x, y)\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mtile_width\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmax_width\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m2\u001b[0m \u001b[0;34m**\u001b[0m \u001b[0mzoom\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mgetData3\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzoom\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mtile_width\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mtile_width\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mtile_width\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mtile_width\u001b[0m \u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 216 | "\u001b[0;32m\u001b[0m in \u001b[0;36mgetData3\u001b[0;34m(f, zoomLevel, startPos1, endPos1, startPos2, endPos2)\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;31m#print(zoomLevel, startPos1, endPos1, startPos2, endPos2)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcooler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCooler\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mzoomLevel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 8\u001b[0m \u001b[0;31m#matrix = c.matrix(balance=True, as_pixels=True, join=True)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;31m#cooler_matrix = {'cooler': c, 'matrix': matrix}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 217 | "\u001b[0;32m/Users/pkerp/.virtualenvs/django/lib/python2.7/site-packages/h5py/_objects.so\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper (/Users/travis/build/MacPython/h5py-wheels/h5py/h5py/_objects.c:2687)\u001b[0;34m()\u001b[0m\n", 218 | "\u001b[0;32m/Users/pkerp/.virtualenvs/django/lib/python2.7/site-packages/h5py/_objects.so\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper (/Users/travis/build/MacPython/h5py-wheels/h5py/h5py/_objects.c:2645)\u001b[0;34m()\u001b[0m\n", 219 | "\u001b[0;32m/Users/pkerp/.virtualenvs/django/lib/python2.7/site-packages/h5py/_hl/group.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m 164\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Invalid HDF5 object reference\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 165\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 166\u001b[0;31m \u001b[0moid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5o\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_e\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlapl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lapl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 167\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 168\u001b[0m \u001b[0motype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh5i\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 220 | "\u001b[0;32m/Users/pkerp/.virtualenvs/django/lib/python2.7/site-packages/h5py/_objects.so\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper (/Users/travis/build/MacPython/h5py-wheels/h5py/h5py/_objects.c:2687)\u001b[0;34m()\u001b[0m\n", 221 | "\u001b[0;32m/Users/pkerp/.virtualenvs/django/lib/python2.7/site-packages/h5py/_objects.so\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper (/Users/travis/build/MacPython/h5py-wheels/h5py/h5py/_objects.c:2645)\u001b[0;34m()\u001b[0m\n", 222 | "\u001b[0;32m/Users/pkerp/.virtualenvs/django/lib/python2.7/site-packages/h5py/h5o.so\u001b[0m in \u001b[0;36mh5py.h5o.open (/Users/travis/build/MacPython/h5py-wheels/h5py/h5py/h5o.c:3573)\u001b[0;34m()\u001b[0m\n", 223 | "\u001b[0;31mKeyError\u001b[0m: \"Unable to open object (Object '0' doesn't exist)\"" 224 | ] 225 | } 226 | ], 227 | "prompt_number": 9 228 | }, 229 | { 230 | "cell_type": "code", 231 | "collapsed": false, 232 | "input": [], 233 | "language": "python", 234 | "metadata": {}, 235 | "outputs": [] 236 | } 237 | ], 238 | "metadata": {} 239 | } 240 | ] 241 | } -------------------------------------------------------------------------------- /notebooks/Register url test.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "%load_ext autoreload\n", 10 | "%autoreload 2" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 11, 16 | "metadata": {}, 17 | "outputs": [ 18 | { 19 | "data": { 20 | "text/plain": [ 21 | "405" 22 | ] 23 | }, 24 | "execution_count": 11, 25 | "metadata": {}, 26 | "output_type": "execute_result" 27 | } 28 | ], 29 | "source": [ 30 | "import requests\n", 31 | "\n", 32 | "req = requests.post('http://localhost:8000/api/v1/register_url',\n", 33 | " json={\n", 34 | " 'fileUrl': 'https://pkerp.s3.amazonaws.com/public/bamfile_test/SRR1770413.sorted.bam'\n", 35 | " })\n", 36 | "req.status_code" 37 | ] 38 | }, 39 | { 40 | "cell_type": "code", 41 | "execution_count": null, 42 | "metadata": {}, 43 | "outputs": [], 44 | "source": [] 45 | } 46 | ], 47 | "metadata": { 48 | "kernelspec": { 49 | "display_name": "Python 3", 50 | "language": "python", 51 | "name": "python3" 52 | }, 53 | "language_info": { 54 | "codemirror_mode": { 55 | "name": "ipython", 56 | "version": 3 57 | }, 58 | "file_extension": ".py", 59 | "mimetype": "text/x-python", 60 | "name": "python", 61 | "nbconvert_exporter": "python", 62 | "pygments_lexer": "ipython3", 63 | "version": "3.6.2" 64 | } 65 | }, 66 | "nbformat": 4, 67 | "nbformat_minor": 2 68 | } 69 | -------------------------------------------------------------------------------- /notebooks/Rename resolutions.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 3, 6 | "metadata": {}, 7 | "outputs": [ 8 | { 9 | "name": "stdout", 10 | "output_type": "stream", 11 | "text": [ 12 | "res: 1000\n", 13 | "res: 1024000\n", 14 | "res: 128000\n", 15 | "res: 16000\n", 16 | "res: 16384000\n", 17 | "res: 2000\n", 18 | "res: 2048000\n", 19 | "res: 256000\n", 20 | "res: 32000\n", 21 | "res: 4000\n", 22 | "res: 4096000\n", 23 | "res: 512000\n", 24 | "res: 64000\n", 25 | "res: 8000\n", 26 | "res: 8192000\n" 27 | ] 28 | } 29 | ], 30 | "source": [ 31 | "import h5py\n", 32 | "\n", 33 | "filename = '../media/uploads/my_file_genome_wide.multires'\n", 34 | "f = h5py.File(filename, 'r+')\n", 35 | "#f.move('resolutions/1000', 'resolutions/1')\n", 36 | "\n", 37 | "for res in f['resolutions']:\n", 38 | " print(\"res:\", int(res))\n", 39 | " #f.move('resolutions/{}'.format(res), 'resolutions/{}'.format(int(res) * 1000))\n", 40 | "f.close()" 41 | ] 42 | }, 43 | { 44 | "cell_type": "markdown", 45 | "metadata": { 46 | "collapsed": true 47 | }, 48 | "source": [ 49 | "python manage.py ingest_tileset --filetype multivec --datatype multivec --no-upload --filename uploads/my_file_genome_wide.multires" 50 | ] 51 | }, 52 | { 53 | "cell_type": "code", 54 | "execution_count": null, 55 | "metadata": { 56 | "collapsed": true 57 | }, 58 | "outputs": [], 59 | "source": [] 60 | } 61 | ], 62 | "metadata": { 63 | "kernelspec": { 64 | "display_name": "Python 3", 65 | "language": "python", 66 | "name": "python3" 67 | }, 68 | "language_info": { 69 | "codemirror_mode": { 70 | "name": "ipython", 71 | "version": 3 72 | }, 73 | "file_extension": ".py", 74 | "mimetype": "text/x-python", 75 | "name": "python", 76 | "nbconvert_exporter": "python", 77 | "pygments_lexer": "ipython3", 78 | "version": "3.6.2" 79 | } 80 | }, 81 | "nbformat": 4, 82 | "nbformat_minor": 2 83 | } 84 | -------------------------------------------------------------------------------- /notebooks/stuff.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "metadata": { 3 | "name": "", 4 | "signature": "sha256:d5708c2fa602f9114565230875e8d2a05d857e94499adafaf1620430353ee257" 5 | }, 6 | "nbformat": 3, 7 | "nbformat_minor": 0, 8 | "worksheets": [ 9 | { 10 | "cells": [ 11 | { 12 | "cell_type": "code", 13 | "collapsed": false, 14 | "input": [ 15 | "import base64\n", 16 | "import numpy as np\n", 17 | "\n", 18 | "t = np.arange(25, dtype=np.float64)\n", 19 | "s = base64.b64encode(t)\n", 20 | "r = base64.decodestring(s)\n", 21 | "q = np.frombuffer(r, dtype=np.float64)\n", 22 | "\n", 23 | "print(np.allclose(q, t))" 24 | ], 25 | "language": "python", 26 | "metadata": {}, 27 | "outputs": [ 28 | { 29 | "output_type": "stream", 30 | "stream": "stdout", 31 | "text": [ 32 | "True\n" 33 | ] 34 | } 35 | ], 36 | "prompt_number": 1 37 | }, 38 | { 39 | "cell_type": "code", 40 | "collapsed": false, 41 | "input": [ 42 | "%timeit base64.b64encode(t)" 43 | ], 44 | "language": "python", 45 | "metadata": {}, 46 | "outputs": [ 47 | { 48 | "output_type": "stream", 49 | "stream": "stdout", 50 | "text": [ 51 | "100000 loops, best of 3: 2 \u00b5s per loop\n" 52 | ] 53 | } 54 | ], 55 | "prompt_number": 2 56 | }, 57 | { 58 | "cell_type": "code", 59 | "collapsed": false, 60 | "input": [ 61 | "len(base64.b64encode(np.arange(25, dtype=np.float64)))" 62 | ], 63 | "language": "python", 64 | "metadata": {}, 65 | "outputs": [ 66 | { 67 | "metadata": {}, 68 | "output_type": "pyout", 69 | "prompt_number": 6, 70 | "text": [ 71 | "268" 72 | ] 73 | } 74 | ], 75 | "prompt_number": 6 76 | }, 77 | { 78 | "cell_type": "code", 79 | "collapsed": false, 80 | "input": [ 81 | "len(base64.b64encode(np.arange(25, dtype=np.float32)))" 82 | ], 83 | "language": "python", 84 | "metadata": {}, 85 | "outputs": [ 86 | { 87 | "metadata": {}, 88 | "output_type": "pyout", 89 | "prompt_number": 7, 90 | "text": [ 91 | "136" 92 | ] 93 | } 94 | ], 95 | "prompt_number": 7 96 | }, 97 | { 98 | "cell_type": "code", 99 | "collapsed": false, 100 | "input": [ 101 | "import json\n", 102 | "\n", 103 | "a = np.linspace(0,10,2**16, dtype=np.float32)\n", 104 | "print len(base64.b64encode(a))\n", 105 | "print len(json.dumps(map(float, a)))" 106 | ], 107 | "language": "python", 108 | "metadata": {}, 109 | "outputs": [ 110 | { 111 | "output_type": "stream", 112 | "stream": "stdout", 113 | "text": [ 114 | "349528\n", 115 | "1262542" 116 | ] 117 | }, 118 | { 119 | "output_type": "stream", 120 | "stream": "stdout", 121 | "text": [ 122 | "\n" 123 | ] 124 | } 125 | ], 126 | "prompt_number": 20 127 | }, 128 | { 129 | "cell_type": "code", 130 | "collapsed": false, 131 | "input": [ 132 | "%timeit base64.b64encode(a)" 133 | ], 134 | "language": "python", 135 | "metadata": {}, 136 | "outputs": [ 137 | { 138 | "output_type": "stream", 139 | "stream": "stdout", 140 | "text": [ 141 | "1000 loops, best of 3: 1.03 ms per loop\n" 142 | ] 143 | } 144 | ], 145 | "prompt_number": 21 146 | }, 147 | { 148 | "cell_type": "code", 149 | "collapsed": false, 150 | "input": [ 151 | "%timeit json.dumps(map(float, a))" 152 | ], 153 | "language": "python", 154 | "metadata": {}, 155 | "outputs": [ 156 | { 157 | "output_type": "stream", 158 | "stream": "stdout", 159 | "text": [ 160 | "10 loops, best of 3: 88.2 ms per loop\n" 161 | ] 162 | } 163 | ], 164 | "prompt_number": 22 165 | }, 166 | { 167 | "cell_type": "code", 168 | "collapsed": false, 169 | "input": [], 170 | "language": "python", 171 | "metadata": {}, 172 | "outputs": [] 173 | } 174 | ], 175 | "metadata": {} 176 | } 177 | ] 178 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "higlass-server", 3 | "description": "Server for HiGlass serving cooler files", 4 | "repository": { 5 | "type": "git", 6 | "url": "https://github.com/higlass/higlass-server.git" 7 | }, 8 | "license": "MIT", 9 | "scripts": { 10 | "start": "python manage.py runserver localhost:8001", 11 | "startmac": "brew services start redis && npm start", 12 | "test": "./test.sh" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | #### test requirements 2 | asynctest>=0.13.0 3 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pybbi==0.2.2 2 | bumpversion==0.5.3 3 | CacheControl==0.12.4 4 | cooler==0.8.6 5 | django-cors-headers==3.0.2 6 | django-guardian==1.5.1 7 | django-rest-swagger==2.2.0 8 | django==2.2.26 9 | djangorestframework==3.11.2 10 | h5py>=3.0.0 11 | higlass-python==0.4.7 12 | jsonschema==3.2.0 13 | numba==0.46.0 14 | numpy==1.17.3 15 | pandas>=0.23.4 16 | Pillow>=9.0.0 17 | pybase64==0.2.1 18 | redis==2.10.5 19 | requests==2.26.0 20 | scikit-learn==1.0.2 21 | slugid==2.0.0 22 | redis==2.10.5 23 | clodius==0.18.0 24 | simple-httpfs==0.4.2 25 | pyppeteer==0.0.25 26 | urllib3==1.26.7 27 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/higlass/higlass-server/cbfe79fe3ae0e844b4c0c78142a83733c8cc66a2/scripts/__init__.py -------------------------------------------------------------------------------- /scripts/add_attr_to_hdf5.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import h5py 4 | import sys 5 | import argparse 6 | 7 | def main(): 8 | parser = argparse.ArgumentParser(description=""" 9 | 10 | python add_attr_to_hdf5.py file.hdf5 attr_name attr_value 11 | 12 | Add an attribute to an HDF5 file. 13 | """) 14 | 15 | parser.add_argument('filepath') 16 | parser.add_argument('attr_name') 17 | parser.add_argument('attr_value') 18 | #parser.add_argument('-o', '--options', default='yo', 19 | # help="Some option", type='str') 20 | #parser.add_argument('-u', '--useless', action='store_true', 21 | # help='Another useless option') 22 | 23 | args = parser.parse_args() 24 | 25 | with h5py.File(args.filepath) as f: 26 | f.attrs[args.attr_name] = args.attr_value 27 | 28 | 29 | if __name__ == '__main__': 30 | main() 31 | 32 | 33 | -------------------------------------------------------------------------------- /scripts/benchmark_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | from __future__ import print_function 4 | 5 | import os.path as op 6 | import requests 7 | import sys 8 | import argparse 9 | from multiprocessing import Pool 10 | 11 | def main(): 12 | parser = argparse.ArgumentParser(description=""" 13 | 14 | Usage: 15 | 16 | cp api/db.sqlite3 api/db.sqlite3.bak 17 | wget https://s3.amazonaws.com/pkerp/public/db.sqlite3 18 | mv db.sqlite3 api 19 | 20 | python benchmark_server.py url path tileset-id [tile_ids] 21 | """) 22 | 23 | parser.add_argument('url') 24 | parser.add_argument('tileset_id') 25 | parser.add_argument('tile_ids', nargs='*') 26 | parser.add_argument('--tile-id-file') 27 | parser.add_argument('--iterations') 28 | parser.add_argument('--at-once', action='store_true') 29 | parser.add_argument('--multi', action='store_true') 30 | 31 | #parser.add_argument('-o', '--options', default='yo', 32 | # help="Some option", type='str') 33 | #parser.add_argument('-u', '--useless', action='store_true', 34 | # help='Another useless option') 35 | args = parser.parse_args() 36 | tile_ids = args.tile_ids 37 | 38 | # parse requests on the command line 39 | for tile_id in args.tile_ids: 40 | get_url = op.join(args.url, 'tilesets/x/render/?d=' + args.tileset_id + '.' + tile_id) 41 | 42 | r = requests.get(get_url) 43 | print("r:", r) 44 | 45 | # parse requests from a file 46 | if args.tile_id_file is not None: 47 | with open(args.tile_id_file, 'r') as f: 48 | for line in f: 49 | tile_ids += [line.strip()] 50 | 51 | if args.at_once: 52 | url_arg = "&d=".join([args.tileset_id + '.' + tile_id for tile_id in tile_ids]) 53 | get_url = op.join(args.url, 'tilesets/x/render/?d=' + url_arg) 54 | 55 | print("get_url:", get_url) 56 | r = requests.get(get_url) 57 | print("r:", r, len(r.text)) 58 | 59 | else: 60 | arr = [] 61 | for tile_id in tile_ids: 62 | get_url = op.join(args.url, 'tilesets/x/render/?d=' + args.tileset_id + '.' + tile_id) 63 | arr.append(get_url) 64 | 65 | if args.multi: 66 | print("Using pool...") 67 | p = Pool(4) 68 | r = p.map(requests.get, arr) 69 | else: 70 | for a in arr: 71 | requests.get(a) 72 | 73 | if __name__ == '__main__': 74 | main() 75 | 76 | 77 | -------------------------------------------------------------------------------- /scripts/format_upload_command.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | from __future__ import print_function 4 | 5 | import sys 6 | import argparse 7 | 8 | def main(): 9 | parser = argparse.ArgumentParser(description=""" 10 | 11 | python format_upload_command.py formatted_filename 12 | 13 | Create higlass server curl command to upload the file with this filename. 14 | 15 | Example filename: 16 | 17 | Dixon2012-IMR90-HindIII-allreps-filtered.1kb.multires.cool 18 | 19 | Example output: 20 | 21 | ... 22 | """) 23 | 24 | parser.add_argument('filename') 25 | #parser.add_argument('argument', nargs=1) 26 | #parser.add_argument('-o', '--options', default='yo', 27 | # help="Some option", type='str') 28 | #parser.add_argument('-u', '--useless', action='store_true', 29 | # help='Another useless option') 30 | 31 | args = parser.parse_args() 32 | 33 | parts = args.filename.split('-') 34 | 35 | try: 36 | name = parts[0][:-4] 37 | year = parts[0][-4:] 38 | celltype = parts[1] 39 | enzyme = parts[2] 40 | resolution = parts[4].split('.')[1] 41 | 42 | out_txt = """ 43 | curl -u `cat ~/.higlass-server-login` \ 44 | -F 'datafile=@/data/downloads/hg19/{filename}' \ 45 | -F 'filetype=cooler' \ 46 | -F 'datatype=matrix' \ 47 | -F 'name={name} et al. ({year}) {celltype} {enzyme} (allreps) {resolution}' \ 48 | -F 'coordSystem=hg19' \ 49 | localhost:8000/api/v1/tilesets/""".format(filename=args.filename, name=name, year=year, celltype=celltype, enzyme=enzyme, resolution=resolution) 50 | 51 | print(out_txt, end="") 52 | except: 53 | print("ERROR:", args.filename) 54 | 55 | 56 | if __name__ == '__main__': 57 | main() 58 | 59 | 60 | -------------------------------------------------------------------------------- /scripts/test_aws_bigWig_fetch.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import math 3 | import argparse 4 | import clodius.tiles.bigwig as hgbi 5 | import clodius.tiles.utils as hgut 6 | import time 7 | 8 | def get_bigwig_tile_by_id(bwpath, zoom_level, tile_pos): 9 | ''' 10 | Get the data for a bigWig tile given a tile id. 11 | 12 | Parameters 13 | ---------- 14 | bwpath: string 15 | The path to the bigWig file (can be remote) 16 | zoom_level: int 17 | The zoom level to get the data for 18 | tile_pos: int 19 | The position of the tile 20 | ''' 21 | chromsizes = hgbi.get_chromsizes(bwpath) 22 | max_depth = hgut.get_quadtree_depth(chromsizes, hgbi.TILE_SIZE) 23 | tile_size = hgbi.TILE_SIZE * 2 ** (max_depth - zoom_level) 24 | 25 | start_pos = tile_pos * tile_size 26 | end_pos = start_pos + tile_size 27 | 28 | return hgbi.get_bigwig_tile(bwpath, zoom_level, start_pos, end_pos) 29 | 30 | def main(): 31 | parser = argparse.ArgumentParser(description=""" 32 | 33 | python test_bigwig_tile_fetch.py filename zoom_level tile_pos 34 | """) 35 | 36 | parser.add_argument('filename') 37 | parser.add_argument('zoom_level') 38 | parser.add_argument('tile_pos') 39 | parser.add_argument('--num-requests', default=1, type=int) 40 | #parser.add_argument('argument', nargs=1) 41 | #parser.add_argument('-o', '--options', default='yo', 42 | # help="Some option", type='str') 43 | #parser.add_argument('-u', '--useless', action='store_true', 44 | # help='Another useless option') 45 | args = parser.parse_args() 46 | 47 | print("fetching:", args.filename) 48 | if args.num_requests == 1: 49 | tile = get_bigwig_tile_by_id(args.filename, int(args.zoom_level), 50 | int(args.tile_pos)) 51 | else: 52 | zoom_level = math.ceil(math.log(args.num_requests) / math.log(2)) 53 | 54 | for tn in range(0, args.num_requests): 55 | print("fetching:", zoom_level, tn) 56 | t1 = time.time() 57 | tile = get_bigwig_tile_by_id(args.filename, int(zoom_level), 58 | int(tn)) 59 | t2 = time.time() 60 | print("fetched: {:.2f}".format(t2 - t1), "tile", len(tile)) 61 | 62 | 63 | if __name__ == "__main__": 64 | main() 65 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | umount media/http 4 | umount media/https 5 | 6 | simple-httpfs.py media/http 7 | simple-httpfs.py media/https 8 | 9 | python manage.py runserver 10 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | # kill previous instances 5 | # ps aux | grep runserver | grep 6000 | awk '{print $2}' | xargs kill 6 | # rm db_test.sqlite3 7 | 8 | ### Build and test from the inside out: 9 | ### 1) Unit tests 10 | 11 | # clear previous db 12 | rm db_test.sqlite3 ||: 13 | 14 | COOLER=dixon2012-h1hesc-hindiii-allreps-filtered.1000kb.multires.cool 15 | HITILE=wgEncodeCaltechRnaSeqHuvecR1x75dTh1014IlnaPlusSignalRep2.hitile 16 | 17 | FILES=$(cat < data/tiny.txt 56 | 57 | SETTINGS=higlass_server.settings_test 58 | 59 | python manage.py migrate --settings=$SETTINGS 60 | 61 | export SITE_URL="somesite.com" 62 | PORT=6000 63 | python manage.py runserver localhost:$PORT --settings=$SETTINGS & 64 | 65 | #DJANGO_PID=$! 66 | TILESETS_URL="http://localhost:$PORT/api/v1/tilesets/" 67 | until $(curl --output /dev/null --silent --fail --globoff $TILESETS_URL); do echo '.'; sleep 1; done 68 | # Server is needed for higlass_server tests 69 | 70 | python manage.py test -v 2 tilesets higlass_server fragments --settings=$SETTINGS 71 | 72 | echo 'PASS!' 73 | 74 | # kill all child processes of this bash script 75 | # e.g.: the server 76 | kill $(ps -o pid= --ppid $$) 77 | -------------------------------------------------------------------------------- /tilesets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/higlass/higlass-server/cbfe79fe3ae0e844b4c0c78142a83733c8cc66a2/tilesets/__init__.py -------------------------------------------------------------------------------- /tilesets/admin.py: -------------------------------------------------------------------------------- 1 | from django.contrib import admin 2 | from tilesets.models import Tileset 3 | from tilesets.models import ViewConf 4 | from tilesets.models import Project 5 | # Register your models here. 6 | 7 | 8 | class TilesetAdmin(admin.ModelAdmin): 9 | list_display = [ 10 | 'created', 11 | 'uuid', 12 | 'datafile', 13 | 'filetype', 14 | 'datatype', 15 | 'coordSystem', 16 | 'coordSystem2', 17 | 'owner', 18 | 'private', 19 | 'name', 20 | ] 21 | 22 | 23 | class ViewConfAdmin(admin.ModelAdmin): 24 | list_display = [ 25 | 'created', 26 | 'uuid', 27 | 'higlassVersion', 28 | ] 29 | 30 | class ProjectConfAdmin(admin.ModelAdmin): 31 | list_display = [ 32 | 'created', 33 | 'uuid', 34 | 'name', 35 | 'description', 36 | ] 37 | 38 | 39 | admin.site.register(Tileset, TilesetAdmin) 40 | admin.site.register(ViewConf, ViewConfAdmin) 41 | admin.site.register(Project, ProjectConfAdmin) 42 | -------------------------------------------------------------------------------- /tilesets/apps.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | from django.apps import AppConfig 4 | 5 | 6 | class TilesetsConfig(AppConfig): 7 | name = 'tilesets' 8 | -------------------------------------------------------------------------------- /tilesets/chromsizes.py: -------------------------------------------------------------------------------- 1 | import csv 2 | import h5py 3 | import logging 4 | import numpy as np 5 | import pandas as pd 6 | 7 | from fragments.utils import get_cooler 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | def chromsizes_array_to_series(chromsizes): 12 | ''' 13 | Convert an array of [[chrname, size]...] values to a series 14 | indexed by chrname with size values 15 | ''' 16 | chrnames = [c[0] for c in chromsizes] 17 | chrvalues = [c[1] for c in chromsizes] 18 | 19 | return pd.Series(np.array([int(c) for c in chrvalues]), index=chrnames) 20 | 21 | def get_multivec_chromsizes(filename): 22 | ''' 23 | Get a list of chromosome sizes from this [presumably] multivec 24 | file. 25 | 26 | Parameters: 27 | ----------- 28 | filename: string 29 | The filename of the multivec file 30 | 31 | Returns 32 | ------- 33 | chromsizes: [(name:string, size:int), ...] 34 | An ordered list of chromosome names and sizes 35 | ''' 36 | with h5py.File(filename, 'r') as f: 37 | try: 38 | chrom_names = [t.decode('utf-8') for t in f['chroms']['name'][:]] 39 | chrom_lengths = f['chroms']['length'][:] 40 | 41 | return zip(chrom_names, chrom_lengths) 42 | except Exception as e: 43 | logger.exception(e) 44 | raise Exception( 'Error retrieving multivec chromsizes') 45 | 46 | def get_cooler_chromsizes(filename): 47 | ''' 48 | Get a list of chromosome sizes from this [presumably] cooler 49 | file. 50 | 51 | Parameters: 52 | ----------- 53 | filename: string 54 | The filename of the cooler file 55 | 56 | Returns 57 | ------- 58 | chromsizes: [(name:string, size:int), ...] 59 | An ordered list of chromosome names and sizes 60 | ''' 61 | with h5py.File(filename, 'r') as f: 62 | 63 | try: 64 | c = get_cooler(f) 65 | except Exception as e: 66 | logger.error(e) 67 | raise Exception('Yikes... Couldn~\'t init cooler files 😵') 68 | 69 | try: 70 | data = [] 71 | for chrom, size in c.chromsizes.iteritems(): 72 | data.append([chrom, size]) 73 | return data 74 | except Exception as e: 75 | logger.error(e) 76 | raise Exception( 'Cooler file has no `chromsizes` attribute 🤔') 77 | 78 | def get_tsv_chromsizes(filename): 79 | ''' 80 | Get a list of chromosome sizes from this [presumably] tsv 81 | chromsizes file file. 82 | 83 | Parameters: 84 | ----------- 85 | filename: string 86 | The filename of the tsv file 87 | 88 | Returns 89 | ------- 90 | chromsizes: [(name:string, size:int), ...] 91 | An ordered list of chromosome names and sizes 92 | ''' 93 | try: 94 | with open(filename, 'r') as f: 95 | reader = csv.reader(f, delimiter='\t') 96 | 97 | data = [] 98 | for row in reader: 99 | data.append(row) 100 | return data 101 | except Exception as ex: 102 | logger.error(ex) 103 | 104 | err_msg = 'WHAT?! Could not load file %s. 😤 (%s)' % ( 105 | filename, ex 106 | ) 107 | 108 | raise Exception(err_msg) 109 | 110 | -------------------------------------------------------------------------------- /tilesets/exceptions.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from rest_framework.exceptions import APIException 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | class CoolerFileBroken(APIException): 9 | status_code = 500 10 | default_detail = 'The cooler file is broken.' 11 | default_code = 'cooler_file_broken' 12 | -------------------------------------------------------------------------------- /tilesets/generate_tiles.py: -------------------------------------------------------------------------------- 1 | import base64 2 | #import tilesets.bigwig_tiles as bwt 3 | import clodius.db_tiles as cdt 4 | import clodius.hdf_tiles as hdft 5 | import collections as col 6 | 7 | import clodius.tiles.bam as ctb 8 | import clodius.tiles.beddb as hgbe 9 | import clodius.tiles.bigwig as hgbi 10 | import clodius.tiles.fasta as hgfa 11 | import clodius.tiles.bigbed as hgbb 12 | import clodius.tiles.cooler as hgco 13 | import clodius.tiles.geo as hggo 14 | import clodius.tiles.imtiles as hgim 15 | 16 | import h5py 17 | import itertools as it 18 | import numpy as np 19 | import os 20 | import os.path as op 21 | import shutil 22 | import time 23 | import tempfile 24 | import tilesets.models as tm 25 | import tilesets.chromsizes as tcs 26 | 27 | import higlass.tilesets as hgti 28 | 29 | import clodius.tiles.multivec as ctmu 30 | 31 | import higlass_server.settings as hss 32 | 33 | def get_tileset_datatype(tileset): 34 | ''' 35 | Extract the filetype for the tileset 36 | 37 | This should be encoded in one of the tags. If there are multiple 38 | "datatype" tags, use the most recent one. 39 | ''' 40 | if tileset.datatype is not None and len(tileset.datatype) > 0: 41 | return tileset.datatype 42 | 43 | for tag in tileset.tags.all(): 44 | parts = tag.name.split(':') 45 | if parts[0] == 'datatype': 46 | return parts[1] 47 | 48 | # fall back to the filetype attribute of the tileset 49 | return tileset.datatype 50 | 51 | def get_cached_datapath(path): 52 | ''' 53 | Check if we need to cache this file or if we have a cached copy 54 | 55 | Parameters 56 | ---------- 57 | filename: str 58 | The original filename 59 | 60 | Returns 61 | ------- 62 | filename: str 63 | Either the cached filename if we're caching or the original 64 | filename 65 | ''' 66 | if hss.CACHE_DIR is None: 67 | # no caching requested 68 | return path 69 | 70 | orig_path = path 71 | cached_path = op.join(hss.CACHE_DIR, path) 72 | 73 | if op.exists(cached_path): 74 | # this file has already been cached 75 | return cached_path 76 | 77 | with tempfile.TemporaryDirectory() as dirpath: 78 | tmp = op.join(dirpath, 'cached_file') 79 | shutil.copyfile(orig_path, tmp) 80 | 81 | # check to make sure the destination directory exists 82 | dest_dir = op.dirname(cached_path) 83 | 84 | if not op.exists(dest_dir): 85 | os.makedirs(dest_dir) 86 | 87 | shutil.move(tmp, cached_path) 88 | 89 | return cached_path 90 | 91 | def extract_tileset_uid(tile_id): 92 | ''' 93 | Get the tileset uid from a tile id. Should usually be all the text 94 | before the first dot. 95 | 96 | Parameters 97 | ---------- 98 | tile_id : str 99 | The id of the tile we're getting the tileset info for (e.g. xyz.0.0.1) 100 | Returns 101 | ------- 102 | tileset_uid : str 103 | The uid of the tileset that this tile comes from 104 | ''' 105 | tile_id_parts = tile_id.split('.') 106 | tileset_uuid = tile_id_parts[0] 107 | 108 | return tileset_uuid 109 | 110 | 111 | def get_tileset_filetype(tileset): 112 | return tileset.filetype 113 | 114 | def generate_1d_tiles(filename, tile_ids, get_data_function, tileset_options): 115 | ''' 116 | Generate a set of tiles for the given tile_ids. 117 | 118 | Parameters 119 | ---------- 120 | filename: str 121 | The file containing the multiresolution data 122 | tile_ids: [str,...] 123 | A list of tile_ids (e.g. xyx.0.0) identifying the tiles 124 | to be retrieved 125 | get_data_function: lambda 126 | A function which retrieves the data for this tile 127 | tileset_options: dict or None 128 | An optional dict containing options, including aggregation options. 129 | 130 | Returns 131 | ------- 132 | tile_list: [(tile_id, tile_data),...] 133 | A list of tile_id, tile_data tuples 134 | ''' 135 | 136 | agg_func_map = { 137 | "sum": lambda x: np.sum(x, axis=0), 138 | "mean": lambda x: np.mean(x, axis=0), 139 | "median": lambda x: np.median(x, axis=0), 140 | "std": lambda x: np.std(x, axis=0), 141 | "var": lambda x: np.var(x, axis=0), 142 | "max": lambda x: np.amax(x, axis=0), 143 | "min": lambda x: np.amin(x, axis=0), 144 | } 145 | 146 | generated_tiles = [] 147 | 148 | for tile_id in tile_ids: 149 | tile_id_parts = tile_id.split('.') 150 | tile_position = list(map(int, tile_id_parts[1:3])) 151 | 152 | dense = get_data_function(filename, tile_position) 153 | 154 | if tileset_options != None and "aggGroups" in tileset_options and "aggFunc" in tileset_options: 155 | agg_func_name = tileset_options["aggFunc"] 156 | agg_group_arr = [ x if type(x) == list else [x] for x in tileset_options["aggGroups"] ] 157 | dense = np.array(list(map(agg_func_map[agg_func_name], [ dense[arr] for arr in agg_group_arr ]))) 158 | 159 | if len(dense): 160 | max_dense = max(dense.reshape(-1,)) 161 | min_dense = min(dense.reshape(-1,)) 162 | else: 163 | max_dense = 0 164 | min_dense = 0 165 | 166 | min_f16 = np.finfo('float16').min 167 | max_f16 = np.finfo('float16').max 168 | 169 | has_nan = len([d for d in dense.reshape((-1,)) if np.isnan(d)]) > 0 170 | 171 | if ( 172 | not has_nan and 173 | max_dense > min_f16 and max_dense < max_f16 and 174 | min_dense > min_f16 and min_dense < max_f16 175 | ): 176 | tile_value = { 177 | 'dense': base64.b64encode(dense.reshape((-1,)).astype('float16')).decode('utf-8'), 178 | 'dtype': 'float16', 179 | 'shape': dense.shape 180 | } 181 | else: 182 | tile_value = { 183 | 'dense': base64.b64encode(dense.reshape((-1,)).astype('float32')).decode('utf-8'), 184 | 'dtype': 'float32', 185 | 'shape': dense.shape 186 | } 187 | 188 | generated_tiles += [(tile_id, tile_value)] 189 | 190 | return generated_tiles 191 | 192 | def get_chromsizes(tileset): 193 | ''' 194 | Get a set of chromsizes matching the coordSystem of this 195 | tileset. 196 | 197 | Parameters 198 | ---------- 199 | tileset: A tileset DJango model object 200 | 201 | Returns 202 | ------- 203 | chromsizes: [[chrom, sizes]] 204 | A set of chromsizes to be used with this bigWig file. 205 | None if no chromsizes tileset with this coordSystem 206 | exists or if two exist with this coordSystem. 207 | ''' 208 | if tileset.coordSystem is None or len(tileset.coordSystem) == None: 209 | return None 210 | 211 | try: 212 | chrom_info_tileset = tm.Tileset.objects.get(coordSystem=tileset.coordSystem, 213 | datatype='chromsizes') 214 | except: 215 | return None 216 | 217 | return tcs.get_tsv_chromsizes(chrom_info_tileset.datafile.path) 218 | 219 | def generate_hitile_tiles(tileset, tile_ids): 220 | ''' 221 | Generate tiles from a hitile file. 222 | 223 | Parameters 224 | ---------- 225 | tileset: tilesets.models.Tileset object 226 | The tileset that the tile ids should be retrieved from 227 | tile_ids: [str,...] 228 | A list of tile_ids (e.g. xyx.0.0) identifying the tiles 229 | to be retrieved 230 | 231 | Returns 232 | ------- 233 | tile_list: [(tile_id, tile_data),...] 234 | A list of tile_id, tile_data tuples 235 | ''' 236 | generated_tiles = [] 237 | 238 | for tile_id in tile_ids: 239 | tile_id_parts = tile_id.split('.') 240 | tile_position = list(map(int, tile_id_parts[1:3])) 241 | 242 | dense = hdft.get_data( 243 | h5py.File( 244 | tileset.datafile.path 245 | ), 246 | tile_position[0], 247 | tile_position[1] 248 | ) 249 | 250 | if len(dense): 251 | max_dense = max(dense) 252 | min_dense = min(dense) 253 | else: 254 | max_dense = 0 255 | min_dense = 0 256 | 257 | min_f16 = np.finfo('float16').min 258 | max_f16 = np.finfo('float16').max 259 | 260 | has_nan = len([d for d in dense if np.isnan(d)]) > 0 261 | 262 | if ( 263 | not has_nan and 264 | max_dense > min_f16 and max_dense < max_f16 and 265 | min_dense > min_f16 and min_dense < max_f16 266 | ): 267 | tile_value = { 268 | 'dense': base64.b64encode(dense.astype('float16')).decode('utf-8'), 269 | 'dtype': 'float16' 270 | } 271 | else: 272 | tile_value = { 273 | 'dense': base64.b64encode(dense.astype('float32')).decode('utf-8'), 274 | 'dtype': 'float32' 275 | } 276 | 277 | generated_tiles += [(tile_id, tile_value)] 278 | 279 | return generated_tiles 280 | 281 | def generate_bed2ddb_tiles(tileset, tile_ids, retriever=cdt.get_2d_tiles): 282 | ''' 283 | Generate tiles from a bed2db file. 284 | 285 | Parameters 286 | ---------- 287 | tileset: tilesets.models.Tileset object 288 | The tileset that the tile ids should be retrieved from 289 | tile_ids: [str,...] 290 | A list of tile_ids (e.g. xyx.0.0.1) identifying the tiles 291 | to be retrieved 292 | 293 | Returns 294 | ------- 295 | generated_tiles: [(tile_id, tile_data),...] 296 | A list of tile_id, tile_data tuples 297 | ''' 298 | generated_tiles = [] 299 | 300 | tile_ids_by_zoom = bin_tiles_by_zoom(tile_ids).values() 301 | partitioned_tile_ids = list(it.chain(*[partition_by_adjacent_tiles(t) 302 | for t in tile_ids_by_zoom])) 303 | 304 | for tile_group in partitioned_tile_ids: 305 | zoom_level = int(tile_group[0].split('.')[1]) 306 | tileset_id = tile_group[0].split('.')[0] 307 | 308 | tile_positions = [[int(x) for x in t.split('.')[2:4]] for t in tile_group] 309 | 310 | # filter for tiles that are in bounds for this zoom level 311 | tile_positions = list(filter(lambda x: x[0] < 2 ** zoom_level, tile_positions)) 312 | tile_positions = list(filter(lambda x: x[1] < 2 ** zoom_level, tile_positions)) 313 | 314 | if len(tile_positions) == 0: 315 | # no in bounds tiles 316 | continue 317 | 318 | minx = min([t[0] for t in tile_positions]) 319 | maxx = max([t[0] for t in tile_positions]) 320 | 321 | miny = min([t[1] for t in tile_positions]) 322 | maxy = max([t[1] for t in tile_positions]) 323 | 324 | cached_datapath = get_cached_datapath(tileset.datafile.path) 325 | tile_data_by_position = retriever( 326 | cached_datapath, 327 | zoom_level, 328 | minx, miny, 329 | maxx - minx + 1, 330 | maxy - miny + 1 331 | ) 332 | 333 | tiles = [(".".join(map(str, [tileset_id] + [zoom_level] + list(position))), tile_data) 334 | for (position, tile_data) in tile_data_by_position.items()] 335 | 336 | generated_tiles += tiles 337 | 338 | return generated_tiles 339 | 340 | def generate_hibed_tiles(tileset, tile_ids): 341 | ''' 342 | Generate tiles from a hibed file. 343 | 344 | Parameters 345 | ---------- 346 | tileset: tilesets.models.Tileset object 347 | The tileset that the tile ids should be retrieved from 348 | tile_ids: [str,...] 349 | A list of tile_ids (e.g. xyx.0.0.1) identifying the tiles 350 | to be retrieved 351 | 352 | Returns 353 | ------- 354 | generated_tiles: [(tile_id, tile_data),...] 355 | A list of tile_id, tile_data tuples 356 | ''' 357 | generated_tiles = [] 358 | for tile_id in tile_ids: 359 | tile_id_parts = tile_id.split('.') 360 | tile_position = list(map(int, tile_id_parts[1:3])) 361 | dense = hdft.get_discrete_data( 362 | h5py.File( 363 | tileset.datafile.path 364 | ), 365 | tile_position[0], 366 | tile_position[1] 367 | ) 368 | 369 | tile_value = {'discrete': list([list([x.decode('utf-8') for x in d]) for d in dense])} 370 | 371 | generated_tiles += [(tile_id, tile_value)] 372 | 373 | return generated_tiles 374 | 375 | def bin_tiles_by_zoom(tile_ids): 376 | ''' 377 | Place these tiles into separate lists according to their 378 | zoom level. 379 | 380 | Parameters 381 | ---------- 382 | tile_ids: [str,...] 383 | A list of tile_ids (e.g. xyx.0.0.1) identifying the tiles 384 | to be retrieved 385 | 386 | Returns 387 | ------- 388 | tile_lists: {zoomLevel: [tile_id, tile_id]} 389 | A dictionary of tile lists 390 | ''' 391 | tile_id_lists = col.defaultdict(set) 392 | 393 | for tile_id in tile_ids: 394 | tile_id_parts = tile_id.split('.') 395 | tile_position = list(map(int, tile_id_parts[1:4])) 396 | zoom_level = tile_position[0] 397 | 398 | tile_id_lists[zoom_level].add(tile_id) 399 | 400 | return tile_id_lists 401 | 402 | 403 | def bin_tiles_by_zoom_level_and_transform(tile_ids): 404 | ''' 405 | Place these tiles into separate lists according to their 406 | zoom level and transform type 407 | 408 | Parameters 409 | ---------- 410 | tile_ids: [str,...] 411 | A list of tile_ids (e.g. xyx.0.0.1) identifying the tiles 412 | to be retrieved 413 | 414 | Returns 415 | ------- 416 | tile_lists: {(zoomLevel, transformType): [tile_id, tile_id]} 417 | A dictionary of tile ids 418 | ''' 419 | tile_id_lists = col.defaultdict(set) 420 | 421 | for tile_id in tile_ids: 422 | tile_id_parts = tile_id.split('.') 423 | tile_position = list(map(int, tile_id_parts[1:4])) 424 | zoom_level = tile_position[0] 425 | 426 | transform_method = hgco.get_transform_type(tile_id) 427 | 428 | tile_id_lists[(zoom_level, transform_method)].add(tile_id) 429 | 430 | return tile_id_lists 431 | 432 | def partition_by_adjacent_tiles(tile_ids, dimension=2): 433 | ''' 434 | Partition a set of tile ids into sets of adjacent tiles 435 | 436 | Parameters 437 | ---------- 438 | tile_ids: [str,...] 439 | A list of tile_ids (e.g. xyx.0.0.1) identifying the tiles 440 | to be retrieved 441 | dimension: int 442 | The dimensionality of the tiles 443 | 444 | Returns 445 | ------- 446 | tile_lists: [tile_ids, tile_ids] 447 | A list of tile lists, all of which have tiles that 448 | are within 1 position of another tile in the list 449 | ''' 450 | tile_id_lists = [] 451 | 452 | for tile_id in sorted(tile_ids, key=lambda x: [int(p) for p in x.split('.')[2:2+dimension]]): 453 | tile_id_parts = tile_id.split('.') 454 | 455 | # exclude the zoom level in the position 456 | # because the tiles should already have been partitioned 457 | # by zoom level 458 | tile_position = list(map(int, tile_id_parts[2:4])) 459 | 460 | added = False 461 | 462 | for tile_id_list in tile_id_lists: 463 | # iterate over each group of adjacent tiles 464 | has_close_tile = False 465 | 466 | for ct_tile_id in tile_id_list: 467 | ct_tile_id_parts = ct_tile_id.split('.') 468 | ct_tile_position = list(map(int, ct_tile_id_parts[2:2+dimension])) 469 | far_apart = False 470 | 471 | # iterate over each dimension and see if this tile is close 472 | for p1,p2 in zip(tile_position, ct_tile_position): 473 | if abs(int(p1) - int(p2)) > 1: 474 | # too far apart can't be part of the same group 475 | far_apart = True 476 | 477 | if not far_apart: 478 | # no position was too far 479 | tile_id_list += [tile_id] 480 | added = True 481 | break 482 | 483 | if added: 484 | break 485 | if not added: 486 | tile_id_lists += [[tile_id]] 487 | 488 | return tile_id_lists 489 | 490 | def generate_tiles(tileset_tile_ids): 491 | ''' 492 | Generate a tiles for the give tile_ids. 493 | 494 | All of the tile_ids must come from the same tileset. This function 495 | will determine the appropriate handler this tile given the tileset's 496 | filetype and datatype 497 | 498 | Parameters 499 | ---------- 500 | tileset_tile_ids: tuple 501 | A four-tuple containing the following parameters. 502 | tileset: tilesets.models.Tileset object 503 | The tileset that the tile ids should be retrieved from 504 | tile_ids: [str,...] 505 | A list of tile_ids (e.g. xyx.0.0.1) identifying the tiles 506 | to be retrieved 507 | raw: str or False 508 | The value of the GET request parameter `raw`. 509 | tileset_options: dict or None 510 | An optional dict containing tileset options, including aggregation options. 511 | 512 | Returns 513 | ------- 514 | tile_list: [(tile_id, tile_data),...] 515 | A list of tile_id, tile_data tuples 516 | ''' 517 | tileset, tile_ids, raw, tileset_options = tileset_tile_ids 518 | 519 | if tileset.filetype == 'hitile': 520 | return generate_hitile_tiles(tileset, tile_ids) 521 | elif tileset.filetype == 'beddb': 522 | return hgbe.tiles(tileset.datafile.path, tile_ids) 523 | elif tileset.filetype == 'bed2ddb' or tileset.filetype == '2dannodb': 524 | return generate_bed2ddb_tiles(tileset, tile_ids) 525 | elif tileset.filetype == 'geodb': 526 | return generate_bed2ddb_tiles(tileset, tile_ids, hggo.get_tiles) 527 | elif tileset.filetype == 'hibed': 528 | return generate_hibed_tiles(tileset, tile_ids) 529 | elif tileset.filetype == 'cooler': 530 | return hgco.generate_tiles(tileset.datafile.path, tile_ids) 531 | elif tileset.filetype == 'bigwig': 532 | chromsizes = get_chromsizes(tileset) 533 | return hgbi.tiles(tileset.datafile.path, tile_ids, chromsizes=chromsizes) 534 | elif tileset.filetype == 'fasta': 535 | chromsizes = get_chromsizes(tileset) 536 | return hgfa.tiles( 537 | tileset.datafile.path, 538 | tile_ids, 539 | chromsizes=chromsizes, 540 | max_tile_width=hss.MAX_FASTA_TILE_WIDTH 541 | ) 542 | elif tileset.filetype == 'bigbed': 543 | chromsizes = get_chromsizes(tileset) 544 | return hgbb.tiles(tileset.datafile.path, tile_ids, chromsizes=chromsizes) 545 | elif tileset.filetype == 'multivec': 546 | return generate_1d_tiles( 547 | tileset.datafile.path, 548 | tile_ids, 549 | ctmu.get_single_tile, 550 | tileset_options) 551 | elif tileset.filetype == 'imtiles': 552 | return hgim.get_tiles(tileset.datafile.path, tile_ids, raw) 553 | elif tileset.filetype == 'bam': 554 | return ctb.tiles( 555 | tileset.datafile.path, 556 | tile_ids, 557 | index_filename=tileset.indexfile.path, 558 | max_tile_width=hss.MAX_BAM_TILE_WIDTH 559 | ) 560 | else: 561 | filetype = tileset.filetype 562 | filepath = tileset.datafile.path 563 | 564 | if filetype in hgti.by_filetype: 565 | return hgti.by_filetype[filetype](filepath).tiles(tile_ids) 566 | 567 | return [(ti, {'error': 'Unknown tileset filetype: {}'.format(tileset.filetype)}) for ti in tile_ids] 568 | 569 | 570 | -------------------------------------------------------------------------------- /tilesets/json_schemas.py: -------------------------------------------------------------------------------- 1 | tiles_post_schema = { 2 | "$schema": "http://json-schema.org/draft-07/schema#", 3 | "definitions": { 4 | "multivecRowAggregationOptions": { 5 | "type": "object", 6 | "required": ["aggGroups", "aggFunc"], 7 | "additionalProperties": False, 8 | "properties": { 9 | "aggGroups": { 10 | "type": "array", 11 | "items": { 12 | "oneOf": [ 13 | { "type": "integer" }, 14 | { "type": "array", "items": { "type": "integer" }} 15 | ] 16 | } 17 | }, 18 | "aggFunc": { 19 | "type": "string", 20 | "enum": ["sum", "mean", "median", "std", "var", "min", "max"] 21 | } 22 | } 23 | } 24 | }, 25 | "type": "array", 26 | "items": { 27 | "type": "object", 28 | "required": ["tilesetUid", "tileIds"], 29 | "properties": { 30 | "tilesetUid": { "type": "string" }, 31 | "tileIds": { "type": "array", "items": { "type": "string" }}, 32 | "options": { 33 | "oneOf": [ 34 | { "$ref": "#/definitions/multivecRowAggregationOptions" } 35 | ] 36 | } 37 | } 38 | } 39 | } -------------------------------------------------------------------------------- /tilesets/management/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/higlass/higlass-server/cbfe79fe3ae0e844b4c0c78142a83733c8cc66a2/tilesets/management/__init__.py -------------------------------------------------------------------------------- /tilesets/management/commands/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/higlass/higlass-server/cbfe79fe3ae0e844b4c0c78142a83733c8cc66a2/tilesets/management/commands/__init__.py -------------------------------------------------------------------------------- /tilesets/management/commands/delete_tileset.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | from django.db.models import ProtectedError 3 | from django.conf import settings 4 | import tilesets.models as tm 5 | import os 6 | 7 | class Command(BaseCommand): 8 | def add_arguments(self, parser): 9 | parser.add_argument('--uuid', type=str, required=True) 10 | 11 | def handle(self, *args, **options): 12 | uuid = options.get('uuid') 13 | 14 | # search for Django object, remove associated file and record 15 | instance = tm.Tileset.objects.get(uuid=uuid) 16 | if not instance: 17 | raise CommandError('Instance for specified uuid ({}) was not found'.format(uuid)) 18 | else: 19 | filename = instance.datafile.name 20 | filepath = os.path.join(settings.MEDIA_ROOT, filename) 21 | if not os.path.isfile(filepath): 22 | raise CommandError('File does not exist under media root') 23 | try: 24 | os.remove(filepath) 25 | except OSError: 26 | raise CommandError('File under media root could not be removed') 27 | try: 28 | instance.delete() 29 | except ProtectedError: 30 | raise CommandError('Instance for specified uuid ({}) could not be deleted'.format(uuid)) -------------------------------------------------------------------------------- /tilesets/management/commands/ingest_tileset.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | import django.core.exceptions as dce 3 | from django.core.files import File 4 | 5 | import clodius.tiles.bigwig as hgbi 6 | import slugid 7 | import tilesets.models as tm 8 | import django.core.files.uploadedfile as dcfu 9 | import logging 10 | import os 11 | import os.path as op 12 | import tilesets.chromsizes as tcs 13 | from django.conf import settings 14 | 15 | logger = logging.getLogger(__name__) 16 | 17 | def remote_to_local(filename, no_upload): 18 | if filename[:7] == 'http://': 19 | filename = "{}..".format(filename.replace('http:/', 'http')) 20 | no_upload=True 21 | if filename[:8] == 'https://': 22 | filename = "{}..".format(filename.replace('https:/', 'https')) 23 | no_upload=True 24 | if filename[:6] == 'ftp://': 25 | filename = "{}..".format(filename.replace('ftp:/', 'ftp')) 26 | no_upload=True 27 | 28 | return (filename, no_upload) 29 | 30 | def ingest(filename=None, datatype=None, filetype=None, coordSystem='', coordSystem2='', 31 | uid=None, name=None, no_upload=False, project_name='', 32 | indexfile=None, temporary=False, **ignored): 33 | uid = uid or slugid.nice() 34 | name = name or op.split(filename)[1] 35 | 36 | if not filetype: 37 | raise CommandError('Filetype has to be specified') 38 | 39 | django_file = None 40 | 41 | # bamfiles need to be ingested with an index, if it's not 42 | # specified as a parameter, try to find it at filename + ".bai" 43 | # and complain if it can't be found 44 | if filetype == 'bam': 45 | if indexfile is None: 46 | indexfile = filename + '.bai' 47 | if not op.exists(indexfile): 48 | print(f'Index for bamfile {indexfile} not found. '+ 49 | 'Either specify explicitly using the --indexfile parameter ' + 50 | 'or create it in the expected location.') 51 | return 52 | 53 | # if we're ingesting a url, place it relative to the httpfs directories 54 | # and append two dots at the end 55 | 56 | filename, no_upload = remote_to_local(filename, no_upload) 57 | if indexfile: 58 | indexfile, _ = remote_to_local(indexfile, no_upload) 59 | 60 | # it's a regular file on the filesystem, not a file being entered as a url 61 | if no_upload: 62 | if (not op.isfile(op.join(settings.MEDIA_ROOT, filename)) and 63 | not op.islink(op.join(settings.MEDIA_ROOT, filename)) and 64 | not any([filename.startswith('http/'), filename.startswith('https/'), filename.startswith('ftp/')])): 65 | raise CommandError('File does not exist under media root') 66 | filename = op.join(settings.MEDIA_ROOT, filename) 67 | django_file = filename 68 | if indexfile: 69 | if (not op.isfile(op.join(settings.MEDIA_ROOT, indexfile)) and 70 | not op.islink(op.join(settings.MEDIA_ROOT, indexfile)) and 71 | not any([indexfile.startswith('http/'), indexfile.startswith('https/'), indexfile.startswith('ftp/')])): 72 | raise CommandError('Index file does not exist under media root') 73 | indexfile = op.join(settings.MEDIA_ROOT, indexfile) 74 | else: 75 | if os.path.islink(filename): 76 | django_file = File(open(os.readlink(filename),'rb')) 77 | if indexfile: 78 | indexfile = File(open(os.readlink(indexfile, 'rb'))) 79 | else: 80 | django_file = File(open(filename,'rb')) 81 | if indexfile: 82 | indexfile = File(open(indexfile, 'rb')) 83 | 84 | # remove the filepath of the filename 85 | django_file.name = op.split(django_file.name)[1] 86 | if indexfile: 87 | indexfile.name = op.split(indexfile.name)[1] 88 | 89 | if filetype.lower() == 'bigwig' or filetype.lower() == 'bigbed': 90 | coordSystem = check_for_chromsizes(filename, coordSystem) 91 | 92 | try: 93 | project_obj = tm.Project.objects.get(name=project_name) 94 | except dce.ObjectDoesNotExist: 95 | project_obj = tm.Project.objects.create( 96 | name=project_name 97 | ) 98 | 99 | return tm.Tileset.objects.create( 100 | datafile=django_file, 101 | indexfile=indexfile, 102 | filetype=filetype, 103 | datatype=datatype, 104 | coordSystem=coordSystem, 105 | coordSystem2=coordSystem2, 106 | owner=None, 107 | project=project_obj, 108 | uuid=uid, 109 | temporary=temporary, 110 | name=name) 111 | 112 | def chromsizes_match(chromsizes1, chromsizes2): 113 | pass 114 | 115 | def check_for_chromsizes(filename, coord_system): 116 | ''' 117 | Check to see if we have chromsizes matching the coord system 118 | of the filename. 119 | 120 | Parameters 121 | ---------- 122 | filename: string 123 | The name of the bigwig file 124 | coord_system: string 125 | The coordinate system (assembly) of this bigwig file 126 | ''' 127 | tileset_info = hgbi.tileset_info(filename) 128 | # print("tileset chromsizes:", tileset_info['chromsizes']) 129 | tsinfo_chromsizes = set([(str(chrom), str(size)) for chrom, size in tileset_info['chromsizes']]) 130 | # print("tsinfo_chromsizes:", tsinfo_chromsizes) 131 | 132 | chrom_info_tileset = None 133 | 134 | # check if we have a chrom sizes tileset that matches the coordsystem 135 | # of the input file 136 | if coord_system is not None and len(coord_system) > 0: 137 | try: 138 | chrom_info_tileset = tm.Tileset.objects.filter( 139 | coordSystem=coord_system, 140 | datatype='chromsizes') 141 | 142 | if len(chrom_info_tileset) > 1: 143 | raise CommandError("More than one available set of chromSizes" 144 | + "for this coordSystem ({})".format(coord_system)) 145 | 146 | chrom_info_tileset = chrom_info_tileset.first() 147 | except dce.ObjectDoesNotExist: 148 | chrom_info_tileset = None 149 | 150 | matches = [] 151 | 152 | if chrom_info_tileset is None: 153 | # we haven't found chromsizes matching the coordsystem 154 | # go through every chromsizes file and see if we have a match 155 | for chrom_info_tileset in tm.Tileset.objects.filter(datatype='chromsizes'): 156 | chromsizes_set = set([tuple(t) for 157 | t in tcs.get_tsv_chromsizes(chrom_info_tileset.datafile.path)]) 158 | 159 | matches += [(len(set.intersection(chromsizes_set, tsinfo_chromsizes)), 160 | chrom_info_tileset)] 161 | 162 | # print("chrom_info_tileset:", chromsizes_set) 163 | #print("intersection:", len(set.intersection(chromsizes_set, tsinfo_chromsizes))) 164 | #print("coord_system:", coord_system) 165 | else: 166 | # a set of chromsizes was provided 167 | chromsizes_set = set([tuple(t) for 168 | t in tcs.get_tsv_chromsizes(chrom_info_tileset.datafile.path)]) 169 | matches += [(len(set.intersection(chromsizes_set, tsinfo_chromsizes)), 170 | chrom_info_tileset)] 171 | 172 | # matches that overlap some chromsizes with the bigwig file 173 | overlap_matches = [m for m in matches if m[0] > 0] 174 | 175 | if len(overlap_matches) == 0: 176 | raise CommandError("No chromsizes available which match the chromosomes in this bigwig" 177 | + "See http://docs.higlass.io/data_preparation.html#bigwig-files " 178 | + "for more information" 179 | ) 180 | 181 | if len(overlap_matches) > 1: 182 | raise CommandError("Multiple matching coordSystems:" 183 | + "See http://docs.higlass.io/data_preparation.html#bigwig-files " 184 | + "for more information", 185 | ["({} [{}])".format(t[1].coordSystem, t[0]) for t in overlap_matches]) 186 | 187 | if (coord_system is not None 188 | and len(coord_system) > 0 189 | and overlap_matches[0][1].coordSystem != coord_system): 190 | raise CommandError("Matching chromosome sizes (coordSystem: {}) do not " 191 | + "match the specified coordinate sytem ({}). " 192 | + "Either omit the coordSystem or specify a matching one." 193 | + "See http://docs.higlass.io/data_preparation.html#bigwig-files " 194 | + "for more information".format(overlap_matches[0][1].coordSystem, coord_system)) 195 | 196 | if (coord_system is not None 197 | and len(coord_system) > 0 198 | and overlap_matches[0][1].coordSystem == coord_system): 199 | print("Using coordinates for coordinate system: {}".format(coord_system)) 200 | 201 | if coord_system is None or len(coord_system) == 0: 202 | print("No coordinate system specified, but we found matching " 203 | + "chromsizes. Using coordinate system {}." 204 | .format(overlap_matches[0][1].coordSystem)) 205 | 206 | return overlap_matches[0][1].coordSystem 207 | 208 | class Command(BaseCommand): 209 | def add_arguments(self, parser): 210 | # TODO: filename, datatype, fileType and coordSystem should 211 | # be checked to make sure they have valid values 212 | # for now, coordSystem2 should take the value of coordSystem 213 | # if the datatype is matrix 214 | # otherwise, coordSystem2 should be empty 215 | parser.add_argument('--filename', type=str) 216 | parser.add_argument('--indexfile', type=str) 217 | parser.add_argument('--datatype', type=str) 218 | parser.add_argument('--filetype', type=str) 219 | parser.add_argument('--coordSystem', default='', type=str) 220 | parser.add_argument('--coordSystem2', default='', type=str) 221 | # parser.add_argument('--coord', default='hg19', type=str) 222 | parser.add_argument('--uid', type=str) 223 | parser.add_argument('--name', type=str) 224 | parser.add_argument('--project-name', type=str, default='') 225 | 226 | # Named (optional) arguments 227 | parser.add_argument( 228 | '--no-upload', 229 | action='store_true', 230 | dest='no_upload', 231 | default=False, 232 | help='Skip upload', 233 | ) 234 | 235 | def handle(self, *args, **options): 236 | ingest(**options) 237 | -------------------------------------------------------------------------------- /tilesets/management/commands/list_tilesets.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | from django.core.files import File 3 | import slugid 4 | import tilesets.models as tm 5 | import django.core.files.uploadedfile as dcfu 6 | import os.path as op 7 | from django.conf import settings 8 | 9 | 10 | class Command(BaseCommand): 11 | def add_arguments(self, parser): 12 | pass 13 | 14 | def handle(self, *args, **options): 15 | for tileset in tm.Tileset.objects.all(): 16 | print('tileset:', tileset) 17 | -------------------------------------------------------------------------------- /tilesets/management/commands/modify_tileset.py: -------------------------------------------------------------------------------- 1 | from django.core.management.base import BaseCommand, CommandError 2 | from django.db.models import ProtectedError 3 | from django.conf import settings 4 | import tilesets.models as tm 5 | import os 6 | 7 | class Command(BaseCommand): 8 | def add_arguments(self, parser): 9 | parser.add_argument('--uuid', type=str, required=True) 10 | parser.add_argument('--name', type=str) 11 | 12 | def handle(self, *args, **options): 13 | uuid = options.get('uuid') 14 | name = options.get('name') 15 | 16 | # search for Django object, modify associated record 17 | instance = tm.Tileset.objects.get(uuid=uuid) 18 | if not instance: 19 | raise CommandError('Instance for specified uuid ({}) was not found'.format(uuid)) 20 | else: 21 | try: 22 | instance_dirty = False 23 | 24 | # only change tileset name if specified, and if it is 25 | # different from the current instance name 26 | if name and name != instance.name: 27 | instance.name = name 28 | instance_dirty = True 29 | 30 | # if any changes were applied, persist them 31 | if instance_dirty: 32 | instance.save() 33 | except ProtectedError: 34 | raise CommandError('Instance for specified uuid ({}) could not be modified'.format(uuid)) -------------------------------------------------------------------------------- /tilesets/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.3 on 2017-02-17 16:24 3 | from __future__ import unicode_literals 4 | 5 | from django.conf import settings 6 | from django.db import migrations, models 7 | import django.db.models.deletion 8 | import slugid.slugid 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | initial = True 14 | 15 | dependencies = [ 16 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 17 | ] 18 | 19 | operations = [ 20 | migrations.CreateModel( 21 | name='Tileset', 22 | fields=[ 23 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 24 | ('created', models.DateTimeField(auto_now_add=True)), 25 | ('uuid', models.CharField(default=slugid.slugid.nice, max_length=100, unique=True)), 26 | ('datafile', models.FileField(upload_to='uploads')), 27 | ('filetype', models.TextField()), 28 | ('datatype', models.TextField(default='unknown')), 29 | ('coordSystem', models.TextField()), 30 | ('coordSystem2', models.TextField(default='')), 31 | ('private', models.BooleanField(default=False)), 32 | ('name', models.TextField(blank=True)), 33 | ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tilesets', to=settings.AUTH_USER_MODEL)), 34 | ], 35 | options={ 36 | 'ordering': ('created',), 37 | 'permissions': (('view_tileset', 'View tileset'),), 38 | }, 39 | ), 40 | migrations.CreateModel( 41 | name='ViewConf', 42 | fields=[ 43 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 44 | ('created', models.DateTimeField(auto_now_add=True)), 45 | ('uuid', models.CharField(default=slugid.slugid.nice, max_length=100, unique=True)), 46 | ('viewconf', models.TextField()), 47 | ], 48 | options={ 49 | 'ordering': ('created',), 50 | }, 51 | ), 52 | ] 53 | -------------------------------------------------------------------------------- /tilesets/migrations/0002_auto_20170223_1629.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2017-02-23 16:29 3 | from __future__ import unicode_literals 4 | 5 | from django.conf import settings 6 | from django.db import migrations, models 7 | import django.db.models.deletion 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | ('tilesets', '0001_initial'), 14 | ] 15 | 16 | operations = [ 17 | migrations.AlterField( 18 | model_name='tileset', 19 | name='owner', 20 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tilesets', to=settings.AUTH_USER_MODEL), 21 | ), 22 | ] 23 | -------------------------------------------------------------------------------- /tilesets/migrations/0003_viewconf_higlassversion.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # Generated by Django 1.10.4 on 2017-05-31 16:17 3 | from __future__ import unicode_literals 4 | 5 | from django.db import migrations, models 6 | 7 | 8 | class Migration(migrations.Migration): 9 | 10 | dependencies = [ 11 | ('tilesets', '0002_auto_20170223_1629'), 12 | ] 13 | 14 | operations = [ 15 | migrations.AddField( 16 | model_name='viewconf', 17 | name='higlassVersion', 18 | field=models.CharField(default='', max_length=5), 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /tilesets/migrations/0004_auto_20181115_1744.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.9 on 2018-11-15 17:44 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0003_viewconf_higlassversion'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='tileset', 15 | name='uuid', 16 | field=models.CharField(default='asXQG1k1Rwe1sqzkuvEtvw', max_length=100, unique=True), 17 | ), 18 | migrations.AlterField( 19 | model_name='viewconf', 20 | name='higlassVersion', 21 | field=models.CharField(default='', max_length=16), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /tilesets/migrations/0005_auto_20181127_0239.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.9 on 2018-11-27 02:39 2 | 3 | from django.conf import settings 4 | from django.db import migrations, models 5 | import django.db.models.deletion 6 | import django.utils.timezone 7 | import tilesets.models 8 | 9 | 10 | class Migration(migrations.Migration): 11 | 12 | dependencies = [ 13 | migrations.swappable_dependency(settings.AUTH_USER_MODEL), 14 | ('tilesets', '0004_auto_20181115_1744'), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name='Project', 20 | fields=[ 21 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 22 | ('created', models.DateTimeField(auto_now_add=True)), 23 | ('last_viewed_time', models.DateTimeField(default=django.utils.timezone.now)), 24 | ('name', models.TextField()), 25 | ('description', models.TextField(blank=True)), 26 | ('uuid', models.CharField(default=tilesets.models.decoded_slugid, max_length=100, unique=True)), 27 | ('private', models.BooleanField(default=False)), 28 | ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), 29 | ], 30 | options={ 31 | 'ordering': ('created',), 32 | 'permissions': (('read', 'Read permission'), ('write', 'Modify tileset'), ('admin', 'Administrator priviliges')), 33 | }, 34 | ), 35 | migrations.CreateModel( 36 | name='Tag', 37 | fields=[ 38 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 39 | ('created', models.DateTimeField(auto_now_add=True)), 40 | ('name', models.TextField(unique=True)), 41 | ('description', models.TextField(blank=True, default='')), 42 | ('refs', models.IntegerField(default=0)), 43 | ], 44 | ), 45 | migrations.AlterModelOptions( 46 | name='tileset', 47 | options={'ordering': ('created',), 'permissions': (('read', 'Read permission'), ('write', 'Modify tileset'), ('admin', 'Administrator priviliges'))}, 48 | ), 49 | migrations.AlterField( 50 | model_name='tileset', 51 | name='uuid', 52 | field=models.CharField(default=tilesets.models.decoded_slugid, max_length=100, unique=True), 53 | ), 54 | migrations.AddField( 55 | model_name='tileset', 56 | name='project', 57 | field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tilesets.Project'), 58 | ), 59 | migrations.AddField( 60 | model_name='tileset', 61 | name='tags', 62 | field=models.ManyToManyField(blank=True, to='tilesets.Tag'), 63 | ), 64 | ] 65 | -------------------------------------------------------------------------------- /tilesets/migrations/0006_tileset_description.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.9 on 2018-11-27 02:43 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0005_auto_20181127_0239'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='tileset', 15 | name='description', 16 | field=models.TextField(blank=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /tilesets/migrations/0007_auto_20181127_0254.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.9 on 2018-11-27 02:54 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0006_tileset_description'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='project', 15 | name='name', 16 | field=models.TextField(unique=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /tilesets/migrations/0008_auto_20181129_1304.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.9 on 2018-11-29 13:04 2 | 3 | from django.db import migrations 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0007_auto_20181127_0254'), 10 | ] 11 | 12 | operations = [ 13 | migrations.RemoveField( 14 | model_name='tileset', 15 | name='tags', 16 | ), 17 | migrations.DeleteModel( 18 | name='Tag', 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /tilesets/migrations/0009_tileset_temporary.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.9 on 2018-12-13 22:16 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0008_auto_20181129_1304'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='tileset', 15 | name='temporary', 16 | field=models.BooleanField(default=False), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /tilesets/migrations/0010_auto_20181228_2250.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.9 on 2018-12-28 22:50 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0009_tileset_temporary'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='tileset', 15 | name='datatype', 16 | field=models.TextField(blank=True, default='unknown'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /tilesets/migrations/0011_auto_20181228_2252.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.0.9 on 2018-12-28 22:52 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0010_auto_20181228_2250'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='tileset', 15 | name='datatype', 16 | field=models.TextField(blank=True, default='unknown', null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /tilesets/migrations/0012_auto_20190923_0257.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 2.1.5 on 2019-09-23 02:57 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0011_auto_20181228_2252'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AddField( 14 | model_name='tileset', 15 | name='indexfile', 16 | field=models.FileField(default=None, null=True, upload_to='uploads'), 17 | ), 18 | migrations.AlterField( 19 | model_name='tileset', 20 | name='coordSystem2', 21 | field=models.TextField(blank=True, default=''), 22 | ), 23 | ] 24 | -------------------------------------------------------------------------------- /tilesets/migrations/0013_auto_20211119_1935.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.1 on 2021-11-19 19:35 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0012_auto_20190923_0257'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='tileset', 15 | name='indexfile', 16 | field=models.FileField(blank=True, default=None, null=True, upload_to='uploads'), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /tilesets/migrations/0014_auto_20211119_1939.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.1 on 2021-11-19 19:39 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | dependencies = [ 9 | ('tilesets', '0013_auto_20211119_1935'), 10 | ] 11 | 12 | operations = [ 13 | migrations.AlterField( 14 | model_name='viewconf', 15 | name='higlassVersion', 16 | field=models.CharField(blank=True, default='', max_length=16, null=True), 17 | ), 18 | ] 19 | -------------------------------------------------------------------------------- /tilesets/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/higlass/higlass-server/cbfe79fe3ae0e844b4c0c78142a83733c8cc66a2/tilesets/migrations/__init__.py -------------------------------------------------------------------------------- /tilesets/models.py: -------------------------------------------------------------------------------- 1 | from __future__ import unicode_literals 2 | 3 | import django 4 | import django.contrib.auth.models as dcam 5 | from django.db import models 6 | 7 | import slugid 8 | 9 | 10 | class ViewConf(models.Model): 11 | created = models.DateTimeField(auto_now_add=True) 12 | higlassVersion = models.CharField(max_length=16, default="", null=True, blank=True) 13 | uuid = models.CharField(max_length=100, unique=True, default=slugid.nice) 14 | viewconf = models.TextField() 15 | 16 | class Meta: 17 | ordering = ("created",) 18 | 19 | def __str__(self): 20 | """ 21 | Get a string representation of this model. Hopefully useful for the 22 | admin interface. 23 | """ 24 | return "Viewconf [uuid: {}]".format(self.uuid) 25 | 26 | 27 | def decoded_slugid(): 28 | return slugid.nice() 29 | 30 | 31 | class Project(models.Model): 32 | created = models.DateTimeField(auto_now_add=True) 33 | last_viewed_time = models.DateTimeField(default=django.utils.timezone.now) 34 | 35 | owner = models.ForeignKey( 36 | dcam.User, on_delete=models.CASCADE, blank=True, null=True 37 | ) 38 | name = models.TextField(unique=True) 39 | description = models.TextField(blank=True) 40 | uuid = models.CharField(max_length=100, unique=True, default=decoded_slugid) 41 | private = models.BooleanField(default=False) 42 | 43 | class Meta: 44 | ordering = ("created",) 45 | permissions = ( 46 | ("read", "Read permission"), 47 | ("write", "Modify tileset"), 48 | ("admin", "Administrator priviliges"), 49 | ) 50 | 51 | def __str__(self): 52 | return "Project [name: " + self.name + "]" 53 | 54 | 55 | class Tileset(models.Model): 56 | created = models.DateTimeField(auto_now_add=True) 57 | 58 | uuid = models.CharField(max_length=100, unique=True, default=decoded_slugid) 59 | 60 | # processed_file = models.TextField() 61 | datafile = models.FileField(upload_to="uploads") 62 | 63 | # indexfile is used for bam files 64 | indexfile = models.FileField( 65 | upload_to="uploads", default=None, blank=True, null=True 66 | ) 67 | filetype = models.TextField() 68 | datatype = models.TextField(default="unknown", blank=True, null=True) 69 | project = models.ForeignKey( 70 | Project, on_delete=models.CASCADE, blank=True, null=True 71 | ) 72 | description = models.TextField(blank=True) 73 | 74 | coordSystem = models.TextField() 75 | coordSystem2 = models.TextField(default="", blank=True) 76 | temporary = models.BooleanField(default=False) 77 | 78 | owner = models.ForeignKey( 79 | "auth.User", 80 | related_name="tilesets", 81 | on_delete=models.CASCADE, 82 | blank=True, 83 | null=True, # Allow anonymous owner 84 | ) 85 | private = models.BooleanField(default=False) 86 | name = models.TextField(blank=True) 87 | 88 | class Meta: 89 | ordering = ("created",) 90 | permissions = ( 91 | ("read", "Read permission"), 92 | ("write", "Modify tileset"), 93 | ("admin", "Administrator priviliges"), 94 | ) 95 | 96 | def __str__(self): 97 | """ 98 | Get a string representation of this model. Hopefully useful for the 99 | admin interface. 100 | """ 101 | return "Tileset [name: {}] [ft: {}] [uuid: {}]".format( 102 | self.name, self.filetype, self.uuid 103 | ) 104 | -------------------------------------------------------------------------------- /tilesets/permissions.py: -------------------------------------------------------------------------------- 1 | from rest_framework import permissions 2 | 3 | 4 | class IsRequestMethodGet(permissions.BasePermission): 5 | """The request is a GET request.""" 6 | 7 | def has_permission(self, request, view): 8 | if request.method == 'GET': 9 | return True 10 | 11 | return False 12 | 13 | 14 | class IsOwnerOrReadOnly(permissions.BasePermission): 15 | """Custom permission to only allow owners of an object to edit it.""" 16 | 17 | def has_object_permission(self, request, view, obj): 18 | # Read permissions are allowed to any request, 19 | # so we'll always allow GET, HEAD or OPTIONS requests. 20 | # if request.method in permissions.SAFE_METHODS: 21 | # Write permissions are only allowed to the owner of the snippet. 22 | if request.user.is_staff: 23 | return True 24 | else: 25 | return obj.owner == request.user 26 | 27 | 28 | class UserPermission(permissions.BasePermission): 29 | # Taken from http://stackoverflow.com/a/34162842/899470 30 | 31 | def has_permission(self, request, view): 32 | if view.action in ['retrieve', 'list']: 33 | return True 34 | elif view.action in ['create', 'update', 'partial_update', 'destroy']: 35 | return request.user.is_authenticated 36 | else: 37 | return False 38 | 39 | def has_object_permission(self, request, view, obj): 40 | if view.action == 'retrieve': 41 | return ( 42 | request.user.is_authenticated and 43 | (obj == request.user or request.user.is_superuser) 44 | ) 45 | elif view.action in ['update', 'partial_update', 'destroy']: 46 | return request.user.is_authenticated and ( 47 | request.user.is_superuser or request.user == obj.owner) 48 | else: 49 | return False 50 | 51 | 52 | class UserPermissionReadOnly(UserPermission): 53 | """Custom permission to only allow read requests.""" 54 | 55 | def has_permission(self, request, view): 56 | if view.action in ['retrieve', 'list']: 57 | return True 58 | else: 59 | return False 60 | 61 | def has_object_permission(self, request, view, obj): 62 | if view.action == 'retrieve': 63 | return ( 64 | request.user.is_authenticated and 65 | (obj == request.user or request.user.is_superuser) 66 | ) 67 | else: 68 | return False 69 | -------------------------------------------------------------------------------- /tilesets/serializers.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | from rest_framework import serializers 4 | from tilesets.models import Tileset, ViewConf 5 | from django.contrib.auth.models import User 6 | import tilesets.generate_tiles as tgt 7 | import tilesets.models as tm 8 | import rest_framework.utils as rfu 9 | from django.core.files.base import File 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | class ProjectsSerializer(serializers.HyperlinkedModelSerializer): 14 | class Meta: 15 | model = tm.Project 16 | fields = ( 17 | 'uuid', 18 | 'name', 19 | 'description', 20 | 'private' 21 | ) 22 | 23 | class UserSerializer(serializers.ModelSerializer): 24 | tilesets = serializers.PrimaryKeyRelatedField( 25 | many=True, queryset=Tileset.objects.all() 26 | ) 27 | 28 | class Meta: 29 | model = User 30 | fields = ('id', 'username') 31 | 32 | 33 | class ViewConfSerializer(serializers.ModelSerializer): 34 | class Meta: 35 | model = ViewConf 36 | fields = ('uuid', 'viewconf') 37 | 38 | 39 | class TilesetSerializer(serializers.ModelSerializer): 40 | project = serializers.SlugRelatedField( 41 | queryset=tm.Project.objects.all(), 42 | slug_field='uuid', 43 | allow_null=True, 44 | required=False) 45 | project_name = serializers.SerializerMethodField('retrieve_project_name') 46 | 47 | def retrieve_project_name(self, obj): 48 | if obj.project is None: 49 | return '' 50 | 51 | return obj.project.name 52 | 53 | class Meta: 54 | owner = serializers.ReadOnlyField(source='owner.username') 55 | model = tm.Tileset 56 | fields = ( 57 | 'uuid', 58 | 'datafile', 59 | 'filetype', 60 | 'datatype', 61 | 'name', 62 | 'coordSystem', 63 | 'coordSystem2', 64 | 'created', 65 | 'project', 66 | 'project_name', 67 | 'description', 68 | 'private', 69 | ) 70 | 71 | 72 | class UserFacingTilesetSerializer(TilesetSerializer): 73 | owner = serializers.ReadOnlyField(source='owner.username') 74 | project_name = serializers.SerializerMethodField('retrieve_project_name') 75 | project_owner = serializers.SerializerMethodField('retrieve_project_owner') 76 | 77 | def retrieve_project_name(self, obj): 78 | if obj.project is None: 79 | return '' 80 | 81 | return obj.project.name 82 | 83 | def retrieve_project_owner(self, obj): 84 | if obj.project is None: 85 | return '' 86 | 87 | if obj.project.owner is None: 88 | return '' 89 | 90 | return obj.project.owner.username 91 | 92 | class Meta: 93 | model = tm.Tileset 94 | fields = ( 95 | 'uuid', 96 | 'filetype', 97 | 'datatype', 98 | 'private', 99 | 'name', 100 | 'coordSystem', 101 | 'coordSystem2', 102 | 'created', 103 | 'owner', 104 | 'project_name', 105 | 'project_owner', 106 | 'description', 107 | ) 108 | -------------------------------------------------------------------------------- /tilesets/storage.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | from errno import EEXIST 4 | import hashlib 5 | import os 6 | import sys 7 | 8 | from django.core.files import File 9 | from django.core.files.storage import FileSystemStorage 10 | from django.utils.encoding import force_unicode 11 | 12 | 13 | class NoAvailableName(Exception): 14 | pass 15 | 16 | 17 | def HashedFilenameMetaStorage(storage_class): 18 | class HashedFilenameStorage(storage_class): 19 | def __init__(self, *args, **kwargs): 20 | # Try to tell storage_class not to uniquify filenames. 21 | # This class will be the one that uniquifies. 22 | try: 23 | new_kwargs = dict(kwargs, uniquify_names=False) 24 | super(HashedFilenameStorage, self).__init__(*args, 25 | **new_kwargs) 26 | except TypeError: 27 | super(HashedFilenameStorage, self).__init__(*args, **kwargs) 28 | 29 | def get_available_name(self, name): 30 | raise NoAvailableName() 31 | 32 | def _get_content_name(self, name, content, chunk_size=None): 33 | dir_name, file_name = os.path.split(name) 34 | file_ext = os.path.splitext(file_name)[1] 35 | file_root = self._compute_hash(content=content, 36 | chunk_size=chunk_size) 37 | # file_ext includes the dot. 38 | return os.path.join(dir_name, file_root) 39 | 40 | def _compute_hash(self, content, chunk_size=None): 41 | if chunk_size is None: 42 | chunk_size = getattr(content, 'DEFAULT_CHUNK_SIZE', 43 | File.DEFAULT_CHUNK_SIZE) 44 | 45 | hasher = hashlib.sha1() 46 | 47 | cursor = content.tell() 48 | content.seek(0) 49 | try: 50 | while True: 51 | data = content.read(chunk_size) 52 | if not data: 53 | break 54 | hasher.update(data) 55 | return hasher.hexdigest() 56 | finally: 57 | content.seek(cursor) 58 | 59 | def save(self, name, content, max_length): 60 | # Get the proper name for the file, as it will actually be saved. 61 | if name is None: 62 | name = content.name 63 | 64 | name = self._get_content_name(name, content) 65 | name = self._save(name, content) 66 | 67 | # Store filenames with forward slashes, even on Windows 68 | return force_unicode(name.replace('\\', '/')) 69 | 70 | def _save(self, name, content, *args, **kwargs): 71 | new_name = self._get_content_name(name=name, content=content) 72 | try: 73 | return super(HashedFilenameStorage, self)._save(new_name, 74 | content, 75 | *args, 76 | **kwargs) 77 | except NoAvailableName: 78 | # File already exists, so we can safely do nothing 79 | # because their contents match. 80 | pass 81 | except OSError as e: 82 | if e.errno == EEXIST: 83 | # We have a safe storage layer and file exists. 84 | pass 85 | else: 86 | raise 87 | return new_name 88 | 89 | HashedFilenameStorage.__name__ = 'HashedFilename' + storage_class.__name__ 90 | return HashedFilenameStorage 91 | 92 | 93 | HashedFilenameFileSystemStorage = HashedFilenameMetaStorage( 94 | storage_class=FileSystemStorage, 95 | ) 96 | -------------------------------------------------------------------------------- /tilesets/suggestions.py: -------------------------------------------------------------------------------- 1 | import sqlite3 2 | 3 | def get_gene_suggestions(db_file, text): 4 | ''' 5 | Get a list of autocomplete suggestions for genes containing 6 | the text. 7 | 8 | Args: 9 | 10 | db_file (string): The filename for the SQLite file containing gene annotations 11 | text (string): The text to be included in the suggestions 12 | 13 | Returns: 14 | 15 | suggestions (list): A list of dictionaries containing the suggestions: 16 | e.g. ([{'txStart': 10, 'txEnd': 20, 'score': 15, 'geneName': 'XV4'}]) 17 | ''' 18 | con = sqlite3.connect(db_file) 19 | c = con.cursor() 20 | 21 | query = """ 22 | SELECT importance, chrOffset, fields FROM intervals 23 | WHERE fields LIKE '%{}%' 24 | ORDER BY importance DESC 25 | LIMIT 10 26 | """.format(text) 27 | 28 | rows = c.execute(query).fetchall() 29 | 30 | to_return = [] 31 | for (importance, chrOffset, fields) in rows: 32 | field_parts = fields.split('\t') 33 | to_return += [{ 34 | 'chr': field_parts[0], 35 | 'txStart': int(field_parts[1]), 36 | 'txEnd': int(field_parts[2]), 37 | 'score': importance, 38 | 'geneName': field_parts[3]}] 39 | 40 | c.execute(query) 41 | 42 | 43 | 44 | return to_return 45 | -------------------------------------------------------------------------------- /tilesets/test_data/hi.txt: -------------------------------------------------------------------------------- 1 | Hello 2 | -------------------------------------------------------------------------------- /tilesets/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url, include 2 | from tilesets import views 3 | from rest_framework.routers import SimpleRouter 4 | from rest_framework_swagger.views import get_swagger_view 5 | 6 | schema_view = get_swagger_view(title='Pastebin API') 7 | # Create a router and register our viewsets with it. 8 | router = SimpleRouter() 9 | 10 | router.register(r'tilesets', views.TilesetsViewSet, 'tilesets') 11 | #router.register(r'users', views.UserViewSet) 12 | 13 | 14 | # The API URLs are now determined automatically by the router. 15 | # Additionally, we include the login URLs for the browsable API. 16 | urlpatterns = [ 17 | #url(r'^schema', schema_view), 18 | url(r'^viewconf', views.viewconfs), 19 | url(r'^uids_by_filename', views.uids_by_filename), 20 | url(r'^tiles/$', views.tiles), 21 | url(r'^tileset_info/$', views.tileset_info), 22 | url(r'^suggest/$', views.suggest), 23 | url(r'^', include(router.urls)), 24 | url(r'^link_tile/$', views.link_tile), 25 | url(r'^register_url/$', views.register_url), 26 | url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), 27 | url(r'^chrom-sizes/$', views.sizes), 28 | url(r'^available-chrom-sizes/$', views.available_chrom_sizes) 29 | #url(r'^users/$', views.UserList.as_view()), 30 | #url(r'^users/(?P[0-9]+)/$', views.UserDetail.as_view()) 31 | ] 32 | -------------------------------------------------------------------------------- /unit_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | umount media/http 4 | umount media/https 5 | 6 | simple-httpfs.py media/http 7 | simple-httpfs.py media/https 8 | 9 | python manage.py test fragments.tests.FragmentsTest --failfast 10 | python manage.py test tilesets.tests.BamTests --failfast 11 | python manage.py test tilesets.tests.FileUploadTest --failfast 12 | python manage.py test tilesets.tests.MultivecTests --failfast 13 | 14 | #python manage.py test tilesets --failfast 15 | -------------------------------------------------------------------------------- /update.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | git pull 4 | 5 | pip install -r ./requirements.txt 6 | 7 | python manage.py migrate 8 | -------------------------------------------------------------------------------- /uwsgi_params: -------------------------------------------------------------------------------- 1 | uwsgi_param QUERY_STRING $query_string; 2 | uwsgi_param REQUEST_METHOD $request_method; 3 | uwsgi_param CONTENT_TYPE $content_type; 4 | uwsgi_param CONTENT_LENGTH $content_length; 5 | 6 | uwsgi_param REQUEST_URI $request_uri; 7 | uwsgi_param PATH_INFO $document_uri; 8 | uwsgi_param DOCUMENT_ROOT $document_root; 9 | uwsgi_param SERVER_PROTOCOL $server_protocol; 10 | uwsgi_param REQUEST_SCHEME $scheme; 11 | uwsgi_param HTTPS $https if_not_empty; 12 | 13 | uwsgi_param REMOTE_ADDR $remote_addr; 14 | uwsgi_param REMOTE_PORT $remote_port; 15 | uwsgi_param SERVER_PORT $server_port; 16 | uwsgi_param SERVER_NAME $server_name; -------------------------------------------------------------------------------- /website/tests.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import os.path as op 4 | from asynctest import CoroutineMock 5 | 6 | from pathlib import Path 7 | from unittest import TestCase, mock 8 | 9 | import django.contrib.auth.models as dcam 10 | import django.test as dt 11 | import tilesets.models as tm 12 | import website.views as wv 13 | 14 | import higlass_server.settings as hss 15 | 16 | class SiteTests(dt.TestCase): 17 | def setUp(self): 18 | self.user1 = dcam.User.objects.create_user( 19 | username='user1', password='pass' 20 | ) 21 | 22 | upload_json_text = json.dumps({'hi': 'there'}) 23 | 24 | self.viewconf = tm.ViewConf.objects.create( 25 | viewconf=upload_json_text, uuid='md') 26 | 27 | def test_link_url(self): 28 | ret = self.client.get('/link/') 29 | assert "No uuid specified" in ret.content.decode('utf8') 30 | 31 | ret = self.client.get('/link/?d=x') 32 | assert ret.status_code == 404 33 | 34 | ret = self.client.get('/link/?d=md') 35 | assert ret.content.decode('utf8').find('window.location') >= 0 36 | 37 | @mock.patch('website.views.screenshot', new=CoroutineMock()) 38 | def test_thumbnail(self): 39 | uuid = 'some_fake_uid' 40 | output_file = Path(hss.THUMBNAILS_ROOT) / (uuid + ".png") 41 | 42 | if not output_file.exists(): 43 | output_file.touch() 44 | 45 | ret = self.client.get( 46 | f'/thumbnail/?d={uuid}' 47 | ) 48 | 49 | self.assertEqual(ret.status_code, 200) 50 | 51 | ret = self.client.get( 52 | f'/t/?d=..file' 53 | ) 54 | 55 | self.assertEqual(ret.status_code, 400) 56 | -------------------------------------------------------------------------------- /website/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import url, include 2 | from website import views 3 | 4 | # The API URLs are now determined automatically by the router. 5 | # Additionally, we include the login URLs for the browsable API. 6 | urlpatterns = [ 7 | #url(r'^schema', schema_view), 8 | url(r'^link/$', views.link), 9 | url(r'^l/$', views.link), 10 | url(r'^thumbnail/$', views.thumbnail), 11 | url(r'^t/$', views.thumbnail) 12 | ] 13 | -------------------------------------------------------------------------------- /website/views.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import pyppeteer 3 | import asyncio 4 | import logging 5 | import os 6 | import os.path as op 7 | from pyppeteer import launch 8 | import tempfile 9 | 10 | import tilesets.models as tm 11 | 12 | import higlass_server.settings as hss 13 | 14 | from django.core.exceptions import ObjectDoesNotExist 15 | from django.http import HttpRequest, HttpResponse, \ 16 | HttpResponseNotFound, HttpResponseBadRequest 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | def link(request): 21 | '''Generate a small page containing the metadata necessary for 22 | link unfurling by Slack or Twitter. The generated page will 23 | point to a screenshot of the rendered viewconf. The page will automatically 24 | redirect to the rendering so that if anybody clicks on this link 25 | they'll be taken to an interactive higlass view. 26 | 27 | The viewconf to render should be specified with the d= html parameter. 28 | 29 | Args: 30 | request: The incoming http request. 31 | Returns: 32 | A response containing an html page with metadata 33 | ''' 34 | # the uuid of the viewconf to render 35 | uuid = request.GET.get('d') 36 | 37 | if not uuid: 38 | # if there's no uuid specified, return an empty page 39 | return HttpResponseNotFound('

No uuid specified

') 40 | 41 | try: 42 | obj = tm.ViewConf.objects.get(uuid=uuid) 43 | except ObjectDoesNotExist: 44 | return HttpResponseNotFound('

No such uuid

') 45 | 46 | # Temporarily deactivate the thumbnail generation (crashes the server) 47 | # thumb_url=f'{request.scheme}://{request.get_host()}/thumbnail/?d={uuid}' 48 | # Removed META tags: 49 | # 50 | # 51 | # 52 | 53 | # the page to redirect to for interactive explorations 54 | redirect_url=f'{request.scheme}://{request.get_host()}/app/?config={uuid}' 55 | 56 | # Simple html page. Not a template just for simplicity's sake. 57 | # If it becomes more complex, we can make it into a template. 58 | html = f""" 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 79 | 80 | """ 81 | 82 | return HttpResponse(html) 83 | 84 | def thumbnail(request: HttpRequest): 85 | '''Retrieve a thumbnail for the viewconf specified by the d= 86 | parameter. 87 | 88 | Args: 89 | request: The incoming request. 90 | Returns: 91 | A response of either 404 if there's no uuid provided or an 92 | image containing a screenshot of the rendered viewconf with 93 | that uuid. 94 | ''' 95 | uuid = request.GET.get('d') 96 | 97 | base_url = f'{request.scheme}://localhost/app/' 98 | 99 | if not uuid: 100 | return HttpResponseNotFound('

No uuid specified

') 101 | 102 | if '.' in uuid or '/' in uuid: 103 | # no funny business 104 | logger.warning('uuid contains . or /: %s', uuid) 105 | return HttpResponseBadRequest("uuid can't contain . or /") 106 | 107 | if not op.exists(hss.THUMBNAILS_ROOT): 108 | os.makedirs(hss.THUMBNAILS_ROOT) 109 | 110 | output_file = op.abspath(op.join(hss.THUMBNAILS_ROOT, uuid + ".png")) 111 | thumbnails_base = op.abspath(hss.THUMBNAILS_ROOT) 112 | 113 | if output_file.find(thumbnails_base) != 0: 114 | logger.warning('Thumbnail file is not in thumbnail_base: %s uuid: %s', 115 | output_file, uuid) 116 | return HttpResponseBadRequest('Strange path') 117 | 118 | if not op.exists(output_file): 119 | loop = asyncio.new_event_loop() 120 | asyncio.set_event_loop(loop) 121 | loop.run_until_complete( 122 | screenshot( 123 | base_url, 124 | uuid, 125 | output_file)) 126 | loop.close() 127 | 128 | with open(output_file, 'rb') as file: 129 | return HttpResponse( 130 | file.read(), 131 | content_type="image/jpeg") 132 | 133 | async def screenshot( 134 | base_url: str, 135 | uuid: str, 136 | output_file: str 137 | ): 138 | '''Take a screenshot of a rendered viewconf. 139 | 140 | Args: 141 | base_url: The url to use for rendering the viewconf 142 | uuid: The uuid of the viewconf to render 143 | output_file: The location on the local filesystem to cache 144 | the thumbnail. 145 | Returns: 146 | Nothing, just stores the screenshot at the given location. 147 | ''' 148 | browser = await launch( 149 | headless=True, 150 | args=['--no-sandbox'], 151 | handleSIGINT=False, 152 | handleSIGTERM=False, 153 | handleSIGHUP=False 154 | ) 155 | url = f'{base_url}?config={uuid}' 156 | page = await browser.newPage() 157 | await page.goto(url, { 158 | 'waitUntil': 'networkidle0', 159 | }) 160 | await page.screenshot({'path': output_file}) 161 | await browser.close() 162 | --------------------------------------------------------------------------------