├── .github └── workflows │ └── workflow.yml ├── .gitignore ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── appveyor.yml ├── isyntax2raw ├── __init__.py ├── cli │ ├── __init__.py │ └── isyntax2raw.py └── resources │ ├── __init__.py │ └── ome_template.xml ├── setup.cfg ├── setup.py └── version.py /.github/workflows/workflow.yml: -------------------------------------------------------------------------------- 1 | # https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions 2 | 3 | name: Build 4 | on: 5 | push: 6 | pull_request: 7 | 8 | jobs: 9 | 10 | test: 11 | name: Test 12 | strategy: 13 | # Keep running so we can see if other tests pass 14 | fail-fast: false 15 | matrix: 16 | python-version: 17 | - '3.6' 18 | - '3.7' 19 | - '3.8' 20 | - '3.9' 21 | os: 22 | - ubuntu-20.04 23 | include: 24 | - python-version: '3.7' 25 | os: macos-latest 26 | - python-version: '3.7' 27 | os: windows-latest 28 | runs-on: ${{ matrix.os }} 29 | steps: 30 | - uses: actions/checkout@v2 31 | - uses: actions/setup-python@v2 32 | with: 33 | python-version: ${{ matrix.python-version }} 34 | - name: Install dependencies 35 | run: python -mpip install -U wheel flake8 virtualenv 36 | - name: Run tests 37 | run: | 38 | # actions/checkout#206 39 | git fetch --prune --unshallow --tags --force 40 | git describe 41 | flake8 42 | python setup.py build 43 | 44 | # https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/ 45 | publish-pypi: 46 | name: Pypi 47 | if: startsWith(github.ref, 'refs/tags') 48 | needs: 49 | # Only publish if other jobs passed 50 | - test 51 | runs-on: ubuntu-latest 52 | steps: 53 | - uses: actions/checkout@v2 54 | - uses: actions/setup-python@v2 55 | - name: Build package 56 | run: | 57 | # actions/checkout#206 58 | git fetch --prune --unshallow --tags --force 59 | python -mpip install wheel 60 | python setup.py sdist bdist_wheel 61 | - name: Publish to PyPI 62 | uses: pypa/gh-action-pypi-publish@v1.3.0 63 | with: 64 | password: ${{ secrets.PYPI_API_TOKEN }} 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | __pycache__ 3 | build 4 | dist 5 | isyntax_to_raw.egg-info 6 | isyntax2raw/version.py 7 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2019 Glencoe Software, Inc. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation 11 | and/or other materials provided with the distribution. 12 | 13 | 3. Neither the name of the copyright holder nor the names of its contributors 14 | may be used to endorse or promote products derived from this software without 15 | specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 20 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 21 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 23 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 24 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 25 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include README.md 2 | include LICENSE.txt 3 | include version.py 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![AppVeyor status](https://ci.appveyor.com/api/projects/status/github/isyntax2raw)](https://ci.appveyor.com/project/gs-jenkins/isyntax2raw) 2 | 3 | # iSyntax Converter 4 | 5 | Python tool that uses Philips' SDK to write slides in an intermediary raw format. 6 | 7 | ## Requirements 8 | 9 | * Python 3.6+ 10 | * Philips iSyntax SDK (https://www.openpathology.philips.com) 11 | 12 | The iSyntax SDK __must__ be downloaded separately from Philips and the 13 | relevant license agreement agreed to before any conversion can take place. 14 | 15 | As of version 0.4.0, which has a Python 3.6+ requirement, the supported 16 | iSyntax SDK versions and environments are as follows: 17 | 18 | * iSyntax SDK 1.2.1 (CentOS 7, Ubuntu 18.04, Windows 10 64-bit) 19 | * iSyntax SDK 2.0 (CentOS 8, Ubuntu 18.04, Windows 10 64-bit) 20 | 21 | ## Usage 22 | 23 | Basic usage is: 24 | 25 | isyntax2raw write_tiles /path/to/input.isyntax /path/to/directory.zarr 26 | 27 | Please see `isyntax2raw write_tiles --help` for detailed information. 28 | 29 | Output tile width and height can optionally be specified; default values are 30 | detailed in `--help`. 31 | 32 | A directory structure containing the pyramid tiles at all resolutions and 33 | macro/label images will be created. The default format is Zarr compliant with https://ngff.openmicroscopy.org/0.4. 34 | Additional metadata is written to a JSON file. Be mindful of available disk space, as 35 | larger .isyntax files can result in >20 GB of tiles. 36 | 37 | Use of the Zarr file type will result in losslessly compressed output. This 38 | is the only format currently supported by the downstream `raw2ometiff` (as of 39 | version 0.3.0). 40 | 41 | ## Background color 42 | 43 | Any missing tiles are filled with 0 by default, which displays as black. 44 | The fill value can be changed using the `--fill_color` option, which accepts 45 | a single integer between 0 and 255 inclusive. Setting `--fill_color=255` 46 | will cause any missing tiles to display as white. 47 | 48 | ## Performance 49 | 50 | This package is __highly__ sensitive to underlying hardware as well as 51 | the following configuration options: 52 | 53 | * `--max_workers` 54 | * `--tile_width` 55 | * `--tile_height` 56 | * `--batch_size` 57 | 58 | On systems with significant I/O bandwidth, particularly SATA or 59 | NVMe based storage, we have found sharply diminishing returns with worker 60 | counts > 4. There are significant performance gains to be had utilizing 61 | larger tile sizes but be mindful of the consequences on the downstream 62 | workflow. You may find increasing the batch size on systems with very 63 | high single core performance to give modest performance gains. 64 | 65 | In general, expect to need to tune the above settings and measure 66 | relative performance. 67 | 68 | ## License 69 | 70 | The iSyntax converter is distributed under the terms of the BSD license. 71 | Please see `LICENSE.txt` for further details. 72 | 73 | ## Areas to improve 74 | 75 | * Currently assumes brightfield (RGB, 8 bits per channel) without really 76 | checking the metadata. Probably should check bit depths etc. 77 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | image: ubuntu1804 2 | 3 | build: off 4 | 5 | install: 6 | - sudo apt-get update 7 | - sudo apt-get install -y python3-pip 8 | 9 | build_script: 10 | - pip3 install --user -U pip setuptools wheel flake8 11 | - export PATH="$(python3 -m site --user-base)/bin:${PATH}" 12 | - python3 setup.py build 13 | 14 | test_script: 15 | - flake8 . 16 | 17 | after_test: 18 | - python3 setup.py bdist_wheel 19 | 20 | artifacts: 21 | - path: dist/* 22 | 23 | cache: 24 | - ${HOME}/.eggs -> setup.py 25 | -------------------------------------------------------------------------------- /isyntax2raw/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (c) 2019 Glencoe Software, Inc. All rights reserved. 5 | # 6 | # This software is distributed under the terms described by the LICENSE.txt 7 | # file you can find at the root of the distribution bundle. If the file is 8 | # missing please request a copy by contacting info@glencoesoftware.com 9 | 10 | from io import BytesIO 11 | import json 12 | import logging 13 | import math 14 | import os 15 | 16 | import numpy as np 17 | import pixelengine 18 | import softwarerendercontext 19 | import softwarerenderbackend 20 | import zarr 21 | 22 | from dateutil.parser import parse 23 | 24 | from datetime import datetime 25 | from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait 26 | from threading import BoundedSemaphore 27 | 28 | from PIL import Image 29 | from kajiki import PackageLoader 30 | from zarr.storage import FSStore 31 | 32 | 33 | log = logging.getLogger(__name__) 34 | 35 | # version of the Zarr layout 36 | LAYOUT_VERSION = 3 37 | 38 | 39 | class MaxQueuePool(object): 40 | """This Class wraps a concurrent.futures.Executor 41 | limiting the size of its task queue. 42 | If `max_queue_size` tasks are submitted, the next call to submit will 43 | block until a previously submitted one is completed. 44 | 45 | Brought in from: 46 | * https://gist.github.com/noxdafox/4150eff0059ea43f6adbdd66e5d5e87e 47 | 48 | See also: 49 | * https://www.bettercodebytes.com/ 50 | theadpoolexecutor-with-a-bounded-queue-in-python/ 51 | * https://pypi.org/project/bounded-pool-executor/ 52 | * https://bugs.python.org/issue14119 53 | * https://bugs.python.org/issue29595 54 | * https://github.com/python/cpython/pull/143 55 | """ 56 | def __init__(self, executor, max_queue_size, max_workers=None): 57 | if max_workers is None: 58 | max_workers = max_queue_size 59 | self.pool = executor(max_workers=max_workers) 60 | self.pool_queue = BoundedSemaphore(max_queue_size) 61 | 62 | def submit(self, function, *args, **kwargs): 63 | """Submits a new task to the pool, blocks if Pool queue is full.""" 64 | self.pool_queue.acquire() 65 | 66 | future = self.pool.submit(function, *args, **kwargs) 67 | future.add_done_callback(self.pool_queue_callback) 68 | 69 | return future 70 | 71 | def pool_queue_callback(self, _): 72 | """Called once task is done, releases one queue slot.""" 73 | self.pool_queue.release() 74 | 75 | def __enter__(self): 76 | return self 77 | 78 | def __exit__(self, exception_type, exception_value, traceback): 79 | self.pool.__exit__(exception_type, exception_value, traceback) 80 | 81 | 82 | class WriteTiles(object): 83 | 84 | def __init__( 85 | self, tile_width, tile_height, resolutions, max_workers, 86 | batch_size, fill_color, nested, input_path, output_path 87 | ): 88 | self.tile_width = tile_width 89 | self.tile_height = tile_height 90 | self.resolutions = resolutions 91 | self.max_workers = max_workers 92 | self.batch_size = batch_size 93 | self.fill_color = fill_color 94 | self.nested = nested 95 | self.input_path = input_path 96 | self.slide_directory = output_path 97 | 98 | render_context = softwarerendercontext.SoftwareRenderContext() 99 | render_backend = softwarerenderbackend.SoftwareRenderBackend() 100 | 101 | self.pixel_engine = pixelengine.PixelEngine( 102 | render_backend, render_context 103 | ) 104 | self.pixel_engine["in"].open(input_path, "ficom") 105 | self.sdk_v1 = hasattr(self.pixel_engine["in"], "BARCODE") 106 | 107 | def __enter__(self): 108 | return self 109 | 110 | def __exit__(self, exception_type, exception_value, traceback): 111 | self.pixel_engine["in"].close() 112 | 113 | def get_metadata(self): 114 | if self.sdk_v1: 115 | return self.get_metadata_sdk_v1() 116 | else: 117 | return self.get_metadata_sdk_v2() 118 | 119 | def get_metadata_sdk_v1(self): 120 | pe_in = self.pixel_engine["in"] 121 | return { 122 | "Barcode": 123 | self.barcode(), 124 | "DICOM acquisition date": 125 | self.acquisition_datetime().isoformat(), 126 | "DICOM last calibration date": 127 | pe_in.DICOM_DATE_OF_LAST_CALIBRATION, 128 | "DICOM time of last calibration": 129 | pe_in.DICOM_TIME_OF_LAST_CALIBRATION, 130 | "DICOM manufacturer": 131 | pe_in.DICOM_MANUFACTURER, 132 | "DICOM manufacturer model name": 133 | pe_in.DICOM_MANUFACTURERS_MODEL_NAME, 134 | "DICOM device serial number": 135 | pe_in.DICOM_DEVICE_SERIAL_NUMBER, 136 | "Color space transform": 137 | pe_in.colorspaceTransform(), 138 | "Block size": 139 | pe_in.blockSize(), 140 | "Number of tiles": 141 | pe_in.numTiles(), 142 | "Bits stored": 143 | pe_in.bitsStored(), 144 | "Derivation description": 145 | self.derivation_description(), 146 | "DICOM software version": 147 | pe_in.DICOM_SOFTWARE_VERSIONS, 148 | "Number of images": self.num_images() 149 | } 150 | 151 | def get_metadata_sdk_v2(self): 152 | pe_in = self.pixel_engine["in"] 153 | return { 154 | "Pixel engine version": 155 | self.pixel_engine.version, 156 | "Barcode": 157 | self.barcode(), 158 | "Acquisition datetime": 159 | self.acquisition_datetime().isoformat(), 160 | "Date of last calibration": 161 | pe_in.date_of_last_calibration, 162 | "Time of last calibration": 163 | pe_in.time_of_last_calibration, 164 | "Manufacturer": 165 | pe_in.manufacturer, 166 | "Model name": 167 | pe_in.model_name, 168 | "Device serial number": 169 | pe_in.device_serial_number, 170 | "Derivation description": 171 | self.derivation_description(), 172 | "Software versions": 173 | pe_in.software_versions, 174 | "Number of images": 175 | self.num_images(), 176 | "Scanner calibration status": 177 | pe_in.scanner_calibration_status, 178 | "Scanner operator ID": 179 | pe_in.scanner_operator_id, 180 | "Scanner rack number": 181 | pe_in.scanner_rack_number, 182 | "Scanner rack priority": 183 | pe_in.scanner_rack_priority, 184 | "Scanner slot number": 185 | pe_in.scanner_slot_number, 186 | "iSyntax file version": 187 | pe_in.isyntax_file_version, 188 | # Could also add: 'is_UFS', 'is_UFSb', 'is_UVS', 'is_philips' 189 | } 190 | 191 | def get_image_metadata(self, image_no): 192 | if self.sdk_v1: 193 | return self.get_image_metadata_sdk_v1(image_no) 194 | else: 195 | return self.get_image_metadata_sdk_v2(image_no) 196 | 197 | def get_image_metadata_sdk_v1(self, image_no): 198 | pe_in = self.pixel_engine["in"] 199 | img = pe_in[image_no] 200 | image_type = self.image_type(image_no) 201 | image_metadata = { 202 | "Image type": 203 | image_type, 204 | "DICOM lossy image compression method": 205 | img.DICOM_LOSSY_IMAGE_COMPRESSION_METHOD, 206 | "DICOM lossy image compression ratio": 207 | img.DICOM_LOSSY_IMAGE_COMPRESSION_RATIO, 208 | "DICOM derivation description": 209 | img.DICOM_DERIVATION_DESCRIPTION, 210 | "Image dimension names": 211 | img.IMAGE_DIMENSION_NAMES, 212 | "Image dimension types": 213 | img.IMAGE_DIMENSION_TYPES, 214 | "Image dimension units": 215 | img.IMAGE_DIMENSION_UNITS, 216 | "Image dimension ranges": 217 | img.IMAGE_DIMENSION_RANGES, 218 | "Image dimension discrete values": 219 | img.IMAGE_DIMENSION_DISCRETE_VALUES_STRING, 220 | "Image scale factor": 221 | img.IMAGE_SCALE_FACTOR 222 | } 223 | if image_type == "WSI": 224 | self.pixel_size_x = img.IMAGE_SCALE_FACTOR[0] 225 | self.pixel_size_y = img.IMAGE_SCALE_FACTOR[1] 226 | 227 | view = pe_in.SourceView() 228 | image_metadata["Bits allocated"] = view.bitsAllocated() 229 | image_metadata["Bits stored"] = view.bitsStored() 230 | image_metadata["High bit"] = view.highBit() 231 | image_metadata["Pixel representation"] = \ 232 | view.pixelRepresentation() 233 | image_metadata["Planar configuration"] = \ 234 | view.planarConfiguration() 235 | image_metadata["Samples per pixel"] = \ 236 | view.samplesPerPixel() 237 | image_metadata["Number of levels"] = \ 238 | pe_in.numLevels() 239 | 240 | for resolution in range(pe_in.numLevels()): 241 | dim_ranges = view.dimensionRanges(resolution) 242 | level_size_x = self.get_size(dim_ranges[0]) 243 | level_size_y = self.get_size(dim_ranges[1]) 244 | image_metadata["Level sizes #%s" % resolution] = { 245 | "X": level_size_x, 246 | "Y": level_size_y 247 | } 248 | if resolution == 0: 249 | self.size_x = level_size_x 250 | self.size_y = level_size_y 251 | elif image_type == "LABELIMAGE": 252 | self.label_x = self.get_size(img.IMAGE_DIMENSION_RANGES[0]) + 1 253 | self.label_y = self.get_size(img.IMAGE_DIMENSION_RANGES[1]) + 1 254 | elif image_type == "MACROIMAGE": 255 | self.macro_x = self.get_size(img.IMAGE_DIMENSION_RANGES[0]) + 1 256 | self.macro_y = self.get_size(img.IMAGE_DIMENSION_RANGES[1]) + 1 257 | return image_metadata 258 | 259 | def get_image_metadata_sdk_v2(self, image_no): 260 | pe_in = self.pixel_engine["in"] 261 | img = pe_in[image_no] 262 | image_type = self.image_type(image_no) 263 | view = img.source_view 264 | image_scale_factor = view.scale 265 | image_metadata = { 266 | "Image type": 267 | image_type, 268 | "Lossy image compression method": 269 | img.lossy_image_compression_method, 270 | "Lossy image compression ratio": 271 | img.lossy_image_compression_ratio, 272 | "Image dimension names": 273 | view.dimension_names, 274 | "Image dimension types": 275 | view.dimension_types, 276 | "Image dimension units": 277 | view.dimension_units, 278 | "Image dimension discrete values": 279 | view.dimension_discrete_values, 280 | "Image scale factor": 281 | image_scale_factor, 282 | "Block size": 283 | img.block_size(), 284 | } 285 | if image_type == "WSI": 286 | image_metadata["Color space transform"] = \ 287 | img.colorspace_transform 288 | image_metadata["Number of tiles"] = img.num_tiles 289 | 290 | self.pixel_size_x = image_scale_factor[0] 291 | self.pixel_size_y = image_scale_factor[1] 292 | 293 | image_metadata["Bits allocated"] = view.bits_allocated 294 | image_metadata["Bits stored"] = view.bits_stored 295 | image_metadata["High bit"] = view.high_bit 296 | image_metadata["Pixel representation"] = \ 297 | view.pixel_representation 298 | image_metadata["Planar configuration"] = \ 299 | view.planar_configuration 300 | image_metadata["Samples per pixel"] = \ 301 | view.samples_per_pixel 302 | image_metadata["Number of derived levels"] = \ 303 | self.num_derived_levels(img) 304 | 305 | for resolution in range(self.num_derived_levels(img)): 306 | dim_ranges = self.dimension_ranges(img, resolution) 307 | level_size_x = self.get_size(dim_ranges[0]) 308 | level_size_y = self.get_size(dim_ranges[1]) 309 | image_metadata["Level sizes #%s" % resolution] = { 310 | "X": level_size_x, 311 | "Y": level_size_y 312 | } 313 | if resolution == 0: 314 | self.size_x = level_size_x 315 | self.size_y = level_size_y 316 | elif image_type == "LABELIMAGE": 317 | self.label_x = self.get_size(view.dimension_ranges(0)[0]) + 1 318 | self.label_y = self.get_size(view.dimension_ranges(0)[1]) + 1 319 | elif image_type == "MACROIMAGE": 320 | self.macro_x = self.get_size(view.dimension_ranges(0)[0]) + 1 321 | self.macro_y = self.get_size(view.dimension_ranges(0)[1]) + 1 322 | return image_metadata 323 | 324 | def acquisition_datetime(self): 325 | pe_in = self.pixel_engine["in"] 326 | if self.sdk_v1: 327 | timestamp = str(pe_in.DICOM_ACQUISITION_DATETIME).strip() 328 | else: 329 | timestamp = pe_in.acquisition_datetime.strip() 330 | # older files store the date time in YYYYmmddHHMMSS.ffffff format 331 | # newer files use ISO 8601, i.e. YYYY-mm-ddTHH:mm:ss 332 | # other timestamp formats may be used in the future 333 | try: 334 | # Handle "special" isyntax date/time format 335 | return datetime.strptime(timestamp, "%Y%m%d%H%M%S.%f") 336 | except ValueError: 337 | # Handle other date/time formats (such as ISO 8601) 338 | return parse(timestamp) 339 | 340 | def barcode(self): 341 | pe_in = self.pixel_engine["in"] 342 | if self.sdk_v1: 343 | return pe_in.BARCODE 344 | else: 345 | return pe_in.barcode 346 | 347 | def data_envelopes(self, image, resolution): 348 | pe_in = self.pixel_engine["in"] 349 | if self.sdk_v1: 350 | return pe_in.SourceView().dataEnvelopes(resolution) 351 | else: 352 | return image.source_view.data_envelopes(resolution) 353 | 354 | def derivation_description(self): 355 | pe_in = self.pixel_engine["in"] 356 | if self.sdk_v1: 357 | return pe_in.DICOM_DERIVATION_DESCRIPTION 358 | else: 359 | return pe_in.derivation_description 360 | 361 | def dimension_ranges(self, image, resolution): 362 | pe_in = self.pixel_engine["in"] 363 | if self.sdk_v1: 364 | return pe_in.SourceView().dimensionRanges(resolution) 365 | else: 366 | return image.source_view.dimension_ranges(resolution) 367 | 368 | def image_data(self, image): 369 | if self.sdk_v1: 370 | return image.IMAGE_DATA 371 | else: 372 | return image.image_data 373 | 374 | def image_type(self, image_no): 375 | pe_in = self.pixel_engine["in"] 376 | if self.sdk_v1: 377 | return pe_in[image_no].IMAGE_TYPE 378 | else: 379 | return pe_in[image_no].image_type 380 | 381 | def num_derived_levels(self, image): 382 | pe_in = self.pixel_engine["in"] 383 | if self.sdk_v1: 384 | return pe_in.numLevels() 385 | else: 386 | return image.source_view.num_derived_levels 387 | 388 | def num_images(self): 389 | pe_in = self.pixel_engine["in"] 390 | if self.sdk_v1: 391 | return pe_in.numImages() 392 | else: 393 | return pe_in.num_images 394 | 395 | def wait_any(self, regions): 396 | if self.sdk_v1: 397 | return self.pixel_engine.waitAny(regions) 398 | else: 399 | return self.pixel_engine.wait_any(regions) 400 | 401 | def write_image_metadata(self, resolutions, series): 402 | # OK to hard-code axes; this matches DimensionOrder in ome_template.xml 403 | axes = { 404 | 't': 'time', 405 | 'c': 'channel', 406 | 'z': 'space', 407 | 'y': 'space', 408 | 'x': 'space' 409 | } 410 | multiscale_axes = [{'name': x, 'type': axes[x]} for x in axes] 411 | if series == 0: 412 | for axis in multiscale_axes: 413 | if axis['name'] == 'x' or axis['name'] == 'y': 414 | axis['unit'] = 'micrometer' 415 | 416 | metadata = self.get_image_metadata(0) 417 | scale_level = [ 418 | self.size_x / metadata['Level sizes #%s' % v]['X'] 419 | for v in resolutions] 420 | 421 | pixel_size_x = self.pixel_size_x if series == 0 else 1.0 422 | pixel_size_y = self.pixel_size_y if series == 0 else 1.0 423 | 424 | multiscales = [{ 425 | 'metadata': { 426 | 'method': 'pixelengine', 427 | 'version': str(self.pixel_engine.version) 428 | }, 429 | 'axes': multiscale_axes, 430 | 'version': '0.4', 431 | 'datasets': [{ 432 | 'path': str(v), 433 | 'coordinateTransformations': [{ 434 | 'scale': [ 435 | 1.0, 1.0, 1.0, 436 | pixel_size_y * scale_level[v], 437 | pixel_size_x * scale_level[v]], 438 | 'type': 'scale' 439 | }] 440 | } for v in resolutions] 441 | }] 442 | z = self.zarr_group["%d" % series] 443 | z.attrs['multiscales'] = multiscales 444 | 445 | def write_metadata_json(self, metadata_file): 446 | '''write metadata to a JSON file''' 447 | 448 | with open(metadata_file, "w", encoding="utf-8") as f: 449 | metadata = self.get_metadata() 450 | 451 | for image in range(self.num_images()): 452 | image_metadata = self.get_image_metadata(image) 453 | metadata["Image #" + str(image)] = image_metadata 454 | 455 | json.dump(metadata, f) 456 | 457 | def write_metadata_xml(self, metadata_file): 458 | ome_timestamp = self.acquisition_datetime() 459 | 460 | xml_values = { 461 | 'image': { 462 | 'name': self.barcode(), 463 | 'acquisitionDate': ome_timestamp.isoformat(), 464 | 'description': self.derivation_description(), 465 | 'pixels': { 466 | 'sizeX': int(self.size_x), 467 | 'sizeY': int(self.size_y), 468 | 'physicalSizeX': self.pixel_size_x, 469 | 'physicalSizeY': self.pixel_size_y 470 | } 471 | }, 472 | 'label': { 473 | 'pixels': { 474 | 'sizeX': int(self.label_x), 475 | 'sizeY': int(self.label_y) 476 | } 477 | }, 478 | 'macro': { 479 | 'pixels': { 480 | 'sizeX': int(self.macro_x), 481 | 'sizeY': int(self.macro_y) 482 | } 483 | } 484 | } 485 | loader = PackageLoader() 486 | template = loader.import_("isyntax2raw.resources.ome_template") 487 | xml = template(xml_values).render() 488 | with open(metadata_file, "w", encoding="utf-8") as omexml: 489 | omexml.write(xml) 490 | 491 | def write_metadata(self): 492 | os.makedirs(os.path.join(self.slide_directory, "OME"), exist_ok=True) 493 | 494 | metadata_file = os.path.join( 495 | self.slide_directory, "OME", "METADATA.json" 496 | ) 497 | self.write_metadata_json(metadata_file) 498 | 499 | metadata_file = os.path.join( 500 | self.slide_directory, "OME", "METADATA.ome.xml" 501 | ) 502 | self.write_metadata_xml(metadata_file) 503 | 504 | def get_size(self, dim_range): 505 | '''calculate the length in pixels of a dimension''' 506 | v = (dim_range[2] - dim_range[0]) / dim_range[1] 507 | if not v.is_integer(): 508 | # isyntax infrastructure should ensure this always divides 509 | # evenly 510 | raise ValueError( 511 | '(%d - %d) / %d results in remainder!' % ( 512 | dim_range[2], dim_range[0], dim_range[1] 513 | ) 514 | ) 515 | return v 516 | 517 | def write_label_image(self): 518 | '''write the label image (if present) as a JPEG file''' 519 | self.write_image_type("LABELIMAGE", 1) 520 | 521 | def write_macro_image(self): 522 | '''write the macro image (if present) as a JPEG file''' 523 | self.write_image_type("MACROIMAGE", 2) 524 | 525 | def find_image_type(self, image_type): 526 | '''look up a given image type in the pixel engine''' 527 | pe_in = self.pixel_engine["in"] 528 | for index in range(self.num_images()): 529 | if image_type == self.image_type(index): 530 | return pe_in[index] 531 | return None 532 | 533 | def write_image_type(self, image_type, series): 534 | '''write an image of the specified type''' 535 | image = self.find_image_type(image_type) 536 | if image is not None: 537 | pixels = self.image_data(image) 538 | 539 | # pixels are JPEG compressed, need to decompress first 540 | img = Image.open(BytesIO(pixels)) 541 | width = img.width 542 | height = img.height 543 | 544 | self.create_tile_directory(series, 0, width, height) 545 | tile = self.zarr_group["%d/0" % series] 546 | tile.attrs['image type'] = image_type 547 | for channel in range(0, 3): 548 | band = np.array(img.getdata(band=channel)) 549 | band.shape = (height, width) 550 | tile[0, channel, 0] = band 551 | self.write_image_metadata(range(1), series) 552 | 553 | log.info("wrote %s image" % image_type) 554 | 555 | def create_tile_directory(self, series, resolution, width, height): 556 | dimension_separator = '/' 557 | if not self.nested: 558 | dimension_separator = '.' 559 | self.zarr_store = FSStore( 560 | self.slide_directory, 561 | dimension_separator=dimension_separator, 562 | normalize_keys=True, 563 | auto_mkdir=True 564 | ) 565 | self.zarr_group = zarr.group(store=self.zarr_store) 566 | self.zarr_group.attrs['bioformats2raw.layout'] = LAYOUT_VERSION 567 | 568 | # important to explicitly set the chunk size to 1 for non-XY dims 569 | # setting to None may cause all planes to be chunked together 570 | # ordering is TCZYX and hard-coded since Z and T are not present 571 | self.zarr_group.create_dataset( 572 | "%s/%s" % (str(series), str(resolution)), 573 | shape=(1, 3, 1, height, width), 574 | chunks=(1, 1, 1, self.tile_height, self.tile_width), dtype='B' 575 | ) 576 | 577 | def make_planar(self, pixels, tile_width, tile_height): 578 | r = pixels[0::3] 579 | g = pixels[1::3] 580 | b = pixels[2::3] 581 | for v in (r, g, b): 582 | v.shape = (tile_height, tile_width) 583 | return np.array([r, g, b]) 584 | 585 | def write_pyramid(self): 586 | '''write the slide's pyramid as a set of tiles''' 587 | pe_in = self.pixel_engine["in"] 588 | image = self.find_image_type("WSI") 589 | 590 | scanned_areas = self.data_envelopes(image, 0) 591 | if scanned_areas is None: 592 | raise RuntimeError("No valid data envelopes") 593 | 594 | if self.resolutions is None: 595 | resolutions = range(self.num_derived_levels(image)) 596 | else: 597 | resolutions = range(self.resolutions) 598 | 599 | def write_tile( 600 | pixels, resolution, x_start, y_start, tile_width, tile_height, 601 | ): 602 | x_end = x_start + tile_width 603 | y_end = y_start + tile_height 604 | try: 605 | # Zarr has a single n-dimensional array representation on 606 | # disk (not interleaved RGB) 607 | pixels = self.make_planar(pixels, tile_width, tile_height) 608 | z = self.zarr_group["0/%d" % resolution] 609 | z[0, :, 0, y_start:y_end, x_start:x_end] = pixels 610 | except Exception: 611 | log.error( 612 | "Failed to write tile [:, %d:%d, %d:%d]" % ( 613 | x_start, x_end, y_start, y_end 614 | ), exc_info=True 615 | ) 616 | 617 | for resolution in resolutions: 618 | # assemble data envelopes (== scanned areas) to extract for 619 | # this level 620 | dim_ranges = self.dimension_ranges(image, resolution) 621 | log.info("dimension ranges = %s" % dim_ranges) 622 | resolution_x_size = self.get_size(dim_ranges[0]) 623 | resolution_y_size = self.get_size(dim_ranges[1]) 624 | scale_x = dim_ranges[0][1] 625 | scale_y = dim_ranges[1][1] 626 | 627 | x_tiles = math.ceil(resolution_x_size / self.tile_width) 628 | y_tiles = math.ceil(resolution_y_size / self.tile_height) 629 | 630 | log.info("# of X (%d) tiles = %d" % (self.tile_width, x_tiles)) 631 | log.info("# of Y (%d) tiles = %d" % (self.tile_height, y_tiles)) 632 | 633 | # create one tile directory per resolution level if required 634 | tile_directory = self.create_tile_directory( 635 | 0, resolution, resolution_x_size, resolution_y_size 636 | ) 637 | 638 | patches, patch_ids = self.create_patch_list( 639 | dim_ranges, [x_tiles, y_tiles], 640 | [self.tile_width, self.tile_height], 641 | tile_directory 642 | ) 643 | envelopes = self.data_envelopes(image, resolution) 644 | jobs = [] 645 | with MaxQueuePool(ThreadPoolExecutor, self.max_workers) as pool: 646 | for i in range(0, len(patches), self.batch_size): 647 | # requestRegions( 648 | # self: pixelengine.PixelEngine.View, 649 | # region: List[List[int]], 650 | # dataEnvelopes: pixelengine.PixelEngine.DataEnvelopes, 651 | # enableAsyncRendering: bool=True, 652 | # backgroundColor: List[int]=[0, 0, 0], 653 | # bufferType: 654 | # pixelengine.PixelEngine.BufferType=BufferType.RGB 655 | # ) -> list 656 | if self.sdk_v1: 657 | request_regions = pe_in.SourceView().requestRegions 658 | else: 659 | request_regions = image.source_view.request_regions 660 | regions = request_regions( 661 | patches[i:i + self.batch_size], envelopes, True, 662 | [self.fill_color] * 3 663 | ) 664 | while regions: 665 | regions_ready = self.wait_any(regions) 666 | 667 | for region_index, region in enumerate(regions_ready): 668 | view_range = region.range 669 | log.debug( 670 | "processing tile %s (%s regions ready; " 671 | "%s regions left; %s jobs)" % ( 672 | view_range, len(regions_ready), 673 | len(regions), len(jobs) 674 | ) 675 | ) 676 | x_start, x_end, y_start, y_end, level = view_range 677 | width = 1 + (x_end - x_start) / scale_x 678 | # isyntax infrastructure should ensure this always 679 | # divides evenly 680 | if not width.is_integer(): 681 | raise ValueError( 682 | '(1 + (%d - %d) / %d results in ' 683 | 'remainder!' % ( 684 | x_end, x_start, scale_x 685 | ) 686 | ) 687 | width = int(width) 688 | height = 1 + (y_end - y_start) / scale_y 689 | # isyntax infrastructure should ensure this always 690 | # divides evenly 691 | if not height.is_integer(): 692 | raise ValueError( 693 | '(1 + (%d - %d) / %d results in ' 694 | 'remainder!' % ( 695 | y_end, y_start, scale_y 696 | ) 697 | ) 698 | height = int(height) 699 | pixel_buffer_size = width * height * 3 700 | pixels = np.empty(pixel_buffer_size, dtype='B') 701 | patch_id = patch_ids.pop(regions.index(region)) 702 | x_start, y_start = patch_id 703 | x_start *= self.tile_width 704 | y_start *= self.tile_height 705 | 706 | region.get(pixels) 707 | regions.remove(region) 708 | 709 | jobs.append(pool.submit( 710 | write_tile, pixels, resolution, 711 | x_start, y_start, width, height 712 | )) 713 | wait(jobs, return_when=ALL_COMPLETED) 714 | self.write_image_metadata(resolutions, 0) 715 | 716 | def create_patch_list( 717 | self, dim_ranges, tiles, tile_size, tile_directory 718 | ): 719 | resolution_x_end = dim_ranges[0][2] 720 | resolution_y_end = dim_ranges[1][2] 721 | origin_x = dim_ranges[0][0] 722 | origin_y = dim_ranges[1][0] 723 | tiles_x, tiles_y = tiles 724 | 725 | patches = [] 726 | patch_ids = [] 727 | scale_x = dim_ranges[0][1] 728 | scale_y = dim_ranges[1][1] 729 | # We'll use the X scale to calculate our level. If the X and Y scales 730 | # are not eqivalent or not a power of two this will not work but that 731 | # seems *highly* unlikely 732 | level = math.log2(scale_x) 733 | if scale_x != scale_y or not level.is_integer(): 734 | raise ValueError( 735 | "scale_x=%d scale_y=%d do not match isyntax format " 736 | "assumptions!" % ( 737 | scale_x, scale_y 738 | ) 739 | ) 740 | level = int(level) 741 | tile_size_x = tile_size[0] * scale_x 742 | tile_size_y = tile_size[1] * scale_y 743 | for y in range(tiles_y): 744 | y_start = origin_y + (y * tile_size_y) 745 | # Subtracting "scale_y" here makes no sense but it works and 746 | # reflects the isyntax SDK examples 747 | y_end = min( 748 | (y_start + tile_size_y) - scale_y, resolution_y_end - scale_y 749 | ) 750 | for x in range(tiles_x): 751 | x_start = origin_x + (x * tile_size_x) 752 | # Subtracting "scale_x" here makes no sense but it works and 753 | # reflects the isyntax SDK examples 754 | x_end = min( 755 | (x_start + tile_size_x) - scale_x, 756 | resolution_x_end - scale_x 757 | ) 758 | patch = [x_start, x_end, y_start, y_end, level] 759 | patches.append(patch) 760 | # Associating spatial information (tile X and Y offset) in 761 | # order to identify the patches returned asynchronously 762 | patch_ids.append((x, y)) 763 | return patches, patch_ids 764 | -------------------------------------------------------------------------------- /isyntax2raw/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # 3 | # Copyright (c) 2017 Glencoe Software, Inc. All rights reserved. 4 | # 5 | # This software is distributed under the terms described by the LICENCE file 6 | # you can find at the root of the distribution bundle. 7 | # If the file is missing please request a copy by contacting 8 | # support@glencoesoftware.com. 9 | -------------------------------------------------------------------------------- /isyntax2raw/cli/isyntax2raw.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # 3 | # Copyright (c) 2019 Glencoe Software, Inc. All rights reserved. 4 | # 5 | # This software is distributed under the terms described by the LICENCE file 6 | # you can find at the root of the distribution bundle. 7 | # If the file is missing please request a copy by contacting 8 | # support@glencoesoftware.com. 9 | 10 | import click 11 | import logging 12 | 13 | from .. import WriteTiles 14 | 15 | 16 | def setup_logging(debug): 17 | level = logging.INFO 18 | if debug: 19 | level = logging.DEBUG 20 | logging.basicConfig( 21 | level=level, 22 | format="%(asctime)s %(levelname)-7s [%(name)16s] " 23 | "(%(thread)10s) %(message)s" 24 | ) 25 | 26 | 27 | @click.group() 28 | def cli(): 29 | pass 30 | 31 | 32 | @cli.command(name='write_tiles') 33 | @click.option( 34 | "--tile_width", default=512, type=int, show_default=True, 35 | help="tile width in pixels" 36 | ) 37 | @click.option( 38 | "--tile_height", default=512, type=int, show_default=True, 39 | help="tile height in pixels" 40 | ) 41 | @click.option( 42 | "--resolutions", type=int, 43 | help="number of pyramid resolutions to generate [default: all]" 44 | ) 45 | @click.option( 46 | "--max_workers", default=4, type=int, 47 | show_default=True, 48 | help="maximum number of tile workers that will run at one time", 49 | ) 50 | @click.option( 51 | "--batch_size", default=250, type=int, show_default=True, 52 | help="number of patches fed into the iSyntax SDK at one time" 53 | ) 54 | @click.option( 55 | "--fill_color", type=click.IntRange(min=0, max=255), default=0, 56 | show_default=True, 57 | help="background color for missing tiles (0-255)" 58 | ) 59 | @click.option( 60 | "--nested/--no-nested", default=True, show_default=True, 61 | help="Whether to use '/' as the chunk path separator" 62 | ) 63 | @click.option( 64 | "--debug", is_flag=True, 65 | help="enable debugging", 66 | ) 67 | @click.argument("input_path") 68 | @click.argument("output_path") 69 | def write_tiles( 70 | tile_width, tile_height, resolutions, max_workers, batch_size, 71 | fill_color, nested, debug, input_path, output_path 72 | ): 73 | setup_logging(debug) 74 | with WriteTiles( 75 | tile_width, tile_height, resolutions, max_workers, 76 | batch_size, fill_color, nested, input_path, output_path 77 | ) as wt: 78 | wt.write_metadata() 79 | wt.write_label_image() 80 | wt.write_macro_image() 81 | wt.write_pyramid() 82 | 83 | 84 | @cli.command(name='write_metadata') 85 | @click.option( 86 | "--debug", is_flag=True, 87 | help="enable debugging", 88 | ) 89 | @click.argument('input_path') 90 | @click.argument('output_file') 91 | def write_metadata(debug, input_path, output_file): 92 | setup_logging(debug) 93 | with WriteTiles( 94 | None, None, None, None, 95 | None, None, None, input_path, None 96 | ) as wt: 97 | wt.write_metadata_json(output_file) 98 | 99 | 100 | def main(): 101 | cli() 102 | -------------------------------------------------------------------------------- /isyntax2raw/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/glencoesoftware/isyntax2raw/7c11bdb9e90e88d79fd294e55f6554752bb91097/isyntax2raw/resources/__init__.py -------------------------------------------------------------------------------- /isyntax2raw/resources/ome_template.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | ${image['acquisitionDate']} 7 | ${image['description']} 8 | 9 | 10 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | ${image['acquisitionDate']} 28 | 29 | 30 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | ${image['acquisitionDate']} 42 | 43 | 44 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | # 3 | # Copyright (c) 2019 Glencoe Software, Inc. All rights reserved. 4 | # 5 | # This software is distributed under the terms described by the LICENCE file 6 | # you can find at the root of the distribution bundle. 7 | # If the file is missing please request a copy by contacting 8 | # support@glencoesoftware.com. 9 | 10 | import os 11 | import sys 12 | 13 | from setuptools import setup, find_packages 14 | from setuptools.command.test import test as TestCommand 15 | 16 | import version 17 | 18 | # Hack to prevent stupid "TypeError: 'NoneType' object is not callable" error 19 | # in multiprocessing/util.py _exit_function when running `python 20 | # setup.py test` or `python setup.py flake8`. See: 21 | # * http://www.eby-sarna.com/pipermail/peak/2010-May/003357.html) 22 | # * https://github.com/getsentry/raven-python/blob/master/setup.py 23 | import multiprocessing 24 | assert multiprocessing # silence flake8 25 | 26 | 27 | def get_requirements(suffix=''): 28 | with open('requirements%s.txt' % suffix) as f: 29 | rv = f.read().splitlines() 30 | return rv 31 | 32 | 33 | class PyTest(TestCommand): 34 | 35 | user_options = [('pytest-args=', 'a', 'Arguments to pass to py.test')] 36 | 37 | def initialize_options(self): 38 | TestCommand.initialize_options(self) 39 | self.pytest_args = [] 40 | 41 | def finalize_options(self): 42 | TestCommand.finalize_options(self) 43 | self.test_args = [] 44 | self.test_suite = True 45 | 46 | def run_tests(self): 47 | import pytest 48 | if isinstance(self.pytest_args, str): 49 | # pytest requires arguments as a list or tuple even if singular 50 | self.pytest_args = [self.pytest_args] 51 | errno = pytest.main(self.pytest_args) 52 | sys.exit(errno) 53 | 54 | 55 | def read(fname): 56 | """ 57 | Utility function to read the README file. 58 | :rtype : String 59 | """ 60 | return open(os.path.join(os.path.dirname(__file__), fname)).read() 61 | 62 | 63 | setup(name='isyntax2raw', 64 | version=version.getVersion(), 65 | python_requires='>=3.6', 66 | description='iSyntax to raw format converter', 67 | long_description=read('README.md'), 68 | long_description_content_type='text/markdown', 69 | classifiers=[], # Get strings from 70 | # http://pypi.python.org/pypi?%3Aaction=list_classifiers 71 | keywords='', 72 | author='Glencoe Software, Inc.', 73 | author_email='info@glencoesoftware.com', 74 | url='https://github.com/glencoesoftware/isyntax2raw', 75 | license='License :: OSI Approved :: BSD License', 76 | packages=find_packages(), 77 | package_data={'isyntax2raw': ['resources/*.xml']}, 78 | zip_safe=True, 79 | include_package_data=True, 80 | platforms='any', 81 | setup_requires=['flake8'], 82 | install_requires=[ 83 | 'click==7.0', 84 | 'pillow>=7.1.0', 85 | 'numpy==1.17.3', 86 | 'zarr==2.8.1', 87 | 'kajiki==0.8.2', 88 | 'fsspec>=0.9.0', 89 | 'python-dateutil>=2.8.2' 90 | ], 91 | tests_require=[ 92 | 'flake8', 93 | 'pytest', 94 | ], 95 | cmdclass={'test': PyTest}, 96 | entry_points={ 97 | 'console_scripts': [ 98 | 'isyntax2raw = isyntax2raw.cli.isyntax2raw:main', 99 | ] 100 | } 101 | ) 102 | -------------------------------------------------------------------------------- /version.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """Calculates the current version number. 4 | 5 | If possible, uses output of “git describe” modified to conform to the 6 | visioning scheme that setuptools uses (see PEP 386). Releases must be 7 | labelled with annotated tags (signed tags are annotated) of the following 8 | format: 9 | 10 | v(.)+ [ {a|b|c|rc} (.)* ] 11 | 12 | If “git describe” returns an error (likely because we're in an unpacked copy 13 | of a release tarball, rather than a git working copy), or returns a tag that 14 | does not match the above format, version is read from RELEASE-VERSION file. 15 | 16 | To use this script, simply import it your setup.py file, and use the results 17 | of getVersion() as your package version: 18 | 19 | import version 20 | setup( 21 | version=version.getVersion(), 22 | . 23 | . 24 | . 25 | ) 26 | 27 | This will automatically update the RELEASE-VERSION file. The RELEASE-VERSION 28 | file should *not* be checked into git but it *should* be included in sdist 29 | tarballs (as should version.py file). To do this, run: 30 | 31 | echo include RELEASE-VERSION version.py >>MANIFEST.in 32 | echo RELEASE-VERSION >>.gitignore 33 | 34 | With that setup, a new release can be labelled by simply invoking: 35 | 36 | git tag -s v1.0 37 | """ 38 | 39 | __author__ = ('Douglas Creager ', 40 | 'Michal Nazarewicz ') 41 | __license__ = 'This file is placed into the public domain.' 42 | __maintainer__ = 'Michal Nazarewicz' 43 | __email__ = 'mina86@mina86.com' 44 | 45 | __all__ = ('getVersion') 46 | 47 | 48 | import re 49 | import subprocess 50 | import sys 51 | 52 | 53 | RELEASE_VERSION_FILE = 'isyntax2raw/version.py' 54 | 55 | # http://www.python.org/dev/peps/pep-0386/ 56 | _PEP386_SHORT_VERSION_RE = r'\d+(?:\.\d+)+(?:(?:[abc]|rc)\d+(?:\.\d+)*)?' 57 | _PEP386_VERSION_RE = r'^%s(?:\.post\d+)?(?:\.dev\d+)?$' % ( 58 | _PEP386_SHORT_VERSION_RE) 59 | _GIT_DESCRIPTION_RE = r'^v(?P%s)-(?P\d+)-g(?P[\da-f]+)$' % ( 60 | _PEP386_SHORT_VERSION_RE) 61 | 62 | 63 | def readGitVersion(): 64 | try: 65 | proc = subprocess.Popen(('git', 'describe', '--long', 66 | '--match', 'v[0-9]*.*'), 67 | stdout=subprocess.PIPE, stderr=subprocess.PIPE) 68 | data, _ = proc.communicate() 69 | if proc.returncode: 70 | return None 71 | ver = data.decode('utf-8').splitlines()[0].strip() 72 | except Exception: 73 | return None 74 | 75 | if not ver: 76 | return None 77 | m = re.search(_GIT_DESCRIPTION_RE, ver) 78 | if not m: 79 | sys.stderr.write('version: git description (%s) is invalid, ' 80 | 'ignoring\n' % ver) 81 | return None 82 | 83 | commits = int(m.group('commits')) 84 | if not commits: 85 | return m.group('ver') 86 | else: 87 | return '%s.post%d.dev%d' % ( 88 | m.group('ver'), commits, int(m.group('sha'), 16)) 89 | 90 | 91 | def readReleaseVersion(): 92 | try: 93 | fd = open(RELEASE_VERSION_FILE) 94 | try: 95 | ver = fd.readline().rsplit('= ', 1)[-1].strip().strip('\'') 96 | finally: 97 | fd.close() 98 | if not re.search(_PEP386_VERSION_RE, ver): 99 | sys.stderr.write('version: release version (%s) is invalid, ' 100 | 'will use it anyway\n' % ver) 101 | return ver 102 | except Exception: 103 | return None 104 | 105 | 106 | def writeReleaseVersion(version): 107 | fd = open(RELEASE_VERSION_FILE, 'w') 108 | fd.write("VERSION = '%s'\n" % version) 109 | fd.close() 110 | 111 | 112 | def getVersion(): 113 | release_version = readReleaseVersion() 114 | version = readGitVersion() or release_version 115 | if not version: 116 | version = '0.0.0' 117 | elif version != release_version: 118 | writeReleaseVersion(version) 119 | return version 120 | 121 | 122 | if __name__ == '__main__': 123 | print(getVersion()) 124 | --------------------------------------------------------------------------------