├── .gitignore ├── .idea ├── misc.xml ├── modules.xml ├── pyclearvolume.iml └── workspace.xml ├── README.md ├── build └── windows │ ├── test.spec │ └── watchfolder.spec ├── examples └── serve_from_numpy.py ├── images ├── logo.png └── logo64.png ├── pyclearvolume ├── __init__.py ├── __main__.py ├── _dataserver.py ├── _serialize.py ├── folder_watch_thread.py ├── image_file_reader.py ├── lib │ ├── __init__.py │ ├── czifile.py │ └── tifffile.py └── pyclearvolume_serve.py ├── setup.py └── tests ├── test_client.py ├── test_folder_watch.py └── test_server.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | pyclearvolume.egg-info 3 | tests/data -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/pyclearvolume.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 10 | 11 | 13 | 14 | 15 | 16 | 17 | 19 | 20 | 21 | 1485184252639 22 | 26 | 27 | 28 | 29 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![GitHub Logo](/images/logo64.png) 2 | # pyclearvolume 3 | 4 | Python bindings to the [ClearVolume project](https://bitbucket.org/clearvolume/clearvolume). 5 | 6 | With it you can either serve numpy data directly from within python to any running ClearVolume client or send/watch a folder via *pyclearvolume_serve* and serve its content to the client (making it useful e.g. for remote scopes) 7 | 8 | ## Prerequisites 9 | 10 | A working installation of [ClearVolume](https://bitbucket.org/clearvolume/clearvolume). 11 | 12 | 13 | ## Installing 14 | 15 | 16 | Either via pip 17 | 18 | > pip install git+https://github.com/ClearVolume/pyclearvolume 19 | 20 | or classically 21 | 22 | > git clone https://github.com/ClearVolume/pyclearvolume 23 | 24 | > cd pyclearvolume 25 | 26 | > python setup.py install 27 | 28 | 29 | to test if everything worked just run from the command line 30 | 31 | 32 | > python -m pyclearvolume 33 | 34 | which will serve some dummy data to the default port (9140 on localhost) ClearVolume is listening to. After that you can open the ClearVolume client and should see some colorful volumes popping up. 35 | 36 | 37 | ## Usage 38 | 39 | ### standalone utility 40 | 41 | Pyclearvolume comes with an executable script *pyclearvolume_serve* that provides a fast way to serve single files (tif or czi) or folder to the client renderer. It further allows to watch a folder for changes and stream any newly arriving data. 42 | See the options for all available parameters: 43 | 44 | ``` 45 | pyclearvolume_serve [-h] [-a ADDRESS] [-p PORT] [-w] [-t DTIME] 46 | [-u UNITS UNITS UNITS] [-c COLOR COLOR COLOR] 47 | files [files ...] 48 | 49 | serves 3d image files or folder to a clearvolume client 50 | 51 | pyclearvolume_serve myfile.tif 52 | 53 | pyclearvolume_serve -u 1. 1. 2. -a remote.domain.com -p 9140 myfile.tif 54 | 55 | 56 | 57 | positional arguments: 58 | files image files or folder to send/watch (currently 59 | supported: ['.tif', '.czi', '.tiff'] 60 | 61 | optional arguments: 62 | -h, --help show this help message and exit 63 | -a ADDRESS, --address ADDRESS 64 | address to bind to (default: ) 65 | -p PORT, --port PORT port to bind to (default: 9140) 66 | -w, --watch watch folder (default: False) 67 | -t DTIME, --time DTIME 68 | time in secs in watch mode to wait for file not having 69 | changed (default: 1.0) 70 | -u UNITS UNITS UNITS, --units UNITS UNITS UNITS 71 | relative units of voxels e.g. -u 1. 1. 2. (default: 72 | [1.0, 1.0, 1.0]) 73 | -c COLOR COLOR COLOR, --color COLOR COLOR COLOR 74 | color rgb in 0..1 (default: [1.0, 1.0, 1.0]) 75 | ``` 76 | 77 | ### from within python 78 | 79 | To create a instance of the data server just do 80 | ```python 81 | d = pyclearvolume.DataServer( 82 | address = "localhost", 83 | port = 9140, 84 | maxVolumeNumber = 20, 85 | dropVolumeOnFull = True) 86 | ``` 87 | 88 | 89 | then start the server 90 | 91 | ```python 92 | d.start() 93 | ``` 94 | 95 | and send some data 96 | 97 | ```python 98 | d.sendData(data, time = 0, channel = 1, color ="1.0 0.4 0.2 1.0") 99 | ``` 100 | 101 | 102 | ###Example 103 | 104 | ```python 105 | import numpy as np 106 | import time 107 | 108 | import pyclearvolume 109 | 110 | print "creating the server" 111 | 112 | d = pyclearvolume.DataServer(maxVolumeNumber=20) 113 | 114 | print "starting the server" 115 | 116 | d.start() 117 | 118 | print "starting to serve data" 119 | 120 | N = 128 121 | data = np.linspace(0,65000,N**3).reshape((N,)*3).astype(np.uint16) 122 | 123 | t = 0 124 | while True: 125 | args = {} 126 | args["color"] = "%s %s %s 1."%tuple([str(c) for c in np.random.uniform(0,1,3)]) 127 | args["voxelwidth"] = np.random.uniform(.2,1.6) 128 | args["voxelheight"] = np.random.uniform(.2,1.6) 129 | args["voxeldepth"] = np.random.uniform(.2,1.6) 130 | args["index"] = t 131 | 132 | print "sending..." 133 | 134 | d.sendData(data,**args) 135 | time.sleep(2) 136 | t += 1 137 | ``` 138 | 139 | ## Acknowledgements 140 | For opening tif and czi files pyclearvolume uses the excellent modules "tifffile" and "czifile" from [Christoph Gohlke](http://www.lfd.uci.edu/~gohlke/). 141 | -------------------------------------------------------------------------------- /build/windows/test.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | a = Analysis(['../../pyclearvolume/test.py'], 3 | pathex=['c:\\Users\\myerslab\\python\\pyclearvolume\\build\\windows'], 4 | hiddenimports=[], 5 | hookspath=None, 6 | runtime_hooks=None) 7 | pyz = PYZ(a.pure) 8 | exe = EXE(pyz, 9 | a.scripts, 10 | exclude_binaries=True, 11 | name='test.exe', 12 | debug=False, 13 | strip=None, 14 | upx=True, 15 | console=True) 16 | coll = COLLECT(exe, 17 | a.binaries, 18 | a.zipfiles, 19 | a.datas, 20 | strip=None, 21 | upx=True, 22 | name='test') 23 | -------------------------------------------------------------------------------- /build/windows/watchfolder.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | a = Analysis(['../../pyclearvolume/watchfolder.py'], 3 | pathex=['c:\\Users\\myerslab\\python\\pyclearvolume\\build\\windows'], 4 | hiddenimports=[], 5 | hookspath=None, 6 | runtime_hooks=None) 7 | pyz = PYZ(a.pure) 8 | exe = EXE(pyz, 9 | a.scripts, 10 | exclude_binaries=True, 11 | name='watchfolder.exe', 12 | debug=False, 13 | strip=None, 14 | upx=True, 15 | console=True ) 16 | coll = COLLECT(exe, 17 | a.binaries, 18 | a.zipfiles, 19 | a.datas, 20 | strip=None, 21 | upx=True, 22 | name='watchfolder') 23 | -------------------------------------------------------------------------------- /examples/serve_from_numpy.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division 2 | from __future__ import absolute_import 3 | 4 | import numpy as np 5 | import time 6 | 7 | import pyclearvolume 8 | 9 | print("creating the server") 10 | 11 | d = pyclearvolume.DataServer(port=9140, maxVolumeNumber=20) 12 | 13 | print("starting the server") 14 | 15 | d.start() 16 | 17 | print("starting to serve data") 18 | 19 | N = 128 20 | data = np.linspace(0, 65000, N ** 3).reshape((N,) * 3).astype(np.uint16) 21 | 22 | 23 | t = 0 24 | while True: 25 | args = {} 26 | args["color"] = "%s %s %s 1." % tuple([str(c) for c in np.random.uniform(0, 1, 3)]) 27 | args["voxelwidth"] = np.random.uniform(.2, 1.6) 28 | args["voxelheight"] = np.random.uniform(.2, 1.6) 29 | args["voxeldepth"] = np.random.uniform(.2, 1.6) 30 | args["index"] = t 31 | 32 | print("sending...") 33 | 34 | d.sendData(data, **args) 35 | time.sleep(2) 36 | t += 1 37 | -------------------------------------------------------------------------------- /images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ClearVolume/PyClearVolume/2979da3747e6c0b8b6e8fc9cbaba7963f37046b5/images/logo.png -------------------------------------------------------------------------------- /images/logo64.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ClearVolume/PyClearVolume/2979da3747e6c0b8b6e8fc9cbaba7963f37046b5/images/logo64.png -------------------------------------------------------------------------------- /pyclearvolume/__init__.py: -------------------------------------------------------------------------------- 1 | from ._dataserver import * 2 | from .image_file_reader import ImageFileReader 3 | -------------------------------------------------------------------------------- /pyclearvolume/__main__.py: -------------------------------------------------------------------------------- 1 | 2 | from __future__ import absolute_import 3 | from __future__ import print_function 4 | import numpy as np 5 | import time 6 | import pyclearvolume 7 | 8 | print("creating the server") 9 | 10 | d = pyclearvolume.DataServer(maxVolumeNumber=2) 11 | 12 | d.start() 13 | 14 | time.sleep(1) 15 | 16 | print("starting to serve data") 17 | 18 | N = 128 19 | data = np.linspace(0,255,N**3).reshape((N,)*3).astype(np.uint16) 20 | 21 | t = 0 22 | while True: 23 | if d.is_connected(): 24 | print("connected to %s %s"%(d.client_address())) 25 | args = {} 26 | args["color"] = "%s %s %s 1."%tuple([str(c) for c in np.random.uniform(0,1,3)]) 27 | args["voxelwidth"] = np.random.uniform(.2,1.6) 28 | args["voxelheight"] = np.random.uniform(.2,1.6) 29 | args["voxeldepth"] = np.random.uniform(.2,1.6) 30 | args["index"] = t 31 | print("sending...") 32 | print(args) 33 | d.sendData(data,**args) 34 | t += 1 35 | 36 | # d.sendData(data) 37 | time.sleep(2) 38 | -------------------------------------------------------------------------------- /pyclearvolume/_dataserver.py: -------------------------------------------------------------------------------- 1 | """ 2 | the data server providing the main functionality 3 | 4 | author: Martin Weigert 5 | email: mweigert@mpi-cbg.de 6 | """ 7 | 8 | from __future__ import absolute_import 9 | from __future__ import print_function 10 | import socket 11 | import six.moves.queue 12 | import threading 13 | import time 14 | 15 | 16 | from pyclearvolume._serialize import _serialize_data 17 | 18 | 19 | ###### logging stuff 20 | import logging 21 | logging.basicConfig() 22 | logger = logging.getLogger(__name__) 23 | logger.setLevel(logging.DEBUG) 24 | logger.setLevel(logging.INFO) 25 | 26 | ###### 27 | 28 | __all__ = ["DataServer"] 29 | 30 | 31 | 32 | class DataServer: 33 | """ 34 | The main data serving object. 35 | 36 | Basic usage: 37 | 38 | 39 | d = DataServer() 40 | 41 | d.start() 42 | 43 | data = linspace(0,100,100**3).reshape((100,100,100)) 44 | 45 | d.sendData(data.astype(uint16)) 46 | 47 | d.stop() 48 | 49 | """ 50 | 51 | _DEFAULT_ADDRESS = "" 52 | _DEFAULT_PORT = 9140 53 | 54 | _TIMEOUT = .001 55 | 56 | FullQueueError = Exception("Data queue is full and policy was not set to drop!") 57 | 58 | 59 | def __init__(self, 60 | address = _DEFAULT_ADDRESS, 61 | port = _DEFAULT_PORT, 62 | maxVolumeNumber = 20, 63 | dropVolumeOnFull = True, 64 | blocking = True): 65 | print("creating a server at address '%s' and port '%s'"%(address,port)) 66 | 67 | self.sock = socket.socket() 68 | self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 69 | self.sock.setblocking(True) 70 | 71 | self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) 72 | self.dataQueue = six.moves.queue.Queue(maxsize = max(1,maxVolumeNumber)) 73 | self.dataThread = _DataServerThread(self.sock, self.dataQueue) 74 | self.dropVolumeOnFull = dropVolumeOnFull 75 | self._bind(address,port) 76 | 77 | def _bind(self, address = _DEFAULT_ADDRESS, port = _DEFAULT_PORT): 78 | logger.debug("binding with address %s at port %s "%(address,port)) 79 | try: 80 | self.sock.bind((address,port)) 81 | except Exception as e: 82 | print(e) 83 | 84 | self.sock.listen(10) 85 | 86 | def sendData(self, data, **kwargs): 87 | """sends array data to the server 88 | 89 | data : a 3d uint16 numpy array 90 | 91 | supported keyword arguments with its defaults: 92 | 93 | "index":0, 94 | "time": 0, 95 | "channel": 0, 96 | "channelname": "python source", 97 | "color": "1. 1. 1. 1.", 98 | "viewmatrix": "1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.", 99 | "dim": 3, 100 | "type": "Unsigned Byte", 101 | "bytespervoxel":1, 102 | "elementsize": 1, 103 | "voxelwidth": 1, 104 | "voxelheight": 1, 105 | "voxeldepth": 1, 106 | "realunit":1 107 | 108 | """ 109 | 110 | logger.debug("put data of shape %s in queue"%str(data.shape)) 111 | logger.debug("meta: %s"%kwargs) 112 | 113 | if self.dataQueue.full(): 114 | if self.dropVolumeOnFull: 115 | while self.dataQueue.full(): 116 | self.dataQueue.get(block=True,timeout = self._TIMEOUT) 117 | else: 118 | raise self.FullQueueError 119 | 120 | self.dataQueue.put((data, kwargs)) 121 | 122 | def is_connected(self): 123 | return self.dataThread.isconnected 124 | 125 | def client_address(self): 126 | if self.dataThread.clientAddress: 127 | try: 128 | clientIP = self.dataThread.clientAddress 129 | clientName = socket.gethostbyaddr(clientIP)[0] 130 | return clientIP, clientName 131 | except Exception as e: 132 | print(e) 133 | return None, None 134 | 135 | 136 | def start(self): 137 | logger.debug("starting server") 138 | self.dataThread.start() 139 | 140 | def stop(self,blocking = False): 141 | self.dataThread.stop(blocking = blocking) 142 | self.sock.close() 143 | 144 | def serveUntilEmpty(self): 145 | while not self.dataThread.isempty: 146 | logger.debug("waiting until empty") 147 | time.sleep(.5) 148 | 149 | 150 | def __del__(self): 151 | self.stop() 152 | 153 | 154 | class _DataServerThread(threading.Thread): 155 | """ 156 | """ 157 | _TIMEOUT = 0.001 158 | def __init__(self, sock, dataQueue): 159 | threading.Thread.__init__ (self) 160 | self.sock = sock 161 | self.dataQueue = dataQueue 162 | self.setDaemon(True) 163 | self.isempty = False 164 | self.isconnected = False 165 | self.clientAddress = None 166 | 167 | 168 | def run(self): 169 | self.isRunning = True 170 | while self.isRunning: 171 | logger.debug("[thread] waiting for connection...") 172 | self.isconnected = False 173 | conn, addr = self.sock.accept() 174 | self.isconnected = True 175 | self.clientAddress = addr[0] 176 | logger.debug("...connected!") 177 | 178 | logger.debug("[thread] now serving the data...") 179 | while True: 180 | try: 181 | self.isempty = False 182 | data, meta = self.dataQueue.get(block = True, timeout = self._TIMEOUT) 183 | logger.debug("[thread] got data in thread...") 184 | self.send_data(conn,data, meta) 185 | self.dataQueue.task_done() 186 | except six.moves.queue.Empty: 187 | logger.debug("[thread] Queue empty") 188 | self.isempty = True 189 | # logger.debug("no data :(") 190 | # if not self.isRunning: 191 | # break 192 | except socket.error: 193 | logger.debug("[thread] socket broken") 194 | break 195 | time.sleep(.1) 196 | logger.debug("[thread] closing socket") 197 | self.sock.close() 198 | 199 | def stop(self, blocking): 200 | logger.debug("[thread] stopping") 201 | self.isRunning = False 202 | 203 | def send_data(self,conn,data, meta = {}): 204 | # print "SEEEEND ", data.shape, meta 205 | logger.debug("[thread] send_data()") 206 | conn.sendall(_serialize_data(data, meta)) 207 | #_serialize_data 208 | 209 | 210 | 211 | def test_full(): 212 | import numpy as np 213 | 214 | d = DataServer(maxVolumeNumber=2) 215 | d.start() 216 | data = np.zeros((10,)*3) 217 | 218 | d.sendData(data) 219 | d.sendData(data) 220 | d.sendData(data) 221 | 222 | 223 | 224 | def test_serve_forever(): 225 | import numpy as np 226 | import time 227 | 228 | d = DataServer(maxVolumeNumber=2) 229 | d.start() 230 | N = 128 231 | 232 | data = np.linspace(0,65000,N**3).reshape((N,)*3).astype(np.uint16) 233 | 234 | t = 0 235 | while True: 236 | args = {} 237 | args["color"] = "%s %s %s 1."%tuple([str(c) for c in np.random.uniform(0,1,3)]) 238 | args["voxelwidth"] = np.random.uniform(.2,1.6) 239 | args["voxelheight"] = np.random.uniform(.2,1.6) 240 | args["voxeldepth"] = np.random.uniform(.2,1.6) 241 | args["time"] = t 242 | 243 | print("sending...") 244 | print(args) 245 | d.sendData(data,**args) 246 | # d.sendData(data) 247 | time.sleep(2) 248 | t += 1 249 | 250 | 251 | if __name__ == '__main__': 252 | 253 | 254 | # test_full() 255 | 256 | 257 | test_serve_forever() 258 | -------------------------------------------------------------------------------- /pyclearvolume/_serialize.py: -------------------------------------------------------------------------------- 1 | """ 2 | packages data into the raw format the clearvolume client expects 3 | 4 | author: Martin Weigert 5 | email: mweigert@mpi-cbg.de 6 | """ 7 | 8 | from __future__ import absolute_import 9 | from __future__ import print_function 10 | import numpy as np 11 | import six 12 | from six.moves import zip 13 | 14 | DEFAULT_METADATA = { 15 | "index": 0, 16 | "time": 0, 17 | "channel": 0, 18 | "channelname": "python source", 19 | "viewmatrix": "1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.", 20 | "dim": 3, 21 | "color": (1., 1., 1., 1.), 22 | "type": "UnsignedShort", 23 | "bytespervoxel": 2, 24 | "elementsize": 1, 25 | "voxelwidth": 1, 26 | "voxelheight": 1, 27 | "voxeldepth": 1, 28 | "realunit": 1 29 | } 30 | 31 | _SUPPORTED_TYPES = {np.uint8: "UnsignedByte", 32 | np.uint16: "UnsignedShort"} 33 | 34 | 35 | def _serialize_data(data, meta=DEFAULT_METADATA): 36 | """returns serialized version of data for clearvolume data viewer""" 37 | 38 | if not isinstance(data, np.ndarray): 39 | raise TypeError("data should be a numpy array (but is %s)" % type(data)) 40 | 41 | if not data.dtype.type in _SUPPORTED_TYPES: 42 | raise ValueError("data type should be in (%s) (but is %s)" % (list(_SUPPORTED_TYPES.keys()), data.dtype)) 43 | 44 | LenInt64 = len(np.int64(1).tostring()) 45 | 46 | Ns = data.shape 47 | 48 | metaData = DEFAULT_METADATA.copy() 49 | 50 | # prepare header.... 51 | 52 | metaData["type"] = _SUPPORTED_TYPES[data.dtype.type] 53 | 54 | for attrName, N in zip(["width", "height", "depth"], Ns[::-1]): 55 | metaData[attrName] = meta.get(attrName, N) 56 | 57 | for key, val in six.iteritems(meta): 58 | if key not in metaData: 59 | raise KeyError(" '%s' (= %s) as is an unknown property!" % (key, val)) 60 | else: 61 | metaData[key] = val 62 | 63 | print(metaData) 64 | 65 | keyValPairs = [str(key) + ":" + str(val) for key, val in six.iteritems(metaData)] 66 | headerStr = ",".join(keyValPairs) 67 | headerStr = "[" + headerStr + "]" 68 | 69 | # headerStr = str(metaData).replace("{","[").replace("}","]").replace("'",'')#.replace(" ",'') 70 | 71 | 72 | headerLength = len(headerStr) 73 | 74 | dataStr = data.tostring() 75 | dataLength = len(dataStr) 76 | 77 | neededBufferLength = 3 * LenInt64 + headerLength + dataLength 78 | 79 | return "%s%s%s%s%s" % (np.int64(neededBufferLength).tostring(), np.int64(headerLength).tostring(), headerStr, 80 | np.int64(dataLength).tostring(), dataStr) 81 | 82 | 83 | if __name__ == '__main__': 84 | Ns = [1, 2, 3] 85 | 86 | d = (123 * np.linspace(0, 200, np.prod(Ns))).reshape(Ns).astype(np.uint8) 87 | 88 | # dStr = _serialize_data(d,{"width": 5.,"color":"1. .5 .2 1."}) 89 | 90 | dStr = _serialize_data(d, {"width": "5", "color": "1. .5 .2 1."}) 91 | 92 | # print dStr 93 | -------------------------------------------------------------------------------- /pyclearvolume/folder_watch_thread.py: -------------------------------------------------------------------------------- 1 | """ 2 | 3 | a simple utility that watches a folder for new image files and 4 | returns the data 5 | 6 | """ 7 | 8 | 9 | from __future__ import absolute_import 10 | from __future__ import print_function 11 | import os 12 | import six.moves.queue 13 | from threading import Thread 14 | 15 | from sortedcontainers import SortedList, SortedDict 16 | from collections import Counter 17 | 18 | 19 | from pyclearvolume.image_file_reader import ImageFolderReader, ImageFileReader 20 | 21 | import time 22 | import numpy as np 23 | import six 24 | from six.moves import range 25 | 26 | 27 | class WatchThread(Thread): 28 | """ 29 | """ 30 | TIME_STEP = .1 31 | def __init__(self,dirName, wildcard="*", deltaTime = 1., func = os.path.getsize): 32 | maxCount = int(np.ceil(deltaTime/self.TIME_STEP)) 33 | print("setting up watchthread with maxCount %s"%maxCount) 34 | super(WatchThread,self).__init__() 35 | self.folderReader = ImageFolderReader(dirName,wildcard, func) 36 | self.goodFiles = six.moves.queue.Queue() 37 | self.updatelist = UpdateCountList(maxCount) 38 | self.processed = set() 39 | self.setDaemon(True) 40 | 41 | 42 | def run(self): 43 | while True: 44 | allfiles = dict([(k,v) for k,v in self.folderReader.list_files() if not k in self.processed]) 45 | self.updatelist.update(allfiles) 46 | 47 | newfiles = self.updatelist.filter_names() 48 | if newfiles: 49 | fName = newfiles[0] 50 | print("newfile: %s %s"%(fName, self.updatelist.properties[fName])) 51 | self.goodFiles.put(fName) 52 | self.processed.add(fName) 53 | 54 | time.sleep(self.TIME_STEP) 55 | 56 | def empty(self): 57 | return self.goodFiles.empty() 58 | 59 | def get(self): 60 | fName = self.goodFiles.get() 61 | print("getting data from ", fName) 62 | data, meta = ImageFileReader.load_file(fName) 63 | print("shape: ", data.shape) 64 | self.goodFiles.task_done() 65 | return data, meta 66 | 67 | 68 | class UpdateCountList(object): 69 | """ implements a list of objects with properties that can be updated""" 70 | 71 | def __init__(self,maxCount = 10): 72 | self.maxCount = maxCount 73 | self.properties = SortedDict() 74 | self.counts = Counter() 75 | 76 | def add(self,name,prop): 77 | if name in self.properties: 78 | if prop == self.properties[name]: 79 | self.counts[name] += 1 80 | else: 81 | self.properties[name] = prop 82 | else: 83 | self.properties[name] = prop 84 | self.counts[name] = 0 85 | 86 | def is_empty(): 87 | return len(self.properties)==0 88 | 89 | def update(self,namedProperties): 90 | """adds names of a name-property dict 91 | and removes the ones that are not present""" 92 | 93 | namesNotPresent = [n for n in self.properties.keys() 94 | if n not in namedProperties] 95 | 96 | for n in namesNotPresent: 97 | self.pop(n) 98 | 99 | for n,p in six.iteritems(namedProperties): 100 | 101 | self.add(n,p) 102 | 103 | def __repr__(self): 104 | return "\n".join([str(self.properties),str(self.counts)]) 105 | 106 | def pop(self,name): 107 | return self.properties.pop(name),self.counts.pop(name) 108 | 109 | 110 | def filter_names(self): 111 | """returns the names whose counts are at least maxCount""" 112 | return [n for n, c in six.iteritems(self.counts) if c>= self.maxCount] 113 | 114 | 115 | def test_updatelist(): 116 | u = UpdateCountList(maxCount=2) 117 | alist = dict([(chr(65+i),i) for i in range(10)]) 118 | 119 | u.update(alist) 120 | u.update(alist) 121 | u.pop(list(alist.keys())[0]) 122 | u.update(alist) 123 | print(u.counts) 124 | 125 | 126 | 127 | if __name__ == "__main__": 128 | 129 | # test_updatelist() 130 | 131 | a = WatchThread("tests_watch/", deltaTime = 1.) 132 | a.start() 133 | 134 | while True: 135 | try: 136 | fName = a.goodFiles.get(timeout = 1.) 137 | print("a new file arrived! ", fName) 138 | except six.moves.queue.Empty: 139 | pass 140 | time.sleep(.1) 141 | 142 | 143 | -------------------------------------------------------------------------------- /pyclearvolume/image_file_reader.py: -------------------------------------------------------------------------------- 1 | """ 2 | some functions to open 3d biological volume files 3 | 4 | supported as of now: 5 | 6 | - 3d Tif 7 | - CZI 8 | 9 | it uses the tifffile and czifile libraries of 10 | Christoph Gohlke http://www.lfd.uci.edu/~gohlke/ 11 | (for copyright and license see tifffile.py) 12 | and python-bioformats 13 | 14 | """ 15 | 16 | 17 | from __future__ import absolute_import 18 | from __future__ import print_function 19 | import os 20 | import glob 21 | 22 | #surpress warning of tifffile.... 23 | import warnings 24 | warnings.filterwarnings("ignore") 25 | 26 | from pyclearvolume.lib import tifffile 27 | from pyclearvolume.lib import czifile 28 | 29 | def _load_tif_file(fName): 30 | with tifffile.TiffFile(fName) as imgFile: 31 | data = imgFile.asarray() 32 | meta = None 33 | return data ,meta 34 | 35 | def _load_czi_file(fName): 36 | with czifile.CziFile(fName) as imgFile: 37 | data = imgFile.asarray() 38 | meta = None 39 | return data ,meta 40 | 41 | class ImageFileReader(object): 42 | """the container class for reading an image""" 43 | 44 | _supported = {".tif":_load_tif_file, 45 | ".tiff":_load_tif_file, 46 | ".czi":_load_czi_file} 47 | 48 | def __init__(self,fName = None): 49 | self.data = None 50 | self.meta = None 51 | if fName is not None: 52 | self.data, self.meta = self.load_file(fName) 53 | 54 | @classmethod 55 | def is_supported(cls,fName): 56 | basename, ext = os.path.splitext(fName) 57 | return ext in cls._supported 58 | 59 | @classmethod 60 | def load_func_for(cls,fName): 61 | basename, ext = os.path.splitext(fName) 62 | return cls._supported[ext] 63 | 64 | @classmethod 65 | def load_file(cls,fName): 66 | if not cls.is_supported(fName): 67 | raise ValueError("Image format of %s not in supported formats (%s)!" 68 | %(fName,list(cls._supported.keys()))) 69 | 70 | return cls.load_func_for(fName)(fName) 71 | 72 | @classmethod 73 | def _list_supported_files(cls,dirName, wildcard = "*", func = None): 74 | """ list all supported files in directory dirName 75 | 76 | func can be a function of the filename, in which case the result is 77 | the list (filename, func(filename)) 78 | """ 79 | if hasattr(func, '__call__'): 80 | return [(f, func(f)) 81 | for f in glob.glob(os.path.join(dirName,wildcard)) 82 | if cls.is_supported(f)] 83 | else: 84 | return [f for f in glob.glob(os.path.join(dirName,wildcard)) 85 | if cls.is_supported(f)] 86 | 87 | 88 | 89 | class ImageFolderReader(object): 90 | def __init__(self,dirName = None, wildcard = "*", func = None): 91 | self.dirName = dirName 92 | self.wildcard = wildcard 93 | self.func= func 94 | 95 | def list_files(self): 96 | return ImageFileReader._list_supported_files( 97 | self.dirName, self.wildcard, self.func) 98 | 99 | def __iter__(self): 100 | for fName in self.list_files(): 101 | if isinstance(fName,(list,tuple)): 102 | fName = fName[0] 103 | data, meta = ImageFileReader.load_file(fName) 104 | yield data, meta 105 | 106 | 107 | 108 | if __name__ == "__main__": 109 | 110 | # fName = "data/retina.czi" 111 | 112 | a = ImageFolderReader("/Users/mweigert/Tmp/CVTmp/") 113 | 114 | print(a.list_files()) 115 | for data, meta in a: 116 | print(data.shape,meta) 117 | -------------------------------------------------------------------------------- /pyclearvolume/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ClearVolume/PyClearVolume/2979da3747e6c0b8b6e8fc9cbaba7963f37046b5/pyclearvolume/lib/__init__.py -------------------------------------------------------------------------------- /pyclearvolume/lib/czifile.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # czifile.py 3 | 4 | # Copyright (c) 2013-2014, Christoph Gohlke 5 | # Copyright (c) 2013-2014, The Regents of the University of California 6 | # Produced at the Laboratory for Fluorescence Dynamics. 7 | # All rights reserved. 8 | # 9 | # Redistribution and use in source and binary forms, with or without 10 | # modification, are permitted provided that the following conditions are met: 11 | # 12 | # * Redistributions of source code must retain the above copyright 13 | # notice, this list of conditions and the following disclaimer. 14 | # * Redistributions in binary form must reproduce the above copyright 15 | # notice, this list of conditions and the following disclaimer in the 16 | # documentation and/or other materials provided with the distribution. 17 | # * Neither the name of the copyright holders nor the names of any 18 | # contributors may be used to endorse or promote products derived 19 | # from this software without specific prior written permission. 20 | # 21 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 22 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 | # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 25 | # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 26 | # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 27 | # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 28 | # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 29 | # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 30 | # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 31 | # POSSIBILITY OF SUCH DAMAGE. 32 | 33 | """Read image and metadata from Carl Zeiss(r) ZISRAW (CZI) files. 34 | 35 | CZI is the native image file format of the ZEN(r) software by the Carl Zeiss 36 | Microscopy GmbH. It stores multidimensional images and metadata from 37 | microscopy experiments. 38 | 39 | :Author: 40 | `Christoph Gohlke `_ 41 | 42 | :Organization: 43 | Laboratory for Fluorescence Dynamics, University of California, Irvine 44 | 45 | :Version: 2013.12.04 46 | 47 | Requirements 48 | ------------ 49 | * `CPython 2.7 or 3.3 `_ 50 | * `Numpy 1.7 `_ 51 | * `Scipy 0.13 `_ 52 | * `Tifffile.py 2013.11.03 `_ 53 | * `Czifle.pyx 2013.12.04 `_ 54 | (for decoding JpegXrFile and JpgFile images) 55 | 56 | Revisions 57 | --------- 58 | 2013.12.04 59 | Decode JpegXrFile and JpgFile via _czifle extension module. 60 | Attempt to reconstruct tiled mosaic images. 61 | 2013.11.20 62 | Initial release. 63 | 64 | Notes 65 | ----- 66 | The API is not stable yet and might change between revisions. 67 | 68 | The file format design specification [1] is confidential and the licence 69 | agreement does not permit to write data into CZI files. 70 | 71 | Only a subset of the 2012 specification is implemented in the initial release. 72 | Specifically, multifile images are not yet supported. 73 | 74 | Tested on Windows with a few example files only. 75 | 76 | References 77 | ---------- 78 | (1) ZISRAW (CZI) File Format Design specification Release Version 1.1 for 79 | ZEN 2012. DS_ZISRAW-FileFormat_Rel_ZEN2012.doc (Confidential) 80 | Documentation can be requested at 81 | 82 | (2) CZI The File Format for the Microscope | ZEISS International 83 | 85 | 86 | Examples 87 | -------- 88 | >>> with CziFile('test.czi') as czi: 89 | ... image = czi.asarray() 90 | >>> image.shape 91 | (3, 3, 3, 250, 200, 3) 92 | >>> image[0, 0, 0, 0, 0] 93 | array([10, 10, 10], dtype=uint8) 94 | 95 | """ 96 | 97 | from __future__ import division, print_function 98 | 99 | from __future__ import absolute_import 100 | import sys 101 | import os 102 | import re 103 | import uuid 104 | import struct 105 | import warnings 106 | import tempfile 107 | import six 108 | from six.moves import range 109 | from six.moves import zip 110 | 111 | try: 112 | from lxml import etree 113 | except ImportError: 114 | from xml.etree import cElementTree as etree 115 | 116 | import numpy 117 | from scipy.ndimage.interpolation import zoom 118 | 119 | from .tifffile import decodelzw, lazyattr, stripnull 120 | 121 | try: 122 | import _czifile 123 | _have_czifile = True 124 | except ImportError: 125 | _have_czifile = False 126 | warnings.warn( 127 | "failed to import the optional _czifile C extension module.\n" 128 | "Decoding of JXR and JPG encoded images will be unavailable.\n" 129 | "Czifile.pyx can be obtained at http://www.lfd.uci.edu/~gohlke/") 130 | 131 | __version__ = '2013.12.04' 132 | __docformat__ = 'restructuredtext en' 133 | __all__ = 'imread', 'CziFile' 134 | 135 | 136 | def imread(filename, *args, **kwargs): 137 | """Return image data from CZI file as numpy array. 138 | 139 | 'args' and 'kwargs' are arguments to CziFile.asarray(). 140 | 141 | Examples 142 | -------- 143 | >>> image = imread('test.czi') 144 | >>> image.shape 145 | (3, 3, 3, 250, 200, 3) 146 | >>> image.dtype 147 | dtype('uint8') 148 | 149 | """ 150 | with CziFile(filename) as czi: 151 | result = czi.asarray(*args, **kwargs) 152 | return result 153 | 154 | 155 | class CziFile(object): 156 | """Carl Zeiss Image (CZI) file. 157 | 158 | Attributes 159 | ---------- 160 | header : FileHeaderSegment 161 | Global file metadata such as file version and GUID. 162 | metadata : etree.ElementTree.Element 163 | Global image metadata in UTF-8 encoded XML format. 164 | 165 | All attributes are read-only. 166 | 167 | """ 168 | 169 | def __init__(self, arg, multifile=True, filesize=None, detectmosaic=True): 170 | """Open CZI file and read header. 171 | 172 | Raise ValueError if file is not a ZISRAW file. 173 | 174 | Parameters 175 | ---------- 176 | multifile : bool 177 | If True (default), the master file of a multifile CZI file 178 | will be opened if applicable. 179 | filesize : int 180 | Size of file if arg is a file handle pointing to an 181 | embedded CZI file. 182 | detectmosaic : bool 183 | If True (default), mosaic images will be reconstructed from 184 | SubBlocks with a tile index. 185 | 186 | Notes 187 | ----- 188 | CziFile instances created from file name must be closed using the 189 | 'close' method, which is automatically called when using the 190 | 'with' statement. 191 | 192 | """ 193 | self._fh = FileHandle(arg, size=filesize) 194 | try: 195 | if self._fh.read(10) != b'ZISRAWFILE': 196 | raise ValueError("not a CZI file") 197 | self.header = Segment(self._fh, 0).data() 198 | except Exception: 199 | self._fh.close() 200 | raise 201 | 202 | if multifile and self.header.file_part and isinstance(arg, six.string_types): 203 | # open master file instead 204 | self._fh.close() 205 | name, _ = match_filename(arg) 206 | self._fh = FileHandle(name) 207 | self.header = Segment(self._fh, 0).data() 208 | assert(self.header.primary_file_guid == self.header.file_guid) 209 | assert(self.header.file_part == 0) 210 | 211 | if self.header.update_pending: 212 | warnings.warn("file is pending update") 213 | self._filter_mosaic = detectmosaic 214 | 215 | def segments(self, kind=None): 216 | """Return iterator over Segment data of specified kind. 217 | 218 | Parameters 219 | ---------- 220 | kind : bytestring or sequence thereof 221 | Segment id(s) as listed in SEGMENT_ID. 222 | If None (default), all segments are returned. 223 | 224 | """ 225 | fpos = 0 226 | while True: 227 | self._fh.seek(fpos) 228 | try: 229 | segment = Segment(self._fh) 230 | except SegmentNotFoundError: 231 | break 232 | if (kind is None) or (segment.sid in kind): 233 | yield segment.data() 234 | fpos = segment.data_offset + segment.allocated_size 235 | 236 | @lazyattr 237 | def metadata(self): 238 | """Return data from MetadataSegment as xml.ElementTree root Element. 239 | 240 | Return None if no Metadata segment is found. 241 | 242 | """ 243 | if self.header.metadata_position: 244 | segment = Segment(self._fh, self.header.metadata_position) 245 | if segment.sid == MetadataSegment.SID: 246 | return etree.fromstring(segment.data().data()) 247 | warnings.warn("Metadata segment not found") 248 | try: 249 | metadata = next(self.segments(MetadataSegment.SID)) 250 | return etree.fromstring(metadata.data()) 251 | except StopIteration: 252 | pass 253 | 254 | @lazyattr 255 | def subblock_directory(self): 256 | """Return list of all DirectoryEntryDV in file. 257 | 258 | Use SubBlockDirectorySegment if exists, else find SubBlockSegments. 259 | 260 | """ 261 | if self.header.directory_position: 262 | segment = Segment(self._fh, self.header.directory_position) 263 | if segment.sid == SubBlockDirectorySegment.SID: 264 | return segment.data().entries 265 | warnings.warn("SubBlockDirectory segment not found") 266 | return list(segment.directory_entry for segment in 267 | self.segments(SubBlockSegment.SID)) 268 | 269 | @lazyattr 270 | def attachment_directory(self): 271 | """Return list of all AttachmentEntryA1 in file. 272 | 273 | Use AttachmentDirectorySegment if exists, else find AttachmentSegments. 274 | 275 | """ 276 | if self.header.attachment_directory_position: 277 | segment = Segment(self._fh, 278 | self.header.attachment_directory_position) 279 | if segment.sid == AttachmentDirectorySegment.SID: 280 | return segment.data().entries 281 | warnings.warn("AttachmentDirectory segment not found") 282 | return list(segment.attachment_entry for segment in 283 | self.segments(AttachmentSegment.SID)) 284 | 285 | def subblocks(self): 286 | """Return iterator over all SubBlock segments in file.""" 287 | for entry in self.subblock_directory: 288 | yield entry.data_segment() 289 | 290 | def attachments(self): 291 | """Return iterator over all Attachment segments in file.""" 292 | for entry in self.attachment_directory: 293 | yield entry.data_segment() 294 | 295 | def save_attachments(self, directory=None): 296 | """Save all attachments to files.""" 297 | if directory is None: 298 | directory = self._fh.filename + '.attachments' 299 | if not os.path.exists(directory): 300 | os.makedirs(directory) 301 | for attachment in self.attachments(): 302 | attachment.save(directory=directory) 303 | 304 | @lazyattr 305 | def filtered_subblock_directory(self): 306 | """Return sorted list of DirectoryEntryDV if mosaic, else all.""" 307 | if not self._filter_mosaic: 308 | return self.subblock_directory 309 | filtered = [directory_entry 310 | for directory_entry in self.subblock_directory 311 | if directory_entry.mosaic_index is not None] 312 | if not filtered: 313 | return self.subblock_directory 314 | return list(sorted(filtered, key=lambda x: x.mosaic_index)) 315 | 316 | @lazyattr 317 | def shape(self): 318 | """Return shape of image data in file.""" 319 | shape = [[dim.start + dim.size 320 | for dim in directory_entry.dimension_entries 321 | if dim.dimension != b'M'] 322 | for directory_entry in self.filtered_subblock_directory] 323 | shape = numpy.max(shape, axis=0) 324 | shape = tuple(i-j for i, j in zip(shape, self.start[:-1])) 325 | dtype = self.filtered_subblock_directory[0].dtype 326 | sampleshape = numpy.dtype(dtype).shape 327 | shape = shape + (sampleshape if sampleshape else (1,)) 328 | return shape 329 | 330 | @lazyattr 331 | def start(self): 332 | """Return minimum start indices per dimension of sub images in file.""" 333 | start = [[dim.start 334 | for dim in directory_entry.dimension_entries 335 | if dim.dimension != b'M'] 336 | for directory_entry in self.filtered_subblock_directory] 337 | start = tuple(numpy.min(start, axis=0)) + (0,) 338 | return start 339 | 340 | @lazyattr 341 | def axes(self): 342 | """Return axes of image data in file.""" 343 | return self.filtered_subblock_directory[0].axes 344 | 345 | @lazyattr 346 | def dtype(self): 347 | """Return dtype of image data in file.""" 348 | # subblock data can be of different pixel type 349 | dtype = self.filtered_subblock_directory[0].dtype[-2:] 350 | for directory_entry in self.filtered_subblock_directory: 351 | dtype = numpy.promote_types(dtype, directory_entry.dtype[-2:]) 352 | return dtype 353 | 354 | def asarray(self, bgr2rgb=False, resize=True, order=1): 355 | """Return image data from file(s) as numpy array. 356 | 357 | Parameters 358 | ---------- 359 | bgr2rgb : bool 360 | If True, exchange red and blue samples if applicable. 361 | resize : bool 362 | If True (default), resize sub/supersampled subblock data. 363 | order : int 364 | The order of spline interpolation used to resize sub/supersampled 365 | subblock data. Default is 1 (bilinear). 366 | 367 | """ 368 | image = numpy.zeros(self.shape, self.dtype) 369 | for directory_entry in self.filtered_subblock_directory: 370 | subblock = directory_entry.data_segment() 371 | tile = subblock.data(bgr2rgb=bgr2rgb, resize=resize, order=order) 372 | index = [slice(i-j, i-j+k) for i, j, k in 373 | zip(directory_entry.start, self.start, tile.shape)] 374 | try: 375 | image[index] = tile 376 | except ValueError as e: 377 | warnings.warn(str(e)) 378 | return image 379 | 380 | def close(self): 381 | self._fh.close() 382 | 383 | def __enter__(self): 384 | return self 385 | 386 | def __exit__(self, exc_type, exc_value, traceback): 387 | self.close() 388 | 389 | def __str__(self): 390 | return '\n '.join(( 391 | self._fh.name.capitalize(), 392 | "(Carl Zeiss Image File)", 393 | str(self.header), 394 | "MetadataSegment", 395 | str(self.axes), 396 | str(self.shape), 397 | str(self.dtype), 398 | str(etree.tostring(self.metadata)))) 399 | 400 | 401 | class FileHandle(object): 402 | """Binary file handle that can handle a file within a file. 403 | 404 | Only binary read, seek, tell and close are supported on embedded files. 405 | 406 | Attributes 407 | ---------- 408 | name : str 409 | file name 410 | path : str 411 | Absolute path to file. 412 | 413 | All attributes are read-only. 414 | 415 | """ 416 | 417 | def __init__(self, arg, mode='rb', name=None, offset=None, size=None): 418 | """Initialize file handle from file name or another file handle. 419 | 420 | Parameters 421 | ---------- 422 | arg : str, File, or FileHandle 423 | File name or open file handle. 424 | mode : str 425 | File open mode in case 'arg' is filename. 426 | name : str 427 | Optional name of file in case 'arg' is file handle. 428 | offset : int 429 | Optional start position of file in the file. By default the 430 | current file position is used as offset. 431 | size : int 432 | Optional size of file in the file. By default the number of 433 | bytes from the current file position to the end of the file 434 | is used. 435 | 436 | """ 437 | if isinstance(arg, six.string_types): # file name 438 | filename = os.path.abspath(arg) 439 | self.path, self.name = os.path.split(filename) 440 | self._fh = open(filename, mode) 441 | self._close = True 442 | if offset is None: 443 | offset = 0 444 | elif isinstance(arg, FileHandle): 445 | if offset is None: 446 | offset = arg.tell() 447 | else: 448 | offset = arg._offset + offset 449 | self._fh = arg._fh 450 | self._close = False 451 | if name: 452 | self.name = name 453 | else: 454 | name, ext = os.path.splitext(arg.name) 455 | self.name = "%s@%i%s" % (name, offset, ext) 456 | self.path = arg.path 457 | else: # file handle 458 | if offset is None: 459 | offset = arg.tell() 460 | self._fh = arg 461 | self._close = False 462 | self.name = name if name else self._fh.name 463 | self.path = '' 464 | 465 | self._offset = offset 466 | if size is not None: 467 | self.size = size 468 | 469 | @property 470 | def filename(self): 471 | return os.path.join(self.path, self.name) 472 | 473 | def read(self, size=-1): 474 | if size < 0 and self._offset: 475 | size = self.size 476 | return self._fh.read(size) 477 | 478 | def fromfile(self, dtype, count=-1, sep=""): 479 | return numpy.fromfile(self._fh, dtype, count, sep) 480 | 481 | def tell(self): 482 | return self._fh.tell() - self._offset 483 | 484 | def seek(self, offset, whence=0): 485 | if self._offset: 486 | if whence == 0: 487 | self._fh.seek(self._offset + offset, whence) 488 | return 489 | elif whence == 2: 490 | self._fh.seek(self._offset + self.size - offset, 0) 491 | return 492 | self._fh.seek(offset, whence) 493 | 494 | def close(self): 495 | if self._close: 496 | self._fh.close() 497 | 498 | def __getattr__(self, name): 499 | if name == 'size': 500 | self._fh.seek(self._offset, 2) 501 | self.size = self._fh.tell() 502 | return self.size 503 | return getattr(self._fh, name) 504 | 505 | def __enter__(self): 506 | return self 507 | 508 | def __exit__(self, exc_type, exc_value, traceback): 509 | self.close() 510 | 511 | 512 | class Segment(object): 513 | """ZISRAW Segment.""" 514 | 515 | __slots__ = 'sid', 'allocated_size', 'used_size', 'data_offset', '_fh' 516 | 517 | def __init__(self, fh, fpos=None): 518 | """Read segment header from file.""" 519 | if fpos is not None: 520 | fh.seek(fpos) 521 | try: 522 | (self.sid, 523 | self.allocated_size, 524 | self.used_size 525 | ) = struct.unpack('<16sqq', fh.read(32)) 526 | except struct.error: 527 | raise SegmentNotFoundError("can not read ZISRAW segment") 528 | self.sid = stripnull(self.sid) 529 | if self.sid not in SEGMENT_ID: 530 | if not self.sid.startswith(b'ZISRAW'): 531 | raise SegmentNotFoundError("not a ZISRAW segment") 532 | warnings.warn("unknown segment type %s" % self.sid) 533 | self.data_offset = fh.tell() 534 | self._fh = fh 535 | 536 | def data(self): 537 | """Read segment data from file and return as \*Segment instance.""" 538 | self._fh.seek(self.data_offset) 539 | return SEGMENT_ID.get(self.sid, UnknownSegment)(self._fh) 540 | 541 | def __str__(self): 542 | return "Segment %s %i of %i" % ( 543 | self.sid, self.used_size, self.allocated_size) 544 | 545 | 546 | class SegmentNotFoundError(Exception): 547 | """Exception to indicate that file position doesn't contain Segment.""" 548 | pass 549 | 550 | 551 | class FileHeaderSegment(object): 552 | """ZISRAWFILE file header segment data. 553 | 554 | Contains global file metadata such as file version and GUID. 555 | 556 | """ 557 | __slots__ = ('version', 'primary_file_guid', 'file_guid', 558 | 'file_part', 'directory_position', 'metadata_position', 559 | 'update_pending', 'attachment_directory_position') 560 | 561 | SID = b'ZISRAWFILE' 562 | 563 | def __init__(self, fh): 564 | (major, 565 | minor, 566 | reserved1, 567 | reserved2, 568 | primary_file_guid, 569 | file_guid, 570 | self.file_part, 571 | self.directory_position, 572 | self.metadata_position, 573 | self.update_pending, 574 | self.attachment_directory_position, 575 | ) = struct.unpack(' 1 else 0 1185 | return name, part 1186 | 1187 | 1188 | def decodejxr(data): 1189 | """Decode JXR data stream into ndarray via temporary file.""" 1190 | fd, filename = tempfile.mkstemp(suffix='.jxr') 1191 | with os.fdopen(fd, 'wb') as fh: 1192 | fh.write(data) 1193 | if isinstance(filename, six.text_type): 1194 | filename = filename.encode('ascii') 1195 | try: 1196 | out = _czifile.decodejxr(filename) 1197 | finally: 1198 | os.remove(filename) 1199 | return out 1200 | 1201 | 1202 | def decodejpg(data): 1203 | """Decode JPG data stream into ndarray.""" 1204 | return _czifile.decodejpg(data, len(data)) 1205 | 1206 | 1207 | # map Segment.sid to data reader 1208 | SEGMENT_ID = { 1209 | FileHeaderSegment.SID: FileHeaderSegment, 1210 | SubBlockDirectorySegment.SID: SubBlockDirectorySegment, 1211 | SubBlockSegment.SID: SubBlockSegment, 1212 | MetadataSegment.SID: MetadataSegment, 1213 | AttachmentSegment.SID: AttachmentSegment, 1214 | AttachmentDirectorySegment.SID: AttachmentDirectorySegment, 1215 | DeletedSegment.SID: DeletedSegment, 1216 | } 1217 | 1218 | # map AttachmentEntryA1.content_file_type to attachment reader. 1219 | CONTENT_FILE_TYPE = { 1220 | b'CZI': CziFile, 1221 | b'ZISRAW': CziFile, 1222 | b'CZTIMS': TimeStamps, 1223 | b'CZEVL': EventList, 1224 | b'CZLUT': LookupTables, 1225 | b'CZFOC': FocusPositions, 1226 | b'CZEXP': xml_reader, # Experiment 1227 | b'CZHWS': xml_reader, # HardwareSetting 1228 | b'CZMVM': xml_reader, # MultiviewMicroscopy 1229 | # b'CZPML': PalMoleculeList, # undocumented 1230 | # b'ZIP' 1231 | # b'JPG' 1232 | } 1233 | 1234 | # map DirectoryEntryDV.pixeltype to numpy dtypes 1235 | PIXEL_TYPE = { 1236 | 0: ' 2: 1287 | six.text_type = str 1288 | six.string_types = str, bytes 1289 | 1290 | if __name__ == "__main__": 1291 | import doctest 1292 | numpy.set_printoptions(suppress=True, precision=5) 1293 | doctest.testmod() 1294 | 1295 | -------------------------------------------------------------------------------- /pyclearvolume/pyclearvolume_serve.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python 2 | 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | import numpy as np 6 | import time 7 | import argparse 8 | import sys 9 | import os 10 | 11 | from pyclearvolume.folder_watch_thread import WatchThread 12 | from pyclearvolume.image_file_reader import ImageFileReader 13 | 14 | import pyclearvolume 15 | import six 16 | 17 | 18 | def _extract_3d(data): 19 | """ in case data has more then 3 dims, extract the 3 biggest ones 20 | e.g. for czi files 21 | """ 22 | return np.squeeze(data) 23 | 24 | 25 | def _prepare_data(data): 26 | """ensure data is a nice uint16 array""" 27 | if data.ndim < 3: 28 | raise ValueError("data has invalid dimensions %s"%(data.shape,)) 29 | if data.ndim > 3: 30 | data = _extract_3d(data) 31 | 32 | data = data.astype(np.uint16) 33 | data = 255.*(data-np.amin(data))/(np.amax(data)-np.amin(data)) 34 | return data.astype(np.uint8) 35 | 36 | 37 | 38 | def _watch_folder(dirName, addr, port, dtime,**kwargs): 39 | d = pyclearvolume.DataServer(maxVolumeNumber=20) 40 | 41 | d.start() 42 | 43 | try: 44 | watchThread = WatchThread(dirName, deltaTime = dtime) 45 | except Exception as e: 46 | sys.exit(str(e)) 47 | 48 | watchThread.start() 49 | 50 | t = 0 51 | while True: 52 | if not watchThread.empty(): 53 | print("connected to %s %s"%(d.client_address())) 54 | try: 55 | data, meta = watchThread.get() 56 | print("getting: ", data.shape) 57 | data = _prepare_data(data) 58 | args = {} 59 | args["index"] = t 60 | args.update(kwargs) 61 | print("sending data of shape %s"%(data.shape,)) 62 | d.sendData(data,**args) 63 | d.serveUntilEmpty() 64 | t += 1 65 | except Exception as e: 66 | print(e) 67 | 68 | time.sleep(1.) 69 | 70 | d.stop() 71 | 72 | 73 | 74 | def _send_files(fNames, addr, port, **kwargs): 75 | files = list(fNames) 76 | d = pyclearvolume.DataServer(maxVolumeNumber=20) 77 | 78 | d.start() 79 | 80 | t = 0 81 | print(files) 82 | for fName in files: 83 | print(fName) 84 | try: 85 | data, meta = ImageFileReader.load_file(fName) 86 | data = _prepare_data(data) 87 | 88 | args = {} 89 | args["index"] = t 90 | args.update(kwargs) 91 | 92 | print("sending data of shape %s"%(data.shape,)) 93 | d.sendData(data,**args) 94 | d.serveUntilEmpty() 95 | time.sleep(.1) 96 | t += 1 97 | except Exception as e: 98 | print(e) 99 | 100 | d.stop() 101 | 102 | def _listdir_full(dirName): 103 | return [os.path.join(dirName, fName) for fName in os.listdir(dirName)] 104 | 105 | def _iter_file_folder_list(fNames): 106 | for fName in fNames: 107 | if os.path.isdir(fName): 108 | for f in _iter_file_folder_list(_listdir_full(fName)): 109 | yield f 110 | else: 111 | yield fName 112 | 113 | 114 | 115 | class MyFormatter(argparse.ArgumentDefaultsHelpFormatter): 116 | 117 | def _fill_text(self, text, width, indent): 118 | 119 | return ''.join([indent + line for line in text.splitlines(True)]) 120 | 121 | 122 | def main(): 123 | parser = argparse.ArgumentParser(formatter_class=MyFormatter, 124 | description=""" 125 | serves 3d image files or folder to a clearvolume client 126 | 127 | pyclearvolume_serve myfile.tif\n 128 | pyclearvolume_serve -u 1. 1. 2. -a remote.domain.com -p 9140 myfile.tif 129 | 130 | """) 131 | 132 | parser.add_argument("-a","--address",dest="address", 133 | help = """address to bind to""", 134 | type=str,default = "", required = False) 135 | 136 | parser.add_argument("-p","--port",dest="port", 137 | help = """port to bind to""", 138 | type=int,default = 9140, required = False) 139 | 140 | parser.add_argument("-w","--watch",dest="watch", 141 | help = """watch folder""", 142 | action="store_true") 143 | 144 | parser.add_argument("-t","--time",dest="dtime", 145 | help = """time in secs in watch mode to 146 | wait for file not having changed""", 147 | type=float,default = 1., required = False) 148 | 149 | parser.add_argument("-u","--units",dest="units", 150 | help = """relative units of voxels e.g. -u 1. 1. 2.""", 151 | type=float,nargs= 3 ,default = [1.,1.,1.], required=False) 152 | 153 | parser.add_argument("-c","--color",dest="color", 154 | help = """color rgb in 0..1""", 155 | type=float,nargs= 3 ,default = [1.,1.,1.], required=False) 156 | 157 | parser.add_argument("files", help="image files or folder to send/watch (currently supported: %s"%(str(list(ImageFileReader._supported.keys()))), nargs="+") 158 | 159 | 160 | 161 | 162 | 163 | if len(sys.argv)==1: 164 | parser.print_help() 165 | sys.exit() 166 | 167 | args = parser.parse_args() 168 | 169 | for k,v in six.iteritems(vars(args)): 170 | print(k,v) 171 | 172 | kwargs = dict() 173 | 174 | kwargs["voxelwidth"] = args.units[0] 175 | kwargs["voxelheight"] = args.units[1] 176 | kwargs["voxeldepth"] = args.units[2] 177 | kwargs["color"] = "%s %s %s 1."%tuple(args.color) 178 | 179 | 180 | if args.watch: 181 | dirName = args.files[0] 182 | if not os.path.isdir(dirName): 183 | raise ValueError("not a valid directory: %s"%dirName) 184 | else: 185 | _watch_folder(dirName,args.address,args.port,args.dtime,**kwargs) 186 | 187 | else: 188 | fNames = [f for f in _iter_file_folder_list(args.files)] 189 | print(fNames) 190 | _send_files(fNames,args.address,args.port,**kwargs) 191 | 192 | 193 | if __name__ == "__main__": 194 | main() 195 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from setuptools import setup 3 | 4 | 5 | setup(name='pyclearvolume', 6 | version='0.1', 7 | description='python binding for the ClearVolume Renderer', 8 | url='http://bitbucket.org/clearvolume/pyclearvolume', 9 | author='Martin Weigert', 10 | author_email='mweigert@mpi-cbg.de', 11 | license='MIT', 12 | packages=['pyclearvolume'], 13 | install_requires=["numpy", 14 | "scipy", 15 | "sortedcontainers", 16 | "tifffile"], 17 | entry_points={ 18 | 'console_scripts': [ 19 | 'pycleartest=pyclearvolume.test:main', 20 | 'pyclearvolume_serve=pyclearvolume.pyclearvolume_serve:main', 21 | ], 22 | } 23 | 24 | 25 | # package_data={"":['']}, 26 | ) 27 | -------------------------------------------------------------------------------- /tests/test_client.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import socket 4 | 5 | 6 | if __name__ == '__main__': 7 | sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 8 | server_address = ('localhost', 9140) 9 | sock.connect(server_address) 10 | 11 | while True: 12 | s = sock.recv(1024) 13 | print(s) 14 | -------------------------------------------------------------------------------- /tests/test_folder_watch.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | from __future__ import print_function 3 | import os 4 | import time 5 | import six.moves.queue 6 | import threading 7 | import numpy as np 8 | 9 | from pyclearvolume.folder_watch_thread import WatchThread 10 | from six.moves import range 11 | 12 | 13 | 14 | class SlowlyFileWriteThread(threading.Thread): 15 | """ 16 | simulates a slowly write of an image file 17 | """ 18 | def __init__(self,dirName, time_to_write = 10., chunksize = 64): 19 | super(SlowlyFileWriteThread,self).__init__() 20 | self.dirName = dirName 21 | self.timestep = 0.1 22 | self.chunksize = chunksize 23 | self.nsteps = int(np.ceil(time_to_write/self.timestep)) 24 | self.setDaemon(True) 25 | 26 | 27 | def run(self): 28 | """write constantly files to dir""" 29 | while True: 30 | fName = os.path.join(self.dirName,"RANDOM_%d.tif"%(time.time())) 31 | print("writing to %s\n"%fName) 32 | #erase file 33 | with open(fName,"w") as f: 34 | pass 35 | for i in range(self.nsteps): 36 | with open(fName,"a") as f: 37 | f.write("1"*self.chunksize) 38 | time.sleep(self.timestep) 39 | time.sleep(.5) 40 | 41 | 42 | 43 | if __name__ == "__main__": 44 | 45 | dirName = "data/random/" 46 | 47 | 48 | 49 | writeSimulator = SlowlyFileWriteThread(os.path.join(dirName)) 50 | 51 | writeSimulator.start() 52 | 53 | 54 | a = WatchThread(dirName, deltaTime = .1) 55 | a.start() 56 | 57 | while True: 58 | try: 59 | fName = a.goodFiles.get(timeout = 1.) 60 | print("a new file arrived! ", fName) 61 | except six.moves.queue.Empty: 62 | pass 63 | time.sleep(.1) 64 | 65 | 66 | -------------------------------------------------------------------------------- /tests/test_server.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function, division 2 | from __future__ import absolute_import 3 | import numpy as np 4 | import time 5 | 6 | import pyclearvolume 7 | 8 | 9 | 10 | def test_server(dtype = np.uint8, N = 128): 11 | print("creating the server") 12 | 13 | d = pyclearvolume.DataServer(maxVolumeNumber=2) 14 | 15 | d.start() 16 | 17 | time.sleep(1) 18 | 19 | print("starting to serve data") 20 | 21 | typeinfo = np.iinfo(dtype) 22 | 23 | data = np.linspace(typeinfo.min,typeinfo.max,N**3).reshape((N,)*3).astype(dtype) 24 | 25 | 26 | t = 0 27 | while True: 28 | if d.is_connected(): 29 | print("connected to %s %s"%(d.client_address())) 30 | args = {} 31 | args["color"] = "%s %s %s 1."%tuple([str(c) for c in np.random.uniform(0,1,3)]) 32 | args["voxelwidth"] = np.random.uniform(.2,1.6) 33 | args["voxelheight"] = np.random.uniform(.2,1.6) 34 | args["voxeldepth"] = np.random.uniform(.2,1.6) 35 | args["index"] = t 36 | print("sending...") 37 | print(args) 38 | d.sendData(data,**args) 39 | t += 1 40 | 41 | # d.sendData(data) 42 | time.sleep(2) 43 | 44 | 45 | 46 | 47 | if __name__ == '__main__': 48 | test_server(dtype = np.uint16) 49 | --------------------------------------------------------------------------------