├── pystp ├── __init__.py ├── utils.py └── client.py ├── setup.py └── README.md /pystp/__init__.py: -------------------------------------------------------------------------------- 1 | from .client import STPClient 2 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | Settings for installing pystp with pip. 3 | """ 4 | from setuptools import setup, find_packages 5 | 6 | setup( 7 | name = "PySTP", 8 | version = "0.1", 9 | packages = find_packages(), 10 | setup_requires = ["wheel"], 11 | install_requires = ["obspy>=1.2.0"] 12 | ) 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PySTP 2 | 3 | PySTP is a Python module for connecting to [STP (Seismogram Transfer Program)](https://scedc.caltech.edu/research-tools/stp/) servers and downloading event metadata, phase picks, and waveforms triggered by seismic events. 4 | 5 | PySTP requires [ObsPy](https://www.obspy.org). 6 | 7 | ## Installation 8 | 9 | Clone the PySTP repository: 10 | 11 | ``` 12 | git clone https://github.com/SCEDC/pystp.git 13 | ``` 14 | 15 | Navigate to the `pystp` directory and run `pip install .` to install the pystp module in your Python environment's path. 16 | 17 | ``` 18 | cd pystp 19 | pip install . 20 | ``` 21 | 22 | ## Basic STPClient Functions 23 | 24 | `get_events` - Downloads an ObsPy catalog either by event IDs or by search parameters of time window, magnitude range, and location boundaries. 25 | 26 | `get_phases` - Downloads an ObsPy catalog containing phase picks. 27 | 28 | `get_trig` - Downloads waveforms for one or events as a Python dictionary with event IDs as keys and ObsPy Stream objects as values. 29 | 30 | ## Usage Example 31 | 32 | ```python 33 | from pystp import STPClient 34 | 35 | client = STPClient() 36 | client.connect() # Open a connection. 37 | 38 | # Download a catalog. 39 | events = client.get_events(times=[datetime.datetime(2019, 10, 17), datetime.datetime(2019, 10, 17, 23, 59, 59)], mags=[2, 4]) 40 | # Get event IDs. 41 | evids = [ev.resource_id.id for ev in events] 42 | # Download all CI.CLC.BH waveforms for the events in the catalog. 43 | waveforms = client.get_trig(evids, net='CI', sta='CLC', chan= 'BH_') 44 | 45 | # Disconnect from the STP server. 46 | client.disconnect() 47 | ``` 48 | 49 | ## Tutorials 50 | 51 | [Downloading waveforms](https://github.com/SCEDC/pystp/blob/master/Example%20Notebook.ipynb) 52 | 53 | [Downloading events and picks](https://github.com/SCEDC/pystp/blob/master/Example%20Notebook%202%20-%20Events%20and%20Phases.ipynb) 54 | 55 | ## Links 56 | 57 | [STP](https://scedc.caltech.edu/research-tools/stp/) 58 | 59 | [ObsPy](https://github.com/obspy/obspy/wiki) -------------------------------------------------------------------------------- /pystp/utils.py: -------------------------------------------------------------------------------- 1 | from obspy.core.event import Event 2 | from obspy.core.event.base import ResourceIdentifier 3 | from obspy.core.event.origin import Origin 4 | from obspy.core.event.origin import Pick 5 | from obspy.core.event.base import WaveformStreamID 6 | from obspy.core.event.magnitude import Magnitude 7 | from obspy.core.utcdatetime import UTCDateTime 8 | from obspy.core.event.base import QuantityError 9 | from datetime import datetime 10 | 11 | 12 | # Mapping of STP magnitude types to obspy.core.event.magnitude.Magnitude.magnitude_type values. 13 | # magnitude_type is a free-text field, so this mapping uses the strings specifically mentioned 14 | # in https://docs.obspy.org/packages/autogen/obspy.core.event.magnitude.Magnitude.html#obspy.core.event.magnitude.Magnitude 15 | # or that seem reasonable. 16 | MAGTYPE_MAPPING = { 'b': 'Mb', \ 17 | 'e': 'Me', \ 18 | 'l': 'ML', \ 19 | 's': 'MS', \ 20 | 'c': 'Mc', 21 | 'n': '', \ 22 | 'w': 'Mw', \ 23 | 'h': 'Mh', \ 24 | 'd': 'Md', \ 25 | 'un': 'M', \ 26 | 'lr': 'Mlr' 27 | } 28 | 29 | # Mapping of STP event types to values allowed in obspy.core.event.header.EventType. 30 | ETYPE_MAPPING = { 'eq': 'earthquake', \ 31 | 'qb': 'quarry blast', \ 32 | 'sn': 'sonic boom', \ 33 | 'nt': 'nuclear blast', \ 34 | 'uk': 'not reported'} 35 | 36 | # Mapping of STP first motion values to polarity for obspy.core.event.origin.Pick. 37 | POLARITY_MAPPING = { 'c': 'positive', \ 38 | 'u': 'positive', \ 39 | 'd': 'negative', \ 40 | 'r': 'dilation', \ 41 | '.': '' \ 42 | } 43 | 44 | def make_event(catalog_entry): 45 | """ Creates an ObsPy Event object from 46 | a line of STP event output. 47 | """ 48 | #print(catalog_entry) 49 | fields = catalog_entry.split() 50 | 51 | evid = fields[0] 52 | etype = fields[1] 53 | origin_time = UTCDateTime(datetime.strptime(fields[3], "%Y/%m/%d,%H:%M:%S.%f")) 54 | 55 | lat = float(fields[4]) 56 | lon = float(fields[5]) 57 | depth = float(fields[6]) 58 | mag = float(fields[7]) 59 | magtype = fields[8] 60 | 61 | res_id = ResourceIdentifier(id=evid) 62 | origin = Origin(latitude=lat, longitude=lon, depth=depth, time=origin_time) 63 | 64 | magnitude = Magnitude(mag=mag, magnitude_type=MAGTYPE_MAPPING[magtype]) 65 | event = Event(resource_id=res_id, event_type=ETYPE_MAPPING[etype], origins=[origin], magnitudes=[magnitude]) 66 | return event 67 | 68 | 69 | def make_pick(pick_str, origin_time): 70 | """ Creates an ObsPy Pick object from a line of STP 71 | phase output. 72 | 73 | Sample pick_str: 74 | CI CLC HHZ -- 35.8157 -117.5975 775.0 P c. i 1.0 6.46 1.543 75 | """ 76 | 77 | fields = pick_str.split() 78 | if len(fields) != 13: 79 | raise Exception('Invalid STP phase output') 80 | 81 | new_pick = Pick() 82 | (net, sta, chan, loc) = fields[:4] 83 | new_pick.waveform_id = WaveformStreamID(network_code=net, station_code=sta, channel_code=chan, location_code=loc) 84 | 85 | # Set phase_hint to the wave type. 86 | new_pick.phase_hint = fields[7] 87 | 88 | # Determine polarity from first motion. 89 | polarity = POLARITY_MAPPING[fields[8][0]] 90 | if polarity == '': 91 | polarity = POLARITY_MAPPING[fields[8][1]] 92 | if polarity != '': 93 | new_pick.polarity = polarity 94 | 95 | # Determine signal onset. 96 | if fields[9] == 'i': 97 | new_pick.onset = 'impulsive' 98 | elif fields[9] == 'e': 99 | new_pick.onset = 'emergent' 100 | else: 101 | new_pick.onset = 'questionable' 102 | 103 | # Determine time error from STP quality. 104 | # Use Jiggle standard and assume sample rate of 100 sps. 105 | quality = float(fields[10]) 106 | if quality == 0.0: 107 | new_pick.time_errors = QuantityError(lower_uncertainty=0.03) 108 | elif quality <= 0.3: 109 | new_pick.time_errors = QuantityError(upper_uncertainty=0.03) 110 | elif quality <= 0.5: 111 | new_pick.time_errors = QuantityError(upper_uncertainty=0.02) 112 | elif quality <= 0.8: 113 | new_pick.time_errors = QuantityError(upper_uncertainty=0.01) 114 | elif quality == 1.0: 115 | new_pick.time_errors = QuantityError(upper_uncertainty=0.0) 116 | 117 | # Determine pick time. 118 | offset = float(fields[12]) 119 | new_pick.time = origin_time + offset 120 | 121 | return new_pick -------------------------------------------------------------------------------- /pystp/client.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import socket 4 | import struct 5 | import re 6 | import os 7 | import obspy.core 8 | from obspy.core import Stream 9 | from obspy.core.event import Catalog 10 | from . import utils 11 | 12 | VALID_FORMATS = ['sac', 'mseed', 'seed', 'ascii', 'v0', 'v1'] 13 | 14 | class STPClient: 15 | 16 | def __init__(self, host='stp.gps.caltech.edu', port=9999, output_dir='.', verbose=False): 17 | """ Set up a new STPClient object. 18 | """ 19 | self.host = host 20 | self.port = port 21 | self.socket = None 22 | self.fdr = None # File handle to the socket 23 | self.fdout = None # Output file handle 24 | self.output_dir = '.' 25 | self.message = '' # Most recent message from the server 26 | self.motd = '' # Message of the Day 27 | self.verbose = verbose 28 | self.connected = False 29 | 30 | 31 | def _send_sample(self): 32 | """ Send the integer 2 to the server 33 | to verify endianness. 34 | """ 35 | 36 | two = struct.Struct('I').pack(2) 37 | nbytes = self.socket.send(two) 38 | 39 | 40 | def _read_message(self): 41 | """ Read and store text sent from the STP server delimited 42 | by MESS and ENDmess. 43 | """ 44 | 45 | message = '' 46 | while True: 47 | line = self.fdr.readline() 48 | 49 | if not line or line == b'OVER\n' or line == b'ENDmess\n': 50 | break 51 | 52 | message += line.decode('ascii') 53 | return message 54 | 55 | 56 | def _process_error(self, fields): 57 | """ Display STP error messages. 58 | """ 59 | 60 | err_msg = ' '.join(fields[1:]) 61 | print(err_msg) 62 | 63 | 64 | def _set_motd(self): 65 | """ Reads the message of the day from the server 66 | and stores it in self.motd. 67 | """ 68 | 69 | line = self.fdr.readline() 70 | if line == b'MESS\n': 71 | self.motd = self._read_message() 72 | else: 73 | words = line.split() 74 | if words[0] == b'ERR': 75 | self._process_error(words) 76 | self.fdr.readline() # Read the b'OVER\n' 77 | 78 | 79 | 80 | def _receive_data(self, dir_lst=[], file_lst=[]): 81 | """ Process results sent by the STP server. 82 | """ 83 | 84 | if self.verbose: 85 | print("STPClient._receive_data()") 86 | while True: 87 | line = self.fdr.readline() 88 | if self.verbose: 89 | print('_receive_data: Received line ', line) 90 | if not line: 91 | self.output_dir = '.' 92 | break 93 | 94 | line_words = line.decode('ascii').split() 95 | if len(line_words) == 0: 96 | continue 97 | if self.verbose: 98 | print('_receive_data: ', line_words) 99 | 100 | if line_words[0] == 'OVER': 101 | self.output_dir = '.' 102 | break 103 | elif line_words[0] == 'FILE': 104 | if self.fdout: 105 | self.fdout.close() 106 | outfile = os.path.join(self.output_dir, line_words[1]) 107 | if self.verbose: 108 | print('Opening {} for writing'.format(outfile)) 109 | self.fdout = open(outfile, 'wb') 110 | if not self.fdout: 111 | print('Could not open {} for writing'.format(outfile)) 112 | else: 113 | file_lst.append(outfile) 114 | elif line_words[0] == 'DIR': 115 | # Create output directory 116 | self.output_dir = os.path.join(self.output_dir, line_words[1]) 117 | if not os.path.isdir(self.output_dir): 118 | os.mkdir(self.output_dir) 119 | dir_lst.append(self.output_dir) 120 | elif line_words[0] == 'MESS': 121 | msg = self._read_message() 122 | self.message += msg 123 | #print(msg, end='') 124 | 125 | elif line_words[0] == 'DATA': 126 | ndata = int(line_words[1]) 127 | # Read ndata bytes. 128 | data = self.fdr.read(ndata) 129 | # Write ndata bytes to the output file handle. 130 | if self.fdout: 131 | self.fdout.write(data) 132 | elif line_words[0] == 'ENDdata': 133 | continue 134 | elif line_words[0] == 'ERR': 135 | self._process_error(line_words) 136 | 137 | 138 | def set_verbose(self, verbose): 139 | self.verbose = verbose 140 | 141 | 142 | def set_output_dir(self, output_dir): 143 | """ Change the base output directory. 144 | """ 145 | self.output_dir = output_dir 146 | 147 | 148 | def set_nevntmax(self, value=100): 149 | """ Set the value of the nevntmax parameter, the maximum 150 | number of events returned by the event command. 151 | """ 152 | self.socket.sendall('set nevntmax {}\n'.format(value).encode('utf-8')) 153 | self.fdr.readline() 154 | 155 | def set_gaincorr(self, value='on'): 156 | """ Set the value of the gain parameter to on or off. 157 | """ 158 | self.socket.sendall('gain {}\n'.format(value).encode('utf-8')) 159 | line = self.fdr.readline() 160 | if line == b'MESS\n': 161 | self.message = self._read_message() 162 | else: 163 | words = line.split() 164 | if words[0] == b'ERR': 165 | self._process_error(words) 166 | self.fdr.readline() # Read the b'OVER\n' 167 | if self.verbose: 168 | print (self.message) 169 | self._clear_message() 170 | 171 | 172 | def connect(self, show_motd=True): 173 | """ Connect to STP server. 174 | """ 175 | 176 | if self.connected: 177 | print('Already connected') 178 | return 179 | 180 | self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 181 | self.socket.connect((self.host, self.port)) 182 | self.fdr = self.socket.makefile(mode='rb') 183 | 184 | self.socket.sendall(b'STP stpisgreat 1.6.3 stpc\n') 185 | 186 | line = self.fdr.readline() 187 | if line != b'CONNECTED\n': 188 | print(line) 189 | raise Exception('Failed to connect') 190 | 191 | self._send_sample() 192 | 193 | self._set_motd() 194 | if show_motd: 195 | print(self.motd, end='') 196 | self.connected = True 197 | self._clear_message() 198 | 199 | 200 | def _send_data_command(self, cmd, data_format, as_stream=True, keep_files=False): 201 | """ Send a waveform request command and process the results. 202 | """ 203 | 204 | data_format = data_format.lower() 205 | if data_format not in VALID_FORMATS: 206 | raise Exception('Invalid data format') 207 | if self.verbose: 208 | print("data_format={} cmd={}".format(data_format, cmd)) 209 | 210 | file_lst = [] 211 | dir_lst = [] 212 | self.socket.sendall('{}\n'.format(data_format).encode('utf-8')) 213 | self._receive_data(dir_lst, file_lst) 214 | self.socket.sendall(cmd.encode('utf-8')) 215 | self._receive_data(dir_lst, file_lst) 216 | 217 | waveform_stream = None 218 | if as_stream: 219 | waveform_stream = Stream() 220 | ntraces = 0 221 | for f in file_lst: 222 | try: 223 | if self.verbose: 224 | print('Reading {}'.format(f)) 225 | tr = obspy.core.read(f) 226 | waveform_stream += tr 227 | ntraces += 1 228 | except TypeError: 229 | if self.verbose: 230 | print('{} is in unknown format. Skipping.'.format(f)) 231 | 232 | 233 | if not keep_files: 234 | if self.verbose: 235 | print("Removing {} after reading".format(f)) 236 | if os.path.isfile(f): 237 | os.remove(f) 238 | print('Processed {} waveform traces'.format(ntraces)) 239 | 240 | return waveform_stream 241 | 242 | 243 | def _end_command(self): 244 | """ Perform cleanup after an STP command is ended. 245 | """ 246 | 247 | if self.fdout: 248 | self.fdout.close() 249 | self._clear_message() 250 | 251 | def _clear_message(self): 252 | self.message = '' 253 | 254 | def get_trig(self, evids, net='%', sta='%', chan='%', loc='%', radius=None, data_format='sac', as_stream=True, keep_files=False): 255 | """ Download triggered waveforms from STP using the TRIG command. 256 | """ 257 | 258 | if not self.connected: 259 | print('STP is not connected') 260 | return None 261 | 262 | base_cmd = 'trig ' 263 | if net != '%': 264 | base_cmd += ' -net {}'.format(net) 265 | if sta != '%': 266 | base_cmd += ' -sta {}'.format(sta) 267 | if chan != '%': 268 | base_cmd += ' -chan {}'.format(chan) 269 | if loc != '%': 270 | base_cmd += ' -loc {}'.format(loc) 271 | if radius is not None: 272 | base_cmd += ' -radius {}'.format(radius) 273 | 274 | result = {} 275 | 276 | def request_event(evid): 277 | cmd = "{} {}\n".format(base_cmd, evid) 278 | result[evid] = self._send_data_command(cmd, data_format, as_stream, keep_files) 279 | 280 | if isinstance(evids, list): 281 | for ev in evids: 282 | #cmd = "{} {}\n".format(base_cmd, ev) 283 | #result[ev] = self._send_data_command(cmd, data_format, as_stream, keep_files) 284 | request_event(ev) 285 | else: 286 | request_event(evids) 287 | 288 | self._end_command() 289 | 290 | return result 291 | 292 | def get_window(self, start_time, end_time, net='%', sta='%', chan='%', loc='%', data_format='sac', as_stream=True, keep_files=False): 293 | """ Download continuous waveforms from STP using the WIND command. 294 | positional params expected by WIND: 295 | net sta chan loc time_on, time_off 296 | """ 297 | 298 | if not self.connected: 299 | print('STP is not connected') 300 | return None 301 | if not start_time or not end_time: 302 | print ('Time range is required') 303 | return None 304 | # at least one of net, sta, chan is required 305 | if net=='%' and sta=='%' and chan=='%': 306 | print ('At least one of net/sta/chan is required.' ) 307 | return None 308 | 309 | start = start_time.strftime("%Y/%m/%d,%H:%M:%S.%f") 310 | end = end_time.strftime("%Y/%m/%d,%H:%M:%S.%f") 311 | 312 | base_cmd = f'wind {net} {sta} {chan} {loc} {start} {end} \n' 313 | result = self._send_data_command(base_cmd, data_format, as_stream, keep_files) 314 | self._end_command() 315 | return result 316 | 317 | 318 | def get_continuous(self, net='%', sta='%', chan='%', loc='%', data_format='sac', as_stream=True, keep_files=False): 319 | pass 320 | 321 | 322 | def _get_event_phase(self, cmd, evids, times=None, lats=None, lons=None, mags=None, depths=None, types=None, gtypes=None, output_file=None, is_xml=False): 323 | """ Helper function that handles the event and phase commands, 324 | which have similar syntax. 325 | """ 326 | 327 | if output_file is not None: 328 | cmd += ' -f {} '.format(output_file) 329 | if evids is not None: 330 | evids_str = [str(e) for e in evids] 331 | cmd += ' -e {} '.format(' '.join(evids_str)) 332 | else: 333 | if times is not None: 334 | start_time = times[0].strftime("%Y/%m/%d,%H:%M:%S.%f") 335 | end_time = times[1].strftime("%Y/%m/%d,%H:%M:%S.%f") 336 | cmd += ' -t0 {} {}'.format(start_time, end_time) 337 | if lats is not None: 338 | cmd += ' -lat {} {}'.format(lats[0], lats[1]) 339 | if lons is not None: 340 | cmd += ' -lon {} {}'.format(lons[0], lons[1]) 341 | if mags is not None: 342 | cmd += ' -mag {} {}'.format(mags[0], mags[1]) 343 | if depths is not None: 344 | cmd += ' -depth {} {}'.format(depths[0], depths[1]) 345 | if types is not None: 346 | cmd += ' -type {} '.format(','.join(types)) 347 | if gtypes is not None: 348 | cmd += ' -gtype {} '.format(','.join(gtypes)) 349 | 350 | if self.verbose: 351 | print(cmd) 352 | cmd += '\n' 353 | if self.verbose: 354 | print('Sending command') 355 | self.socket.send(cmd.encode('utf-8')) 356 | self._receive_data() 357 | 358 | 359 | def get_eavail(self, evid, net='', sta='', chan='', loc='', format='s', as_list=True): 360 | """ Get the available data for an event. 361 | """ 362 | 363 | if not self.connected: 364 | print('STP is not connected') 365 | return None 366 | 367 | cmd = 'eavail' 368 | 369 | if net != '': 370 | cmd += ' -net {}'.format(net) 371 | if sta != '': 372 | cmd += ' -sta {}'.format(sta) 373 | if chan != '': 374 | cmd += ' -chan {}'.format(chan) 375 | if loc != '': 376 | cmd += ' -loc {}'.format(loc) 377 | if format == 'l' or format == 'long': 378 | cmd += ' -l' 379 | 380 | cmd += ' ' + str(evid) 381 | cmd += '\n' 382 | if self.verbose: 383 | print(cmd) 384 | self.socket.send(cmd.encode('utf-8')) 385 | self._receive_data() 386 | 387 | if as_list: 388 | # Remove the comment with the number of seismograms, which will not be part of the list. 389 | self.message = self.message.split('#')[0] 390 | 391 | if as_list and (format == 'l' or format== 'long'): 392 | eavail_listing = [line.strip().split() for line in self.message.split('\n') if not line.strip().startswith('#') and not line == ''] 393 | elif as_list and (format == 's' or format == 'short'): 394 | eavail_listing = [line.strip().split('.') for line in self.message.split() if not line.strip().startswith('#') and not line == ''] 395 | else: 396 | eavail_listing = self.message 397 | self._end_command() 398 | return eavail_listing 399 | 400 | 401 | def get_events(self, evids=None, times=None, lats=None, lons=None, mags=None, depths=None, types=None, gtypes=None, output_file=None, is_xml=False): 402 | """ Download events from STP using the EVENT command. 403 | """ 404 | 405 | if not self.connected: 406 | print('STP is not connected') 407 | return None 408 | self._get_event_phase('event', evids, times, lats, lons, mags, depths, types, gtypes, output_file) 409 | catalog = Catalog() 410 | for line in self.message.splitlines(): 411 | if not line.startswith('#'): 412 | catalog.append(utils.make_event(line)) 413 | self._end_command() 414 | return catalog 415 | 416 | 417 | def get_phases(self, evids=None, times=None, lats=None, lons=None, mags=None, depths=None, types=None, gtypes=None, output_file=None, is_xml=False): 418 | """ Download events and phase picks from STP using the PHASE command. 419 | """ 420 | 421 | if not self.connected: 422 | print('STP is not connected') 423 | return None 424 | self._get_event_phase('phase', evids, times, lats, lons, mags, depths, types, gtypes, output_file) 425 | evid_pattern = re.compile('^[1-9]+') 426 | catalog = Catalog() 427 | event = None 428 | for line in self.message.splitlines(): 429 | line = line.strip() 430 | if not line.startswith('#'): 431 | #print(evid_pattern.match(line)) 432 | if evid_pattern.match(line) is not None: 433 | #print('Creating event') 434 | event = utils.make_event(line) 435 | catalog.append(event) 436 | else: 437 | #print('Creating phase pick') 438 | pick = utils.make_pick(line.strip(), event.origins[0].time) 439 | if event is None: 440 | raise Exception('Error parsing phase output') 441 | event.picks.append(pick) 442 | self._end_command() 443 | return catalog 444 | 445 | 446 | def disconnect(self): 447 | """ Disconnect from the STP server. 448 | """ 449 | 450 | if self.fdr: 451 | self.fdr.close() 452 | if self.socket: 453 | self.socket.close() 454 | self.connected = False 455 | 456 | if __name__ == '__main__': 457 | stp = STPClient('athabasca.gps.caltech.edu', 9999) 458 | stp.connect(True) 459 | stp.disconnect() 460 | --------------------------------------------------------------------------------