├── quakenet_ingv ├── tflib │ ├── __init__.py │ ├── layers.py │ └── model.py ├── quakenet │ ├── __init__.py │ ├── config.py │ ├── util.py │ ├── data_pipeline.py │ ├── plot.py │ └── models.py ├── .DS_Store └── bin │ ├── .DS_Store │ ├── train │ ├── preprocess │ ├── get_events.py │ ├── convert_streams.py │ └── get_streams.py │ ├── plot_events │ ├── plot_event_stats │ ├── plot_event_results │ └── evaluate ├── .DS_Store ├── INSTALLATION.txt ├── .gitignore ├── RUNNING_MN3_ConvNetQuake9.txt └── README.md /quakenet_ingv/tflib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /quakenet_ingv/quakenet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alomax/ConvNetQuake_INGV/HEAD/.DS_Store -------------------------------------------------------------------------------- /quakenet_ingv/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alomax/ConvNetQuake_INGV/HEAD/quakenet_ingv/.DS_Store -------------------------------------------------------------------------------- /quakenet_ingv/bin/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alomax/ConvNetQuake_INGV/HEAD/quakenet_ingv/bin/.DS_Store -------------------------------------------------------------------------------- /quakenet_ingv/quakenet/config.py: -------------------------------------------------------------------------------- 1 | class Config(object): 2 | def __init__(self): 3 | self.learning_rate = 0.001 4 | self.batch_size = 128 5 | self.win_size = 1001 6 | self.n_traces = 3 7 | self.display_step = 50 8 | self.n_threads = 2 9 | self.n_epochs = None 10 | self.regularization = 1e-3 11 | self.n_distances = None 12 | self.n_magnitudes = None 13 | self.n_depths = None 14 | self.n_azimuths = None 15 | 16 | # Number of epochs, None is infinite 17 | n_epochs = None 18 | -------------------------------------------------------------------------------- /INSTALLATION.txt: -------------------------------------------------------------------------------- 1 | # install environment 2 | # https://www.tensorflow.org/install/install_mac 3 | sudo easy_install pip 4 | pip install --upgrade virtualenv 5 | virtualenv --system-site-packages . 6 | source ./bin/activate 7 | easy_install -U pip 8 | pip install grpcio==1.9.1 9 | pip install https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl 10 | pip install -r requirements.txt # needed? 11 | pip install -r requirements2.txt # needed? 12 | pip install python-gflags 13 | pip install -e git+https://github.com/gem/oq-hazardlib.git@589fa31ddca3697e6b167082136dc0174a77bc16#egg=openquake.hazardlib 14 | pip install dask --upgrade 15 | pip install argparse 16 | pip install geojson 17 | pip install geopy 18 | pip install folium 19 | 20 | 21 | ln -s /Users/anthony/opt/ConvNetQuake-master_tf1.0/bin cnqbin 22 | ln -s /Volumes/RAID_2TB_2015A/anthony_big_stuff/ConvNetQuake/data . 23 | ln -s /Users/anthony/opt/ConvNetQuake-master_tf1.0/quakenet . 24 | ln -s /Users/anthony/opt/ConvNetQuake-master_tf1.0/tflib . 25 | 26 | # convert code to TensorFlow 1.2 27 | # upgrade a whole directory of 0.n TensorFlow programs to 1.0, enter a command having the following format: 28 | $ python tf_upgrade.py --intree InputDir --outtree OutputDir 29 | # 30 | chmod a+x ./cnqbin/*/*.py 31 | 32 | 33 | # run codes 34 | source ./bin/activate 35 | export PYTHONPATH=./:./quakenet_ingv:/Users/anthony/opt/ConvNetQuake-master_tf1.0:$PYTHONPATH 36 | 37 | ... 38 | 39 | see RUNNING_*.txt 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | -------------------------------------------------------------------------------- /quakenet_ingv/quakenet/util.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 3 Apr 2018 3 | 4 | @author: anthony 5 | ''' 6 | 7 | 8 | import math 9 | 10 | 11 | 12 | def distance2classification(distance, num_dist_classes): 13 | 14 | dist_id = int(round(float(num_dist_classes) * math.pow(distance / 180.0, 0.5))) 15 | dist_id = min(max(dist_id, 0), num_dist_classes - 1) 16 | 17 | return dist_id 18 | 19 | 20 | def classification2distance(dist_id, num_dist_classes): 21 | 22 | distance = 180.0 * math.pow(float(dist_id) / float(num_dist_classes), 2.0) 23 | 24 | return distance 25 | 26 | 27 | def magntiude2classification(magnitude, num_mag_classes): 28 | 29 | mstep = 9.5 / float(num_mag_classes - 1); 30 | mag_id = int(round(magnitude / mstep)) 31 | mag_id = min(max(mag_id, 0), num_mag_classes - 1) 32 | 33 | return mag_id 34 | 35 | 36 | def classification2magnitude(mag_id, num_mag_classes): 37 | 38 | mstep = 9.5 / float(num_mag_classes - 1); 39 | magnitude = mstep * float(mag_id) 40 | 41 | return magnitude 42 | 43 | 44 | def depth2classification(depth, num_depth_classes): 45 | 46 | depth_id = int(round(float(num_depth_classes) * math.pow(depth / 700.0, 0.5))) 47 | depth_id = min(max(depth_id, 0), num_depth_classes - 1) 48 | 49 | return depth_id 50 | 51 | 52 | def classification2depth(depth_id, num_depth_classes): 53 | 54 | depth = 700.0 * math.pow(float(depth_id) / float(num_depth_classes), 2.0) 55 | 56 | return depth 57 | 58 | 59 | def azimuth2classification(azimuth, num_az_classes): 60 | 61 | mstep = 360.0 / float(num_az_classes); 62 | mag_id = int(round(azimuth / mstep)) 63 | mag_id = min(max(mag_id, 0), num_az_classes - 1) 64 | 65 | return mag_id 66 | 67 | 68 | def classification2azimuth(az_id, num_az_classes): 69 | 70 | mstep = 360.0 / float(num_az_classes); 71 | azimuth = mstep * float(az_id) 72 | 73 | return azimuth 74 | 75 | 76 | 77 | -------------------------------------------------------------------------------- /RUNNING_MN3_ConvNetQuake9.txt: -------------------------------------------------------------------------------- 1 | #install 2 | see INSTALLATION.txt 3 | 4 | # activate environment 5 | source ./bin/activate; export PYTHONPATH=./:./quakenet_ingv 6 | #:/Users/anthony/opt/ConvNetQuake-master_tf1.0:$PYTHONPATH 7 | 8 | 9 | # get channel_file 10 | http://webservices.ingv.it/fdsnws/station/1/query?network=MN&channel=BH?&starttime=2009-01-01&endtime=2018-12-31&level=channel 11 | # save as: data/MN/MN_stations_2009-2018.xml 12 | 13 | # get train and validation events for circular distances centred on each station in channel_file 14 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2010-01-01 --endtime 2018-12-31 --channel_prefix BH --sampling_rate 20 --minradius 0.0 --maxradius 2.0 --minmagnitude 3.0 --event_files_path output/MN3/events 15 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2010-01-01 --endtime 2018-12-31 --channel_prefix BH --sampling_rate 20 --minradius 2.0 --maxradius 20.0 --minmagnitude 4.0 --event_files_path output/MN3/events 16 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2010-01-01 --endtime 2018-12-31 --channel_prefix BH --sampling_rate 20 --minradius 20.0 --maxradius 70.0 --minmagnitude 5.0 --event_files_path output/MN3/events 17 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2010-01-01 --endtime 2018-12-31 --channel_prefix BH --sampling_rate 20 --minradius 70.0 --maxradius 180.0 --minmagnitude 6.0 --event_files_path output/MN3/events 18 | 19 | python quakenet_ingv/bin/preprocess/get_streams.py --event_files_path output/MN3/events --outpath output/MN3/streams --base_url INGV --channel_file data/MN/MN_stations_2009-2018.xml --channel_prefix BH --sampling_rate 20 --n_streams 5000 --noise_fraction 0.4 --validation_fraction 0.1 --test_fraction 0.0 --snr_accept 3.0 --window_length 50 --window_start 5 --n_distances 50 --n_magnitudes 20 --n_depths 20 --n_azimuths 36 --event_fraction 0.25 20 | 21 | #train 22 | quakenet_ingv/bin/train --outpath output/MN3/streams --dataset output/MN3/streams/train --model ConvNetQuake9 --checkpoint_dir output/MN3/ConvNetQuake9 --use_magnitudes --use_depths --use_azimuths 23 | tensorboard --logdir output/MN3/ConvNetQuake9/ConvNetQuake9 24 | quakenet_ingv/bin/evaluate --outpath output/MN3/streams --model ConvNetQuake9 --checkpoint_dir output/MN3/ConvNetQuake9/ConvNetQuake9 --dataset output/MN3/streams/validate --eval_interval -1 --use_magnitudes --use_depths --use_azimuths --events 25 | 26 | 27 | # get test events for circular distances centred on each station in channel_file 28 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2007-01-01 --endtime 2009-12-31 --channel_prefix BH --sampling_rate 20 --minradius 0.0 --maxradius 2.0 --minmagnitude 4.0 --event_files_path output/MN/test_events 29 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2007-01-01 --endtime 2009-12-31 --channel_prefix BH --sampling_rate 20 --minradius 2.0 --maxradius 20.0 --minmagnitude 5.0 --event_files_path output/MN/test_events 30 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2007-01-01 --endtime 2009-12-31 --channel_prefix BH --sampling_rate 20 --minradius 20.0 --maxradius 50.0 --minmagnitude 6.0 --event_files_path output/MN/test_events 31 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2007-01-01 --endtime 2009-12-31 --channel_prefix BH --sampling_rate 20 --minradius 50.0 --maxradius 180.0 --minmagnitude 6.5 --event_files_path output/MN/test_events 32 | # 33 | python quakenet_ingv/bin/preprocess/get_streams.py --event_files_path output/MN/test_events --outpath output/MN3/test_streams --base_url INGV --channel_file data/MN/MN_stations_2009-2018.xml --channel_prefix BH --sampling_rate 20 --n_streams 999 --noise_fraction 0.0 --validation_fraction 0.0 --test_fraction 1.0 --snr_accept -1.0 --window_length 50 --window_start 5 --n_distances 50 --n_magnitudes 20 --n_depths 20 --n_azimuths 36 --systematic 34 | # 35 | quakenet_ingv/bin/evaluate --outpath output/MN3/test_streams --model ConvNetQuake9 --checkpoint_dir output/MN3/ConvNetQuake9/ConvNetQuake9_NNN --dataset output/MN3/test_streams_YEARS/test --eval_interval -1 --use_magnitudes --use_depths --use_azimuths --channel_file data/MN/MN_stations_2007-2018.xml --events --save_event_results --step NNN 36 | # NOTE: move event_results to event_results_YEARS 37 | # 38 | ./run_event_results_batch.bash output/MN3/ConvNetQuake9/ConvNetQuake9_NNN/event_results_YEARS 39 | 40 | quakenet_ingv/bin/plot_events --event_files_path output/MN3/streams/train/events/xml --channel_file data/MN/MN_stations_2007-2018.xml --channel_prefix BH --sampling_rate 20 --n_magnitudes 20 --n_depths 20 41 | quakenet_ingv/bin/plot_events --event_files_path output/MN3/test_streams_2009/test/events/xml --channel_file data/MN/MN_stations_2007-2018.xml --channel_prefix BH --sampling_rate 20 --n_magnitudes 20 --n_depths 20 42 | 43 | quakenet_ingv/bin/plot_event_stats --evaluate_event_results_path output/MN3/ConvNetQuake9/ConvNetQuake9_NNN/event_results_YEARS --train_events_path output/MN3/streams/train/events/xml 44 | 45 | 46 | -------------------------------------------------------------------------------- /quakenet_ingv/bin/train: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | # ------------------------------------------------------------------- 4 | # File: train.py 5 | # Author: Michael Gharbi 6 | # Created: 2016-10-25 7 | # ------------------------------------------------------------------- 8 | # 9 | # 10 | # 11 | # ------------------------------------------------------------------# 12 | """Train a model.""" 13 | 14 | import argparse 15 | import os 16 | import cPickle as pickle 17 | 18 | import tensorflow as tf 19 | import setproctitle 20 | 21 | import quakenet.models as models 22 | import quakenet.data_pipeline as dp 23 | import quakenet.config as config 24 | 25 | 26 | def main(args): 27 | setproctitle.setproctitle('quakenet') 28 | 29 | tf.set_random_seed(1234) 30 | 31 | # get streams parameters 32 | with open(os.path.join(args.outpath, 'params.pkl'), 'r') as file: 33 | stream_params = pickle.load(file) 34 | 35 | 36 | n_distances = stream_params.n_distances 37 | 38 | if args.use_magnitudes: 39 | n_magnitudes = stream_params.n_magnitudes 40 | else: 41 | n_magnitudes = 0 42 | 43 | if args.use_depths: 44 | n_depths = stream_params.n_depths 45 | else: 46 | n_depths = 0 47 | 48 | if args.use_azimuths: 49 | n_azimuths = stream_params.n_azimuths 50 | else: 51 | n_azimuths = 0 52 | 53 | cfg = config.Config() 54 | cfg.batch_size = args.batch_size 55 | cfg.regularization = args.regularization 56 | 57 | cfg.n_distances = n_distances 58 | cfg.n_distances += 1 59 | cfg.add = 1 60 | 61 | cfg.n_magnitudes = n_magnitudes 62 | cfg.n_depths = n_depths 63 | cfg.n_azimuths = n_azimuths 64 | 65 | cfg.unique_station = args.unique_station 66 | 67 | pos_path = os.path.join(args.dataset,"events") 68 | neg_path = os.path.join(args.dataset,"noise") 69 | 70 | # data pipeline for positive and negative examples 71 | pos_pipeline = dp.DataPipeline(pos_path, cfg, True) 72 | neg_pipeline = dp.DataPipeline(neg_path, cfg, True) 73 | 74 | pos_samples = { 75 | 'data': pos_pipeline.samples, 76 | 'stream_max': pos_pipeline.stream_max, 77 | 'distance_id': pos_pipeline.distance_id, 78 | 'magnitude_id': pos_pipeline.magnitude_id, 79 | 'depth_id': pos_pipeline.depth_id, 80 | 'azimuth_id': pos_pipeline.azimuth_id 81 | } 82 | neg_samples = { 83 | 'data': neg_pipeline.samples, 84 | 'stream_max': neg_pipeline.stream_max, 85 | 'distance_id': neg_pipeline.distance_id, 86 | 'magnitude_id': neg_pipeline.magnitude_id, 87 | 'depth_id': neg_pipeline.depth_id, 88 | 'azimuth_id': neg_pipeline.azimuth_id 89 | } 90 | 91 | # 20180319 AJL - Fix to support later tensorflow versions (e.g. 1.6.0) 92 | # samples = { 93 | # "data": tf.concat(0,[pos_samples["data"],neg_samples["data"]]), 94 | # "distance_id" : tf.concat(0,[pos_samples["distance_id"],neg_samples["distance_id"]]) 95 | # } 96 | samples = { 97 | "data": tf.concat([pos_samples["data"],neg_samples["data"]], 0), 98 | "stream_max": tf.concat([pos_samples["stream_max"],neg_samples["stream_max"]], 0), 99 | "distance_id" : tf.concat([pos_samples["distance_id"],neg_samples["distance_id"]], 0), 100 | "magnitude_id" : tf.concat([pos_samples["magnitude_id"],neg_samples["magnitude_id"]], 0), 101 | "depth_id" : tf.concat([pos_samples["depth_id"],neg_samples["depth_id"]], 0), 102 | "azimuth_id" : tf.concat([pos_samples["azimuth_id"],neg_samples["azimuth_id"]], 0) 103 | } 104 | 105 | # 20180521 AJL 106 | # slice data to requested number of points 107 | if args.ndatapoints > 0: 108 | samples['data'] = samples['data'][ : , 0:args.ndatapoints] 109 | 110 | # model 111 | model = models.get(args.model, samples, cfg, args.checkpoint_dir, is_training=True) 112 | 113 | # train loop 114 | model.train( 115 | args.learning_rate, 116 | resume=args.resume, 117 | profiling=args.profiling, 118 | summary_step=10) 119 | 120 | if __name__ == '__main__': 121 | parser = argparse.ArgumentParser() 122 | parser.add_argument('--model', type=str, default='ConvNetQuake') 123 | parser.add_argument('--outpath', type=str, help='Path for stream output') 124 | parser.add_argument('--checkpoint_dir', type=str, default='output/checkpoints') 125 | parser.add_argument('--dataset', type=str, default='data/hackathon/train') 126 | parser.add_argument('--unique_station', type=str, default=None) 127 | parser.add_argument('--ndatapoints', type=int, default=-1) 128 | parser.add_argument('--batch_size', type=int, default=64) 129 | parser.add_argument('--regularization', type=float, default=1e-3) 130 | parser.add_argument('--learning_rate', type=float, default=1e-4) 131 | parser.add_argument('--resume', action='store_true') 132 | parser.set_defaults(resume=False) 133 | parser.add_argument('--profiling', action='store_true') 134 | parser.add_argument('--use_magnitudes', action='store_true') 135 | parser.set_defaults(use_magnitudes=False) 136 | parser.add_argument('--use_depths', action='store_true') 137 | parser.set_defaults(use_depths=False) 138 | parser.add_argument('--use_azimuths', action='store_true') 139 | parser.set_defaults(use_azimuths=False) 140 | parser.set_defaults(profiling=False) 141 | args = parser.parse_args() 142 | 143 | args.checkpoint_dir = os.path.join(args.checkpoint_dir, args.model) 144 | 145 | main(args) 146 | -------------------------------------------------------------------------------- /quakenet_ingv/bin/preprocess/get_events.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 28 Mar 2018 3 | 4 | @author: Anthony Lomax - ALomax Scientific 5 | ''' 6 | from obspy.core.utcdatetime import UTCDateTime 7 | 8 | """Get events from FDSN web service.""" 9 | 10 | import os 11 | import argparse 12 | 13 | from obspy import read_inventory 14 | import obspy.clients.fdsn as fdsn 15 | 16 | 17 | def main(args): 18 | 19 | dirname = args.event_files_path 20 | if not os.path.exists(dirname): 21 | os.makedirs(dirname) 22 | 23 | # read list of channels to use 24 | inventory = read_inventory(args.channel_file) 25 | inventory = inventory.select(channel=args.channel_prefix+'Z', sampling_rate=args.sampling_rate) 26 | 27 | depth_str = '' if (args.mindepth == None and args.maxdepth == None) else \ 28 | (str(args.mindepth) if args.mindepth != None else (str(-999.9) + '-' + str(args.maxdepth) if args.maxdepth != None else '')) + 'km_' 29 | filename_base = '' \ 30 | + (str(args.minradius) if args.minradius != None else str(0.0)) + '-' + str(args.maxradius) + 'deg_' \ 31 | + depth_str \ 32 | + 'M' + str(args.minmagnitude) + '-' \ 33 | + (str(args.maxmagnitude) if args.maxmagnitude != None else '') 34 | 35 | print 'filename_base', filename_base 36 | 37 | events_starttime = UTCDateTime(args.starttime) 38 | events_endtime = UTCDateTime(args.endtime) 39 | print 'events_starttime', events_starttime 40 | print 'events_endtime', events_endtime 41 | 42 | for net in inventory: 43 | for sta in net: 44 | outfile = os.path.join(args.event_files_path, net.code + '_' + sta.code + '_' \ 45 | + filename_base + '.xml') 46 | client = fdsn.Client(args.base_url) 47 | print 'net_sta', net.code + '_' + sta.code 48 | print 'sta.start_date', sta.start_date 49 | print 'sta.end_date', sta.end_date 50 | tstart = sta.start_date if sta.start_date > events_starttime else events_starttime 51 | tend = sta.end_date if sta.end_date < events_endtime else events_endtime 52 | print 'tstart', tstart 53 | print 'tend', tend 54 | if not tstart < tend: 55 | continue 56 | try: 57 | catalog = client.get_events(latitude=sta.latitude, longitude=sta.longitude, \ 58 | starttime=tstart, endtime=tend, \ 59 | minradius=args.minradius, maxradius=args.maxradius, \ 60 | mindepth=args.mindepth, maxdepth=args.maxdepth, \ 61 | minmagnitude=args.minmagnitude, maxmagnitude=args.maxmagnitude, \ 62 | includeallorigins=False, includeallmagnitudes= False, includearrivals=False) 63 | except Exception as ex: 64 | print 'Skipping net:', net.code, 'sta:', sta.code, 'Exception:', ex, 65 | continue 66 | #, filename=args.outfile) 67 | catalog.write(outfile, 'QUAKEML') 68 | print catalog.count(), 'events:', 'written to:', outfile 69 | #catalog.plot(outfile=outfile + '.png') 70 | 71 | 72 | if __name__ == '__main__': 73 | 74 | parser = argparse.ArgumentParser() 75 | 76 | parser.add_argument('--base_url', type=str, default='IRIS', 77 | help="Base URL of FDSN web service or key string for recognized server") 78 | parser.add_argument('--starttime', type=str, default=None, 79 | help='Limit to events on or after the specified start time') 80 | parser.add_argument('--endtime', type=str, default=None, 81 | help='Limit to events on or before the specified end time') 82 | parser.add_argument('--channel_file', type=str, 83 | help='File containing FDSNStationXML list of net/station/location/channel to retrieve') 84 | parser.add_argument('--channel_prefix', type=str, 85 | help='Prefix of the channel code to retrieve (e.g. BH)') 86 | parser.add_argument('--sampling_rate', type=float, 87 | help='Channel sample rate to retrieve (e.g. 20)') 88 | parser.add_argument('--minradius', type=float, default=None, 89 | help='Limit to events within the specified minimum number of degrees from the geographic point defined by the latitude and longitude parameters') 90 | parser.add_argument('--maxradius', type=float, 91 | help='Limit to events within the specified maximum number of degrees from the geographic point defined by the latitude and longitude parameters') 92 | parser.add_argument('--mindepth', type=float, default=None, 93 | help='Limit to events with depth, in kilometers, larger than the specified minimum') 94 | parser.add_argument('--maxdepth', type=float, default=None, 95 | help='Limit to events with depth, in kilometers, larger than the specified maximum') 96 | parser.add_argument('--minmagnitude', type=float, 97 | help='Limit to events with a magnitude larger than the specified minimum') 98 | parser.add_argument('--maxmagnitude', type=float, default=None, 99 | help='Limit to events with a magnitude larger than the specified maximum') 100 | parser.add_argument('--event_files_path', type=str, 101 | help='Path to the directory to write event QuakeML event xml files') 102 | 103 | args = parser.parse_args() 104 | 105 | main(args) 106 | -------------------------------------------------------------------------------- /quakenet_ingv/bin/plot_events: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | # ------------------------------------------------------------------- 4 | # File: plot_event_results 5 | # Author: Anthony Lomax 6 | # Created: 2018-07-23 7 | # ------------------------------------------------------------------# 8 | from _pytest.runner import Skipped 9 | 10 | """ Plot events on an interactive map""" 11 | 12 | # import sys 13 | import argparse 14 | import os 15 | 16 | from obspy.core import UTCDateTime 17 | from obspy.core import event 18 | from obspy import read_inventory 19 | 20 | import json 21 | import pandas as pd 22 | import numpy as np 23 | import matplotlib.pyplot as plt 24 | import matplotlib as mpl 25 | import folium 26 | from folium.plugins import HeatMap 27 | #from folium.plugins import MarkerCluster 28 | 29 | import quakenet.util as util 30 | import quakenet.plot as qnplot 31 | 32 | 33 | 34 | def main(args): 35 | 36 | # get event files 37 | feature_group_events = folium.map.FeatureGroup(name=args.event_files_path, overlay=True, control=True) 38 | heat_map_data = [] 39 | total_prob_magnitude = [] 40 | total_prob_depth = [] 41 | event_xml_file = None 42 | processed_origin_times=set() 43 | have_skipped=False 44 | for root, dirs, files in os.walk(args.event_files_path): 45 | for f in files: 46 | if f.endswith('.xml'): 47 | event_xml_file = f 48 | catalog = event.read_events(os.path.join(args.event_files_path, event_xml_file)) 49 | for evt in catalog: 50 | origin = evt.preferred_origin() 51 | origin_time_isoformat = origin.time.isoformat() 52 | if origin_time_isoformat in processed_origin_times: 53 | print '.', 54 | have_skipped = True 55 | continue 56 | if have_skipped: 57 | print '' 58 | have_skipped=False 59 | event_preferred_magnitude = evt.preferred_magnitude().mag 60 | print origin_time_isoformat + ' M=' + str(event_preferred_magnitude) + ' ' + evt.event_descriptions[0].text 61 | processed_origin_times.add(origin_time_isoformat) 62 | origin_str = 'M=' + str(event_preferred_magnitude) + ' ' + evt.event_descriptions[0].text + '\n' 63 | origin_str += ' '+ origin_time_isoformat + '\n' 64 | origin_str += ' '+ 'Lat:' + str(origin.latitude) + ' Lon:' + str(origin.longitude) + ' Depth:' + str(origin.depth / 1000.0) + 'km' + '\n' 65 | marker = folium.CircleMarker([origin.latitude, origin.longitude], \ 66 | radius=3, popup=origin_str, stroke=True, weight=1, color='red', fill=True, fillColor='red', fillOpacity=1.0) 67 | feature_group_events.add_child(marker) 68 | # accumulate heatmap points 69 | heat_map_data.append([origin.latitude, origin.longitude]) 70 | # accumulate magnitude 71 | prob_magnitude = [0] * args.n_magnitudes 72 | prob_magnitude[util.magntiude2classification(event_preferred_magnitude, args.n_magnitudes)] = 1.0 73 | total_prob_magnitude.append(prob_magnitude) 74 | # accumulate depth 75 | prob_depth = [0] * args.n_depths 76 | prob_depth[util.depth2classification(origin.depth / 1000.0, args.n_depths)] = 1.0 77 | total_prob_depth.append(prob_depth) 78 | 79 | if have_skipped: 80 | print '' 81 | 82 | events_heat_map = HeatMap(heat_map_data, name="Events heatmap", 83 | min_opacity=0.5, max_zoom=18, max_val=1.0, radius=10, blur=10, gradient={0.2: 'blue', 0.6: 'lime', 0.9: 'yellow'}, overlay=True) 84 | #min_opacity=0.5, max_zoom=18, max_val=1.0, radius=25, blur=15, gradient=None, overlay=True) 85 | 86 | # read list of channels to use 87 | inventory = read_inventory(args.channel_file) 88 | inventory = inventory.select(channel=args.channel_prefix+'Z', sampling_rate=args.sampling_rate) 89 | feature_group_stations = folium.map.FeatureGroup(name=args.channel_file, overlay=True, control=True) 90 | for net in inventory: 91 | for sta in net: 92 | net_sta = net.code + '_' + sta.code 93 | print 'net_sta', net_sta 94 | marker = folium.CircleMarker([sta.latitude, sta.longitude], \ 95 | radius=1, popup=net_sta, stroke=True, weight=2, color='black') 96 | feature_group_stations.add_child(marker) 97 | 98 | #folium_map = folium.Map(tiles='OpenStreetMap', attr='OpenStreetMap attribution', \ 99 | # prefer_canvas=True, control_scale=True, zoom_start = 4, location=[origin.latitude, origin.longitude]) 100 | folium_map = folium.Map(tiles='OpenStreetMap', attr='OpenStreetMap attribution', \ 101 | prefer_canvas=True, control_scale=True, zoom_start = 2, location=[0.0, 15.0]) 102 | folium_map.add_child(feature_group_events) 103 | folium_map.add_child(feature_group_stations) 104 | folium_map.add_child(events_heat_map) 105 | folium.LayerControl().add_to(folium_map) 106 | htmlfile = os.path.join(args.event_files_path, 'events.html') 107 | folium_map.save(htmlfile) 108 | print 'HTML output written to' 109 | print htmlfile 110 | 111 | # magnitude bar chart 112 | v_axis_direction = -1 113 | qnplot.write_bar_chart_html('Magnitude Distribution', '', "Relative frequency", None, os.path.join(args.event_files_path, 'magnitudes.html'), \ 114 | total_prob_magnitude, 0.0, util.classification2magnitude, 115 | v_axis_direction, normalize=True) 116 | 117 | # depth bar chart 118 | v_axis_direction = -1 119 | qnplot.write_bar_chart_html('Depth Distribution', '', "Relative frequency", None, os.path.join(args.event_files_path, 'depths.html'), \ 120 | total_prob_depth, 0.0, util.classification2depth, 121 | v_axis_direction, normalize=True) 122 | 123 | 124 | 125 | if __name__ == '__main__': 126 | 127 | parser = argparse.ArgumentParser() 128 | parser.add_argument('--event_files_path', type=str, 129 | help='Path for event files input') 130 | parser.add_argument('--channel_file', type=str, 131 | help='File containing FDSNStationXML list of net/station/location/channel to retrieve') 132 | parser.add_argument('--channel_prefix', type=str, 133 | help='Prefix of the channel code to retrieve (e.g. BH)') 134 | parser.add_argument('--sampling_rate', type=float, 135 | help='Channel sample rate to retrieve (e.g. 20)') 136 | parser.add_argument('--n_magnitudes', type=int, default=None, 137 | help='Number of magnitude classes, 0=none') 138 | parser.add_argument('--n_depths', type=int, default=None, 139 | help='Number of depth classes, 0=none') 140 | 141 | args = parser.parse_args() 142 | main(args) 143 | -------------------------------------------------------------------------------- /quakenet_ingv/quakenet/data_pipeline.py: -------------------------------------------------------------------------------- 1 | """Classes and functions to read, write and feed data.""" 2 | 3 | import os 4 | import re 5 | import numpy as np 6 | import tensorflow as tf 7 | from tqdm import tqdm 8 | import csv 9 | import json 10 | from obspy.core.utcdatetime import UTCDateTime 11 | 12 | POSITIVE_EXAMPLES_PATH = 'positive' 13 | NEGATIVE_EXAMPLES_PATH = 'negative' 14 | # RECORD_REGEXP = re.compile(r'\d+\.tfrecords') 15 | 16 | 17 | class DataWriter(object): 18 | 19 | """ Writes .tfrecords file to disk from window Stream objects. 20 | """ 21 | 22 | def __init__(self, filename): 23 | self._writer = None 24 | self._filename = filename 25 | self._written = 0 26 | self._writer = tf.python_io.TFRecordWriter(self._filename) 27 | 28 | def write(self, sample_window, stream_max, distance_id, magnitude_id, depth_id, azimuth_id, distance, magnitude, depth, azimuth): 29 | n_traces = len(sample_window) 30 | n_samples = len(sample_window[0].data) 31 | # Extract data 32 | data = np.zeros((n_traces, n_samples), dtype=np.float32) 33 | for i in range(n_traces): 34 | data[i, :] = sample_window[i].data[...] 35 | # Extract metadata 36 | start_time = np.int64(sample_window[0].stats.starttime.timestamp) 37 | end_time = np.int64(sample_window[0].stats.endtime.timestamp) 38 | # print('starttime {}, endtime {}'.format(UTCDateTime(start_time), 39 | # UTCDateTime(end_time))) 40 | 41 | example = tf.train.Example(features=tf.train.Features(feature={ 42 | 'window_size': self._int64_feature(n_samples), 43 | 'n_traces': self._int64_feature(n_traces), 44 | 'data': self._bytes_feature(data.tobytes()), 45 | 'stream_max': self._float_feature(stream_max), 46 | 'distance_id': self._int64_feature(distance_id), 47 | 'magnitude_id': self._int64_feature(magnitude_id), 48 | 'depth_id': self._int64_feature(depth_id), 49 | 'azimuth_id': self._int64_feature(azimuth_id), 50 | 'distance': self._float_feature(distance), 51 | 'magnitude': self._float_feature(magnitude), 52 | 'depth': self._float_feature(depth), 53 | 'azimuth': self._float_feature(azimuth), 54 | 'start_time': self._int64_feature(start_time), 55 | 'end_time': self._int64_feature(end_time), 56 | })) 57 | self._writer.write(example.SerializeToString()) 58 | self._written += 1 59 | 60 | def close(self): 61 | self._writer.close() 62 | 63 | def _int64_feature(self, value): 64 | return tf.train.Feature(int64_list=tf.train.Int64List(value=[value])) 65 | 66 | def _float_feature(self, value): 67 | return tf.train.Feature(float_list=tf.train.FloatList(value=[value])) 68 | 69 | def _bytes_feature(self, value): 70 | return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) 71 | 72 | 73 | class DataReader(object): 74 | 75 | def __init__(self, path, config, shuffle=True): 76 | self._path = path 77 | self._shuffle = shuffle 78 | self._config = config 79 | self.win_size = config.win_size 80 | self.n_traces = config.n_traces 81 | 82 | 83 | self._reader = tf.TFRecordReader() 84 | 85 | def read(self): 86 | filename_queue = self._filename_queue() 87 | example_key, serialized_example = self._reader.read(filename_queue) 88 | example = self._parse_example(example_key, serialized_example) 89 | return example 90 | 91 | def _filename_queue(self): 92 | fnames = [] 93 | for root, dirs, files in os.walk(self._path): 94 | for f in files: 95 | if f.endswith(".tfrecords"): 96 | if self._config.unique_station is None or self._config.unique_station in f: 97 | fnames.append(os.path.join(root, f)) 98 | fname_q = tf.train.string_input_producer(fnames, 99 | shuffle=self._shuffle, 100 | num_epochs=self._config.n_epochs) 101 | return fname_q 102 | 103 | def _parse_example(self, example_key, serialized_example): 104 | features = tf.parse_single_example( 105 | serialized_example, 106 | features={ 107 | 'window_size': tf.FixedLenFeature([], tf.int64), 108 | 'n_traces': tf.FixedLenFeature([], tf.int64), 109 | 'data': tf.FixedLenFeature([], tf.string), 110 | 'stream_max': tf.FixedLenFeature([], tf.float32), 111 | 'distance_id': tf.FixedLenFeature([], tf.int64), 112 | 'magnitude_id': tf.FixedLenFeature([], tf.int64), 113 | 'depth_id': tf.FixedLenFeature([], tf.int64), 114 | 'azimuth_id': tf.FixedLenFeature([], tf.int64), 115 | 'distance': tf.FixedLenFeature([], tf.float32), 116 | 'magnitude': tf.FixedLenFeature([], tf.float32), 117 | 'depth': tf.FixedLenFeature([], tf.float32), 118 | 'azimuth': tf.FixedLenFeature([], tf.float32), 119 | 'start_time': tf.FixedLenFeature([],tf.int64), 120 | 'end_time': tf.FixedLenFeature([], tf.int64)}) 121 | 122 | # Convert and reshape 123 | data = tf.decode_raw(features['data'], tf.float32) 124 | data.set_shape([self.n_traces * self.win_size]) 125 | data = tf.reshape(data, [self.n_traces, self.win_size]) 126 | data = tf.transpose(data, [1, 0]) 127 | 128 | # Pack 129 | features['data'] = data 130 | 131 | features['name'] = example_key 132 | 133 | return features 134 | 135 | 136 | class DataPipeline(object): 137 | 138 | """Creates a queue op to stream data for training. 139 | 140 | Attributes: 141 | samples: Tensor(float). batch of input samples [batch_size, n_channels, n_points] 142 | labels: Tensor(int32). Corresponding batch 0 or 1 labels, [batch_size,] 143 | 144 | """ 145 | 146 | def __init__(self, dataset_path, config, is_training): 147 | 148 | min_after_dequeue = 1000 149 | capacity = 1000 + 3 * config.batch_size 150 | 151 | if is_training: 152 | 153 | with tf.name_scope('inputs'): 154 | self._reader = DataReader(dataset_path, config=config) 155 | samples = self._reader.read() 156 | tensors = [samples['name'], samples['data'], samples['stream_max'], samples["distance_id"], samples["magnitude_id"], samples["depth_id"], samples["azimuth_id"]] 157 | 158 | self.names, self.samples, self.stream_max, self.distance_id, self.magnitude_id, self.depth_id, self.azimuth_id = tf.train.shuffle_batch( 159 | tensors, 160 | batch_size=config.batch_size, 161 | capacity=capacity, 162 | min_after_dequeue=min_after_dequeue, 163 | allow_smaller_final_batch=False) 164 | 165 | elif not is_training: 166 | 167 | with tf.name_scope('validation_inputs'): 168 | self._reader = DataReader(dataset_path, config=config) 169 | samples = self._reader.read() 170 | 171 | tensors = [samples['name'], samples['data'], samples['stream_max'], 172 | samples["distance_id"], samples["magnitude_id"], samples["depth_id"], samples["azimuth_id"], 173 | samples["distance"], samples["magnitude"], samples["depth"], samples["azimuth"], 174 | samples["start_time"], samples["end_time"]] 175 | 176 | self.names, self.samples, self.stream_max, self.distance_id, self.magnitude_id, self.depth_id, self.azimuth_id, \ 177 | self.distance, self.magnitude, self.depth, self.azimuth, self.start_time, self.end_time = tf.train.batch( 178 | tensors, 179 | batch_size=config.batch_size, 180 | capacity=capacity, 181 | num_threads=config.n_threads, 182 | allow_smaller_final_batch=False) 183 | else: 184 | raise ValueError( 185 | "is_training flag is not defined, set True for training and False for testing") 186 | 187 | -------------------------------------------------------------------------------- /quakenet_ingv/bin/preprocess/convert_streams.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 29 Mar 2018 3 | 4 | @author: Anthony Lomax - ALomax Scientific 5 | ''' 6 | from h5py._hl import dataset 7 | 8 | """Get streams from FDSN web service.""" 9 | 10 | import os 11 | import argparse 12 | import cPickle as pickle 13 | 14 | from shutil import copy 15 | 16 | import numpy as np 17 | 18 | from obspy import read 19 | 20 | from quakenet.data_pipeline import DataWriter 21 | 22 | import tensorflow as tf 23 | 24 | 25 | 26 | 27 | 28 | def main(args): 29 | 30 | 31 | if not os.path.exists(args.outroot): 32 | os.makedirs(args.outroot) 33 | 34 | # copy some files 35 | copy(os.path.join(args.inroot, 'params.pkl'), args.outroot) 36 | copy(os.path.join(args.inroot, 'event_channel_dict.pkl'), args.outroot) 37 | 38 | if not os.path.exists(args.outroot): 39 | os.makedirs(args.outroot) 40 | 41 | for dataset in ['train', 'validate', 'test']: 42 | for datatype in ['events', 'noise']: 43 | inpath = os.path.join(args.inroot, dataset, datatype) 44 | outpath = os.path.join(args.outroot, dataset, datatype) 45 | if not os.path.exists(outpath): 46 | os.makedirs(outpath) 47 | mseedpath = os.path.join(outpath, 'mseed') 48 | if not os.path.exists(mseedpath): 49 | os.makedirs(mseedpath) 50 | mseedpath = os.path.join(outpath, 'mseed_raw') 51 | if not os.path.exists(mseedpath): 52 | os.makedirs(mseedpath) 53 | if datatype == 'events': 54 | xmlpath = os.path.join(outpath, 'xml') 55 | if not os.path.exists(xmlpath): 56 | os.makedirs(xmlpath) 57 | 58 | # inroot example: output/MN/streams 59 | # inpath example: output/MN/streams/train/events 60 | for dirpath, dirnames, filenames in os.walk(inpath): 61 | for name in filenames: 62 | if name.endswith(".tfrecords"): 63 | filename_root = name.replace('.tfrecords', '') 64 | print 'Processing:', name, os.path.join(outpath, filename_root + '.tfrecords') 65 | 66 | # copy some files 67 | copy(os.path.join(inpath, 'mseed_raw', filename_root + '.mseed'), os.path.join(outpath, 'mseed_raw')) 68 | if datatype == 'events': 69 | copy(os.path.join(inpath, 'xml', filename_root + '.xml'), os.path.join(outpath, 'xml')) 70 | 71 | # read raw mseed 72 | stream = read(os.path.join(inpath, 'mseed_raw', filename_root + '.mseed'), format='MSEED') 73 | # store absolute maximum 74 | stream_max = np.absolute(stream.max()).max() 75 | # normalize by absolute maximum 76 | stream.normalize(global_max = True) 77 | # write new processed miniseed 78 | streamfile = os.path.join(outpath, 'mseed', filename_root + '.mseed') 79 | stream.write(streamfile, format='MSEED', encoding='FLOAT32') 80 | 81 | n_traces = 3 82 | win_size = 10001 83 | 84 | # read old tfrecords 85 | # https://www.kaggle.com/mpekalski/reading-tfrecord 86 | record_iterator = tf.python_io.tf_record_iterator(path=os.path.join(inpath, filename_root + '.tfrecords')) 87 | for string_record in record_iterator: 88 | example = tf.train.Example() 89 | example.ParseFromString(string_record) 90 | distance_id = int(example.features.feature['distance_id'].int64_list.value[0]) 91 | magnitude_id = int(example.features.feature['magnitude_id'].int64_list.value[0]) 92 | depth_id = int(example.features.feature['depth_id'].int64_list.value[0]) 93 | azimuth_id = int(example.features.feature['azimuth_id'].int64_list.value[0]) 94 | distance = float(example.features.feature['distance'].float_list.value[0]) 95 | magnitude = float(example.features.feature['magnitude'].float_list.value[0]) 96 | depth = float(example.features.feature['depth'].float_list.value[0]) 97 | azimuth = float(example.features.feature['azimuth'].float_list.value[0]) 98 | print 'id', distance_id, 'im', magnitude_id, 'ide', depth_id, 'iaz', azimuth_id, \ 99 | 'd', distance, 'm', magnitude, 'de', depth, 'az', azimuth 100 | 101 | # filename_queue = tf.train.string_input_producer([os.path.join(inpath, filename_root + '.tfrecords')], shuffle=False) 102 | # reader = tf.TFRecordReader() 103 | # example_key, serialized_example = reader.read(filename_queue) 104 | # # data_pipeline._parse_example() 105 | # features = tf.parse_single_example( 106 | # serialized_example, 107 | # features={ 108 | # 'window_size': tf.FixedLenFeature([], tf.int64), 109 | # 'n_traces': tf.FixedLenFeature([], tf.int64), 110 | # 'data': tf.FixedLenFeature([], tf.string), 111 | # #'stream_max': tf.FixedLenFeature([], tf.float32), 112 | # 'distance_id': tf.FixedLenFeature([], tf.int64), 113 | # 'magnitude_id': tf.FixedLenFeature([], tf.int64), 114 | # 'depth_id': tf.FixedLenFeature([], tf.int64), 115 | # 'azimuth_id': tf.FixedLenFeature([], tf.int64), 116 | # 'distance': tf.FixedLenFeature([], tf.float32), 117 | # 'magnitude': tf.FixedLenFeature([], tf.float32), 118 | # 'depth': tf.FixedLenFeature([], tf.float32), 119 | # 'azimuth': tf.FixedLenFeature([], tf.float32), 120 | # 'start_time': tf.FixedLenFeature([],tf.int64), 121 | # 'end_time': tf.FixedLenFeature([], tf.int64)}) 122 | # features['name'] = example_key 123 | # # END - data_pipeline._parse_example() 124 | # 125 | # print "features['distance_id']", features['distance_id'] 126 | # print 'distance_id shape', tf.shape(features['distance_id']) 127 | # with tf.Session() as sess: 128 | # print sess.run(features['distance_id']) 129 | # #print 'distance_id', distance_id 130 | # #print 'distance_id shape', tf.shape(distance_id) 131 | # magnitude_id = features['magnitude_id'] 132 | # depth_id = features['depth_id'] 133 | # azimuth_id = features['azimuth_id'] 134 | # distance = features['distance'] 135 | # magnitude = features['magnitude'] 136 | # depth = features['depth'] 137 | # azimuth = features['azimuth'] 138 | 139 | # write new tfrecords 140 | writer = DataWriter(os.path.join(outpath, filename_root + '.tfrecords')) 141 | writer.write(stream, stream_max, distance_id, magnitude_id, depth_id, azimuth_id, distance, magnitude, depth, azimuth) 142 | 143 | 144 | 145 | 146 | if __name__ == '__main__': 147 | 148 | parser = argparse.ArgumentParser() 149 | 150 | parser.add_argument('--inroot', type=str, 151 | help='Path for input') 152 | parser.add_argument('--outroot', type=str, 153 | help='Path for output') 154 | 155 | 156 | args = parser.parse_args() 157 | 158 | main(args) 159 | -------------------------------------------------------------------------------- /quakenet_ingv/tflib/layers.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | 4 | def conv(inputs, 5 | nfilters, 6 | ksize, 7 | stride=1, 8 | padding='SAME', 9 | use_bias=True, 10 | activation_fn=tf.nn.relu, 11 | initializer=tf.contrib.layers.variance_scaling_initializer(), 12 | regularizer=None, 13 | scope=None, 14 | reuse=None): 15 | with tf.variable_scope(scope, reuse=reuse): 16 | n_in = inputs.get_shape().as_list()[-1] 17 | weights = tf.get_variable( 18 | 'weights', 19 | shape=[ksize, ksize, n_in, nfilters], 20 | dtype=inputs.dtype.base_dtype, 21 | initializer=initializer, 22 | collections=[tf.GraphKeys.WEIGHTS, tf.GraphKeys.GLOBAL_VARIABLES], 23 | regularizer=regularizer) 24 | 25 | strides = [1, stride, stride, 1] 26 | current_layer = tf.nn.conv2d(inputs, weights, strides, padding=padding) 27 | 28 | if use_bias: 29 | biases = tf.get_variable( 30 | 'biases', 31 | shape=[nfilters,], 32 | dtype=inputs.dtype.base_dtype, 33 | initializer=tf.constant_initializer(0.0), 34 | collections=[tf.GraphKeys.BIASES, tf.GraphKeys.GLOBAL_VARIABLES]) 35 | current_layer = tf.nn.bias_add(current_layer, biases) 36 | 37 | if activation_fn is not None: 38 | current_layer = activation_fn(current_layer) 39 | 40 | return current_layer 41 | 42 | def transpose_conv(inputs, 43 | nfilters, 44 | ksize, 45 | stride=1, 46 | padding='SAME', 47 | use_bias=True, 48 | activation_fn=tf.nn.relu, 49 | initializer=tf.contrib.layers.variance_scaling_initializer(), 50 | regularizer=None, 51 | scope=None, 52 | reuse=None): 53 | with tf.variable_scope(scope, reuse=reuse): 54 | n_in = inputs.get_shape().as_list()[-1] 55 | weights = tf.get_variable( 56 | 'weights', 57 | shape=[ksize, ksize, n_in, nfilters], 58 | dtype=inputs.dtype.base_dtype, 59 | initializer=initializer, 60 | collections=[tf.GraphKeys.WEIGHTS, tf.GraphKeys.GLOBAL_VARIABLES], 61 | regularizer=regularizer) 62 | 63 | bs, h, w, c = inputs.get_shape().as_list() 64 | strides = [1, stride, stride, 1] 65 | out_shape = [bs, stride*h, stride*w, c] 66 | current_layer = tf.nn.conv2d_transpose(inputs, weights, out_shape, strides, padding=padding) 67 | 68 | if use_bias: 69 | biases = tf.get_variable( 70 | 'biases', 71 | shape=[nfilters,], 72 | dtype=inputs.dtype.base_dtype, 73 | initializer=tf.constant_initializer(0.0), 74 | collections=[tf.GraphKeys.BIASES, tf.GraphKeys.GLOBAL_VARIABLES]) 75 | current_layer = tf.nn.bias_add(current_layer, biases) 76 | 77 | if activation_fn is not None: 78 | current_layer = activation_fn(current_layer) 79 | 80 | return current_layer 81 | 82 | def conv1(inputs, 83 | nfilters, 84 | ksize, 85 | stride=1, 86 | padding='SAME', 87 | use_bias=True, 88 | activation_fn=tf.nn.relu, 89 | initializer=tf.contrib.layers.variance_scaling_initializer(), 90 | regularizer=None, 91 | scope=None, 92 | reuse=None): 93 | with tf.variable_scope(scope, reuse=reuse): 94 | n_in = inputs.get_shape().as_list()[-1] 95 | weights = tf.get_variable( 96 | 'weights', 97 | shape=[ksize, n_in, nfilters], 98 | dtype=inputs.dtype.base_dtype, 99 | initializer=initializer, 100 | collections=[tf.GraphKeys.WEIGHTS, tf.GraphKeys.GLOBAL_VARIABLES], 101 | regularizer=regularizer) 102 | 103 | current_layer = tf.nn.conv1d(inputs, weights, stride, padding=padding) 104 | 105 | if use_bias: 106 | biases = tf.get_variable( 107 | 'biases', 108 | shape=[nfilters,], 109 | dtype=inputs.dtype.base_dtype, 110 | initializer=tf.constant_initializer(0.0), 111 | collections=[tf.GraphKeys.BIASES, tf.GraphKeys.GLOBAL_VARIABLES]) 112 | current_layer = tf.nn.bias_add(current_layer, biases) 113 | 114 | if activation_fn is not None: 115 | current_layer = activation_fn(current_layer) 116 | 117 | return current_layer 118 | 119 | def atrous_conv1d(inputs, 120 | nfilters, 121 | ksize, 122 | rate=1, 123 | padding='SAME', 124 | use_bias=True, 125 | activation_fn=tf.nn.relu, 126 | initializer=tf.contrib.layers.variance_scaling_initializer(), 127 | regularizer=None, 128 | scope=None, 129 | reuse=None): 130 | """ Use tf.nn.atrous_conv2d and adapt to 1d""" 131 | with tf.variable_scope(scope, reuse=reuse): 132 | # from (bs,width,c) to (bs,width,1,c) 133 | inputs = tf.expand_dims(inputs,2) 134 | 135 | n_in = inputs.get_shape().as_list()[-1] 136 | weights = tf.get_variable( 137 | 'weights', 138 | shape=[ksize, 1, n_in, nfilters], 139 | dtype=inputs.dtype.base_dtype, 140 | initializer=initializer, 141 | collections=[tf.GraphKeys.WEIGHTS, tf.GraphKeys.GLOBAL_VARIABLES], 142 | regularizer=regularizer) 143 | 144 | current_layer = tf.nn.atrous_conv2d(inputs,weights, rate, padding=padding) 145 | 146 | # Resize into (bs,width,c) 147 | current_layer = tf.squeeze(current_layer,axis=[2]) 148 | 149 | if use_bias: 150 | biases = tf.get_variable( 151 | 'biases', 152 | shape=[nfilters,], 153 | dtype=inputs.dtype.base_dtype, 154 | initializer=tf.constant_initializer(0.0), 155 | collections=[tf.GraphKeys.BIASES, tf.GraphKeys.GLOBAL_VARIABLES]) 156 | current_layer = tf.nn.bias_add(current_layer, biases) 157 | 158 | if activation_fn is not None: 159 | current_layer = activation_fn(current_layer) 160 | 161 | return current_layer 162 | 163 | 164 | 165 | def conv3(inputs, 166 | nfilters, 167 | ksize, 168 | stride=1, 169 | padding='SAME', 170 | use_bias=True, 171 | activation_fn=tf.nn.relu, 172 | initializer=tf.contrib.layers.variance_scaling_initializer(), 173 | regularizer=None, 174 | scope=None, 175 | reuse=None): 176 | with tf.variable_scope(scope, reuse=reuse): 177 | n_in = inputs.get_shape().as_list()[-1] 178 | weights = tf.get_variable( 179 | 'weights', 180 | shape=[ksize, ksize, ksize, n_in, nfilters], 181 | dtype=inputs.dtype.base_dtype, 182 | initializer=initializer, 183 | collections=[tf.GraphKeys.WEIGHTS, tf.GraphKeys.GLOBAL_VARIABLES], 184 | regularizer=regularizer) 185 | 186 | strides = [1, stride, stride, stride, 1] 187 | current_layer = tf.nn.conv3d(inputs, weights, strides, padding=padding) 188 | 189 | if use_bias: 190 | biases = tf.get_variable( 191 | 'biases', 192 | shape=[nfilters,], 193 | dtype=inputs.dtype.base_dtype, 194 | initializer=tf.constant_initializer(0.0), 195 | collections=[tf.GraphKeys.BIASES, tf.GraphKeys.GLOBAL_VARIABLES]) 196 | current_layer = tf.nn.bias_add(current_layer, biases) 197 | 198 | if activation_fn is not None: 199 | current_layer = activation_fn(current_layer) 200 | 201 | return current_layer 202 | 203 | 204 | def fc(inputs, nfilters, use_bias=True, activation_fn=tf.nn.relu, 205 | initializer=tf.contrib.layers.variance_scaling_initializer(), 206 | regularizer=None, scope=None, reuse=None): 207 | with tf.variable_scope(scope, reuse=reuse): 208 | n_in = inputs.get_shape().as_list()[-1] 209 | weights = tf.get_variable( 210 | 'weights', 211 | shape=[n_in, nfilters], 212 | dtype=inputs.dtype.base_dtype, 213 | initializer=initializer, 214 | regularizer=regularizer) 215 | 216 | current_layer = tf.matmul(inputs, weights) 217 | 218 | if use_bias: 219 | biases = tf.get_variable( 220 | 'biases', 221 | shape=[nfilters,], 222 | dtype=inputs.dtype.base_dtype, 223 | initializer=tf.constant_initializer(0)) 224 | current_layer = tf.nn.bias_add(current_layer, biases) 225 | 226 | if activation_fn is not None: 227 | current_layer = activation_fn(current_layer) 228 | 229 | return current_layer 230 | 231 | def batch_norm(inputs, center=False, scale=False, 232 | decay=0.999, epsilon=0.001, reuse=None, 233 | scope=None, is_training=False): 234 | return tf.contrib.layers.batch_norm( 235 | inputs, center=center, scale=scale, 236 | decay=decay, epsilon=epsilon, activation_fn=None, 237 | reuse=reuse,trainable=False, scope=scope, is_training=is_training) 238 | 239 | relu = tf.nn.relu 240 | 241 | def crop_like(inputs, like, name=None): 242 | with tf.name_scope(name): 243 | _, h, w, _ = inputs.get_shape().as_list() 244 | _, new_h, new_w, _ = like.get_shape().as_list() 245 | crop_h = (h-new_h)/2 246 | crop_w = (w-new_w)/2 247 | cropped = inputs[:, crop_h:crop_h+new_h, crop_w:crop_w+new_w, :] 248 | return cropped 249 | -------------------------------------------------------------------------------- /quakenet_ingv/quakenet/plot.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 23 July 2018 3 | 4 | @author: anthony 5 | ''' 6 | 7 | import math 8 | 9 | 10 | def point_color(red, green, blue): 11 | 12 | hexstr = "#{0:02x}{1:02x}{2:02x}".format(red, green, blue) 13 | 14 | return hexstr 15 | 16 | 17 | def write_scatter_chart_html(name, title, xaxis_title, xaxis_log, yaxis_title, yaxis_log, data, htmlfile): 18 | 19 | DEFAULT_POINT_SIZE = 3 20 | 21 | with open(htmlfile, 'w') as html_file: 22 | 23 | html_file.write('\n') 24 | html_file.write(' \n') 25 | html_file.write(' \n') 26 | html_file.write(' \n') 73 | html_file.write(' \n') 74 | html_file.write(' \n') 75 | html_file.write('
\n') 76 | html_file.write(' \n') 77 | html_file.write('\n') 78 | 79 | 80 | def write_bar_chart_html(name, units, bar_title, target_value, htmlfile, probabilities, prob_cutoff, class2value_method, 81 | v_axis_direction, normalize=False): 82 | 83 | tot_prob = [0] * len(probabilities[0]) 84 | for prob_set in probabilities: 85 | for classification in range(len(prob_set)): 86 | if prob_set[classification] >= prob_cutoff: 87 | tot_prob[classification] += prob_set[classification] 88 | 89 | prob_scale = 0.0 90 | nclass = len(tot_prob) 91 | for classification in range(nclass): 92 | if normalize: 93 | prob_scale = max(prob_scale, tot_prob[classification]) 94 | else: 95 | prob_scale += tot_prob[classification] 96 | # TEST 97 | #prob_scale = len(prob_set) 98 | 99 | with open(htmlfile, 'w') as html_file: 100 | 101 | html_file.write('\n') 102 | html_file.write(' \n') 103 | html_file.write(' \n') 104 | html_file.write(' \n') 158 | html_file.write(' \n') 159 | html_file.write(' \n') 160 | html_file.write('
\n') 161 | html_file.write(' \n') 162 | html_file.write('\n') 163 | 164 | 165 | -------------------------------------------------------------------------------- /quakenet_ingv/tflib/model.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import os 3 | import time 4 | 5 | import numpy as np 6 | import tensorflow as tf 7 | from tensorflow.python.client import timeline 8 | 9 | class BaseModel(object): 10 | """Base model implementing the training loop and general model interface.""" 11 | __metaclass__ = abc.ABCMeta 12 | 13 | def __init__(self, inputs, checkpoint_dir, is_training=False, 14 | reuse=False): 15 | """Creates a model mapped to a directory on disk for I/O: 16 | 17 | Args: 18 | inputs: input tensor(s), can be placeholders (e.g. for runtime prediction) or 19 | a queued data_pipeline. 20 | checkpoint_dir: directory where the trained parameters will be saved/loaded from. 21 | is_training: allows to parametrize certain layers differently when training (e.g. batchnorm). 22 | reuse: whether to reuse weights defined by another model. 23 | """ 24 | 25 | self.inputs = inputs 26 | self.checkpoint_dir = checkpoint_dir 27 | self.is_training = is_training 28 | 29 | self.layers = {} 30 | self.summaries = [] 31 | self.eval_summaries = [] 32 | 33 | self.global_step = tf.Variable(0, name='global_step', trainable=False) 34 | self._setup_prediction() 35 | self.saver = tf.train.Saver(tf.global_variables(),max_to_keep=100) 36 | 37 | @abc.abstractmethod 38 | def _setup_prediction(self): 39 | """Core layers for model prediction.""" 40 | pass 41 | 42 | @abc.abstractmethod 43 | def _setup_loss(self): 44 | """Loss function to minimize.""" 45 | pass 46 | 47 | @abc.abstractmethod 48 | def _setup_optimizer(self, learning_rate): 49 | """Optimizer.""" 50 | pass 51 | 52 | @abc.abstractmethod 53 | def _tofetch(self): 54 | """Tensors to run/fetch at each training step. 55 | Returns: 56 | tofetch: (dict) of Tensors/Ops. 57 | """ 58 | pass 59 | 60 | def _train_step(self, sess, start_time, run_options=None, run_metadata=None): 61 | """Step of the training loop. 62 | 63 | Returns: 64 | data (dict): data from useful for printing in 'summary_step'. 65 | Should contain field "step" with the current_step. 66 | """ 67 | tofetch = self._tofetch() 68 | tofetch['step'] = self.global_step 69 | tofetch['summaries'] = self.merged_summaries 70 | data = sess.run(tofetch, options=run_options, run_metadata=run_metadata) 71 | data['duration'] = time.time()-start_time 72 | return data 73 | 74 | def _test_step(self, sess, start_time, run_options=None, run_metadata=None): 75 | """Step of the training loop. 76 | 77 | Returns: 78 | data (dict): data from useful for printing in 'summary_step'. 79 | Should contain field "step" with the current_step. 80 | """ 81 | tofetch = self._tofetch() 82 | tofetch['step'] = self.global_step 83 | tofetch['is_correct'] = self.is_correct[0] 84 | data = sess.run(tofetch, options=run_options, run_metadata=run_metadata) 85 | data['duration'] = time.time()-start_time 86 | return data 87 | 88 | @abc.abstractmethod 89 | def _summary_step(self, data): 90 | """Information form data printed at each 'summary_step'. 91 | 92 | Returns: 93 | message (str): string printed at each summary step. 94 | """ 95 | pass 96 | 97 | def load(self, sess, step=None): 98 | """Loads the latest checkpoint from disk. 99 | 100 | Args: 101 | sess (tf.Session): current session in which the parameters are imported. 102 | step: specific step to load. 103 | """ 104 | if step==None: 105 | checkpoint_path = tf.train.latest_checkpoint(self.checkpoint_dir) 106 | else: 107 | checkpoint_path = os.path.join(self.checkpoint_dir,"model-"+str(step)) 108 | self.saver.restore(sess, checkpoint_path) 109 | step = tf.train.global_step(sess, self.global_step) 110 | print 'Loaded model at step {} from snapshot {}.'.format(step, checkpoint_path) 111 | 112 | def save(self, sess): 113 | """Saves a checkpoint to disk. 114 | 115 | Args: 116 | sess (tf.Session): current session from which the parameters are saved. 117 | """ 118 | checkpoint_path = os.path.join(self.checkpoint_dir, 'model') 119 | if not os.path.exists(self.checkpoint_dir): 120 | os.makedirs(self.checkpoint_dir) 121 | self.saver.save(sess, checkpoint_path, global_step=self.global_step) 122 | 123 | def test(self,n_val_steps): 124 | """Run predictions and print accuracy 125 | Args: 126 | n_val_steps (int): number of steps to run for testing 127 | (if is_training=False), n_val_steps=n_examples 128 | """ 129 | targets = tf.add(self.inputs['distance_id'],self.config.add) 130 | 131 | self.optimizer = tf.no_op(name="optimizer") 132 | self.loss = tf.no_op(name="loss") 133 | with tf.name_scope('accuracy'): 134 | is_correct = tf.equal(self.layers['distance_prediction'], targets) 135 | self.is_correct = tf.to_float(is_correct) 136 | self.accuracy = tf.reduce_mean(self.is_correct) 137 | 138 | with tf.Session() as sess: 139 | coord = tf.train.Coordinator() 140 | threads = tf.train.start_queue_runners(sess=sess, coord=coord) 141 | 142 | self.load(sess) 143 | 144 | print 'Starting prediction on testing set.' 145 | start_time = time.time() 146 | correct_prediction = 0 147 | for step in range(n_val_steps): 148 | step_data = self._test_step(sess, start_time, None, None) 149 | correct_prediction += step_data["is_correct"] 150 | accuracy = float(correct_prediction / n_val_steps) 151 | print 'Accuracy on testing set = {:.1f}%'.format(100*accuracy) 152 | 153 | 154 | coord.request_stop() 155 | coord.join(threads) 156 | 157 | def train(self, learning_rate, resume=False, summary_step=100, 158 | checkpoint_step=500, profiling=False): 159 | """Main training loop. 160 | 161 | Args: 162 | learning_rate (float): global learning rate used for the optimizer. 163 | resume (bool): whether to resume training from a checkpoint. 164 | summary_step (int): frequency at which log entries are added. 165 | checkpoint_step (int): frequency at which checkpoints are saved to disk. 166 | profiling: whether to save profiling trace at each summary step. (used for perf. debugging). 167 | """ 168 | lr = tf.Variable(learning_rate, name='learning_rate', 169 | trainable=False, 170 | collections=[tf.GraphKeys.GLOBAL_VARIABLES]) 171 | self.summaries.append(tf.summary.scalar('learning_rate', lr)) 172 | 173 | # Optimizer 174 | self._setup_loss() 175 | self._setup_optimizer(lr) 176 | 177 | # Profiling 178 | if profiling: 179 | run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE) 180 | run_metadata = tf.RunMetadata() 181 | else: 182 | run_options = None 183 | run_metadata = None 184 | 185 | # Summaries 186 | self.merged_summaries = tf.summary.merge(self.summaries) 187 | 188 | with tf.Session() as sess: 189 | self.summary_writer = tf.summary.FileWriter(self.checkpoint_dir, sess.graph) 190 | 191 | print 'Initializing all variables.' 192 | tf.local_variables_initializer().run() 193 | tf.global_variables_initializer().run() 194 | if resume: 195 | self.load(sess) 196 | 197 | print 'Starting data threads coordinator.' 198 | coord = tf.train.Coordinator() 199 | threads = tf.train.start_queue_runners(sess=sess, coord=coord) 200 | 201 | print 'Starting optimization.' 202 | start_time = time.time() 203 | try: 204 | while not coord.should_stop(): # Training loop 205 | step_data = self._train_step(sess, start_time, run_options, run_metadata) 206 | step = step_data['step'] 207 | 208 | if step > 0 and step % summary_step == 0: 209 | if profiling: 210 | self.summary_writer.add_run_metadata(run_metadata, 'step%d' % step) 211 | tl = timeline.Timeline(run_metadata.step_stats) 212 | ctf = tl.generate_chrome_trace_format() 213 | with open(os.path.join(self.checkpoint_dir, 'timeline.json'), 'w') as fid: 214 | print ('Writing trace.') 215 | fid.write(ctf) 216 | 217 | print self._summary_step(step_data) 218 | self.summary_writer.add_summary(step_data['summaries'], global_step=step) 219 | 220 | # Save checkpoint every `checkpoint_step` 221 | if checkpoint_step is not None and ( 222 | step > 0) and step % checkpoint_step == 0: 223 | print 'Step {} | Saving checkpoint.'.format(step) 224 | self.save(sess) 225 | 226 | except KeyboardInterrupt: 227 | print 'Interrupted training at step {}.'.format(step) 228 | self.save(sess) 229 | 230 | except tf.errors.OutOfRangeError: 231 | print 'Training completed at step {}.'.format(step) 232 | self.save(sess) 233 | 234 | finally: 235 | print 'Shutting down data threads.' 236 | coord.request_stop() 237 | self.summary_writer.close() 238 | 239 | # Wait for data threads 240 | print 'Waiting for all threads.' 241 | coord.join(threads) 242 | 243 | print 'Optimization done.' 244 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ConvNetQuake_INGV 2 | ## A python package for rapid earthquake characterization using single station waveforms and a convolutional neural network. 3 | 4 | 5 | ## Citation 6 | We have a manuscript submitted on the development and application of ConvNetQuake_INGV, if you reference or make use of ConvNetQuake_INGV please cite the folloing paper: 7 | 8 | Anthony Lomax, Alberto Michelini and Dario Jozinović, 2018. **An investigation of rapid earthquake characterization using single station waveforms and a convolutional neural network**, Seismological Research Letters 9 | 10 | 11 | ## Overview 12 | 13 | ConvNetQuake_INGV, derived from ConvNetQuake (Perol et al., 2018; http://advances.sciencemag.org/content/4/2/e1700578.full), implements a CNN to characterize earthquakes at any distance (local to far teleseismic). ConvNetQuake_INGV operates on 50sec, 3-component, broadband, single-station waveforms to detect seismic events and obtain binned, probabilistic estimates of the distance, azimuth, depth and magnitude of the event. ConvNetQuake_INGV is trained through supervised learning using waveform windows containing a diverse set of known event and noise waveforms. 14 | 15 | For ConvNetQuake_INGV, we modify the procedures and codes of Perol et al. (2018) and develop new tools to retrieve events, and noise and event waveforms from FDSN web-services (http://www.fdsn.org/webservices). 16 | 17 | For details see The ConvNetQuake_INGV section below. 18 | 19 | 20 | ## Installation 21 | 22 | # install environment 23 | # https://www.tensorflow.org/install/install_mac 24 | sudo easy_install pip 25 | pip install --upgrade virtualenv 26 | virtualenv --system-site-packages . 27 | source ./bin/activate 28 | easy_install -U pip 29 | pip install grpcio==1.9.1 30 | pip install https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl 31 | pip install -r requirements.txt # needed? 32 | pip install -r requirements2.txt # needed? 33 | pip install python-gflags 34 | pip install -e git+https://github.com/gem/oq-hazardlib.git@589fa31ddca3697e6b167082136dc0174a77bc16#egg=openquake.hazardlib 35 | pip install dask --upgrade 36 | pip install argparse 37 | pip install geojson 38 | pip install geopy 39 | pip install folium 40 | 41 | # install ConvNetQuake -> https://github.com/tperol/ConvNetQuake 42 | # link ConvNetQuake code in current working directory 43 | ln -s /Users/anthony/opt/ConvNetQuake-master_tf1.0/tflib . 44 | # 45 | # convert ConvNetQuake code to TensorFlow 1.2 46 | # upgrade a whole directory of 0.n TensorFlow programs to 1.0, enter a command having the following format: 47 | # $ python tf_upgrade.py --intree InputDir --outtree OutputDir 48 | # e.g. 49 | python tf_upgrade.py --intree ./tflib --outtree ./tflib 50 | chmod a+x ./tflib/*/*.py 51 | 52 | 53 | # run ConvNetQuake_INGV codes 54 | source ./bin/activate; export PYTHONPATH=./:./quakenet_ingv 55 | ... 56 | 57 | 58 | ## Run examples 59 | 60 | #install 61 | see INSTALLATION.txt 62 | 63 | # activate environment 64 | source ./bin/activate; export PYTHONPATH=./:./quakenet_ingv 65 | 66 | # get channel_file 67 | http://webservices.ingv.it/fdsnws/station/1/query?network=MN&channel=BH?&starttime=2009-01-01&endtime=2018-12-31&level=channel 68 | # save as: data/MN/MN_stations_2009-2018.xml 69 | 70 | # get train and validation events for circular distances centred on each station in channel_file 71 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2010-01-01 --endtime 2018-12-31 --channel_prefix BH --sampling_rate 20 --minradius 0.0 --maxradius 2.0 --minmagnitude 3.0 --event_files_path output/MN3/events 72 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2010-01-01 --endtime 2018-12-31 --channel_prefix BH --sampling_rate 20 --minradius 2.0 --maxradius 20.0 --minmagnitude 4.0 --event_files_path output/MN3/events 73 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2010-01-01 --endtime 2018-12-31 --channel_prefix BH --sampling_rate 20 --minradius 20.0 --maxradius 70.0 --minmagnitude 5.0 --event_files_path output/MN3/events 74 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2010-01-01 --endtime 2018-12-31 --channel_prefix BH --sampling_rate 20 --minradius 70.0 --maxradius 180.0 --minmagnitude 6.0 --event_files_path output/MN3/events 75 | 76 | python quakenet_ingv/bin/preprocess/get_streams.py --event_files_path output/MN3/events --outpath output/MN3/streams --base_url INGV --channel_file data/MN/MN_stations_2009-2018.xml --channel_prefix BH --sampling_rate 20 --n_streams 5000 --noise_fraction 0.4 --validation_fraction 0.1 --test_fraction 0.0 --snr_accept 3.0 --window_length 50 --window_start 5 --n_distances 50 --n_magnitudes 20 --n_depths 20 --n_azimuths 36 --event_fraction 0.25 77 | 78 | # train 79 | quakenet_ingv/bin/train --outpath output/MN3/streams --dataset output/MN3/streams/train --model ConvNetQuake9 --checkpoint_dir output/MN3/ConvNetQuake9 --use_magnitudes --use_depths --use_azimuths 80 | tensorboard --logdir output/MN3/ConvNetQuake9/ConvNetQuake9 81 | quakenet_ingv/bin/evaluate --outpath output/MN3/streams --model ConvNetQuake9 --checkpoint_dir output/MN3/ConvNetQuake9/ConvNetQuake9 --dataset output/MN3/streams/validate --eval_interval -1 --use_magnitudes --use_depths --use_azimuths --events 82 | 83 | 84 | # get test events for circular distances centred on each station in channel_file 85 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2007-01-01 --endtime 2009-12-31 --channel_prefix BH --sampling_rate 20 --minradius 0.0 --maxradius 2.0 --minmagnitude 4.0 --event_files_path output/MN/test_events 86 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2007-01-01 --endtime 2009-12-31 --channel_prefix BH --sampling_rate 20 --minradius 2.0 --maxradius 20.0 --minmagnitude 5.0 --event_files_path output/MN/test_events 87 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2007-01-01 --endtime 2009-12-31 --channel_prefix BH --sampling_rate 20 --minradius 20.0 --maxradius 50.0 --minmagnitude 6.0 --event_files_path output/MN/test_events 88 | python quakenet_ingv/bin/preprocess/get_events.py --base_url IRIS --channel_file data/MN/MN_stations_2009-2018.xml --starttime 2007-01-01 --endtime 2009-12-31 --channel_prefix BH --sampling_rate 20 --minradius 50.0 --maxradius 180.0 --minmagnitude 6.5 --event_files_path output/MN/test_events 89 | # 90 | python quakenet_ingv/bin/preprocess/get_streams.py --event_files_path output/MN/test_events --outpath output/MN3/test_streams --base_url INGV --channel_file data/MN/MN_stations_2009-2018.xml --channel_prefix BH --sampling_rate 20 --n_streams 999 --noise_fraction 0.0 --validation_fraction 0.0 --test_fraction 1.0 --snr_accept -1.0 --window_length 50 --window_start 5 --n_distances 50 --n_magnitudes 20 --n_depths 20 --n_azimuths 36 --systematic 91 | # 92 | quakenet_ingv/bin/evaluate --outpath output/MN3/test_streams --model ConvNetQuake9 --checkpoint_dir output/MN3/ConvNetQuake9/ConvNetQuake9_NNN --dataset output/MN3/test_streams_YEARS/test --eval_interval -1 --use_magnitudes --use_depths --use_azimuths --channel_file data/MN/MN_stations_2007-2018.xml --events --save_event_results --step NNN 93 | # NOTE: move event_results to event_results_YEARS 94 | # 95 | ./run_event_results_batch.bash output/MN3/ConvNetQuake9/ConvNetQuake9_NNN/event_results_YEARS 96 | 97 | quakenet_ingv/bin/plot_events --event_files_path output/MN3/streams/train/events/xml --channel_file data/MN/MN_stations_2007-2018.xml --channel_prefix BH --sampling_rate 20 --n_magnitudes 20 --n_depths 20 98 | quakenet_ingv/bin/plot_events --event_files_path output/MN3/test_streams_2009/test/events/xml --channel_file data/MN/MN_stations_2007-2018.xml --channel_prefix BH --sampling_rate 20 --n_magnitudes 20 --n_depths 20 99 | 100 | quakenet_ingv/bin/plot_event_stats --evaluate_event_results_path output/MN3/ConvNetQuake9/ConvNetQuake9_NNN/event_results_YEARS --train_events_path output/MN3/streams/train/events/xml 101 | 102 | 103 | ## The ConvNetQuake_INGV algorithm 104 | 105 | The procedures and Python codes (https://github.com/tperol/ConvNetQuake) of Perol et al. (2018; http://advances.sciencemag.org/content/4/2/e1700578.full) for ConvNetQuake are based on several technologies: 106 | 107 | 1. The Python (https://www.python.org) programming language, with the NumPy (http://www.numpy.org) package for scientific computing, and many other packages. 108 | 2. The ObsPy (http://obspy.org) Python framework for processing seismological data. 109 | 3. TensorFlow (Abadi et al. 2015; https://www.tensorflow.org), an open source software library for high performance numerical computation with strong support for machine learning and deep learning. 110 | 111 | For ConvNetQuake_INGV, we modify the procedures and codes of Perol et al. (2018) and develop new Python tools to: 112 | 113 | 1. Retrieve events in QuakeML format from an FDSN “event” web-service for specified magnitude and distance ranges for each station in a QuakeML channel file, possibly retrieved from an FDSN “station” web-service. (quakenet_ingv/bin/preprocess/get_events.py) 114 | 2. Retrieve from an FDSN “dataselect” web-service noise and event waveforms (Figures 1 in the main text) for training, validation and test data sets with specified proportions of noise/event samples. For the training and validation data sets, waveforms are obtained for randomly chosen stations from the QuakeML channel file and randomly chosen events from the QuakeML event files generated by get_events.py. For the test data set, waveforms are obtained for all available stations in the QuakeML channel file and randomly chosen events from the QuakeML event files generated by get_events.py. Waveforms are labelled with binned (Table S1) station-event distance, station-event azimuth, event magnitude and event depth. (quakenet_ingv/bin/preprocess/get_streams.py) 115 | 3. Set up the neural network architecture (Figure 2 and description in main text), define the loss function, setup the optimizer (inverts for network node weights and biases), and evaluate validation metrics. (quakenet_ingv/quakenet/models.py) 116 | 4. Set-up and train the neural network by feeding batches of training event and noise waveforms stored by get_streams.py to routines defined in models.py (quakenet_ingv/bin/train) 117 | 5. Evaluate results of network training by feeding specified validation event or noise waveforms stored by get_streams.py to trained network, outputs various metrics showing quality of match between predicted and true classification values for detection, station-event distance, azimuth, event magnitude and depth. (quakenet_ingv/bin/evaluate) 118 | 6. Plot Evaluate results output for the test data set using Folium (http://python-visualization.github.io/folium) to generate leaflet.js (https://leafletjs.com) html pages showing, for each event: 1) an interactive map of stations, true epicenter and predicted distance-azimuth bins with finite probability, 2) a histogram of true and predicted magnitude bins, 2) a histogram of true and predicted depth bins. (quakenet_ingv/bin/plot_event_results) 119 | 120 | ### Target waveform labels 121 | 122 | The class labels or classification for each waveform are: event or noise, binned station-event distance (0-180°), station-event azimuth (0-360°, 10° step), event magnitude (0-10, 0.5mu step) and event depth (0-700km). The bin steps for distance and depth are geometrically increasing to give higher weight and precision to nearer and shallower events. Table S1 lists the binned, target labels. Following Perol et al. (2018), the event or noise label is a binary classification, it is implemented as a -1 value for the station-event distance label. 123 | 124 | ### Waveform datasets 125 | 126 | The event and noise waveforms are identical except for being labelled as event or noise, and their data files are organized in different event or noise sub-directories in train, validate and test stream directories. 127 | 128 | 3-component (BHZ/N/E) waveforms are retrieved from an FDSN “dataselect” web-service files for specified proportions of noise/event and train/validate/test samples. The training earthquake waveforms should span, as well as possible, the distance, azimuth, depth and magnitude range of target events for application of the trained ConvNetQuake_INGV. Waveforms are obtained for randomly chosen network-station specified in a QuakeML format channel file. Then, to define a channel event waveforms, events are randomly chosen from QuakeML format event files generated by get_events.py. For noise waveforms, a reference time is chosen between consecutive events randomly chosen from the QuakeML format event files. 129 | 130 | Waveform quality control and pre-processing includes: 131 | 132 | 1. checking that all 3 components are available, 133 | 2. resampling to 20Hz if necessary, 134 | 3. checking that event waveforms have signal-to-noise ratio (SNR) greater than a specified threshold [3.0] for at least one broadband or 1Hz, 4-pole high-pass filtered component 135 | 4. checking that noise waveforms have signal-to-noise ratio (SNR) less than a specified threshold for all components in broadband or with 1Hz, 4-pole high-pass, 136 | 5. trim event waveforms to start at a specified time interval [5 sec] before predicted P arrival time (using ak135 model) and event and noise waveforms to a specified total window length [50 sec], 137 | 6. correct amplitudes to sensitivity (gain) for each component, 138 | 7. normalized to the global maximum of all 3 traces, store the normalization value, stream_max. 139 | 8. for event waveforms, determine true binned distance, azimuth, magnitude and depth class. 140 | 141 | Since the pre-processing includes normalized to the global maximum of all 3 traces, the normalization value, stream_max, is used by the neural network to provide absolute trace amplitude information to aid in magnitude estimation. 142 | 143 | After quality control and pre-processing, the windowed noise and event waveforms are saved to the stream directories in miniseed format (IRIS, 2012) and, along with the true waveform classification value, in TensorFlow “TFRecord” format (https://www.tensorflow.org). For event waveforms, event meta-data in QuakeML are also saved to the stream directories. 144 | 145 | ### Loss function and optimizer 146 | 147 | ConvNetQuake_INGV uses an L2-regularized cross-entropy loss (misfit) function and the Adam Optimizer algorithm, as in ConvNetQuake (Perol et al., 2018; Mehta et al., 2018; Kingma and Ba, 2017). The Adam Optimizer performs first-order, gradient-based optimization of stochastic objective functions, based on adaptive estimates of lower-order moments. The method is straightforward to implement, is computationally efficient, has little memory requirements, is invariant to diagonal rescaling of the gradients, and is well suited for problems that are large in terms of data or parameters. 148 | 149 | -------------------------------------------------------------------------------- /quakenet_ingv/bin/plot_event_stats: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | # ------------------------------------------------------------------- 4 | # File: plot_event_results 5 | # Author: Anthony Lomax 6 | # Created: 2018-05-09 7 | # ------------------------------------------------------------------# 8 | 9 | """ Plot event results on an interactive map""" 10 | 11 | # import sys 12 | import argparse 13 | import os 14 | # import shutil 15 | # import time 16 | import cPickle as pickle 17 | 18 | import math 19 | import geopy 20 | 21 | from obspy.core import event 22 | import obspy.geodetics.base as geo 23 | 24 | import folium 25 | #from folium.plugins import MarkerCluster 26 | 27 | import quakenet.util as util 28 | import quakenet.plot as qnplot 29 | 30 | PROB_CUTOFF = 0.01 31 | 32 | 33 | def get_event_density(sigma, latitude, longitude, origins): 34 | 35 | two_sigma_sqr = (2.0 * sigma * sigma) 36 | density_sum = 0.0 37 | closest = 999.9 38 | for origin in origins: 39 | dist_meters, azim, bazim = geo.gps2dist_azimuth(latitude, longitude, origin.latitude, origin.longitude) 40 | dist = geo.kilometer2degrees(dist_meters / 1000.0, radius=6371) 41 | closest = min(closest, dist) 42 | weight = math.exp(-(dist * dist) / two_sigma_sqr) # https://en.wikipedia.org/wiki/Gaussian_blur 43 | density_sum += weight 44 | 45 | return density_sum, closest 46 | 47 | 48 | 49 | def main(args): 50 | 51 | # get train events 52 | print 'get train events' 53 | origins = {} 54 | all_origins = [] 55 | all_origins_ids = [] 56 | for root, dirs, files in os.walk(args.train_events_path): 57 | for f in files: 58 | if f.endswith('.xml'): 59 | # if not '_VTS_' in f: # DEBUG!!! 60 | # continue 61 | event_xml_file = f 62 | # MN_PDG__BH.2009.12.17.01.42.47.815.xml 63 | values = event_xml_file.split('.') 64 | net, sta, loc, chan = values[0].split('_') 65 | seed_id = net + '.' + sta + '.' + loc + '.' + chan + 'Z' 66 | print event_xml_file + ' \r', 67 | events = event.read_events(os.path.join(root, event_xml_file)) 68 | for evt in events: 69 | if not seed_id in origins.keys(): 70 | origins[seed_id] = [] 71 | origins[seed_id].append(evt.preferred_origin()) 72 | if not str(evt.preferred_origin().resource_id) in all_origins_ids: 73 | all_origins.append(evt.preferred_origin()) 74 | all_origins_ids.append(str(evt.preferred_origin().resource_id)) 75 | print '' 76 | 77 | # get evaluate event stats 78 | print 'get evaluate event stats' 79 | data_relative = [] 80 | data_absolute = [] 81 | data_relative_closest = [] 82 | data_absolute_closest = [] 83 | data_distance = [] 84 | data_azimuth = [] 85 | data_epi_sta_stats_relative = [] 86 | data_epi_sta_stats_absolute = [] 87 | data_epi_sta_stats_relative_closest = [] 88 | data_epi_sta_stats_absolute_closest = [] 89 | data_epi_sta_stats_distance = [] 90 | SIGMA_DIST_FRACTION = 20.0 91 | for root, dirs, files in os.walk(args.evaluate_event_results_path): 92 | for f in files: 93 | if f.endswith('.epi_sta_stats.pkl'): 94 | epi_sta_stats_pkl_file = f 95 | print 'Processing:', epi_sta_stats_pkl_file 96 | with open(os.path.join(root, epi_sta_stats_pkl_file), 'r') as file: 97 | (origin_latitude, origin_longitude, origin_depth, mean_distance_error, tot_prob_sum, mean_origin_sta_distance, tot_sta_sum) = pickle.load(file) 98 | 99 | # find training event density around origin 100 | # sigma is 1/20 station-event distance 101 | sigma = mean_origin_sta_distance / SIGMA_DIST_FRACTION 102 | train_event_density, closest = get_event_density(sigma, origin_latitude, origin_longitude, all_origins) 103 | # get measure of distance error 104 | distance_error_relative = mean_distance_error / mean_origin_sta_distance 105 | point_size = tot_prob_sum / float(tot_sta_sum) 106 | #color_value = mean_origin_sta_distance / 90.0 107 | #color_value = min(color_value, 1.0) 108 | color_value = [255, 0, 0] 109 | if (mean_origin_sta_distance < 3.5): 110 | color_value = [0, 127, 0] 111 | elif (mean_origin_sta_distance > 30.0): 112 | color_value = [0, 0, 255] 113 | # print 'DEBUG: ' + str(origin_latitude) + ' '+ str(origin_longitude) + ' den=' + str(train_event_density) + \ 114 | # ' prob=' + str(tot_prob) + ' size=' + str(point_size) 115 | data_epi_sta_stats_relative.append([train_event_density, distance_error_relative, point_size, color_value[0], color_value[1], color_value[2]]) 116 | data_epi_sta_stats_absolute.append([train_event_density, mean_distance_error, point_size, color_value[0], color_value[1], color_value[2]]) 117 | data_epi_sta_stats_relative_closest.append([closest, distance_error_relative, point_size, color_value[0], color_value[1], color_value[2]]) 118 | data_epi_sta_stats_absolute_closest.append([closest, mean_distance_error, point_size, color_value[0], color_value[1], color_value[2]]) 119 | data_epi_sta_stats_distance.append([mean_origin_sta_distance, mean_distance_error, point_size, color_value[0], color_value[1], color_value[2]]) 120 | if f.endswith('.epi_sta_poly.pkl'): 121 | epi_sta_poly_pkl_file = f 122 | print 'Processing:', epi_sta_poly_pkl_file 123 | seed_id_last = '$NULL$' 124 | train_event_density, closest = (-1.0, -1.0) 125 | with open(os.path.join(root, epi_sta_poly_pkl_file), 'r') as file: 126 | epi_sta_poly_coords = pickle.load(file) 127 | for coords in epi_sta_poly_coords: 128 | (origin_latitude, origin_longitude, origin_depth, 129 | seed_id, sta_latitude, sta_longitude, 130 | lat_cent, lon_cent, tot_prob) = coords 131 | if not seed_id in origins.keys(): 132 | print '\nERROR: ' + seed_id + ' not in origins list!\n' 133 | continue 134 | if not seed_id == seed_id_last: 135 | #origin_sta_distance = distance((origin_latitude, origin_longitude), (sta_latitude, sta_longitude)).degrees 136 | dist_meters, azim, bazim = geo.gps2dist_azimuth(origin_latitude, origin_longitude, sta_latitude, sta_longitude) 137 | origin_sta_distance = geo.kilometer2degrees(dist_meters / 1000.0, radius=6371) 138 | # find training event density around origin 139 | # sigma is 1/10 station-event distance 140 | sigma = origin_sta_distance / SIGMA_DIST_FRACTION 141 | train_event_density, closest = get_event_density(sigma, origin_latitude, origin_longitude, origins[seed_id]) 142 | seed_id_last = seed_id 143 | # get measure of distance error 144 | #distance_error =distance((origin_latitude, origin_longitude), (lat_cent, lon_cent)).degrees / origin_sta_distance 145 | dist_meters, azim, bazim = geo.gps2dist_azimuth(origin_latitude, origin_longitude, lat_cent, lon_cent) 146 | distance_error = geo.kilometer2degrees(dist_meters / 1000.0, radius=6371) 147 | distance_error_relative = distance_error / origin_sta_distance 148 | # 149 | dist_meters, azim1, bazim = geo.gps2dist_azimuth(sta_latitude, sta_longitude, origin_latitude, origin_longitude) 150 | dist_meters, azim2, bazim = geo.gps2dist_azimuth(sta_latitude, sta_longitude, lat_cent, lon_cent) 151 | azimuth_error = azim2 - azim1 152 | if azimuth_error < -180.0: 153 | azimuth_error += 360 154 | elif azimuth_error > 180.0: 155 | azimuth_error -= 360 156 | # 157 | point_size = tot_prob 158 | #color_value = origin_sta_distance / 90.0 159 | #color_value = min(color_value, 1.0) 160 | color_value = [255, 0, 0] 161 | if (origin_sta_distance < 3.5): 162 | color_value = [0, 127, 0] 163 | elif (origin_sta_distance > 30.0): 164 | color_value = [0, 0, 255] 165 | # print 'DEBUG: ' + str(origin_latitude) + ' '+ str(origin_longitude) + ' den=' + str(train_event_density) + \ 166 | # ' prob=' + str(tot_prob) + ' size=' + str(point_size) 167 | data_relative.append([train_event_density, distance_error_relative, point_size, color_value[0], color_value[1], color_value[2]]) 168 | data_absolute.append([train_event_density, distance_error, point_size, color_value[0], color_value[1], color_value[2]]) 169 | data_relative_closest.append([closest, distance_error_relative, point_size, color_value[0], color_value[1], color_value[2]]) 170 | data_absolute_closest.append([closest, distance_error, point_size, color_value[0], color_value[1], color_value[2]]) 171 | data_distance.append([origin_sta_distance, distance_error, point_size, color_value[0], color_value[1], color_value[2]]) 172 | # azimuth 173 | color_value = [255, 0, 0] 174 | #if (origin_sta_distance < 20.0): 175 | if (origin_sta_distance < 3.5): 176 | color_value = [0, 127, 0] 177 | elif (origin_sta_distance > 80.0): 178 | color_value = [0, 0, 255] 179 | data_azimuth.append([origin_sta_distance, azimuth_error, point_size, color_value[0], color_value[1], color_value[2]]) 180 | print '' 181 | 182 | outpath = os.path.join(args.evaluate_event_results_path, 'html') 183 | if not os.path.exists(outpath): 184 | os.makedirs(outpath) 185 | 186 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_density_relative.html') 187 | qnplot.write_scatter_chart_html('distance_error_event_density_relative', 'Distance error vs. Training event density', 188 | 'Training event density around true epicenter', False, 'Relative distance error ( error / station-event distance)', True, data_relative, htmlfile) 189 | print 'HTML written to: ' + htmlfile 190 | # 191 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_density_absolute.html') 192 | qnplot.write_scatter_chart_html('distance_error_event_density_absolute', 'Distance error vs. Training event density', 193 | 'Training event density around true epicenter', False, 'Distance error (degrees)', True, data_absolute, htmlfile) 194 | print 'HTML written to: ' + htmlfile 195 | # 196 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_closest_relative.html') 197 | qnplot.write_scatter_chart_html('distance_error_event_closest_relative', 'Distance error vs. Training event closest distance', 198 | 'Training event closest distance to true epicenter (degrees)', True, 'Relative distance error ( error / station-event distance)', True, data_relative_closest, htmlfile) 199 | print 'HTML written to: ' + htmlfile 200 | # 201 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_closest_absolute.html') 202 | qnplot.write_scatter_chart_html('distance_error_event_closest_absolute', 'Distance error vs. Training event closest distance', 203 | 'Training event closest distance to true epicenter (degrees)', True, 'Distance error (degrees)', True, data_absolute_closest, htmlfile) 204 | print 'HTML written to: ' + htmlfile 205 | # 206 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_distance.html') 207 | qnplot.write_scatter_chart_html('distance_error_event_distance', 'Distance error vs. Event distance', 208 | 'Station-event distance (degrees)', True, 'Distance error (degrees)', True, data_distance, htmlfile) 209 | print 'HTML written to: ' + htmlfile 210 | 211 | # 212 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.azimuth_error_event_distance.html') 213 | qnplot.write_scatter_chart_html('azimuth_error_event_distance', 'Azimuth error vs. Event distance', 214 | 'Station-event distance (degrees)', True, 'Azimuth error (degrees)', True, data_azimuth, htmlfile) 215 | print 'HTML written to: ' + htmlfile 216 | 217 | 218 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_density_relative_data_epi_sta_stats.html') 219 | qnplot.write_scatter_chart_html('distance_error_event_density_relative', 'Distance error vs. Training event density (event mean over stations)', 220 | 'Training event density around true epicenter', False, 'Relative distance error ( error / station-event distance)', True, data_epi_sta_stats_relative, htmlfile) 221 | print 'HTML written to: ' + htmlfile 222 | # 223 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_density_absolute_data_epi_sta_stats.html') 224 | qnplot.write_scatter_chart_html('distance_error_event_density_absolute', 'Distance error vs. Training event density (event mean over stations)', 225 | 'Training event density around true epicenter', False, 'Distance error (degrees)', True, data_epi_sta_stats_absolute, htmlfile) 226 | print 'HTML written to: ' + htmlfile 227 | # 228 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_closest_relative_data_epi_sta_stats.html') 229 | qnplot.write_scatter_chart_html('distance_error_event_closest_relative', 'Distance error vs. Training event closest distance (event mean over stations)', 230 | 'Training event closest distance to true epicenter (degrees)', True, 'Relative distance error ( error / station-event distance)', True, data_epi_sta_stats_relative_closest, htmlfile) 231 | print 'HTML written to: ' + htmlfile 232 | # 233 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_closest_absolute_data_epi_sta_stats.html') 234 | qnplot.write_scatter_chart_html('distance_error_event_closest_absolute', 'Distance error vs. Training event closest distance (event mean over stations)', 235 | 'Training event closest distance to true epicenter (degrees)', True, 'Distance error (degrees)', True, data_epi_sta_stats_absolute_closest, htmlfile) 236 | print 'HTML written to: ' + htmlfile 237 | # 238 | htmlfile = os.path.join(args.evaluate_event_results_path, 'html', 'events.distance_error_event_distance_data_epi_sta_stats.html') 239 | qnplot.write_scatter_chart_html('distance_error_event_distance', 'Distance error vs. Event distance (event mean over stations)', 240 | 'Station-event distance (degrees)', True, 'Distance error (degrees)', True, data_epi_sta_stats_distance, htmlfile) 241 | print 'HTML written to: ' + htmlfile 242 | 243 | 244 | if __name__ == '__main__': 245 | 246 | parser = argparse.ArgumentParser() 247 | parser.add_argument('--evaluate_event_results_path', type=str, 248 | help='Path for evaluated event files input') 249 | parser.add_argument('--train_events_path', type=str, 250 | help='Path for training event files input') 251 | 252 | args = parser.parse_args() 253 | main(args) 254 | -------------------------------------------------------------------------------- /quakenet_ingv/quakenet/models.py: -------------------------------------------------------------------------------- 1 | # ------------------------------------------------------------------- 2 | # File Name : models.py 3 | # Creation Date : 11-27-16 4 | # Last Modified : Fri Jan 6 13:38:15 2017 5 | # Author: Thibaut Perol & Michael Gharbi 6 | # ------------------------------------------------------------------- 7 | """ConvNetQuake model. 8 | the function get is implemented to help prototyping other models. 9 | One can create a subclass 10 | class Proto(ConvNetQuake) 11 | and overwrite the _setup_prediction method to change the network 12 | architecture 13 | """ 14 | 15 | import os 16 | import time 17 | 18 | import numpy as np 19 | import tensorflow as tf 20 | from tensorflow.python.client import timeline 21 | 22 | import tflib.model 23 | import tflib.layers as layers 24 | 25 | 26 | def get(model_name, inputs, config, checkpoint_dir, is_training=False): 27 | """Returns a Model instance instance by model name. 28 | 29 | Args: 30 | name: class name of the model to instantiate. 31 | inputs: inputs as returned by a DataPipeline. 32 | params: dict of model parameters. 33 | """ 34 | 35 | return globals()[model_name](inputs, config, checkpoint_dir, is_training=is_training) 36 | 37 | 38 | 39 | class ConvNetQuake(tflib.model.BaseModel): 40 | 41 | def __init__(self, inputs, config, checkpoint_dir, n_channels=32, n_conv_layers=8, n_fc_layers=2, is_training=False, reuse=False): 42 | 43 | self.n_channels = n_channels 44 | self.n_conv_layers = n_conv_layers 45 | self.n_fc_layers = n_fc_layers 46 | self.is_training = is_training 47 | self.cfg = config 48 | super(ConvNetQuake, self).__init__(inputs, checkpoint_dir, is_training=is_training, reuse=reuse) 49 | 50 | def _setup_prediction(self): 51 | 52 | self.batch_size = self.inputs['data'].get_shape().as_list()[0] 53 | 54 | current_layer = self.inputs['data'] 55 | #n_channels = 32 # number of channels per conv layer 56 | ksize = 3 # size of the convolution kernel 57 | for i in range(self.n_conv_layers): 58 | current_layer = layers.conv1(current_layer, self.n_channels, ksize, stride=2, scope='conv{}'.format(i + 1), padding='SAME') 59 | tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) 60 | self.layers['conv{}'.format(i + 1)] = current_layer 61 | 62 | bs, width, _ = current_layer.get_shape().as_list() 63 | n_fc_nodes = width * self.n_channels 64 | current_layer = tf.reshape(current_layer, [bs, n_fc_nodes], name="reshape") 65 | 66 | # 20180916 AJL - include stram_max in fc layers 67 | stream_max_tensor = tf.expand_dims(self.inputs['stream_max'], 1) 68 | current_layer = tf.concat([current_layer, stream_max_tensor], 1) 69 | n_fc_nodes += 1 70 | 71 | for i in range(self.n_fc_layers - 1): 72 | current_layer = layers.fc(current_layer, n_fc_nodes, scope='fc{}'.format(i + 1), activation_fn=tf.nn.relu) 73 | current_layer = layers.fc(current_layer, self.cfg.n_distances + self.cfg.n_magnitudes + self.cfg.n_depths + self.cfg.n_azimuths, \ 74 | scope='logits', activation_fn=None) 75 | 76 | istart = 0 77 | iend = self.cfg.n_distances 78 | self.layers['distance_logits'] = current_layer[ : , istart : iend] 79 | self.layers['distance_prob'] = tf.nn.softmax(self.layers['distance_logits'], name='distance_prob') 80 | self.layers['distance_prediction'] = tf.argmax(self.layers['distance_prob'], 1, name='distance_pred') 81 | istart = iend 82 | 83 | self.layers['magnitude_logits'] = tf.constant(-1) 84 | self.layers['magnitude_prob'] = tf.constant(-1) 85 | self.layers['magnitude_prediction'] = tf.constant(-1) 86 | if self.cfg.n_magnitudes > 0: 87 | iend += self.cfg.n_magnitudes 88 | self.layers['magnitude_logits'] = current_layer[ : , istart : iend] 89 | self.layers['magnitude_prob'] = tf.nn.softmax(self.layers['magnitude_logits'], name='magnitude_prob') 90 | self.layers['magnitude_prediction'] = tf.argmax(self.layers['magnitude_prob'], 1, name='magnitude_pred') 91 | istart = iend 92 | 93 | self.layers['depth_logits'] = tf.constant(-1) 94 | self.layers['depth_prob'] = tf.constant(-1) 95 | self.layers['depth_prediction'] = tf.constant(-1) 96 | if self.cfg.n_depths > 0: 97 | iend += self.cfg.n_depths 98 | self.layers['depth_logits'] = current_layer[ : , istart : iend] 99 | self.layers['depth_prob'] = tf.nn.softmax(self.layers['depth_logits'], name='depth_prob') 100 | self.layers['depth_prediction'] = tf.argmax(self.layers['depth_prob'], 1, name='depth_pred') 101 | istart = iend 102 | 103 | self.layers['azimuth_logits'] = tf.constant(-1) 104 | self.layers['azimuth_prob'] = tf.constant(-1) 105 | self.layers['azimuth_prediction'] = tf.constant(-1) 106 | if self.cfg.n_azimuths > 0: 107 | iend += self.cfg.n_azimuths 108 | self.layers['azimuth_logits'] = current_layer[ : , istart : iend] 109 | self.layers['azimuth_prob'] = tf.nn.softmax(self.layers['azimuth_logits'], name='azimuth_prob') 110 | self.layers['azimuth_prediction'] = tf.argmax(self.layers['azimuth_prob'], 1, name='azimuth_pred') 111 | istart = iend 112 | 113 | 114 | tf.add_to_collection(tf.GraphKeys.ACTIVATIONS, current_layer) 115 | 116 | tf.contrib.layers.apply_regularization( 117 | tf.contrib.layers.l2_regularizer(self.cfg.regularization), 118 | weights_list=tf.get_collection(tf.GraphKeys.WEIGHTS)) 119 | 120 | def validation_metrics(self): 121 | if not hasattr(self, '_validation_metrics'): 122 | self._setup_loss() 123 | 124 | self._validation_metrics = { 125 | 'loss': self.loss, 126 | 'distance_loss': self.distance_loss, 127 | 'magnitude_loss': self.magnitude_loss, 128 | 'depth_loss': self.depth_loss, 129 | 'azimuth_loss': self.azimuth_loss, 130 | 'detection_accuracy': self.detection_accuracy, 131 | 'distance_accuracy': self.distance_accuracy, 132 | 'magnitude_accuracy': self.magnitude_accuracy, 133 | 'depth_accuracy': self.depth_accuracy, 134 | 'azimuth_accuracy': self.azimuth_accuracy 135 | } 136 | return self._validation_metrics 137 | 138 | def validation_metrics_message(self, metrics): 139 | s = 'loss = {:.5f} | acc = det:{:.1f}% dist:{:.1f}% mag:{:.1f}% dep:{:.1f}% az:{:.1f}%'.format(metrics['loss'], 140 | metrics['detection_accuracy'] * 100, metrics['distance_accuracy'] * 100, \ 141 | metrics['magnitude_accuracy'] * 100, metrics['depth_accuracy'] * 100, metrics['azimuth_accuracy'] * 100) 142 | return s 143 | 144 | def _setup_loss(self): 145 | with tf.name_scope('loss'): 146 | # distance loss 147 | # change target range from -1:n_distances-1 to 0:n_distances 148 | targets_distance_id = tf.add(self.inputs['distance_id'], self.cfg.add) 149 | self.distance_loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.layers['distance_logits'], labels=targets_distance_id)) 150 | raw_loss = self.distance_loss 151 | # magnitude loss 152 | targets_magnitude_id = self.inputs['magnitude_id'] 153 | self.magnitude_loss = tf.constant(0.0) 154 | if self.cfg.n_magnitudes > 0: 155 | vloss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.layers['magnitude_logits'], labels=targets_magnitude_id) 156 | vloss = tf.where(tf.greater(targets_distance_id, tf.zeros_like(targets_distance_id)), vloss, tf.zeros_like(vloss)) 157 | self.magnitude_loss = tf.reduce_mean(vloss) 158 | raw_loss += self.magnitude_loss 159 | # depth loss 160 | targets_depth_id = self.inputs['depth_id'] 161 | # # TEMPORARY BUG FIX 162 | # t_clip = tf.clip_by_value(targets_depth_id, 0, self.cfg.n_depths -1) 163 | # targets_depth_id = t_clip 164 | # # END TEMPORARY BUG FIX 165 | self.depth_loss = tf.constant(0.0) 166 | if self.cfg.n_depths > 0: 167 | vloss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.layers['depth_logits'], labels=targets_depth_id) 168 | vloss = tf.where(tf.greater(targets_distance_id, tf.zeros_like(targets_distance_id)), vloss, tf.zeros_like(vloss)) 169 | self.depth_loss = tf.reduce_mean(vloss) 170 | raw_loss += self.depth_loss 171 | # azimuth loss 172 | targets_azimuth_id = self.inputs['azimuth_id'] 173 | self.azimuth_loss = tf.constant(0.0) 174 | if self.cfg.n_azimuths > 0: 175 | vloss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.layers['azimuth_logits'], labels=targets_azimuth_id) 176 | vloss = tf.where(tf.greater(targets_distance_id, tf.zeros_like(targets_distance_id)), vloss, tf.zeros_like(vloss)) 177 | self.azimuth_loss = tf.reduce_mean(vloss) 178 | raw_loss += self.azimuth_loss 179 | # 180 | self.summaries.append(tf.summary.scalar('loss/train_raw', raw_loss)) 181 | 182 | self.loss = raw_loss 183 | 184 | reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES) 185 | if reg_losses: 186 | with tf.name_scope('regularizers'): 187 | reg_loss = sum(reg_losses) 188 | self.summaries.append(tf.summary.scalar('loss/regularization', reg_loss)) 189 | self.loss += reg_loss 190 | 191 | self.summaries.append(tf.summary.scalar('loss/train', self.loss)) 192 | 193 | with tf.name_scope('accuracy'): 194 | # change target range from -1:n_distances-1 to 0:n_distances 195 | targets = tf.add(self.inputs['distance_id'], self.cfg.add) 196 | is_true_event_logical = tf.greater(targets, tf.zeros_like(targets)) 197 | is_true_event = tf.cast(is_true_event_logical, tf.int64) 198 | is_pred_event = tf.cast(tf.greater(self.layers['distance_prediction'], tf.zeros_like(targets)), tf.int64) 199 | detection_is_correct = tf.equal(is_true_event, is_pred_event) 200 | self.detection_accuracy = tf.reduce_mean(tf.to_float(detection_is_correct)) 201 | self.summaries.append(tf.summary.scalar('detection_accuracy/train', self.detection_accuracy)) 202 | # 203 | self.distance_accuracy = tf.reduce_mean(tf.to_float(tf.boolean_mask(tf.equal(self.layers['distance_prediction'], targets), is_true_event_logical))) 204 | #self.distance_accuracy = tf.reduce_mean(tf.to_float(tf.equal(self.layers['distance_prediction'], targets))) 205 | self.summaries.append(tf.summary.scalar('distance_accuracy/train', self.distance_accuracy)) 206 | self.magnitude_accuracy = tf.reduce_mean(tf.to_float(tf.boolean_mask(tf.equal(self.layers['magnitude_prediction'], self.inputs['magnitude_id']), is_true_event_logical))) 207 | #self.magnitude_accuracy = tf.reduce_mean(tf.to_float(tf.equal(self.layers['magnitude_prediction'], self.inputs['magnitude_id']))) 208 | self.summaries.append(tf.summary.scalar('magnitude_accuracy/train', self.magnitude_accuracy)) 209 | self.depth_accuracy = tf.reduce_mean(tf.to_float(tf.boolean_mask(tf.equal(self.layers['depth_prediction'], self.inputs['depth_id']), is_true_event_logical))) 210 | #self.depth_accuracy = tf.reduce_mean(tf.to_float(tf.equal(self.layers['depth_prediction'], self.inputs['depth_id']))) 211 | self.summaries.append(tf.summary.scalar('depth_accuracy/train', self.depth_accuracy)) 212 | self.azimuth_accuracy = tf.reduce_mean(tf.to_float(tf.boolean_mask(tf.equal(self.layers['azimuth_prediction'], self.inputs['azimuth_id']), is_true_event_logical))) 213 | #self.azimuth_accuracy = tf.reduce_mean(tf.to_float(tf.equal(self.layers['azimuth_prediction'], self.inputs['azimuth_id']))) 214 | self.summaries.append(tf.summary.scalar('azimuth_accuracy/train', self.azimuth_accuracy)) 215 | 216 | def _setup_optimizer(self, learning_rate): 217 | update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) 218 | if update_ops: 219 | updates = tf.group(*update_ops, name='update_ops') 220 | with tf.control_dependencies([updates]): 221 | self.loss = tf.identity(self.loss) 222 | optim = tf.train.AdamOptimizer(learning_rate).minimize( 223 | self.loss, name='optimizer', global_step=self.global_step) 224 | self.optimizer = optim 225 | 226 | def _tofetch(self): 227 | return { 228 | 'optimizer': self.optimizer, 229 | 'loss': self.loss, 230 | 'distance_loss': self.distance_loss, 231 | 'magnitude_loss': self.magnitude_loss, 232 | 'depth_loss': self.depth_loss, 233 | 'azimuth_loss': self.azimuth_loss, 234 | 'detection_accuracy': self.detection_accuracy, 235 | 'distance_accuracy': self.distance_accuracy, 236 | 'magnitude_accuracy': self.magnitude_accuracy, 237 | 'depth_accuracy': self.depth_accuracy, 238 | 'azimuth_accuracy': self.azimuth_accuracy 239 | } 240 | 241 | def _summary_step(self, step_data): 242 | step = step_data['step'] 243 | loss = step_data['loss'] 244 | distance_loss = step_data['distance_loss'] 245 | magnitude_loss = step_data['magnitude_loss'] 246 | depth_loss = step_data['depth_loss'] 247 | azimuth_loss = step_data['azimuth_loss'] 248 | det_accuracy = step_data['detection_accuracy'] 249 | dist_accuracy = step_data['distance_accuracy'] 250 | mag_accuracy = step_data['magnitude_accuracy'] 251 | dep_accuracy = step_data['depth_accuracy'] 252 | az_accuracy = step_data['azimuth_accuracy'] 253 | duration = step_data['duration'] 254 | avg_duration = 1000 * duration / step 255 | 256 | if self.is_training: 257 | toprint = 'Step {} | {:.0f}s ({:.0f}ms) | loss = {:.4f} (dist:{:.2f} mag:{:.2f} dep:{:.2f} az:{:.2f}) | acc = det:{:.1f}% d:{:.1f}% m:{:.1f}% dp:{:.1f}% az:{:.1f}%'.format( 258 | step, duration, avg_duration, loss, distance_loss, magnitude_loss, depth_loss, azimuth_loss, \ 259 | 100 * det_accuracy, 100 * dist_accuracy, 100 * mag_accuracy, 100 * dep_accuracy, 100 * az_accuracy) 260 | else: 261 | toprint = 'Step {} | {:.0f}s ({:.0f}ms) | acc = det:{:.1f}% dist:{:.1f}% mag:{:.1f}% dep:{:.1f}% az:{:.1f}%'.format( 262 | step, duration, avg_duration, 100 * det_accuracy, 100 * dist_accuracy, 100 * mag_accuracy, 100 * dep_accuracy, 100 * az_accuracy) 263 | 264 | return toprint 265 | 266 | 267 | 268 | class ConvNetQuake7(ConvNetQuake): 269 | 270 | def __init__(self, inputs, config, checkpoint_dir, is_training=False, reuse=False): 271 | 272 | n_conv_layers = 7 273 | super(ConvNetQuake7, self).__init__(inputs, config, checkpoint_dir, n_conv_layers=n_conv_layers, is_training=is_training, 274 | reuse=reuse) 275 | 276 | 277 | class ConvNetQuake6(ConvNetQuake): 278 | 279 | def __init__(self, inputs, config, checkpoint_dir, is_training=False, reuse=False): 280 | 281 | n_conv_layers = 6 282 | super(ConvNetQuake6, self).__init__(inputs, config, checkpoint_dir, n_conv_layers=n_conv_layers, is_training=is_training, 283 | reuse=reuse) 284 | 285 | class ConvNetQuake9(ConvNetQuake): 286 | 287 | def __init__(self, inputs, config, checkpoint_dir, is_training=False, reuse=False): 288 | 289 | n_conv_layers = 9 290 | super(ConvNetQuake9, self).__init__(inputs, config, checkpoint_dir, n_conv_layers=n_conv_layers, is_training=is_training, 291 | reuse=reuse) 292 | 293 | 294 | class ConvNetQuake_ch64_cv8_fc4(ConvNetQuake): 295 | 296 | def __init__(self, inputs, config, checkpoint_dir, is_training=False, reuse=False): 297 | 298 | n_channels=64 299 | n_conv_layers=8 300 | n_fc_layers=4 301 | super(ConvNetQuake_ch64_cv8_fc4, self).__init__(inputs, config, checkpoint_dir, 302 | n_channels=n_channels, n_conv_layers=n_conv_layers, n_fc_layers= n_fc_layers, is_training=is_training, 303 | reuse=reuse) 304 | 305 | 306 | class ConvNetQuake_ch32_cv8_fc4(ConvNetQuake): 307 | 308 | def __init__(self, inputs, config, checkpoint_dir, is_training=False, reuse=False): 309 | 310 | n_channels=32 311 | n_conv_layers=8 312 | n_fc_layers=4 313 | super(ConvNetQuake_ch32_cv8_fc4, self).__init__(inputs, config, checkpoint_dir, 314 | n_channels=n_channels, n_conv_layers=n_conv_layers, n_fc_layers= n_fc_layers, is_training=is_training, 315 | reuse=reuse) 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | -------------------------------------------------------------------------------- /quakenet_ingv/bin/plot_event_results: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | # ------------------------------------------------------------------- 4 | # File: plot_event_results 5 | # Author: Anthony Lomax 6 | # Created: 2018-05-09 7 | # ------------------------------------------------------------------# 8 | 9 | """ Plot event results on an interactive map""" 10 | 11 | # import sys 12 | import argparse 13 | import os 14 | # import shutil 15 | # import time 16 | import cPickle as pickle 17 | 18 | import math 19 | # 20 | # import numpy as np 21 | # import skimage 22 | # import skimage.io 23 | # import skimage.transform 24 | # import tensorflow as tf 25 | # from sklearn.metrics import confusion_matrix 26 | import geopy 27 | from geopy.distance import distance 28 | from geopy.distance import EARTH_RADIUS 29 | from geojson import Polygon 30 | 31 | import obspy.geodetics.base as geo 32 | 33 | from obspy.core import UTCDateTime 34 | from obspy.core import event 35 | from obspy import read_inventory 36 | 37 | import json 38 | import pandas as pd 39 | import numpy as np 40 | import matplotlib.pyplot as plt 41 | import matplotlib as mpl 42 | import folium 43 | #from folium.plugins import MarkerCluster 44 | 45 | import quakenet.util as util 46 | import quakenet.plot as qnplot 47 | 48 | PROB_CUTOFF = 0.01 49 | PROB_CUTOFF_STATS = 0.01 50 | DIST_STEP_REF = 0.1 # deg 51 | AZ_STEP_REF = 1.0 # deg at 90 deg 52 | 53 | DEG2KM = math.pi * EARTH_RADIUS / 180.0 54 | 55 | def get_great_cicle_path(sta_coordinates, distance0, azimuth0): 56 | 57 | origin = geopy.Point(sta_coordinates['latitude'], sta_coordinates['longitude']) 58 | 59 | num_dist = 2 + int((distance0) / DIST_STEP_REF) 60 | dist_step = distance0 / float(num_dist) 61 | 62 | great_circle = [] 63 | dist = dist_step 64 | destination = distance(kilometers=dist * DEG2KM).destination(origin, azimuth0) 65 | last_longitude = destination.longitude 66 | while dist < distance0 + dist_step / 2.0: 67 | destination = distance(kilometers=dist * DEG2KM).destination(origin, azimuth0) 68 | corrected_longitude = destination.longitude \ 69 | if math.fabs(destination.longitude - last_longitude) < 180.0 \ 70 | else destination.longitude - 360.0 if destination.longitude > 0.0 else destination.longitude + 360.0 71 | great_circle.append([destination.latitude, corrected_longitude]) 72 | dist += dist_step 73 | 74 | return great_circle 75 | 76 | 77 | def get_polygon(sta_coordinates, distance0, distance1, azimuth0, azimuth1): 78 | 79 | origin = geopy.Point(sta_coordinates['latitude'], sta_coordinates['longitude']) 80 | 81 | num_dist = 2 + int((distance1 - distance0) / DIST_STEP_REF) 82 | dist_step = (distance1 - distance0) / float(num_dist) 83 | num_az = 2 + int((distance0 / 90.0) * (azimuth1 - azimuth0) / AZ_STEP_REF) 84 | az_step = (azimuth1 - azimuth0) / float(num_az) 85 | 86 | polygon = [] 87 | # start at lower left corner 88 | dist = distance0 89 | az = azimuth0 90 | destination = distance(kilometers=dist * DEG2KM).destination(origin, az) 91 | last_longitude = destination.longitude 92 | # move CCW from lower left to lower right 93 | while az < azimuth1 + az_step / 2.0: 94 | destination = distance(kilometers=dist * DEG2KM).destination(origin, az) 95 | corrected_longitude = destination.longitude \ 96 | if math.fabs(destination.longitude - last_longitude) < 180.0 \ 97 | else destination.longitude - 360.0 if destination.longitude > 0.0 else destination.longitude + 360.0 98 | polygon.append(tuple([round(corrected_longitude, 3), round(destination.latitude, 3)])) 99 | az += az_step 100 | # lower right corner 101 | az = azimuth1 102 | # move CCW from lower right to upper right 103 | while dist < distance1 + dist_step / 2.0: 104 | destination = distance(kilometers=dist * DEG2KM).destination(origin, az) 105 | corrected_longitude = destination.longitude \ 106 | if math.fabs(destination.longitude - last_longitude) < 180.0 \ 107 | else destination.longitude - 360.0 if destination.longitude > 0.0 else destination.longitude + 360.0 108 | polygon.append(tuple([round(corrected_longitude, 3), round(destination.latitude, 3)])) 109 | dist += dist_step 110 | # upper right corner 111 | dist = distance1 112 | # move CCW from upper right to upper left 113 | while az > azimuth0 - az_step / 2.0: 114 | destination = distance(kilometers=dist * DEG2KM).destination(origin, az) 115 | corrected_longitude = destination.longitude \ 116 | if math.fabs(destination.longitude - last_longitude) < 180.0 \ 117 | else destination.longitude - 360.0 if destination.longitude > 0.0 else destination.longitude + 360.0 118 | polygon.append(tuple([round(corrected_longitude, 3), round(destination.latitude, 3)])) 119 | az -= az_step 120 | # upper right corner 121 | az = azimuth0 122 | # move CCW from lower right to upper right 123 | while dist > distance0 - dist_step / 2.0: 124 | destination = distance(kilometers=dist * DEG2KM).destination(origin, az) 125 | corrected_longitude = destination.longitude \ 126 | if math.fabs(destination.longitude - last_longitude) < 180.0 \ 127 | else destination.longitude - 360.0 if destination.longitude > 0.0 else destination.longitude + 360.0 128 | polygon.append(tuple([round(corrected_longitude, 3), round(destination.latitude, 3)])) 129 | dist -= dist_step 130 | # close with lower left corner 131 | dist = distance0 132 | destination = distance(kilometers=dist * DEG2KM).destination(origin, az) 133 | corrected_longitude = destination.longitude \ 134 | if math.fabs(destination.longitude - last_longitude) < 180.0 \ 135 | else destination.longitude - 360.0 if destination.longitude > 0.0 else destination.longitude + 360.0 136 | polygon.append(tuple([round(corrected_longitude, 3), round(destination.latitude, 3)])) 137 | 138 | # get polygon center 139 | az = (azimuth0 + azimuth1) / 2.0 140 | dist = (distance0 + distance1) / 2.0 141 | center_destination = distance(kilometers=dist * DEG2KM).destination(origin, az) 142 | 143 | #print 'polygon', polygon 144 | return Polygon([polygon]), center_destination.latitude, center_destination.longitude 145 | 146 | 147 | 148 | 149 | def main(args): 150 | 151 | # get event file 152 | event_xml_file = None 153 | for root, dirs, files in os.walk(args.eventpath): 154 | for f in files: 155 | if f.endswith('.event.xml'): 156 | event_xml_file = f 157 | break 158 | if event_xml_file == None: 159 | return 160 | 161 | event_root_name = event_xml_file.replace('.event.xml', '') 162 | catalog = event.read_events(os.path.join(args.eventpath, event_xml_file)) 163 | evt = catalog[0] 164 | origin = evt.preferred_origin() 165 | print 'evt', evt 166 | event_preferred_magnitude = evt.preferred_magnitude().mag 167 | print 'event_preferred_magnitude', event_preferred_magnitude 168 | origin_str = 'M=' + str(event_preferred_magnitude) + ' ' + evt.event_descriptions[0].text + '\n' 169 | origin_str += ' '+ origin.time.isoformat() + '\n' 170 | origin_str += ' '+ 'Lat:' + str(origin.latitude) + ' Lon:' + str(origin.longitude) + ' Depth:' + str(origin.depth / 1000.0) + 'km' + '\n' 171 | print origin_str 172 | origin_str_line = origin_str.replace('\n', ' ') 173 | with open(os.path.join(args.eventpath, event_root_name + '.txt'), 'w') as file: 174 | file.write(origin_str_line) 175 | file.write('\n') 176 | 177 | 178 | # get channel files 179 | sta_probs_files = [] 180 | for root, dirs, files in os.walk(args.eventpath): 181 | for f in files: 182 | if f.endswith('.probs.pkl'): 183 | sta_probs_files.append(f) 184 | num_sta_probs = len(sta_probs_files) 185 | 186 | event_prob_magnitude = [] 187 | event_prob_depth = [] 188 | 189 | epi_sta_poly_coords = [] 190 | 191 | distance_error_sum = 0.0 192 | tot_prob_sum = 0.0 193 | distance_sum = 0.0 194 | tot_sta_sum = 0.0 195 | 196 | sta_feature_groups = [] 197 | feature_group_gc_lines = folium.map.FeatureGroup(name='sta great-circles', overlay=True, control=True) 198 | 199 | # process each channel, build probability feature polygons 200 | for sta_prob_file in sta_probs_files: 201 | 202 | sta_root_name = sta_prob_file.replace('.probs.pkl', '') 203 | print sta_root_name 204 | # 2009-01-06T16-09-06.750.MN.DIVS..BHZ.probs.pkl 205 | 206 | event_date, dsec, net, sta, loc, chan = sta_root_name.split('.') 207 | seed_id = net + '.' + sta + '.' + loc + '.' + chan 208 | 209 | feature_group = folium.map.FeatureGroup(name=seed_id, overlay=True, control=True) 210 | 211 | # get station coordinates 212 | sta_xml_file = os.path.join(args.eventpath, sta_root_name + '.station.xml') 213 | inventory = read_inventory(sta_xml_file) 214 | sta_coordinates = inventory.get_coordinates(seed_id, UTCDateTime(event_date)) 215 | #print ' ', seed_id, sta_coordinates['latitude'], sta_coordinates['longitude'], sta_coordinates['elevation'], sta_coordinates['local_depth'] 216 | 217 | marker = folium.CircleMarker([sta_coordinates['latitude'], sta_coordinates['longitude']], \ 218 | radius=5, popup=seed_id, stroke=True, weight=5, color='green') 219 | feature_group.add_child(marker) 220 | 221 | 222 | # get probabilities 223 | with open(os.path.join(args.eventpath, sta_prob_file), 'r') as file: 224 | stream_params = pickle.load(file) 225 | pred_prob_distance = pickle.load(file) 226 | pred_prob_magnitude = pickle.load(file) 227 | pred_prob_depth = pickle.load(file) 228 | pred_prob_azimuth = pickle.load(file) 229 | 230 | print ' ', 'n_distance', stream_params.n_distances, len(pred_prob_distance), \ 231 | 'n_magnitudes', stream_params.n_magnitudes, len(pred_prob_magnitude), \ 232 | 'n_depths', stream_params.n_depths, len(pred_prob_depth), \ 233 | 'n_azimuths', stream_params.n_azimuths, len(pred_prob_azimuth) 234 | 235 | dist_az_prob_accepted = False 236 | 237 | for ndist in range(1, len(pred_prob_distance)): 238 | distance0 = util.classification2distance(ndist - 1, len(pred_prob_distance)) 239 | distance1 = util.classification2distance(ndist - 1 + 1, len(pred_prob_distance)) 240 | dist_prob = pred_prob_distance[ndist] 241 | #print ' ', ' ', 'dist:', ndist, distance, dist_prob 242 | for naz in range(len(pred_prob_azimuth)): 243 | az_prob = pred_prob_azimuth[naz] 244 | tot_prob = dist_prob * az_prob 245 | # plot if total prob high enough 246 | if tot_prob >= PROB_CUTOFF: 247 | dist_az_prob_accepted = True 248 | azimuth0 = util.classification2azimuth(naz, len(pred_prob_azimuth)) 249 | azimuth1 = util.classification2azimuth(naz + 1, len(pred_prob_azimuth)) 250 | opacity = float(tot_prob) 251 | # line from epicenter to center of polygon 252 | line_opacity = min(opacity, 0.8) 253 | line_opacity = max(line_opacity, 0.1) 254 | great_cicle_path = get_great_cicle_path(sta_coordinates, distance0 , (azimuth0 + azimuth1) / 2.0) 255 | polyline = folium.PolyLine(great_cicle_path, color='#888888', weight=2, opacity=line_opacity) 256 | feature_group_gc_lines.add_child(polyline) 257 | # dist-az geographic polygon 258 | polygon, lat_cent, lon_cent = get_polygon(sta_coordinates, distance0, distance1, azimuth0, azimuth1) 259 | geo_json = folium.GeoJson(polygon, 260 | style_function=lambda feature, opacity=opacity: { 261 | 'weight': 0, 262 | 'fillColor': 'blue', 263 | 'fillOpacity': opacity 264 | }) 265 | feature_group.add_child(geo_json) 266 | # calc stats if total prob high enough 267 | if tot_prob >= PROB_CUTOFF_STATS: 268 | # accumulate epicenter, station and polygon coordinates 269 | coords = (origin.latitude, origin.longitude, origin.depth / 1000.0, 270 | seed_id, sta_coordinates['latitude'], sta_coordinates['longitude'], 271 | lat_cent, lon_cent, tot_prob) 272 | epi_sta_poly_coords.append(coords) 273 | # accumulate mean epicenter station stats 274 | dist_meters, azim, bazim = geo.gps2dist_azimuth(origin.latitude, origin.longitude, lat_cent, lon_cent) 275 | distance_error = geo.kilometer2degrees(dist_meters / 1000.0, radius=6371) 276 | distance_error_sum += distance_error * tot_prob 277 | tot_prob_sum += tot_prob 278 | 279 | dist_meters, azim, bazim = geo.gps2dist_azimuth(origin.latitude, origin.longitude, sta_coordinates['latitude'], sta_coordinates['longitude']) 280 | dist = geo.kilometer2degrees(dist_meters / 1000.0, radius=6371) 281 | distance_sum += dist 282 | tot_sta_sum += 1 283 | 284 | sta_feature_groups.append(feature_group) 285 | # only use magnitude and depth probs if at least one dist-az bin had tot_prob >= PROB_CUTOFF 286 | if dist_az_prob_accepted: 287 | event_prob_magnitude.append(pred_prob_magnitude) 288 | event_prob_depth.append(pred_prob_depth) 289 | 290 | # save epicenter, station stats 291 | mean_distance_error = 0.0 292 | if tot_prob_sum > 0.0: 293 | mean_distance_error= distance_error_sum / tot_prob_sum 294 | mean_sta_epi_distance = 0.0 295 | if tot_sta_sum > 0: 296 | mean_sta_epi_distance = distance_sum / float(tot_sta_sum) 297 | epi_sta_stats = (origin.latitude, origin.longitude, origin.depth / 1000.0, mean_distance_error, tot_prob_sum, mean_sta_epi_distance, tot_sta_sum) 298 | filename = os.path.join(args.eventpath, event_root_name + '.epi_sta_stats.pkl') 299 | with open(filename, 'w') as file: 300 | file.write(pickle.dumps(epi_sta_stats)) # use `pickle.loads` to do the reverse 301 | 302 | # plot 303 | zoom_start = 4 304 | location = [origin.latitude, origin.longitude] 305 | if mean_sta_epi_distance > 90.0: 306 | zoom_start = 0 307 | location = [0.0, 0.0] 308 | elif mean_sta_epi_distance > 60.0: 309 | zoom_start = 1 310 | location = [0.0, 0.0] 311 | elif mean_sta_epi_distance > 40.0: 312 | zoom_start = 2 313 | elif mean_sta_epi_distance > 20.0: 314 | zoom_start = 3 315 | folium_map = folium.Map(tiles='CartoDB positron', \ 316 | prefer_canvas=True, control_scale=True, zoom_start = zoom_start, location=location) 317 | lower_left = [-90, -180] 318 | upper_right = [90, 180] 319 | 320 | feature_group = folium.map.FeatureGroup(name=event_root_name, overlay=True, control=True) 321 | folium_map.add_child(feature_group) 322 | 323 | folium_map.add_child(feature_group_gc_lines) 324 | for feature_group in sta_feature_groups: 325 | folium_map.add_child(feature_group) 326 | 327 | #marker = folium.features.Marker([origin.latitude, origin.longitude], \ 328 | # icon=folium.Icon(prefix='fa', icon='fa-star', color='red'), popup=origin_str) 329 | marker = folium.CircleMarker([origin.latitude, origin.longitude], \ 330 | radius=10, popup=origin_str, stroke=True, weight=6, color='red') 331 | feature_group.add_child(marker) 332 | 333 | folium.LayerControl().add_to(folium_map) 334 | htmlfile = event_root_name + '.html' 335 | folium_map.save(os.path.join(args.eventpath, htmlfile)) 336 | print 'HTML output written to' 337 | print htmlfile 338 | 339 | 340 | # magnitude bar chart 341 | v_axis_direction = -1 342 | qnplot.write_bar_chart_html('Magnitude', '', "Relative probability", event_preferred_magnitude, os.path.join(args.eventpath, event_root_name + '.magnitude.html'), \ 343 | event_prob_magnitude, PROB_CUTOFF, util.classification2magnitude, 344 | v_axis_direction) 345 | 346 | # depth bar chart 347 | v_axis_direction = 1 348 | qnplot.write_bar_chart_html('Depth', '(km)', "Relative probability", origin.depth / 1000.0, os.path.join(args.eventpath, event_root_name + '.depth.html'), \ 349 | event_prob_depth, PROB_CUTOFF, util.classification2depth, 350 | v_axis_direction) 351 | 352 | # save epicenter, station and polygon coordinates list 353 | filename = os.path.join(args.eventpath, event_root_name + '.epi_sta_poly.pkl') 354 | with open(filename, 'w') as file: 355 | file.write(pickle.dumps(epi_sta_poly_coords)) # use `pickle.loads` to do the reverse 356 | 357 | 358 | if __name__ == '__main__': 359 | 360 | parser = argparse.ArgumentParser() 361 | parser.add_argument('--eventpath', type=str, 362 | help='Path for event input') 363 | 364 | args = parser.parse_args() 365 | main(args) 366 | -------------------------------------------------------------------------------- /quakenet_ingv/bin/evaluate: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # encoding: utf-8 3 | # Created: 2016-10-25 4 | # ------------------------------------------------------------------- 5 | # File: evaluate 6 | # Author: Thibaut Perol 7 | # Created: 2016-11-16 8 | # ------------------------------------------------------------------# 9 | 10 | """ Test a model on tfrecords""" 11 | 12 | import sys 13 | import argparse 14 | import os 15 | import shutil 16 | import time 17 | import json 18 | import pandas as pd 19 | import cPickle as pickle 20 | 21 | import numpy as np 22 | import skimage 23 | import skimage.io 24 | import skimage.transform 25 | import tensorflow as tf 26 | from sklearn.metrics import confusion_matrix 27 | from obspy.core import UTCDateTime 28 | from obspy.core import event 29 | from obspy import read_inventory 30 | 31 | import quakenet.util as util 32 | import quakenet.models as models 33 | from quakenet.data_pipeline import DataPipeline 34 | import quakenet.config as config 35 | 36 | 37 | def main(args): 38 | 39 | # get streams parameters 40 | with open(os.path.join(args.outpath, 'params.pkl'), 'r') as file: 41 | stream_params = pickle.load(file) 42 | 43 | 44 | n_distances = stream_params.n_distances 45 | 46 | if args.use_magnitudes: 47 | n_magnitudes = stream_params.n_magnitudes 48 | else: 49 | n_magnitudes = 0 50 | 51 | if args.use_depths: 52 | n_depths = stream_params.n_depths 53 | else: 54 | n_depths = 0 55 | 56 | if args.use_azimuths: 57 | n_azimuths = stream_params.n_azimuths 58 | else: 59 | n_azimuths = 0 60 | 61 | 62 | if not args.noise and not args.events: 63 | raise ValueError('Define if evaluating accuracy on noise or events') 64 | 65 | if args.noise and args.events: 66 | raise ValueError('Define if evaluating accuracy on either noise or events') 67 | 68 | 69 | # Directory in which the evaluation summaries are written 70 | if args.noise: 71 | summary_dir = os.path.join(args.checkpoint_dir,'noise') 72 | elif args.events: 73 | summary_dir = os.path.join(args.checkpoint_dir,'events') 74 | 75 | if args.save_false: 76 | false_start = [] 77 | false_end = [] 78 | false_origintime =[] 79 | false_dir = os.path.join('output','false_predictions') 80 | if not os.path.exists(false_dir): 81 | os.makedirs(false_dir) 82 | 83 | while True: 84 | ckpt = tf.train.get_checkpoint_state(args.checkpoint_dir) 85 | if args.eval_interval < 0 or ckpt: 86 | print 'Evaluating model' 87 | break 88 | print 'Waiting for training job to save a checkpoint' 89 | time.sleep(args.eval_interval) 90 | 91 | cfg = config.Config() 92 | if args.noise: 93 | cfg.batch_size = 256 94 | if args.events: 95 | cfg.batch_size = 1 96 | if args.save_false: 97 | cfg.batch_size = 1 98 | cfg.n_epochs = 1 99 | 100 | cfg.n_distances = n_distances 101 | cfg.n_distances += 1 102 | cfg.add = 1 103 | 104 | cfg.n_magnitudes = n_magnitudes 105 | cfg.n_depths = n_depths 106 | cfg.n_azimuths = n_azimuths 107 | 108 | cfg.unique_station = args.unique_station 109 | 110 | # read list of channels to use 111 | inventory_full = read_inventory(args.channel_file) 112 | 113 | while True: 114 | try: 115 | # data pipeline 116 | data_pipeline = DataPipeline(args.dataset, config=cfg, is_training=False) 117 | samples = { 118 | 'name': data_pipeline.names, 119 | 'data': data_pipeline.samples, 120 | 'stream_max': data_pipeline.stream_max, 121 | 'distance_id': data_pipeline.distance_id, 122 | 'magnitude_id': data_pipeline.magnitude_id, 123 | 'depth_id': data_pipeline.depth_id, 124 | 'azimuth_id': data_pipeline.azimuth_id, 125 | 'start_time': data_pipeline.start_time, 126 | 'end_time': data_pipeline.end_time, 127 | 'distance': data_pipeline.distance, 128 | 'magnitude': data_pipeline.magnitude, 129 | 'depth': data_pipeline.depth, 130 | 'azimuth': data_pipeline.azimuth 131 | } 132 | 133 | # 20180521 AJL 134 | # slice data to requested number of points 135 | if args.ndatapoints > 0: 136 | samples['data'] = samples['data'][ : , 0:args.ndatapoints] 137 | 138 | # set up model and validation metrics 139 | model = models.get(args.model, samples, cfg, 140 | args.checkpoint_dir, 141 | is_training=False) 142 | metrics = model.validation_metrics() 143 | # Validation summary writer 144 | # 20180417 AJL summary_writer = tf.train.SummaryWriter(summary_dir, None) 145 | summary_writer = tf.summary.FileWriter(summary_dir, None) 146 | 147 | with tf.Session() as sess: 148 | coord = tf.train.Coordinator() 149 | tf.initialize_local_variables().run() 150 | threads = tf.train.start_queue_runners(sess=sess, coord=coord) 151 | 152 | model.load(sess, args.step) 153 | print 'Evaluating at step {}'.format(sess.run(model.global_step)) 154 | 155 | step = tf.train.global_step(sess, model.global_step) 156 | mean_metrics = {} 157 | for key in metrics: 158 | mean_metrics[key] = 0 159 | 160 | # open results to file 161 | evaluate_results_file_name = os.path.join(args.checkpoint_dir,'evaluate_results.txt') 162 | evaluate_results_file = open(evaluate_results_file_name,'w') 163 | 164 | if args.save_event_results: 165 | event_results_dirname = os.path.join(args.checkpoint_dir, 'event_results') 166 | if not os.path.exists(event_results_dirname): 167 | os.makedirs(event_results_dirname) 168 | 169 | n_metrics = 0 170 | pred_labels_distance = np.empty(1) 171 | true_labels_distance = np.empty(1) 172 | pred_labels_magnitude = np.empty(1) 173 | true_labels_magnitude = np.empty(1) 174 | pred_labels_depth = np.empty(1) 175 | true_labels_depth = np.empty(1) 176 | pred_labels_azimuth = np.empty(1) 177 | true_labels_azimuth = np.empty(1) 178 | while True: 179 | try: 180 | to_fetch = [metrics, 181 | samples['name'], 182 | model.layers['distance_logits'], 183 | model.layers['distance_prob'], 184 | model.layers['distance_prediction'], 185 | samples['distance_id'], 186 | samples['distance'], 187 | model.layers['magnitude_logits'], 188 | model.layers['magnitude_prob'], 189 | model.layers['magnitude_prediction'], 190 | samples['magnitude_id'], 191 | samples['magnitude'], 192 | model.layers['depth_logits'], 193 | model.layers['depth_prob'], 194 | model.layers['depth_prediction'], 195 | samples['depth_id'], 196 | samples['depth'], 197 | model.layers['azimuth_logits'], 198 | model.layers['azimuth_prob'], 199 | model.layers['azimuth_prediction'], 200 | samples['azimuth_id'], 201 | samples['azimuth'], 202 | samples['start_time'], 203 | samples['end_time'] 204 | ] 205 | metrics_, \ 206 | batch_names, \ 207 | batch_pred_logits_distance, batch_pred_prob_distance, batch_pred_label_distance, batch_true_label_distance, batch_true_distance, \ 208 | batch_pred_logits_magnitude, batch_pred_prob_magnitude, batch_pred_label_magnitude, batch_true_label_magnitude, batch_true_magnitude, \ 209 | batch_pred_logits_depth, batch_pred_prob_depth, batch_pred_label_depth, batch_true_label_depth, batch_true_depth, \ 210 | batch_pred_logits_azimuth, batch_pred_prob_azimuth, batch_pred_label_azimuth, batch_true_label_azimuth, batch_true_azimuth, \ 211 | starttime, endtime = sess.run(to_fetch) 212 | 213 | batch_pred_label_distance -=1 214 | pred_labels_distance = np.append(pred_labels_distance, batch_pred_label_distance) 215 | true_labels_distance = np.append(true_labels_distance, batch_true_label_distance) 216 | 217 | pred_labels_magnitude = np.append(pred_labels_magnitude, batch_pred_label_magnitude) 218 | true_labels_magnitude = np.append(true_labels_magnitude, batch_true_label_magnitude) 219 | 220 | pred_labels_depth = np.append(pred_labels_depth, batch_pred_label_depth) 221 | true_labels_depth = np.append(true_labels_depth, batch_true_label_depth) 222 | 223 | pred_labels_azimuth = np.append(pred_labels_azimuth, batch_pred_label_azimuth) 224 | true_labels_azimuth = np.append(true_labels_azimuth, batch_true_label_azimuth) 225 | 226 | # Save times of false preds 227 | if args.save_false and \ 228 | batch_pred_label_distance != batch_true_label_distance: 229 | print '---False prediction---' 230 | print UTCDateTime(starttime), UTCDateTime(endtime) 231 | false_origintime.append((starttime[0]+endtime[0])/2) 232 | false_end.append(UTCDateTime(endtime)) 233 | false_start.append(UTCDateTime(starttime)) 234 | 235 | # print results to file 236 | for n in range(len(batch_names)): 237 | mag = -999.0 238 | depth = -999.0 239 | azimuth = -999.0 240 | vsum = 0.0 241 | wt = 0.0 242 | for m in range(len(batch_pred_prob_distance[n])): 243 | vsum += batch_pred_prob_distance[n, m] * util.classification2distance(float(m) + 0.5, cfg.n_distances) 244 | wt += batch_pred_prob_distance[n, m] 245 | dist_deg = vsum / wt 246 | #print 'Dist_t/p/pv/l', batch_true_label_distance[n], batch_pred_label_distance[n], dist_deg, metrics_['distance_loss'], 247 | evaluate_results_file.write( 'Dist_t/p/pv/l {:d} {:d} {:.2f} {:.5f}'.format( \ 248 | batch_true_label_distance[n], batch_pred_label_distance[n], dist_deg, metrics_['distance_loss'])) 249 | if cfg.n_magnitudes > 0: 250 | vsum = 0.0 251 | wt = 0.0 252 | for m in range(len(batch_pred_prob_magnitude[n])): 253 | vsum += batch_pred_prob_magnitude[n, m] * util.classification2magnitude(float(m) + 0.5, cfg.n_magnitudes) 254 | wt += batch_pred_prob_magnitude[n, m] 255 | mag = vsum / wt 256 | #print 'Mag_t/p/pv/l', batch_true_label_magnitude[n], batch_pred_label_magnitude[n], mag, metrics_['magnitude_loss'], 257 | evaluate_results_file.write( ' Mag_t/p/pv/l {:d} {:d} {:.2f} {:.5f}'.format( \ 258 | batch_true_label_magnitude[n], batch_pred_label_magnitude[n], mag, metrics_['magnitude_loss'])) 259 | if cfg.n_depths > 0: 260 | vsum = 0.0 261 | wt = 0.0 262 | for m in range(len(batch_pred_prob_depth[n])): 263 | vsum += batch_pred_prob_depth[n, m] * util.classification2depth(float(m) + 0.5, cfg.n_depths) 264 | wt += batch_pred_prob_depth[n, m] 265 | depth = vsum / wt 266 | #print 'Dep_t/p/pv/l', batch_true_label_depth[n], batch_pred_label_depth[n], depth, metrics_['depth_loss'], 267 | evaluate_results_file.write( ' Dep_t/p/pv/l {:d} {:d} {:.1f} {:.5f}'.format( \ 268 | batch_true_label_depth[n], batch_pred_label_depth[n], depth, metrics_['depth_loss'])) 269 | if cfg.n_azimuths > 0: 270 | vsum = 0.0 271 | wt = 0.0 272 | for m in range(len(batch_pred_prob_azimuth[n])): 273 | vsum += batch_pred_prob_azimuth[n, m] * util.classification2azimuth(float(m) + 0.5, cfg.n_azimuths) 274 | wt += batch_pred_prob_azimuth[n, m] 275 | azimuth = vsum / wt 276 | #print 'Az_t/p/pv/l', batch_true_label_azimuth[n], batch_pred_label_azimuth[n], azimuth, metrics_['azimuth_loss'], 277 | evaluate_results_file.write( ' Az_t/p/pv/l {:d} {:d} {:.1f} {:.5f}'.format( \ 278 | batch_true_label_azimuth[n], batch_pred_label_azimuth[n], azimuth, metrics_['azimuth_loss'])) 279 | evaluate_results_file.write(' {:s}'.format(batch_names[n])) 280 | evaluate_results_file.write('\n') 281 | 282 | # write event results to event directory 283 | if args.save_event_results and batch_true_distance[n] > 0: 284 | fpath, fname = os.path.split(batch_names[n]) 285 | event_name = fname.replace('.tfrecords:0', '') 286 | xml_path = os.path.join(fpath, 'xml', event_name + '.xml') 287 | catalog = event.read_events(xml_path, 'QUAKEML') 288 | evt = catalog[0] 289 | origin = evt.preferred_origin() 290 | event_name = origin.time.format_iris_web_service().replace(':', '-') 291 | event_dirname = os.path.join(event_results_dirname, event_name) 292 | if not os.path.exists(event_dirname): 293 | os.makedirs(event_dirname) 294 | # save event information 295 | filename = os.path.join(event_dirname, event_name + '.event.xml') 296 | evt.write(filename, format='QUAKEML') 297 | # write channel event results 298 | net, sta, loc_raw, remainder = fname.split('_') 299 | tokens = remainder.split('.') 300 | chan = tokens[0] 301 | timestr = remainder.replace(chan + '.', '').replace('.tfrecords:0', '') 302 | loc = '-' if loc_raw == '' else loc_raw 303 | # write summary results file 304 | filename = os.path.join(event_dirname, event_name + '.sum.txt') 305 | with open(filename, 'a') as file: 306 | file.write('{:s} {:s} {:s} {:s} {:s} Dist_t/p {:.2f} {:.2f} Mag_t/p {:.2f} {:.2f} Dep_t/p {:.1f} {:.1f} Az_t/p {:.1f} {:.1f}' \ 307 | .format(net, sta, loc, chan, timestr, \ 308 | batch_true_distance[n], dist_deg, batch_true_magnitude[n], mag, \ 309 | batch_true_depth[n], depth, batch_true_azimuth[n], azimuth)) 310 | file.write('\n') 311 | # write detailed results file 312 | filename = os.path.join(event_dirname, event_name + '.details.txt') 313 | with open(filename, 'a') as file: 314 | file.write('{:s} {:s} {:s} {:s} {:s} Dist_t/p {:.2f} {:.2f} Mag_t/p {:.2f} {:.2f} Dep_t/p {:.1f} {:.1f} Az_t/p {:.1f} {:.1f}' \ 315 | .format(net, sta, loc, chan, timestr, \ 316 | batch_true_distance[n], dist_deg, batch_true_magnitude[n], mag, \ 317 | batch_true_depth[n], depth, batch_true_azimuth[n], azimuth)) 318 | file.write('\n') 319 | for vlist in [batch_pred_prob_distance[n], batch_pred_prob_magnitude[n], batch_pred_prob_depth[n], batch_pred_prob_azimuth[n]]: 320 | for m in range(len(vlist)): 321 | file.write(' {:.2f}'.format(vlist[m])) 322 | file.write('\n') 323 | # pickle detailed results 324 | filename_root = event_name + '.' + net + '.' + sta + '.' + loc_raw + '.' + chan + 'Z' 325 | filename = os.path.join(event_dirname, filename_root + '.probs.pkl') 326 | with open(filename, 'w') as file: 327 | file.write(pickle.dumps(stream_params)) # use `pickle.loads` to do the reverse 328 | for vlist in [batch_pred_prob_distance[n], batch_pred_prob_magnitude[n], batch_pred_prob_depth[n], batch_pred_prob_azimuth[n]]: 329 | file.write(pickle.dumps(vlist)) # use `pickle.loads` to do the reverse 330 | # save channel information 331 | chan_inv = inventory_full.select(network=net, station=sta, location=loc_raw, channel=chan + 'Z', time=origin.time) 332 | filename = os.path.join(event_dirname, filename_root + '.station.xml') 333 | chan_inv.write(filename, format='STATIONXML') 334 | 335 | #print 'logits', batch_pred_logits[n] 336 | #print 'prob', batch_pred_prob_distance[n] 337 | 338 | for key in metrics: 339 | mean_metrics[key] += cfg.batch_size*metrics_[key] 340 | n_metrics += cfg.batch_size 341 | 342 | mess = model.validation_metrics_message(metrics_) 343 | # 20180420 AJL print '{:03d} | '.format(n)+mess 344 | 345 | except KeyboardInterrupt: 346 | print 'stopping evaluation' 347 | break 348 | 349 | except tf.errors.OutOfRangeError: 350 | print 'Evaluation completed ({} epochs).'.format(cfg.n_epochs) 351 | print '{} windows seen'.format(n_metrics) 352 | break 353 | 354 | 355 | evaluate_results_file.close() 356 | print 'Evaluation results written to: ', evaluate_results_file_name 357 | 358 | 359 | if n_metrics > 0: 360 | for key in metrics: 361 | mean_metrics[key] /= n_metrics 362 | summary = tf.Summary(value=[tf.Summary.Value( 363 | tag='{}/val'.format(key), simple_value=mean_metrics[key])]) 364 | if args.save_summary: 365 | summary_writer.add_summary(summary, global_step=step) 366 | 367 | summary_writer.flush() 368 | 369 | mess = model.validation_metrics_message(mean_metrics) 370 | print 'Average | '+mess 371 | 372 | if args.eval_interval < 0 or args.save_event_results: 373 | print 'End of evaluation' 374 | break 375 | 376 | tf.reset_default_graph() 377 | print 'Sleeping for {}s'.format(args.eval_interval) 378 | time.sleep(args.eval_interval) 379 | 380 | except KeyboardInterrupt: 381 | print 'stopping evaluation' 382 | break 383 | 384 | finally: 385 | print 'joining data threads' 386 | coord.request_stop() 387 | 388 | 389 | if args.save_false: 390 | false_preds = {} 391 | false_preds['start_time'] = false_start 392 | false_preds['end_time'] = false_end 393 | false_preds['origintime'] = false_origintime 394 | # false_preds = np.array((false_start, false_end)).transpose()[0] 395 | # print 'shape', false_preds.shape 396 | df = pd.DataFrame(false_preds) 397 | df.to_csv(os.path.join(false_dir,'false_preds.csv')) 398 | 399 | if args.save_event_results: 400 | return 401 | 402 | pred_labels_distance = pred_labels_distance[1::] 403 | true_labels_distance = true_labels_distance[1::] 404 | # np.save('output/pred_labels_noise.npy',pred_labels) 405 | # np.save('output/true_labels_noise.npy',true_labels) 406 | print '---Confusion Matrix Distance----' 407 | print confusion_matrix(true_labels_distance, pred_labels_distance) 408 | pred_labels_magnitude = pred_labels_magnitude[1::] 409 | true_labels_magnitude = true_labels_magnitude[1::] 410 | print '---Confusion Matrix Magnitude----' 411 | print confusion_matrix(true_labels_magnitude, pred_labels_magnitude) 412 | pred_labels_depth = pred_labels_depth[1::] 413 | true_labels_depth = true_labels_depth[1::] 414 | print '---Confusion Matrix Depth----' 415 | print confusion_matrix(true_labels_depth, pred_labels_depth) 416 | pred_labels_azimuth = pred_labels_azimuth[1::] 417 | true_labels_azimuth = true_labels_azimuth[1::] 418 | print '---Confusion Matrix Azimuth----' 419 | print confusion_matrix(true_labels_azimuth, pred_labels_azimuth) 420 | 421 | coord.join(threads) 422 | 423 | # if args.save_false: 424 | # false_preds = np.array((false_start, false_end)).transpose() 425 | # df = pd.Dataframe(false_preds, columns=['start_time, end_time']) 426 | # df.to_csv(os.path.join(false_dir,'false_preds.csv') 427 | 428 | if __name__ == '__main__': 429 | 430 | parser = argparse.ArgumentParser() 431 | parser.add_argument('--outpath', type=str, 432 | help='Path for stream output') 433 | parser.add_argument('--dataset', type=str,default=None, 434 | help='path to the records to evaluate') 435 | parser.add_argument('--unique_station', type=str, default=None) 436 | parser.add_argument('--ndatapoints', type=int, default=-1) 437 | parser.add_argument('--channel_file', type=str, 438 | help='File containing FDSNStationXML list of net/station/location/channel to retrieve') 439 | parser.add_argument('--checkpoint_dir',default='model/convnetquake', 440 | type=str, help='path to checkpoints directory') 441 | parser.add_argument('--step',type=int,default=None, 442 | help='step to load') 443 | parser.add_argument('--model',type=str,default='ConvNetQuake', 444 | help='model to load') 445 | parser.add_argument('--eval_interval',type=int,default=-1, 446 | help='sleep time between evaluations') 447 | parser.add_argument('--save_summary',type=bool,default=True, 448 | help='True to save summary in tensorboard') 449 | parser.add_argument('--noise', action='store_true', 450 | help='pass this flag if evaluate acc on noise') 451 | parser.set_defaults(noise=False) 452 | parser.add_argument('--events', action='store_true', 453 | help='pass this flag if evaluate acc on events') 454 | parser.set_defaults(events=True) 455 | parser.add_argument('--save_event_results', action='store_true', 456 | help='save detailed even results') 457 | parser.set_defaults(save_event_results=False) 458 | parser.add_argument('--save_false', action='store_true', 459 | help='pass this flag to save times of false preds') 460 | parser.set_defaults(profiling=False) 461 | parser.add_argument('--use_magnitudes', action='store_true') 462 | parser.set_defaults(use_magnitudes=False) 463 | parser.add_argument('--use_depths', action='store_true') 464 | parser.set_defaults(use_depths=False) 465 | parser.add_argument('--use_azimuths', action='store_true') 466 | parser.set_defaults(use_azimuths=False) 467 | 468 | args = parser.parse_args() 469 | main(args) 470 | -------------------------------------------------------------------------------- /quakenet_ingv/bin/preprocess/get_streams.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Created on 29 Mar 2018 3 | 4 | @author: Anthony Lomax - ALomax Scientific 5 | ''' 6 | 7 | """Get streams from FDSN web service.""" 8 | 9 | import os 10 | import traceback 11 | import argparse 12 | import random 13 | from datetime import datetime 14 | import cPickle as pickle 15 | 16 | import math 17 | import numpy as np 18 | 19 | from obspy import read_inventory 20 | from obspy.core.event import Catalog 21 | from obspy.core.event.catalog import read_events 22 | import obspy.clients.fdsn as fdsn 23 | from obspy.clients.fdsn.header import (DEFAULT_PARAMETERS) 24 | from obspy.taup import TauPyModel 25 | import obspy.geodetics.base as geo 26 | 27 | import quakenet.util as util 28 | from quakenet.data_pipeline import DataWriter 29 | 30 | 31 | WINDOW_PADDING_FDSN = 60.0 # pad requested time window to avoid truncated waveforms 32 | MIN_INTER_EVENT_TIME = 60.0*120.0 # 2 hours, minimum inter event time in seconds to process for noise waveform 33 | MAX_SNR_NOISE = 1.5 34 | TAU_PY_MODEL = TauPyModel(model='ak135') 35 | 36 | 37 | 38 | def get_first_P_travel_time(origin, channel): 39 | 40 | # get first P arrival travel-time 41 | arrivals = TAU_PY_MODEL.get_travel_times_geo( \ 42 | origin.depth / 1000.0, origin.latitude, origin.longitude, \ 43 | channel.latitude, channel.longitude, \ 44 | phase_list=('ttp', )) 45 | 46 | print 'arrivals', arrivals 47 | 48 | ttime = arrivals[0].time 49 | if arrivals[0].name == 'Pdiff' and len(arrivals) > 1: 50 | ttime = arrivals[1].time 51 | print 'using ', arrivals[1].name 52 | else: 53 | print 'using ', arrivals[0].name 54 | 55 | return ttime 56 | 57 | 58 | def get_systematic_channel(inventory, catalog_all, is_noise, event_ndx, net_ndx, sta_ndx, channel_ndx): 59 | 60 | print 'get_systematic_channel', 'event_ndx', event_ndx, 'net_ndx', net_ndx, 'sta_ndx', sta_ndx, 'channel_ndx', channel_ndx 61 | 62 | net = inventory[net_ndx] 63 | sta = net[sta_ndx] 64 | channel_ndx = channel_ndx + 1 65 | if channel_ndx >= len(sta): 66 | channel_ndx = 0 67 | sta_ndx = sta_ndx + 1 68 | if sta_ndx >= len(net): 69 | sta_ndx = 0 70 | net_ndx = net_ndx + 1 71 | if net_ndx >= len(inventory): 72 | net_ndx = 0 73 | event_ndx = event_ndx + 1 74 | if event_ndx >= len(catalog_all): 75 | raise ValueError("finished processing inventory and events") 76 | net = inventory[net_ndx] 77 | sta = net[sta_ndx] 78 | channel = sta[channel_ndx] 79 | 80 | event = catalog_all[event_ndx] 81 | origin = event.preferred_origin() 82 | 83 | return catalog_all, event_ndx, event, origin, channel, net_ndx, net, sta_ndx, sta, channel_ndx 84 | 85 | 86 | def get_random_channel(inventory, catalog_dict, is_noise): 87 | 88 | nnet = len(inventory) 89 | if nnet > 0: 90 | net_ndx = random.randint(0, nnet - 1) 91 | net = inventory[net_ndx] 92 | nsta = len(net) 93 | if nsta > 0: 94 | sta_ndx = random.randint(0, len(net) - 1) 95 | sta = net[sta_ndx] 96 | net_sta = net.code + '_' + sta.code 97 | if not net_sta in catalog_dict: 98 | raise ValueError("net_sta not in catalog_dict") 99 | nchans = len(sta) 100 | if nchans > 0: 101 | channel_ndx = random.randint(0, nchans - 1) 102 | channel = sta[channel_ndx] 103 | 104 | # get random event from Catalog 105 | catalog = catalog_dict[net_sta] 106 | imax = catalog.count() - 1 107 | if is_noise: 108 | imax = catalog.count() - 2 109 | event_ndx = random.randint(0, imax) 110 | event = catalog[event_ndx] 111 | origin = event.preferred_origin() 112 | 113 | return catalog, event_ndx, event, origin, channel, net_ndx, net, sta_ndx, sta, channel_ndx 114 | 115 | 116 | 117 | def main(args): 118 | 119 | random.seed(datetime.now()) 120 | 121 | if args.n_distances < 1: 122 | args.n_distances = None 123 | # print distance classifications 124 | if args.n_distances != None: 125 | print 'dist_class, dist_deg, dist_km' 126 | for dclass in range(0, args.n_distances, 1): 127 | dist_deg = util.classification2distance(dclass, args.n_distances) 128 | dist_km = geo.degrees2kilometers(dist_deg) 129 | print "{} {:.2f} {:.1f}".format(dclass, dist_deg, dist_km) 130 | print '' 131 | 132 | if args.n_magnitudes < 1: 133 | args.n_magnitudes = None 134 | # print magtitude classifications 135 | if args.n_magnitudes != None: 136 | print 'mag_class, mag' 137 | for mclass in range(0, args.n_magnitudes, 1): 138 | mag = util.classification2magnitude(mclass, args.n_magnitudes) 139 | print "{} {:.2f}".format(mclass, mag) 140 | print '' 141 | 142 | if args.n_depths < 1: 143 | args.n_depths = None 144 | # print depth classifications 145 | if args.n_depths != None: 146 | print 'depth_class, depth' 147 | for dclass in range(0, args.n_depths, 1): 148 | depth = util.classification2depth(dclass, args.n_depths) 149 | print "{} {:.1f}".format(dclass, depth) 150 | print '' 151 | 152 | if args.n_azimuths < 1: 153 | args.n_azimuths = None 154 | # print azimuth classifications 155 | if args.n_azimuths != None: 156 | print 'azimuth_class, azimuth' 157 | for aclass in range(0, args.n_azimuths, 1): 158 | azimuth = util.classification2azimuth(aclass, args.n_azimuths) 159 | print "{} {:.1f}".format(aclass, azimuth) 160 | print '' 161 | 162 | 163 | if not os.path.exists(args.outpath): 164 | os.makedirs(args.outpath) 165 | 166 | # save arguments 167 | with open(os.path.join(args.outpath, 'params.pkl'), 'w') as file: 168 | file.write(pickle.dumps(args)) # use `pickle.loads` to do the reverse 169 | 170 | for dataset in ['train', 'validate', 'test']: 171 | for datatype in ['events', 'noise']: 172 | datapath = os.path.join(args.outpath, dataset, datatype) 173 | if not os.path.exists(datapath): 174 | os.makedirs(datapath) 175 | mseedpath = os.path.join(datapath, 'mseed') 176 | if not os.path.exists(mseedpath): 177 | os.makedirs(mseedpath) 178 | mseedpath = os.path.join(datapath, 'mseed_raw') 179 | if not os.path.exists(mseedpath): 180 | os.makedirs(mseedpath) 181 | if datatype == 'events': 182 | xmlpath = os.path.join(datapath, 'xml') 183 | if not os.path.exists(xmlpath): 184 | os.makedirs(xmlpath) 185 | 186 | 187 | # read catalog of events 188 | #filenames = args.event_files_path + os.sep + '*.xml' 189 | catalog_dict = {} 190 | catalog_all = [] 191 | for dirpath, dirnames, filenames in os.walk(args.event_files_path): 192 | for name in filenames: 193 | if name.endswith(".xml"): 194 | file = os.path.join(dirpath, name) 195 | catalog = read_events(file) 196 | target_count = int(args.event_fraction * float(catalog.count())) 197 | print catalog.count(), 'events:', 'read from:', file, 'will use:', target_count, 'since args.event_fraction=', args.event_fraction 198 | if (args.event_fraction < 1.0): 199 | while catalog.count() > target_count: 200 | del catalog[random.randint(0, catalog.count() - 1)] 201 | if not args.systematic: 202 | tokens = name.split('_') 203 | net_sta = tokens[0] + '_' + tokens[1] 204 | if not net_sta in catalog_dict: 205 | catalog_dict[net_sta] = catalog 206 | else: 207 | catalog_dict[net_sta] += catalog 208 | # sort catalog by date 209 | catalog_dict[net_sta] = Catalog(sorted(catalog_dict[net_sta], key=lambda e: e.origins[0].time)) 210 | else: 211 | catalog_all += catalog 212 | 213 | # read list of channels to use 214 | inventory_full = read_inventory(args.channel_file) 215 | inventory_full = inventory_full.select(channel=args.channel_prefix+'Z', sampling_rate=args.sampling_rate) 216 | #print(inventory) 217 | 218 | client = fdsn.Client(args.base_url) 219 | 220 | # get existing already processed event channel dictionary 221 | try: 222 | with open(os.path.join(args.outpath, 'event_channel_dict.pkl'), 'r') as file: 223 | event_channel_dict = pickle.load(file) 224 | except IOError: 225 | event_channel_dict = {} 226 | print 'Existing event_channel_dict size:', len(event_channel_dict) 227 | 228 | n_noise = int(0.5 + float(args.n_streams) * args.noise_fraction) 229 | n_events = args.n_streams - n_noise 230 | n_validate = int(0.5 + float(n_events) * args.validation_fraction) 231 | n_test = int(0.5 + float(n_events) * args.test_fraction) 232 | n_train = n_events - n_validate - n_test 233 | n_count = 0; 234 | n_streams = 0 235 | 236 | if args.systematic: 237 | event_ndx = 0 238 | net_ndx = 0 239 | sta_ndx = 0 240 | channel_ndx = -1 241 | 242 | 243 | 244 | # distance_id_count = {} 245 | # max_num_for_distance_id = {} 246 | # if args.n_distances != None: 247 | # # train 248 | # distance_id_count['train'] = [0] * args.n_distances 249 | # max_num_for_distance_id['train'] = 1 + int(2.0 * float(n_train) / float(args.n_distances)) 250 | # print 'Maximum number events for each distance bin train:', max_num_for_distance_id['train'] 251 | # # validate 252 | # distance_id_count['validate'] = [0] * args.n_distances 253 | # max_num_for_distance_id['validate'] = 1 + int(2.0 * float(n_validate) / float(args.n_distances)) 254 | # print 'Maximum number events for each distance bin validate:', max_num_for_distance_id['validate'] 255 | # # test 256 | # distance_id_count['test'] = [0] * args.n_distances 257 | # max_num_for_distance_id['test'] = 1 + int(2.0 * float(n_test) / float(args.n_distances)) 258 | # print 'Maximum number events for each distance bin test:', max_num_for_distance_id['test'] 259 | 260 | while args.systematic or n_streams < args.n_streams: 261 | 262 | try: 263 | 264 | # choose event or noise 265 | is_noise = n_streams >= n_events 266 | 267 | # reset validate test count if switching from event to noise 268 | if n_streams == n_events: 269 | n_validate = int(0.5 + float(n_noise) * args.validation_fraction) 270 | n_test = int(0.5 + float(n_noise) * args.test_fraction) 271 | n_train = n_noise - n_validate - n_test 272 | n_count = 0; 273 | 274 | # set out paths 275 | if is_noise: 276 | datatype = 'noise' 277 | else: 278 | datatype = 'events' 279 | if n_count < n_train: 280 | dataset = 'train' 281 | elif n_count < n_train + n_validate: 282 | dataset = 'validate' 283 | else: 284 | dataset = 'test' 285 | datapath = os.path.join(args.outpath, dataset, datatype) 286 | 287 | # get random channel from Inventory 288 | #inventory = inventory_full.select(time=origin.time) 289 | inventory = inventory_full 290 | 291 | if args.systematic: 292 | try: 293 | catalog, event_ndx, event, origin, channel, net_ndx, net, sta_ndx, sta, channel_ndx \ 294 | = get_systematic_channel(inventory, catalog_all, is_noise, event_ndx, net_ndx, sta_ndx, channel_ndx) 295 | except ValueError: 296 | break 297 | else: 298 | try: 299 | catalog, event_ndx, event, origin, channel, net_ndx, net, sta_ndx, sta, channel_ndx = get_random_channel(inventory, catalog_dict, is_noise) 300 | except ValueError: 301 | continue 302 | 303 | distance_id = 0 304 | distance = -999.0 305 | magnitude = -999.0 306 | depth = -999.0 307 | azimuth = -999.0 308 | if not is_noise: 309 | dist_meters, azim, bazim = geo.gps2dist_azimuth(channel.latitude, channel.longitude, origin.latitude, origin.longitude, a=geo.WGS84_A, f=geo.WGS84_F) 310 | distance = geo.kilometer2degrees(dist_meters / 1000.0, radius=6371) 311 | azimuth = azim 312 | magnitude = event.preferred_magnitude().mag 313 | depth = origin.depth / 1000.0 314 | if args.n_distances != None: 315 | distance_id = util.distance2classification(distance, args.n_distances) 316 | # if distance_id_count[dataset][distance_id] >= max_num_for_distance_id[dataset]: 317 | # print 'Skipping event_channel: distance bin', distance_id, 'for', dataset, 'already full:', \ 318 | # distance_id_count[dataset][distance_id], '/', max_num_for_distance_id[dataset] 319 | # continue 320 | 321 | print '' 322 | print 'Event:', origin.time.isoformat(), event.event_descriptions[0].text, \ 323 | ', Dist(deg): {:.2f} Dist(km): {:.1f} ID: {}'.format(distance, geo.degrees2kilometers(distance), distance_id), \ 324 | ', Mag: {:.2f}'.format(magnitude), \ 325 | ', Depth(km): {:.1f}'.format(depth), \ 326 | ', Az(deg): {:.1f}'.format(azimuth) 327 | print 'Retrieving channels:', (n_streams + 1), '/ ', args.n_streams, (', NOISE, ' if is_noise else ', EVENT, '), 'event', event_ndx, origin.time, \ 328 | ', net', net_ndx, ', sta', sta_ndx, ', chan', channel_ndx, \ 329 | ', ', net.code, sta.code, \ 330 | channel.code, channel.location_code, \ 331 | channel.sample_rate 332 | # check station was available at origin.time 333 | if not sta.is_active(time=origin.time): 334 | print 'Skipping event_channel: station not active at origin.time:' 335 | continue 336 | #key = str(event_ndx) + '_' + str(net_ndx) + '_' + str(sta_ndx) + '_' + str(channel_ndx) + '_' + str(is_noise) 337 | key = str(event_ndx) + '_' + net.code + '_' + sta.code + '_' + channel.code + '_' + str(is_noise) 338 | if key in event_channel_dict: 339 | print 'Skipping event_channel: already processed.' 340 | continue 341 | event_channel_dict[key] = 1 342 | 343 | # get start time for waveform request 344 | ttime = get_first_P_travel_time(origin, channel) 345 | arrival_time = origin.time + ttime 346 | if is_noise: 347 | # get start time of next event 348 | event2 = catalog[event_ndx + 1] 349 | origin2 = event2.preferred_origin() 350 | # check that origins are at least min time apart 351 | if origin2.time - origin.time < MIN_INTER_EVENT_TIME: 352 | print 'Skipping noise event_channel: inter event time too small: ', str(origin2.time - origin.time), \ 353 | origin2.time, origin.time 354 | continue 355 | ttime2 = get_first_P_travel_time(origin2, channel) 356 | arrival_time2 = origin2.time + ttime2 357 | arrival_time = (arrival_time + ((arrival_time2 - arrival_time) / 2.0)) - args.window_start 358 | 359 | start_time = arrival_time - args.window_start 360 | 361 | # request data for 3 channels 362 | 363 | #for orientation in ['Z', 'N', 'E', '1', '2']: 364 | # req_chan = args.channel_prefix + orientation 365 | channel_name = net.code + '_' + sta.code + '_' + channel.location_code + '_' + args.channel_prefix 366 | padded_start_time = start_time - WINDOW_PADDING_FDSN 367 | padded_end_time = start_time + args.window_length + 2.0 * WINDOW_PADDING_FDSN 368 | chan_param = args.channel_prefix + '?' 369 | # kluge to get url used for data request 370 | kwargs = {'network': net.code, 'station': sta.code, 'location': channel.location_code, 'channel': chan_param, 371 | 'starttime': padded_start_time, 'endtime': padded_end_time} 372 | #url = client._create_url_from_parameters('dataselect', DEFAULT_PARAMETERS['dataselect'], **kwargs) 373 | url = fdsn.client.build_url(client.base_url, 'dataselect', client.major_versions['dataselect'], "query", parameters=kwargs) 374 | print ' java net.alomax.seisgram2k.SeisGram2K', '\"', url, '\"' 375 | try: 376 | stream = client.get_waveforms( \ 377 | net.code, sta.code, channel.location_code, chan_param, \ 378 | padded_start_time, padded_end_time, \ 379 | attach_response=True) 380 | 381 | except fdsn.header.FDSNException as ex: 382 | print 'Skipping channel:', channel_name, 'FDSNException:', ex, 383 | continue 384 | 385 | print stream 386 | # TEST 387 | # for trace in stream: 388 | # print '==========> trace.stats', trace.stats 389 | 390 | # check some things 391 | if (len(stream) != 3): 392 | print 'Skipping channel: len(stream) != 3:', channel_name 393 | continue 394 | ntrace = 0 395 | for trace in stream: 396 | if (len(trace) < 1): 397 | print 'Skipping trace: len(trace) < 1:', channel_name 398 | continue 399 | if (trace.stats.starttime > start_time or trace.stats.endtime < start_time + args.window_length): 400 | print 'Skipping trace: does not contain required time window:', channel_name 401 | continue 402 | ntrace += 1 403 | if (ntrace != 3): 404 | print 'Skipping channel: ntrace != 3:', channel_name 405 | continue 406 | 407 | # pre-process streams 408 | # sort so that channels will be ingested in NN always in same order ENZ 409 | stream.sort(['channel']) 410 | # detrend - this is meant to be equivalent to detrend or a long period low-pass (e.g. at 100sec) applied to real-time data 411 | stream.detrend(type='linear') 412 | for trace in stream: 413 | # correct for required sampling rate 414 | if abs(trace.stats.sampling_rate - args.sampling_rate) / args.sampling_rate > 0.01: 415 | trace.resample(args.sampling_rate) 416 | 417 | # apply high-pass filter if requested 418 | if args.hp_filter_freq > 0.0: 419 | stream.filter('highpass', freq=args.hp_filter_freq, corners=args.hp_filter_corners) 420 | 421 | # check signal to noise ratio, if fail, repeat on 1sec hp data to capture local/regional events in longer period microseismic noise 422 | sn_type = 'BRB' 423 | first_pass = True; 424 | while True: 425 | if is_noise: 426 | snrOK = True 427 | else: 428 | snrOK = False 429 | for trace in stream: 430 | # slice with 1sec margin of error for arrival time to: 1) avoid increasing noise amplitude with signal, 2) avoid missing first P in signal 431 | if (first_pass): 432 | signal_slice = trace.slice(starttime=arrival_time - 1.0, endtime=arrival_time - 1.0 + args.snr_window_length) 433 | noise_slice = trace.slice(endtime=arrival_time - 1.0) 434 | else: 435 | # highpass at 1sec 436 | filt_trace = trace.copy() 437 | filt_trace.filter('highpass', freq=1.0, corners=4) 438 | signal_slice = filt_trace.slice(starttime=arrival_time - 1.0, endtime=arrival_time - 1.0 + args.snr_window_length) 439 | noise_slice = filt_trace.slice(endtime=arrival_time - 1.0) 440 | sn_type = '1HzHP' 441 | # check signal to noise around arrival_time 442 | # ratio of std 443 | asignal = signal_slice.std() 444 | anoise = noise_slice.std() 445 | snr = asignal / anoise 446 | print trace.id, sn_type, 'snr:', snr, 'std_signal:', asignal, 'std_noise:', anoise 447 | # ratio of peak amplitudes (DO NOT USE, GIVE UNSTABLE RESULTS!) 448 | # asignal = signal_slice.max() 449 | # anoise = noise_slice.max() 450 | # snr = np.absolute(asignal / anoise) 451 | # print trace.id, sn_type, 'snr:', snr, 'amax_signal:', asignal, 'amax_noise:', anoise 452 | if is_noise: 453 | snrOK = snrOK and snr <= MAX_SNR_NOISE 454 | if not snrOK: 455 | break 456 | else: 457 | snrOK = snrOK or snr >= args.snr_accept 458 | if (first_pass and not snrOK and args.hp_filter_freq < 0.0): 459 | first_pass = False; 460 | continue 461 | else: 462 | break 463 | 464 | if (not snrOK): 465 | if is_noise: 466 | print 'Skipping channel:', sn_type, 'snr >', MAX_SNR_NOISE, 'on one or more traces:', channel_name 467 | else: 468 | print 'Skipping channel:', sn_type, 'snr < args.snr_accept:', args.snr_accept, 'on all traces:', channel_name 469 | continue 470 | 471 | # trim data to required window 472 | # try to make sure samples and start/end times align as closely as possible to first trace 473 | trace = stream.traces[0] 474 | trace = trace.slice(starttime=start_time, endtime=start_time + args.window_length, nearest_sample=True) 475 | start_time = trace.stats.starttime 476 | stream = stream.slice(starttime=start_time, endtime=start_time + args.window_length, nearest_sample=True) 477 | 478 | cstart_time = '%04d.%02d.%02d.%02d.%02d.%02d.%03d' % \ 479 | (start_time.year, start_time.month, start_time.day, start_time.hour, start_time.minute, \ 480 | start_time.second, start_time.microsecond // 1000) 481 | 482 | # process each trace 483 | try: 484 | for trace in stream: 485 | # correct for overall sensitivity or gain 486 | trace.normalize(trace.stats.response.instrument_sensitivity.value) 487 | trace.data = trace.data.astype(np.float32) 488 | # write miniseed 489 | #tracefile = os.path.join(datapath, 'mseed', trace.id + '.' + cstart_time + '.mseed') 490 | #trace.write(tracefile, format='MSEED', encoding='FLOAT32') 491 | #print 'Channel written:', tracefile, trace.count(), 'samples' 492 | except AttributeError as err: 493 | print 'Skipping channel:', channel_name, ': Error applying trace.normalize():' , err 494 | 495 | filename_root = channel_name + '.' + cstart_time 496 | 497 | # write raw miniseed 498 | streamfile = os.path.join(datapath, 'mseed_raw', filename_root + '.mseed') 499 | stream.write(streamfile, format='MSEED', encoding='FLOAT32') 500 | print 'Stream written:', stream.count(), 'traces:' 501 | print ' java net.alomax.seisgram2k.SeisGram2K', streamfile 502 | 503 | # store absolute maximum 504 | stream_max = np.absolute(stream.max()).max() 505 | # normalize by absolute maximum 506 | stream.normalize(global_max = True) 507 | 508 | # 20180521 AJL 509 | # spherical coordinates 510 | # raw data always in same order ENZ 511 | # tensor indexing is [traces, datapoints, comps] 512 | if args.spherical: 513 | rad2deg = 180.0 / math.pi 514 | # calculate modulus 515 | temp_square = np.add(np.square(stream.traces[0].data), np.add(np.square(stream.traces[1].data), np.square(stream.traces[2].data))) 516 | temp_modulus = np.sqrt(temp_square) 517 | # calculate azimuth 518 | temp_azimuth = np.add( np.multiply(np.arctan2(stream.traces[0].data, stream.traces[1].data), rad2deg), 180.0) 519 | # calculate inclination 520 | temp_inclination = np.multiply(np.arcsin(np.divide(stream.traces[2].data, temp_modulus)), rad2deg) 521 | # reset stream data to spherical coordinates 522 | stream.traces[0].data = temp_inclination 523 | stream.traces[1].data = temp_azimuth 524 | temp_modulus = np.multiply(temp_modulus, 100.0) # increase scale for plotting purposes 525 | stream.traces[2].data = temp_modulus 526 | 527 | 528 | # put absolute maximum normalization in first element of data array, to seed NN magnitude estimation 529 | # 20180816 AJL - do not mix max with data 530 | # for trace in stream: 531 | # trace.data[0] = stream_max 532 | print 'stream_max', stream_max 533 | 534 | 535 | # write processed miniseed 536 | streamfile = os.path.join(datapath, 'mseed', filename_root + '.mseed') 537 | stream.write(streamfile, format='MSEED', encoding='FLOAT32') 538 | print 'Stream written:', stream.count(), 'traces:' 539 | print ' java net.alomax.seisgram2k.SeisGram2K', streamfile 540 | 541 | # write event waveforms and distance_id in .tfrecords 542 | magnitude_id = 0 543 | depth_id = 0 544 | azimuth_id = 0 545 | if not is_noise: 546 | # if args.n_distances != None: 547 | # distance_id_count[dataset][distance_id] += 1 548 | if args.n_magnitudes != None: 549 | magnitude_id = util.magntiude2classification(magnitude, args.n_magnitudes) 550 | if args.n_depths != None: 551 | depth_id = util.depth2classification(depth, args.n_depths) 552 | if args.n_azimuths != None: 553 | azimuth_id = util.azimuth2classification(azimuth, args.n_azimuths) 554 | else: 555 | distance_id = -1 556 | distance = 0.0 557 | output_name = filename_root + '.tfrecords' 558 | output_path = os.path.join(datapath, output_name) 559 | writer = DataWriter(output_path) 560 | writer.write(stream, stream_max, distance_id, magnitude_id, depth_id, azimuth_id, distance, magnitude, depth, azimuth) 561 | if not is_noise: 562 | print '==== Event stream tfrecords written:', output_name, \ 563 | 'Dist(deg): {:.2f} Dist(km): {:.1f} ID: {}'.format(distance, geo.degrees2kilometers(distance), distance_id), \ 564 | ', Mag: {:.2f} ID: {}'.format(magnitude, magnitude_id), \ 565 | ', Depth(km): {:.1f} ID: {}'.format(depth, depth_id), \ 566 | ', Az(deg): {:.1f} ID: {}'.format(azimuth, azimuth_id) 567 | else: 568 | print '==== Noise stream tfrecords written:', output_name, 'ID: Dist {}, Mag {}, Depth {}, Az {}'.format(distance_id, magnitude_id, depth_id, azimuth_id) 569 | 570 | # write event data 571 | if not is_noise: 572 | filename = os.path.join(datapath, 'xml', filename_root + '.xml') 573 | event.write(filename, 'QUAKEML') 574 | 575 | n_streams += 1 576 | n_count += 1 577 | 578 | except KeyboardInterrupt: 579 | print 'Stopping: KeyboardInterrupt' 580 | break 581 | 582 | except Exception as ex: 583 | print 'Skipping stream: Exception:', ex 584 | traceback.print_exc() 585 | continue 586 | 587 | print n_streams, 'streams:', 'written to:', args.outpath 588 | 589 | # save event_channel_dict 590 | with open(os.path.join(args.outpath, 'event_channel_dict.pkl'), 'w') as file: 591 | file.write(pickle.dumps(event_channel_dict)) 592 | 593 | 594 | if __name__ == '__main__': 595 | 596 | parser = argparse.ArgumentParser() 597 | 598 | parser.add_argument('--event_files_path', type=str, 599 | help='Path to the directory of event QuakeML event xml files to process') 600 | parser.add_argument('--event_fraction', type=float, default=1.0, 601 | help='Fraction of events to use, randomly removes events to achieve this fraction of events in xml files') 602 | parser.add_argument('--base_url', type=str, default='IRIS', 603 | help='Base URL of FDSN web service or key string for recognized server') 604 | parser.add_argument('--channel_file', type=str, 605 | help='File containing FDSNStationXML list of net/station/location/channel to retrieve') 606 | parser.add_argument('--channel_prefix', type=str, 607 | help='Prefix of the channel code to retrieve (e.g. BH)') 608 | parser.add_argument('--sampling_rate', type=float, 609 | help='Channel sample rate to retrieve (e.g. 20)') 610 | parser.add_argument('--n_streams', type=int, 611 | help='Number of streams to retrieve') 612 | parser.add_argument('--noise_fraction', type=float, 613 | help='Fraction of streams that are noise') 614 | parser.add_argument('--validation_fraction', type=float, 615 | help='Fraction of streams that are validation data') 616 | parser.add_argument('--test_fraction', type=float, 617 | help='Fraction of streams that are testing data') 618 | parser.add_argument('--window_start', type=float, default=1.0, 619 | help='Start time before first-arriving P/PKP of the trace window (in seconds)') 620 | parser.add_argument('--window_length', type=float, default=50, 621 | help='Length of the trace window (in seconds)') 622 | parser.add_argument('--snr_accept', type=float, default=3.0, 623 | help='Minimum acceptable SNR') 624 | parser.add_argument('--snr_window_length', type=float, default=10, 625 | help='Length of the window after arrival for SNR (in seconds)') 626 | parser.add_argument('--outpath', type=str, 627 | help='Path for stream output') 628 | 629 | parser.add_argument('--hp_filter_freq', type=float, default=-1.0, 630 | help='High-pass filter corner frequency (in Hz)') 631 | parser.add_argument('--hp_filter_corners', type=int, default=4, 632 | help='High-pass filter number of corners (poles)') 633 | 634 | parser.add_argument('--spherical', action='store_true') 635 | parser.set_defaults(spherical=False) 636 | 637 | parser.add_argument('--n_distances', type=int, default=None, 638 | help='Number of distance classes, 0=none') 639 | parser.add_argument('--n_magnitudes', type=int, default=None, 640 | help='Number of magnitude classes, 0=none') 641 | parser.add_argument('--n_azimuths', type=int, default=None, 642 | help='Number of azimuth classes, 0=none') 643 | parser.add_argument('--n_depths', type=int, default=None, 644 | help='Number of depth classes, 0=none') 645 | parser.add_argument('--systematic', action='store_true', help='Attempt to get up to --n_streams streams for all net/sta/chan') 646 | parser.set_defaults(systematic=False) 647 | 648 | 649 | args = parser.parse_args() 650 | 651 | main(args) 652 | --------------------------------------------------------------------------------