From a6fe11d4ec53f0627a203c7008f0683f54310a60 Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Wed, 16 Jun 2021 12:50:08 +1000 Subject: [PATCH 1/9] Modified imports to suit relative import. Fixed bug in missing 'phase' column when reading picks ensemble file into Pandas data frame. Fixed bug where script will crash if there are no common events (parameter df_plot = None). So just need to catch the df_plot = None case. --- .gitignore | 1 + .../gps_corrections/relative_tt_residuals_plotter.py | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index f753958bc..a6d260d5f 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,4 @@ seismic/xcorqc/**/*.pbs **/junk.* **/LOG_CODE_DUPES_*.txt **/LOG_LOCATION_DUPES_*.txt +/seismic/gps_corrections/p_combined.txt diff --git a/seismic/gps_corrections/relative_tt_residuals_plotter.py b/seismic/gps_corrections/relative_tt_residuals_plotter.py index 3911718ec..4c8920080 100644 --- a/seismic/gps_corrections/relative_tt_residuals_plotter.py +++ b/seismic/gps_corrections/relative_tt_residuals_plotter.py @@ -33,10 +33,7 @@ else: import pathlib # pylint: disable=import-error -from seismic.gps_corrections.picks_reader_utils import (read_picks_ensemble, - get_network_stations, - compute_matching_network_mask, - generate_large_events_catalog) +from picks_reader_utils import (read_picks_ensemble, get_network_stations, compute_matching_network_mask, generate_large_events_catalog) # pylint: disable=invalid-name, fixme, too-many-locals, too-many-statements # pylint: disable=attribute-defined-outside-init, logging-format-interpolation, logging-not-lazy @@ -318,11 +315,13 @@ def _plot_network_relative_to_ref_station(df_plot, ref, target_stns, batch_optio :type display_options: class DisplayOptions """ register_matplotlib_converters() + if df_plot is None: + return df_plot = df_plot.assign(relTtResidual=(df_plot['ttResidual'] - df_plot['ttResidualRef'])) # Re-order columns df_plot = df_plot[['#eventID', 'originTimestamp', 'mag', 'originLon', 'originLat', 'originDepthKm', - 'net', 'sta', 'cha', 'pickTimestamp', 'phase', 'stationLon', 'stationLat', + 'net', 'sta', 'cha', 'pickTimestamp', 'stationLon', 'stationLat', 'distance', 'snr', 'ttResidual', 'ttResidualRef', 'relTtResidual', 'qualityMeasureCWT', 'qualityMeasureSlope', 'nSigma']] @@ -704,7 +703,7 @@ def main(picks_file, network1, networks2, stations1=None, stations2=None, df_raw_picks = read_picks_ensemble(picks_file) # Remove unused columns df_raw_picks = df_raw_picks[['#eventID', 'originTimestamp', 'mag', 'originLon', 'originLat', 'originDepthKm', - 'net', 'sta', 'cha', 'pickTimestamp', 'phase', 'stationLon', 'stationLat', 'az', + 'net', 'sta', 'cha', 'pickTimestamp', 'stationLon', 'stationLat', 'az', 'baz', 'distance', 'ttResidual', 'snr', 'qualityMeasureCWT', 'qualityMeasureSlope', 'nSigma']] log.debug("Number of raw picks: {}".format(len(df_raw_picks))) From 180767c1a064a6c0831a48689bbcd69f7049b924 Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Fri, 23 Jul 2021 13:09:51 +1000 Subject: [PATCH 2/9] added in requiements to get the GPS correction workflow working on the NCI VDI --- .../gps_clock_correction_gui.py | 2 +- seismic/gps_corrections/requirementsVDI.txt | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 seismic/gps_corrections/requirementsVDI.txt diff --git a/seismic/gps_corrections/gps_clock_correction_gui.py b/seismic/gps_corrections/gps_clock_correction_gui.py index 5e1829f7f..36d6da2d4 100644 --- a/seismic/gps_corrections/gps_clock_correction_gui.py +++ b/seismic/gps_corrections/gps_clock_correction_gui.py @@ -143,7 +143,7 @@ def _update_next_button(self, *_unused): nc_file_value = self.nc_file.get() nc_file_valid = bool(nc_file_value) and os.path.isfile(nc_file_value) if nc_file_valid: - self.station_code= self._extract_code_from_filename(nc_file_value) + self.station_code = self._extract_code_from_filename(nc_file_value) else: self.station_code = '' self.stn_code_label['text'] = "Station code: " + self.station_code diff --git a/seismic/gps_corrections/requirementsVDI.txt b/seismic/gps_corrections/requirementsVDI.txt new file mode 100644 index 000000000..2997ed774 --- /dev/null +++ b/seismic/gps_corrections/requirementsVDI.txt @@ -0,0 +1,27 @@ +# module load python3 +# python3 -m venv venv +# module purge +# module load openmpi +# module load mpi4py +# module load hdf5/1.10.5p +# pip install -r requirements.txt +pyasdf +numpy +obspy +pandas +matplotlib +ordered-set +psutil +click +ujson +sklearn +netcdf4 +tqdm +shapely +descartes +pyproj +pytz +pathlib2 +jupyter +# python3 -m seismic.gps_corrections.gps_clock_correction_gui +# env PYTHONPATH=`pwd` jupyter notebook \ No newline at end of file From 8c9db67c3831a28a1f11fd03818200ce3e38bb0c Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Fri, 23 Jul 2021 13:12:12 +1000 Subject: [PATCH 3/9] changed back to top level imports --- seismic/gps_corrections/relative_tt_residuals_plotter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seismic/gps_corrections/relative_tt_residuals_plotter.py b/seismic/gps_corrections/relative_tt_residuals_plotter.py index 4c8920080..22f9e6014 100644 --- a/seismic/gps_corrections/relative_tt_residuals_plotter.py +++ b/seismic/gps_corrections/relative_tt_residuals_plotter.py @@ -33,7 +33,7 @@ else: import pathlib # pylint: disable=import-error -from picks_reader_utils import (read_picks_ensemble, get_network_stations, compute_matching_network_mask, generate_large_events_catalog) +from seismic.gps_corrections.picks_reader_utils import (read_picks_ensemble, get_network_stations, compute_matching_network_mask, generate_large_events_catalog) # pylint: disable=invalid-name, fixme, too-many-locals, too-many-statements # pylint: disable=attribute-defined-outside-init, logging-format-interpolation, logging-not-lazy From 62abc08dc41f6941b5fcfe21b1c0aec68e525790 Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Fri, 23 Jul 2021 13:46:47 +1000 Subject: [PATCH 4/9] updated requirementsVDI --- seismic/gps_corrections/requirementsVDI.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/seismic/gps_corrections/requirementsVDI.txt b/seismic/gps_corrections/requirementsVDI.txt index 2997ed774..256ea8ca1 100644 --- a/seismic/gps_corrections/requirementsVDI.txt +++ b/seismic/gps_corrections/requirementsVDI.txt @@ -1,10 +1,11 @@ +# module purge # module load python3 # python3 -m venv venv -# module purge +# source venv/bin/activate # module load openmpi # module load mpi4py # module load hdf5/1.10.5p -# pip install -r requirements.txt +# pip install -r requirementsVDI.txt pyasdf numpy obspy From 329a664661d9182e03f2f79aff0610aba534092a Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Fri, 23 Jul 2021 14:16:33 +1000 Subject: [PATCH 5/9] updated requirementsVDI --- seismic/gps_corrections/requirementsVDI.txt | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/seismic/gps_corrections/requirementsVDI.txt b/seismic/gps_corrections/requirementsVDI.txt index 256ea8ca1..cf2327686 100644 --- a/seismic/gps_corrections/requirementsVDI.txt +++ b/seismic/gps_corrections/requirementsVDI.txt @@ -1,11 +1,13 @@ +# cd to hiperseis directory # module purge -# module load python3 -# python3 -m venv venv -# source venv/bin/activate # module load openmpi # module load mpi4py # module load hdf5/1.10.5p -# pip install -r requirementsVDI.txt +# module load python3-as-python +# alias python=python3 +# python -m venv venv +# source venv/bin/activate +# pip install -r seismic/gps_corrections/requirementsVDI.txt pyasdf numpy obspy @@ -24,5 +26,6 @@ pyproj pytz pathlib2 jupyter +# export PYTHONPATH=$PYTHONPATH:'pwd' # python3 -m seismic.gps_corrections.gps_clock_correction_gui -# env PYTHONPATH=`pwd` jupyter notebook \ No newline at end of file +# jupyter notebook \ No newline at end of file From 890a5dd763c27f96da1043ce8df24ec162ea27b4 Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Tue, 27 Jul 2021 12:39:33 +1000 Subject: [PATCH 6/9] fixed bug with num parameter input to np.linspace function. Type needs to be integer. --- seismic/xcorqc/xcorr_station_clock_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seismic/xcorqc/xcorr_station_clock_analysis.py b/seismic/xcorqc/xcorr_station_clock_analysis.py index cc6efe217..9dafa2de9 100644 --- a/seismic/xcorqc/xcorr_station_clock_analysis.py +++ b/seismic/xcorqc/xcorr_station_clock_analysis.py @@ -150,7 +150,7 @@ def do_spline_regression(self, group_ids, regression_degree): y = self.corrections_clean[mask_group] # Constrain the knot frequency to be proportional to the order, so that we don't overfit # to high frequency variations in the data just for the sake of lowering the spline residual. - t = np.linspace(x[0], x[-1], degree - 1) + t = np.linspace(x[0], x[-1], int(degree - 1)) r = LSQUnivariateSpline(x, y, t[1:-1], k=degree) regressions[i] = r From 41d18d6ed690c2cbd8d56f88650e1d52955a9651 Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Tue, 27 Jul 2021 17:34:08 +1000 Subject: [PATCH 7/9] fixed bug with num parameter input to np.linspace function. Type needs to be integer. --- seismic/xcorqc/xcorr_station_clock_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seismic/xcorqc/xcorr_station_clock_analysis.py b/seismic/xcorqc/xcorr_station_clock_analysis.py index 9dafa2de9..d043945e2 100644 --- a/seismic/xcorqc/xcorr_station_clock_analysis.py +++ b/seismic/xcorqc/xcorr_station_clock_analysis.py @@ -176,7 +176,7 @@ def do_spline_resampling(self, group_ids, regressors, sampling_period_seconds): timestamp_min = min(x) timestamp_max = max(x) num_samples = np.round((timestamp_max - timestamp_min) / sampling_period_seconds) - lin_times = np.linspace(timestamp_min, timestamp_max, num_samples + 1) + lin_times = np.linspace(timestamp_min, timestamp_max, int(num_samples + 1)) lin_corrections = regressors[i](lin_times) regular_corrections[i] = {'times': lin_times, 'corrections': lin_corrections} From fdf1e2cb78a16a070f7aa03f45d7cf48f7bcd1d9 Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Mon, 23 Aug 2021 13:15:37 +1000 Subject: [PATCH 8/9] update how to run gps corrections --- seismic/gps_corrections/requirementsVDI.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/seismic/gps_corrections/requirementsVDI.txt b/seismic/gps_corrections/requirementsVDI.txt index cf2327686..93430119b 100644 --- a/seismic/gps_corrections/requirementsVDI.txt +++ b/seismic/gps_corrections/requirementsVDI.txt @@ -28,4 +28,5 @@ pathlib2 jupyter # export PYTHONPATH=$PYTHONPATH:'pwd' # python3 -m seismic.gps_corrections.gps_clock_correction_gui +# python -m seismic.gps_corrections.relative_tt_residuals_plotter seismic/gps_corrections/p_combined.txt --network1=1Q --networks2=AU --no-strict-filtering # jupyter notebook \ No newline at end of file From 40c47a68df2da71d276fe778a9fff66f22702db3 Mon Sep 17 00:00:00 2001 From: outdoorpet Date: Mon, 30 Aug 2021 13:44:59 +1000 Subject: [PATCH 9/9] added in flag for y axis scale on tt residual plots --- .../relative_tt_residuals_plotter.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/seismic/gps_corrections/relative_tt_residuals_plotter.py b/seismic/gps_corrections/relative_tt_residuals_plotter.py index 22f9e6014..d4129664c 100644 --- a/seismic/gps_corrections/relative_tt_residuals_plotter.py +++ b/seismic/gps_corrections/relative_tt_residuals_plotter.py @@ -198,9 +198,11 @@ def pandas_timestamp_to_plottable_datetime(data): return data.transform(datetime.datetime.utcfromtimestamp).astype('datetime64[ms]').dt.to_pydatetime() -def _plot_target_network_rel_residuals(df, target, ref, batch_options, filter_options, tt_scale=50, +def _plot_target_network_rel_residuals(df, target, ref, batch_options, filter_options, tt_scale=False, snr_scale=(0, 60), annotator=None): + if tt_scale is False: + tt_scale = 50 file_label = batch_options.batch_label save_file = batch_options.save_file @@ -296,7 +298,7 @@ def _plot_dataset(ds, net_code, ref_code): df_export.to_csv(fname, index=False) -def _plot_network_relative_to_ref_station(df_plot, ref, target_stns, batch_options, filter_options, display_options): +def _plot_network_relative_to_ref_station(df_plot, ref, target_stns, batch_options, filter_options, display_options, tt_scale): """ Compute relative residuals and send to plotting function. @@ -334,7 +336,7 @@ def _plot_decorator(opts): if opts.deployments is not None: _add_temporary_deployment_intervals(opts.deployments) - _plot_target_network_rel_residuals(df_plot, target_stns, ref, batch_options, filter_options, + _plot_target_network_rel_residuals(df_plot, target_stns, ref, batch_options, filter_options, tt_scale, annotator=lambda: _plot_decorator(display_options)) deregister_matplotlib_converters() @@ -665,13 +667,16 @@ def _get_known_temporary_deployments(): @click.option('--interactive', is_flag=True, default=False, show_default=True, help='If True, plots will be displayed as popup windows instead of saving to file. ' 'Use this option to interact with the data.') +@click.option('--tt-scale', type=float, default=50, show_default=True, + help='y-axis limits for TT residual plots') + def main(picks_file, network1, networks2, stations1=None, stations2=None, min_distance=DEFAULT_MIN_DISTANCE, max_distance=DEFAULT_MAX_DISTANCE, min_event_snr=DEFAULT_MIN_EVENT_SNR, cwt_cutoff=DEFAULT_CWT_CUTOFF, slope_cutoff=DEFAULT_SLOPE_CUTOFF, nsigma_cutoff=DEFAULT_NSIGMA_CUTOFF, min_event_magnitude=DEFAULT_MIN_EVENT_MAG, strict_filtering=DEFAULT_STRICT_FILTERING, show_deployments=False, show_historical=True, include_alternate_catalog=True, - export_path=None, interactive=False): # pragma: no cover + export_path=None, interactive=False, tt_scale=False): # pragma: no cover """ Main function for running relative traveltime residual plotting. The picks ensemble file should have column headings in accordance with picks_reader_utils.PICKS_TABLE_SCHEMA. @@ -788,7 +793,7 @@ def main(picks_file, network1, networks2, stations1=None, stations2=None, df_final = analyze_target_relative_to_ref(df_picks, single_ref, TARGET_STNS, filter_options) # Plot results _plot_network_relative_to_ref_station(df_final, single_ref, TARGET_STNS, batch_options, - filter_options, display_options) + filter_options, display_options, tt_scale) # end for # end for