run_DARNtids.py

#!/usr/bin/env python3
import sys
import os
import datetime
import subprocess

import matplotlib
matplotlib.use('Agg')

import multiprocessing

import mstid
from mstid import run_helper

years = list(range(2015,2017))
#years = [2015]
radars = []

# radars.append('ade')
# radars.append('adw')
# radars.append('bks')
# radars.append('cve')
# radars.append('cvw')
# radars.append('cly')
# radars.append('fhe')
# radars.append('fhw')
# radars.append('gbr')
# radars.append('han')
# radars.append('hok')
# radars.append('hkw')
# radars.append('inv')
# radars.append('jme')
# radars.append('kap')
# radars.append('ksr')
# radars.append('kod')
# radars.append('lyr')
# radars.append('pyk')
# radars.append('pgr')
# radars.append('rkn')
# radars.append('sas')
# radars.append('sch')
# radars.append('sto')
# radars.append('wal')

## Standard North American Radars
radars.append('cvw')
radars.append('cve')
radars.append('fhw')
radars.append('fhe')
radars.append('bks')
radars.append('wal')

radars.append('sas')
radars.append('pgr')
radars.append('kap')
radars.append('gbr')

# # European Radars
radars.append('han')
radars.append('pyk')

# #Southern Radars
# radars.append('san')
# radars.append('fir')
# radars.append('tig')
# radars.append('unw')
# #buckland park
# #radars.append('bkp')

db_name                     = 'mstid_GSMR_fitexfilter_HDF5_fig3'
base_dir                    = os.path.join('mstid_data',db_name)
# Used for creating an SSH tunnel when running the MSTID database on a remote machine.
#tunnel,mongo_port           = mstid.createTunnel() 

for year in years:
    dct                             = {}
#    dct['fovModel']                 = 'HALF_SLANT'
    dct['fovModel']                 = 'GS'
    dct['radars']                   = radars
#    dct['list_sDate']               = datetime.datetime(year,  11,1)
#    dct['list_eDate']               = datetime.datetime(year+1, 5,1)
#    dct['list_sDate']               = datetime.datetime(year,12,1)
#    dct['list_eDate']               = datetime.datetime(year + 1,1,31)
    dct['list_sDate']               = datetime.datetime(year,1,1)
    dct['list_eDate']               = datetime.datetime(year,12,31)
    dct['hanning_window_space']     = False # Set to False for MSTID Index Calculation
    dct['bad_range_km']             = None  # Set to None for MSTID Index Calculation
    #dct['mongo_port']              = mongo_port
    dct['db_name']                  = db_name
    dct['data_path']                = os.path.join(base_dir,'mstid_index')
    dct['boxcar_filter']            = False
    dct['fitacf_dir']               = '/data/sd-data_fitexfilter'
    dct['slt_range']                     = None # Default is (6,18) # Range of local times sent to mongo_tools.generate_mongo_list()
    dct['rti_fraction_threshold']        = 0.25 # Default is 0.675 was used in Frissel et al. [2016]; 0.25 used in Frissel et al. [2025]
    dct['terminator_fraction_threshold'] = 1.0. # Default is 0.0

    # Takes dct and explodes it into run_helper function
    dct_list                        = run_helper.create_music_run_list(**dct)
    #import ipdb; ipdb.set_trace()

    mstid_index         = True
    new_list            = True      # Create a completely fresh list of events in MongoDB. Delete an old list if it exists.
    recompute           = False     # Recalculate all events from raw data. If False, use existing cached hdf5 files.
    reupdate_db         = True 

    music_process       = False
    music_new_list      = True
    music_reupdate_db   = True

    nprocs              = 60
    multiproc           = True

    # Classification parameters go here. ###########################################
    classification_path = os.path.join(base_dir,'classification')

    #******************************************************************************#
    # No User Input Below This Line ***********************************************#
    #******************************************************************************#
    if mstid_index:
        # Generate MSTID List and do rti_interp level processing.
        run_helper.get_events_and_run(dct_list,process_level='rti_interp',new_list=new_list,
                recompute=recompute,multiproc=multiproc,nprocs=nprocs,**dct)
        # Reload RTI Data into MongoDb. ################################################
        if reupdate_db:
            for dct in dct_list:
                mstid.updateDb_mstid_list(multiproc=multiproc,nprocs=nprocs,**dct)

        for dct in dct_list:
            # Determine if each event is good or bad based on:
            #   1. Whether or not data is available.
            #   2. Results of pyDARNmusic.utils.checkDataQuality()
            #       (Ensures radar is operational for a minimum amount of time during the data window.
            #        Default is to require the radar to be turned off no more than 10 minutes in the
            #        data window.)
            #   3. The fraction of radar scatter points present in the data window.
            #       (Default requires minimum 67.5% data coverage.)
            #   4. The percentage of daylight in the data window.
            #       (Default requires 100% daylight in the data window.)
            mstid.classify.classify_none_events(**dct) 

            # Generate a web page and copy select figures into new directory to make it easier
            # to evaluate data and see if classification algorithm is working.
            mstid.classify.rcgb(classification_path=classification_path,**dct)

        # Run FFT Level processing on unclassified events.
        run_helper.get_events_and_run(dct_list,process_level='fft',category='unclassified',
                multiproc=multiproc,nprocs=nprocs)

        # Now run the real MSTID classification.
        mstid.classify.run_mstid_classification(dct_list,classification_path=classification_path,
                multiproc=multiproc,nprocs=5)
        print("----------- DCT LIST ---------")
        print(dct_list)
        print('Plotting calendar plot...')
        calendar_output_dir = os.path.join(base_dir,'calendar')
        mstid.calendar_plot(dct_list,db_name=db_name,output_dir=calendar_output_dir)

    # Run actual MUSIC Processing ##################################################
    if music_process:
        for dct in dct_list:
            dct['input_mstid_list']     = dct['mstid_list']
            dct['input_db_name']        = dct['db_name']
            dct['input_mongo_port']     = dct['mongo_port']
            dct['mstid_list']           = 'music_'+dct['mstid_list']
            dct['data_path']            = os.path.join(base_dir,'music_data')
            dct['hanning_window_space'] = True
    #        dct['bad_range_km']         = 500 # Set to 500 for MUSIC Calculation
            dct['bad_range_km']         = None # Set to None to match original calculations
        run_helper.get_events_and_run(dct_list,process_level='music',
                new_list=music_new_list,category=['mstid','quiet'],
                multiproc=multiproc,nprocs=nprocs)

        if music_reupdate_db:
            for dct in dct_list:
                mstid.updateDb_mstid_list(multiproc=multiproc,nprocs=nprocs,**dct)

#tunnel.kill()
print("I'm done!")