Skip to content

Commit b418ae1

Browse files
jiandewangjiandewang
authored
Merge pull request mom-ocean#67 from jiandewang/feature/update-to-GFDL-20210503
Feature/update to gfdl 20210503
2 parents 00ea3fd + 819267f commit b418ae1

6 files changed

Lines changed: 238 additions & 63 deletions

File tree

‎config_src/drivers/nuopc_cap/mom_cap.F90‎

Lines changed: 148 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -35,22 +35,25 @@ module MOM_cap_mod
3535
use MOM_ocean_model_nuopc, only: ocean_model_init, update_ocean_model, ocean_model_end
3636
use MOM_ocean_model_nuopc, only: get_ocean_grid, get_eps_omesh
3737
use MOM_cap_time, only: AlarmInit
38-
use MOM_cap_methods, only: mom_import, mom_export, mom_set_geomtype, state_diagnose
38+
use MOM_cap_methods, only: mom_import, mom_export, mom_set_geomtype, mod2med_areacor
39+
use MOM_cap_methods, only: med2mod_areacor, state_diagnose
3940
use MOM_cap_methods, only: ChkErr
41+
4042
#ifdef CESMCOUPLED
4143
use shr_file_mod, only: shr_file_setLogUnit, shr_file_getLogUnit
44+
use shr_mpi_mod, only : shr_mpi_min, shr_mpi_max
4245
#endif
4346
use time_utils_mod, only: esmf2fms_time
4447

4548
use, intrinsic :: iso_fortran_env, only: output_unit
4649

47-
use ESMF, only: ESMF_ClockAdvance, ESMF_ClockGet, ESMF_ClockPrint
50+
use ESMF, only: ESMF_ClockAdvance, ESMF_ClockGet, ESMF_ClockPrint, ESMF_VMget
4851
use ESMF, only: ESMF_ClockGetAlarm, ESMF_ClockGetNextTime, ESMF_ClockAdvance
4952
use ESMF, only: ESMF_ClockSet, ESMF_Clock, ESMF_GeomType_Flag, ESMF_LOGMSG_INFO
5053
use ESMF, only: ESMF_Grid, ESMF_GridCreate, ESMF_GridAddCoord
5154
use ESMF, only: ESMF_GridGetCoord, ESMF_GridAddItem, ESMF_GridGetItem
5255
use ESMF, only: ESMF_GridComp, ESMF_GridCompSetEntryPoint, ESMF_GridCompGet
53-
use ESMF, only: ESMF_LogFoundError, ESMF_LogWrite, ESMF_LogSetError
56+
use ESMF, only: ESMF_LogWrite, ESMF_LogSetError
5457
use ESMF, only: ESMF_LOGERR_PASSTHRU, ESMF_KIND_R8, ESMF_RC_VAL_WRONG
5558
use ESMF, only: ESMF_GEOMTYPE_MESH, ESMF_GEOMTYPE_GRID, ESMF_SUCCESS
5659
use ESMF, only: ESMF_METHOD_INITIALIZE, ESMF_MethodRemove, ESMF_State
@@ -69,9 +72,10 @@ module MOM_cap_mod
6972
use ESMF, only: ESMF_COORDSYS_SPH_DEG, ESMF_GridCreate, ESMF_INDEX_DELOCAL
7073
use ESMF, only: ESMF_MESHLOC_ELEMENT, ESMF_RC_VAL_OUTOFRANGE, ESMF_StateGet
7174
use ESMF, only: ESMF_TimePrint, ESMF_AlarmSet, ESMF_FieldGet, ESMF_Array
75+
use ESMF, only: ESMF_FieldRegridGetArea
7276
use ESMF, only: ESMF_ArrayCreate
7377
use ESMF, only: ESMF_RC_FILE_OPEN, ESMF_RC_FILE_READ, ESMF_RC_FILE_WRITE
74-
use ESMF, only: ESMF_VMBroadcast
78+
use ESMF, only: ESMF_VMBroadcast, ESMF_VMReduce, ESMF_REDUCE_MAX, ESMF_REDUCE_MIN
7579
use ESMF, only: ESMF_AlarmCreate, ESMF_ClockGetAlarmList, ESMF_AlarmList_Flag
7680
use ESMF, only: ESMF_AlarmGet, ESMF_AlarmIsCreated, ESMF_ALARMLIST_ALL, ESMF_AlarmIsEnabled
7781
use ESMF, only: ESMF_STATEITEM_NOTFOUND, ESMF_FieldWrite
@@ -93,6 +97,7 @@ module MOM_cap_mod
9397
use NUOPC_Model, only: model_label_SetRunClock => label_SetRunClock
9498
use NUOPC_Model, only: model_label_Finalize => label_Finalize
9599
use NUOPC_Model, only: SetVM
100+
!$use omp_lib , only : omp_set_num_threads
96101

97102
implicit none; private
98103

@@ -141,6 +146,7 @@ module MOM_cap_mod
141146
integer :: scalar_field_count = 0
142147
integer :: scalar_field_idx_grid_nx = 0
143148
integer :: scalar_field_idx_grid_ny = 0
149+
integer :: nthrds !< number of openmp threads per task
144150
character(len=*),parameter :: u_FILE_u = &
145151
__FILE__
146152

@@ -412,10 +418,12 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
412418
character(len=512) :: diro
413419
character(len=512) :: logfile
414420
character(ESMF_MAXSTR) :: cvalue
421+
character(len=64) :: logmsg
415422
logical :: isPresent, isPresentDiro, isPresentLogfile, isSet
416423
logical :: existflag
417424
integer :: userRc
418425
integer :: localPet
426+
integer :: localPeCount
419427
integer :: iostat
420428
integer :: readunit
421429
character(len=512) :: restartfile ! Path/Name of restart file
@@ -440,7 +448,7 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
440448
call ESMF_VMGetCurrent(vm, rc=rc)
441449
if (ChkErr(rc,__LINE__,u_FILE_u)) return
442450

443-
call ESMF_VMGet(VM, mpiCommunicator=mpi_comm_mom, rc=rc)
451+
call ESMF_VMGet(VM, mpiCommunicator=mpi_comm_mom, localPet=localPet, rc=rc)
444452
if (ChkErr(rc,__LINE__,u_FILE_u)) return
445453

446454
call ESMF_ClockGet(CLOCK, currTIME=MyTime, TimeStep=TINT, RC=rc)
@@ -452,7 +460,30 @@ subroutine InitializeAdvertise(gcomp, importState, exportState, clock, rc)
452460
CALL ESMF_TimeIntervalGet(TINT, S=DT_OCEAN, RC=rc)
453461
if (ChkErr(rc,__LINE__,u_FILE_u)) return
454462

455-
!TODO: next two lines not present in NCAR
463+
!---------------------------------
464+
! openmp threads
465+
!---------------------------------
466+
467+
call ESMF_VMGet(vm, pet=localPet, peCount=localPeCount, rc=rc)
468+
if (ChkErr(rc,__LINE__,u_FILE_u)) return
469+
470+
if(localPeCount == 1) then
471+
call NUOPC_CompAttributeGet(gcomp, name="nthreads", value=cvalue, &
472+
isPresent=isPresent, isSet=isSet, rc=rc)
473+
if (ChkErr(rc,__LINE__,u_FILE_u)) return
474+
if (isPresent .and. isSet) then
475+
read(cvalue,*) nthrds
476+
else
477+
nthrds = localPeCount
478+
endif
479+
else
480+
nthrds = localPeCount
481+
endif
482+
write(logmsg,*) nthrds
483+
call ESMF_LogWrite(trim(subname)//': nthreads = '//trim(logmsg), ESMF_LOGMSG_INFO)
484+
485+
!$ call omp_set_num_threads(nthrds)
486+
456487
call fms_init(mpi_comm_mom)
457488
call constants_init
458489
call field_manager_init
@@ -799,6 +830,7 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
799830
integer :: lbnd3,ubnd3,lbnd4,ubnd4
800831
integer :: nblocks_tot
801832
logical :: found
833+
logical :: isPresent, isSet
802834
integer(ESMF_KIND_I4), pointer :: dataPtr_mask(:,:)
803835
real(ESMF_KIND_R8), pointer :: dataPtr_area(:,:)
804836
real(ESMF_KIND_R8), pointer :: dataPtr_xcen(:,:)
@@ -807,23 +839,37 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
807839
real(ESMF_KIND_R8), pointer :: dataPtr_ycor(:,:)
808840
integer :: mpicom
809841
integer :: localPet
842+
integer :: localPeCount
810843
integer :: lsize
811844
integer :: ig,jg, ni,nj,k
812845
integer, allocatable :: gindex(:) ! global index space
813846
character(len=128) :: fldname
814847
character(len=256) :: cvalue
815848
character(len=256) :: frmt ! format specifier for several error msgs
816849
character(len=512) :: err_msg ! error messages
850+
integer :: spatialDim
851+
integer :: numOwnedElements
852+
type(ESMF_Array) :: elemMaskArray
853+
real(ESMF_KIND_R8) , pointer :: ownedElemCoords(:)
854+
real(ESMF_KIND_R8) , pointer :: lat(:), latMesh(:)
855+
real(ESMF_KIND_R8) , pointer :: lon(:), lonMesh(:)
856+
integer(ESMF_KIND_I4) , pointer :: mask(:), maskMesh(:)
857+
real(ESMF_KIND_R8) :: diff_lon, diff_lat
858+
real :: eps_omesh
859+
real(ESMF_KIND_R8) :: L2_to_rad2
860+
type(ESMF_Field) :: lfield
861+
real(ESMF_KIND_R8), allocatable :: mesh_areas(:)
862+
real(ESMF_KIND_R8), allocatable :: model_areas(:)
863+
real(ESMF_KIND_R8), pointer :: dataPtr_mesh_areas(:)
864+
real(ESMF_KIND_R8) :: max_mod2med_areacor
865+
real(ESMF_KIND_R8) :: max_med2mod_areacor
866+
real(ESMF_KIND_R8) :: min_mod2med_areacor
867+
real(ESMF_KIND_R8) :: min_med2mod_areacor
868+
real(ESMF_KIND_R8) :: max_mod2med_areacor_glob
869+
real(ESMF_KIND_R8) :: max_med2mod_areacor_glob
870+
real(ESMF_KIND_R8) :: min_mod2med_areacor_glob
871+
real(ESMF_KIND_R8) :: min_med2mod_areacor_glob
817872
character(len=*), parameter :: subname='(MOM_cap:InitializeRealize)'
818-
integer :: spatialDim
819-
integer :: numOwnedElements
820-
type(ESMF_Array) :: elemMaskArray
821-
real(ESMF_KIND_R8) , pointer :: ownedElemCoords(:)
822-
real(ESMF_KIND_R8) , pointer :: lat(:), latMesh(:)
823-
real(ESMF_KIND_R8) , pointer :: lon(:), lonMesh(:)
824-
integer(ESMF_KIND_I4) , pointer :: mask(:), maskMesh(:)
825-
real(ESMF_KIND_R8) :: diff_lon, diff_lat
826-
real :: eps_omesh
827873
!--------------------------------
828874

829875
rc = ESMF_SUCCESS
@@ -851,6 +897,28 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
851897
call ESMF_VMGet(vm, petCount=npet, mpiCommunicator=mpicom, localPet=localPet, rc=rc)
852898
if (ChkErr(rc,__LINE__,u_FILE_u)) return
853899

900+
!---------------------------------
901+
! openmp threads
902+
!---------------------------------
903+
904+
call ESMF_VMGet(vm, pet=localPet, peCount=localPeCount, rc=rc)
905+
if (ChkErr(rc,__LINE__,u_FILE_u)) return
906+
907+
if(localPeCount == 1) then
908+
call NUOPC_CompAttributeGet(gcomp, name="nthreads", value=cvalue, &
909+
isPresent=isPresent, isSet=isSet, rc=rc)
910+
if (ChkErr(rc,__LINE__,u_FILE_u)) return
911+
if (isPresent .and. isSet) then
912+
read(cvalue,*) nthrds
913+
else
914+
nthrds = localPeCount
915+
endif
916+
else
917+
nthrds = localPeCount
918+
endif
919+
920+
!$ call omp_set_num_threads(nthrds)
921+
854922
!---------------------------------
855923
! global mom grid size
856924
!---------------------------------
@@ -992,17 +1060,76 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
9921060
end if
9931061
end do
9941062

995-
deallocate(ownedElemCoords)
996-
deallocate(lonMesh , lon )
997-
deallocate(latMesh , lat )
998-
deallocate(maskMesh, mask)
9991063
! realize the import and export fields using the mesh
10001064
call MOM_RealizeFields(importState, fldsToOcn_num, fldsToOcn, "Ocn import", mesh=Emesh, rc=rc)
10011065
if (ChkErr(rc,__LINE__,u_FILE_u)) return
10021066

10031067
call MOM_RealizeFields(exportState, fldsFrOcn_num, fldsFrOcn, "Ocn export", mesh=Emesh, rc=rc)
10041068
if (ChkErr(rc,__LINE__,u_FILE_u)) return
10051069

1070+
!---------------------------------
1071+
! determine flux area correction factors - module variables in mom_cap_methods
1072+
!---------------------------------
1073+
! Area correction factors are ONLY valid for meshes that are read in - so do not need them for
1074+
! grids that are calculated internally
1075+
1076+
! Determine mesh areas for regridding
1077+
call ESMF_MeshGet(Emesh, numOwnedElements=numOwnedElements, spatialDim=spatialDim, rc=rc)
1078+
if (ChkErr(rc,__LINE__,u_FILE_u)) return
1079+
1080+
allocate (mod2med_areacor(numOwnedElements))
1081+
allocate (med2mod_areacor(numOwnedElements))
1082+
mod2med_areacor(:) = 1._ESMF_KIND_R8
1083+
med2mod_areacor(:) = 1._ESMF_KIND_R8
1084+
1085+
#ifdef CESMCOUPLED
1086+
! Determine model areas and flux correction factors (module variables in mom_)
1087+
call ESMF_StateGet(exportState, itemName=trim(fldsFrOcn(2)%stdname), field=lfield, rc=rc)
1088+
if (ChkErr(rc,__LINE__,u_FILE_u)) return
1089+
call ESMF_FieldRegridGetArea(lfield, rc=rc)
1090+
if (ChkErr(rc,__LINE__,u_FILE_u)) return
1091+
call ESMF_FieldGet(lfield, farrayPtr=dataPtr_mesh_areas, rc=rc)
1092+
if (ChkErr(rc,__LINE__,u_FILE_u)) return
1093+
1094+
allocate(mesh_areas(numOwnedElements))
1095+
allocate(model_areas(numOwnedElements))
1096+
k = 0
1097+
do j = ocean_grid%jsc, ocean_grid%jec
1098+
do i = ocean_grid%isc, ocean_grid%iec
1099+
k = k + 1 ! Increment position within gindex
1100+
if (mask(k) /= 0) then
1101+
mesh_areas(k) = dataPtr_mesh_areas(k)
1102+
model_areas(k) = ocean_grid%AreaT(i,j) / ocean_grid%Rad_Earth**2
1103+
mod2med_areacor(k) = model_areas(k) / mesh_areas(k)
1104+
med2mod_areacor(k) = mesh_areas(k) / model_areas(k)
1105+
end if
1106+
end do
1107+
end do
1108+
deallocate(mesh_areas)
1109+
deallocate(model_areas)
1110+
1111+
! Write diagnostic output for correction factors
1112+
min_mod2med_areacor = minval(mod2med_areacor)
1113+
max_mod2med_areacor = maxval(mod2med_areacor)
1114+
min_med2mod_areacor = minval(med2mod_areacor)
1115+
max_med2mod_areacor = maxval(med2mod_areacor)
1116+
call shr_mpi_max(max_mod2med_areacor, max_mod2med_areacor_glob, mpicom)
1117+
call shr_mpi_min(min_mod2med_areacor, min_mod2med_areacor_glob, mpicom)
1118+
call shr_mpi_max(max_med2mod_areacor, max_med2mod_areacor_glob, mpicom)
1119+
call shr_mpi_min(min_med2mod_areacor, min_med2mod_areacor_glob, mpicom)
1120+
if (localPet == 0) then
1121+
write(logunit,'(2A,2g23.15,A )') trim(subname),' : min_mod2med_areacor, max_mod2med_areacor ',&
1122+
min_mod2med_areacor_glob, max_mod2med_areacor_glob, 'MOM6'
1123+
write(logunit,'(2A,2g23.15,A )') trim(subname),' : min_med2mod_areacor, max_med2mod_areacor ',&
1124+
min_med2mod_areacor_glob, max_med2mod_areacor_glob, 'MOM6'
1125+
end if
1126+
#endif
1127+
1128+
deallocate(ownedElemCoords)
1129+
deallocate(lonMesh , lon )
1130+
deallocate(latMesh , lat )
1131+
deallocate(maskMesh, mask)
1132+
10061133
else if (geomtype == ESMF_GEOMTYPE_GRID) then
10071134

10081135
!---------------------------------
@@ -1229,7 +1356,6 @@ subroutine InitializeRealize(gcomp, importState, exportState, clock, rc)
12291356

12301357
call MOM_RealizeFields(exportState, fldsFrOcn_num, fldsFrOcn, "Ocn export", grid=gridOut, rc=rc)
12311358
if (ChkErr(rc,__LINE__,u_FILE_u)) return
1232-
12331359
endif
12341360

12351361
!---------------------------------
@@ -1405,6 +1531,8 @@ subroutine ModelAdvance(gcomp, rc)
14051531

14061532
call shr_file_setLogUnit (logunit)
14071533

1534+
!$ call omp_set_num_threads(nthrds)
1535+
14081536
! query the Component for its clock, importState and exportState
14091537
call ESMF_GridCompGet(gcomp, clock=clock, importState=importState, &
14101538
exportState=exportState, rc=rc)

0 commit comments

Comments
 (0)
⚡